path: root/kernel/main.c

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/vfio.h>
#include <linux/iommufd.h>
#include <linux/vfio_pci_core.h>
#include <linux/mdev.h>
#include <linux/slab.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/iommu.h>
#include <linux/vmalloc.h>
#include <linux/version.h>
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/completion.h>

#include "nvidia-vgpu-vfio.h"
#include "bar_emulation.h"
#include "gsp_comm.h"
#include "gsp_emulation.h"
#include "gsp_rm_emulation.h"
#include "gsp_sema_service.h"
#include "memory.h"
#include "irq.h"
#include "migration.h"
#include "ioctl.h"

#include "nv-vgpu-vfio-interface.h"
#include "gsp_phys_gpu.h"
#include "gsp_phys_booter.h"

#define NV_VERSION_STRING "550.163.01"

/* ============================================================
 * Forward declarations
 * ============================================================ */

static int nv_vgpu_mdev_create(struct mdev_device *mdev);
static void nv_vgpu_mdev_remove(struct mdev_device *mdev);
static void nv_vgpu_device_init_work(struct work_struct *work);
static void nv_vgpu_device_cleanup_work(struct work_struct *work);

/* VFIO device operations */
static int nv_vgpu_vfio_dev_match(struct vfio_device *vdev, char *buf);
static int nv_vgpu_vfio_dev_init(struct vfio_device *vdev);
static void nv_vgpu_vfio_dev_release(struct vfio_device *vdev);
static int nv_vgpu_vfio_dev_open(struct vfio_device *vdev);
static void nv_vgpu_vfio_dev_close(struct vfio_device *vdev);
static ssize_t nv_vgpu_vfio_dev_read(struct vfio_device *vdev, char __user *buf, size_t count, loff_t *ppos);
static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __user *buf, size_t count, loff_t *ppos);
static long nv_vgpu_vfio_dev_ioctl(struct vfio_device *vdev, unsigned int cmd, unsigned long arg);
static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct *vma);
static void nv_vgpu_vfio_dev_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length);

/* PCI Config Space handlers for VFIO mdev */
static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev *pdev);
static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos);
static ssize_t nv_vgpu_config_write(struct nv_vgpu_device *vgpu, const char __user *buf, size_t count, loff_t *ppos);

/* vGPU type management */
int nv_vgpu_register_type_info(struct nv_vgpu_type_info *info);
struct nv_vgpu_type_info *nv_vgpu_find_type_info(const char *name);

struct mdev_type *nv_vgpu_mdev_types[16];
struct nv_vgpu_type_info g_vgpu_type_infos[16];
int g_num_vgpu_types = 0;
static DEFINE_MUTEX(g_vgpu_type_lock);

/*
 * get_available - report available instances for an mdev type.
 *
 * The mdev core's shared parent->available_instances counter can drift
 * (e.g. if a remove path ever fails to complete). Compute availability
 * from our own authoritative device list instead.
 */
static unsigned int nv_vgpu_get_available(struct mdev_type *mtype)
{
	struct nv_vgpu_type_info *info;
	struct nv_vgpu_device *vgpu;
	const char *name = mtype->sysfs_name;
	unsigned int active = 0, max = 1;

	/* Type dirs may carry an extra "nvidia-" prefix vs our type names */
	if (str_has_prefix(name, "nvidia-"))
		name += strlen("nvidia-");
	info = nv_vgpu_find_type_info(name);
	if (info && info->max_instances)
		max = info->max_instances;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		if (vgpu->mdev && vgpu->mdev->type == mtype)
			active++;
	}
	mutex_unlock(&nv_vgpu_driver.lock);

	return active >= max ? 0 : max - active;
}

/*
 * show_description - mdev type description for management tools.
 *
 * Management tools parse "framebuffer=<n>M" from this string for every
 * supported type before creating an mdev, so the field is mandatory
 * (a missing description fails type enumeration).
 * 256M matches the vGPU BAR1 (VRAM aperture) size we expose to guests.
 */
static ssize_t nv_vgpu_show_description(struct mdev_type *mtype, char *buf)
{
	return sysfs_emit(buf, "num_heads=1, frl_config=60, framebuffer=256M, "
			       "max_resolution=5120x2880, max_instance=1\n");
}

struct mdev_driver nv_vgpu_mdev_driver = {
	.device_api = VFIO_DEVICE_API_PCI_STRING,
	.max_instances = 8,
	.probe = nv_vgpu_mdev_create,
	.remove = nv_vgpu_mdev_remove,
	.get_available = nv_vgpu_get_available,
	.show_description = nv_vgpu_show_description,
	.driver = {
		.name = "nvidia-vgpu-vfio-mdev",
	},
};

/* VFIO device operations for vGPU */
static const struct vfio_device_ops nv_vgpu_vfio_dev_ops = {
	.name		= "nvidia-vgpu-vfio",
	.init		= nv_vgpu_vfio_dev_init,
	.release	= nv_vgpu_vfio_dev_release,
	.open_device	= nv_vgpu_vfio_dev_open,
	.close_device	= nv_vgpu_vfio_dev_close,
	.read		= nv_vgpu_vfio_dev_read,
	.write		= nv_vgpu_vfio_dev_write,
	.ioctl		= nv_vgpu_vfio_dev_ioctl,
	.mmap		= nv_vgpu_vfio_dev_mmap,
	.dma_unmap	= nv_vgpu_vfio_dev_dma_unmap,
	.match		= nv_vgpu_vfio_dev_match,
	.bind_iommufd	= vfio_iommufd_emulated_bind,
	.unbind_iommufd = vfio_iommufd_emulated_unbind,
	.attach_ioas	= vfio_iommufd_emulated_attach_ioas,
	.detach_ioas	= vfio_iommufd_emulated_detach_ioas,
};

/* ============================================================
 * Symbols we provide to nvidia.ko
 * ============================================================ */

static vgpu_vfio_ops_t *g_vgpu_vfio_ops = NULL;
static rm_vgpu_vfio_ops_t g_rm_vgpu_vfio_ops;

/* mdev parent structure for our physical device */
static struct mdev_parent *g_mdev_parent = NULL;

static bool gsp_phys_gpu_param;

/* VFIO ops implementation */
static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id);
static void nv_vgpu_vfio_remove(struct pci_dev *pdev);
static NV_STATUS nv_vgpu_vfio_inject_interrupt(void *vgpu_dev);

static vgpu_vfio_ops_t nv_vgpu_vfio_ops = {
	.probe = nv_vgpu_vfio_probe,
	.remove = nv_vgpu_vfio_remove,
	.inject_interrupt = nv_vgpu_vfio_inject_interrupt,
};

/* RM ops implementation - callbacks from nvidia.ko */
static NV_STATUS nv_vgpu_create_callback(struct pci_dev *pdev, const NvU8 *uuid,
                                         NvU32 type_id, NvU16 *vgpu_id,
                                         NvU32 *gpu_pci_id, NvU32 gpu_pci_bdf)
{
	pr_info("vgsp: vgpu_create callback: type_id=%u\n", type_id);
	if (vgpu_id)
		*vgpu_id = 1;
	if (gpu_pci_id)
		*gpu_pci_id = 0x10de1ed3;
	return NV_OK;
}

static NV_STATUS nv_vgpu_delete_callback(const NvU8 *uuid, NvU16 vgpu_id)
{
	pr_info("vgsp: vgpu_delete callback: vgpu_id=%u\n", vgpu_id);
	return NV_OK;
}

static NV_STATUS nv_vgpu_bar_info_callback(struct pci_dev *pdev, const NvU8 *uuid,
                                           NvU64 *size, NvU32 bar_index, void *vgpu_dev)
{
	pr_info("vgsp: vgpu_bar_info callback: bar_index=%u\n", bar_index);
	if (size) {
		switch (bar_index) {
		case 0:
			*size = NV_VGPU_BAR0_SIZE;  /* 16MB */
			break;
		case 1:
			*size = NV_VGPU_BAR1_SIZE;  /* 256MB */
			break;
		case 2:
			*size = NV_VGPU_BAR2_SIZE;  /* 32MB */
			break;
		case 3:
			*size = NV_VGPU_BAR3_SIZE;  /* 32MB */
			break;
		default:
			*size = 0;
			break;
		}
	}
	return NV_OK;
}

static NV_STATUS nv_vgpu_get_name_callback(struct pci_dev *pdev, NvU32 type_id, char *buf)
{
	pr_info("vgsp: get_name callback: type_id=%u\n", type_id);
	if (buf) {
		switch (type_id) {
		case 0: /* Ampere */
			strcpy(buf, "NVIDIA A100");
			break;
		case 1: /* Hopper */
			strcpy(buf, "NVIDIA H100");
			break;
		case 2: /* Turing */
			strcpy(buf, "NVIDIA T4");
			break;
		case 3: /* Pascal */
			strcpy(buf, "NVIDIA P4");
			break;
		default:
			strcpy(buf, "NVIDIA vGPU");
			break;
		}
	}
	return NV_OK;
}

static NV_STATUS nv_vgpu_get_description_callback(struct pci_dev *pdev, NvU32 type_id, char *buf)
{
	pr_info("vgsp: get_description callback: type_id=%u\n", type_id);
	if (buf) {
		switch (type_id) {
		case 0:
			strcpy(buf, "NVIDIA Ampere Architecture vGPU");
			break;
		case 1:
			strcpy(buf, "NVIDIA Hopper Architecture vGPU");
			break;
		case 2:
			strcpy(buf, "NVIDIA Turing Architecture vGPU");
			break;
		case 3:
			strcpy(buf, "NVIDIA Pascal Architecture vGPU");
			break;
		default:
			strcpy(buf, "NVIDIA vGPU Description");
			break;
		}
	}
	return NV_OK;
}

static NV_STATUS nv_vgpu_get_instances_callback(struct pci_dev *pdev, NvU32 type_id, char *buf)
{
	pr_info("vgsp: get_instances callback: type_id=%u\n", type_id);
	if (buf)
		strcpy(buf, "1");
	return NV_OK;
}

static void nv_vgpu_update_request_callback(const NvU8 *uuid, NvU64 *vgpu_dev,
                                            NvU64 *vgpu_vm, VGPU_DEVICE_STATE device_state,
                                            const char *params)
{
	pr_info("vgsp: update_request callback: state=%u\n", device_state);
}

static NV_STATUS nv_vgpu_start_callback(const NvU8 *uuid, void *wait_queue, NvS32 *return_status,
                                        NvU8 *vm_name, NvU32 qemu_pid)
{
	pr_info("vgsp: vgpu_start callback: qemu_pid=%u\n", qemu_pid);
	if (return_status)
		*return_status = 0;
	return NV_OK;
}

static NV_STATUS nv_vgpu_sparse_mmap_callback(struct pci_dev *pdev, const NvU8 *uuid,
                                              NvU64 **pages, NvU64 **offsets, NvU32 *count)
{
	pr_info("vgsp: sparse_mmap callback\n");
	if (count)
		*count = 0;
	return NV_OK;
}

/* Build RM ops structure */
static void populate_rm_ops(void)
{
	memset(&g_rm_vgpu_vfio_ops, 0, sizeof(g_rm_vgpu_vfio_ops));
	g_rm_vgpu_vfio_ops.version_string = NV_VERSION_STRING;
	g_rm_vgpu_vfio_ops.vgpu_create = nv_vgpu_create_callback;
	g_rm_vgpu_vfio_ops.vgpu_delete = nv_vgpu_delete_callback;
	g_rm_vgpu_vfio_ops.vgpu_bar_info = nv_vgpu_bar_info_callback;
	g_rm_vgpu_vfio_ops.vgpu_start = nv_vgpu_start_callback;
	g_rm_vgpu_vfio_ops.get_name = nv_vgpu_get_name_callback;
	g_rm_vgpu_vfio_ops.get_description = nv_vgpu_get_description_callback;
	g_rm_vgpu_vfio_ops.get_instances = nv_vgpu_get_instances_callback;
	g_rm_vgpu_vfio_ops.get_sparse_mmap = nv_vgpu_sparse_mmap_callback;
	g_rm_vgpu_vfio_ops.update_request = nv_vgpu_update_request_callback;
}

/* ============================================================
 * vGPU Type Management (userspace provides info via ioctl)
 * ============================================================ */

int nv_vgpu_register_type_info(struct nv_vgpu_type_info *info)
{
    int i;
    
    mutex_lock(&g_vgpu_type_lock);
    
    /* Check if already registered */
    for (i = 0; i < g_num_vgpu_types; i++) {
        if (strcmp(g_vgpu_type_infos[i].name, info->name) == 0) {
            /* Update existing */
            g_vgpu_type_infos[i] = *info;
            mutex_unlock(&g_vgpu_type_lock);
            return 0;
        }
    }
    
    /* Add new */
    if (g_num_vgpu_types >= 16) {
        mutex_unlock(&g_vgpu_type_lock);
        return -ENOSPC;
    }
    
    g_vgpu_type_infos[g_num_vgpu_types] = *info;
    g_num_vgpu_types++;
    
    mutex_unlock(&g_vgpu_type_lock);
    
    pr_info("vgsp: Registered vGPU type: name='%s', vdev=0x%llx, pdev=0x%llx, max_inst=%u\n",
            info->name, info->vdev_id, info->pdev_id, info->max_instances);
    return 0;
}
EXPORT_SYMBOL(nv_vgpu_register_type_info);

struct nv_vgpu_type_info *nv_vgpu_find_type_info(const char *name)
{
    int i;
    
    mutex_lock(&g_vgpu_type_lock);
    for (i = 0; i < g_num_vgpu_types; i++) {
        if (strcmp(g_vgpu_type_infos[i].name, name) == 0) {
            mutex_unlock(&g_vgpu_type_lock);
            return &g_vgpu_type_infos[i];
        }
    }
    mutex_unlock(&g_vgpu_type_lock);
    return NULL;
}
EXPORT_SYMBOL(nv_vgpu_find_type_info);

static void nv_vgpu_create_static_types(void)
{
    int i;
    struct nv_vgpu_type_info static_types[4] = {
        { .name = "nvidia-vgpu-ampere", .description = "NVIDIA Ampere vGPU", .vdev_id = 0x10de2235, .pdev_id = 0x10de2235, .max_instances = 1 },
        { .name = "nvidia-vgpu-hopper", .description = "NVIDIA Hopper vGPU", .vdev_id = 0x10de2331, .pdev_id = 0x10de2331, .max_instances = 1 },
        { .name = "nvidia-vgpu-turing", .description = "NVIDIA Turing vGPU", .vdev_id = 0x10de1ed3, .pdev_id = 0x10de1ed3, .max_instances = 1 },
        { .name = "nvidia-vgpu-pascal", .description = "NVIDIA Pascal vGPU", .vdev_id = 0x10de1db1, .pdev_id = 0x10de1db1, .max_instances = 1 },
    };
    
    for (i = 0; i < 4; i++) {
        nv_vgpu_register_type_info(&static_types[i]);
    }
}

int nv_vgpu_create_mdev_types(void)
{
    int i, ret;
    
    for (i = 0; i < g_num_vgpu_types; i++) {
        nv_vgpu_mdev_types[i] = kzalloc(sizeof(struct mdev_type), GFP_KERNEL);
        if (!nv_vgpu_mdev_types[i]) {
            ret = -ENOMEM;
            goto err_types;
        }
        nv_vgpu_mdev_types[i]->sysfs_name = kstrdup(g_vgpu_type_infos[i].name, GFP_KERNEL);
        nv_vgpu_mdev_types[i]->pretty_name = kstrdup(g_vgpu_type_infos[i].description[0] ? g_vgpu_type_infos[i].description : g_vgpu_type_infos[i].name, GFP_KERNEL);
        if (!nv_vgpu_mdev_types[i]->sysfs_name || !nv_vgpu_mdev_types[i]->pretty_name) {
            ret = -ENOMEM;
            goto err_types;
        }
    }
    
    return 0;
    
err_types:
    for (i = 0; i < g_num_vgpu_types; i++) {
        kfree(nv_vgpu_mdev_types[i]->sysfs_name);
        kfree(nv_vgpu_mdev_types[i]->pretty_name);
        kfree(nv_vgpu_mdev_types[i]);
        nv_vgpu_mdev_types[i] = NULL;
    }
    return ret;
}
EXPORT_SYMBOL(nv_vgpu_create_mdev_types);

void nv_vgpu_free_type_info(void)
{
    int i;
    for (i = 0; i < g_num_vgpu_types; i++) {
        if (!nv_vgpu_mdev_types[i])
            continue;
        kfree(nv_vgpu_mdev_types[i]->sysfs_name);
        kfree(nv_vgpu_mdev_types[i]->pretty_name);
        kfree(nv_vgpu_mdev_types[i]);
        nv_vgpu_mdev_types[i] = NULL;
    }
    g_num_vgpu_types = 0;
}
EXPORT_SYMBOL(nv_vgpu_free_type_info);

/* get_ops: called by nvidia-vgpu-vfio to get RM ops */
NV_STATUS nvidia_vgpu_vfio_get_ops(rm_vgpu_vfio_ops_t *ops)
{
	if (!ops)
		return NV_ERR_INVALID_ARGUMENT;
	
	populate_rm_ops();
	memcpy(ops, &g_rm_vgpu_vfio_ops, sizeof(rm_vgpu_vfio_ops_t));
	pr_info("vgsp: get_ops provided RM ops\n");
	return NV_OK;
}
EXPORT_SYMBOL(nvidia_vgpu_vfio_get_ops);

/* set_ops: called by nvidia-vgpu-vfio to provide VFIO ops */
NV_STATUS nvidia_vgpu_vfio_set_ops(vgpu_vfio_ops_t *ops)
{
	if (!ops)
		return NV_ERR_INVALID_ARGUMENT;
	
	g_vgpu_vfio_ops = ops;
	pr_info("vgsp: set_ops received VFIO ops\n");
	return NV_OK;
}
EXPORT_SYMBOL(nvidia_vgpu_vfio_set_ops);

/* mdev driver registration - called by nvidia.ko */
int nv_vgpu_mdev_driver_register(void)
{
	pr_info("vgsp: mdev driver register\n");
	return mdev_register_driver(&nv_vgpu_mdev_driver);
}
EXPORT_SYMBOL(nv_vgpu_mdev_driver_register);

void nv_vgpu_mdev_driver_unregister(void)
{
	pr_info("vgsp: mdev driver unregister\n");
	mdev_unregister_driver(&nv_vgpu_mdev_driver);
}
EXPORT_SYMBOL(nv_vgpu_mdev_driver_unregister);

/* mdev parent device registration - called by nvidia.ko */

int nv_vgpu_mdev_register_parent(struct pci_dev *pdev)
{
	struct mdev_parent *parent;
	int ret;

	pr_info("vgsp: mdev parent register for %s\n", pci_name(pdev));

	if (g_mdev_parent) {
		pr_warn("vgsp: mdev parent already registered\n");
		return 0;
	}

	parent = kzalloc(sizeof(*parent), GFP_KERNEL);
	if (!parent)
		return -ENOMEM;

	parent->dev = &pdev->dev;

	ret = mdev_register_parent(parent, &pdev->dev, &nv_vgpu_mdev_driver,
				   nv_vgpu_mdev_types, g_num_vgpu_types);
	if (ret) {
		pr_err("vgsp: Failed to register mdev parent: %d\n", ret);
		kfree(parent);
		return ret;
	}

	g_mdev_parent = parent;
	pr_info("vgsp: mdev parent registered successfully with %d types\n", g_num_vgpu_types);
	return 0;
}
EXPORT_SYMBOL(nv_vgpu_mdev_register_parent);

void nv_vgpu_mdev_unregister_parent(struct pci_dev *pdev)
{
	pr_info("vgsp: mdev parent unregister for %s\n", pci_name(pdev));

	if (g_mdev_parent) {
		pr_info("vgsp: mdev_unregister_parent: entering\n");
		mdev_unregister_parent(g_mdev_parent);
		pr_info("vgsp: mdev_unregister_parent: returned\n");
		kfree(g_mdev_parent);
		g_mdev_parent = NULL;
	}
}
EXPORT_SYMBOL(nv_vgpu_mdev_unregister_parent);

/* NOTE: No MODULE_SOFTDEP on nvidia - this module is fully standalone.
 * It exports the nvidia_vgpu_vfio_* interface (nvidia.ko imports it when
 * present) and registers the mdev parent on the physical GPU directly.
 * Load order for the matched stack: vgsp.ko BEFORE nvidia.ko. */

MODULE_AUTHOR("Open-IOV Project");
MODULE_DESCRIPTION("Open-IOV vGPU VFIO Mediated Device Driver");
MODULE_LICENSE("GPL v2");
MODULE_VERSION(VGSP_VERSION);

struct nv_vgpu_driver_data nv_vgpu_driver = {
	.lock = __MUTEX_INITIALIZER(nv_vgpu_driver.lock),
	.devices = LIST_HEAD_INIT(nv_vgpu_driver.devices),
	.device_count = 0,
};

static int nv_vgpu_mdev_create(struct mdev_device *mdev)
{
	struct nv_vgpu_device *vgpu;
	struct pci_dev *pdev;
	struct mdev_parent *parent;
	int ret;

	vgpu = vfio_alloc_device(nv_vgpu_device, vfio_dev, &mdev->dev,
				 &nv_vgpu_vfio_dev_ops);
	if (IS_ERR(vgpu))
		return PTR_ERR(vgpu);

 	vgpu->mdev = mdev;
 	vgpu->state = NV_VGPU_STATE_UNINITIALIZED;
 	mutex_init(&vgpu->lock);
 	INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work);
 	INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work);

 	/* Initialize VFIO device registration state */
 	vgpu->vfio_dev_registered = false;

	/* No physical backend reference until init_work starts one */
	vgpu->phys_backend_ref = false;

 	/* Initialize VFIO device open/close tracking */
 	atomic_set(&vgpu->vfio_dev_open_count, 0);
 	init_waitqueue_head(&vgpu->vfio_dev_close_wait);

/* Set drvdata so we can retrieve it in remove */
	dev_set_drvdata(&mdev->dev, vgpu);

	parent = mdev->type->parent;
	if (!parent || !parent->dev) {
		ret = -ENODEV;
		goto err_put_vdev;
	}
	pdev = to_pci_dev(parent->dev);
	if (!pdev) {
		ret = -ENODEV;
		goto err_put_vdev;
	}
vgpu->pdev = pci_dev_get(pdev);

	/* Initialize PCI config space for VFIO mdev */
	ret = nv_vgpu_config_space_init(vgpu, pdev);
	if (ret) {
		nv_vgpu_err(vgpu, "PCI config space init failed: %d", ret);
		goto err_put;
	}

	mutex_lock(&nv_vgpu_driver.lock);
	list_add_tail(&vgpu->list, &nv_vgpu_driver.devices);
	nv_vgpu_driver.device_count++;
	mutex_unlock(&nv_vgpu_driver.lock);

	/* Register VFIO device (required for VFIO_DEVICE_API_PCI_STRING).
	 * This creates a new IOMMU group for the emulated device.
	 * Do NOT add mdev to parent's IOMMU group first - vfio_register_emulated_iommu_dev
	 * creates its own IOMMU group for the emulated device. */
	ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_dev);
	if (ret) {
		nv_vgpu_err(vgpu, "Failed to register VFIO device: %d", ret);
		goto err_list_del;
	}
	vgpu->vfio_dev_registered = true;

	schedule_work(&vgpu->init_work);

	return 0;

 err_list_del:
 	mutex_lock(&nv_vgpu_driver.lock);
 	list_del(&vgpu->list);
 	nv_vgpu_driver.device_count--;
 	mutex_unlock(&nv_vgpu_driver.lock);
 err_put:
 	pci_dev_put(vgpu->pdev);
 err_put_vdev:
 	vfio_put_device(&vgpu->vfio_dev);
 	return ret;
}

static void nv_vgpu_mdev_remove(struct mdev_device *mdev)
{
	struct nv_vgpu_device *vgpu = dev_get_drvdata(&mdev->dev);
	long timeout;

	if (!vgpu) {
		printk(KERN_ERR "vgsp: No vGPU device data found for mdev %s\n",
		       dev_name(&mdev->dev));
		return;
	}

	pr_info("vgsp: mdev remove started for %s\n", dev_name(&mdev->dev));

	/* Cancel pending/running init work first */
	cancel_work_sync(&vgpu->init_work);

	/* Wait for any open file descriptors to close before proceeding.
	 * The VFIO device must not be unregistered while userspace has it open,
	 * otherwise vfio_device_group_unregister can crash with NULL pointer deref.
	 * vfio_unregister_group_dev() itself blocks until fds drain, so this
	 * pre-wait exists to surface a clear warning if userspace is stuck. */
	pr_info("vgsp: waiting for vfio_dev_open_count to reach 0 (current: %d)\n",
		atomic_read(&vgpu->vfio_dev_open_count));

	timeout = wait_event_timeout(vgpu->vfio_dev_close_wait,
				     atomic_read(&vgpu->vfio_dev_open_count) == 0,
				     msecs_to_jiffies(5000));
	if (timeout == 0) {
		pr_warn("vgsp: timeout waiting for vfio device close (open_count=%d), proceeding anyway\n",
			atomic_read(&vgpu->vfio_dev_open_count));
	} else {
		pr_info("vgsp: all vfio device file descriptors closed\n");
	}

	/* Unregister the VFIO device. This removes it from its IOMMU group,
	 * deletes the cdev, and waits for any remaining userspace fds to
	 * close. After this returns, no userspace can access the device. */
	if (vgpu->vfio_dev_registered) {
		vfio_unregister_group_dev(&vgpu->vfio_dev);
		vgpu->vfio_dev_registered = false;
	}

	/* Now that no userspace can touch the device, release all per-device
	 * resources (GSP emulation thread, memory, IRQs, BARs, migration).
	 * cleanup_work is never scheduled elsewhere, so invoke it directly. */
	nv_vgpu_device_cleanup_work(&vgpu->cleanup_work);

	/* Remove from driver device list (get_available accounts per-type) */
	mutex_lock(&nv_vgpu_driver.lock);
	list_del(&vgpu->list);
	nv_vgpu_driver.device_count--;
	mutex_unlock(&nv_vgpu_driver.lock);

	pci_dev_put(vgpu->pdev);
	vgpu->pdev = NULL;

	/* Balance vfio_alloc_device() in create; frees the vgpu struct once
	 * the vfio_device refcount drops to zero (ops->release fires first). */
	vfio_put_device(&vgpu->vfio_dev);

	pr_info("vgsp: mdev remove callback completed for %s\n", dev_name(&mdev->dev));
}

/* VFIO device operations implementations */
static int nv_vgpu_vfio_dev_init(struct vfio_device *vdev)
{
	pr_info("vgsp: VFIO device init\n");
	return 0;
}

static void nv_vgpu_vfio_dev_release(struct vfio_device *vdev)
{
	pr_info("vgsp: VFIO device release\n");
}

static int nv_vgpu_vfio_dev_open(struct vfio_device *vdev)
{
	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
	
	pr_info("vgsp: VFIO device open\n");
	atomic_inc(&vgpu->vfio_dev_open_count);
	return 0;
}

static void nv_vgpu_vfio_dev_close(struct vfio_device *vdev)
{
	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
	int i;

	pr_info("vgsp: VFIO device close\n");

	/* Release eventfd trigger references wired via SET_IRQS; the next
	 * opener re-wires its own. Prevents fd/ctx leaks across VM restarts. */
	for (i = 0; i < NV_VGPU_MAX_IRQS; i++) {
		if (vgpu->irqs[i].trigger) {
			eventfd_ctx_put(vgpu->irqs[i].trigger);
			vgpu->irqs[i].trigger = NULL;
		}
	}

	atomic_dec(&vgpu->vfio_dev_open_count);
	wake_up(&vgpu->vfio_dev_close_wait);
}

static ssize_t nv_vgpu_vfio_dev_read(struct vfio_device *vdev, char __user *buf,
				     size_t count, loff_t *ppos)
{
	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
	loff_t pos = *ppos;
	
	if (gsp_sema_dbg)
		pr_info("vgsp: VFIO device read count=%zu pos=%lld (0x%llx)\n",
			count, pos, pos);

	/* Config space region: VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_CONFIG_REGION_INDEX) = 0x70000000000 */
	if (pos >= 0x70000000000ULL && pos < 0x70000000000ULL + 4096) {
		loff_t cfg_pos = pos - 0x70000000000ULL;
		size_t to_read = min(count, (size_t)(4096 - cfg_pos));
		if (copy_to_user(buf, vgpu->pci_config + cfg_pos, to_read))
			return -EFAULT;
		*ppos += to_read;
		return to_read;
	}

	/* BAR0 region (register space). Most guest accesses hit the mmap'd shadow
	 * directly, but region reads also arrive here. They must be served from
	 * the shadow (and the trap table) - refusing them makes the guest driver
	 * poll a register forever. */
	if (pos >= 0 && (u64)pos < vgpu->bars[0].size && count <= 8) {
		u8 tmp[8];
		int ret;

		if (!vgpu->bar_emul_ctx)
			return -ENODEV;
		if (count > sizeof(tmp))
			return -EINVAL;

		ret = nv_vgpu_bar_emul_read(vgpu, 0, (u64)pos, tmp, count);
		if (ret)
			return ret;
		if (copy_to_user(buf, tmp, count))
			return -EFAULT;
		*ppos += count;
		return count;
	}

	/* BAR1 region (VRAM): read from the emulated vidmem backing. */
	if (pos >= (1ULL << 40) &&
	    (u64)pos < (1ULL << 40) + vgpu->bars[1].size &&
	    vgpu->bars[1].vaddr && count <= 8) {
		u64 bar1_off = (u64)pos - (1ULL << 40);

		if (copy_to_user(buf, vgpu->bars[1].vaddr + bar1_off, count))
			return -EFAULT;
		*ppos += count;
		return count;
	}

	return 0;
}

static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __user *buf,
				      size_t count, loff_t *ppos)
{
	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
	loff_t pos = *ppos;
	
	if (gsp_sema_dbg)
		pr_info_ratelimited("vgsp: VFIO device write count=%zu pos=%lld (0x%llx)\n",
				    count, pos, pos);

	/* Config space region - route through the per-register filter so the
	 * guest can only modify writable registers (not vendor/device IDs,
	 * BAR addresses, capabilities structures, etc.) */
	if (pos >= 0x70000000000ULL && pos < 0x70000000000ULL + 4096) {
		loff_t cfg_pos = pos - 0x70000000000ULL;
		ssize_t ret;

		ret = nv_vgpu_config_write(vgpu, buf, count, &cfg_pos);
		if (ret > 0)
			*ppos += ret;
		return ret;
	}

	/* BAR0 region (register space). Most guest accesses hit the mmap'd
	 * shadow directly, but region writes also arrive here (e.g. when QEMU
	 * handles a trapped access). They must be applied to the shadow and to
	 * the trap table - dropping them makes the guest driver poll a
	 * register that never changes and spin forever. */
	if (pos >= 0 && (u64)pos < vgpu->bars[0].size && count <= 8) {
		u8 tmp[8];
		ssize_t ret;
		static u32 bar0_log;

		if (!vgpu->bar_emul_ctx)
			return -ENODEV;
		if (copy_from_user(tmp, buf, count))
			return -EFAULT;

		if ((u64)pos == 0x88704 || (gsp_sema_dbg && bar0_log < 64)) {
			static u32 db_log;
			u64 v = 0;

			memcpy(&v, tmp, count);

			if ((u64)pos == 0x88704) {
				if (gsp_sema_dbg && db_log < 512) {
					pr_info("vgsp: DOORBELL write pos=0x%llx size=%zu val=0x%llx\n",
						(u64)pos, count, v);
					db_log++;
				}
				if (count == 4)
					gsp_sema_service_doorbell((u32)v);
			} else {
				pr_info("vgsp: BAR0 write pos=0x%llx size=%zu val=0x%llx\n",
					(u64)pos, count, v);
				bar0_log++;
			}
		}

		ret = nv_vgpu_bar_emul_write(vgpu, 0, (u64)pos, tmp, count);
		if (ret)
			return ret;
		*ppos += count;
		return count;
	}

	/* BAR1 region (VRAM): same for the emulated vidmem backing. */
	if (pos >= (1ULL << 40) &&
	    (u64)pos < (1ULL << 40) + vgpu->bars[1].size &&
	    vgpu->bars[1].vaddr && count <= 8) {
		u64 bar1_off = (u64)pos - (1ULL << 40);
		static u32 bar1_log;

		if (gsp_sema_dbg && bar1_log < 64) {
			u64 v = 0;

			memcpy(&v, buf, count);

			pr_info("vgsp: BAR1 write off=0x%llx size=%zu val=0x%llx\n",
				bar1_off, count, v);
			bar1_log++;
		}

		if (copy_from_user(vgpu->bars[1].vaddr + bar1_off, buf, count))
			return -EFAULT;
		*ppos += count;
		return count;
	}

	return 0;
}

static long nv_vgpu_vfio_dev_ioctl(struct vfio_device *vdev, unsigned int cmd,
				   unsigned long arg)
{
	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
	unsigned long minsz;
	int ret;
	
	pr_info("vgsp: VFIO device ioctl cmd=0x%x\n", cmd);

	if (cmd == VFIO_DEVICE_GET_INFO) {
		struct vfio_device_info info;

		minsz = offsetofend(struct vfio_device_info, num_irqs);

		pr_info("vgsp: VFIO_DEVICE_GET_INFO minsz=%lu argsz=%u\n", minsz, info.argsz);

		if (copy_from_user(&info, (void __user *)arg, minsz)) {
			pr_err("vgsp: VFIO_DEVICE_GET_INFO copy_from_user failed\n");
			return -EFAULT;
		}

		pr_info("vgsp: VFIO_DEVICE_GET_INFO argsz=%u minsz=%lu\n", info.argsz, minsz);

		if (info.argsz < minsz) {
			pr_err("vgsp: VFIO_DEVICE_GET_INFO argsz too small\n");
			return -EINVAL;
		}

		info.flags = VFIO_DEVICE_FLAGS_PCI | VFIO_DEVICE_FLAGS_RESET;
		info.num_regions = VFIO_PCI_NUM_REGIONS;
		info.num_irqs = VFIO_PCI_NUM_IRQS;

		pr_info("vgsp: VFIO_DEVICE_GET_INFO returning flags=0x%x num_regions=%u num_irqs=%u\n",
			info.flags, info.num_regions, info.num_irqs);

		ret = copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		pr_info("vgsp: VFIO_DEVICE_GET_INFO copy_to_user ret=%d\n", ret);
		return ret;
	}

	if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
		struct vfio_region_info info;

		minsz = offsetofend(struct vfio_region_info, offset);

		if (copy_from_user(&info, (void __user *)arg, minsz))
			return -EFAULT;

		if (info.argsz < minsz)
			return -EINVAL;

		/* PCI config space region */
		if (info.index == VFIO_PCI_CONFIG_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = 4096; /* 4KB config space */
			info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE;
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* BAR0 - GPU registers */
		if (info.index == VFIO_PCI_BAR0_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = vgpu->bars[0].size ? vgpu->bars[0].size : (256 * 1024 * 1024); /* 256MB default */
			info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* BAR1 - VRAM (if present) */
		if (info.index == VFIO_PCI_BAR1_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = vgpu->bars[1].size ? vgpu->bars[1].size : 0;
			if (info.size) {
				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
			} else {
				info.flags = 0;
			}
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* BAR2 - Additional BAR (if present) */
		if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = vgpu->bars[2].size ? vgpu->bars[2].size : 0;
			if (info.size) {
				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
			} else {
				info.flags = 0;
			}
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* BAR3 - Additional BAR (if present) */
		if (info.index == VFIO_PCI_BAR3_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = vgpu->bars[3].size ? vgpu->bars[3].size : 0;
			if (info.size) {
				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
			} else {
				info.flags = 0;
			}
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* BAR4 - Additional BAR (if present) */
		if (info.index == VFIO_PCI_BAR4_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = vgpu->bars[4].size ? vgpu->bars[4].size : 0;
			if (info.size) {
				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
			} else {
				info.flags = 0;
			}
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* BAR5 - Additional BAR (if present) */
		if (info.index == VFIO_PCI_BAR5_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = vgpu->bars[5].size ? vgpu->bars[5].size : 0;
			if (info.size) {
				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
			} else {
				info.flags = 0;
			}
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* ROM region */
		if (info.index == VFIO_PCI_ROM_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = 0;
			info.flags = 0;
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* VGA region */
		if (info.index == VFIO_PCI_VGA_REGION_INDEX) {
			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
			info.size = 0;
			info.flags = 0;
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		return -EINVAL;
	}

	if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
		struct vfio_irq_info info;

		minsz = offsetofend(struct vfio_irq_info, count);

		if (copy_from_user(&info, (void __user *)arg, minsz))
			return -EFAULT;

		if (info.argsz < minsz)
			return -EINVAL;

		/* INTx */
		if (info.index == VFIO_PCI_INTX_IRQ_INDEX) {
			info.flags = VFIO_IRQ_INFO_EVENTFD;
			info.count = 1;
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* MSI */
		if (info.index == VFIO_PCI_MSI_IRQ_INDEX) {
			info.flags = VFIO_IRQ_INFO_EVENTFD;
			info.count = 1;
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* MSI-X */
		if (info.index == VFIO_PCI_MSIX_IRQ_INDEX) {
			info.flags = VFIO_IRQ_INFO_EVENTFD | VFIO_IRQ_INFO_AUTOMASKED;
			info.count = 32; /* 32 MSI-X vectors */
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		/* ERR IRQ */
		if (info.index == VFIO_PCI_ERR_IRQ_INDEX) {
			info.flags = VFIO_IRQ_INFO_EVENTFD;
			info.count = 1;
			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
		}

		return -EINVAL;
	}

	if (cmd == VFIO_DEVICE_SET_IRQS) {
		struct vfio_irq_set *irq_set_ptr;
		struct vfio_irq_set irq_set_hdr;
		unsigned long minsz = offsetof(struct vfio_irq_set, data);
		int *fds = NULL;

		pr_info("vgsp: VFIO_DEVICE_SET_IRQS\n");

		/* First copy the fixed part to get argsz */
		if (copy_from_user(&irq_set_hdr, (void __user *)arg, minsz)) {
			pr_err("vgsp: SET_IRQS copy_from_user hdr failed\n");
			return -EFAULT;
		}

		pr_info("vgsp: SET_IRQS hdr: argsz=%u flags=0x%x index=%u start=%u count=%u\n",
			irq_set_hdr.argsz, irq_set_hdr.flags, irq_set_hdr.index, irq_set_hdr.start, irq_set_hdr.count);

		if (irq_set_hdr.argsz < minsz) {
			pr_err("vgsp: SET_IRQS argsz too small: %u < %lu\n", irq_set_hdr.argsz, minsz);
			return -EINVAL;
		}

		/* Allocate buffer for full struct including data array */
		irq_set_ptr = kmalloc(irq_set_hdr.argsz, GFP_KERNEL);
		if (!irq_set_ptr) {
			pr_err("vgsp: SET_IRQS kmalloc failed\n");
			return -ENOMEM;
		}

		if (copy_from_user(irq_set_ptr, (void __user *)arg, irq_set_hdr.argsz)) {
			pr_err("vgsp: SET_IRQS copy_from_user full failed\n");
			kfree(irq_set_ptr);
			return -EFAULT;
		}

		pr_info("vgsp: SET_IRQS full: argsz=%u flags=0x%x index=%u start=%u count=%u\n",
			irq_set_ptr->argsz, irq_set_ptr->flags, irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count);

		/* Validate data type */
		if ((irq_set_ptr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) == 0 ||
		    (irq_set_ptr->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) == 0) {
			pr_err("vgsp: SET_IRQS invalid flags\n");
			kfree(irq_set_ptr);
			return -EINVAL;
		}

		/* Handle DATA_EVENTFD */
		if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_EVENTFD) {
			int i, idx, wired = 0;

			if (irq_set_ptr->count == 0) {
				kfree(irq_set_ptr);
				return -EINVAL;
			}

			if (irq_set_ptr->index > VFIO_PCI_REQ_IRQ_INDEX ||
			    irq_set_ptr->start + irq_set_ptr->count > NV_VGPU_MAX_IRQS) {
				pr_err("vgsp: SET_IRQS out of range: index=%u start=%u count=%u\n",
				       irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count);
				kfree(irq_set_ptr);
				return -EINVAL;
			}

			fds = kcalloc(irq_set_ptr->count, sizeof(int), GFP_KERNEL);
			if (!fds) {
				kfree(irq_set_ptr);
				return -ENOMEM;
			}

			/* Copy eventfds directly from userspace (after the fixed header) */
			if (copy_from_user(fds, (void __user *)arg + minsz, irq_set_ptr->count * sizeof(int))) {
				pr_err("vgsp: SET_IRQS copy_from_user data failed\n");
				kfree(fds);
				kfree(irq_set_ptr);
				return -EFAULT;
			}

			/* Wire each eventfd to its interrupt vector trigger.
			 * fd >= 0: take a reference and store; fd == -1: clear.
			 * The GSP emulation engine fires vector 0 via
			 * gsp_emu_inject_completion_interrupt(). */
			for (i = 0; i < irq_set_ptr->count; i++) {
				idx = irq_set_ptr->start + i;

				if (fds[i] >= 0) {
					struct eventfd_ctx *trigger;

					trigger = eventfd_ctx_fdget(fds[i]);
					if (IS_ERR(trigger)) {
						pr_err("vgsp: SET_IRQS bad eventfd %d for vector %d\n",
						       fds[i], idx);
						continue;
					}
					if (vgpu->irqs[idx].trigger)
						eventfd_ctx_put(vgpu->irqs[idx].trigger);
					vgpu->irqs[idx].trigger = trigger;
					vgpu->irqs[idx].type = irq_set_ptr->index;
					wired++;
				} else if (fds[i] == -1) {
					if (vgpu->irqs[idx].trigger) {
						eventfd_ctx_put(vgpu->irqs[idx].trigger);
						vgpu->irqs[idx].trigger = NULL;
					}
				}
			}

			pr_info("vgsp: SET_IRQS wired %d/%d eventfds (index=%d start=%d)\n",
				wired, irq_set_ptr->count, irq_set_ptr->index, irq_set_ptr->start);

			kfree(fds);
			kfree(irq_set_ptr);
			return 0;
		}

		/* Handle DATA_NONE (mask/unmask) */
		if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_NONE) {
			pr_info("vgsp: SET_IRQS DATA_NONE index=%d start=%d count=%d action=%x\n",
				irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count, irq_set_ptr->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK);
			kfree(irq_set_ptr);
			return 0;
		}

		/* Handle DATA_BOOL */
		if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_BOOL) {
			pr_info("vgsp: SET_IRQS DATA_BOOL index=%d start=%d count=%d\n",
				irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count);
			kfree(irq_set_ptr);
			return 0;
		}

		kfree(irq_set_ptr);
		return -EINVAL;
	}

	return -ENOTTY;
}

/* Convert any kernel virtual address (lowmem or vmalloc) to a PFN for mmap.
 * virt_to_phys() is only valid for direct-mapped lowmem; vmalloc/vmap
 * addresses (vzalloc, ioremap, kvzalloc fallback) must go through
 * vmalloc_to_pfn(). Using virt_to_phys() on a vmalloc address produces a
 * garbage PFN, which KVM then refuses to map (EPT reserved-bit WARN) and
 * the guest faults/hangs touching the BAR. */
static unsigned long nv_vgpu_mmap_virt_to_pfn(const void *vaddr)
{
	if (is_vmalloc_addr(vaddr))
		return vmalloc_to_pfn(vaddr);
	return virt_to_phys((void *)vaddr) >> PAGE_SHIFT;
}

static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct *vma)
{
	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
	unsigned long size = vma->vm_end - vma->vm_start;
	unsigned long pfn;
	void *vaddr;
	struct nv_vgpu_bar_emul_ctx *bar_ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
	unsigned long region_base;
	unsigned long region_size;

	pr_info("vgsp: VFIO device mmap offset=0x%lx size=0x%lx\n", offset, size);

	/* Config space region */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_CONFIG_REGION_INDEX);
	region_size = 4096;
	if (offset >= region_base && offset < region_base + region_size) {
		if (offset + size > region_base + region_size)
			return -EINVAL;
		vaddr = vgpu->pci_config + (offset - region_base);
		pfn = nv_vgpu_mmap_virt_to_pfn(vaddr);
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
	}

	/* BAR0 region - map to BAR0 emulation shadow */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);
	region_size = bar_ctx ? bar_ctx->bar0_shadow_size : (256 * 1024 * 1024);
	if (offset >= region_base && offset < region_base + region_size) {
		unsigned long shadow_off = offset - region_base;
		unsigned long pos;

		if (!bar_ctx || !bar_ctx->bar0_shadow)
			return -ENODEV;
		if (offset + size > region_base + region_size)
			return -EINVAL;
		/* bar0_shadow is vzalloc'd: physically non-contiguous pages, so
		 * map one page at a time via vmalloc_to_pfn(). */
		vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
		for (pos = 0; pos < size; pos += PAGE_SIZE) {
			pfn = vmalloc_to_pfn(bar_ctx->bar0_shadow + shadow_off + pos);
			if (remap_pfn_range(vma, vma->vm_start + pos, pfn, PAGE_SIZE,
					    vma->vm_page_prot))
				return -EAGAIN;
		}
		/* Debug: log PRAMIN window mapping for BAR2 verify debugging */
		if (shadow_off <= 0x70e000 && shadow_off + size > 0x70e000) {
			pr_info("vgsp: BAR0 mmap covers PRAMIN[0xe000] shadow_off=0x%lx size=0x%lx\n",
				shadow_off, size);
		}
		return 0;
	}

	/* BAR1 region - VRAM (emulated vidmem buffer, per-page vmalloc_to_pfn) */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR1_REGION_INDEX);
	region_size = vgpu->bars[1].size;
	if (offset >= region_base && offset < region_base + region_size) {
		unsigned long vram_off = offset - region_base;
		unsigned long pos;

		if (!vgpu->bars[1].vaddr || region_size == 0)
			return -ENODEV;
		if (offset + size > region_base + region_size)
			return -EINVAL;
		/* vidmem buffer is vzalloc'd: physically non-contiguous pages, so
		 * map one page at a time via vmalloc_to_pfn() (like BAR0 shadow).
		 * Guest VRAM accesses hit our clean host-RAM backing, not the
		 * host GPU's real BAR1. */
		vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
		for (pos = 0; pos < size; pos += PAGE_SIZE) {
			pfn = vmalloc_to_pfn(vgpu->bars[1].vaddr + vram_off + pos);
			if (remap_pfn_range(vma, vma->vm_start + pos, pfn, PAGE_SIZE,
					    vma->vm_page_prot))
				return -EAGAIN;
		}
		return 0;
	}

	/* BAR2 region - emulated shadow (emulation-only) or real BAR (claim) */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR2_REGION_INDEX);
	region_size = vgpu->bars[2].size;
	if (offset >= region_base && offset < region_base + region_size) {
		unsigned long bar2_off = offset - region_base;
		unsigned long pos;

		if (!vgpu->bars[2].vaddr || region_size == 0)
			return -ENODEV;
		if (offset + size > region_base + region_size)
			return -EINVAL;
		if (vgpu->bars[2].emulated) {
			/* Emulation-only: map the private shadow's own pages,
			 * never the physical GPU's BAR2 PFNs. */
			vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
			vma->vm_page_prot =
				pgprot_writecombine(vma->vm_page_prot);
			for (pos = 0; pos < size; pos += PAGE_SIZE) {
				pfn = vmalloc_to_pfn(vgpu->bars[2].vaddr +
						     bar2_off + pos);
				if (remap_pfn_range(vma, vma->vm_start + pos,
						    pfn, PAGE_SIZE,
						    vma->vm_page_prot))
					return -EAGAIN;
			}
			return 0;
		}
		pfn = (vgpu->bars[2].gpa + bar2_off) >> PAGE_SHIFT;
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
	}

	/* BAR3 region - emulated shadow buffer (not physical GPU BAR) */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR3_REGION_INDEX);
	region_size = vgpu->bars[3].size;
	if (offset >= region_base && offset < region_base + region_size) {
		unsigned long bar3_off = offset - region_base;
		unsigned long pos;

		if (!vgpu->bars[3].vaddr || region_size == 0)
			return -ENODEV;
		if (offset + size > region_base + region_size)
			return -EINVAL;
		/* BAR3 shadow is vzalloc'd: per-page vmalloc_to_pfn like BAR0 shadow.
		 * Guest BAR3 accesses hit our clean host-RAM backing, not the
		 * host GPU's real instance memory aperture. */
		vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
		for (pos = 0; pos < size; pos += PAGE_SIZE) {
			pfn = vmalloc_to_pfn(vgpu->bars[3].vaddr + bar3_off + pos);
			if (remap_pfn_range(vma, vma->vm_start + pos, pfn, PAGE_SIZE,
					    vma->vm_page_prot))
				return -EAGAIN;
		}
		return 0;
	}

	/* BAR4 region - map backing physical address if present */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR4_REGION_INDEX);
	region_size = vgpu->bars[4].size;
	if (offset >= region_base && offset < region_base + region_size) {
		if (!vgpu->bars[4].vaddr || region_size == 0)
			return -ENODEV;
		if (offset + size > region_base + region_size)
			return -EINVAL;
		pfn = (vgpu->bars[4].gpa + (offset - region_base)) >> PAGE_SHIFT;
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
	}

	/* BAR5 region - map backing physical address if present */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR5_REGION_INDEX);
	region_size = vgpu->bars[5].size;
	if (offset >= region_base && offset < region_base + region_size) {
		if (!vgpu->bars[5].vaddr || region_size == 0)
			return -ENODEV;
		if (offset + size > region_base + region_size)
			return -EINVAL;
		pfn = (vgpu->bars[5].gpa + (offset - region_base)) >> PAGE_SHIFT;
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
	}

	/* ROM region */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);
	region_size = 0;
	if (offset >= region_base && offset < region_base + region_size) {
		return -EINVAL;
	}

	/* VGA region */
	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_VGA_REGION_INDEX);
	region_size = 0;
	if (offset >= region_base && offset < region_base + region_size) {
		return -EINVAL;
	}

	pr_err("vgsp: mmap failed for unknown offset 0x%lx\n", offset);
	return -EINVAL;
}

static void nv_vgpu_vfio_dev_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)
{
	pr_info("vgsp: VFIO device dma_unmap iova=0x%llx len=0x%llx\n", iova, length);
}

static int nv_vgpu_vfio_dev_match(struct vfio_device *vdev, char *buf)
{
	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
	
	/* Match by mdev UUID */
	if (vgpu->mdev && !strcmp(dev_name(&vgpu->mdev->dev), buf)) {
		return 1;
	}
	
	/* Also match by vfio device name (vfio0, vfio1, etc.) */
	if (!strcmp(dev_name(&vdev->device), buf)) {
		return 1;
	}
	
	return 0;
}

static int __init nv_vgpu_init(void)
{
	int ret;
	struct pci_dev *pdev = NULL;

	pr_info("vgsp: init started\n");

	nv_vgpu_driver.class = class_create("nvidia-vgpu");
	if (IS_ERR(nv_vgpu_driver.class)) {
		ret = PTR_ERR(nv_vgpu_driver.class);
		pr_err("vgsp: Failed to create nvidia-vgpu class: %d\n", ret);
		return ret;
	}
	pr_info("vgsp: class created\n");

	/* Create static vGPU types as fallback (userspace can override via ioctl) */
	nv_vgpu_create_static_types();
	pr_info("vgsp: static types created\n");

	/* Create mdev types from registered vGPU types */
	ret = nv_vgpu_create_mdev_types();
	if (ret) {
		pr_err("vgsp: Failed to create mdev types: %d\n", ret);
		goto err_class;
	}
	pr_info("vgsp: mdev types created\n");

	/* Find the parent PCI device to register mdev parent */
	pdev = pci_get_device(0x10de, 0x1ed3, NULL);
	if (!pdev) {
		pr_err("vgsp: Failed to find NVIDIA GPU (10de:1ed3) for mdev parent registration\n");
		ret = -ENODEV;
		goto err_types;
	}
	pr_info("vgsp: found GPU %s\n", pci_name(pdev));

	/* Register mdev parent via nvidia.ko */
	ret = nv_vgpu_mdev_register_parent(pdev);
	if (ret) {
		pr_err("vgsp: Failed to register mdev parent: %d\n", ret);
		goto err_put_pdev;
	}
	pr_info("vgsp: mdev parent registered\n");

	ret = mdev_register_driver(&nv_vgpu_mdev_driver);
	if (ret) {
		pr_err("vgsp: Failed to register mdev driver: %d\n", ret);
		goto err_types;
	}
	pr_info("vgsp: mdev driver registered\n");

	pci_dev_put(pdev);
	pdev = NULL;

	/* Provide VFIO ops to host driver (nvidia.ko) */
	ret = nvidia_vgpu_vfio_set_ops(&nv_vgpu_vfio_ops);
	if (ret) {
		pr_err("vgsp: Failed to set VFIO ops: %d\n", ret);
		goto err_driver;
	}
	pr_info("vgsp: VFIO ops set\n");

	if (gsp_phys_gpu_param && !gsp_phys_mdev_owned_enabled()) {
		ret = gsp_phys_gpu_claim();
		if (ret) {
			pr_warn("vgsp: physical GPU claim failed: %d\n",
				ret);
		} else {
			/*
			 * G0.4 (P2 prep): expose the load-time module-owned
			 * claim as a READY backend session. The join path only
			 * marks the state and schedules the static-config
			 * snapshot probe (a work item); failure is non-fatal,
			 * module load keeps the emulation path. The claim
			 * itself is still released at rmmod.
			 */
			ret = gsp_phys_backend_start();
			if (ret)
				pr_warn("vgsp: physical backend start failed: %d\n",
					ret);
		}
	} else if (gsp_phys_gpu_param) {
		pr_info("vgsp: gsp_phys_mdev_owned=1: physical GPU claim+boot deferred to the first mdev create\n");
	} else if (gsp_phys_mdev_owned_enabled()) {
		pr_warn("vgsp: gsp_phys_mdev_owned=1 requires gsp_phys_gpu=1; physical backend stays off\n");
	}

	/* Test-enabling trigger for the GSP-RM unload RPC plus the read-only
	 * state snapshot; inert unless a stage-3d boot ran on a live claim
	 * (gsp_phys_boot=1). Removed on module exit. Creation failure is
	 * non-fatal. */
	if (gsp_phys_boot_debugfs_init())
		pr_warn("vgsp: gsp-phys debugfs files unavailable\n");

	pr_info("vgsp: NVIDIA vGPU VFIO driver loaded (version %s)\n", NVIDIA_VGPU_VFIO_VERSION);
	/* Start the emulated-GPU progress service: it discovers the guest's
	 * GPU-VA -> guest-PA map and applies copy-engine semaphore releases so
	 * the guest's UVM waits complete. Idle until a VM opens a device. */
	ret = gsp_sema_service_start();
	if (ret)
		pr_warn("vgsp: sema service start failed: %d\n", ret);

	pr_info("vgsp: init complete\n");
	return 0;

err_driver:
	mdev_unregister_driver(&nv_vgpu_mdev_driver);
err_types:
	nv_vgpu_free_type_info();
err_put_pdev:
	if (pdev)
		pci_dev_put(pdev);
err_class:
	class_destroy(nv_vgpu_driver.class);
	return ret;
}

static void __exit nv_vgpu_exit(void)
{
	struct pci_dev *pdev = NULL;
	bool mdev_owned = gsp_phys_mdev_owned_enabled();

	pr_info("vgsp: nv_vgpu_exit: starting cleanup\n");

	/* Break circular dependency with nvidia.ko FIRST
	 * by clearing the VFIO ops pointer. This signals nvidia.ko
	 * to release its references to our module. */
	pr_info("vgsp: nv_vgpu_exit: clearing VFIO ops to break circular dependency\n");
	g_vgpu_vfio_ops = NULL;

	gsp_phys_boot_debugfs_fini();

	if (mdev_owned) {
		/*
		 * Mdev-owned session: gate every live forward first (STOPPING
		 * under the session lock, generation bump), drain the
		 * emulation threads with the mdev unregistration below, and
		 * only then release the claim - release must not race a live
		 * forward. The mdev remove path releases when live mdevs
		 * exist; end_stop() below covers the no-mdev case.
		 */
		gsp_phys_backend_begin_stop();
	} else {
		/*
		 * Module-owned claim (gsp_phys_gpu=1, mdev_owned=0): flush the
		 * static-config probe work item (begin_stop's only effect for a
		 * module-owned session), then keep today's release position
		 * relative to the debugfs teardown and the emulation threads.
		 */
		gsp_phys_backend_begin_stop();
		gsp_phys_gpu_release();
	}

	if (nv_vgpu_driver.device_count > 0) {
		struct nv_vgpu_device *vgpu;
		mutex_lock(&nv_vgpu_driver.lock);
		list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
			if (vgpu->pdev) {
				/*
				 * Take our own reference: the reference held
				 * by the vgpu is dropped by the mdev remove
				 * path, so using it here and putting it below
				 * would underflow the pci_dev refcount.
				 */
				pdev = pci_dev_get(vgpu->pdev);
				break;
			}
		}
		mutex_unlock(&nv_vgpu_driver.lock);
	}

	if (!pdev) {
		pdev = pci_get_device(0x10de, 0x1ed3, NULL);
	}

	pr_info("vgsp: nv_vgpu_exit: stopping sema service\n");
	gsp_sema_service_stop();
	pr_info("vgsp: nv_vgpu_exit: sema service stopped\n");

	pr_info("vgsp: nv_vgpu_exit: unregistering mdev driver\n");
	/* Unregister mdev driver FIRST, then mdev parent. This drains the
	 * mdev remove callbacks, hence the emulation threads. */
	mdev_unregister_driver(&nv_vgpu_mdev_driver);
	pr_info("vgsp: nv_vgpu_exit: mdev driver unregistered\n");

	if (mdev_owned) {
		/* Drain complete: release the mdev-owned claim (no-op when
		 * the mdev remove path already released it). */
		gsp_phys_backend_end_stop();
	}

	/* Unregister mdev parent via nvidia.ko */
	if (pdev) {
		pr_info("vgsp: nv_vgpu_exit: unregistering mdev parent\n");
		nv_vgpu_mdev_unregister_parent(pdev);
		pr_info("vgsp: mdev parent unregistered for %s\n", dev_name(&pdev->dev));
		pci_dev_put(pdev);
		pdev = NULL;
	}

	pr_info("vgsp: nv_vgpu_exit: destroying class\n");
	class_destroy(nv_vgpu_driver.class);
	pr_info("vgsp: nv_vgpu_exit: class destroyed\n");

	pr_info("vgsp: nv_vgpu_exit: freeing type info\n");
	nv_vgpu_free_type_info();
	pr_info("vgsp: nv_vgpu_exit: type info freed\n");

	pr_info("vgsp: NVIDIA vGPU VFIO driver unloaded\n");
}

static void nv_vgpu_device_init_work(struct work_struct *work)
{
    struct nv_vgpu_device *vgpu = container_of(work, struct nv_vgpu_device, init_work);
    int ret;

    mutex_lock(&vgpu->lock);
    vgpu->state = NV_VGPU_STATE_INITIALIZING;
    mutex_unlock(&vgpu->lock);

    ret = nv_vgpu_bar_init(vgpu);
    if (ret) {
        nv_vgpu_err(vgpu, "BAR init failed: %d", ret);
        goto err_state;
    }

    ret = nv_vgpu_irq_init(vgpu);
    if (ret) {
        nv_vgpu_err(vgpu, "IRQ init failed: %d", ret);
        goto err_bar;
    }

    ret = nv_vgpu_memory_init(vgpu);
    if (ret) {
        nv_vgpu_err(vgpu, "Memory init failed: %d", ret);
        goto err_irq;
    }

    /* Physical GSP is managed by host nvidia.ko - skip nv_vgpu_gsp_init to avoid conflict */
    nv_vgpu_info(vgpu, "Skipping physical GSP init (host nvidia.ko manages physical GSP)");

    /* Initialize GSP emulation engine (host-side) for guest GSP RPC handling */
    ret = gsp_emu_init(vgpu);
    if (ret) {
        nv_vgpu_err(vgpu, "GSP emulation init failed: %d", ret);
        goto err_memory;
    }

    ret = nv_vgpu_migration_init(vgpu);
    if (ret) {
        nv_vgpu_warn(vgpu, "Migration init failed: %d", ret);
    }

    /*
     * mdev-owned physical backend: the first mdev create (refcount 0 -> 1)
     * claims and boots the real GPU asynchronously (gsp_phys_backend_start()
     * only schedules the claim/boot work item, so this never blocks). Any
     * failure is fail-open: log it, leave the backend OFF and keep serving
     * the guest from the emulation path.
     */
    if (gsp_phys_mdev_owned_enabled()) {
        ret = gsp_phys_backend_start();
        if (ret) {
            nv_vgpu_warn(vgpu, "Physical backend start refused/failed: %d; continuing on the emulation path",
                         ret);
        } else {
            vgpu->phys_backend_ref = true;
        }
    }

    mutex_lock(&vgpu->lock);
    vgpu->state = NV_VGPU_STATE_READY;
    mutex_unlock(&vgpu->lock);

    nv_vgpu_info(vgpu, "vGPU device initialized successfully");
    return;

err_memory:
    nv_vgpu_memory_cleanup(vgpu);
err_irq:
    nv_vgpu_irq_cleanup(vgpu);
err_bar:
    nv_vgpu_bar_cleanup(vgpu);
err_state:
    mutex_lock(&vgpu->lock);
    vgpu->state = NV_VGPU_STATE_ERROR;
    mutex_unlock(&vgpu->lock);
}

static void nv_vgpu_device_cleanup_work(struct work_struct *work)
{
    struct nv_vgpu_device *vgpu = container_of(work, struct nv_vgpu_device, cleanup_work);

    nv_vgpu_migration_cleanup(vgpu);
    gsp_emu_cleanup(vgpu);

    /*
     * Last-reference backend teardown: gsp_emu_cleanup() above has stopped
     * this mdev's emulation thread, so no forward can be in flight when the
     * claim is released. The backend session stays up until the last mdev
     * removes (refcount 1 -> 0).
     */
    if (vgpu->phys_backend_ref) {
        vgpu->phys_backend_ref = false;
        gsp_phys_backend_stop();
    }

    nv_vgpu_memory_cleanup(vgpu);
    nv_vgpu_irq_cleanup(vgpu);
    nv_vgpu_bar_cleanup(vgpu);

    mutex_lock(&vgpu->lock);
    vgpu->state = NV_VGPU_STATE_UNINITIALIZED;
    mutex_unlock(&vgpu->lock);

    nv_vgpu_info(vgpu, "vGPU device cleaned up");
}

static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id)
{
	pr_info("vgsp: VFIO probe called for vGPU type %u\n", vgpu_type_id);
	return NV_OK;
}

static void nv_vgpu_vfio_remove(struct pci_dev *pdev)
{
	pr_info("vgsp: VFIO remove called\n");
	/* Clear VFIO ops to break circular dependency with nvidia.ko */
	g_vgpu_vfio_ops = NULL;
}

static NV_STATUS nv_vgpu_vfio_inject_interrupt(void *vgpu_dev)
{
	pr_info("vgsp: VFIO inject interrupt called\n");
	return NV_OK;
}

/* ============================================================
 * PCI Config Space for VFIO mdev
 * ============================================================ */

static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev *pdev)
{
	u8 *cfg = vgpu->pci_config;
	u16 vendor_id = 0x10de;   /* NVIDIA */
	u16 device_id = 0x1ed3;   /* TU104 vGPU (default) */
	u16 subsys_vendor = 0x10de;
	u16 subsys_device = 0x12b4;
	u8 revision = 0xa1;
	u8 prog_if = 0x00;
	u8 subclass = 0x00;
	u8 class_code = 0x03;     /* Display controller */
	u8 cache_line = 0x10;
	u8 latency = 0x00;
	u8 header_type = 0x00;    /* Normal device */
	u8 bist = 0x00;
	int i;

	memset(cfg, 0, 4096);

	/* Look up vGPU type info from mdev type name */
	if (vgpu->mdev && vgpu->mdev->type && vgpu->mdev->type->sysfs_name) {
		for (i = 0; i < g_num_vgpu_types; i++) {
			if (strcmp(vgpu->mdev->type->sysfs_name, g_vgpu_type_infos[i].name) == 0) {
				vendor_id = (u16)((g_vgpu_type_infos[i].vdev_id >> 16) & 0xFFFF);
				device_id = (u16)(g_vgpu_type_infos[i].vdev_id & 0xFFFF);
				subsys_vendor = (u16)((g_vgpu_type_infos[i].pdev_id >> 16) & 0xFFFF);
				subsys_device = (u16)(g_vgpu_type_infos[i].pdev_id & 0xFFFF);
				nv_vgpu_info(vgpu, "Using vGPU type: name='%s', vendor=0x%04x, device=0x%04x, subsys_vendor=0x%04x, subsys_device=0x%04x",
					g_vgpu_type_infos[i].name, vendor_id, device_id, subsys_vendor, subsys_device);
				break;
			}
		}
	}

	/* PCI Header (offset 0x00-0x3F) */
	cfg[0x00] = vendor_id & 0xff;
	cfg[0x01] = (vendor_id >> 8) & 0xff;
	cfg[0x02] = device_id & 0xff;
	cfg[0x03] = (device_id >> 8) & 0xff;

	/* Command register (0x04) - Memory + IO + BusMaster */
	cfg[0x04] = 0x07;
	cfg[0x05] = 0x00;

	/* Status register (0x06) */
	cfg[0x06] = 0x10;
	cfg[0x07] = 0x00;

	/* Revision ID (0x08) */
	cfg[0x08] = revision;

	/* Class code (0x09-0x0B) */
	cfg[0x09] = prog_if;
	cfg[0x0a] = subclass;
	cfg[0x0b] = class_code;

	/* Cache line size (0x0C) */
	cfg[0x0c] = cache_line;

	/* Latency timer (0x0D) */
	cfg[0x0d] = latency;

	/* Header type (0x0E) */
	cfg[0x0e] = header_type;

	/* BIST (0x0F) */
	cfg[0x0f] = bist;

	/* BAR0 - 16MB Memory space (32-bit, non-prefetchable) - matches TU104 */
	cfg[0x10] = 0x00;  /* 32-bit, non-prefetchable */
	cfg[0x11] = 0x00;
	cfg[0x12] = 0x00;
	cfg[0x13] = 0x00;

	/* BAR1 - 256MB Memory space (64-bit, prefetchable) - VRAM */
	cfg[0x14] = 0x0c;  /* 64-bit, prefetchable */
	cfg[0x15] = 0x00;
	cfg[0x16] = 0x00;
	cfg[0x17] = 0x00;
	cfg[0x18] = 0x00;  /* Upper 32 bits */
	cfg[0x19] = 0x00;
	cfg[0x1a] = 0x00;
	cfg[0x1b] = 0x00;

	/* BAR2 - Not present (consumed by BAR1 upper half) */
	cfg[0x1c] = 0x00;
	cfg[0x1d] = 0x00;
	cfg[0x1e] = 0x00;
	cfg[0x1f] = 0x00;

	/* BAR3 - 32MB Memory space (64-bit, prefetchable) */
	cfg[0x20] = 0x0c;  /* 64-bit, prefetchable */
	cfg[0x21] = 0x00;
	cfg[0x22] = 0x00;
	cfg[0x23] = 0x00;
	cfg[0x24] = 0x00;  /* Upper 32 bits */
	cfg[0x25] = 0x00;
	cfg[0x26] = 0x00;
	cfg[0x27] = 0x00;

	/* BAR4 - Not present (consumed by BAR3 upper half) */
	cfg[0x28] = 0x00;
	cfg[0x29] = 0x00;
	cfg[0x2a] = 0x00;
	cfg[0x2b] = 0x00;

	/* CardBus CIS Pointer (0x2C) - Not used */
	cfg[0x2c] = 0x00;
	cfg[0x2d] = 0x00;
	cfg[0x2e] = 0x00;
	cfg[0x2f] = 0x00;

	/* Subsystem Vendor ID (0x2E) and Subsystem ID (0x2F) - wait, correct offsets: */
	/* Subsystem Vendor ID at 0x2C, Subsystem ID at 0x2E */
	cfg[0x2c] = subsys_vendor & 0xff;
	cfg[0x2d] = (subsys_vendor >> 8) & 0xff;
	cfg[0x2e] = subsys_device & 0xff;
	cfg[0x2f] = (subsys_device >> 8) & 0xff;

	/* Expansion ROM Base Address (0x30-0x33) - Not used */
	cfg[0x30] = 0x00;
	cfg[0x31] = 0x00;
	cfg[0x32] = 0x00;
	cfg[0x33] = 0x00;

	/* Capabilities Pointer (0x34) - Point to Power Management at 0x40 */
	cfg[0x34] = 0x40;
	cfg[0x35] = 0x00;
	cfg[0x36] = 0x00;
	cfg[0x37] = 0x00;

	/* Interrupt Line (0x3C) - Will be set by guest */
	cfg[0x3c] = 0x00;
	/* Interrupt Pin (0x3D) - INTA */
	cfg[0x3d] = 0x01;
	/* Min_Gnt (0x3E) */
	cfg[0x3e] = 0x00;
	/* Max_Lat (0x3F) */
	cfg[0x3f] = 0x00;

	/* Subsystem Vendor ID (0x3C) */
	cfg[0x3c] = subsys_vendor & 0xff;
	cfg[0x3d] = (subsys_vendor >> 8) & 0xff;

	/* Subsystem ID (0x3E) */
	cfg[0x3e] = subsys_device & 0xff;
	cfg[0x3f] = (subsys_device >> 8) & 0xff;

	/* Expansion ROM Base Address (0x40-0x43) - Not used */

	/* Capabilities Pointer (0x34) - Point to Power Management at 0x40 */
	cfg[0x34] = 0x40;
	cfg[0x35] = 0x00;
	cfg[0x36] = 0x00;
	cfg[0x37] = 0x00;

	/* Interrupt Line (0x3C) - Will be set by guest */
	cfg[0x3c] = 0x00;
	/* Interrupt Pin (0x3D) - INTA */
	cfg[0x3d] = 0x01;
	/* Min_Gnt (0x3E) */
	cfg[0x3e] = 0x00;
	/* Max_Lat (0x3F) */
	cfg[0x3f] = 0x00;

	/* Power Management Capability (0x40) */
	cfg[0x40] = 0x01;  /* Capability ID: Power Management */
	cfg[0x41] = 0x48;  /* Next capability: MSI at 0x48 */
	cfg[0x42] = 0x03;  /* PME support from D0, D3hot, D3cold */
	cfg[0x43] = 0x00;
	cfg[0x44] = 0x00;  /* PM Control/Status */
	cfg[0x45] = 0x00;
	cfg[0x46] = 0x00;
	cfg[0x47] = 0x00;

	/* MSI Capability (0x48) */
	cfg[0x48] = 0x05;  /* Capability ID: MSI */
	cfg[0x49] = 0x68;  /* Next capability: MSI-X at 0x68 */
	cfg[0x4a] = 0x80;  /* 64-bit capable, 1 vector */
	cfg[0x4b] = 0x00;
	cfg[0x4c] = 0x00;  /* Message Address (low) - set by guest */
	cfg[0x4d] = 0x00;
	cfg[0x4e] = 0x00;
	cfg[0x4f] = 0x00;
	cfg[0x50] = 0x00;  /* Message Address (high) - 64-bit */
	cfg[0x51] = 0x00;
	cfg[0x52] = 0x00;
	cfg[0x53] = 0x00;
	cfg[0x54] = 0x01;  /* Message Data - set by guest */
	cfg[0x55] = 0x00;

	/* MSI-X Capability (0x68) - real TU104 puts the PCIe capability at
	 * 0x78, so the MSI-X cap lives at 0x68 (0x68-0x73). This matters:
	 * the driver's NV_PCFG mirror (BAR0+0x88000, mirror size 0x1000)
	 * is intercepted by vgpuDevReadReg032, which forwards a BAR0 read
	 * at 0x88000+off to PCI config space offset `off`
	 * (kbifGetPciConfigSpacePriMirror_GM107 = NV_PCFG = 0x88000). The
	 * driver reads NV_XVE_LINK_CAPABILITIES = 0x84 expecting the PCIe
	 * Link Capabilities register there, which is only true when the
	 * PCIe capability base is 0x78 (cap+0x0C = 0x84). */
	cfg[0x68] = 0x11;  /* Capability ID: MSI-X */
	cfg[0x69] = 0x78;  /* Next capability: PCI Express at 0x78 */
	cfg[0x6a] = 0x1f;  /* Table size = 31 (32 vectors), 64-bit */
	cfg[0x6b] = 0x00;
	/* Table Offset/BIR (0x6c) - Table at BAR0 offset 0x2000 */
	cfg[0x6c] = 0x00;
	cfg[0x6d] = 0x20;
	cfg[0x6e] = 0x00;
	cfg[0x6f] = 0x00;
	/* PBA Offset/BIR (0x70) - PBA at BAR0 offset 0x3000 */
	cfg[0x70] = 0x00;
	cfg[0x71] = 0x30;
	cfg[0x72] = 0x00;
	cfg[0x73] = 0x00;

	/* PCI Express Capability (0x78) - matches the real TU104 layout:
	 *   cap+0x00 0x78 cap header, cap+0x04 0x7C Device Capabilities,
	 *   cap+0x08 0x80 Device Control/Status,
	 *   cap+0x0C 0x84 Link Capabilities  <- NV_XVE_LINK_CAPABILITIES
	 *   cap+0x10 0x88 Link Control/Status <- NV_XVE_LINK_CONTROL_STATUS
	 * Gen3 x16 (MAX_SPEED 3 = 8 GT/s, MAX_WIDTH 0x10 = x16), so UVM's
	 * getPCIELinkRateMBps -> calculatePCIELinkRateMBps succeeds. */
	cfg[0x78] = 0x10;  /* Capability ID: PCI Express */
	cfg[0x79] = 0x00;  /* Next capability: None (end of list) */
	cfg[0x7a] = 0x02;  /* Device type: PCIe Endpoint, version 2 */
	cfg[0x7b] = 0x00;
	cfg[0x7c] = 0x00;  /* Device Capabilities */
	cfg[0x7d] = 0x00;
	cfg[0x7e] = 0x00;
	cfg[0x7f] = 0x00;
	cfg[0x80] = 0x00;  /* Device Control */
	cfg[0x81] = 0x00;
	cfg[0x82] = 0x10;  /* Device Status */
	cfg[0x83] = 0x00;
	cfg[0x84] = 0x03;  /* Link Capabilities: MAX_SPEED 3:0 = 3 (8 GT/s) */
	cfg[0x85] = 0x01;  /* ... MAX_WIDTH 9:4 = 0x10 (x16) -> 0x103 */
	cfg[0x86] = 0x00;
	cfg[0x87] = 0x00;
	cfg[0x88] = 0x00;  /* Link Control */
	cfg[0x89] = 0x00;
	cfg[0x8a] = 0x03;  /* Link Status: speed 3:0 = 3 (Gen3) */
	cfg[0x8b] = 0x01;  /* ... width 9:4 = 0x10 (x16) */
	cfg[0x8c] = 0x00;  /* Slot Capabilities */
	cfg[0x8d] = 0x00;
	cfg[0x8e] = 0x00;
	cfg[0x8f] = 0x00;
	cfg[0x90] = 0x00;  /* Slot Control */
	cfg[0x91] = 0x00;
	cfg[0x92] = 0x00;  /* Slot Status */
	cfg[0x93] = 0x00;
	cfg[0x94] = 0x00;  /* Root Capabilities */
	cfg[0x95] = 0x00;
	cfg[0x96] = 0x00;  /* Root Control */
	cfg[0x97] = 0x00;
	cfg[0x98] = 0x00;  /* Root Status */
	cfg[0x99] = 0x00;
	cfg[0x9a] = 0x00;
	cfg[0x9b] = 0x00;
	cfg[0x9c] = 0x00;  /* Device Capabilities 2 */
	cfg[0x9d] = 0x00;
	cfg[0x9e] = 0x00;
	cfg[0x9f] = 0x00;
	cfg[0xa0] = 0x00;  /* Device Control 2 */
	cfg[0xa1] = 0x00;
	cfg[0xa2] = 0x00;
	cfg[0xa3] = 0x00;
	cfg[0xa4] = 0x00;  /* Link Control 2 */
	cfg[0xa5] = 0x00;
	cfg[0xa6] = 0x00;  /* Link Status 2 */
	cfg[0xa7] = 0x00;

	vgpu->config_space_dirty = false;
	nv_vgpu_info(vgpu, "PCI config space initialized (vendor=0x%04x, device=0x%04x, subsys_vendor=0x%04x, subsys_device=0x%04x)", vendor_id, device_id, subsys_vendor, subsys_device);
	nv_vgpu_info(vgpu, "PCIe cap: base=0x%02x hdr=%02x%02x%02x%02x devcap=%02x%02x%02x%02x devctl=%02x%02x linkcap=%02x%02x%02x%02x linkctl=%02x%02x linksta=%02x%02x",
		     cfg[0x78], cfg[0x78], cfg[0x79], cfg[0x7a], cfg[0x7b],
		     cfg[0x7c], cfg[0x7d], cfg[0x7e], cfg[0x7f],
		     cfg[0x80], cfg[0x81],
		     cfg[0x84], cfg[0x85], cfg[0x86], cfg[0x87],
		     cfg[0x88], cfg[0x89], cfg[0x8a], cfg[0x8b]);
	return 0;
}

static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos)
{
	loff_t pos = *ppos;

	if (pos >= 256)
		return 0;

	if (pos + count > 256)
		count = 256 - pos;

	if (copy_to_user(buf, vgpu->pci_config + pos, count))
		return -EFAULT;

	*ppos += count;
	return count;
}

static ssize_t nv_vgpu_config_write(struct nv_vgpu_device *vgpu, const char __user *buf, size_t count, loff_t *ppos)
{
	loff_t pos = *ppos;
	size_t written = 0;
	u8 val;

	if (pos >= 4096)
		return -EINVAL;

	if (pos + count > 4096)
		count = 4096 - pos;

	/* Handle specific writable registers */
	while (count--) {
		if (copy_from_user(&val, buf++, 1))
			return -EFAULT;

		switch (pos) {
		case 0x04: /* Command register */
			/* Allow Memory, IO, BusMaster */
			vgpu->pci_config[pos] = val & 0x07;
			break;
		case 0x0c: /* Cache line size */
			vgpu->pci_config[pos] = val;
			break;
		case 0x0d: /* Latency timer */
			vgpu->pci_config[pos] = val;
			break;
		case 0x3c: /* Interrupt line */
			vgpu->pci_config[pos] = val;
			break;
		case 0x4c: case 0x4d: case 0x4e: case 0x4f: /* MSI Message Address Low */
		case 0x50: case 0x51: case 0x52: case 0x53: /* MSI Message Address High */
		case 0x54: case 0x55: /* MSI Message Data */
			vgpu->pci_config[pos] = val;
			break;
		case 0x6a: case 0x6b: /* MSI-X Message Control */
			vgpu->pci_config[pos] = val;
			break;
		case 0x6c: case 0x6d: case 0x6e: case 0x6f: /* MSI-X Table Offset/BIR */
			vgpu->pci_config[pos] = val;
			break;
		case 0x70: case 0x71: case 0x72: case 0x73: /* MSI-X PBA Offset/BIR */
			vgpu->pci_config[pos] = val;
			break;
		case 0x80: case 0x81: /* Device Control */
		case 0x88: case 0x89: /* Link Control */
		case 0x90: case 0x91: /* Slot Control */
		case 0x96: case 0x97: /* Root Control */
		case 0xa0: case 0xa1: /* Device Control 2 */
		case 0xa4: case 0xa5: /* Link Control 2 */
			vgpu->pci_config[pos] = val;
			break;
		case 0x100 ... 0xfff:
			/* Extended capability space: accept and store. The guest
			 * driver programs registers here (e.g. 0x704) and retries
			 * forever if the write is refused. */
			vgpu->pci_config[pos] = val;
			break;
		default:
			/* Legacy identity/read-only registers stay fixed */
			break;
		}
		pos++;
		written++;
	}

	vgpu->config_space_dirty = true;
	*ppos = pos;
	return written;
}

/* ============================================================
 * Exported symbols for nvidia.ko (vGPU VFIO interface)
 * ============================================================ */

NV_STATUS nvidia_vgpu_vfio_probe(struct pci_dev *pdev)
{
	pr_info("vgsp: nvidia_vgpu_vfio_probe called for %s\n", pci_name(pdev));
	return NV_OK;
}
EXPORT_SYMBOL(nvidia_vgpu_vfio_probe);

void nvidia_vgpu_vfio_remove(struct pci_dev *pdev, NvBool remove_device)
{
	pr_info("vgsp: nvidia_vgpu_vfio_remove called for %s, remove_device=%d\n",
		pci_name(pdev), remove_device);
}
EXPORT_SYMBOL(nvidia_vgpu_vfio_remove);

/*
 * Debug: scan the guest's RAM for a 64-bit value to locate guest structures
 * without knowing the guest's GPU page tables (see handoff PART 6).
 *   echo 0x121000000 > /sys/module/vgsp/parameters/scan_guest_ram
 * Scans 0..8GB of guest physical address space, 4KB at a time.
 *
 * The scan MUST run in a kthread: vfio_iommu_type1_dma_rw only works when
 * current->mm is NULL (kthread) or matches the mapping's mm (QEMU). From a
 * user process it fails with -EFAULT for every page.
 */
struct nv_vgpu_scan_req {
	u64 needle;
	struct completion done;
};

static int nv_vgpu_scan_thread(void *arg)
{
	struct nv_vgpu_scan_req *req = arg;
	struct nv_vgpu_device *vgpu;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		gsp_rm_scan_guest_ram(vgpu, req->needle, 0, 8ULL << 30, 32);
	}
	mutex_unlock(&nv_vgpu_driver.lock);

	complete(&req->done);
	return 0;
}

static int nv_vgpu_scan_set(const char *val, const struct kernel_param *kp)
{
	static struct nv_vgpu_scan_req req;
	struct task_struct *task;
	u64 needle = 0;
	int ret;

	ret = kstrtoull(val, 0, &needle);
	if (ret)
		return ret;

	req.needle = needle;
	init_completion(&req.done);

	task = kthread_run(nv_vgpu_scan_thread, &req, "vgpu-scan");
	if (IS_ERR(task))
		return PTR_ERR(task);

	wait_for_completion(&req.done);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_scan_ops = {
	.set = nv_vgpu_scan_set,
};

module_param_cb(scan_guest_ram, &nv_vgpu_scan_ops, NULL, 0200);
MODULE_PARM_DESC(scan_guest_ram, "Scan guest RAM for a 64-bit value (hex)");

/*
 * Debug: scan the emulated vidmem (BAR1 shadow, 256MB host buffer) for a
 * 32/64-bit value. Used to find guest page tables / channel structures that
 * the guest writes into its vidmem aperture.
 *   echo 0x21000000 > /sys/module/vgsp/parameters/scan_vidmem
 */
static int nv_vgpu_scan_vidmem_set(const char *val, const struct kernel_param *kp)
{
	struct nv_vgpu_device *vgpu;
	u64 needle = 0;
	int ret;

	ret = kstrtoull(val, 0, &needle);
	if (ret)
		return ret;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;
		size_t off;
		u32 hits = 0;

		if (!base || !size)
			continue;

		for (off = 0; off + 8 <= size && hits < 16; off += 4) {
			u64 v64;
			u32 v32;

			memcpy(&v64, base + off, 8);
			memcpy(&v32, base + off, 4);
			if (v64 == needle) {
				pr_info("vgsp: vidmem scan: u64 0x%llx at vidmem off 0x%zx\n",
					needle, off);
				hits++;
			}
			if (v32 == (u32)needle) {
				pr_info("vgsp: vidmem scan: u32 0x%x at vidmem off 0x%zx\n",
					(u32)needle, off);
				hits++;
			}
		}
		pr_info("vgsp: vidmem scan: needle=0x%llx size=0x%zx hits=%u\n",
			needle, size, hits);
	}
	mutex_unlock(&nv_vgpu_driver.lock);

	return 0;
}

static const struct kernel_param_ops nv_vgpu_scan_vidmem_ops = {
	.set = nv_vgpu_scan_vidmem_set,
};

module_param_cb(scan_vidmem, &nv_vgpu_scan_vidmem_ops, NULL, 0200);
MODULE_PARM_DESC(scan_vidmem, "Scan emulated vidmem for a 32/64-bit value (hex)");

/*
 * Debug: scan guest RAM and emulated vidmem for a sequence of up to 4 u32
 * values (comma-separated hex), e.g. a CE semaphore-release method sequence:
 *   echo 0x240,0x1,0x244,0x21000000 > .../parameters/scan_seq
 */
struct nv_vgpu_seq_req {
	u32 pat[4];
	u32 npat;
	struct completion done;
};

static int nv_vgpu_seq_thread(void *arg)
{
	struct nv_vgpu_seq_req *req = arg;
	struct nv_vgpu_device *vgpu;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		gsp_rm_scan_guest_ram_seq(vgpu, req->pat, req->npat,
					  0, 8ULL << 30, 8);
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	complete(&req->done);
	return 0;
}

static int nv_vgpu_scan_seq_set(const char *val, const struct kernel_param *kp)
{
	static struct nv_vgpu_seq_req req;
	struct task_struct *task;
	char buf[128];
	char *tok, *cur;

	memset(&req, 0, sizeof(req));
	strscpy(buf, val, sizeof(buf));
	cur = buf;
	while (req.npat < 4 && (tok = strsep(&cur, ",")) != NULL) {
		u32 v;

		if (kstrtou32(tok, 0, &v))
			break;
		req.pat[req.npat++] = v;
	}
	if (!req.npat)
		return -EINVAL;

	init_completion(&req.done);
	task = kthread_run(nv_vgpu_seq_thread, &req, "vgpu-seq");
	if (IS_ERR(task))
		return PTR_ERR(task);
	wait_for_completion(&req.done);

	/* Also scan the emulated vidmem (plain memory, no kthread needed) */
	mutex_lock(&nv_vgpu_driver.lock);
	{
		struct nv_vgpu_device *vgpu;

		list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;
			size_t off;
			u32 hits = 0;

			if (!base || !size)
				continue;
			for (off = 0; off + req.npat * 4 <= size; off += 4) {
				u32 i, ok = 1;

				for (i = 0; i < req.npat; i++) {
					u32 v;

					memcpy(&v, base + off + i * 4, 4);
					if (v != req.pat[i]) {
						ok = 0;
						break;
					}
				}
				if (ok && hits < 8) {
					pr_info("vgsp: seq scan: hit in vidmem at off 0x%zx\n",
						off);
					hits++;
				}
			}
			pr_info("vgsp: seq scan: vidmem hits=%u\n", hits);
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);

	return 0;
}

static const struct kernel_param_ops nv_vgpu_scan_seq_ops = {
	.set = nv_vgpu_scan_seq_set,
};

module_param_cb(scan_seq, &nv_vgpu_scan_seq_ops, NULL, 0200);
MODULE_PARM_DESC(scan_seq, "Scan vidmem for a u32 sequence (hex,comma separated)");

/*
 * Debug: dump a BAR shadow (0/1/3) as u32 words:
 *   echo <bar>,<offset>[,<count>] > .../parameters/shadow_dump
 */
static int nv_vgpu_shadow_dump_set(const char *val, const struct kernel_param *kp)
{
	char buf[64];
	char *c1, *c2;
	u32 bar = 0, n = 16;
	u64 off = 0;
	struct nv_vgpu_device *vgpu;

	strscpy(buf, val, sizeof(buf));
	c1 = strchr(buf, ',');
	if (!c1)
		return -EINVAL;
	*c1 = '\0';
	c2 = strchr(c1 + 1, ',');
	if (c2)
		*c2 = '\0';
	if (kstrtou32(buf, 0, &bar) || kstrtoull(c1 + 1, 0, &off))
		return -EINVAL;
	if (c2 && kstrtou32(c2 + 1, 0, &n))
		return -EINVAL;
	if (bar > 3)
		return -EINVAL;
	if (n > 64)
		n = 64;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u8 *base = vgpu->bars[bar].vaddr;
		u64 size = vgpu->bars[bar].size;
		u32 i;

		if (!base || off + (u64)n * 4 > size) {
			pr_info("vgsp: shadow dump: bar%u bad (base=%p size=0x%llx)\n",
				bar, base, size);
			continue;
		}
		for (i = 0; i < n; i++) {
			u32 v;

			memcpy(&v, base + off + i * 4, 4);
			pr_info("vgsp: shadow bar%u[0x%llx+%u] = 0x%08x\n",
				bar, off, i, v);
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_shadow_dump_ops = {
	.set = nv_vgpu_shadow_dump_set,
};
module_param_cb(shadow_dump, &nv_vgpu_shadow_dump_ops, NULL, 0200);
MODULE_PARM_DESC(shadow_dump, "Dump BAR shadow words: bar,offset[,count]");

/*
 * Debug: scan every BAR shadow (BAR0/BAR1/BAR3) for up to 4 u32 patterns:
 *   echo 0x2001001e[,p2,p3,p4] > .../parameters/shadow_scan
 */
static int nv_vgpu_shadow_scan_set(const char *val, const struct kernel_param *kp)
{
	char buf[128];
	char *tok, *cur;
	u32 pat[4], npat = 0;
	struct nv_vgpu_device *vgpu;

	strscpy(buf, val, sizeof(buf));
	cur = buf;
	while (npat < 4 && (tok = strsep(&cur, ",")) != NULL) {
		if (kstrtou32(tok, 0, &pat[npat]))
			break;
		npat++;
	}
	if (!npat)
		return -EINVAL;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u32 b;

		for (b = 0; b < 4; b++) {
			u8 *base = vgpu->bars[b].vaddr;
			u64 size = vgpu->bars[b].size, off;
			u32 hits = 0;

			if (!base || !size)
				continue;
			for (off = 0; off + (u64)npat * 4 <= size; off += 4) {
				u32 i, ok = 1;

				for (i = 0; i < npat; i++) {
					u32 v;

					memcpy(&v, base + off + i * 4, 4);
					if (v != pat[i]) {
						ok = 0;
						break;
					}
				}
				if (ok && hits < 8) {
					pr_info("vgsp: shadow scan: bar%u hit at 0x%llx\n",
						b, off);
					hits++;
				}
			}
			if (hits)
				pr_info("vgsp: shadow scan: bar%u hits=%u\n",
					b, hits);
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_shadow_scan_ops = {
	.set = nv_vgpu_shadow_scan_set,
};
module_param_cb(shadow_scan, &nv_vgpu_shadow_scan_ops, NULL, 0200);
MODULE_PARM_DESC(shadow_scan, "Scan BAR shadows (0/1/3) for a u32 sequence");

/*
 * Debug: dump the semaphore service's view of a GPU VA (VA->PA map entry and
 * the current semaphore slot values):
 *   echo 0x121000010[,count] > .../parameters/sema_probe
 */
static int nv_vgpu_sema_probe_set(const char *val, const struct kernel_param *kp)
{
	char buf[64];
	char *comma;
	u64 va = 0;
	u32 n = 8;

	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (comma) {
		*comma = '\0';
		if (kstrtou32(comma + 1, 0, &n))
			return -EINVAL;
	}
	if (kstrtoull(buf, 0, &va))
		return -EINVAL;

	return gsp_sema_service_probe(va, n);
}

static const struct kernel_param_ops nv_vgpu_sema_probe_ops = {
	.set = nv_vgpu_sema_probe_set,
};

module_param_cb(sema_probe, &nv_vgpu_sema_probe_ops, NULL, 0200);
MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA");

/*
 * Debug: run the page-table walks for a GPU VA with per-level tracing,
 * bypassing the pin/lat/pair map (see gsp_sema_service_walk_probe):
 *   echo 0x20400f000 > .../parameters/walk_probe
 */
static int nv_vgpu_walk_probe_set(const char *val, const struct kernel_param *kp)
{
	u64 va = 0;

	if (kstrtoull(val, 0, &va))
		return -EINVAL;
	return gsp_sema_service_walk_probe(va);
}

static const struct kernel_param_ops nv_vgpu_walk_probe_ops = {
	.set = nv_vgpu_walk_probe_set,
};

module_param_cb(walk_probe, &nv_vgpu_walk_probe_ops, NULL, 0200);
MODULE_PARM_DESC(walk_probe, "Trace the page-table walks for a GPU VA (hex)");

/*
 * Debug: dump the semaphore service's channel table (GPFIFO/USERD state and
 * the two NvNotification slots each channel exposes):
 *   echo 1 > .../parameters/chan_dump
 */
static int nv_vgpu_chan_dump_set(const char *val, const struct kernel_param *kp)
{
	return gsp_sema_service_dump_channels();
}

static const struct kernel_param_ops nv_vgpu_chan_dump_ops = {
	.set = nv_vgpu_chan_dump_set,
};

module_param_cb(chan_dump, &nv_vgpu_chan_dump_ops, NULL, 0200);
MODULE_PARM_DESC(chan_dump, "Dump the sema service channel table (any value)");

/*
 * Physical GPU ownership for the real-GSP execution backend (bring-up rung
 * 1: claim). Default off. On claim the distribution driver is unbound from
 * the physical GPU, this module's pci_driver binds it and the BARs are
 * mapped. Only set when no VM holds an mdev; release happens on unload.
 */
module_param_named(gsp_phys_gpu, gsp_phys_gpu_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_gpu, "Claim the physical GPU for the real-GSP backend (default 0)");

/* Read by the backend session (gsp_phys_gpu.c) to enforce the mdev-owned
 * start gate; the parameter itself stays owned here. */
bool gsp_phys_gpu_enabled(void)
{
	return gsp_phys_gpu_param;
}

/*
 * GR ring discovery: pin the compute channels' rings/USERDs, which follow the
 * CE rings in the channel video-memory allocation. Default on; set 0 to leave
 * GR channels untranslated when debugging push-execution side effects.
 */
module_param_named(sema_gr_rings, gsp_sema_gr_rings, bool, 0644);
MODULE_PARM_DESC(sema_gr_rings, "Auto-discover+pin GR channel rings (default 1)");

/*
 * In-kernel GR completion-counter slot derivation from the structural
 * translations (published page directories, self-discovered directory,
 * targeted rm_mem record), used when a release's counter page has no injected
 * pin yet. Default on; set 0 to restore the injected-pin-only behaviour for
 * an A/B without a rebuild.
 */
module_param_named(sema_counter_derive, gsp_sema_counter_derive, bool, 0644);
MODULE_PARM_DESC(sema_counter_derive, "Derive GR completion-counter slots in-kernel from structural translations (default 1)");

/*
 * Acknowledge releases from unreadable CE pushes directly (tracking-semaphore
 * fallback). The push's data writes are lost either way when the content
 * rescue cannot find the page; writing the expected payload keeps the
 * driver's progress waits from timing out on the rescue latency.
 */
module_param_named(sema_fake_rel, gsp_sema_fake_rel, bool, 0644);
MODULE_PARM_DESC(sema_fake_rel, "Acknowledge unreadable CE releases directly (default 0)");
module_param_named(sema_rel_recover, gsp_sema_rel_recover, bool, 0644);
MODULE_PARM_DESC(sema_rel_recover, "Recover unreadable CE releases after grace + rescue (default 0)");

/*
 * Skip executing the pushed work (CE memsets/copies) while still applying the
 * semaphore releases. The client only observes releases; skipping the shadow
 * writes removes the per-push execution cost that dominates the context-init
 * latency budget.
 */
module_param_named(sema_skip_ops, gsp_sema_skip_ops, bool, 0644);
MODULE_PARM_DESC(sema_skip_ops, "Apply releases without executing pushed memsets (default 0)");

/*
 * Master diagnostic switch for the per-push/per-doorbell/per-op tracing
 * (sema-push, sema-lda, sema-chan rel-apply, sema-grce, DOORBELL, the
 * GSPRM per-RPC lines, ...). Off by default so a healthy boot leaves the
 * kernel log quiet; turn it on when instrumenting a cycle. The watch knobs
 * below (ops_log, copy_watch, copy_pat, copy_dst) only take effect while
 * this is on.
 */
module_param_named(sema_dbg, gsp_sema_dbg, bool, 0644);
MODULE_PARM_DESC(sema_dbg, "Verbose per-push/per-RPC emulation tracing (default 0)");

/*
 * Executed-op log cap: how many CE fills/copies are logged with full
 * operands. Raise it (e.g. 4096) to audit a data path; the default keeps a
 * run quiet.
 */
module_param_named(sema_ops_log, gsp_sema_ops_log, uint, 0644);
MODULE_PARM_DESC(sema_ops_log, "Max executed CE ops logged with operands (default 8)");

/*
 * Data-path watch: log the first source/destination bytes of every CE copy
 * of exactly this size (0 = off), to audit the push-data chain:
 *   echo 256 > .../parameters/sema_copy_watch
 */
module_param_named(sema_copy_watch, gsp_sema_copy_watch, uint, 0644);
MODULE_PARM_DESC(sema_copy_watch, "Copy size whose first bytes are logged (default 0)");

/*
 * Data-path watch: 8-byte value matched against every CE copy's first source
 * and destination words, and scanned for inside every decoded push, so the
 * submission carrying a known payload is named even when its size/encoding is
 * unknown:
 *   echo 0x0000000400000001 > .../parameters/sema_copy_pat
 */
module_param_named(sema_copy_pat, gsp_sema_copy_pat, ullong, 0644);
MODULE_PARM_DESC(sema_copy_pat, "8-byte pattern matched in copy src/dst and pushes (default 0)");

/*
 * Data-path watch: dump the source bytes of copies whose destination falls
 * in this address's page (up to 32 samples):
 *   echo 0x200000 > .../parameters/sema_copy_dst
 */
module_param_named(sema_copy_dst, gsp_sema_copy_dst, ullong, 0644);
MODULE_PARM_DESC(sema_copy_dst, "Dump source bytes of copies landing in this page (default 0)");

/*
 * Same watch, second 16 KB-aligned window (0 = off). Two windows let one
 * run cover both the page-directory table and the page-table page it points
 * into, e.g. 0x208000 and 0x230000.
 */
module_param_named(sema_copy_dst2, gsp_sema_copy_dst2, ullong, 0644);
MODULE_PARM_DESC(sema_copy_dst2, "Second destination window for sema_copy_dst (default 0)");

/*
 * Grace window (ms) an untranslatable CE entry stays pending before the
 * fallback acknowledges its release and consumes it. Its work must run before
 * the release the client synchronizes on; the translation usually lands within
 * a service pass.
 */
module_param_named(sema_ce_grace_ms, gsp_sema_ce_grace_ms, uint, 0644);
MODULE_PARM_DESC(sema_ce_grace_ms, "CE pending-entry grace window in ms (default 8000)");

/*
 * Bounded work-before-release hold (ms) for an unreadable CE entry; the
 * deferred retry probe shows translations/pins land 0.5-2 s late. Absolute
 * per-entry deadline, never reset; 0 disables.
 */
module_param_named(sema_push_hold_ms, gsp_sema_push_hold_ms, uint, 0644);
MODULE_PARM_DESC(sema_push_hold_ms, "CE unreadable-entry work-before-release hold in ms (default 2500, 0 disables)");

/*
 * Grace window (ms) an untranslatable GR entry stays pending before the pump
 * consumes it as stale, so a late push-buffer pin can still make it readable.
 */
module_param_named(sema_gr_grace_ms, gsp_sema_gr_grace_ms, uint, 0644);
MODULE_PARM_DESC(sema_gr_grace_ms, "GR pending-entry grace window in ms (default 8000)");

/*
 * Debug: inject a verified {GPU VA -> guest PA} translation into the
 * semaphore service (used when the client's mapping cannot be discovered
 * from guest structures):
 *   echo 0x<va>,0x<pa> > .../parameters/sema_map_add
 */
static int nv_vgpu_sema_map_add_set(const char *val, const struct kernel_param *kp)
{
	char buf[64];
	char *comma, *comma2, *comma3;
	u64 va = 0, pa = 0, len = 0, ap = 0;

	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (!comma)
		return -EINVAL;
	*comma = '\0';
	comma2 = strchr(comma + 1, ',');
	if (comma2)
		*comma2 = '\0';
	if (kstrtoull(buf, 0, &va) || kstrtoull(comma + 1, 0, &pa))
		return -EINVAL;
	if (comma2) {
		comma3 = strchr(comma2 + 1, ',');
		if (comma3)
			*comma3 = '\0';
		if (kstrtoull(comma2 + 1, 0, &len))
			return -EINVAL;
		if (comma3 && kstrtoull(comma3 + 1, 0, &ap))
			return -EINVAL;
	}

	return gsp_sema_service_add_map(va, pa, len, (u32)ap);
}

static const struct kernel_param_ops nv_vgpu_sema_map_add_ops = {
	.set = nv_vgpu_sema_map_add_set,
};

module_param_cb(sema_map_add, &nv_vgpu_sema_map_add_ops, NULL, 0200);
MODULE_PARM_DESC(sema_map_add, "Inject a GPU VA -> guest PA mapping (hex va,hex pa)");

/*
 * Debug: register a channel's GPU completion counter (guest PA of the u32
 * tracking counter the GPU would write when a push completes):
 *   echo <h_object>,<pa> > .../parameters/sema_counter
 */
static int nv_vgpu_sema_counter_set(const char *val, const struct kernel_param *kp)
{
	char buf[64];
	char *comma;
	u64 h_object = 0;
	u64 pa = 0;

	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (!comma)
		return -EINVAL;
	*comma = '\0';
	/* kstrtoull tolerates the trailing newline sysfs writes carry. */
	if (kstrtoull(buf, 0, &h_object) || kstrtoull(comma + 1, 0, &pa))
		return -EINVAL;
	if (!h_object || h_object > 0xffffffffULL)
		return -EINVAL;

	return gsp_sema_service_counter_add((u32)h_object, pa);
}

static const struct kernel_param_ops nv_vgpu_sema_counter_ops = {
	.set = nv_vgpu_sema_counter_set,
};

module_param_cb(sema_counter, &nv_vgpu_sema_counter_ops, NULL, 0200);
MODULE_PARM_DESC(sema_counter, "Register channel GPU completion counter (h_object,pa)");

/*
 * Debug: dump a range of the emulated vidmem (BAR1 shadow) into the kernel
 * log, to verify where a decoded CE copy actually landed:
 *   echo 0x227000,256 > .../parameters/sema_vidmem_dump
 */
static int nv_vgpu_sema_vidmem_dump_set(const char *val,
					const struct kernel_param *kp)
{
	char buf[64];
	char *comma;
	u64 addr = 0, len = 0;

	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (!comma)
		return -EINVAL;
	*comma = '\0';
	if (kstrtoull(buf, 0, &addr) || kstrtoull(comma + 1, 0, &len))
		return -EINVAL;

	return gsp_sema_service_dump_vidmem(addr, len);
}

static const struct kernel_param_ops nv_vgpu_sema_vidmem_dump_ops = {
	.set = nv_vgpu_sema_vidmem_dump_set,
};

module_param_cb(sema_vidmem_dump, &nv_vgpu_sema_vidmem_dump_ops, NULL, 0200);
MODULE_PARM_DESC(sema_vidmem_dump, "Dump emulated vidmem (addr,len) to the kernel log");

/*
 * Debug: search the whole emulated vidmem for an 8-byte pattern:
 *   echo 0x0000000400000001,16 > .../parameters/sema_vidmem_find
 */
static int nv_vgpu_sema_vidmem_find_set(const char *val,
					const struct kernel_param *kp)
{
	char buf[64];
	char *comma;
	u64 pattern = 0, max_hits = 16;

	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (comma) {
		*comma = '\0';
		if (kstrtoull(comma + 1, 0, &max_hits))
			return -EINVAL;
	}
	if (kstrtoull(buf, 0, &pattern))
		return -EINVAL;

	return gsp_sema_service_find_vidmem(pattern, max_hits);
}

static const struct kernel_param_ops nv_vgpu_sema_vidmem_find_ops = {
	.set = nv_vgpu_sema_vidmem_find_set,
};

module_param_cb(sema_vidmem_find, &nv_vgpu_sema_vidmem_find_ops, NULL, 0200);
MODULE_PARM_DESC(sema_vidmem_find, "Search emulated vidmem for an 8-byte pattern (pattern, max_hits)");

/*
 * Debug: find page-table entries pointing at a physical address, in guest RAM
 * and emulated vidmem:
 *   echo 0x12442b000 > .../parameters/scan_pt
 */
struct nv_vgpu_pt_req {
	u64 pa;
	struct completion done;
};

static int nv_vgpu_pt_thread(void *arg)
{
	struct nv_vgpu_pt_req *req = arg;
	struct nv_vgpu_device *vgpu;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list)
		gsp_rm_scan_guest_ram_pt(vgpu, req->pa, 0, 8ULL << 30, 16);
	mutex_unlock(&nv_vgpu_driver.lock);
	complete(&req->done);
	return 0;
}

static int nv_vgpu_scan_pt_set(const char *val, const struct kernel_param *kp)
{
	static struct nv_vgpu_pt_req req;
	struct task_struct *task;
	u64 pa = 0;
	int ret;

	ret = kstrtoull(val, 0, &pa);
	if (ret)
		return ret;

	req.pa = pa;
	init_completion(&req.done);
	task = kthread_run(nv_vgpu_pt_thread, &req, "vgpu-pt");
	if (IS_ERR(task))
		return PTR_ERR(task);
	wait_for_completion(&req.done);

	/* vidmem shadow scan (plain memory) */
	mutex_lock(&nv_vgpu_driver.lock);
	{
		struct nv_vgpu_device *vgpu;

		list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;
			size_t off;
			u32 hits = 0;
			u64 pa_page = pa & ~0xfffULL;

			if (!base || !size)
				continue;
			for (off = 0; off + 8 <= size; off += 4) {
				u64 v64;
				u32 v32;

				memcpy(&v64, base + off, 8);
				memcpy(&v32, base + off, 4);
				if ((v64 & 0xfffffffffffff000ULL) == pa_page) {
					pr_info("vgsp: pt scan: vidmem u64 pte 0x%llx at off 0x%zx\n",
						v64, off);
					if (++hits >= 16)
						break;
				}
				if ((v32 & 0xfffff000U) == (u32)pa_page) {
					pr_info("vgsp: pt scan: vidmem u32 pte 0x%x at off 0x%zx\n",
						v32, off);
					if (++hits >= 16)
						break;
				}
			}
			pr_info("vgsp: pt scan: vidmem pa=0x%llx hits=%u\n",
				pa, hits);
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);

	return 0;
}

static const struct kernel_param_ops nv_vgpu_scan_pt_ops = {
	.set = nv_vgpu_scan_pt_set,
};

module_param_cb(scan_pt, &nv_vgpu_scan_pt_ops, NULL, 0200);
MODULE_PARM_DESC(scan_pt, "Find PTEs pointing at a physical address (hex)");

/*
 * Debug: scan guest RAM for (v & mask) == value (kthread context):
 *   echo 0x121000000,0xfffff000 > .../parameters/scan_masked
 * Used to find stored page-base VAs (base = VA - index*size).
 */
struct nv_vgpu_masked_req {
	u64 value;
	u64 mask;
	struct completion done;
};

static int nv_vgpu_masked_thread(void *arg)
{
	struct nv_vgpu_masked_req *req = arg;
	struct nv_vgpu_device *vgpu;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list)
		gsp_rm_scan_guest_ram_masked(vgpu, req->value, req->mask,
					     0, 8ULL << 30, 12);
	mutex_unlock(&nv_vgpu_driver.lock);
	complete(&req->done);
	return 0;
}

static int nv_vgpu_scan_masked_set(const char *val, const struct kernel_param *kp)
{
	static struct nv_vgpu_masked_req req;
	struct task_struct *task;
	char buf[96];
	char *comma;

	memset(&req, 0, sizeof(req));
	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (!comma)
		return -EINVAL;
	*comma = '\0';
	if (kstrtoull(buf, 0, &req.value) || kstrtoull(comma + 1, 0, &req.mask))
		return -EINVAL;

	init_completion(&req.done);
	task = kthread_run(nv_vgpu_masked_thread, &req, "vgpu-mask");
	if (IS_ERR(task))
		return PTR_ERR(task);
	wait_for_completion(&req.done);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_scan_masked_ops = {
	.set = nv_vgpu_scan_masked_set,
};

module_param_cb(scan_masked, &nv_vgpu_scan_masked_ops, NULL, 0200);
MODULE_PARM_DESC(scan_masked, "Scan guest RAM for (v & mask) == value (hex,hex)");

/*
 * Debug: fill the entire emulated vidmem (BAR1 shadow) with a u32 value:
 *   echo 0x1 > .../parameters/fill_vidmem
 * Used to test whether a guest-visible structure (e.g. a UVM semaphore)
 * lives in the vidmem aperture.
 */
static int nv_vgpu_fill_vidmem_set(const char *val, const struct kernel_param *kp)
{
	struct nv_vgpu_device *vgpu;
	u32 value = 0;
	int ret;

	ret = kstrtou32(val, 0, &value);
	if (ret)
		return ret;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;
		size_t o;

		if (!base || !size)
			continue;
		for (o = 0; o + 4 <= size; o += 4)
			memcpy(base + o, &value, 4);
		pr_info("vgsp: fill_vidmem value=0x%x size=0x%zx done\n",
			value, size);
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_fill_vidmem_ops = {
	.set = nv_vgpu_fill_vidmem_set,
};

module_param_cb(fill_vidmem, &nv_vgpu_fill_vidmem_ops, NULL, 0200);
MODULE_PARM_DESC(fill_vidmem, "Fill emulated vidmem with a u32 value (hex)");

/*
 * Debug: write a u32 to a guest physical address (kthread context, via
 * vfio_dma_rw). Used to complete an emulated GPU semaphore release:
 *   echo 0x121000000,0x1 > .../parameters/poke_guest
 */
struct nv_vgpu_poke_req {
	u64 gpa;
	u32 value;
	struct completion done;
	int rc;
};

static int nv_vgpu_poke_thread(void *arg)
{
	struct nv_vgpu_poke_req *req = arg;
	struct nv_vgpu_device *vgpu;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) {
			req->rc = -ENODEV;
			continue;
		}
		req->rc = vfio_dma_rw(&vgpu->vfio_dev, req->gpa, &req->value,
				      sizeof(req->value), true);
		pr_info("vgsp: poke gpa=0x%llx value=0x%x rc=%d\n",
			req->gpa, req->value, req->rc);
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	complete(&req->done);
	return 0;
}

static int nv_vgpu_poke_set(const char *val, const struct kernel_param *kp)
{
	static struct nv_vgpu_poke_req req;
	struct task_struct *task;
	char buf[64];
	char *comma;

	memset(&req, 0, sizeof(req));
	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (!comma)
		return -EINVAL;
	*comma = '\0';
	if (kstrtoull(buf, 0, &req.gpa) || kstrtou32(comma + 1, 0, &req.value))
		return -EINVAL;

	init_completion(&req.done);
	task = kthread_run(nv_vgpu_poke_thread, &req, "vgpu-poke");
	if (IS_ERR(task))
		return PTR_ERR(task);
	wait_for_completion(&req.done);

	return req.rc;
}

static const struct kernel_param_ops nv_vgpu_poke_ops = {
	.set = nv_vgpu_poke_set,
};

module_param_cb(poke_guest, &nv_vgpu_poke_ops, NULL, 0200);
MODULE_PARM_DESC(poke_guest, "Write u32 to guest PA (hex gpa,hex value)");

/*
 * Debug: fill a guest physical range with a u32 value (kthread context):
 *   echo 0x121000000,0x1,1024 > .../parameters/poke_fill
 * (gpa, value, count of dwords)
 */
static u32 *nv_vgpu_fill_buf;
static u32 nv_vgpu_fill_count;

static int nv_vgpu_poke_fill_thread(void *arg)
{
	struct nv_vgpu_poke_req *req = arg;
	struct nv_vgpu_device *vgpu;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) {
			req->rc = -ENODEV;
			continue;
		}
		req->rc = vfio_dma_rw(&vgpu->vfio_dev, req->gpa,
				      nv_vgpu_fill_buf,
				      nv_vgpu_fill_count * sizeof(u32), true);
		pr_info("vgsp: poke_fill gpa=0x%llx count=%u rc=%d\n",
			req->gpa, nv_vgpu_fill_count, req->rc);
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	complete(&req->done);
	return 0;
}

static int nv_vgpu_poke_fill_set(const char *val, const struct kernel_param *kp)
{
	u64 gpa = 0;
	u32 value = 0, count = 0;
	char buf[96];
	char *tok, *cur;
	struct task_struct *task;
	static struct nv_vgpu_poke_req req;
	int rc;

	strscpy(buf, val, sizeof(buf));
	cur = buf;
	tok = strsep(&cur, ",");
	if (!tok || kstrtoull(tok, 0, &gpa))
		return -EINVAL;
	tok = strsep(&cur, ",");
	if (!tok || kstrtou32(tok, 0, &value))
		return -EINVAL;
	tok = strsep(&cur, ",");
	if (!tok || kstrtou32(tok, 0, &count) || !count || count > 4096)
		return -EINVAL;

	nv_vgpu_fill_buf = kmalloc_array(count, sizeof(u32), GFP_KERNEL);
	if (!nv_vgpu_fill_buf)
		return -ENOMEM;
	for (u32 i = 0; i < count; i++)
		nv_vgpu_fill_buf[i] = value;
	nv_vgpu_fill_count = count;

	memset(&req, 0, sizeof(req));
	req.gpa = gpa;
	init_completion(&req.done);

	task = kthread_run(nv_vgpu_poke_fill_thread, &req, "vgpu-fill");
	if (IS_ERR(task)) {
		kfree(nv_vgpu_fill_buf);
		nv_vgpu_fill_buf = NULL;
		return PTR_ERR(task);
	}
	wait_for_completion(&req.done);
	rc = req.rc;
	kfree(nv_vgpu_fill_buf);
	nv_vgpu_fill_buf = NULL;
	return rc;
}

static const struct kernel_param_ops nv_vgpu_poke_fill_ops = {
	.set = nv_vgpu_poke_fill_set,
};

module_param_cb(poke_fill, &nv_vgpu_poke_fill_ops, NULL, 0200);
MODULE_PARM_DESC(poke_fill, "Fill guest PA range with u32 (gpa,value,count)");

/*
 * Debug: dump a range of the emulated vidmem (BAR1 shadow):
 *   echo 0x20000,0x200 > .../parameters/dump_vidmem
 *   echo summary       > .../parameters/dump_vidmem   (find nonzero areas)
 */
static int nv_vgpu_dump_vidmem_set(const char *val, const struct kernel_param *kp)
{
	struct nv_vgpu_device *vgpu;
	unsigned long off = 0, len = 0;
	char buf[64];
	char *comma;

	strscpy(buf, val, sizeof(buf));
	if (!strncmp(buf, "summary", 7)) {
		mutex_lock(&nv_vgpu_driver.lock);
		list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;
			size_t o;
			u32 nz = 0;

			if (!base || !size)
				continue;
			for (o = 0; o + 4 <= size; o += 4) {
				u32 v;

				memcpy(&v, base + o, 4);
				if (v) {
					if (nz < 32)
						pr_info("vgsp: vidmem nz[0x%zx]=0x%x\n",
							o, v);
					nz++;
				}
			}
			pr_info("vgsp: vidmem summary: size=0x%zx nonzero_dwords=%u\n",
				size, nz);
		}
		mutex_unlock(&nv_vgpu_driver.lock);
		return 0;
	}

	comma = strchr(buf, ',');
	if (comma)
		*comma = '\0';
	if (kstrtoul(buf, 0, &off))
		return -EINVAL;
	if (comma && kstrtoul(comma + 1, 0, &len))
		return -EINVAL;
	if (!len)
		len = 0x100;
	if (len > 0x1000)
		len = 0x1000;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;
		unsigned long o;

		if (!base || !size || off + len > size)
			continue;
		for (o = 0; o < len; o += 64) {
			u32 n = min_t(unsigned long, len - o, 64);

			pr_info("vgsp: vidmem[0x%lx..0x%lx]=%*phN\n",
				off + o, off + o + n, (int)n, base + off + o);
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_dump_vidmem_ops = {
	.set = nv_vgpu_dump_vidmem_set,
};

module_param_cb(dump_vidmem, &nv_vgpu_dump_vidmem_ops, NULL, 0200);
MODULE_PARM_DESC(dump_vidmem, "Dump emulated vidmem range (hex offset,hex len)");

/*
 * Debug: hex-dump or scan the BAR0 register shadow. Guest MMIO writes that go
 * through the mapped shadow never reach the write callback, so this is the
 * only way to observe registers the guest writes directly (e.g. the
 * work-submission doorbell/notify registers, client region).
 *   echo 0x88700,0x100 > .../parameters/dump_bar0
 *   echo 0x00040022 > .../parameters/scan_bar0
 */
static int nv_vgpu_dump_bar0_set(const char *val, const struct kernel_param *kp)
{
	struct nv_vgpu_device *vgpu;
	struct nv_vgpu_bar_emul_ctx *ctx;
	unsigned long off = 0, len = 0;
	char buf[64];
	char *comma;

	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (comma)
		*comma = '\0';
	if (kstrtoul(buf, 0, &off))
		return -EINVAL;
	if (comma && kstrtoul(comma + 1, 0, &len))
		return -EINVAL;
	if (!len)
		len = 0x100;
	if (len > 0x1000)
		len = 0x1000;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u8 *base;
		unsigned long o;

		ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
		if (!ctx || !ctx->bar0_shadow)
			continue;
		base = (u8 *)ctx->bar0_shadow;
		if (off + len > ctx->bar0_shadow_size)
			continue;
		for (o = 0; o < len; o += 64) {
			u32 n = min_t(unsigned long, len - o, 64);

			pr_info("vgsp: bar0[0x%lx..0x%lx]=%*phN\n",
				off + o, off + o + n, (int)n, base + off + o);
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_dump_bar0_ops = {
	.set = nv_vgpu_dump_bar0_set,
};

module_param_cb(dump_bar0, &nv_vgpu_dump_bar0_ops, NULL, 0200);
MODULE_PARM_DESC(dump_bar0, "Dump BAR0 shadow range (hex offset,hex len)");

static int nv_vgpu_scan_bar0_set(const char *val, const struct kernel_param *kp)
{
	struct nv_vgpu_device *vgpu;
	struct nv_vgpu_bar_emul_ctx *ctx;
	unsigned long needle = 0;
	u32 hits = 0;

	if (kstrtoul(val, 0, &needle))
		return -EINVAL;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u8 *base;
		size_t size, o;

		ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
		if (!ctx || !ctx->bar0_shadow)
			continue;
		base = (u8 *)ctx->bar0_shadow;
		size = ctx->bar0_shadow_size;
		for (o = 0; o + 4 <= size && hits < 64; o += 4) {
			u32 v;

			memcpy(&v, base + o, 4);
			if (v == (u32)needle) {
				pr_info("vgsp: bar0 scan: 0x%x at off 0x%zx\n",
					v, o);
				hits++;
			}
		}
		pr_info("vgsp: bar0 scan: needle=0x%lx hits=%u\n",
			needle, hits);
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_scan_bar0_ops = {
	.set = nv_vgpu_scan_bar0_set,
};

module_param_cb(scan_bar0, &nv_vgpu_scan_bar0_ops, NULL, 0200);
MODULE_PARM_DESC(scan_bar0, "Scan BAR0 shadow for a 32-bit value (hex)");

/*
 * Debug: find channel USERD blocks in the emulated vidmem. A USERD page has a
 * GPGet at +0x88 and a GPPut at +0x8c; report every page whose GPPut is a
 * plausible entry index (1..65535) with GPGet <= GPPut. This is how the real
 * vidmem offset of a channel's control block is discovered when the offset
 * carried in the allocation parameters is relative to a pool the emulation
 * cannot see.
 */
static int nv_vgpu_find_userd_set(const char *val, const struct kernel_param *kp)
{
	struct nv_vgpu_device *vgpu;
	int shown = 0;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;
		size_t off;

		if (!base || !size)
			continue;
		for (off = 0; off + 0x90 <= size; off += 4) {
			u32 get, put;

			memcpy(&put, base + off + 0x8c, 4);
			if (put < 1 || put > 65535)
				continue;
			memcpy(&get, base + off + 0x88, 4);
			if (get > put)
				continue;
			if (shown < 32)
				pr_info("vgsp: userd cand off=0x%zx get=%u put=%u\n",
					off, get, put);
			shown++;
		}
		pr_info("vgsp: userd scan: candidates=%d\n", shown);
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_find_userd_ops = {
	.set = nv_vgpu_find_userd_set,
};

module_param_cb(find_userd, &nv_vgpu_find_userd_ops, NULL, 0200);
MODULE_PARM_DESC(find_userd, "Find channel USERD blocks in emulated vidmem");

/*
 * Debug: write a u32 into the emulated vidmem (BAR1 backing) at an offset:
 *   echo 0x<offset>,0x<value> > .../parameters/poke_vidmem
 */
static int nv_vgpu_poke_vidmem_set(const char *val, const struct kernel_param *kp)
{
	struct nv_vgpu_device *vgpu;
	u64 off = 0, value = 0;
	char *end;
	int ret = 0;

	if (sscanf(val, "%lli,%lli", &off, &value) != 2)
		return -EINVAL;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
			continue;
		if (!vgpu->bars[1].vaddr || off + 4 > vgpu->bars[1].size) {
			ret = -EINVAL;
			break;
		}
		memcpy(vgpu->bars[1].vaddr + off, &value, 4);
		pr_info("vgsp-poke-vidmem: [0x%llx] = 0x%llx\n", off, value);
		break;
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	(void)end;
	return ret;
}

static const struct kernel_param_ops nv_vgpu_poke_vidmem_ops = {
	.set = nv_vgpu_poke_vidmem_set,
};

module_param_cb(poke_vidmem, &nv_vgpu_poke_vidmem_ops, NULL, 0200);
MODULE_PARM_DESC(poke_vidmem, "Write u32 to emulated vidmem (hex offset,hex value)");

/*
 * Debug: hex-dump a guest physical range (kthread: guest-DMA needs it):
 *   echo 0x<gpa>,0x<len> > .../parameters/dump_guest
 */
struct nv_vgpu_dump_guest_req {
	u64 gpa;
	unsigned long len;
	struct completion done;
};

static int nv_vgpu_dump_guest_thread(void *arg)
{
	struct nv_vgpu_dump_guest_req *req = arg;
	struct nv_vgpu_device *vgpu;
	u32 *buf;
	unsigned long i;

	buf = kmalloc(256, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		u64 o;

		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
			continue;
		pr_info("vgsp: guest dump 0x%llx..0x%llx\n",
			req->gpa, req->gpa + req->len);
		for (o = 0; o < req->len; o += 256) {
			unsigned long chunk = min_t(unsigned long, 256,
						    req->len - o);

			if (vfio_dma_rw(&vgpu->vfio_dev, req->gpa + o, buf,
					chunk, false)) {
				pr_info("vgsp: guest dump read failed at 0x%llx\n",
					req->gpa + o);
				break;
			}
			for (i = 0; i < chunk / 4; i += 4)
				pr_info("vgsp: 0x%llx: %08x %08x %08x %08x\n",
					req->gpa + o + i * 4, buf[i], buf[i + 1],
					buf[i + 2], buf[i + 3]);
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);

	kfree(buf);
	complete(&req->done);
	return 0;
}

static int nv_vgpu_dump_guest_set(const char *val, const struct kernel_param *kp)
{
	static struct nv_vgpu_dump_guest_req req;
	struct task_struct *task;
	char buf[64];
	char *comma;

	memset(&req, 0, sizeof(req));
	strscpy(buf, val, sizeof(buf));
	comma = strchr(buf, ',');
	if (comma)
		*comma = '\0';
	if (kstrtoull(buf, 0, &req.gpa))
		return -EINVAL;
	req.len = 0x100;
	if (comma && kstrtoul(comma + 1, 0, &req.len))
		return -EINVAL;
	if (!req.len)
		req.len = 0x100;
	if (req.len > 0x4000)
		req.len = 0x4000;

	init_completion(&req.done);
	task = kthread_run(nv_vgpu_dump_guest_thread, &req, "vgpu-dump");
	if (IS_ERR(task))
		return PTR_ERR(task);
	wait_for_completion(&req.done);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_dump_guest_ops = {
	.set = nv_vgpu_dump_guest_set,
};

module_param_cb(dump_guest, &nv_vgpu_dump_guest_ops, NULL, 0200);
MODULE_PARM_DESC(dump_guest, "Hex-dump a guest physical range (hex gpa,hex len)");

/*
 * Debug: deliver one MSI-X vector to the guest. QEMU wires each vector's
 * eventfd via VFIO_DEVICE_SET_IRQS; signaling it makes QEMU raise the guest
 * interrupt. The emulated GPU consumes work without raising the interrupts a
 * real GPU would, so sessions use this to drive the guest's interrupt-driven
 * progress paths (channel/runlist events) and find out which vector an event
 * needs.
 */
static int nv_vgpu_inject_irq_set(const char *val, const struct kernel_param *kp)
{
	struct nv_vgpu_device *vgpu;
	unsigned long idx = 0;
	int ret = -ENODEV;

	if (kstrtoul(val, 0, &idx) || idx >= NV_VGPU_MAX_IRQS)
		return -EINVAL;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		struct eventfd_ctx *trigger;

		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
			continue;

		if (vgpu->bar_emul_ctx && idx < NV_VGPU_MAX_IRQS) {
			u32 bit = 1u << (idx % 32);

			nv_vgpu_bar_emul_write(vgpu, 0,
					       GSP_EMU_MSIX_PBA_OFFSET +
					       (idx / 32) * 4,
					       &bit, sizeof(bit));
		}

		trigger = vgpu->irqs[idx].trigger;
		if (trigger)
			eventfd_signal(trigger);
		pr_info("vgsp: inject_irq %lu (trigger %s)\n",
			idx, trigger ? "fired" : "missing");
		ret = 0;
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	return ret;
}

static const struct kernel_param_ops nv_vgpu_inject_irq_ops = {
	.set = nv_vgpu_inject_irq_set,
};

module_param_cb(inject_irq, &nv_vgpu_inject_irq_ops, NULL, 0200);
MODULE_PARM_DESC(inject_irq, "Deliver MSI-X vector N to the guest (debug)");

/*
 * Debug: dump the GSP-RM msgq cursors (ours and the guest's) plus the
 * registered event objects:
 *   echo 1 > .../parameters/rmstate
 *
 * Runs in a kthread: vfio_iommu_type1_dma_rw only works when current->mm is
 * NULL (kthread) or matches the mapping's mm (QEMU).
 */
static int nv_vgpu_rmstate_thread(void *arg)
{
	struct completion *done = arg;
	struct nv_vgpu_device *vgpu;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
			continue;
		gsp_rm_dump_state(vgpu);
	}
	mutex_unlock(&nv_vgpu_driver.lock);

	complete(done);
	return 0;
}

static int nv_vgpu_rmstate_set(const char *val, const struct kernel_param *kp)
{
	DECLARE_COMPLETION_ONSTACK(done);
	struct task_struct *task;

	task = kthread_run(nv_vgpu_rmstate_thread, &done, "vgpu-rmstate");
	if (IS_ERR(task))
		return PTR_ERR(task);
	wait_for_completion(&done);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_rmstate_ops = {
	.set = nv_vgpu_rmstate_set,
};

module_param_cb(rmstate, &nv_vgpu_rmstate_ops, NULL, 0200);
MODULE_PARM_DESC(rmstate, "Dump GSP-RM msgq cursors and event table (debug)");

/*
 * Debug: post an event record to the guest (idx < events_n, or 0xffffffff
 * for every registered event):
 *   echo 0 > .../parameters/rmpost
 */
struct nv_vgpu_rmpost_req {
	u32 idx;
	int rc;
	struct completion done;
};

static int nv_vgpu_rmpost_thread(void *arg)
{
	struct nv_vgpu_rmpost_req *req = arg;
	struct nv_vgpu_device *vgpu;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
			continue;
		req->rc = gsp_rm_force_post(vgpu, req->idx);
		pr_info("vgsp: rmpost idx=%u -> %d\n",
			req->idx, req->rc);
	}
	mutex_unlock(&nv_vgpu_driver.lock);

	complete(&req->done);
	return 0;
}

static int nv_vgpu_rmpost_set(const char *val, const struct kernel_param *kp)
{
	static struct nv_vgpu_rmpost_req req;
	struct task_struct *task;

	memset(&req, 0, sizeof(req));
	req.rc = -ENODEV;
	if (kstrtou32(val, 0, &req.idx))
		return -EINVAL;

	init_completion(&req.done);
	task = kthread_run(nv_vgpu_rmpost_thread, &req, "vgpu-rmpost");
	if (IS_ERR(task))
		return PTR_ERR(task);
	wait_for_completion(&req.done);
	return 0;
}

static const struct kernel_param_ops nv_vgpu_rmpost_ops = {
	.set = nv_vgpu_rmpost_set,
};

module_param_cb(rmpost, &nv_vgpu_rmpost_ops, NULL, 0200);
MODULE_PARM_DESC(rmpost, "Post a GSP-RM event to the guest by event index (debug)");

module_init(nv_vgpu_init);
module_exit(nv_vgpu_exit);