path: root/legacy-driver-guest/include/gsp_core.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * GSP vGPU-IO Driver - Core Definitions
 * NVIDIA GSP 570.144W (TU102/TU104) Firmware Interface
 */

#ifndef _GSP_CORE_H_
#define _GSP_CORE_H_

#include <linux/types.h>
#include <linux/pci.h>
#include <linux/mutex.h>
#include <linux/idr.h>
#include <linux/list.h>
#include <linux/kref.h>
#include <linux/completion.h>
#include <linux/wait.h>
#include <linux/spinlock.h>
#include <linux/miscdevice.h>

/* Firmware Constants */
#define GSP_FW_BASE_ADDR          0x04000000ULL
#define GSP_FW_SIZE               0x02000000ULL
#define GSP_ENTRY_POINT           0x04009000ULL
#define GSP_PRIMARY_DISPATCHER    0x057A1AC0ULL
#define GSP_SECONDARY_DISPATCHER  0x04ED39DCULL
#define GSP_RPC_JUMP_TABLE        0x059BFEB0ULL

/* PCIe BAR Offsets */
#define GSP_BAR0_FALCON_ENGINE     0x1103c0
#define GSP_BAR0_FALCON_MAILBOX0   0x110040
#define GSP_BAR0_FALCON_MAILBOX1   0x110044
#define GSP_BAR0_MAILBOX(i)        (0x110804 + (i) * 4)
#define GSP_BAR0_QUEUE_HEAD(i)     (0x110c00 + (i) * 8)

/* IOCTL definitions (must match userspace libvgpu.h) */
#define GSP_IOCTL_MAGIC 'G'
#define GSP_IOCTL_CREATE_VF   _IOW(GSP_IOCTL_MAGIC, 0x01, struct gsp_vf_config)
#define GSP_IOCTL_DESTROY_VF  _IOW(GSP_IOCTL_MAGIC, 0x02, uint32_t)
#define GSP_IOCTL_ALLOC       _IOWR(GSP_IOCTL_MAGIC, 0x10, struct gsp_alloc_args)
#define GSP_IOCTL_FREE        _IOW(GSP_IOCTL_MAGIC, 0x11, struct gsp_free_args)
#define GSP_IOCTL_MAP         _IOWR(GSP_IOCTL_MAGIC, 0x12, struct gsp_map_args)
#define GSP_IOCTL_UNMAP       _IOW(GSP_IOCTL_MAGIC, 0x13, struct gsp_unmap_args)
#define GSP_IOCTL_PIN         _IOWR(GSP_IOCTL_MAGIC, 0x14, struct gsp_pin_args)
#define GSP_IOCTL_UNPIN       _IOW(GSP_IOCTL_MAGIC, 0x15, struct gsp_unpin_args)
#define GSP_IOCTL_BAR_CONFIG  _IOWR(GSP_IOCTL_MAGIC, 0x16, struct gsp_bar_config_args)
#define GSP_IOCTL_BAR_FORCE   _IOW(GSP_IOCTL_MAGIC, 0x17, struct gsp_bar_force_args)
#define GSP_IOCTL_BAR_SAVE    _IOWR(GSP_IOCTL_MAGIC, 0x18, struct gsp_bar_save_args)
#define GSP_IOCTL_BAR_RESTORE _IOWR(GSP_IOCTL_MAGIC, 0x19, struct gsp_bar_restore_args)
#define GSP_IOCTL_WPR_PROTECT  _IOWR(GSP_IOCTL_MAGIC, 0x1A, struct gsp_wpr_args)
#define GSP_IOCTL_WPR_UNPROTECT _IOW(GSP_IOCTL_MAGIC, 0x1B, struct gsp_wpr_args)
#define GSP_IOCTL_WPR_TIMEOUT _IOW(GSP_IOCTL_MAGIC, 0x1C, struct gsp_wpr_timeout_args)
#define GSP_IOCTL_SCHEDULE    _IOW(GSP_IOCTL_MAGIC, 0x1D, struct gsp_schedule_args)
#define GSP_IOCTL_YIELD       _IOW(GSP_IOCTL_MAGIC, 0x1E, uint32_t)
#define GSP_IOCTL_MIGRATE     _IOWR(GSP_IOCTL_MAGIC, 0x1F, struct gsp_migrate_args)
#define GSP_IOCTL_GET_INFO    _IOR(GSP_IOCTL_MAGIC, 0x20, char[256])

/* Kernel argument structures (matching userspace) */
struct gsp_vf_config {
    uint32_t id;
    uint64_t bar1_offset;
    uint64_t bar1_size;
    uint64_t pt_base;
    uint64_t pt_size;
    uint64_t cmd_base;
    uint64_t cmd_size;
    uint32_t max_memory_mb;
    uint32_t num_engines;
};

struct gsp_alloc_args {
    uint32_t vf_id;
    uint64_t size;
    uint32_t type;
    uint64_t alignment;
    uint64_t va_hint;
    uint32_t flags;
    uint32_t pid;
    uint64_t handle;
    uint64_t va;
    uint64_t actual_size;
};

struct gsp_free_args {
    uint32_t vf_id;
    uint64_t handle;
};

struct gsp_map_args {
    uint32_t vf_id;
    uint64_t handle;
    uint64_t gpu_va;
    uint64_t size;
    uint32_t prot;
    uint32_t flags;
    uint64_t offset;
    uint64_t mapped_va;
};

struct gsp_unmap_args {
    uint32_t vf_id;
    uint64_t handle;
    uint64_t gpu_va;
    uint64_t size;
};

struct gsp_pin_args {
    uint32_t vf_id;
    uint64_t handle;
    uint64_t offset;
    uint64_t size;
    uint32_t flags;
    uint64_t dma_addr;
    uint32_t num_pages;
};

struct gsp_unpin_args {
    uint32_t vf_id;
    uint64_t handle;
    uint64_t offset;
    uint64_t size;
};

struct gsp_bar_config_args {
    uint32_t vf_id;
    uint64_t aperture_offset;
    uint64_t aperture_size;
};

struct gsp_bar_force_args {
    uint32_t vf_id;
    uint32_t enable;
};

struct gsp_bar_save_args {
    uint32_t vf_id;
    void *buf;
    size_t *size;
};

struct gsp_bar_restore_args {
    uint32_t vf_id;
    void *buf;
    size_t size;
};

struct gsp_wpr_args {
    uint32_t vf_id;
    uint64_t base;
    uint64_t size;
    uint32_t flags;
};

struct gsp_wpr_timeout_args {
    uint32_t vf_id;
    uint64_t timeout_ns;
};

struct gsp_schedule_args {
    uint32_t vf_id;
    uint32_t priority;
    uint64_t time_slice_ns;
};

struct gsp_migrate_args {
    uint32_t vf_id;
    uint64_t dst_vf;
    uint32_t phase;
    uint32_t flags;
    uint64_t token;
};

/* Message Ring */
#define GSP_MSG_RING_SIZE         256
#define GSP_MSG_MAX_SIZE          256
#define GSP_MAX_PENDING           64

/* RPC Types */
enum gsp_rpc_type {
    GSP_RPC_ALLOC     = 0x01,
    GSP_RPC_FREE      = 0x02,
    GSP_RPC_MAP       = 0x03,
    GSP_RPC_UNMAP     = 0x04,
    GSP_RPC_PIN       = 0x05,
    GSP_RPC_UNPIN     = 0x06,
    GSP_RPC_SCHEDULE  = 0x07,
    GSP_RPC_YIELD     = 0x08,
    GSP_RPC_NOTIFY    = 0x09,
    GSP_RPC_BAR_TRAP  = 0x0A,
    GSP_RPC_MIGRATE   = 0x0B,
    GSP_RPC_DEBUG     = 0x0C,
};

/* RPC Message Flags */
#define GSP_MSG_SYNC        (1 << 0)
#define GSP_MSG_ASYNC       (1 << 1)
#define GSP_MSG_NOREPLY     (1 << 2)

/* Memory Types */
enum gsp_mem_type {
    GSP_MEM_VRAM     = 0,
    GSP_MEM_SYSMEM   = 1,
    GSP_MEM_UNCACHED = 2,
};

/* Pin Flags */
#define GSP_PIN_READ      (1 << 0)
#define GSP_PIN_WRITE     (1 << 1)
#define GSP_PIN_LONGTERM  (1 << 2)

/* BAR Trap Types */
#define GSP_BAR_READ      0
#define GSP_BAR_WRITE     1
#define GSP_BAR_ATOMIC    2

/* Migration Phases */
enum gsp_migrate_phase {
    GSP_MIG_QUERY     = 0,
    GSP_MIG_PREPARE   = 1,
    GSP_MIG_TRANSFER  = 2,
    GSP_MIG_COMMIT    = 3,
    GSP_MIG_CLEANUP   = 4,
};

/* Migration Flags */
#define GSP_MIG_LIVE        (1 << 0)
#define GSP_MIG_POSTCOPY    (1 << 1)
#define GSP_MIG_AUTO_COMMIT (1 << 2)
#define GSP_MIG_VERIFY      (1 << 3)

/* Error Codes (from firmware) */
#define GSP_OK              0
#define GSP_EPERM           0xFFFFFFFF
#define GSP_ENOENT          0xFFFFFFFE
#define GSP_EINVAL          0xFFFFFFFD
#define GSP_ENODEV          0xFFFFFFFA
#define GSP_ENOMEM          0xFFFFFFF4
#define GSP_EBUSY           0xFFFFFFEE
#define GSP_ENOSPC          0xFFFFFFE4
#define GSP_ETIMEDOUT       0x0000001E
#define GSP_EOVERFLOW       0x00000025
#define GSP_EDEV            0x0000001F

/* Forward Declarations */
struct gsp_device;
struct gsp_vf;
struct gsp_rpc_client;
struct gsp_migration;
struct gsp_wpr_region;

/* WPR Region */
struct gsp_wpr_region {
    struct list_head list;
    uint64_t base;
    uint64_t size;
    uint32_t flags;
    uint32_t vf_id;
    void (*completion_cb)(struct gsp_vf *, uint64_t, uint32_t);
    void (*timeout_cb)(struct gsp_vf *, uint64_t, uint32_t);
    uint64_t timeout_ns;
};

/* WPR Flags */
#define GSP_WPR_WRITE      (1 << 0)
#define GSP_WPR_READ       (1 << 1)
#define GSP_WPR_EXECUTE    (1 << 2)

/* WPR Event Types */
enum gsp_wpr_event {
    GSP_WPR_WRITE_COMPLETE    = 0,
    GSP_WPR_READ_COMPLETE     = 1,
    GSP_WPR_WRITE_TIMEOUT     = 2,
    GSP_WPR_READ_TIMEOUT      = 3,
};

/* Migration State */
enum gsp_mig_state {
    GSP_MIG_S_IDLE = 0,
    GSP_MIG_S_QUERY = 1,
    GSP_MIG_S_PREPARE = 2,
    GSP_MIG_S_TRANSFER = 3,
    GSP_MIG_S_COMMIT = 4,
    GSP_MIG_S_CLEANUP = 5,
    GSP_MIG_S_FAILED = 6,
    GSP_MIG_S_COMPLETED = 7,
};

/* Migration Context */
struct gsp_migration {
    struct gsp_vf *vf;
    struct mutex lock;
    enum gsp_mig_state state;
    uint64_t token;
    uint64_t dst_vf;
    uint32_t flags;
    
    struct {
        uint64_t pages_transferred;
        uint64_t bytes_transferred;
        uint64_t dirty_pages;
        uint64_t *page_list;
        uint32_t page_count;
    } progress;
    
    struct {
        void *buf;
        size_t size;
    } bar_state;
    
    struct task_struct *thread;
    struct completion comp;
    bool cancelled;
    int error;
};

/* Emulation mode: skip GSP RPC for BAR/WPR when GSP lacks vGPU handlers */
extern bool emulation_mode;

/* RPC Message Header (64 bytes) */
struct gsp_rpc_header {
    uint32_t magic;           /* 0x47535052 "GSPR" */
    uint32_t version;         /* Protocol version */
    uint32_t rpc_type;        /* enum gsp_rpc_type */
    uint32_t sequence;        /* Monotonic sequence */
    uint64_t timestamp_ns;    /* Submission timestamp */
    uint32_t payload_size;    /* Payload size in bytes */
    uint32_t flags;           /* GSP_MSG_* */
    uint64_t correlation_id;  /* Async matching */
    uint64_t reserved[3];     /* Padding to 64 bytes */
} __attribute__((packed));

/* Complete RPC Message */
struct gsp_rpc_message {
    struct gsp_rpc_header hdr;
    uint8_t payload[GSP_MSG_MAX_SIZE - sizeof(struct gsp_rpc_header)];
} __attribute__((packed));

/* RPC Response Header */
struct gsp_rpc_response {
    int64_t status;           /* 0 = success, negative = error */
    uint32_t rpc_type;        /* Echo of request */
    uint32_t sequence;        /* Echo of request */
    uint64_t timestamp_ns;    /* Completion timestamp */
    uint32_t payload_size;    /* Response payload size */
    uint32_t reserved;
    uint8_t payload[];
} __attribute__((packed));

/* Primary Dispatcher Sub-Command Descriptor */
struct gsp_subcmd_desc {
    uint32_t subcmd_id;       /* Constant ID (e.g., 0x10, 0x14, 0x40) */
    uint8_t  echo_flags_b;    /* Copy of input flags_b */
    uint8_t  echo_flags_a;    /* Copy of input flags_a */
    uint8_t  echo_flags_c;    /* Copy of input flags_c */
    uint8_t  reserved;
} __attribute__((packed));

/* ALLOC Payload */
struct gsp_alloc_payload {
    uint64_t size;
    uint32_t type;            /* enum gsp_mem_type */
    uint32_t alignment;
    uint64_t va_hint;
    uint32_t flags;
    uint32_t pid;
    uint32_t vf_id;
    uint32_t reserved;
} __attribute__((packed));

struct gsp_alloc_response {
    uint64_t handle;
    uint64_t va;
    uint64_t pa;
} __attribute__((packed));

/* PIN Payload */
struct gsp_pin_payload {
    uint64_t handle;
    uint64_t offset;
    uint64_t size;
    uint32_t vf_id;
    uint32_t flags;           /* GSP_PIN_* */
    uint32_t reserved[2];
} __attribute__((packed));

struct gsp_pin_response {
    uint64_t dma_addr;
} __attribute__((packed));

/* BAR Trap Payload */
struct gsp_bar_trap_payload {
    uint32_t bar_index;
    uint32_t trap_type;       /* GSP_BAR_* */
    uint64_t offset;
    uint64_t value;
    uint32_t size;
    uint32_t vf_id;
    uint32_t reserved;
} __attribute__((packed));

struct gsp_bar_trap_response {
    uint64_t value;
} __attribute__((packed));

/* Migration Payload */
struct gsp_migrate_payload {
    uint32_t phase;           /* enum gsp_migrate_phase */
    uint32_t subcmd;          /* Sub-command index */
    uint64_t src_vf;
    uint64_t dst_vf;
    uint64_t data_ptr;
    uint64_t data_size;
    uint32_t flags;           /* GSP_MIG_* */
    uint32_t reserved;
    uint64_t token;
} __attribute__((packed));

/* Message Ring Buffer */
struct gsp_msg_ring {
    void *base;
    size_t size;
    uint32_t head;
    uint32_t tail;
    spinlock_t lock;
};

/* RPC Client */
struct gsp_rpc_client {
    struct gsp_device *gsp;
    struct mutex lock;
    atomic_t seq;
    wait_queue_head_t wq;
    struct gsp_msg_ring *tx_ring;
    struct gsp_msg_ring *rx_ring;
    dma_addr_t tx_ring_dma;
    dma_addr_t rx_ring_dma;
    struct completion *pending[GSP_MAX_PENDING];
    struct gsp_rpc_response *resp_buf[GSP_MAX_PENDING];
};

/* BAR State per VF */
struct gsp_bar_state {
    uint64_t aperture_offset;
    uint64_t aperture_size;
    bool force_access;
    uint32_t force_vf;
    void *backing_store;
    size_t backing_size;
};

/* VF State */
enum gsp_vf_state {
    GSP_VF_CREATING = 0,
    GSP_VF_ACTIVE,
    GSP_VF_MIGRATING,
    GSP_VF_SUSPENDED,
    GSP_VF_DESTROYING,
};

/* GSP Virtual Function */
struct gsp_vf {
    struct gsp_device *gsp;
    uint32_t id;
    struct kref ref;
    struct mutex lock;
    enum gsp_vf_state state;
    struct gsp_migration *migration;
    
    /* Resource configuration */
    uint64_t bar1_offset;
    uint64_t bar1_size;
    uint64_t pt_base;
    uint64_t pt_size;
    uint64_t cmd_base;
    uint64_t cmd_size;
    
    /* WPR regions */
    struct list_head wpr_list;
    struct mutex wpr_lock;
    
    /* BAR state */
    struct gsp_bar_state bar_state;
    
    atomic_t refcount;
    bool active;
    struct list_head list;
};

/* GSP Device */
struct gsp_device {
    struct pci_dev *pdev;
    struct device *dev;
    void __iomem *bar0;
    void __iomem *bar1;
    
    /* Firmware */
    const struct firmware *fw;
    bool fw_loaded;
    
    /* RPC */
    struct gsp_rpc_client *rpc;
    
    /* VF Management */
    struct mutex vf_lock;
    struct idr vf_idr;
    struct list_head vf_list;
    uint32_t max_vfs;
    
    /* IRQ */
    int irq;
    bool msix_enabled;
    struct msix_entry msix_entries[1];
    
    /* Debug */
    struct dentry *debugfs_root;
    struct dentry *debugfs_vfs_dir;
    
    /* Char device */
    struct miscdevice miscdev;
    struct mutex chardev_lock;
    
    /* Statistics */
    atomic64_t rpc_total;
    atomic64_t rpc_errors;
    atomic64_t bar_traps;
    atomic64_t wpr_events;
    atomic64_t migrations;
};

/* Function Prototypes */

/* Core */
int gsp_probe(struct pci_dev *pdev, const struct pci_device_id *id);
void gsp_remove(struct pci_dev *pdev);
int gsp_firmware_load(struct gsp_device *gsp);
int gsp_rpc_init(struct gsp_device *gsp);
void gsp_shutdown(struct gsp_device *gsp);

/* RPC Client */
struct gsp_rpc_client *gsp_rpc_client_init(struct gsp_device *gsp);
void gsp_rpc_client_fini(struct gsp_rpc_client *rpc);
int gsp_rpc_send_sync(struct gsp_rpc_client *rpc,
                      enum gsp_rpc_type type,
                      void *payload, size_t payload_size,
                      void *response, size_t *response_size);
int gsp_rpc_send_async(struct gsp_rpc_client *rpc,
                       enum gsp_rpc_type type,
                       void *payload, size_t payload_size,
                       void (*callback)(struct gsp_rpc_client *,
                                        struct gsp_rpc_response *,
                                        void *),
                       void *ctx);
irqreturn_t gsp_isr(int irq, void *data);
void gsp_handle_notification(struct gsp_device *gsp, struct gsp_rpc_message *msg);
int gsp_errno_to_errno(int gsp_err);

/* VF Management */
int gsp_vf_create(struct gsp_device *gsp, const struct gsp_vf_config *config);
void gsp_vf_destroy(struct gsp_vf *vf);
struct gsp_vf *gsp_vf_get(struct gsp_device *gsp, uint32_t id);
void gsp_vf_put(struct gsp_vf *vf);

/* Memory Management */
int gsp_vf_mem_alloc(struct gsp_vf *vf, uint64_t size, uint32_t flags,
                     uint64_t *handle, uint64_t *va);
int gsp_vf_mem_free(struct gsp_vf *vf, uint64_t handle);
int gsp_vf_mem_pin(struct gsp_vf *vf, uint64_t handle,
                   uint64_t offset, uint64_t size,
                   uint64_t *dma_addr);
int gsp_vf_mem_unpin(struct gsp_vf *vf, uint64_t handle);
int gsp_vf_mem_map(struct gsp_vf *vf, uint64_t handle, uint64_t gpu_va,
                   uint64_t size, uint32_t prot, uint32_t flags,
                   uint64_t offset, uint64_t *mapped_va);
int gsp_vf_mem_unmap(struct gsp_vf *vf, uint64_t handle, uint64_t gpu_va, uint64_t size);

/* BAR Emulation */
int gsp_vf_bar_init(struct gsp_vf *vf);
void gsp_vf_bar_fini(struct gsp_vf *vf);
int gsp_vf_bar_trap(struct gsp_vf *vf, struct gsp_bar_trap_payload *trap);
int gsp_bar_force_access(struct gsp_vf *vf, bool enable);
int gsp_migration_bar_save(struct gsp_vf *vf, void *buf, size_t *size);
int gsp_migration_bar_restore(struct gsp_vf *vf, void *buf, size_t size);

/* WPR Management */
int gsp_vf_wpr_init(struct gsp_vf *vf);
void gsp_vf_wpr_fini(struct gsp_vf *vf);
int gsp_vf_wpr_protect(struct gsp_vf *vf, uint64_t base, uint64_t size,
                       uint32_t flags);
int gsp_vf_wpr_unprotect(struct gsp_vf *vf, uint64_t base);
int gsp_vf_wpr_protect_write(struct gsp_vf *vf, uint64_t base, uint64_t size);
int gsp_vf_wpr_protect_read(struct gsp_vf *vf, uint64_t base, uint64_t size);
int gsp_vf_wpr_set_timeout(struct gsp_vf *vf, uint64_t timeout_ns);
int gsp_vf_wpr_register_callback(struct gsp_vf *vf, enum gsp_wpr_event type,
                                 void (*cb)(struct gsp_vf *, uint64_t, uint32_t),
                                 void *ctx);
void gsp_wpr_handle_event(struct gsp_device *gsp, struct gsp_rpc_message *msg);

/* Scheduling */
int gsp_vf_schedule(struct gsp_vf *vf, uint32_t priority, uint64_t time_slice_ns);
int gsp_vf_yield(struct gsp_vf *vf);

/* Migration */
int gsp_migration_start(struct gsp_vf *vf, uint64_t dst_vf);
void gsp_migration_cancel(struct gsp_migration *mig);
int gsp_migration_status(struct gsp_migration *mig);

/* Debugfs */
int gsp_debugfs_init(struct gsp_device *gsp);
void gsp_debugfs_fini(struct gsp_device *gsp);
void gsp_debugfs_vf_add(struct gsp_vf *vf);
void gsp_debugfs_vf_remove(struct gsp_vf *vf);

#endif /* _GSP_CORE_H_ */