/* SPDX-License-Identifier: GPL-2.0-only */ /* * GSP vGPU-IO Driver - Core Definitions * NVIDIA GSP 570.144W (TU102/TU104) Firmware Interface */ #ifndef _GSP_CORE_H_ #define _GSP_CORE_H_ #include #include #include #include #include #include #include #include #include #include /* Firmware Constants */ #define GSP_FW_BASE_ADDR 0x04000000ULL #define GSP_FW_SIZE 0x02000000ULL #define GSP_ENTRY_POINT 0x04009000ULL #define GSP_PRIMARY_DISPATCHER 0x057A1AC0ULL #define GSP_SECONDARY_DISPATCHER 0x04ED39DCULL #define GSP_RPC_JUMP_TABLE 0x059BFEB0ULL /* PCIe BAR Offsets */ #define GSP_BAR0_FALCON_ENGINE 0x1103c0 #define GSP_BAR0_FALCON_MAILBOX0 0x110040 #define GSP_BAR0_FALCON_MAILBOX1 0x110044 #define GSP_BAR0_MAILBOX(i) (0x110804 + (i) * 4) #define GSP_BAR0_QUEUE_HEAD(i) (0x110c00 + (i) * 8) /* IOCTL definitions (must match userspace libvgpu.h) */ #define GSP_IOCTL_MAGIC 'G' #define GSP_IOCTL_CREATE_VF _IOW(GSP_IOCTL_MAGIC, 0x01, struct gsp_vf_config) #define GSP_IOCTL_DESTROY_VF _IOW(GSP_IOCTL_MAGIC, 0x02, uint32_t) #define GSP_IOCTL_ALLOC _IOWR(GSP_IOCTL_MAGIC, 0x10, struct gsp_alloc_args) #define GSP_IOCTL_FREE _IOW(GSP_IOCTL_MAGIC, 0x11, struct gsp_free_args) #define GSP_IOCTL_MAP _IOWR(GSP_IOCTL_MAGIC, 0x12, struct gsp_map_args) #define GSP_IOCTL_UNMAP _IOW(GSP_IOCTL_MAGIC, 0x13, struct gsp_unmap_args) #define GSP_IOCTL_PIN _IOWR(GSP_IOCTL_MAGIC, 0x14, struct gsp_pin_args) #define GSP_IOCTL_UNPIN _IOW(GSP_IOCTL_MAGIC, 0x15, struct gsp_unpin_args) #define GSP_IOCTL_BAR_CONFIG _IOWR(GSP_IOCTL_MAGIC, 0x16, struct gsp_bar_config_args) #define GSP_IOCTL_BAR_FORCE _IOW(GSP_IOCTL_MAGIC, 0x17, struct gsp_bar_force_args) #define GSP_IOCTL_BAR_SAVE _IOWR(GSP_IOCTL_MAGIC, 0x18, struct gsp_bar_save_args) #define GSP_IOCTL_BAR_RESTORE _IOWR(GSP_IOCTL_MAGIC, 0x19, struct gsp_bar_restore_args) #define GSP_IOCTL_WPR_PROTECT _IOWR(GSP_IOCTL_MAGIC, 0x1A, struct gsp_wpr_args) #define GSP_IOCTL_WPR_UNPROTECT _IOW(GSP_IOCTL_MAGIC, 0x1B, struct gsp_wpr_args) #define GSP_IOCTL_WPR_TIMEOUT _IOW(GSP_IOCTL_MAGIC, 0x1C, struct gsp_wpr_timeout_args) #define GSP_IOCTL_SCHEDULE _IOW(GSP_IOCTL_MAGIC, 0x1D, struct gsp_schedule_args) #define GSP_IOCTL_YIELD _IOW(GSP_IOCTL_MAGIC, 0x1E, uint32_t) #define GSP_IOCTL_MIGRATE _IOWR(GSP_IOCTL_MAGIC, 0x1F, struct gsp_migrate_args) #define GSP_IOCTL_GET_INFO _IOR(GSP_IOCTL_MAGIC, 0x20, char[256]) /* Kernel argument structures (matching userspace) */ struct gsp_vf_config { uint32_t id; uint64_t bar1_offset; uint64_t bar1_size; uint64_t pt_base; uint64_t pt_size; uint64_t cmd_base; uint64_t cmd_size; uint32_t max_memory_mb; uint32_t num_engines; }; struct gsp_alloc_args { uint32_t vf_id; uint64_t size; uint32_t type; uint64_t alignment; uint64_t va_hint; uint32_t flags; uint32_t pid; uint64_t handle; uint64_t va; uint64_t actual_size; }; struct gsp_free_args { uint32_t vf_id; uint64_t handle; }; struct gsp_map_args { uint32_t vf_id; uint64_t handle; uint64_t gpu_va; uint64_t size; uint32_t prot; uint32_t flags; uint64_t offset; uint64_t mapped_va; }; struct gsp_unmap_args { uint32_t vf_id; uint64_t handle; uint64_t gpu_va; uint64_t size; }; struct gsp_pin_args { uint32_t vf_id; uint64_t handle; uint64_t offset; uint64_t size; uint32_t flags; uint64_t dma_addr; uint32_t num_pages; }; struct gsp_unpin_args { uint32_t vf_id; uint64_t handle; uint64_t offset; uint64_t size; }; struct gsp_bar_config_args { uint32_t vf_id; uint64_t aperture_offset; uint64_t aperture_size; }; struct gsp_bar_force_args { uint32_t vf_id; uint32_t enable; }; struct gsp_bar_save_args { uint32_t vf_id; void *buf; size_t *size; }; struct gsp_bar_restore_args { uint32_t vf_id; void *buf; size_t size; }; struct gsp_wpr_args { uint32_t vf_id; uint64_t base; uint64_t size; uint32_t flags; }; struct gsp_wpr_timeout_args { uint32_t vf_id; uint64_t timeout_ns; }; struct gsp_schedule_args { uint32_t vf_id; uint32_t priority; uint64_t time_slice_ns; }; struct gsp_migrate_args { uint32_t vf_id; uint64_t dst_vf; uint32_t phase; uint32_t flags; uint64_t token; }; /* Message Ring */ #define GSP_MSG_RING_SIZE 256 #define GSP_MSG_MAX_SIZE 256 #define GSP_MAX_PENDING 64 /* RPC Types */ enum gsp_rpc_type { GSP_RPC_ALLOC = 0x01, GSP_RPC_FREE = 0x02, GSP_RPC_MAP = 0x03, GSP_RPC_UNMAP = 0x04, GSP_RPC_PIN = 0x05, GSP_RPC_UNPIN = 0x06, GSP_RPC_SCHEDULE = 0x07, GSP_RPC_YIELD = 0x08, GSP_RPC_NOTIFY = 0x09, GSP_RPC_BAR_TRAP = 0x0A, GSP_RPC_MIGRATE = 0x0B, GSP_RPC_DEBUG = 0x0C, }; /* RPC Message Flags */ #define GSP_MSG_SYNC (1 << 0) #define GSP_MSG_ASYNC (1 << 1) #define GSP_MSG_NOREPLY (1 << 2) /* Memory Types */ enum gsp_mem_type { GSP_MEM_VRAM = 0, GSP_MEM_SYSMEM = 1, GSP_MEM_UNCACHED = 2, }; /* Pin Flags */ #define GSP_PIN_READ (1 << 0) #define GSP_PIN_WRITE (1 << 1) #define GSP_PIN_LONGTERM (1 << 2) /* BAR Trap Types */ #define GSP_BAR_READ 0 #define GSP_BAR_WRITE 1 #define GSP_BAR_ATOMIC 2 /* Migration Phases */ enum gsp_migrate_phase { GSP_MIG_QUERY = 0, GSP_MIG_PREPARE = 1, GSP_MIG_TRANSFER = 2, GSP_MIG_COMMIT = 3, GSP_MIG_CLEANUP = 4, }; /* Migration Flags */ #define GSP_MIG_LIVE (1 << 0) #define GSP_MIG_POSTCOPY (1 << 1) #define GSP_MIG_AUTO_COMMIT (1 << 2) #define GSP_MIG_VERIFY (1 << 3) /* Error Codes (from firmware) */ #define GSP_OK 0 #define GSP_EPERM 0xFFFFFFFF #define GSP_ENOENT 0xFFFFFFFE #define GSP_EINVAL 0xFFFFFFFD #define GSP_ENODEV 0xFFFFFFFA #define GSP_ENOMEM 0xFFFFFFF4 #define GSP_EBUSY 0xFFFFFFEE #define GSP_ENOSPC 0xFFFFFFE4 #define GSP_ETIMEDOUT 0x0000001E #define GSP_EOVERFLOW 0x00000025 #define GSP_EDEV 0x0000001F /* Forward Declarations */ struct gsp_device; struct gsp_vf; struct gsp_rpc_client; struct gsp_migration; struct gsp_wpr_region; /* WPR Region */ struct gsp_wpr_region { struct list_head list; uint64_t base; uint64_t size; uint32_t flags; uint32_t vf_id; void (*completion_cb)(struct gsp_vf *, uint64_t, uint32_t); void (*timeout_cb)(struct gsp_vf *, uint64_t, uint32_t); uint64_t timeout_ns; }; /* WPR Flags */ #define GSP_WPR_WRITE (1 << 0) #define GSP_WPR_READ (1 << 1) #define GSP_WPR_EXECUTE (1 << 2) /* WPR Event Types */ enum gsp_wpr_event { GSP_WPR_WRITE_COMPLETE = 0, GSP_WPR_READ_COMPLETE = 1, GSP_WPR_WRITE_TIMEOUT = 2, GSP_WPR_READ_TIMEOUT = 3, }; /* Migration State */ enum gsp_mig_state { GSP_MIG_S_IDLE = 0, GSP_MIG_S_QUERY = 1, GSP_MIG_S_PREPARE = 2, GSP_MIG_S_TRANSFER = 3, GSP_MIG_S_COMMIT = 4, GSP_MIG_S_CLEANUP = 5, GSP_MIG_S_FAILED = 6, GSP_MIG_S_COMPLETED = 7, }; /* Migration Context */ struct gsp_migration { struct gsp_vf *vf; struct mutex lock; enum gsp_mig_state state; uint64_t token; uint64_t dst_vf; uint32_t flags; struct { uint64_t pages_transferred; uint64_t bytes_transferred; uint64_t dirty_pages; uint64_t *page_list; uint32_t page_count; } progress; struct { void *buf; size_t size; } bar_state; struct task_struct *thread; struct completion comp; bool cancelled; int error; }; /* Emulation mode: skip GSP RPC for BAR/WPR when GSP lacks vGPU handlers */ extern bool emulation_mode; /* RPC Message Header (64 bytes) */ struct gsp_rpc_header { uint32_t magic; /* 0x47535052 "GSPR" */ uint32_t version; /* Protocol version */ uint32_t rpc_type; /* enum gsp_rpc_type */ uint32_t sequence; /* Monotonic sequence */ uint64_t timestamp_ns; /* Submission timestamp */ uint32_t payload_size; /* Payload size in bytes */ uint32_t flags; /* GSP_MSG_* */ uint64_t correlation_id; /* Async matching */ uint64_t reserved[3]; /* Padding to 64 bytes */ } __attribute__((packed)); /* Complete RPC Message */ struct gsp_rpc_message { struct gsp_rpc_header hdr; uint8_t payload[GSP_MSG_MAX_SIZE - sizeof(struct gsp_rpc_header)]; } __attribute__((packed)); /* RPC Response Header */ struct gsp_rpc_response { int64_t status; /* 0 = success, negative = error */ uint32_t rpc_type; /* Echo of request */ uint32_t sequence; /* Echo of request */ uint64_t timestamp_ns; /* Completion timestamp */ uint32_t payload_size; /* Response payload size */ uint32_t reserved; uint8_t payload[]; } __attribute__((packed)); /* Primary Dispatcher Sub-Command Descriptor */ struct gsp_subcmd_desc { uint32_t subcmd_id; /* Constant ID (e.g., 0x10, 0x14, 0x40) */ uint8_t echo_flags_b; /* Copy of input flags_b */ uint8_t echo_flags_a; /* Copy of input flags_a */ uint8_t echo_flags_c; /* Copy of input flags_c */ uint8_t reserved; } __attribute__((packed)); /* ALLOC Payload */ struct gsp_alloc_payload { uint64_t size; uint32_t type; /* enum gsp_mem_type */ uint32_t alignment; uint64_t va_hint; uint32_t flags; uint32_t pid; uint32_t vf_id; uint32_t reserved; } __attribute__((packed)); struct gsp_alloc_response { uint64_t handle; uint64_t va; uint64_t pa; } __attribute__((packed)); /* PIN Payload */ struct gsp_pin_payload { uint64_t handle; uint64_t offset; uint64_t size; uint32_t vf_id; uint32_t flags; /* GSP_PIN_* */ uint32_t reserved[2]; } __attribute__((packed)); struct gsp_pin_response { uint64_t dma_addr; } __attribute__((packed)); /* BAR Trap Payload */ struct gsp_bar_trap_payload { uint32_t bar_index; uint32_t trap_type; /* GSP_BAR_* */ uint64_t offset; uint64_t value; uint32_t size; uint32_t vf_id; uint32_t reserved; } __attribute__((packed)); struct gsp_bar_trap_response { uint64_t value; } __attribute__((packed)); /* Migration Payload */ struct gsp_migrate_payload { uint32_t phase; /* enum gsp_migrate_phase */ uint32_t subcmd; /* Sub-command index */ uint64_t src_vf; uint64_t dst_vf; uint64_t data_ptr; uint64_t data_size; uint32_t flags; /* GSP_MIG_* */ uint32_t reserved; uint64_t token; } __attribute__((packed)); /* Message Ring Buffer */ struct gsp_msg_ring { void *base; size_t size; uint32_t head; uint32_t tail; spinlock_t lock; }; /* RPC Client */ struct gsp_rpc_client { struct gsp_device *gsp; struct mutex lock; atomic_t seq; wait_queue_head_t wq; struct gsp_msg_ring *tx_ring; struct gsp_msg_ring *rx_ring; dma_addr_t tx_ring_dma; dma_addr_t rx_ring_dma; struct completion *pending[GSP_MAX_PENDING]; struct gsp_rpc_response *resp_buf[GSP_MAX_PENDING]; }; /* BAR State per VF */ struct gsp_bar_state { uint64_t aperture_offset; uint64_t aperture_size; bool force_access; uint32_t force_vf; void *backing_store; size_t backing_size; }; /* VF State */ enum gsp_vf_state { GSP_VF_CREATING = 0, GSP_VF_ACTIVE, GSP_VF_MIGRATING, GSP_VF_SUSPENDED, GSP_VF_DESTROYING, }; /* GSP Virtual Function */ struct gsp_vf { struct gsp_device *gsp; uint32_t id; struct kref ref; struct mutex lock; enum gsp_vf_state state; struct gsp_migration *migration; /* Resource configuration */ uint64_t bar1_offset; uint64_t bar1_size; uint64_t pt_base; uint64_t pt_size; uint64_t cmd_base; uint64_t cmd_size; /* WPR regions */ struct list_head wpr_list; struct mutex wpr_lock; /* BAR state */ struct gsp_bar_state bar_state; atomic_t refcount; bool active; struct list_head list; }; /* GSP Device */ struct gsp_device { struct pci_dev *pdev; struct device *dev; void __iomem *bar0; void __iomem *bar1; /* Firmware */ const struct firmware *fw; bool fw_loaded; /* RPC */ struct gsp_rpc_client *rpc; /* VF Management */ struct mutex vf_lock; struct idr vf_idr; struct list_head vf_list; uint32_t max_vfs; /* IRQ */ int irq; bool msix_enabled; struct msix_entry msix_entries[1]; /* Debug */ struct dentry *debugfs_root; struct dentry *debugfs_vfs_dir; /* Char device */ struct miscdevice miscdev; struct mutex chardev_lock; /* Statistics */ atomic64_t rpc_total; atomic64_t rpc_errors; atomic64_t bar_traps; atomic64_t wpr_events; atomic64_t migrations; }; /* Function Prototypes */ /* Core */ int gsp_probe(struct pci_dev *pdev, const struct pci_device_id *id); void gsp_remove(struct pci_dev *pdev); int gsp_firmware_load(struct gsp_device *gsp); int gsp_rpc_init(struct gsp_device *gsp); void gsp_shutdown(struct gsp_device *gsp); /* RPC Client */ struct gsp_rpc_client *gsp_rpc_client_init(struct gsp_device *gsp); void gsp_rpc_client_fini(struct gsp_rpc_client *rpc); int gsp_rpc_send_sync(struct gsp_rpc_client *rpc, enum gsp_rpc_type type, void *payload, size_t payload_size, void *response, size_t *response_size); int gsp_rpc_send_async(struct gsp_rpc_client *rpc, enum gsp_rpc_type type, void *payload, size_t payload_size, void (*callback)(struct gsp_rpc_client *, struct gsp_rpc_response *, void *), void *ctx); irqreturn_t gsp_isr(int irq, void *data); void gsp_handle_notification(struct gsp_device *gsp, struct gsp_rpc_message *msg); int gsp_errno_to_errno(int gsp_err); /* VF Management */ int gsp_vf_create(struct gsp_device *gsp, const struct gsp_vf_config *config); void gsp_vf_destroy(struct gsp_vf *vf); struct gsp_vf *gsp_vf_get(struct gsp_device *gsp, uint32_t id); void gsp_vf_put(struct gsp_vf *vf); /* Memory Management */ int gsp_vf_mem_alloc(struct gsp_vf *vf, uint64_t size, uint32_t flags, uint64_t *handle, uint64_t *va); int gsp_vf_mem_free(struct gsp_vf *vf, uint64_t handle); int gsp_vf_mem_pin(struct gsp_vf *vf, uint64_t handle, uint64_t offset, uint64_t size, uint64_t *dma_addr); int gsp_vf_mem_unpin(struct gsp_vf *vf, uint64_t handle); int gsp_vf_mem_map(struct gsp_vf *vf, uint64_t handle, uint64_t gpu_va, uint64_t size, uint32_t prot, uint32_t flags, uint64_t offset, uint64_t *mapped_va); int gsp_vf_mem_unmap(struct gsp_vf *vf, uint64_t handle, uint64_t gpu_va, uint64_t size); /* BAR Emulation */ int gsp_vf_bar_init(struct gsp_vf *vf); void gsp_vf_bar_fini(struct gsp_vf *vf); int gsp_vf_bar_trap(struct gsp_vf *vf, struct gsp_bar_trap_payload *trap); int gsp_bar_force_access(struct gsp_vf *vf, bool enable); int gsp_migration_bar_save(struct gsp_vf *vf, void *buf, size_t *size); int gsp_migration_bar_restore(struct gsp_vf *vf, void *buf, size_t size); /* WPR Management */ int gsp_vf_wpr_init(struct gsp_vf *vf); void gsp_vf_wpr_fini(struct gsp_vf *vf); int gsp_vf_wpr_protect(struct gsp_vf *vf, uint64_t base, uint64_t size, uint32_t flags); int gsp_vf_wpr_unprotect(struct gsp_vf *vf, uint64_t base); int gsp_vf_wpr_protect_write(struct gsp_vf *vf, uint64_t base, uint64_t size); int gsp_vf_wpr_protect_read(struct gsp_vf *vf, uint64_t base, uint64_t size); int gsp_vf_wpr_set_timeout(struct gsp_vf *vf, uint64_t timeout_ns); int gsp_vf_wpr_register_callback(struct gsp_vf *vf, enum gsp_wpr_event type, void (*cb)(struct gsp_vf *, uint64_t, uint32_t), void *ctx); void gsp_wpr_handle_event(struct gsp_device *gsp, struct gsp_rpc_message *msg); /* Scheduling */ int gsp_vf_schedule(struct gsp_vf *vf, uint32_t priority, uint64_t time_slice_ns); int gsp_vf_yield(struct gsp_vf *vf); /* Migration */ int gsp_migration_start(struct gsp_vf *vf, uint64_t dst_vf); void gsp_migration_cancel(struct gsp_migration *mig); int gsp_migration_status(struct gsp_migration *mig); /* Debugfs */ int gsp_debugfs_init(struct gsp_device *gsp); void gsp_debugfs_fini(struct gsp_device *gsp); void gsp_debugfs_vf_add(struct gsp_vf *vf); void gsp_debugfs_vf_remove(struct gsp_vf *vf); #endif /* _GSP_CORE_H_ */