1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
/*
* GSP Emulation Engine for Host-Side GSP Emulation
* Replaces guest-side GSP communication with host kernel emulation
*/
#ifndef _GSP_EMULATION_H_
#define _GSP_EMULATION_H_
#include <linux/types.h>
#include <linux/device.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
#include <linux/list.h>
#include <linux/kthread.h>
#include <linux/eventfd.h>
#include "nvidia-vgpu-vfio.h"
#include "gsp_rpc_client.h"
#include "bar_emulation.h"
#include "irq.h"
#include "memory.h"
/* ============================================================
* GSP Emulation Constants
* ============================================================ */
#define GSP_EMU_DOORBELL_OFFSET 0x110C00
#define GSP_EMU_SHARED_MEM_OFFSET 0x1000
#define GSP_EMU_SHARED_MEM_SIZE 4096
#define GSP_EMU_MSIX_TABLE_OFFSET 0x2000
#define GSP_EMU_MSIX_PBA_OFFSET 0x3000
#define GSP_EMU_POLL_BUSY_US 100
#define GSP_EMU_POLL_SLEEP_MIN_US 50
#define GSP_EMU_POLL_SLEEP_MAX_US 100
/* GSP falcon boot-watch: registers polled for change logging so the guest's
* physical GSP boot sequence is visible on the host (guest BAR0 access is
* mmap-based and otherwise invisible). Offsets are BAR0-relative, GSP falcon
* block at 0x110000. */
#define GSP_EMU_FALCON_WATCH_COUNT 20
/* BAR0 shadow offsets used by the guest's physical GSP boot */
#define GSP_EMU_REG_GSP_CPUCTL 0x110100 /* bit1 STARTCPU, bit4 HALTED */
#define GSP_EMU_REG_GSP_ENGINE 0x1103c0 /* bit0 RESET */
#define GSP_EMU_REG_GSP_MAILBOX0 0x110040
#define GSP_EMU_REG_GSP_MAILBOX1 0x110044
#define GSP_EMU_REG_VBIOS_SCRATCH_0E 0x001438 /* FRTS err code (0=NONE) */
#define GSP_EMU_REG_WPR2_ADDR_LO 0x1fa824
#define GSP_EMU_REG_WPR2_ADDR_HI 0x1fa828
#define GSP_EMU_REG_RISCV_STATUS 0x111240 /* bit0 ACTIVE (NV_PRISCV_RISCV
CORE_SWITCH_RISCV_STATUS=0x240 at
GSP riscvRegisterBase =
NV_FALCON2_GSP_BASE=0x111000) */
/* SEC2 falcon (Booter Load/Unload runs here, not on the GSP falcon) */
#define GSP_EMU_REG_SEC2_CPUCTL 0x840100
#define GSP_EMU_REG_SEC2_MAILBOX0 0x840040
#define GSP_EMU_REG_SEC2_ENGINE 0x8403c0
/* Base of the virtual-function register window (NV_VIRTUAL_FUNCTION_FULL_PHYS_
* OFFSET). The client RM reaches per-VF registers through GPU_VREG_* =
* BAR0 + sriovState.virtualRegPhysOffset + offset; for a non-SRIOV virtual GPU
* that offset is 0xB80000, and 0 for a VF, hence the plain + windowed aliases
* below. */
#define GSP_EMU_REG_FULL_PHYS_OFFSET 0xB80000
/* NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE: driver sets TRIGGER(bit31) to
* start a TLB invalidate, then polls for bit31==0 (done). We auto-clear it.
* The client RM reaches it through the GPU_VREG_* window (BAR0 + 0xB80000 +
* 0x30B0) and the physical path pokes the plain 0x30B0 alias, so watch both. */
#define GSP_EMU_REG_MMU_INVALIDATE 0x0030b0
#define GSP_EMU_MMU_INVALIDATE_TRIGGER (1U << 31)
#define GSP_EMU_REG_MMU_INVALIDATE_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x0030b0)
/* L2 cache-op completion registers (NV_UFLUSH_*): the driver writes PENDING
* (bit 0) to issue FB flush / L2 invalidate / L2 clean+dirty flush and then
* spins in kmemsysDoCacheOp_GM107 until the polled bits read back 0. On real
* hardware the L2 clears them when the operation completes; the emulated L2
* completes instantly, so auto-clear any pending bits.
*
* On Turing the sysmem/peermem invalidate registers are reached through the
* virtual-register window (GPU_VREG_*: BAR0 + sriovState.virtualRegPhysOffset
* + NV_VIRTUAL_FUNCTION_PRIV_L2_*); the offset is 0xB80000 for a non-SRIOV
* virtual GPU and 0 for a VF, so watch both the plain and windowed aliases. */
#define GSP_EMU_REG_UFLUSH_BASE 0x070000
#define GSP_EMU_REG_UFLUSH_FB_FLUSH (GSP_EMU_REG_UFLUSH_BASE + 0x00)
#define GSP_EMU_REG_UFLUSH_L2_SYSMEM_INV (GSP_EMU_REG_UFLUSH_BASE + 0x04)
#define GSP_EMU_REG_UFLUSH_L2_PEERMEM_INV (GSP_EMU_REG_UFLUSH_BASE + 0x08)
#define GSP_EMU_REG_UFLUSH_L2_CLEAN_COMPTAGS (GSP_EMU_REG_UFLUSH_BASE + 0x0c)
#define GSP_EMU_REG_UFLUSH_L2_FLUSH_DIRTY (GSP_EMU_REG_UFLUSH_BASE + 0x10)
#define GSP_EMU_REG_UFLUSH_L2_WAIT_SYS_READS (GSP_EMU_REG_UFLUSH_BASE + 0x14)
/* NV_VIRTUAL_FUNCTION_PRIV_L2_{SYSMEM,PEERMEM}_INVALIDATE (dev_vm.h) */
#define GSP_EMU_REG_VF_L2_SYSMEM_INV 0x000f00
#define GSP_EMU_REG_VF_L2_PEERMEM_INV 0x000f04
/* NV_PFALCON_FALCON_DEBUGINFO (GSP falcon BAR0+0x110094): holds the CrashCat
* wayfinder-L0 descriptor. If its low 16 bits == 0xdead (NV_CRASHCAT_SIGNATURE),
* the driver's kgspHealthCheck reads crashcat reports from garbage and loops
* forever in nv_open_q (which then can't poll RPCs -> fn-10 FREE timeouts).
* Force it to 0 (invalid wayfinder -> crashcatEngineGetNextCrashReport
* returns NULL) so the health check always passes. */
#define GSP_EMU_REG_FALCON_DEBUGINFO 0x110094
/* CPU interrupt tree (Turing, physical-GPU path): the driver accesses the
* VF window registers at virtualRegPhysOffset = NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET
* (0xB80000) + offset.
* LEAF_TRIGGER 0xB80000+0x1640 - write a vector to fire its software interrupt
* LEAF[i] 0xB80000+0x1000+i*4 - per-32-vector pending bits
* TOP[i] 0xB81600+i*4 - per-32-subtree pending bits
* _osVerifyInterrupts (osVerifySystemEnvironment) fires a doorbell software
* interrupt and waits for its ISR; if we don't set the pending state and
* inject MSI-X, the ISR never runs and RmInitAdapter bails. */
#define GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1640)
#define GSP_EMU_REG_CPU_INTR_LEAF_BASE (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1000)
#define GSP_EMU_REG_CPU_INTR_TOP_BASE (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1600)
/* Windowed aliases of the VF L2 invalidate registers (GPU_VREG_* path). */
#define GSP_EMU_REG_VF_L2_SYSMEM_INV_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x000f00)
#define GSP_EMU_REG_VF_L2_PEERMEM_INV_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x000f04)
#define GSP_EMU_CPUCTL_STARTCPU 0x2
#define GSP_EMU_CPUCTL_HALTED 0x10
/* ============================================================
* GSP Emulation Context
* ============================================================ */
struct gsp_emu_context {
struct nv_vgpu_device *vgpu;
struct task_struct *thread;
struct completion thread_stopped;
void __iomem *bar0_vaddr;
void __iomem *doorbell_reg;
void __iomem *shared_mem;
struct gsp_client *rpc_client;
spinlock_t lock;
bool running;
bool initialized;
bool stopping;
/* Shared memory for RPC communication (4KB at BAR0+0x1000) */
void *shared_mem_virt;
dma_addr_t shared_mem_dma;
size_t shared_mem_size;
/* Statistics */
u64 processed_rpcs;
u64 failed_rpcs;
u64 total_latency_ns;
/* MSI-X injection */
struct eventfd_ctx *msix_trigger[NV_VGPU_MAX_IRQS];
/* Falcon boot-watch: last-sampled values for change detection */
u32 falcon_watch_last[GSP_EMU_FALCON_WATCH_COUNT];
bool falcon_watch_valid;
/* Physical GSP boot emulation state (gsp_emu_poll_boot) */
u32 boot_stage; /* enum gsp_emu_boot_stage */
u32 flcn_kick_count; /* STARTCPU kicks seen since last ENGINE reset */
u32 wpr2_addr_lo; /* WPR2_ADDR_LO value to report after FWSEC */
/* GSP-RM msgq emulation (physical path, guest-sysmem queues) */
struct gsp_rm_emu *rm_emu;
/* PRAMIN window translation state (BAR0 window -> emulated vidmem) */
u32 bar0_window_base; /* last-seen NV_PBUS_BAR0_WINDOW BASE (page) */
u32 bar0_window_target; /* last-seen TARGET (0=vidmem, 2/3=sysmem) */
bool bar0_window_valid; /* base/target captured at least once */
};
/* Shadow accessors shared with gsp_rm_emulation.c */
static inline void gsp_emu_shadow_write32(struct gsp_emu_context *ctx,
u32 offset, u32 val)
{
writel(val, ctx->bar0_vaddr + offset);
}
static inline u32 gsp_emu_shadow_read32(struct gsp_emu_context *ctx, u32 offset)
{
return readl(ctx->bar0_vaddr + offset);
}
/* ============================================================
* Public API
* ============================================================ */
int gsp_emu_init(struct nv_vgpu_device *vgpu);
void gsp_emu_cleanup(struct nv_vgpu_device *vgpu);
/* Clear emulated GSP boot state (WPR2/RISCV/stages/RM queues) on
* ENGINE reset or detected GSP death (stale msgq). */
void gsp_emu_clear_boot_state(struct gsp_emu_context *ctx, const char *reason);
int gsp_emu_process_rpc(struct gsp_emu_context *ctx);
void gsp_emu_inject_completion_interrupt(struct gsp_emu_context *ctx, u32 seq_id);
/* ============================================================
* Debug/Statistics
* ============================================================ */
void gsp_emu_get_stats(struct gsp_emu_context *ctx,
u64 *processed, u64 *failed, u64 *total_latency_ns);
#endif /* _GSP_EMULATION_H_ */
|
