summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_rm_emulation.c273
-rw-r--r--kernel/gsp_rm_emulation.h23
-rw-r--r--kernel/gsp_sema_service.c1218
-rw-r--r--kernel/gsp_sema_service.h23
-rw-r--r--kernel/main.c269
5 files changed, 1744 insertions, 62 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index a21cf28..986132e 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -473,11 +473,17 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
if (!buf)
return -ENOMEM;
+ /* The status queue is shared: RPC replies run on the GSP emulation
+ * thread, channel-completion events are posted from the progress
+ * service's thread. */
+ mutex_lock(&rm->stat_lock);
+
record_size = gsp_rm_build_record(rm, buf, buf_size, function,
sequence, result, data, data_len,
&elem_count);
if (!record_size) {
kfree(buf);
+ mutex_unlock(&rm->stat_lock);
return -EOVERFLOW;
}
@@ -486,6 +492,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
nv_vgpu_err(ctx->vgpu, "GSPRM: record too large: %u elements > queue %u",
elem_count, rm->stat_msgcount);
kfree(buf);
+ mutex_unlock(&rm->stat_lock);
return -EOVERFLOW;
}
@@ -507,6 +514,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
nv_vgpu_err(ctx->vgpu, "GSPRM: status write failed off=0x%llx ret=%d",
rec_off, ret);
kfree(buf);
+ mutex_unlock(&rm->stat_lock);
return ret;
}
@@ -518,6 +526,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
nv_vgpu_err(ctx->vgpu, "GSPRM: status wrap write failed ret=%d",
ret);
kfree(buf);
+ mutex_unlock(&rm->stat_lock);
return ret;
}
}
@@ -538,6 +547,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
&new_wp, sizeof(new_wp));
if (ret) {
nv_vgpu_err(ctx->vgpu, "GSPRM: status writePtr update failed ret=%d", ret);
+ mutex_unlock(&rm->stat_lock);
return ret;
}
@@ -547,9 +557,82 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
* drain while a poller waits for it (same as real GSP-RM). */
gsp_emu_inject_completion_interrupt(ctx, rm->stat_seq);
+ mutex_unlock(&rm->stat_lock);
return 0;
}
+/*
+ * Post the GPU event that tells the guest's CPU-side RM a channel's
+ * submitted work is complete. A real GPU raises an interrupt, the host GSP
+ * turns it into an NV_VGPU_MSG_EVENT_POST_EVENT record for the client's
+ * event object, and the client's RM signals the OS event (eventfd) the
+ * submitting thread polls - this is what makes libcuda's work waits finish.
+ *
+ * Safe to call from the progress service's thread: the status queue write is
+ * serialized by rm->stat_lock and the guest is woken with MSI-X vector 0.
+ */
+int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src)
+{
+ struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL;
+ struct gsp_rm_emu *rm;
+ u8 payload[32];
+ u32 i;
+ int posted = 0;
+
+ if (!ctx)
+ return -ENODEV;
+ rm = ctx->rm_emu;
+ if (!rm || !h_src)
+ return -ENODEV;
+
+ for (i = 0; i < rm->events_n; i++) {
+ if (!rm->events[i].valid || rm->events[i].h_src != h_src)
+ continue;
+
+ memset(payload, 0, sizeof(payload));
+ memcpy(payload + 0, &rm->events[i].h_client, 4);
+ memcpy(payload + 4, &rm->events[i].h_event, 4);
+ memcpy(payload + 8, &rm->events[i].notify_index, 4);
+ memcpy(payload + 12, &rm->events[i].data, 4);
+ /* info16@16, status@20, eventDataSize@24, bNotifyList@28 = 0 */
+ if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
+ payload, sizeof(payload)))
+ continue;
+ posted++;
+ }
+
+ /*
+ * Fallback when the event's source object could not be associated with
+ * the channel: nudge every registered event. The client's dispatcher
+ * re-reads each channel's state on any event, so a spurious wake-up is
+ * harmless (and this only runs when a submission was consumed).
+ */
+ if (!posted) {
+ for (i = 0; i < rm->events_n; i++) {
+ if (!rm->events[i].valid)
+ continue;
+ memset(payload, 0, sizeof(payload));
+ memcpy(payload + 0, &rm->events[i].h_client, 4);
+ memcpy(payload + 4, &rm->events[i].h_event, 4);
+ memcpy(payload + 8, &rm->events[i].notify_index, 4);
+ memcpy(payload + 12, &rm->events[i].data, 4);
+ if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
+ payload, sizeof(payload)))
+ continue;
+ posted++;
+ }
+ }
+
+ if (posted) {
+ rm->events_posted += posted;
+ if (rm->events_posted <= 32 || (rm->events_posted % 5000) == 0)
+ nv_vgpu_info(vgpu,
+ "GSPRM: post_event hSrc=0x%x posted=%d total=%llu",
+ h_src, posted, rm->events_posted);
+ }
+ return posted ? 0 : -ENOENT;
+}
+
/* ============================================================
* Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED)
* ============================================================ */
@@ -714,6 +797,8 @@ static const char *gsp_rm_rpc_name(u32 function)
case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO: return "GSP_SET_SYSTEM_INFO";
case GSPRM_RPC_FUNC_SET_REGISTRY: return "SET_REGISTRY";
case GSPRM_RPC_FUNC_CONTINUATION_RECORD: return "CONTINUATION_RECORD";
+ case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY: return "SET_PAGE_DIRECTORY";
+ case GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY: return "UNSET_PAGE_DIRECTORY";
case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL";
default: return "UNKNOWN";
}
@@ -1949,6 +2034,7 @@ static bool gsp_rm_is_channel_class(u32 class_id)
#define GSPRM_CHAN_GPFIFO_OFFSET 8
#define GSPRM_CHAN_GPFIFO_ENTRIES 16
#define GSPRM_CHAN_ENGINE_TYPE 128
+#define GSPRM_CHAN_CID 132
#define GSPRM_CHAN_INSTANCE_MEM 144
#define GSPRM_CHAN_USERD_MEM 168
#define GSPRM_CHAN_MEM_APERTURE 16 /* offset of addressSpace in the desc */
@@ -1967,11 +2053,14 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8);
memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4);
memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4);
+ memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4);
memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8);
memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8);
memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM +
GSPRM_CHAN_MEM_APERTURE, 4);
memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8);
+ memcpy(&desc.notifier_as, params + GSPRM_CHAN_NOTIFIER_MEM +
+ GSPRM_CHAN_MEM_APERTURE, 4);
gsp_sema_service_channel_add(&desc);
}
@@ -2038,6 +2127,58 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx,
if (gsp_rm_is_channel_class(class_id))
gsp_rm_register_channel(ctx, ho, params, params_size);
+ /*
+ * NV01_EVENT (allocated with class 0x79 in the guest ABI; params are
+ * NV0005_ALLOC_PARAMETERS {hParentClient, hSrcResource, hClass,
+ * notifyIndex, data}). The driver allocates one per channel and waits
+ * for the GPU to post the event before it considers submitted work
+ * complete - remember them so completed work can be signalled back.
+ */
+ if (class_id == 0x79 && params_size >= 24 && rm->events_n < GSPRM_MAX_EVENTS) {
+ u32 h_src = 0, notify_index = 0, hclass = 0;
+ u64 data64 = 0;
+ int hc_off = -1, i;
+
+ /*
+ * Locate hClass (0x79) inside the params instead of trusting a
+ * fixed offset: NV0005_ALLOC_PARAMETERS is
+ * {hParentClient, hSrcResource, hClass, notifyIndex, data},
+ * with hSrcResource immediately before hClass.
+ */
+ for (i = 0; i + 4 <= (int)params_size; i += 4) {
+ u32 v;
+
+ memcpy(&v, params + i, 4);
+ if (v == 0x79) {
+ hc_off = i;
+ break;
+ }
+ }
+ if (hc_off >= 4) {
+ memcpy(&h_src, params + hc_off - 4, 4);
+ memcpy(&hclass, params + hc_off, 4);
+ if (hc_off + 8 <= (int)params_size)
+ memcpy(&notify_index, params + hc_off + 4, 4);
+ if (hc_off + 16 <= (int)params_size)
+ memcpy(&data64, params + hc_off + 8, 8);
+ } else {
+ memcpy(&h_src, params + 4, 4);
+ memcpy(&notify_index, params + 12, 4);
+ memcpy(&data64, params + 16, 8);
+ }
+ rm->events[rm->events_n].h_src = h_src;
+ rm->events[rm->events_n].h_event = ho;
+ rm->events[rm->events_n].h_client = hc;
+ rm->events[rm->events_n].notify_index = notify_index;
+ rm->events[rm->events_n].data = (u32)data64;
+ rm->events[rm->events_n].valid = true;
+ rm->events_n++;
+ nv_vgpu_info(ctx->vgpu,
+ "GSPRM: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d) params=%*phN",
+ ho, h_src, notify_index, hc_off, (int)min(params_size, (u32)24),
+ params);
+ }
+
/* Echo the fixed alloc header + as many param bytes as are contiguous
* in the first queue element. */
echo_len = min(params_size,
@@ -2336,6 +2477,129 @@ static bool gsp_rm_alloc_interesting(u32 class_id)
}
}
+/*
+ * SET_PAGE_DIRECTORY (54) / UNSET_PAGE_DIRECTORY (79)
+ *
+ * The driver builds the page tables of the GPU VA spaces it owns and hands the
+ * top-level directory to the GPU here. The parameters are
+ * { u64 physAddress; u32 numEntries; u32 flags; u32 hVASpace; u32 chId;
+ * u32 subDeviceId; u32 pasid; }
+ * (flags bits 1:0: 0 = vidmem aperture, 1 = sysmem coherent,
+ * 2 = sysmem non-coherent).
+ *
+ * Recording the directory lets the progress service translate that VA space's
+ * addresses directly (four-level walk), which is how the client's semaphore
+ * waits are completed; it also removes the need for heuristic discovery.
+ */
+static void gsp_rm_handle_set_page_directory(struct gsp_emu_context *ctx,
+ const u8 *elem, u32 sequence,
+ u32 length)
+{
+ const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
+ sizeof(struct gsprm_rpc_header);
+ struct sema_vaspace_desc desc;
+ u64 phys = 0;
+ u32 num_entries = 0, flags = 0, h_vas = 0, ch_id = 0, subdev = 0, pasid = 0;
+
+ /*
+ * Wire layout: a 16-byte message header (hClient, hObject, ...)
+ * followed by the control's own parameters
+ * { u64 physAddress; u32 numEntries; u32 flags; u32 hVASpace; u32 chId;
+ * u32 subDeviceId; u32 pasid; }.
+ */
+ if (length >= 44) {
+ memcpy(&phys, p + 16, 8);
+ memcpy(&num_entries, p + 24, 4);
+ memcpy(&flags, p + 28, 4);
+ memcpy(&h_vas, p + 32, 4);
+ memcpy(&ch_id, p + 36, 4);
+ memcpy(&subdev, p + 40, 4);
+ memcpy(&pasid, p + 44, 4);
+ }
+
+ {
+ char dump[200];
+ int o = 0, i;
+ u32 dlen = min_t(u32, length, 80);
+
+ for (i = 0; i < (int)dlen; i += 4) {
+ u32 v = 0;
+ memcpy(&v, p + i, 4);
+ o += snprintf(dump + o, sizeof(dump) - o, "%08x ",
+ v);
+ }
+ nv_vgpu_info(ctx->vgpu,
+ "GSPRM: SET_PAGE_DIRECTORY len=%u payload: %s",
+ length, dump);
+ }
+ nv_vgpu_info(ctx->vgpu,
+ "GSPRM: SET_PAGE_DIRECTORY pdb=0x%llx entries=%u flags=0x%x hVASpace=0x%x chId=0x%x subDev=%u pasid=%u len=%u",
+ phys, num_entries, flags, h_vas, ch_id, subdev, pasid,
+ length);
+
+ if (phys && num_entries) {
+ memset(&desc, 0, sizeof(desc));
+ desc.h_vaspace = h_vas;
+ desc.pdb = phys;
+ desc.num_entries = num_entries;
+ desc.b_vidmem = (flags & 0x3) == 0;
+ gsp_sema_service_vaspace_add(&desc);
+
+ /* Show the first entries of the directory: they are the ground
+ * truth for the walk. */
+ {
+ u64 peek[4] = {0};
+ char b[128];
+ int i, o = 0, rc = -ENODEV;
+
+ if (desc.b_vidmem) {
+ u8 *base = ctx->vgpu->bars[1].vaddr;
+
+ if (base && phys + sizeof(peek) <=
+ ctx->vgpu->bars[1].size) {
+ memcpy(peek, base + phys, sizeof(peek));
+ rc = 0;
+ }
+ } else {
+ rc = gsp_rm_read_dev(ctx->vgpu, phys, peek,
+ sizeof(peek));
+ }
+ if (!rc) {
+ for (i = 0; i < 4; i++)
+ o += snprintf(b + o, sizeof(b) - o,
+ "%016llx ", peek[i]);
+ nv_vgpu_info(ctx->vgpu,
+ "GSPRM: directory[0..3]: %s", b);
+ }
+ }
+ }
+
+ if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY, sequence, 0,
+ NULL, 0))
+ nv_vgpu_err(ctx->vgpu, "GSPRM: SET_PAGE_DIRECTORY reply send failed");
+}
+
+static void gsp_rm_handle_unset_page_directory(struct gsp_emu_context *ctx,
+ const u8 *elem, u32 sequence,
+ u32 length)
+{
+ const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
+ sizeof(struct gsprm_rpc_header);
+ u32 h_vas = 0;
+
+ if (length >= 4)
+ memcpy(&h_vas, p, 4);
+
+ nv_vgpu_info(ctx->vgpu,
+ "GSPRM: UNSET_PAGE_DIRECTORY hVASpace=0x%x len=%u",
+ h_vas, length);
+ gsp_sema_service_vaspace_del(h_vas);
+
+ if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY, sequence, 0,
+ NULL, 0))
+ nv_vgpu_err(ctx->vgpu, "GSPRM: UNSET_PAGE_DIRECTORY reply send failed");
+}
+
static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
{
struct gsp_rm_emu *rm = ctx->rm_emu;
@@ -2376,6 +2640,14 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
gsp_rm_handle_free(ctx, elem, sequence);
return;
+ case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY:
+ gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length);
+ return;
+
+ case GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY:
+ gsp_rm_handle_unset_page_directory(ctx, elem, sequence, hdr->length);
+ return;
+
default:
/* Synchronous RPC: echo function+sequence with success so the
* driver's rpcRecvPoll completes. */
@@ -2499,6 +2771,7 @@ int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx)
return -ENOMEM;
rm->vgpu = vgpu;
+ mutex_init(&rm->stat_lock);
ctx->rm_emu = rm;
return 0;
}
diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h
index a82656f..1ab0a83 100644
--- a/kernel/gsp_rm_emulation.h
+++ b/kernel/gsp_rm_emulation.h
@@ -40,9 +40,13 @@ struct gsp_emu_context;
#define GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO 72
#define GSPRM_RPC_FUNC_SET_REGISTRY 73
#define GSPRM_RPC_FUNC_CONTINUATION_RECORD 71
+#define GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY 54
+#define GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY 79
#define GSPRM_RPC_FUNC_GSP_RM_CONTROL 76
#define GSPRM_RPC_FUNC_GSP_RM_ALLOC 103
#define GSPRM_RPC_FUNC_FREE 10
+#define GSPRM_RPC_EVENT_POST_EVENT 0x1003
+#define GSPRM_MAX_EVENTS 128
#define GSPRM_RPC_EVENT_GSP_INIT_DONE 0x1001
/* rpc_set_guest_system_info_v03_00: version fields the driver reads back
@@ -168,10 +172,29 @@ struct gsp_rm_emu {
u32 alloc_dumps; /* raw GSP_RM_ALLOC dumps emitted */
u32 alloc_param_dumps; /* alloc param dumps emitted */
u32 ctrl_param_dumps; /* control param dumps emitted */
+
+ /*
+ * NV01_EVENT objects (class 0x5, allocated via class 0x79 in the
+ * guest ABI). Each carries {hSrcResource, notifyIndex, data}; the
+ * driver allocates one per channel and waits for the GPU to post the
+ * event before it considers submitted work complete.
+ */
+ struct mutex stat_lock; /* serializes status-queue writers */
+ struct {
+ u32 h_src; /* source object (channel handle) */
+ u32 h_event; /* event handle */
+ u32 h_client; /* owning client */
+ u32 notify_index;
+ u32 data;
+ bool valid;
+ } events[GSPRM_MAX_EVENTS];
+ u32 events_n;
+ u64 events_posted;
};
int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx);
void gsp_rm_emu_fini(struct gsp_emu_context *ctx);
+int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src);
int gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle,
u64 start, u64 end, u32 max_hits);
int gsp_rm_scan_guest_ram_seq(struct nv_vgpu_device *vgpu, const u32 *pat,
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 2f453fa..10d6b80 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -43,6 +43,7 @@
#include "nvidia-vgpu-vfio.h"
#include "gsp_sema_service.h"
+#include "gsp_rm_emulation.h"
#define SEMA_MAP_MAX 65536
#define SEMA_MAP_SIZE_BITS 16
@@ -82,6 +83,18 @@
#define HOST_SEM_OP_RELEASE 1
#define HOST_SEM_OP_REDUCTION 6
+/*
+ * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other
+ * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared
+ * by the TURING and later classes. RELEASE (0) ends the push.
+ */
+#define HOST_M_REPORT_SEM_A 0x1b00
+#define HOST_M_REPORT_SEM_B 0x1b04
+#define HOST_M_REPORT_SEM_C 0x1b08
+#define HOST_M_REPORT_SEM_D 0x1b0c
+#define HOST_RS_OP_MASK 0x3
+#define HOST_RS_OP_RELEASE 0
+
struct sema_map_entry {
u64 gpu_page; /* GPU VA, page aligned */
u64 pa_page; /* guest PA, page aligned */
@@ -91,7 +104,7 @@ struct sema_map_entry {
/* registered channels (GPFIFO-driven execution) */
/* ------------------------------------------------------------------ */
-#define SVC_MAX_CHANNELS 24
+#define SVC_MAX_CHANNELS 128
#define SVC_GPFIFO_ENTRY_SIZE 8
#define SVC_PUSH_MAX (256 * 1024)
@@ -103,12 +116,17 @@ struct svc_channel {
bool valid;
u32 h_object;
u32 engine_type;
+ u32 cid; /* RM channel id (doorbell token vector) */
u64 gpfifo_va;
u32 gpfifo_entries;
u64 userd_pa;
u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */
+ bool userd_ok; /* userd_pa reads a plausible put/get */
+ bool logged; /* one state log per channel */
u64 instance_pa;
- u64 notifier_pa;
+ u64 notifier_pa; /* error notifier: NvNotification slots */
+ u32 notifier_as;
+ u64 notified; /* work-submit-token notifications completed */
u32 last_get; /* next GPFIFO entry we have still to run */
u64 submissions; /* GPFIFO entries executed */
};
@@ -122,6 +140,37 @@ struct svc_channel {
static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)];
static u32 svc_map_hash[SEMA_MAP_MAX * 2];
+/* VA spaces whose page directory the driver published / learned VA->PA pairs. */
+#define SVC_MAX_VASPACES 8
+#define SVC_LAT_MAX 65536
+
+/*
+ * Pinned VA->PA translations: translations a session has verified by hand
+ * (debug injection) and that must not be evicted by the learned cache, which
+ * is content-addressed and gets thrashed by the lookup stream.
+ */
+#define SVC_PIN_SLOTS 4096
+
+struct svc_pin {
+ u64 va_page;
+ u64 pa_page;
+};
+
+struct svc_vaspace {
+ bool valid;
+ u32 h_vaspace;
+ u64 pdb;
+ u32 entries; /* root entry count */
+ int ap; /* SVC_APERTURE_* of the directory itself */
+};
+
+struct svc_lat {
+ bool valid;
+ u64 va;
+ u64 addr;
+ int ap; /* SVC_APERTURE_* */
+};
+
static void svc_mark_hot(u64 pa)
{
u64 page = pa >> SEMA_PAGE_SHIFT;
@@ -133,6 +182,7 @@ static void svc_mark_hot(u64 pa)
struct sema_svc {
struct task_struct *task;
bool stop;
+ struct completion stopped;
u64 direct_base;
u32 map_n;
@@ -144,7 +194,17 @@ struct sema_svc {
spinlock_t chan_lock;
struct svc_channel chan[SVC_MAX_CHANNELS];
+ /* VA spaces whose page directory the driver published */
+ spinlock_t vas_lock;
+ struct svc_vaspace vas[SVC_MAX_VASPACES];
+ struct svc_lat lat[SVC_LAT_MAX];
+ struct svc_pin pin[SVC_PIN_SLOTS];
+ u32 pin_n;
+
/* stats */
+ u64 walks_ok;
+ u64 walks_miss;
+ u64 lat_hits;
u64 pairs_seen;
u64 pages_decoded;
u64 pushes_seen;
@@ -162,6 +222,12 @@ struct sema_svc {
u64 gpfifo_entries_done; /* entries consumed via GPFIFO */
u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */
u64 vidmem_writes; /* semaphores written in vidmem */
+ u64 gpfifo_log; /* channel-state log budget */
+ u64 gpfifo_entry_log; /* per-entry log budget */
+ u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */
+ u32 doorbell_token; /* last token written */
+ u32 doorbell_log; /* token log budget */
+ bool doorbell_pending; /* pump on next service pass */
};
static struct sema_svc svc;
@@ -249,19 +315,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
ch->valid = true;
ch->h_object = desc->h_object;
ch->engine_type = desc->engine_type;
+ ch->cid = desc->cid;
ch->gpfifo_va = desc->gpfifo_va;
ch->gpfifo_entries = desc->gpfifo_entries;
ch->userd_pa = desc->userd_pa;
ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS;
ch->instance_pa = desc->instance_pa;
ch->notifier_pa = desc->notifier_pa;
+ ch->notifier_as = desc->notifier_as ? desc->notifier_as
+ : SVC_APERTURE_SYS;
}
spin_unlock_irqrestore(&svc.chan_lock, flags);
if (ch)
- pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx\n",
+ pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
- desc->userd_pa);
+ desc->userd_pa, desc->engine_type, desc->cid,
+ desc->notifier_pa);
+ else
+ pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n",
+ desc->h_object, SVC_MAX_CHANNELS);
}
void gsp_sema_service_channel_del(u32 h_object)
@@ -285,30 +358,604 @@ void gsp_sema_service_channel_del(u32 h_object)
}
/*
+ * The driver publishes the page directory of a GPU VA space it owns (it builds
+ * the page tables itself and hands their base address to the GPU). Store it so
+ * the service can translate that space's VAs directly.
+ */
+void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
+{
+ struct svc_vaspace *vs = NULL;
+ unsigned long flags;
+ int i;
+
+ if (!desc || !desc->pdb || !desc->num_entries)
+ return;
+
+ spin_lock_irqsave(&svc.vas_lock, flags);
+ for (i = 0; i < SVC_MAX_VASPACES; i++) {
+ if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) {
+ vs = &svc.vas[i];
+ break;
+ }
+ if (!vs && !svc.vas[i].valid)
+ vs = &svc.vas[i];
+ }
+ if (vs) {
+ vs->valid = true;
+ vs->h_vaspace = desc->h_vaspace;
+ vs->pdb = desc->pdb;
+ vs->entries = desc->num_entries;
+ vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
+ /* A new directory invalidates what we learned from the old one. */
+ memset(svc.lat, 0, sizeof(svc.lat));
+ }
+ spin_unlock_irqrestore(&svc.vas_lock, flags);
+
+ if (vs)
+ pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n",
+ desc->h_vaspace, desc->pdb, desc->num_entries,
+ desc->b_vidmem ? "vidmem" : "sysmem");
+}
+
+void gsp_sema_service_vaspace_del(u32 h_vaspace)
+{
+ unsigned long flags;
+ int i;
+
+ spin_lock_irqsave(&svc.vas_lock, flags);
+ for (i = 0; i < SVC_MAX_VASPACES; i++) {
+ if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) {
+ svc.vas[i].valid = false;
+ memset(svc.lat, 0, sizeof(svc.lat));
+ pr_info("sema-svc: VA space 0x%x page directory dropped\n",
+ h_vaspace);
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&svc.vas_lock, flags);
+}
+
+/*
* The client's vidmem VAs are its window base plus the vidmem offset, and the
* backing store is the emulated BAR1 buffer. Everything else that resolves
* goes through the discovered guest-RAM map.
+ *
+ * The window base is not 0x200000000: the observed layout places the first
+ * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame
+ * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000.
+ * Verified against the guest's own view of its GPFIFO rings and USERD.
*/
-#define SVC_VIDMEM_VA_BASE 0x200000000ULL
+#define SVC_VIDMEM_VA_BASE 0x200080000ULL
+#define SVC_VIDMEM_VA_LO 0x200000000ULL /* anything in the window below
+ * the base is still vidmem, but
+ * never maps into the buffer */
static int svc_map_lookup(u64 gpu_va, u64 *pa);
-static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
- u32 *aperture)
+/* ------------------------------------------------------------------ */
+/* client page-table walk */
+/* ------------------------------------------------------------------ */
+/*
+ * The driver builds its own page tables for the GPU VA spaces it owns and
+ * publishes each top-level directory through SET_PAGE_DIRECTORY. With those
+ * directories the service can translate the client's GPU VAs directly - the
+ * authoritative answer, and independent of any heuristic discovery.
+ *
+ * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere:
+ *
+ * PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21]
+ * dual entry (16 bytes):
+ * big half -> PT_BIG [20:16]
+ * small half -> PT_4K [20:12]
+ *
+ * A directory entry carries the aperture in bits 2:1 (1 = video memory,
+ * 2 = system coherent, 3 = system non-coherent) and the child address in bits
+ * 53:8 at 4 KB granularity (bits 53:4 at 256 B granularity in the big half of
+ * a dual entry). Bit 0 of a leaf entry is VALID; bit 0 of a directory entry is
+ * IS_PTE, so a set bit means the level terminates in a page here.
+ */
+#define SVC_MMU_VA_BITS 49
+#define SVC_MMU_AP_VID 1 /* in directory entries */
+#define SVC_MMU_AP_SYS_COH 2
+#define SVC_MMU_AP_SYS_NC 3
+#define SVC_MMU_PTE_AP_VID 0 /* in page table entries */
+#define SVC_MMU_PTE_AP_PEER 1
+#define SVC_MMU_PTE_AP_SYS_COH 2
+#define SVC_MMU_PTE_AP_SYS_NC 3
+
+static u64 svc_pte_addr(u64 e, bool vidmem)
{
- if (!svc_map_lookup(va, addr)) {
- *aperture = 0; /* guest RAM */
+ if (vidmem)
+ return (e & 0x00000001FFFFFFFF00ULL);
+ return (e & 0x003FFFFFFFFF00ULL);
+}
+
+/* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */
+static int svc_pte_aperture(u32 ap)
+{
+ if (ap == SVC_MMU_PTE_AP_VID)
+ return SVC_APERTURE_FB;
+ if (ap == SVC_MMU_PTE_AP_PEER)
+ return -ENODEV;
+ return SVC_APERTURE_SYS;
+}
+
+static int svc_pde_aperture(u32 ap)
+{
+ if (ap == SVC_MMU_AP_VID)
+ return SVC_APERTURE_FB;
+ if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC)
+ return SVC_APERTURE_SYS;
+ return -ENODEV;
+}
+
+/*
+ * A translated address is only usable if it lies inside the aperture that
+ * backs it: guest RAM up to the module's RAM limit for system memory, and the
+ * emulated vidmem buffer for video memory. Rejecting out-of-range results also
+ * stops a bogus page-table chain from being mistaken for a translation.
+ */
+static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap)
+{
+ if (ap == SVC_APERTURE_FB)
+ return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size;
+ return addr + 8 <= SEMA_PA_MAX;
+}
+
+static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off,
+ int ap, u64 *out)
+{
+ if (!svc_ap_ok(vgpu, table + off, ap))
+ return -ENOENT;
+ if (svc_read_ap(vgpu, table + off,
+ ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
+ : SVC_APERTURE_SYS,
+ out, sizeof(*out)))
+ return -ENOENT;
+ return 0;
+}
+
+/*
+ * Leaf lookup under a table: read the PTE at `index`, validate it and return
+ * the backing address plus its aperture.
+ */
+static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap,
+ u32 index, u64 *addr, int *ap)
+{
+ u64 e;
+ int eap;
+
+ if (svc_read_ent(vgpu, table, 8 * index, tap, &e))
+ return -ENOENT;
+ if (!(e & 1)) /* VALID */
+ return -ENOENT;
+ eap = svc_pte_aperture((e >> 1) & 0x3);
+ if (eap < 0)
+ return -ENOENT;
+ *addr = svc_pte_addr(e, eap == SVC_APERTURE_FB);
+ if (!svc_ap_ok(vgpu, *addr, eap))
+ return -ENOENT;
+ *ap = eap;
+ return 0;
+}
+
+/*
+ * Walk the four-level page table rooted at `pdb` for one VA. Directory levels
+ * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry
+ * is the 16-byte dual entry (big half first, small half second).
+ */
+static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries,
+ u64 va, int tap, u64 *addr, int *ap, bool trace)
+{
+ u64 table = pdb;
+ u32 root_bits, idx, level;
+ u64 big, small;
+ int sap, bap;
+
+ root_bits = ilog2(entries ? entries : 1);
+ if (root_bits < 1 || root_bits > 12)
+ return -ENOENT;
+
+ if (!table)
+ return -ENOENT;
+
+ for (level = 0; level < 3; level++) {
+ u64 e;
+ u32 bits, shift;
+ int eap;
+
+ if (level == 0) {
+ bits = root_bits;
+ shift = SVC_MMU_VA_BITS - root_bits;
+ } else {
+ bits = 9;
+ shift = 29 + 9 * (2 - level);
+ }
+ idx = (u32)((va >> shift) & ((1u << bits) - 1));
+ if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) {
+ if (trace)
+ pr_info("sema-walk: L%u read fail table=0x%llx idx=%u\n",
+ level, table, idx);
+ return -ENOENT;
+ }
+ if (trace)
+ pr_info("sema-walk: L%u e=0x%016llx idx=%u\n",
+ level, e, idx);
+ /* A page at a directory level would cover at least 2^38 bytes;
+ * the driver never maps those, so treat it as a false chain. */
+ if (e & 1)
+ return -ENOENT;
+ eap = svc_pde_aperture((e >> 1) & 0x3);
+ if (eap < 0)
+ return -ENOENT;
+ tap = eap;
+ table = svc_pte_addr(e, eap == SVC_APERTURE_FB);
+ if (!svc_ap_ok(vgpu, table, eap))
+ return -ENOENT;
+ }
+
+ /* PD0: one dual entry per index (big half then small half). */
+ idx = (va >> 21) & 0xff;
+ if (svc_read_ap(vgpu, table + 16 * idx,
+ tap == SVC_APERTURE_FB ? SVC_APERTURE_FB
+ : SVC_APERTURE_SYS,
+ &big, sizeof(big)) ||
+ svc_read_ap(vgpu, table + 16 * idx + 8,
+ tap == SVC_APERTURE_FB ? SVC_APERTURE_FB
+ : SVC_APERTURE_SYS,
+ &small, sizeof(small)))
+ return -ENOENT;
+
+ /* Small half: 4 KB page table under it. */
+ sap = svc_pde_aperture(((small >> 1) & 0x3));
+ if (sap >= 0 && !(small & 1)) {
+ u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB);
+
+ if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff,
+ addr, ap)) {
+ *addr += va & 0xfff;
+ return 0;
+ }
+ }
+
+ /* Big half: a 2 MB page here, or a large (64 KB) page table. */
+ if (big & 1) {
+ bap = svc_pte_aperture((big >> 1) & 0x3);
+ if (bap >= 0) {
+ *addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) +
+ (va & ((1ULL << 21) - 1));
+ *ap = bap;
+ return 0;
+ }
+ }
+ bap = svc_pde_aperture((big >> 1) & 0x3);
+ if (bap >= 0) {
+ u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB);
+
+ if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f,
+ addr, ap)) {
+ *addr += va & 0xffff;
+ return 0;
+ }
+ }
+ return -ENOENT;
+}
+
+/*
+ * Page directories the driver published (SET_PAGE_DIRECTORY).
+ */
+static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va,
+ u64 *addr, int *ap)
+{
+ unsigned long flags;
+ int i, rc = -ENOENT;
+ bool trace = svc.walks_ok + svc.walks_miss < 24;
+
+ spin_lock_irqsave(&svc.vas_lock, flags);
+ for (i = 0; i < SVC_MAX_VASPACES; i++) {
+ int alt;
+
+ if (!svc.vas[i].valid)
+ continue;
+ if (va >> SVC_MMU_VA_BITS)
+ break;
+ if (trace)
+ pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n",
+ i, va, svc.vas[i].pdb, svc.vas[i].entries);
+ /*
+ * Try the aperture the driver named first, then the other one:
+ * the aperture encoding has changed between driver generations
+ * and a wrong guess only costs a rejected walk.
+ */
+ alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS
+ : SVC_APERTURE_FB;
+ if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va,
+ svc.vas[i].ap, addr, ap, trace) ||
+ !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va,
+ alt, addr, ap, trace)) {
+ svc.walks_ok++;
+ rc = 0;
+ break;
+ }
+ }
+ if (rc)
+ svc.walks_miss++;
+ spin_unlock_irqrestore(&svc.vas_lock, flags);
+ return rc;
+}
+
+/*
+ * Self-discovering the page directory.
+ *
+ * The published directories are not always usable (a guest can publish a
+ * directory location the host never gets to see written), so as a fallback
+ * find the directory by verification: take pairs of GPU VAs whose backing
+ * guest page is already known (the discovered VA->PA map), and accept the
+ * first guest page that, used as a top-level directory, reproduces several of
+ * those translations exactly. A wrong candidate cannot pass that test, so no
+ * layout assumption is needed to trust the result; only the entry encoding
+ * (which the walk itself applies consistently) is shared.
+ */
+static u64 svc_pd_base;
+static int svc_pd_ap;
+static u32 svc_pd_entries;
+static u8 svc_pd_tried;
+static u8 svc_pd_found;
+#define SVC_PD_SAMPLES 8
+#define SVC_PD_MIN_HITS 4
+
+static u32 svc_pd_cands;
+static u32 svc_pd_best;
+
+static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand,
+ int ap, const u64 *sva, const u64 *spa,
+ u32 nsamp)
+{
+ u32 i, hits = 0;
+
+ svc_pd_cands++;
+
+ for (i = 0; i < nsamp; i++) {
+ u64 a = 0;
+ int eap = 0;
+
+ if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) &&
+ (a & ~0xfffULL) == (spa[i] & ~0xfffULL))
+ hits++;
+ }
+ if (hits > svc_pd_best)
+ svc_pd_best = hits;
+ if (hits >= SVC_PD_MIN_HITS) {
+ svc_pd_base = cand;
+ svc_pd_ap = ap;
+ svc_pd_entries = 4;
+ svc_pd_found = 1;
+ pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n",
+ cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram",
+ hits, nsamp);
+ }
+}
+
+static void svc_pd_discover(struct nv_vgpu_device *vgpu)
+{
+ u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES];
+ u32 i, n = 0, step;
+ u64 chunk = 64 * 1024;
+ u8 *buf;
+ int pass;
+
+ if (svc_pd_tried)
+ return;
+ svc_pd_tried = 1;
+
+ if (svc.map_n < SVC_PD_SAMPLES)
+ return;
+ step = svc.map_n / SVC_PD_SAMPLES;
+ for (i = 0; i < SVC_PD_SAMPLES; i++) {
+ sva[i] = svc.map[i * step].gpu_page;
+ spa[i] = svc.map[i * step].pa_page;
+ if (sva[i] >> SVC_MMU_VA_BITS)
+ return; /* outside the walkable VA range */
+ n++;
+ }
+ if (!n)
+ return;
+
+ /* Guest RAM first, then the emulated vidmem. Page directories are page
+ * aligned, and every VA here is below 2^47, so the root entry that
+ * matters is the first one of the page. */
+ for (pass = 0; pass < 2 && !svc_pd_found; pass++) {
+ int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB;
+ u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size;
+ u8 *fb = (u8 *)vgpu->bars[1].vaddr;
+ u64 gpa;
+
+ buf = kmalloc(chunk, GFP_KERNEL);
+ if (!buf)
+ return;
+ for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) {
+ u32 len = (u32)min_t(u64, chunk, limit - gpa);
+ u64 off;
+
+ if (pass == 0) {
+ if (svc_read(vgpu, gpa, buf, len))
+ continue;
+ } else {
+ if (!fb)
+ break;
+ memcpy(buf, fb + gpa, len);
+ }
+ for (off = 0; off + 8 <= len && !svc_pd_found;
+ off += SEMA_PAGE_SIZE) {
+ u64 e;
+
+ memcpy(&e, buf + off, sizeof(e));
+ if (!(e & 1) && ((e >> 1) & 0x3))
+ svc_pd_try_candidate(vgpu, gpa + off,
+ ap, sva, spa, n);
+ }
+ }
+ kfree(buf);
+ }
+ pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n",
+ svc_pd_found ? "found" : "not found", svc_pd_base,
+ svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram",
+ svc_pd_cands, svc_pd_best, n);
+}
+
+static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va,
+ u64 *addr, int *ap)
+{
+ if (!svc_pd_found)
+ return -ENOENT;
+ return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va,
+ svc_pd_ap, addr, ap, false);
+}
+
+/* Learned translations: a walk costs a handful of guest reads, so cache the
+ * successes and answer repeats without touching the page tables again. A cache
+ * line covers a whole page, so arrays of semaphores in one page all resolve. */
+static int svc_lat_lookup(u64 va, u64 *addr, int *ap)
+{
+ struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];
+
+ if (l->valid &&
+ (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
+ (va & ~(u64)(SEMA_PAGE_SIZE - 1))) {
+ *addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) |
+ (va & (SEMA_PAGE_SIZE - 1));
+ *ap = l->ap;
+ svc.lat_hits++;
+ return 0;
+ }
+ return -ENOENT;
+}
+
+static void svc_lat_store(u64 va, u64 addr, int ap)
+{
+ struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];
+
+ l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1);
+ l->ap = ap;
+ l->valid = true;
+}
+
+static int svc_pin_lookup(u64 va, u64 *pa)
+{
+ u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ u32 mask = SVC_PIN_SLOTS - 1;
+ u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
+ u32 probe;
+
+ for (probe = 0; probe < 8; probe++) {
+ struct svc_pin *p = &svc.pin[(h + probe) & mask];
+
+ if (!p->va_page)
+ return -ENOENT;
+ if (p->va_page == page) {
+ *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1));
+ return 0;
+ }
+ }
+ return -ENOENT;
+}
+
+static void svc_pin_store(u64 va, u64 pa)
+{
+ u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ u32 mask = SVC_PIN_SLOTS - 1;
+ u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
+ u32 probe;
+
+ for (probe = 0; probe < 8; probe++) {
+ struct svc_pin *p = &svc.pin[(h + probe) & mask];
+
+ if (!p->va_page || p->va_page == page) {
+ if (!p->va_page)
+ svc.pin_n++;
+ p->va_page = page;
+ p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+ return;
+ }
+ }
+}
+
+/*
+ * Translate a GPU VA using trusted sources only: a pinned translation, the
+ * learned cache, the discovered map, the directories the driver published and
+ * the self-discovered directory. The final vidmem window convention is
+ * deliberately NOT part of this path: it answers with a vidmem offset for any
+ * VA above the window base, including sysmem-backed VAs, so callers that must
+ * not act on a guess (push execution) use this variant.
+ */
+static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
+ int *ap)
+{
+ u64 pa = 0;
+
+ if (!svc_pin_lookup(va, addr)) {
+ *ap = SVC_APERTURE_SYS;
+ return 0;
+ }
+ if (!svc_lat_lookup(va, addr, ap))
+ return 0;
+ if (!svc_map_lookup(va, &pa)) {
+ *addr = pa;
+ *ap = SVC_APERTURE_SYS;
+ svc_lat_store(va, pa, SVC_APERTURE_SYS);
+ return 0;
+ }
+ if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) {
+ *addr += va & 0xfff;
+ svc_lat_store(va, *addr, *ap);
return 0;
}
+ if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
+ *addr += va & 0xfff;
+ svc_lat_store(va, *addr, *ap);
+ return 0;
+ }
+ if (!svc_pd_tried) {
+ svc_pd_discover(vgpu);
+ if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
+ *addr += va & 0xfff;
+ svc_lat_store(va, *addr, *ap);
+ return 0;
+ }
+ }
+ return -ENOENT;
+}
+
+/*
+ * Translate a GPU VA: the trusted sources, then the vidmem window convention
+ * (a structural guess that is correct for the driver's vidmem window).
+ */
+static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap)
+{
+ if (!svc_lookup_trusted(vgpu, va, addr, ap))
+ return 0;
if (va >= SVC_VIDMEM_VA_BASE &&
va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) {
*addr = va - SVC_VIDMEM_VA_BASE;
- *aperture = 1; /* emulated vidmem */
+ *ap = SVC_APERTURE_FB;
+ svc_lat_store(va, *addr, *ap);
return 0;
}
return -ENOENT;
}
+static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
+ u32 *aperture)
+{
+ int ap = SVC_APERTURE_SYS;
+
+ if (svc_lookup(vgpu, va, addr, &ap))
+ return -ENOENT;
+ *aperture = (ap == SVC_APERTURE_FB) ? 1 : 0;
+ return 0;
+}
+
static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
{
u64 addr = 0;
@@ -316,11 +963,9 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
if (svc_resolve_va(vgpu, va, &addr, &ap))
return -ENOENT;
- if (ap) {
- memcpy(val, vgpu->bars[1].vaddr + addr, sizeof(*val));
- return 0;
- }
- return svc_read(vgpu, addr, val, sizeof(*val));
+ return svc_read_ap(vgpu, addr,
+ ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
+ val, sizeof(*val));
}
static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val)
@@ -330,12 +975,11 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val)
if (svc_resolve_va(vgpu, va, &addr, &ap))
return -ENOENT;
- if (ap) {
- memcpy(vgpu->bars[1].vaddr + addr, &val, sizeof(val));
+ if (ap)
svc.vidmem_writes++;
- return 0;
- }
- return svc_write(vgpu, addr, &val, sizeof(val));
+ return svc_write_ap(vgpu, addr,
+ ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
+ &val, sizeof(val));
}
/* ------------------------------------------------------------------ */
@@ -357,7 +1001,7 @@ struct base_candidate {
* vas[CPU] followed by vas[GPU]), collected raw for pass 2.
*/
#define PAIR_MAX 32768
-struct raw_pair { u64 cpu_va; u64 gpu_va; };
+struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; };
static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
struct base_candidate *cands, u32 *ncand,
@@ -404,8 +1048,32 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
b >= SEMA_VA_LO && b < SEMA_VA_HI &&
(a & 0xfff) == 0 && (b & 0xfff) == 0 &&
*npairs < PAIR_MAX) {
+ u8 ok = 0;
+
+ /*
+ * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU]
+ * (offset 48) directly before vas[UVM_ID_GPU0]
+ * (offset 56), so a pair taken from a live rm_mem
+ * is preceded by the header {u32 type; bitmap
+ * mapped_on}. Validate it when it fits in the same
+ * page: freed allocations and unrelated tables
+ * that happen to hold KVA/VA pairs do not match,
+ * and a stale entry for the same VA must not
+ * shadow the live one.
+ */
+ if (off >= 48) {
+ u32 type;
+ u64 mapped_on;
+
+ memcpy(&type, buf + off - 48, 4);
+ memcpy(&mapped_on, buf + off - 40, 8);
+ if ((type == 0 || type == 1) &&
+ (mapped_on & 0x3) == 0x3)
+ ok = 1;
+ }
pairs[*npairs].cpu_va = a;
pairs[*npairs].gpu_va = b;
+ pairs[*npairs].valid_hdr = ok;
(*npairs)++;
}
}
@@ -424,6 +1092,22 @@ static void svc_add_map(u64 gpu_va, u64 pa)
}
}
+/* Add one scanned {CPU KVA, GPU VA} pair to the map. */
+static void svc_add_pair(u64 base, const struct raw_pair *p)
+{
+ u64 pa;
+
+ if (p->cpu_va < base)
+ return;
+ pa = p->cpu_va - base;
+ if (pa >= 0x140000000ULL || (pa & 0xfff))
+ return;
+ if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI)
+ return;
+ svc_add_map(p->gpu_va, pa);
+ svc.pairs_seen++;
+}
+
static int svc_map_cmp(const void *a, const void *b)
{
const struct sema_map_entry *x = a, *y = b;
@@ -495,6 +1179,28 @@ static int svc_map_lookup(u64 gpu_va, u64 *pa)
return -ENOENT;
}
+/* Binary search over a sorted sub-range (used while the tail is unsorted). */
+static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa)
+{
+ u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ int lo = (int)from, hi = (int)to - 1;
+
+ while (lo <= hi) {
+ int mid = (lo + hi) / 2;
+
+ if (svc.map[mid].gpu_page == gpu_page) {
+ *pa = svc.map[mid].pa_page +
+ (gpu_va & (SEMA_PAGE_SIZE - 1));
+ return 0;
+ }
+ if (svc.map[mid].gpu_page < gpu_page)
+ lo = mid + 1;
+ else
+ hi = mid - 1;
+ }
+ return -ENOENT;
+}
+
static void svc_rebuild(struct nv_vgpu_device *vgpu)
{
static u8 *buf;
@@ -521,32 +1227,47 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
}
if (best_base && best_count >= 2) {
+ u32 n_valid;
+
svc.direct_base = best_base;
+
+ /*
+ * Live rm_mem-style records first (dedup is first-wins), then
+ * headerless candidates only for VAs the validated set does not
+ * cover - a freed allocation or an unrelated table holding a
+ * stale pair for the same VA must not shadow the live record.
+ */
+ for (i = 0; i < npairs; i++) {
+ if (pairs[i].valid_hdr)
+ svc_add_pair(best_base, &pairs[i]);
+ }
+ n_valid = svc.map_n;
+ sort(svc.map, n_valid, sizeof(svc.map[0]), svc_map_cmp, NULL);
for (i = 0; i < npairs; i++) {
- u64 cpu = pairs[i].cpu_va;
- u64 pa;
+ u64 dummy;
- if (cpu < best_base)
- continue;
- pa = cpu - best_base;
- if (pa >= 0x140000000ULL || (pa & 0xfff))
+ if (pairs[i].valid_hdr)
continue;
- if (pairs[i].gpu_va < SEMA_VA_LO ||
- pairs[i].gpu_va >= SEMA_VA_HI)
+ if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid,
+ &dummy))
continue;
- svc_add_map(pairs[i].gpu_va, pa);
- svc.pairs_seen++;
+ svc_add_pair(best_base, &pairs[i]);
}
}
+ /* Stale entries from a previous rebuild (wrong PA for a reused VA)
+ * must not outlive the map they were derived from. */
+ memset(svc.lat, 0, sizeof(svc.lat));
+
svc_map_finalize();
svc.builds++;
svc.last_build = jiffies;
- pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu\n",
+ pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu\n",
svc.builds, best_base, best_count, npairs, svc.map_n,
svc.releases_seen, svc.releases_applied, svc.releases_missed,
- svc.releases_stale, svc.releases_jumped);
+ svc.releases_stale, svc.releases_jumped,
+ svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits);
pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n",
svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
@@ -631,6 +1352,8 @@ struct svc_push_state {
u32 have_remap;
u32 release_dma;
u32 sem_inc;
+ u32 rs_upper, rs_lower, rs_payload;
+ u32 have_rs_addr, have_rs_payload;
};
/* ------------------------------------------------------------------ */
@@ -925,11 +1648,73 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
continue;
}
- if (subch != 4) {
+ /*
+ * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of
+ * four words {offset_upper, offset_lower, payload, operation}
+ * or the same words written one at a time. A RELEASE operation
+ * ends the push: run the collected work, then write the
+ * payload to the semaphore (forward progress only, so recycled
+ * buffers stay harmless).
+ */
+ if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) {
+ u32 vv[4] = { v0, v1, v2, v3 };
+ u32 k, n = count > 4 ? 4 : count;
+
+ for (k = 0; k < n; k++) {
+ switch (addr + 4 * k) {
+ case HOST_M_REPORT_SEM_A:
+ st.rs_upper = vv[k] & 0xff;
+ st.have_rs_addr = 1;
+ break;
+ case HOST_M_REPORT_SEM_B:
+ st.rs_lower = vv[k];
+ st.have_rs_addr = 1;
+ break;
+ case HOST_M_REPORT_SEM_C:
+ st.rs_payload = vv[k];
+ st.have_rs_payload = 1;
+ break;
+ case HOST_M_REPORT_SEM_D: {
+ u64 va;
+ u32 cur = 0;
+
+ if ((vv[k] & HOST_RS_OP_MASK) !=
+ HOST_RS_OP_RELEASE ||
+ !st.have_rs_addr ||
+ !st.have_rs_payload ||
+ !st.rs_payload)
+ break;
+ va = ((u64)st.rs_upper << 32) |
+ st.rs_lower;
+ if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
+ break;
+ if (page_pa)
+ svc_mark_hot(page_pa);
+ svc.pushes_seen++;
+ if (svc_sem_read(vgpu, va, &cur))
+ break;
+ if (st.rs_payload <= cur)
+ break;
+ svc.pushes_pending++;
+ svc_run_ops(vgpu, ops, n_ops);
+ n_ops = 0;
+ if (!svc_sem_write(vgpu, va,
+ st.rs_payload))
+ svc.releases_applied++;
+ break;
+ }
+ default:
+ break;
+ }
+ }
off += 4 + count * 4;
continue;
}
+ if (subch != 4) {
+ off += 4 + count * 4;
+ continue;
+ }
switch (addr) {
case CE_M_SET_SEMAPHORE_A:
st.sem_va_hi = v0;
@@ -1253,11 +2038,181 @@ static void svc_apply_mapped(struct nv_vgpu_device *vgpu)
static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf,
size_t len)
{
- u64 pa;
+ u8 *p = buf;
+
+ while (len) {
+ u64 pa = 0;
+ size_t chunk = min_t(size_t, len,
+ SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
+ int ap = SVC_APERTURE_SYS;
+
+ if (svc_lookup(vgpu, va, &pa, &ap))
+ return -ENOENT;
+ if (svc_read_ap(vgpu, pa,
+ ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
+ : SVC_APERTURE_SYS,
+ p, chunk))
+ return -EIO;
+ p += chunk;
+ va += chunk;
+ len -= chunk;
+ }
+ return 0;
+}
+
+/*
+ * Read `len` bytes at a GPU VA through a trusted translation only. Push text
+ * must never be fetched through the vidmem window convention: a sysmem-backed
+ * pushbuffer VA above the window base answers with a vidmem offset and the
+ * caller would decode zeros and mark work done that never ran.
+ */
+static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+ size_t len)
+{
+ u8 *p = buf;
+
+ while (len) {
+ u64 pa = 0;
+ size_t chunk = min_t(size_t, len,
+ SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
+ int ap = SVC_APERTURE_SYS;
- if (svc_map_lookup(va, &pa))
+ if (svc_lookup_trusted(vgpu, va, &pa, &ap))
+ return -ENOENT;
+ if (svc_read_ap(vgpu, pa,
+ ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
+ : SVC_APERTURE_SYS,
+ p, chunk))
+ return -EIO;
+ p += chunk;
+ va += chunk;
+ len -= chunk;
+ }
+ return 0;
+}
+
+/*
+ * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the
+ * values are plausible for a channel with `entries` GPFIFO slots.
+ */
+static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as,
+ u32 entries, u32 *put, u32 *get)
+{
+ u32 p = 0, g = 0;
+
+ if (!pa)
+ return -EINVAL;
+ if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p)))
+ return -ENOENT;
+ if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g)))
return -ENOENT;
- return svc_read(vgpu, pa, buf, len);
+ if (p > entries || g > p)
+ return -EINVAL;
+ if (!p && !g)
+ return -EAGAIN; /* plausible but idle: keep looking */
+ *put = p;
+ *get = g;
+ return 0;
+}
+
+/*
+ * Locate a channel's USERD. The allocation parameters carry an offset that is
+ * relative to a client pool the emulation cannot see, so it is only one
+ * candidate: for the combined GPFIFO+USERD allocations the control block sits
+ * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout).
+ * Whichever candidate reads a coherent get/put pair wins and is cached. An
+ * idle candidate (0/0) is not accepted until it shows activity, so a zeroed
+ * page is never mistaken for the control block.
+ */
+static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+ u64 ring_pa, int ring_ap, u32 *put, u32 *get)
+{
+ u64 cand[4];
+ u32 as[4], nc = 0, i;
+
+ if (ring_ap == SVC_APERTURE_FB && ring_pa) {
+ cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB;
+ cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB;
+ }
+ cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as;
+
+ for (i = 0; i < nc; i++) {
+ if (!cand[i])
+ continue;
+ if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries,
+ put, get)) {
+ ch->userd_pa = cand[i];
+ ch->userd_as = as[i];
+ ch->userd_ok = true;
+ return 0;
+ }
+ }
+ return -ENOENT;
+}
+
+/*
+ * Complete a channel's work-submission notification the way the GPU does.
+ *
+ * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot
+ * in-progress (status 0xffff, info32 = work-submit token) and then polls the
+ * notification memory for it to come back cleared - the GPU clearing it is
+ * the acknowledgement that the submission was picked up. The emulated GPU
+ * never did that, so libcuda's context setup spun forever. Write the cleared
+ * notification when the slot is busy.
+ *
+ * NvNotification layout (16 bytes): timestamp[2], info32, info16, status.
+ */
+#define SVC_NOTIFY_SIZE 16
+#define SVC_NOTIFY_STATUS_OFF 14
+#define SVC_NOTIFY_INFO32_OFF 8
+#define SVC_NOTIFY_TOKEN_SLOT 1 /* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */
+
+static void svc_notify_complete(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch)
+{
+ u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0};
+ u64 base = ch->notifier_pa;
+ u32 status = 0, info32 = 0;
+ int slot, busy = 0;
+
+ if (!ch->notifier_pa)
+ return;
+
+ /* Both the error slot (0) and the work-submit-token slot (1) must read
+ * as completed; the client polls one of them per submission. */
+ for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) {
+ u64 pa = base + slot * SVC_NOTIFY_SIZE;
+
+ status = 0;
+ info32 = 0;
+ if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF,
+ ch->notifier_as, &status, 2) ||
+ svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF,
+ ch->notifier_as, &info32, 4))
+ continue;
+ if (status || info32)
+ busy = 1;
+ }
+ if (!busy)
+ return;
+
+ /*
+ * Clear both slots: they are adjacent, and whichever one the client is
+ * polling must read as completed for a healthy channel.
+ */
+ if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) {
+ ch->notified++;
+ if (ch->notified <= 4 || (ch->notified % 10000) == 0)
+ pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n",
+ ch->h_object, info32, status, ch->notified);
+
+ /*
+ * The GPU consumed the submission: post the channel's event so
+ * the client's CPU-side RM signals the OS event the submitting
+ * thread waits on (a real GPU would raise an interrupt here).
+ */
+ gsp_rm_post_channel_event(vgpu, ch->h_object);
+ }
}
/*
@@ -1284,18 +2239,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
for (i = 0; i < n; i++) {
struct svc_channel *ch = &local[i];
+ u64 ring_pa = 0;
+ int ring_ap = SVC_APERTURE_SYS;
+ int rrc;
u32 put = 0, get = 0, guard = 0;
- if (!ch->userd_pa)
- continue;
+ svc_notify_complete(vgpu, ch);
- if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT,
- ch->userd_as, &put, sizeof(put))) {
- svc.gpfifo_nomap++;
+ if (!ch->gpfifo_va || !ch->gpfifo_entries)
continue;
+
+ rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap);
+
+ if (!ch->userd_ok) {
+ if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
+ rrc ? -1 : ring_ap, &put, &get))
+ continue;
+ } else {
+ if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT,
+ ch->userd_as, &put, sizeof(put)))
+ continue;
+ (void)svc_read_ap(vgpu,
+ ch->userd_pa + SVC_USERD_GP_GET,
+ ch->userd_as, &get, sizeof(get));
+ }
+
+ if (!ch->logged) {
+ pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n",
+ ch->h_object, ch->engine_type, ch->cid,
+ ch->gpfifo_va,
+ ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
+ put, get, rrc, ring_pa, ring_ap);
+ ch->logged = true;
}
- (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET,
- ch->userd_as, &get, sizeof(get));
if (put >= ch->gpfifo_entries)
put %= ch->gpfifo_entries;
@@ -1305,6 +2281,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
u64 va, idx = ch->last_get;
u32 len, e0, e1, g;
+ if (rrc)
+ break; /* cannot read the ring at all */
+
if (svc_read_va(vgpu,
ch->gpfifo_va +
idx * SVC_GPFIFO_ENTRY_SIZE,
@@ -1320,17 +2299,29 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
/* GP_ENTRY1.LENGTH[30:10], in dwords */
len = ((e1 >> 10) & 0x1fffff) << 2;
+ if (svc.gpfifo_entry_log < 32) {
+ pr_info("sema-chan: entry[%llu] raw=%08x%08x va=0x%llx len=%u\n",
+ idx, e1, e0, va, len);
+ svc.gpfifo_entry_log++;
+ }
+
if (va && len && len <= SVC_PUSH_MAX) {
u8 *pb = kmalloc(len, GFP_KERNEL);
if (pb) {
- if (!svc_read_va(vgpu, va, pb, len)) {
+ if (!svc_read_va_trusted(vgpu, va, pb,
+ len)) {
svc_decode_stream(vgpu, pb,
len, 0);
ch->submissions++;
svc.gpfifo_entries_done++;
} else {
+ /* Keep the entry pending: the
+ * mapping may resolve once the
+ * page tables are filled. */
svc.gpfifo_nomap++;
+ kfree(pb);
+ break;
}
kfree(pb);
}
@@ -1348,6 +2339,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
for (k = 0; k < SVC_MAX_CHANNELS; k++) {
if (svc.chan[k].valid &&
svc.chan[k].h_object == ch->h_object) {
+ svc.chan[k].userd_pa = ch->userd_pa;
+ svc.chan[k].userd_as = ch->userd_as;
+ svc.chan[k].userd_ok = ch->userd_ok;
+ svc.chan[k].logged = ch->logged;
+ svc.chan[k].notified = ch->notified;
svc.chan[k].last_get = ch->last_get;
svc.chan[k].submissions = ch->submissions;
break;
@@ -1383,19 +2379,30 @@ static void svc_handle_probe(struct nv_vgpu_device *vgpu)
probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
svc.direct_base, svc.map_n);
- if (svc_map_lookup(probe_va, &pa)) {
- pr_info("sema-probe: NO MAP ENTRY for page 0x%llx\n",
- probe_va & ~(u64)(SEMA_PAGE_SIZE - 1));
- } else {
- pr_info("sema-probe: pa=0x%llx\n", pa);
- for (i = 0; i < probe_n; i++) {
- u32 v = 0;
+ {
+ int ap = SVC_APERTURE_SYS;
- if (svc_read(vgpu, pa + 4ULL * i, &v, sizeof(v))) {
- pr_info("sema-probe: [%u] read failed\n", i);
- break;
+ if (svc_lookup(vgpu, probe_va, &pa, &ap)) {
+ pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n",
+ probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+ svc.walks_ok, svc.walks_ok + svc.walks_miss);
+ } else {
+ pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa,
+ ap == SVC_APERTURE_FB ? "vidmem" : "ram");
+ for (i = 0; i < probe_n; i++) {
+ u32 v = 0;
+
+ if (svc_read_ap(vgpu, pa + 4ULL * i,
+ ap == SVC_APERTURE_FB ?
+ SVC_APERTURE_FB :
+ SVC_APERTURE_SYS,
+ &v, sizeof(v))) {
+ pr_info("sema-probe: [%u] read failed\n",
+ i);
+ break;
+ }
+ pr_info("sema-probe: [%u] 0x%08x\n", i, v);
}
- pr_info("sema-probe: [%u] 0x%08x\n", i, v);
}
}
@@ -1422,6 +2429,59 @@ int gsp_sema_service_probe(u64 gpu_va, u32 count)
return 0;
}
+/*
+ * Debug injection: teach the service one {GPU VA -> guest PA} translation.
+ * Some client mappings (user VA == GPU VA, sysmem backed) are not
+ * discoverable from the structures the service scans; this lets a session
+ * feed a pair that was verified from the guest and watch push execution use
+ * it.
+ */
+static struct completion map_add_done;
+static u64 map_add_va;
+static u64 map_add_pa;
+static u64 map_add_len;
+static bool map_add_pending;
+
+static void svc_handle_map_add(void)
+{
+ u64 off;
+
+ if (!map_add_pending)
+ return;
+
+ for (off = 0; off < (map_add_len ? map_add_len : 1);
+ off += SEMA_PAGE_SIZE) {
+ svc_pin_store(map_add_va + off, map_add_pa + off);
+ svc_lat_store(map_add_va + off, map_add_pa + off,
+ SVC_APERTURE_SYS);
+ }
+ pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (sysmem)\n",
+ map_add_va, map_add_pa, map_add_len);
+ map_add_pending = false;
+ complete(&map_add_done);
+}
+
+int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len)
+{
+ if (!svc.task)
+ return -ENODEV;
+
+ if (len > (64ULL << 20))
+ return -EINVAL;
+
+ init_completion(&map_add_done);
+ map_add_va = gpu_va;
+ map_add_pa = pa;
+ map_add_len = len;
+ map_add_pending = true;
+
+ if (!wait_for_completion_timeout(&map_add_done, 3 * HZ)) {
+ map_add_pending = false;
+ return -ETIMEDOUT;
+ }
+ return 0;
+}
+
static int svc_thread(void *arg)
{
struct nv_vgpu_device *vgpu;
@@ -1449,6 +2509,7 @@ static int svc_thread(void *arg)
}
svc_handle_probe(vgpu);
+ svc_handle_map_add();
if (!svc.direct_base ||
time_after(jiffies, svc.last_build + 10 * HZ))
@@ -1458,8 +2519,10 @@ static int svc_thread(void *arg)
* Channel-driven execution: notice submissions as they are made
* and run the pushed work immediately, so the guest's UVM
* progress waits complete in milliseconds rather than in the
- * time it takes the fallback scan to find the push.
+ * time it takes the fallback scan to find the push. A doorbell
+ * token means a channel was just scheduled; run everything now.
*/
+ svc.doorbell_pending = false;
svc_pump_channels(vgpu);
/*
@@ -1484,15 +2547,39 @@ static int svc_thread(void *arg)
pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n",
svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n);
+ complete(&svc.stopped);
return 0;
}
+/*
+ * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0
+ * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has
+ * written the puller GPFIFO entry, so a pending token means "run this
+ * channel's entries now". Called from the VFIO write path (atomic context):
+ * record the token and let the service thread do the work.
+ */
+void gsp_sema_service_doorbell(u32 token)
+{
+ svc.doorbell_token = token;
+ svc.doorbell_count++;
+ svc.doorbell_pending = true;
+
+ if (svc.doorbell_log < 64) {
+ pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n",
+ token, (token >> 16) & 0x7f, token & 0xfff,
+ svc.doorbell_count);
+ svc.doorbell_log++;
+ }
+}
+
int gsp_sema_service_start(void)
{
if (svc.task)
return 0;
svc.stop = false;
+ init_completion(&svc.stopped);
spin_lock_init(&svc.chan_lock);
+ spin_lock_init(&svc.vas_lock);
svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
if (IS_ERR(svc.task)) {
int ret = PTR_ERR(svc.task);
@@ -1503,11 +2590,18 @@ int gsp_sema_service_start(void)
return 0;
}
+/*
+ * Stop the service thread without kthread_stop(): the thread can also end on
+ * its own if it ever faults, and joining a task that is already gone faults in
+ * the caller (which wedges the whole module). The completion tells us the
+ * thread function has returned, whatever ended it.
+ */
void gsp_sema_service_stop(void)
{
- if (svc.task) {
- svc.stop = true;
- kthread_stop(svc.task);
- svc.task = NULL;
- }
+ if (!svc.task)
+ return;
+ svc.stop = true;
+ if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ))
+ pr_err("sema-svc: thread did not stop within 5s\n");
+ svc.task = NULL;
}
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 16a045f..a6893bd 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -22,19 +22,42 @@
struct sema_channel_desc {
u32 h_object;
u32 engine_type;
+ u32 cid; /* RM channel id (the doorbell token vector) */
u64 gpfifo_va;
u32 gpfifo_entries;
u64 userd_pa; /* NvA06FControl: GPGet @0x88, GPPut @0x8c */
u32 userd_as; /* NV_ADDR_SYSMEM(1) / NV_ADDR_FBMEM(2) */
u64 instance_pa;
u64 notifier_pa;
+ u32 notifier_as; /* NV_ADDR_SYSMEM(1) / NV_ADDR_FBMEM(2) */
};
void gsp_sema_service_channel_add(const struct sema_channel_desc *desc);
void gsp_sema_service_channel_del(u32 h_object);
+/*
+ * A GPU VA space whose top-level page directory the driver has published (the
+ * driver builds its own page tables for the address spaces it owns, then hands
+ * the directory's physical address to the GPU through SET_PAGE_DIRECTORY).
+ *
+ * pdb is the directory base in the aperture selected by b_vidmem (false =
+ * guest RAM, true = emulated vidmem); num_entries is the root entry count and
+ * fixes the number of VA bits the root covers.
+ */
+struct sema_vaspace_desc {
+ u32 h_vaspace;
+ u64 pdb;
+ u32 num_entries;
+ bool b_vidmem;
+};
+
+void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc);
+void gsp_sema_service_vaspace_del(u32 h_vaspace);
+
int gsp_sema_service_start(void);
void gsp_sema_service_stop(void);
int gsp_sema_service_probe(u64 gpu_va, u32 count);
+int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len);
+void gsp_sema_service_doorbell(u32 token);
#endif /* _GSP_SEMA_SERVICE_H_ */
diff --git a/kernel/main.c b/kernel/main.c
index 84a05ca..05a49b1 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -791,12 +791,34 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use
if (pos >= 0 && (u64)pos < vgpu->bars[0].size && count <= 8) {
u8 tmp[8];
ssize_t ret;
+ static u32 bar0_log;
if (!vgpu->bar_emul_ctx)
return -ENODEV;
if (copy_from_user(tmp, buf, count))
return -EFAULT;
+ if (bar0_log < 64 || (u64)pos == 0x88704) {
+ static u32 db_log;
+ u64 v = 0;
+
+ memcpy(&v, tmp, count);
+
+ if ((u64)pos == 0x88704) {
+ if (db_log < 512) {
+ pr_info("gsp-resman-vfio: DOORBELL write pos=0x%llx size=%zu val=0x%llx\n",
+ (u64)pos, count, v);
+ db_log++;
+ }
+ if (count == 4)
+ gsp_sema_service_doorbell((u32)v);
+ } else {
+ pr_info("gsp-resman-vfio: BAR0 write pos=0x%llx size=%zu val=0x%llx\n",
+ (u64)pos, count, v);
+ bar0_log++;
+ }
+ }
+
ret = nv_vgpu_bar_emul_write(vgpu, 0, (u64)pos, tmp, count);
if (ret)
return ret;
@@ -809,6 +831,17 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use
(u64)pos < (1ULL << 40) + vgpu->bars[1].size &&
vgpu->bars[1].vaddr && count <= 8) {
u64 bar1_off = (u64)pos - (1ULL << 40);
+ static u32 bar1_log;
+
+ if (bar1_log < 64) {
+ u64 v = 0;
+
+ memcpy(&v, buf, count);
+
+ pr_info("gsp-resman-vfio: BAR1 write off=0x%llx size=%zu val=0x%llx\n",
+ bar1_off, count, v);
+ bar1_log++;
+ }
if (copy_from_user(vgpu->bars[1].vaddr + bar1_off, buf, count))
return -EFAULT;
@@ -2215,6 +2248,41 @@ module_param_cb(sema_probe, &nv_vgpu_sema_probe_ops, NULL, 0200);
MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA");
/*
+ * Debug: inject a verified {GPU VA -> guest PA} translation into the
+ * semaphore service (used when the client's mapping cannot be discovered
+ * from guest structures):
+ * echo 0x<va>,0x<pa> > .../parameters/sema_map_add
+ */
+static int nv_vgpu_sema_map_add_set(const char *val, const struct kernel_param *kp)
+{
+ char buf[64];
+ char *comma, *comma2;
+ u64 va = 0, pa = 0, len = 0;
+
+ strscpy(buf, val, sizeof(buf));
+ comma = strchr(buf, ',');
+ if (!comma)
+ return -EINVAL;
+ *comma = '\0';
+ comma2 = strchr(comma + 1, ',');
+ if (comma2)
+ *comma2 = '\0';
+ if (kstrtoull(buf, 0, &va) || kstrtoull(comma + 1, 0, &pa))
+ return -EINVAL;
+ if (comma2 && kstrtoull(comma2 + 1, 0, &len))
+ return -EINVAL;
+
+ return gsp_sema_service_add_map(va, pa, len);
+}
+
+static const struct kernel_param_ops nv_vgpu_sema_map_add_ops = {
+ .set = nv_vgpu_sema_map_add_set,
+};
+
+module_param_cb(sema_map_add, &nv_vgpu_sema_map_add_ops, NULL, 0200);
+MODULE_PARM_DESC(sema_map_add, "Inject a GPU VA -> guest PA mapping (hex va,hex pa)");
+
+/*
* Debug: find page-table entries pointing at a physical address, in guest RAM
* and emulated vidmem:
* echo 0x12442b000 > .../parameters/scan_pt
@@ -2628,6 +2696,156 @@ module_param_cb(dump_vidmem, &nv_vgpu_dump_vidmem_ops, NULL, 0200);
MODULE_PARM_DESC(dump_vidmem, "Dump emulated vidmem range (hex offset,hex len)");
/*
+ * Debug: hex-dump or scan the BAR0 register shadow. Guest MMIO writes that go
+ * through the mapped shadow never reach the write callback, so this is the
+ * only way to observe registers the guest writes directly (e.g. the
+ * work-submission doorbell/notify registers, client region).
+ * echo 0x88700,0x100 > .../parameters/dump_bar0
+ * echo 0x00040022 > .../parameters/scan_bar0
+ */
+static int nv_vgpu_dump_bar0_set(const char *val, const struct kernel_param *kp)
+{
+ struct nv_vgpu_device *vgpu;
+ struct nv_vgpu_bar_emul_ctx *ctx;
+ unsigned long off = 0, len = 0;
+ char buf[64];
+ char *comma;
+
+ strscpy(buf, val, sizeof(buf));
+ comma = strchr(buf, ',');
+ if (comma)
+ *comma = '\0';
+ if (kstrtoul(buf, 0, &off))
+ return -EINVAL;
+ if (comma && kstrtoul(comma + 1, 0, &len))
+ return -EINVAL;
+ if (!len)
+ len = 0x100;
+ if (len > 0x1000)
+ len = 0x1000;
+
+ mutex_lock(&nv_vgpu_driver.lock);
+ list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
+ u8 *base;
+ unsigned long o;
+
+ ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
+ if (!ctx || !ctx->bar0_shadow)
+ continue;
+ base = (u8 *)ctx->bar0_shadow;
+ if (off + len > ctx->bar0_shadow_size)
+ continue;
+ for (o = 0; o < len; o += 64) {
+ u32 n = min_t(unsigned long, len - o, 64);
+
+ pr_info("gsp-resman-vfio: bar0[0x%lx..0x%lx]=%*phN\n",
+ off + o, off + o + n, (int)n, base + off + o);
+ }
+ }
+ mutex_unlock(&nv_vgpu_driver.lock);
+ return 0;
+}
+
+static const struct kernel_param_ops nv_vgpu_dump_bar0_ops = {
+ .set = nv_vgpu_dump_bar0_set,
+};
+
+module_param_cb(dump_bar0, &nv_vgpu_dump_bar0_ops, NULL, 0200);
+MODULE_PARM_DESC(dump_bar0, "Dump BAR0 shadow range (hex offset,hex len)");
+
+static int nv_vgpu_scan_bar0_set(const char *val, const struct kernel_param *kp)
+{
+ struct nv_vgpu_device *vgpu;
+ struct nv_vgpu_bar_emul_ctx *ctx;
+ unsigned long needle = 0;
+ u32 hits = 0;
+
+ if (kstrtoul(val, 0, &needle))
+ return -EINVAL;
+
+ mutex_lock(&nv_vgpu_driver.lock);
+ list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
+ u8 *base;
+ size_t size, o;
+
+ ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
+ if (!ctx || !ctx->bar0_shadow)
+ continue;
+ base = (u8 *)ctx->bar0_shadow;
+ size = ctx->bar0_shadow_size;
+ for (o = 0; o + 4 <= size && hits < 64; o += 4) {
+ u32 v;
+
+ memcpy(&v, base + o, 4);
+ if (v == (u32)needle) {
+ pr_info("gsp-resman-vfio: bar0 scan: 0x%x at off 0x%zx\n",
+ v, o);
+ hits++;
+ }
+ }
+ pr_info("gsp-resman-vfio: bar0 scan: needle=0x%lx hits=%u\n",
+ needle, hits);
+ }
+ mutex_unlock(&nv_vgpu_driver.lock);
+ return 0;
+}
+
+static const struct kernel_param_ops nv_vgpu_scan_bar0_ops = {
+ .set = nv_vgpu_scan_bar0_set,
+};
+
+module_param_cb(scan_bar0, &nv_vgpu_scan_bar0_ops, NULL, 0200);
+MODULE_PARM_DESC(scan_bar0, "Scan BAR0 shadow for a 32-bit value (hex)");
+
+/*
+ * Debug: find channel USERD blocks in the emulated vidmem. A USERD page has a
+ * GPGet at +0x88 and a GPPut at +0x8c; report every page whose GPPut is a
+ * plausible entry index (1..65535) with GPGet <= GPPut. This is how the real
+ * vidmem offset of a channel's control block is discovered when the offset
+ * carried in the allocation parameters is relative to a pool the emulation
+ * cannot see.
+ */
+static int nv_vgpu_find_userd_set(const char *val, const struct kernel_param *kp)
+{
+ struct nv_vgpu_device *vgpu;
+ int shown = 0;
+
+ mutex_lock(&nv_vgpu_driver.lock);
+ list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
+ u8 *base = vgpu->bars[1].vaddr;
+ size_t size = vgpu->bars[1].size;
+ size_t off;
+
+ if (!base || !size)
+ continue;
+ for (off = 0; off + 0x90 <= size; off += 4) {
+ u32 get, put;
+
+ memcpy(&put, base + off + 0x8c, 4);
+ if (put < 1 || put > 65535)
+ continue;
+ memcpy(&get, base + off + 0x88, 4);
+ if (get > put)
+ continue;
+ if (shown < 32)
+ pr_info("gsp-resman-vfio: userd cand off=0x%zx get=%u put=%u\n",
+ off, get, put);
+ shown++;
+ }
+ pr_info("gsp-resman-vfio: userd scan: candidates=%d\n", shown);
+ }
+ mutex_unlock(&nv_vgpu_driver.lock);
+ return 0;
+}
+
+static const struct kernel_param_ops nv_vgpu_find_userd_ops = {
+ .set = nv_vgpu_find_userd_set,
+};
+
+module_param_cb(find_userd, &nv_vgpu_find_userd_ops, NULL, 0200);
+MODULE_PARM_DESC(find_userd, "Find channel USERD blocks in emulated vidmem");
+
+/*
* Debug: write a u32 into the emulated vidmem (BAR1 backing) at an offset:
* echo 0x<offset>,0x<value> > .../parameters/poke_vidmem
*/
@@ -2754,5 +2972,56 @@ static const struct kernel_param_ops nv_vgpu_dump_guest_ops = {
module_param_cb(dump_guest, &nv_vgpu_dump_guest_ops, NULL, 0200);
MODULE_PARM_DESC(dump_guest, "Hex-dump a guest physical range (hex gpa,hex len)");
+/*
+ * Debug: deliver one MSI-X vector to the guest. QEMU wires each vector's
+ * eventfd via VFIO_DEVICE_SET_IRQS; signaling it makes QEMU raise the guest
+ * interrupt. The emulated GPU consumes work without raising the interrupts a
+ * real GPU would, so sessions use this to drive the guest's interrupt-driven
+ * progress paths (channel/runlist events) and find out which vector an event
+ * needs.
+ */
+static int nv_vgpu_inject_irq_set(const char *val, const struct kernel_param *kp)
+{
+ struct nv_vgpu_device *vgpu;
+ unsigned long idx = 0;
+ int ret = -ENODEV;
+
+ if (kstrtoul(val, 0, &idx) || idx >= NV_VGPU_MAX_IRQS)
+ return -EINVAL;
+
+ mutex_lock(&nv_vgpu_driver.lock);
+ list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
+ struct eventfd_ctx *trigger;
+
+ if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+ continue;
+
+ if (vgpu->bar_emul_ctx && idx < NV_VGPU_MAX_IRQS) {
+ u32 bit = 1u << (idx % 32);
+
+ nv_vgpu_bar_emul_write(vgpu, 0,
+ GSP_EMU_MSIX_PBA_OFFSET +
+ (idx / 32) * 4,
+ &bit, sizeof(bit));
+ }
+
+ trigger = vgpu->irqs[idx].trigger;
+ if (trigger)
+ eventfd_signal(trigger);
+ pr_info("gsp-resman-vfio: inject_irq %lu (trigger %s)\n",
+ idx, trigger ? "fired" : "missing");
+ ret = 0;
+ }
+ mutex_unlock(&nv_vgpu_driver.lock);
+ return ret;
+}
+
+static const struct kernel_param_ops nv_vgpu_inject_irq_ops = {
+ .set = nv_vgpu_inject_irq_set,
+};
+
+module_param_cb(inject_irq, &nv_vgpu_inject_irq_ops, NULL, 0200);
+MODULE_PARM_DESC(inject_irq, "Deliver MSI-X vector N to the guest (debug)");
+
module_init(nv_vgpu_init);
module_exit(nv_vgpu_exit);
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 13:12:55 +0000