From 3b9432714a0515b89309ff2e6c1983690fb67761 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 24 Sep 2026 01:58:15 -0400 Subject: kernel: validate learned semaphore maps and post guest GPU events - admit {GPU VA -> PA} pairs only when the preceding bytes form a valid client memory descriptor header (type 0/1, CPU+GPU mapped bits), prefer validated pairs over headerless candidates under first-wins dedup, and invalidate the translation cache on every rebuild; fixes stale identity entries shadowing live pool pages and lets the guest UVM channel bring-up complete on its own - capture OS event allocations (class 0x79) and post event records (function 0x1003) into the guest status queue when a channel's busy notification is consumed, with a broadcast fallback, serialized with replies via a queue lock - add BAR0 shadow introspection (dump_bar0/scan_bar0) and guest-side diagnostic scripts for the VM cycle --- .../06-host-module-and-hybrid-model.md | 18 + kernel/gsp_rm_emulation.c | 273 +++++ kernel/gsp_rm_emulation.h | 23 + kernel/gsp_sema_service.c | 1218 +++++++++++++++++++- kernel/gsp_sema_service.h | 23 + kernel/main.c | 269 +++++ scripts/create_mdev.sh | 6 +- tools/guest-diag/cycle_cdt.sh | 46 + tools/guest-diag/guest_pmap.py | 37 + tools/guest-diag/inject_tracker.sh | 30 + tools/guest-diag/pmap_va.sh | 21 + tools/guest-diag/run_traced.sh | 53 + tools/guest-diag/vmrestart.sh | 31 + 13 files changed, 1983 insertions(+), 65 deletions(-) create mode 100755 tools/guest-diag/cycle_cdt.sh create mode 100644 tools/guest-diag/guest_pmap.py create mode 100755 tools/guest-diag/inject_tracker.sh create mode 100755 tools/guest-diag/pmap_va.sh create mode 100755 tools/guest-diag/run_traced.sh create mode 100755 tools/guest-diag/vmrestart.sh diff --git a/docs/03-architecture/06-host-module-and-hybrid-model.md b/docs/03-architecture/06-host-module-and-hybrid-model.md index 5849071..15545a4 100644 --- a/docs/03-architecture/06-host-module-and-hybrid-model.md +++ b/docs/03-architecture/06-host-module-and-hybrid-model.md @@ -175,6 +175,23 @@ interface). That is enough to: * inspect push buffers and page tables, * and later, create page tables for server-owned ranges. +Releases are recognised in the decoded method stream in both of the encodings +the driver uses: the copy-engine `SET_SEMAPHORE_A/B` + `LAUNCH_DMA` group and +the TURING+ `SET_REPORT_SEMAPHORE_A..D` group +(`{offset_upper, offset_lower, payload, operation}`, operation 0 = release), +which is how the host class ends a push. The learned translation cache is +page-granular, so all semaphore slots of one page resolve from one entry. + +One class of pages is not derivable from the structures above: client +mappings created by `mmap()` on the control device, where the kernel driver +maps sysmem pages into the process and the *same* numeric address doubles as +the GPU VA (observed live: a 2 MB shared mapping of the control device, and +push releases whose semaphore address equals the mapped client VA). Those +pages have no leaf PTEs in the GPU page tables (the client delegates them), +and the kernel-side bookkeeping records physical pages without a GPU VA. +Until a durable join is implemented, a session can feed verified pairs with +the `sema_map_add` debug parameter. + ## 5. The forwarding path (design) ``` @@ -219,6 +236,7 @@ emulated device inspectable without rebuilding: | `poke_fill=,,` | fill a range | | `fill_vidmem=` | fill the VRAM shadow | | `sema_probe=[,count]` | probe a semaphore VA and nearby state in guest memory | +| `sema_map_add=,` | teach the service one verified `{GPU VA -> guest PA}` pair | | `dump_guest=,` | dump guest physical memory (length capped) | These are development-grade but production-useful: they are how the memory diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index a21cf28..986132e 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -473,11 +473,17 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, if (!buf) return -ENOMEM; + /* The status queue is shared: RPC replies run on the GSP emulation + * thread, channel-completion events are posted from the progress + * service's thread. */ + mutex_lock(&rm->stat_lock); + record_size = gsp_rm_build_record(rm, buf, buf_size, function, sequence, result, data, data_len, &elem_count); if (!record_size) { kfree(buf); + mutex_unlock(&rm->stat_lock); return -EOVERFLOW; } @@ -486,6 +492,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, nv_vgpu_err(ctx->vgpu, "GSPRM: record too large: %u elements > queue %u", elem_count, rm->stat_msgcount); kfree(buf); + mutex_unlock(&rm->stat_lock); return -EOVERFLOW; } @@ -507,6 +514,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, nv_vgpu_err(ctx->vgpu, "GSPRM: status write failed off=0x%llx ret=%d", rec_off, ret); kfree(buf); + mutex_unlock(&rm->stat_lock); return ret; } @@ -518,6 +526,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, nv_vgpu_err(ctx->vgpu, "GSPRM: status wrap write failed ret=%d", ret); kfree(buf); + mutex_unlock(&rm->stat_lock); return ret; } } @@ -538,6 +547,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, &new_wp, sizeof(new_wp)); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: status writePtr update failed ret=%d", ret); + mutex_unlock(&rm->stat_lock); return ret; } @@ -547,9 +557,82 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, * drain while a poller waits for it (same as real GSP-RM). */ gsp_emu_inject_completion_interrupt(ctx, rm->stat_seq); + mutex_unlock(&rm->stat_lock); return 0; } +/* + * Post the GPU event that tells the guest's CPU-side RM a channel's + * submitted work is complete. A real GPU raises an interrupt, the host GSP + * turns it into an NV_VGPU_MSG_EVENT_POST_EVENT record for the client's + * event object, and the client's RM signals the OS event (eventfd) the + * submitting thread polls - this is what makes libcuda's work waits finish. + * + * Safe to call from the progress service's thread: the status queue write is + * serialized by rm->stat_lock and the guest is woken with MSI-X vector 0. + */ +int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src) +{ + struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL; + struct gsp_rm_emu *rm; + u8 payload[32]; + u32 i; + int posted = 0; + + if (!ctx) + return -ENODEV; + rm = ctx->rm_emu; + if (!rm || !h_src) + return -ENODEV; + + for (i = 0; i < rm->events_n; i++) { + if (!rm->events[i].valid || rm->events[i].h_src != h_src) + continue; + + memset(payload, 0, sizeof(payload)); + memcpy(payload + 0, &rm->events[i].h_client, 4); + memcpy(payload + 4, &rm->events[i].h_event, 4); + memcpy(payload + 8, &rm->events[i].notify_index, 4); + memcpy(payload + 12, &rm->events[i].data, 4); + /* info16@16, status@20, eventDataSize@24, bNotifyList@28 = 0 */ + if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, + payload, sizeof(payload))) + continue; + posted++; + } + + /* + * Fallback when the event's source object could not be associated with + * the channel: nudge every registered event. The client's dispatcher + * re-reads each channel's state on any event, so a spurious wake-up is + * harmless (and this only runs when a submission was consumed). + */ + if (!posted) { + for (i = 0; i < rm->events_n; i++) { + if (!rm->events[i].valid) + continue; + memset(payload, 0, sizeof(payload)); + memcpy(payload + 0, &rm->events[i].h_client, 4); + memcpy(payload + 4, &rm->events[i].h_event, 4); + memcpy(payload + 8, &rm->events[i].notify_index, 4); + memcpy(payload + 12, &rm->events[i].data, 4); + if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, + payload, sizeof(payload))) + continue; + posted++; + } + } + + if (posted) { + rm->events_posted += posted; + if (rm->events_posted <= 32 || (rm->events_posted % 5000) == 0) + nv_vgpu_info(vgpu, + "GSPRM: post_event hSrc=0x%x posted=%d total=%llu", + h_src, posted, rm->events_posted); + } + return posted ? 0 : -ENOENT; +} + /* ============================================================ * Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED) * ============================================================ */ @@ -714,6 +797,8 @@ static const char *gsp_rm_rpc_name(u32 function) case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO: return "GSP_SET_SYSTEM_INFO"; case GSPRM_RPC_FUNC_SET_REGISTRY: return "SET_REGISTRY"; case GSPRM_RPC_FUNC_CONTINUATION_RECORD: return "CONTINUATION_RECORD"; + case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY: return "SET_PAGE_DIRECTORY"; + case GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY: return "UNSET_PAGE_DIRECTORY"; case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL"; default: return "UNKNOWN"; } @@ -1949,6 +2034,7 @@ static bool gsp_rm_is_channel_class(u32 class_id) #define GSPRM_CHAN_GPFIFO_OFFSET 8 #define GSPRM_CHAN_GPFIFO_ENTRIES 16 #define GSPRM_CHAN_ENGINE_TYPE 128 +#define GSPRM_CHAN_CID 132 #define GSPRM_CHAN_INSTANCE_MEM 144 #define GSPRM_CHAN_USERD_MEM 168 #define GSPRM_CHAN_MEM_APERTURE 16 /* offset of addressSpace in the desc */ @@ -1967,11 +2053,14 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8); memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4); memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4); + memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4); memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8); memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8); memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM + GSPRM_CHAN_MEM_APERTURE, 4); memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8); + memcpy(&desc.notifier_as, params + GSPRM_CHAN_NOTIFIER_MEM + + GSPRM_CHAN_MEM_APERTURE, 4); gsp_sema_service_channel_add(&desc); } @@ -2038,6 +2127,58 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, if (gsp_rm_is_channel_class(class_id)) gsp_rm_register_channel(ctx, ho, params, params_size); + /* + * NV01_EVENT (allocated with class 0x79 in the guest ABI; params are + * NV0005_ALLOC_PARAMETERS {hParentClient, hSrcResource, hClass, + * notifyIndex, data}). The driver allocates one per channel and waits + * for the GPU to post the event before it considers submitted work + * complete - remember them so completed work can be signalled back. + */ + if (class_id == 0x79 && params_size >= 24 && rm->events_n < GSPRM_MAX_EVENTS) { + u32 h_src = 0, notify_index = 0, hclass = 0; + u64 data64 = 0; + int hc_off = -1, i; + + /* + * Locate hClass (0x79) inside the params instead of trusting a + * fixed offset: NV0005_ALLOC_PARAMETERS is + * {hParentClient, hSrcResource, hClass, notifyIndex, data}, + * with hSrcResource immediately before hClass. + */ + for (i = 0; i + 4 <= (int)params_size; i += 4) { + u32 v; + + memcpy(&v, params + i, 4); + if (v == 0x79) { + hc_off = i; + break; + } + } + if (hc_off >= 4) { + memcpy(&h_src, params + hc_off - 4, 4); + memcpy(&hclass, params + hc_off, 4); + if (hc_off + 8 <= (int)params_size) + memcpy(¬ify_index, params + hc_off + 4, 4); + if (hc_off + 16 <= (int)params_size) + memcpy(&data64, params + hc_off + 8, 8); + } else { + memcpy(&h_src, params + 4, 4); + memcpy(¬ify_index, params + 12, 4); + memcpy(&data64, params + 16, 8); + } + rm->events[rm->events_n].h_src = h_src; + rm->events[rm->events_n].h_event = ho; + rm->events[rm->events_n].h_client = hc; + rm->events[rm->events_n].notify_index = notify_index; + rm->events[rm->events_n].data = (u32)data64; + rm->events[rm->events_n].valid = true; + rm->events_n++; + nv_vgpu_info(ctx->vgpu, + "GSPRM: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d) params=%*phN", + ho, h_src, notify_index, hc_off, (int)min(params_size, (u32)24), + params); + } + /* Echo the fixed alloc header + as many param bytes as are contiguous * in the first queue element. */ echo_len = min(params_size, @@ -2336,6 +2477,129 @@ static bool gsp_rm_alloc_interesting(u32 class_id) } } +/* + * SET_PAGE_DIRECTORY (54) / UNSET_PAGE_DIRECTORY (79) + * + * The driver builds the page tables of the GPU VA spaces it owns and hands the + * top-level directory to the GPU here. The parameters are + * { u64 physAddress; u32 numEntries; u32 flags; u32 hVASpace; u32 chId; + * u32 subDeviceId; u32 pasid; } + * (flags bits 1:0: 0 = vidmem aperture, 1 = sysmem coherent, + * 2 = sysmem non-coherent). + * + * Recording the directory lets the progress service translate that VA space's + * addresses directly (four-level walk), which is how the client's semaphore + * waits are completed; it also removes the need for heuristic discovery. + */ +static void gsp_rm_handle_set_page_directory(struct gsp_emu_context *ctx, + const u8 *elem, u32 sequence, + u32 length) +{ + const u8 *p = elem + GSPRM_ELEM_HDR_SIZE + + sizeof(struct gsprm_rpc_header); + struct sema_vaspace_desc desc; + u64 phys = 0; + u32 num_entries = 0, flags = 0, h_vas = 0, ch_id = 0, subdev = 0, pasid = 0; + + /* + * Wire layout: a 16-byte message header (hClient, hObject, ...) + * followed by the control's own parameters + * { u64 physAddress; u32 numEntries; u32 flags; u32 hVASpace; u32 chId; + * u32 subDeviceId; u32 pasid; }. + */ + if (length >= 44) { + memcpy(&phys, p + 16, 8); + memcpy(&num_entries, p + 24, 4); + memcpy(&flags, p + 28, 4); + memcpy(&h_vas, p + 32, 4); + memcpy(&ch_id, p + 36, 4); + memcpy(&subdev, p + 40, 4); + memcpy(&pasid, p + 44, 4); + } + + { + char dump[200]; + int o = 0, i; + u32 dlen = min_t(u32, length, 80); + + for (i = 0; i < (int)dlen; i += 4) { + u32 v = 0; + memcpy(&v, p + i, 4); + o += snprintf(dump + o, sizeof(dump) - o, "%08x ", + v); + } + nv_vgpu_info(ctx->vgpu, + "GSPRM: SET_PAGE_DIRECTORY len=%u payload: %s", + length, dump); + } + nv_vgpu_info(ctx->vgpu, + "GSPRM: SET_PAGE_DIRECTORY pdb=0x%llx entries=%u flags=0x%x hVASpace=0x%x chId=0x%x subDev=%u pasid=%u len=%u", + phys, num_entries, flags, h_vas, ch_id, subdev, pasid, + length); + + if (phys && num_entries) { + memset(&desc, 0, sizeof(desc)); + desc.h_vaspace = h_vas; + desc.pdb = phys; + desc.num_entries = num_entries; + desc.b_vidmem = (flags & 0x3) == 0; + gsp_sema_service_vaspace_add(&desc); + + /* Show the first entries of the directory: they are the ground + * truth for the walk. */ + { + u64 peek[4] = {0}; + char b[128]; + int i, o = 0, rc = -ENODEV; + + if (desc.b_vidmem) { + u8 *base = ctx->vgpu->bars[1].vaddr; + + if (base && phys + sizeof(peek) <= + ctx->vgpu->bars[1].size) { + memcpy(peek, base + phys, sizeof(peek)); + rc = 0; + } + } else { + rc = gsp_rm_read_dev(ctx->vgpu, phys, peek, + sizeof(peek)); + } + if (!rc) { + for (i = 0; i < 4; i++) + o += snprintf(b + o, sizeof(b) - o, + "%016llx ", peek[i]); + nv_vgpu_info(ctx->vgpu, + "GSPRM: directory[0..3]: %s", b); + } + } + } + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY, sequence, 0, + NULL, 0)) + nv_vgpu_err(ctx->vgpu, "GSPRM: SET_PAGE_DIRECTORY reply send failed"); +} + +static void gsp_rm_handle_unset_page_directory(struct gsp_emu_context *ctx, + const u8 *elem, u32 sequence, + u32 length) +{ + const u8 *p = elem + GSPRM_ELEM_HDR_SIZE + + sizeof(struct gsprm_rpc_header); + u32 h_vas = 0; + + if (length >= 4) + memcpy(&h_vas, p, 4); + + nv_vgpu_info(ctx->vgpu, + "GSPRM: UNSET_PAGE_DIRECTORY hVASpace=0x%x len=%u", + h_vas, length); + gsp_sema_service_vaspace_del(h_vas); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY, sequence, 0, + NULL, 0)) + nv_vgpu_err(ctx->vgpu, "GSPRM: UNSET_PAGE_DIRECTORY reply send failed"); +} + static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) { struct gsp_rm_emu *rm = ctx->rm_emu; @@ -2376,6 +2640,14 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) gsp_rm_handle_free(ctx, elem, sequence); return; + case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY: + gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length); + return; + + case GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY: + gsp_rm_handle_unset_page_directory(ctx, elem, sequence, hdr->length); + return; + default: /* Synchronous RPC: echo function+sequence with success so the * driver's rpcRecvPoll completes. */ @@ -2499,6 +2771,7 @@ int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx) return -ENOMEM; rm->vgpu = vgpu; + mutex_init(&rm->stat_lock); ctx->rm_emu = rm; return 0; } diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index a82656f..1ab0a83 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -40,9 +40,13 @@ struct gsp_emu_context; #define GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO 72 #define GSPRM_RPC_FUNC_SET_REGISTRY 73 #define GSPRM_RPC_FUNC_CONTINUATION_RECORD 71 +#define GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY 54 +#define GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY 79 #define GSPRM_RPC_FUNC_GSP_RM_CONTROL 76 #define GSPRM_RPC_FUNC_GSP_RM_ALLOC 103 #define GSPRM_RPC_FUNC_FREE 10 +#define GSPRM_RPC_EVENT_POST_EVENT 0x1003 +#define GSPRM_MAX_EVENTS 128 #define GSPRM_RPC_EVENT_GSP_INIT_DONE 0x1001 /* rpc_set_guest_system_info_v03_00: version fields the driver reads back @@ -168,10 +172,29 @@ struct gsp_rm_emu { u32 alloc_dumps; /* raw GSP_RM_ALLOC dumps emitted */ u32 alloc_param_dumps; /* alloc param dumps emitted */ u32 ctrl_param_dumps; /* control param dumps emitted */ + + /* + * NV01_EVENT objects (class 0x5, allocated via class 0x79 in the + * guest ABI). Each carries {hSrcResource, notifyIndex, data}; the + * driver allocates one per channel and waits for the GPU to post the + * event before it considers submitted work complete. + */ + struct mutex stat_lock; /* serializes status-queue writers */ + struct { + u32 h_src; /* source object (channel handle) */ + u32 h_event; /* event handle */ + u32 h_client; /* owning client */ + u32 notify_index; + u32 data; + bool valid; + } events[GSPRM_MAX_EVENTS]; + u32 events_n; + u64 events_posted; }; int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx); void gsp_rm_emu_fini(struct gsp_emu_context *ctx); +int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src); int gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle, u64 start, u64 end, u32 max_hits); int gsp_rm_scan_guest_ram_seq(struct nv_vgpu_device *vgpu, const u32 *pat, diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 2f453fa..10d6b80 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -43,6 +43,7 @@ #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" +#include "gsp_rm_emulation.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 @@ -82,6 +83,18 @@ #define HOST_SEM_OP_RELEASE 1 #define HOST_SEM_OP_REDUCTION 6 +/* + * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other + * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared + * by the TURING and later classes. RELEASE (0) ends the push. + */ +#define HOST_M_REPORT_SEM_A 0x1b00 +#define HOST_M_REPORT_SEM_B 0x1b04 +#define HOST_M_REPORT_SEM_C 0x1b08 +#define HOST_M_REPORT_SEM_D 0x1b0c +#define HOST_RS_OP_MASK 0x3 +#define HOST_RS_OP_RELEASE 0 + struct sema_map_entry { u64 gpu_page; /* GPU VA, page aligned */ u64 pa_page; /* guest PA, page aligned */ @@ -91,7 +104,7 @@ struct sema_map_entry { /* registered channels (GPFIFO-driven execution) */ /* ------------------------------------------------------------------ */ -#define SVC_MAX_CHANNELS 24 +#define SVC_MAX_CHANNELS 128 #define SVC_GPFIFO_ENTRY_SIZE 8 #define SVC_PUSH_MAX (256 * 1024) @@ -103,12 +116,17 @@ struct svc_channel { bool valid; u32 h_object; u32 engine_type; + u32 cid; /* RM channel id (doorbell token vector) */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ + bool userd_ok; /* userd_pa reads a plausible put/get */ + bool logged; /* one state log per channel */ u64 instance_pa; - u64 notifier_pa; + u64 notifier_pa; /* error notifier: NvNotification slots */ + u32 notifier_as; + u64 notified; /* work-submit-token notifications completed */ u32 last_get; /* next GPFIFO entry we have still to run */ u64 submissions; /* GPFIFO entries executed */ }; @@ -122,6 +140,37 @@ struct svc_channel { static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)]; static u32 svc_map_hash[SEMA_MAP_MAX * 2]; +/* VA spaces whose page directory the driver published / learned VA->PA pairs. */ +#define SVC_MAX_VASPACES 8 +#define SVC_LAT_MAX 65536 + +/* + * Pinned VA->PA translations: translations a session has verified by hand + * (debug injection) and that must not be evicted by the learned cache, which + * is content-addressed and gets thrashed by the lookup stream. + */ +#define SVC_PIN_SLOTS 4096 + +struct svc_pin { + u64 va_page; + u64 pa_page; +}; + +struct svc_vaspace { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; /* root entry count */ + int ap; /* SVC_APERTURE_* of the directory itself */ +}; + +struct svc_lat { + bool valid; + u64 va; + u64 addr; + int ap; /* SVC_APERTURE_* */ +}; + static void svc_mark_hot(u64 pa) { u64 page = pa >> SEMA_PAGE_SHIFT; @@ -133,6 +182,7 @@ static void svc_mark_hot(u64 pa) struct sema_svc { struct task_struct *task; bool stop; + struct completion stopped; u64 direct_base; u32 map_n; @@ -144,7 +194,17 @@ struct sema_svc { spinlock_t chan_lock; struct svc_channel chan[SVC_MAX_CHANNELS]; + /* VA spaces whose page directory the driver published */ + spinlock_t vas_lock; + struct svc_vaspace vas[SVC_MAX_VASPACES]; + struct svc_lat lat[SVC_LAT_MAX]; + struct svc_pin pin[SVC_PIN_SLOTS]; + u32 pin_n; + /* stats */ + u64 walks_ok; + u64 walks_miss; + u64 lat_hits; u64 pairs_seen; u64 pages_decoded; u64 pushes_seen; @@ -162,6 +222,12 @@ struct sema_svc { u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ u64 vidmem_writes; /* semaphores written in vidmem */ + u64 gpfifo_log; /* channel-state log budget */ + u64 gpfifo_entry_log; /* per-entry log budget */ + u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */ + u32 doorbell_token; /* last token written */ + u32 doorbell_log; /* token log budget */ + bool doorbell_pending; /* pump on next service pass */ }; static struct sema_svc svc; @@ -249,19 +315,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->valid = true; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; + ch->cid = desc->cid; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; + ch->notifier_as = desc->notifier_as ? desc->notifier_as + : SVC_APERTURE_SYS; } spin_unlock_irqrestore(&svc.chan_lock, flags); if (ch) - pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx\n", + pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, - desc->userd_pa); + desc->userd_pa, desc->engine_type, desc->cid, + desc->notifier_pa); + else + pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n", + desc->h_object, SVC_MAX_CHANNELS); } void gsp_sema_service_channel_del(u32 h_object) @@ -284,31 +357,605 @@ void gsp_sema_service_channel_del(u32 h_object) spin_unlock_irqrestore(&svc.chan_lock, flags); } +/* + * The driver publishes the page directory of a GPU VA space it owns (it builds + * the page tables itself and hands their base address to the GPU). Store it so + * the service can translate that space's VAs directly. + */ +void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) +{ + struct svc_vaspace *vs = NULL; + unsigned long flags; + int i; + + if (!desc || !desc->pdb || !desc->num_entries) + return; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) { + vs = &svc.vas[i]; + break; + } + if (!vs && !svc.vas[i].valid) + vs = &svc.vas[i]; + } + if (vs) { + vs->valid = true; + vs->h_vaspace = desc->h_vaspace; + vs->pdb = desc->pdb; + vs->entries = desc->num_entries; + vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + /* A new directory invalidates what we learned from the old one. */ + memset(svc.lat, 0, sizeof(svc.lat)); + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + + if (vs) + pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n", + desc->h_vaspace, desc->pdb, desc->num_entries, + desc->b_vidmem ? "vidmem" : "sysmem"); +} + +void gsp_sema_service_vaspace_del(u32 h_vaspace) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) { + svc.vas[i].valid = false; + memset(svc.lat, 0, sizeof(svc.lat)); + pr_info("sema-svc: VA space 0x%x page directory dropped\n", + h_vaspace); + break; + } + } + spin_unlock_irqrestore(&svc.vas_lock, flags); +} + /* * The client's vidmem VAs are its window base plus the vidmem offset, and the * backing store is the emulated BAR1 buffer. Everything else that resolves * goes through the discovered guest-RAM map. + * + * The window base is not 0x200000000: the observed layout places the first + * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame + * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000. + * Verified against the guest's own view of its GPFIFO rings and USERD. */ -#define SVC_VIDMEM_VA_BASE 0x200000000ULL +#define SVC_VIDMEM_VA_BASE 0x200080000ULL +#define SVC_VIDMEM_VA_LO 0x200000000ULL /* anything in the window below + * the base is still vidmem, but + * never maps into the buffer */ static int svc_map_lookup(u64 gpu_va, u64 *pa); -static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, - u32 *aperture) +/* ------------------------------------------------------------------ */ +/* client page-table walk */ +/* ------------------------------------------------------------------ */ +/* + * The driver builds its own page tables for the GPU VA spaces it owns and + * publishes each top-level directory through SET_PAGE_DIRECTORY. With those + * directories the service can translate the client's GPU VAs directly - the + * authoritative answer, and independent of any heuristic discovery. + * + * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere: + * + * PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21] + * dual entry (16 bytes): + * big half -> PT_BIG [20:16] + * small half -> PT_4K [20:12] + * + * A directory entry carries the aperture in bits 2:1 (1 = video memory, + * 2 = system coherent, 3 = system non-coherent) and the child address in bits + * 53:8 at 4 KB granularity (bits 53:4 at 256 B granularity in the big half of + * a dual entry). Bit 0 of a leaf entry is VALID; bit 0 of a directory entry is + * IS_PTE, so a set bit means the level terminates in a page here. + */ +#define SVC_MMU_VA_BITS 49 +#define SVC_MMU_AP_VID 1 /* in directory entries */ +#define SVC_MMU_AP_SYS_COH 2 +#define SVC_MMU_AP_SYS_NC 3 +#define SVC_MMU_PTE_AP_VID 0 /* in page table entries */ +#define SVC_MMU_PTE_AP_PEER 1 +#define SVC_MMU_PTE_AP_SYS_COH 2 +#define SVC_MMU_PTE_AP_SYS_NC 3 + +static u64 svc_pte_addr(u64 e, bool vidmem) { - if (!svc_map_lookup(va, addr)) { - *aperture = 0; /* guest RAM */ + if (vidmem) + return (e & 0x00000001FFFFFFFF00ULL); + return (e & 0x003FFFFFFFFF00ULL); +} + +/* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */ +static int svc_pte_aperture(u32 ap) +{ + if (ap == SVC_MMU_PTE_AP_VID) + return SVC_APERTURE_FB; + if (ap == SVC_MMU_PTE_AP_PEER) + return -ENODEV; + return SVC_APERTURE_SYS; +} + +static int svc_pde_aperture(u32 ap) +{ + if (ap == SVC_MMU_AP_VID) + return SVC_APERTURE_FB; + if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC) + return SVC_APERTURE_SYS; + return -ENODEV; +} + +/* + * A translated address is only usable if it lies inside the aperture that + * backs it: guest RAM up to the module's RAM limit for system memory, and the + * emulated vidmem buffer for video memory. Rejecting out-of-range results also + * stops a bogus page-table chain from being mistaken for a translation. + */ +static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap) +{ + if (ap == SVC_APERTURE_FB) + return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size; + return addr + 8 <= SEMA_PA_MAX; +} + +static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off, + int ap, u64 *out) +{ + if (!svc_ap_ok(vgpu, table + off, ap)) + return -ENOENT; + if (svc_read_ap(vgpu, table + off, + ap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + out, sizeof(*out))) + return -ENOENT; + return 0; +} + +/* + * Leaf lookup under a table: read the PTE at `index`, validate it and return + * the backing address plus its aperture. + */ +static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap, + u32 index, u64 *addr, int *ap) +{ + u64 e; + int eap; + + if (svc_read_ent(vgpu, table, 8 * index, tap, &e)) + return -ENOENT; + if (!(e & 1)) /* VALID */ + return -ENOENT; + eap = svc_pte_aperture((e >> 1) & 0x3); + if (eap < 0) + return -ENOENT; + *addr = svc_pte_addr(e, eap == SVC_APERTURE_FB); + if (!svc_ap_ok(vgpu, *addr, eap)) + return -ENOENT; + *ap = eap; + return 0; +} + +/* + * Walk the four-level page table rooted at `pdb` for one VA. Directory levels + * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry + * is the 16-byte dual entry (big half first, small half second). + */ +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace) +{ + u64 table = pdb; + u32 root_bits, idx, level; + u64 big, small; + int sap, bap; + + root_bits = ilog2(entries ? entries : 1); + if (root_bits < 1 || root_bits > 12) + return -ENOENT; + + if (!table) + return -ENOENT; + + for (level = 0; level < 3; level++) { + u64 e; + u32 bits, shift; + int eap; + + if (level == 0) { + bits = root_bits; + shift = SVC_MMU_VA_BITS - root_bits; + } else { + bits = 9; + shift = 29 + 9 * (2 - level); + } + idx = (u32)((va >> shift) & ((1u << bits) - 1)); + if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) { + if (trace) + pr_info("sema-walk: L%u read fail table=0x%llx idx=%u\n", + level, table, idx); + return -ENOENT; + } + if (trace) + pr_info("sema-walk: L%u e=0x%016llx idx=%u\n", + level, e, idx); + /* A page at a directory level would cover at least 2^38 bytes; + * the driver never maps those, so treat it as a false chain. */ + if (e & 1) + return -ENOENT; + eap = svc_pde_aperture((e >> 1) & 0x3); + if (eap < 0) + return -ENOENT; + tap = eap; + table = svc_pte_addr(e, eap == SVC_APERTURE_FB); + if (!svc_ap_ok(vgpu, table, eap)) + return -ENOENT; + } + + /* PD0: one dual entry per index (big half then small half). */ + idx = (va >> 21) & 0xff; + if (svc_read_ap(vgpu, table + 16 * idx, + tap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + &big, sizeof(big)) || + svc_read_ap(vgpu, table + 16 * idx + 8, + tap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + &small, sizeof(small))) + return -ENOENT; + + /* Small half: 4 KB page table under it. */ + sap = svc_pde_aperture(((small >> 1) & 0x3)); + if (sap >= 0 && !(small & 1)) { + u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB); + + if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff, + addr, ap)) { + *addr += va & 0xfff; + return 0; + } + } + + /* Big half: a 2 MB page here, or a large (64 KB) page table. */ + if (big & 1) { + bap = svc_pte_aperture((big >> 1) & 0x3); + if (bap >= 0) { + *addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) + + (va & ((1ULL << 21) - 1)); + *ap = bap; + return 0; + } + } + bap = svc_pde_aperture((big >> 1) & 0x3); + if (bap >= 0) { + u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB); + + if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f, + addr, ap)) { + *addr += va & 0xffff; + return 0; + } + } + return -ENOENT; +} + +/* + * Page directories the driver published (SET_PAGE_DIRECTORY). + */ +static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, + u64 *addr, int *ap) +{ + unsigned long flags; + int i, rc = -ENOENT; + bool trace = svc.walks_ok + svc.walks_miss < 24; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + int alt; + + if (!svc.vas[i].valid) + continue; + if (va >> SVC_MMU_VA_BITS) + break; + if (trace) + pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n", + i, va, svc.vas[i].pdb, svc.vas[i].entries); + /* + * Try the aperture the driver named first, then the other one: + * the aperture encoding has changed between driver generations + * and a wrong guess only costs a rejected walk. + */ + alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; + if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, + svc.vas[i].ap, addr, ap, trace) || + !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, + alt, addr, ap, trace)) { + svc.walks_ok++; + rc = 0; + break; + } + } + if (rc) + svc.walks_miss++; + spin_unlock_irqrestore(&svc.vas_lock, flags); + return rc; +} + +/* + * Self-discovering the page directory. + * + * The published directories are not always usable (a guest can publish a + * directory location the host never gets to see written), so as a fallback + * find the directory by verification: take pairs of GPU VAs whose backing + * guest page is already known (the discovered VA->PA map), and accept the + * first guest page that, used as a top-level directory, reproduces several of + * those translations exactly. A wrong candidate cannot pass that test, so no + * layout assumption is needed to trust the result; only the entry encoding + * (which the walk itself applies consistently) is shared. + */ +static u64 svc_pd_base; +static int svc_pd_ap; +static u32 svc_pd_entries; +static u8 svc_pd_tried; +static u8 svc_pd_found; +#define SVC_PD_SAMPLES 8 +#define SVC_PD_MIN_HITS 4 + +static u32 svc_pd_cands; +static u32 svc_pd_best; + +static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand, + int ap, const u64 *sva, const u64 *spa, + u32 nsamp) +{ + u32 i, hits = 0; + + svc_pd_cands++; + + for (i = 0; i < nsamp; i++) { + u64 a = 0; + int eap = 0; + + if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) && + (a & ~0xfffULL) == (spa[i] & ~0xfffULL)) + hits++; + } + if (hits > svc_pd_best) + svc_pd_best = hits; + if (hits >= SVC_PD_MIN_HITS) { + svc_pd_base = cand; + svc_pd_ap = ap; + svc_pd_entries = 4; + svc_pd_found = 1; + pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n", + cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram", + hits, nsamp); + } +} + +static void svc_pd_discover(struct nv_vgpu_device *vgpu) +{ + u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES]; + u32 i, n = 0, step; + u64 chunk = 64 * 1024; + u8 *buf; + int pass; + + if (svc_pd_tried) + return; + svc_pd_tried = 1; + + if (svc.map_n < SVC_PD_SAMPLES) + return; + step = svc.map_n / SVC_PD_SAMPLES; + for (i = 0; i < SVC_PD_SAMPLES; i++) { + sva[i] = svc.map[i * step].gpu_page; + spa[i] = svc.map[i * step].pa_page; + if (sva[i] >> SVC_MMU_VA_BITS) + return; /* outside the walkable VA range */ + n++; + } + if (!n) + return; + + /* Guest RAM first, then the emulated vidmem. Page directories are page + * aligned, and every VA here is below 2^47, so the root entry that + * matters is the first one of the page. */ + for (pass = 0; pass < 2 && !svc_pd_found; pass++) { + int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB; + u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size; + u8 *fb = (u8 *)vgpu->bars[1].vaddr; + u64 gpa; + + buf = kmalloc(chunk, GFP_KERNEL); + if (!buf) + return; + for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) { + u32 len = (u32)min_t(u64, chunk, limit - gpa); + u64 off; + + if (pass == 0) { + if (svc_read(vgpu, gpa, buf, len)) + continue; + } else { + if (!fb) + break; + memcpy(buf, fb + gpa, len); + } + for (off = 0; off + 8 <= len && !svc_pd_found; + off += SEMA_PAGE_SIZE) { + u64 e; + + memcpy(&e, buf + off, sizeof(e)); + if (!(e & 1) && ((e >> 1) & 0x3)) + svc_pd_try_candidate(vgpu, gpa + off, + ap, sva, spa, n); + } + } + kfree(buf); + } + pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n", + svc_pd_found ? "found" : "not found", svc_pd_base, + svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram", + svc_pd_cands, svc_pd_best, n); +} + +static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va, + u64 *addr, int *ap) +{ + if (!svc_pd_found) + return -ENOENT; + return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, + svc_pd_ap, addr, ap, false); +} + +/* Learned translations: a walk costs a handful of guest reads, so cache the + * successes and answer repeats without touching the page tables again. A cache + * line covers a whole page, so arrays of semaphores in one page all resolve. */ +static int svc_lat_lookup(u64 va, u64 *addr, int *ap) +{ + struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; + + if (l->valid && + (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (va & ~(u64)(SEMA_PAGE_SIZE - 1))) { + *addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + *ap = l->ap; + svc.lat_hits++; + return 0; + } + return -ENOENT; +} + +static void svc_lat_store(u64 va, u64 addr, int ap) +{ + struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; + + l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1); + l->ap = ap; + l->valid = true; +} + +static int svc_pin_lookup(u64 va, u64 *pa) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < 8; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page) + return -ENOENT; + if (p->va_page == page) { + *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); + return 0; + } + } + return -ENOENT; +} + +static void svc_pin_store(u64 va, u64 pa) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < 8; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page || p->va_page == page) { + if (!p->va_page) + svc.pin_n++; + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + return; + } + } +} + +/* + * Translate a GPU VA using trusted sources only: a pinned translation, the + * learned cache, the discovered map, the directories the driver published and + * the self-discovered directory. The final vidmem window convention is + * deliberately NOT part of this path: it answers with a vidmem offset for any + * VA above the window base, including sysmem-backed VAs, so callers that must + * not act on a guess (push execution) use this variant. + */ +static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + int *ap) +{ + u64 pa = 0; + + if (!svc_pin_lookup(va, addr)) { + *ap = SVC_APERTURE_SYS; + return 0; + } + if (!svc_lat_lookup(va, addr, ap)) + return 0; + if (!svc_map_lookup(va, &pa)) { + *addr = pa; + *ap = SVC_APERTURE_SYS; + svc_lat_store(va, pa, SVC_APERTURE_SYS); + return 0; + } + if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) { + *addr += va & 0xfff; + svc_lat_store(va, *addr, *ap); return 0; } + if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { + *addr += va & 0xfff; + svc_lat_store(va, *addr, *ap); + return 0; + } + if (!svc_pd_tried) { + svc_pd_discover(vgpu); + if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { + *addr += va & 0xfff; + svc_lat_store(va, *addr, *ap); + return 0; + } + } + return -ENOENT; +} + +/* + * Translate a GPU VA: the trusted sources, then the vidmem window convention + * (a structural guess that is correct for the driver's vidmem window). + */ +static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) +{ + if (!svc_lookup_trusted(vgpu, va, addr, ap)) + return 0; if (va >= SVC_VIDMEM_VA_BASE && va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) { *addr = va - SVC_VIDMEM_VA_BASE; - *aperture = 1; /* emulated vidmem */ + *ap = SVC_APERTURE_FB; + svc_lat_store(va, *addr, *ap); return 0; } return -ENOENT; } +static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + u32 *aperture) +{ + int ap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, addr, &ap)) + return -ENOENT; + *aperture = (ap == SVC_APERTURE_FB) ? 1 : 0; + return 0; +} + static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { u64 addr = 0; @@ -316,11 +963,9 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; - if (ap) { - memcpy(val, vgpu->bars[1].vaddr + addr, sizeof(*val)); - return 0; - } - return svc_read(vgpu, addr, val, sizeof(*val)); + return svc_read_ap(vgpu, addr, + ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, + val, sizeof(*val)); } static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) @@ -330,12 +975,11 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; - if (ap) { - memcpy(vgpu->bars[1].vaddr + addr, &val, sizeof(val)); + if (ap) svc.vidmem_writes++; - return 0; - } - return svc_write(vgpu, addr, &val, sizeof(val)); + return svc_write_ap(vgpu, addr, + ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, + &val, sizeof(val)); } /* ------------------------------------------------------------------ */ @@ -357,7 +1001,7 @@ struct base_candidate { * vas[CPU] followed by vas[GPU]), collected raw for pass 2. */ #define PAIR_MAX 32768 -struct raw_pair { u64 cpu_va; u64 gpu_va; }; +struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; }; static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, struct base_candidate *cands, u32 *ncand, @@ -404,8 +1048,32 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, b >= SEMA_VA_LO && b < SEMA_VA_HI && (a & 0xfff) == 0 && (b & 0xfff) == 0 && *npairs < PAIR_MAX) { + u8 ok = 0; + + /* + * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] + * (offset 48) directly before vas[UVM_ID_GPU0] + * (offset 56), so a pair taken from a live rm_mem + * is preceded by the header {u32 type; bitmap + * mapped_on}. Validate it when it fits in the same + * page: freed allocations and unrelated tables + * that happen to hold KVA/VA pairs do not match, + * and a stale entry for the same VA must not + * shadow the live one. + */ + if (off >= 48) { + u32 type; + u64 mapped_on; + + memcpy(&type, buf + off - 48, 4); + memcpy(&mapped_on, buf + off - 40, 8); + if ((type == 0 || type == 1) && + (mapped_on & 0x3) == 0x3) + ok = 1; + } pairs[*npairs].cpu_va = a; pairs[*npairs].gpu_va = b; + pairs[*npairs].valid_hdr = ok; (*npairs)++; } } @@ -424,6 +1092,22 @@ static void svc_add_map(u64 gpu_va, u64 pa) } } +/* Add one scanned {CPU KVA, GPU VA} pair to the map. */ +static void svc_add_pair(u64 base, const struct raw_pair *p) +{ + u64 pa; + + if (p->cpu_va < base) + return; + pa = p->cpu_va - base; + if (pa >= 0x140000000ULL || (pa & 0xfff)) + return; + if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI) + return; + svc_add_map(p->gpu_va, pa); + svc.pairs_seen++; +} + static int svc_map_cmp(const void *a, const void *b) { const struct sema_map_entry *x = a, *y = b; @@ -495,6 +1179,28 @@ static int svc_map_lookup(u64 gpu_va, u64 *pa) return -ENOENT; } +/* Binary search over a sorted sub-range (used while the tail is unsorted). */ +static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa) +{ + u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); + int lo = (int)from, hi = (int)to - 1; + + while (lo <= hi) { + int mid = (lo + hi) / 2; + + if (svc.map[mid].gpu_page == gpu_page) { + *pa = svc.map[mid].pa_page + + (gpu_va & (SEMA_PAGE_SIZE - 1)); + return 0; + } + if (svc.map[mid].gpu_page < gpu_page) + lo = mid + 1; + else + hi = mid - 1; + } + return -ENOENT; +} + static void svc_rebuild(struct nv_vgpu_device *vgpu) { static u8 *buf; @@ -521,32 +1227,47 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) } if (best_base && best_count >= 2) { + u32 n_valid; + svc.direct_base = best_base; + + /* + * Live rm_mem-style records first (dedup is first-wins), then + * headerless candidates only for VAs the validated set does not + * cover - a freed allocation or an unrelated table holding a + * stale pair for the same VA must not shadow the live record. + */ + for (i = 0; i < npairs; i++) { + if (pairs[i].valid_hdr) + svc_add_pair(best_base, &pairs[i]); + } + n_valid = svc.map_n; + sort(svc.map, n_valid, sizeof(svc.map[0]), svc_map_cmp, NULL); for (i = 0; i < npairs; i++) { - u64 cpu = pairs[i].cpu_va; - u64 pa; + u64 dummy; - if (cpu < best_base) - continue; - pa = cpu - best_base; - if (pa >= 0x140000000ULL || (pa & 0xfff)) + if (pairs[i].valid_hdr) continue; - if (pairs[i].gpu_va < SEMA_VA_LO || - pairs[i].gpu_va >= SEMA_VA_HI) + if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid, + &dummy)) continue; - svc_add_map(pairs[i].gpu_va, pa); - svc.pairs_seen++; + svc_add_pair(best_base, &pairs[i]); } } + /* Stale entries from a previous rebuild (wrong PA for a reused VA) + * must not outlive the map they were derived from. */ + memset(svc.lat, 0, sizeof(svc.lat)); + svc_map_finalize(); svc.builds++; svc.last_build = jiffies; - pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu\n", + pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu\n", svc.builds, best_base, best_count, npairs, svc.map_n, svc.releases_seen, svc.releases_applied, svc.releases_missed, - svc.releases_stale, svc.releases_jumped); + svc.releases_stale, svc.releases_jumped, + svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits); pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, @@ -631,6 +1352,8 @@ struct svc_push_state { u32 have_remap; u32 release_dma; u32 sem_inc; + u32 rs_upper, rs_lower, rs_payload; + u32 have_rs_addr, have_rs_payload; }; /* ------------------------------------------------------------------ */ @@ -925,11 +1648,73 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } - if (subch != 4) { + /* + * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of + * four words {offset_upper, offset_lower, payload, operation} + * or the same words written one at a time. A RELEASE operation + * ends the push: run the collected work, then write the + * payload to the semaphore (forward progress only, so recycled + * buffers stay harmless). + */ + if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) { + u32 vv[4] = { v0, v1, v2, v3 }; + u32 k, n = count > 4 ? 4 : count; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case HOST_M_REPORT_SEM_A: + st.rs_upper = vv[k] & 0xff; + st.have_rs_addr = 1; + break; + case HOST_M_REPORT_SEM_B: + st.rs_lower = vv[k]; + st.have_rs_addr = 1; + break; + case HOST_M_REPORT_SEM_C: + st.rs_payload = vv[k]; + st.have_rs_payload = 1; + break; + case HOST_M_REPORT_SEM_D: { + u64 va; + u32 cur = 0; + + if ((vv[k] & HOST_RS_OP_MASK) != + HOST_RS_OP_RELEASE || + !st.have_rs_addr || + !st.have_rs_payload || + !st.rs_payload) + break; + va = ((u64)st.rs_upper << 32) | + st.rs_lower; + if (va < SEMA_VA_LO || va >= SEMA_VA_HI) + break; + if (page_pa) + svc_mark_hot(page_pa); + svc.pushes_seen++; + if (svc_sem_read(vgpu, va, &cur)) + break; + if (st.rs_payload <= cur) + break; + svc.pushes_pending++; + svc_run_ops(vgpu, ops, n_ops); + n_ops = 0; + if (!svc_sem_write(vgpu, va, + st.rs_payload)) + svc.releases_applied++; + break; + } + default: + break; + } + } off += 4 + count * 4; continue; } + if (subch != 4) { + off += 4 + count * 4; + continue; + } switch (addr) { case CE_M_SET_SEMAPHORE_A: st.sem_va_hi = v0; @@ -1253,11 +2038,181 @@ static void svc_apply_mapped(struct nv_vgpu_device *vgpu) static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { - u64 pa; + u8 *p = buf; + + while (len) { + u64 pa = 0; + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); + int ap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa, + ap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * Read `len` bytes at a GPU VA through a trusted translation only. Push text + * must never be fetched through the vidmem window convention: a sysmem-backed + * pushbuffer VA above the window base answers with a vidmem offset and the + * caller would decode zeros and mark work done that never ran. + */ +static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + u64 pa = 0; + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); + int ap = SVC_APERTURE_SYS; - if (svc_map_lookup(va, &pa)) + if (svc_lookup_trusted(vgpu, va, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa, + ap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the + * values are plausible for a channel with `entries` GPFIFO slots. + */ +static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get) +{ + u32 p = 0, g = 0; + + if (!pa) + return -EINVAL; + if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p))) + return -ENOENT; + if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g))) return -ENOENT; - return svc_read(vgpu, pa, buf, len); + if (p > entries || g > p) + return -EINVAL; + if (!p && !g) + return -EAGAIN; /* plausible but idle: keep looking */ + *put = p; + *get = g; + return 0; +} + +/* + * Locate a channel's USERD. The allocation parameters carry an offset that is + * relative to a client pool the emulation cannot see, so it is only one + * candidate: for the combined GPFIFO+USERD allocations the control block sits + * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout). + * Whichever candidate reads a coherent get/put pair wins and is cached. An + * idle candidate (0/0) is not accepted until it shows activity, so a zeroed + * page is never mistaken for the control block. + */ +static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u64 ring_pa, int ring_ap, u32 *put, u32 *get) +{ + u64 cand[4]; + u32 as[4], nc = 0, i; + + if (ring_ap == SVC_APERTURE_FB && ring_pa) { + cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB; + cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB; + } + cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; + + for (i = 0; i < nc; i++) { + if (!cand[i]) + continue; + if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, + put, get)) { + ch->userd_pa = cand[i]; + ch->userd_as = as[i]; + ch->userd_ok = true; + return 0; + } + } + return -ENOENT; +} + +/* + * Complete a channel's work-submission notification the way the GPU does. + * + * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot + * in-progress (status 0xffff, info32 = work-submit token) and then polls the + * notification memory for it to come back cleared - the GPU clearing it is + * the acknowledgement that the submission was picked up. The emulated GPU + * never did that, so libcuda's context setup spun forever. Write the cleared + * notification when the slot is busy. + * + * NvNotification layout (16 bytes): timestamp[2], info32, info16, status. + */ +#define SVC_NOTIFY_SIZE 16 +#define SVC_NOTIFY_STATUS_OFF 14 +#define SVC_NOTIFY_INFO32_OFF 8 +#define SVC_NOTIFY_TOKEN_SLOT 1 /* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */ + +static void svc_notify_complete(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0}; + u64 base = ch->notifier_pa; + u32 status = 0, info32 = 0; + int slot, busy = 0; + + if (!ch->notifier_pa) + return; + + /* Both the error slot (0) and the work-submit-token slot (1) must read + * as completed; the client polls one of them per submission. */ + for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) { + u64 pa = base + slot * SVC_NOTIFY_SIZE; + + status = 0; + info32 = 0; + if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF, + ch->notifier_as, &status, 2) || + svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF, + ch->notifier_as, &info32, 4)) + continue; + if (status || info32) + busy = 1; + } + if (!busy) + return; + + /* + * Clear both slots: they are adjacent, and whichever one the client is + * polling must read as completed for a healthy channel. + */ + if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) { + ch->notified++; + if (ch->notified <= 4 || (ch->notified % 10000) == 0) + pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n", + ch->h_object, info32, status, ch->notified); + + /* + * The GPU consumed the submission: post the channel's event so + * the client's CPU-side RM signals the OS event the submitting + * thread waits on (a real GPU would raise an interrupt here). + */ + gsp_rm_post_channel_event(vgpu, ch->h_object); + } } /* @@ -1284,18 +2239,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + int rrc; u32 put = 0, get = 0, guard = 0; - if (!ch->userd_pa) - continue; + svc_notify_complete(vgpu, ch); - if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, - ch->userd_as, &put, sizeof(put))) { - svc.gpfifo_nomap++; + if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; + + rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); + + if (!ch->userd_ok) { + if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, + rrc ? -1 : ring_ap, &put, &get)) + continue; + } else { + if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, + ch->userd_as, &put, sizeof(put))) + continue; + (void)svc_read_ap(vgpu, + ch->userd_pa + SVC_USERD_GP_GET, + ch->userd_as, &get, sizeof(get)); + } + + if (!ch->logged) { + pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n", + ch->h_object, ch->engine_type, ch->cid, + ch->gpfifo_va, + ch->gpfifo_entries, ch->userd_pa, ch->userd_as, + put, get, rrc, ring_pa, ring_ap); + ch->logged = true; } - (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, - ch->userd_as, &get, sizeof(get)); if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; @@ -1305,6 +2281,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 va, idx = ch->last_get; u32 len, e0, e1, g; + if (rrc) + break; /* cannot read the ring at all */ + if (svc_read_va(vgpu, ch->gpfifo_va + idx * SVC_GPFIFO_ENTRY_SIZE, @@ -1320,17 +2299,29 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* GP_ENTRY1.LENGTH[30:10], in dwords */ len = ((e1 >> 10) & 0x1fffff) << 2; + if (svc.gpfifo_entry_log < 32) { + pr_info("sema-chan: entry[%llu] raw=%08x%08x va=0x%llx len=%u\n", + idx, e1, e0, va, len); + svc.gpfifo_entry_log++; + } + if (va && len && len <= SVC_PUSH_MAX) { u8 *pb = kmalloc(len, GFP_KERNEL); if (pb) { - if (!svc_read_va(vgpu, va, pb, len)) { + if (!svc_read_va_trusted(vgpu, va, pb, + len)) { svc_decode_stream(vgpu, pb, len, 0); ch->submissions++; svc.gpfifo_entries_done++; } else { + /* Keep the entry pending: the + * mapping may resolve once the + * page tables are filled. */ svc.gpfifo_nomap++; + kfree(pb); + break; } kfree(pb); } @@ -1348,6 +2339,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) for (k = 0; k < SVC_MAX_CHANNELS; k++) { if (svc.chan[k].valid && svc.chan[k].h_object == ch->h_object) { + svc.chan[k].userd_pa = ch->userd_pa; + svc.chan[k].userd_as = ch->userd_as; + svc.chan[k].userd_ok = ch->userd_ok; + svc.chan[k].logged = ch->logged; + svc.chan[k].notified = ch->notified; svc.chan[k].last_get = ch->last_get; svc.chan[k].submissions = ch->submissions; break; @@ -1383,19 +2379,30 @@ static void svc_handle_probe(struct nv_vgpu_device *vgpu) probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.direct_base, svc.map_n); - if (svc_map_lookup(probe_va, &pa)) { - pr_info("sema-probe: NO MAP ENTRY for page 0x%llx\n", - probe_va & ~(u64)(SEMA_PAGE_SIZE - 1)); - } else { - pr_info("sema-probe: pa=0x%llx\n", pa); - for (i = 0; i < probe_n; i++) { - u32 v = 0; + { + int ap = SVC_APERTURE_SYS; - if (svc_read(vgpu, pa + 4ULL * i, &v, sizeof(v))) { - pr_info("sema-probe: [%u] read failed\n", i); - break; + if (svc_lookup(vgpu, probe_va, &pa, &ap)) { + pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n", + probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), + svc.walks_ok, svc.walks_ok + svc.walks_miss); + } else { + pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa, + ap == SVC_APERTURE_FB ? "vidmem" : "ram"); + for (i = 0; i < probe_n; i++) { + u32 v = 0; + + if (svc_read_ap(vgpu, pa + 4ULL * i, + ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS, + &v, sizeof(v))) { + pr_info("sema-probe: [%u] read failed\n", + i); + break; + } + pr_info("sema-probe: [%u] 0x%08x\n", i, v); } - pr_info("sema-probe: [%u] 0x%08x\n", i, v); } } @@ -1422,6 +2429,59 @@ int gsp_sema_service_probe(u64 gpu_va, u32 count) return 0; } +/* + * Debug injection: teach the service one {GPU VA -> guest PA} translation. + * Some client mappings (user VA == GPU VA, sysmem backed) are not + * discoverable from the structures the service scans; this lets a session + * feed a pair that was verified from the guest and watch push execution use + * it. + */ +static struct completion map_add_done; +static u64 map_add_va; +static u64 map_add_pa; +static u64 map_add_len; +static bool map_add_pending; + +static void svc_handle_map_add(void) +{ + u64 off; + + if (!map_add_pending) + return; + + for (off = 0; off < (map_add_len ? map_add_len : 1); + off += SEMA_PAGE_SIZE) { + svc_pin_store(map_add_va + off, map_add_pa + off); + svc_lat_store(map_add_va + off, map_add_pa + off, + SVC_APERTURE_SYS); + } + pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (sysmem)\n", + map_add_va, map_add_pa, map_add_len); + map_add_pending = false; + complete(&map_add_done); +} + +int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len) +{ + if (!svc.task) + return -ENODEV; + + if (len > (64ULL << 20)) + return -EINVAL; + + init_completion(&map_add_done); + map_add_va = gpu_va; + map_add_pa = pa; + map_add_len = len; + map_add_pending = true; + + if (!wait_for_completion_timeout(&map_add_done, 3 * HZ)) { + map_add_pending = false; + return -ETIMEDOUT; + } + return 0; +} + static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; @@ -1449,6 +2509,7 @@ static int svc_thread(void *arg) } svc_handle_probe(vgpu); + svc_handle_map_add(); if (!svc.direct_base || time_after(jiffies, svc.last_build + 10 * HZ)) @@ -1458,8 +2519,10 @@ static int svc_thread(void *arg) * Channel-driven execution: notice submissions as they are made * and run the pushed work immediately, so the guest's UVM * progress waits complete in milliseconds rather than in the - * time it takes the fallback scan to find the push. + * time it takes the fallback scan to find the push. A doorbell + * token means a channel was just scheduled; run everything now. */ + svc.doorbell_pending = false; svc_pump_channels(vgpu); /* @@ -1484,15 +2547,39 @@ static int svc_thread(void *arg) pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); + complete(&svc.stopped); return 0; } +/* + * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0 + * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has + * written the puller GPFIFO entry, so a pending token means "run this + * channel's entries now". Called from the VFIO write path (atomic context): + * record the token and let the service thread do the work. + */ +void gsp_sema_service_doorbell(u32 token) +{ + svc.doorbell_token = token; + svc.doorbell_count++; + svc.doorbell_pending = true; + + if (svc.doorbell_log < 64) { + pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n", + token, (token >> 16) & 0x7f, token & 0xfff, + svc.doorbell_count); + svc.doorbell_log++; + } +} + int gsp_sema_service_start(void) { if (svc.task) return 0; svc.stop = false; + init_completion(&svc.stopped); spin_lock_init(&svc.chan_lock); + spin_lock_init(&svc.vas_lock); svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); @@ -1503,11 +2590,18 @@ int gsp_sema_service_start(void) return 0; } +/* + * Stop the service thread without kthread_stop(): the thread can also end on + * its own if it ever faults, and joining a task that is already gone faults in + * the caller (which wedges the whole module). The completion tells us the + * thread function has returned, whatever ended it. + */ void gsp_sema_service_stop(void) { - if (svc.task) { - svc.stop = true; - kthread_stop(svc.task); - svc.task = NULL; - } + if (!svc.task) + return; + svc.stop = true; + if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) + pr_err("sema-svc: thread did not stop within 5s\n"); + svc.task = NULL; } diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 16a045f..a6893bd 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -22,19 +22,42 @@ struct sema_channel_desc { u32 h_object; u32 engine_type; + u32 cid; /* RM channel id (the doorbell token vector) */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; /* NvA06FControl: GPGet @0x88, GPPut @0x8c */ u32 userd_as; /* NV_ADDR_SYSMEM(1) / NV_ADDR_FBMEM(2) */ u64 instance_pa; u64 notifier_pa; + u32 notifier_as; /* NV_ADDR_SYSMEM(1) / NV_ADDR_FBMEM(2) */ }; void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); void gsp_sema_service_channel_del(u32 h_object); +/* + * A GPU VA space whose top-level page directory the driver has published (the + * driver builds its own page tables for the address spaces it owns, then hands + * the directory's physical address to the GPU through SET_PAGE_DIRECTORY). + * + * pdb is the directory base in the aperture selected by b_vidmem (false = + * guest RAM, true = emulated vidmem); num_entries is the root entry count and + * fixes the number of VA bits the root covers. + */ +struct sema_vaspace_desc { + u32 h_vaspace; + u64 pdb; + u32 num_entries; + bool b_vidmem; +}; + +void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); +void gsp_sema_service_vaspace_del(u32 h_vaspace); + int gsp_sema_service_start(void); void gsp_sema_service_stop(void); int gsp_sema_service_probe(u64 gpu_va, u32 count); +int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len); +void gsp_sema_service_doorbell(u32 token); #endif /* _GSP_SEMA_SERVICE_H_ */ diff --git a/kernel/main.c b/kernel/main.c index 84a05ca..05a49b1 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -791,12 +791,34 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use if (pos >= 0 && (u64)pos < vgpu->bars[0].size && count <= 8) { u8 tmp[8]; ssize_t ret; + static u32 bar0_log; if (!vgpu->bar_emul_ctx) return -ENODEV; if (copy_from_user(tmp, buf, count)) return -EFAULT; + if (bar0_log < 64 || (u64)pos == 0x88704) { + static u32 db_log; + u64 v = 0; + + memcpy(&v, tmp, count); + + if ((u64)pos == 0x88704) { + if (db_log < 512) { + pr_info("gsp-resman-vfio: DOORBELL write pos=0x%llx size=%zu val=0x%llx\n", + (u64)pos, count, v); + db_log++; + } + if (count == 4) + gsp_sema_service_doorbell((u32)v); + } else { + pr_info("gsp-resman-vfio: BAR0 write pos=0x%llx size=%zu val=0x%llx\n", + (u64)pos, count, v); + bar0_log++; + } + } + ret = nv_vgpu_bar_emul_write(vgpu, 0, (u64)pos, tmp, count); if (ret) return ret; @@ -809,6 +831,17 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use (u64)pos < (1ULL << 40) + vgpu->bars[1].size && vgpu->bars[1].vaddr && count <= 8) { u64 bar1_off = (u64)pos - (1ULL << 40); + static u32 bar1_log; + + if (bar1_log < 64) { + u64 v = 0; + + memcpy(&v, buf, count); + + pr_info("gsp-resman-vfio: BAR1 write off=0x%llx size=%zu val=0x%llx\n", + bar1_off, count, v); + bar1_log++; + } if (copy_from_user(vgpu->bars[1].vaddr + bar1_off, buf, count)) return -EFAULT; @@ -2214,6 +2247,41 @@ static const struct kernel_param_ops nv_vgpu_sema_probe_ops = { module_param_cb(sema_probe, &nv_vgpu_sema_probe_ops, NULL, 0200); MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA"); +/* + * Debug: inject a verified {GPU VA -> guest PA} translation into the + * semaphore service (used when the client's mapping cannot be discovered + * from guest structures): + * echo 0x,0x > .../parameters/sema_map_add + */ +static int nv_vgpu_sema_map_add_set(const char *val, const struct kernel_param *kp) +{ + char buf[64]; + char *comma, *comma2; + u64 va = 0, pa = 0, len = 0; + + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (!comma) + return -EINVAL; + *comma = '\0'; + comma2 = strchr(comma + 1, ','); + if (comma2) + *comma2 = '\0'; + if (kstrtoull(buf, 0, &va) || kstrtoull(comma + 1, 0, &pa)) + return -EINVAL; + if (comma2 && kstrtoull(comma2 + 1, 0, &len)) + return -EINVAL; + + return gsp_sema_service_add_map(va, pa, len); +} + +static const struct kernel_param_ops nv_vgpu_sema_map_add_ops = { + .set = nv_vgpu_sema_map_add_set, +}; + +module_param_cb(sema_map_add, &nv_vgpu_sema_map_add_ops, NULL, 0200); +MODULE_PARM_DESC(sema_map_add, "Inject a GPU VA -> guest PA mapping (hex va,hex pa)"); + /* * Debug: find page-table entries pointing at a physical address, in guest RAM * and emulated vidmem: @@ -2627,6 +2695,156 @@ static const struct kernel_param_ops nv_vgpu_dump_vidmem_ops = { module_param_cb(dump_vidmem, &nv_vgpu_dump_vidmem_ops, NULL, 0200); MODULE_PARM_DESC(dump_vidmem, "Dump emulated vidmem range (hex offset,hex len)"); +/* + * Debug: hex-dump or scan the BAR0 register shadow. Guest MMIO writes that go + * through the mapped shadow never reach the write callback, so this is the + * only way to observe registers the guest writes directly (e.g. the + * work-submission doorbell/notify registers, client region). + * echo 0x88700,0x100 > .../parameters/dump_bar0 + * echo 0x00040022 > .../parameters/scan_bar0 + */ +static int nv_vgpu_dump_bar0_set(const char *val, const struct kernel_param *kp) +{ + struct nv_vgpu_device *vgpu; + struct nv_vgpu_bar_emul_ctx *ctx; + unsigned long off = 0, len = 0; + char buf[64]; + char *comma; + + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (comma) + *comma = '\0'; + if (kstrtoul(buf, 0, &off)) + return -EINVAL; + if (comma && kstrtoul(comma + 1, 0, &len)) + return -EINVAL; + if (!len) + len = 0x100; + if (len > 0x1000) + len = 0x1000; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base; + unsigned long o; + + ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx; + if (!ctx || !ctx->bar0_shadow) + continue; + base = (u8 *)ctx->bar0_shadow; + if (off + len > ctx->bar0_shadow_size) + continue; + for (o = 0; o < len; o += 64) { + u32 n = min_t(unsigned long, len - o, 64); + + pr_info("gsp-resman-vfio: bar0[0x%lx..0x%lx]=%*phN\n", + off + o, off + o + n, (int)n, base + off + o); + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_dump_bar0_ops = { + .set = nv_vgpu_dump_bar0_set, +}; + +module_param_cb(dump_bar0, &nv_vgpu_dump_bar0_ops, NULL, 0200); +MODULE_PARM_DESC(dump_bar0, "Dump BAR0 shadow range (hex offset,hex len)"); + +static int nv_vgpu_scan_bar0_set(const char *val, const struct kernel_param *kp) +{ + struct nv_vgpu_device *vgpu; + struct nv_vgpu_bar_emul_ctx *ctx; + unsigned long needle = 0; + u32 hits = 0; + + if (kstrtoul(val, 0, &needle)) + return -EINVAL; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base; + size_t size, o; + + ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx; + if (!ctx || !ctx->bar0_shadow) + continue; + base = (u8 *)ctx->bar0_shadow; + size = ctx->bar0_shadow_size; + for (o = 0; o + 4 <= size && hits < 64; o += 4) { + u32 v; + + memcpy(&v, base + o, 4); + if (v == (u32)needle) { + pr_info("gsp-resman-vfio: bar0 scan: 0x%x at off 0x%zx\n", + v, o); + hits++; + } + } + pr_info("gsp-resman-vfio: bar0 scan: needle=0x%lx hits=%u\n", + needle, hits); + } + mutex_unlock(&nv_vgpu_driver.lock); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_scan_bar0_ops = { + .set = nv_vgpu_scan_bar0_set, +}; + +module_param_cb(scan_bar0, &nv_vgpu_scan_bar0_ops, NULL, 0200); +MODULE_PARM_DESC(scan_bar0, "Scan BAR0 shadow for a 32-bit value (hex)"); + +/* + * Debug: find channel USERD blocks in the emulated vidmem. A USERD page has a + * GPGet at +0x88 and a GPPut at +0x8c; report every page whose GPPut is a + * plausible entry index (1..65535) with GPGet <= GPPut. This is how the real + * vidmem offset of a channel's control block is discovered when the offset + * carried in the allocation parameters is relative to a pool the emulation + * cannot see. + */ +static int nv_vgpu_find_userd_set(const char *val, const struct kernel_param *kp) +{ + struct nv_vgpu_device *vgpu; + int shown = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + size_t off; + + if (!base || !size) + continue; + for (off = 0; off + 0x90 <= size; off += 4) { + u32 get, put; + + memcpy(&put, base + off + 0x8c, 4); + if (put < 1 || put > 65535) + continue; + memcpy(&get, base + off + 0x88, 4); + if (get > put) + continue; + if (shown < 32) + pr_info("gsp-resman-vfio: userd cand off=0x%zx get=%u put=%u\n", + off, get, put); + shown++; + } + pr_info("gsp-resman-vfio: userd scan: candidates=%d\n", shown); + } + mutex_unlock(&nv_vgpu_driver.lock); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_find_userd_ops = { + .set = nv_vgpu_find_userd_set, +}; + +module_param_cb(find_userd, &nv_vgpu_find_userd_ops, NULL, 0200); +MODULE_PARM_DESC(find_userd, "Find channel USERD blocks in emulated vidmem"); + /* * Debug: write a u32 into the emulated vidmem (BAR1 backing) at an offset: * echo 0x,0x > .../parameters/poke_vidmem @@ -2754,5 +2972,56 @@ static const struct kernel_param_ops nv_vgpu_dump_guest_ops = { module_param_cb(dump_guest, &nv_vgpu_dump_guest_ops, NULL, 0200); MODULE_PARM_DESC(dump_guest, "Hex-dump a guest physical range (hex gpa,hex len)"); +/* + * Debug: deliver one MSI-X vector to the guest. QEMU wires each vector's + * eventfd via VFIO_DEVICE_SET_IRQS; signaling it makes QEMU raise the guest + * interrupt. The emulated GPU consumes work without raising the interrupts a + * real GPU would, so sessions use this to drive the guest's interrupt-driven + * progress paths (channel/runlist events) and find out which vector an event + * needs. + */ +static int nv_vgpu_inject_irq_set(const char *val, const struct kernel_param *kp) +{ + struct nv_vgpu_device *vgpu; + unsigned long idx = 0; + int ret = -ENODEV; + + if (kstrtoul(val, 0, &idx) || idx >= NV_VGPU_MAX_IRQS) + return -EINVAL; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + struct eventfd_ctx *trigger; + + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + continue; + + if (vgpu->bar_emul_ctx && idx < NV_VGPU_MAX_IRQS) { + u32 bit = 1u << (idx % 32); + + nv_vgpu_bar_emul_write(vgpu, 0, + GSP_EMU_MSIX_PBA_OFFSET + + (idx / 32) * 4, + &bit, sizeof(bit)); + } + + trigger = vgpu->irqs[idx].trigger; + if (trigger) + eventfd_signal(trigger); + pr_info("gsp-resman-vfio: inject_irq %lu (trigger %s)\n", + idx, trigger ? "fired" : "missing"); + ret = 0; + } + mutex_unlock(&nv_vgpu_driver.lock); + return ret; +} + +static const struct kernel_param_ops nv_vgpu_inject_irq_ops = { + .set = nv_vgpu_inject_irq_set, +}; + +module_param_cb(inject_irq, &nv_vgpu_inject_irq_ops, NULL, 0200); +MODULE_PARM_DESC(inject_irq, "Deliver MSI-X vector N to the guest (debug)"); + module_init(nv_vgpu_init); module_exit(nv_vgpu_exit); diff --git a/scripts/create_mdev.sh b/scripts/create_mdev.sh index a6ea91e..4f1cc1c 100755 --- a/scripts/create_mdev.sh +++ b/scripts/create_mdev.sh @@ -5,10 +5,10 @@ set -euo pipefail PCI_ADDR="${1:-}" -VGPU_TYPE="${2:-pci-nvidia-vgpu-turing}" +VGPU_TYPE="${2:-nvidia-nvidia-vgpu-turing}" -# Supported vGPU types (matching the mdev type names) -VALID_TYPES=("pci-nvidia-vgpu-ampere" "pci-nvidia-vgpu-hopper" "pci-nvidia-vgpu-turing" "pci-nvidia-vgpu-pascal") +# Supported vGPU types (matching the mdev type names the driver registers) +VALID_TYPES=("nvidia-nvidia-vgpu-ampere" "nvidia-nvidia-vgpu-hopper" "nvidia-nvidia-vgpu-turing" "nvidia-nvidia-vgpu-pascal") # Find NVIDIA GPU if not specified if [[ -z "$PCI_ADDR" ]]; then diff --git a/tools/guest-diag/cycle_cdt.sh b/tools/guest-diag/cycle_cdt.sh new file mode 100755 index 0000000..009bdfc --- /dev/null +++ b/tools/guest-diag/cycle_cdt.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# cycle_cdt.sh - full test cycle: push tools, start cdt, inject the libcuda +# tracker page mapping when the tracker wait is reached, poke UVM's tracker +# when it stalls, and report where cdt ends up. +set -u +cd "$(dirname "$(readlink -f "$0")")/../.." + +echo "[*] waiting for guest" +for i in $(seq 1 30); do + timeout 20 ./scripts/vgpu_vm.py exec --timeout 12 'uptime >/dev/null 2>&1 && echo READY' 2>/dev/null | grep -q READY && break + sleep 5 +done + +./tools/guest-diag/push_guest_tools.sh >/tmp/opencode/push.log 2>&1 || { echo "PUSH FAILED"; tail -3 /tmp/opencode/push.log; exit 1; } +echo "[*] tools pushed" + +timeout 40 ./scripts/vgpu_vm.py exec --timeout 35 'sudo nvidia-modprobe -c 0; sudo nvidia-modprobe -u; sudo modprobe nvidia-uvm; sudo sh -c "echo 1 > /sys/module/nvidia_uvm/parameters/uvm_debug_prints"; sudo rm -f /tmp/cdt.out; sudo setsid /tmp/cdt > /tmp/cdt.out 2>&1 & sleep 2; tail -1 /tmp/cdt.out' 2>/dev/null | tail -1 +echo "[*] cdt started" + +# wait for the libcuda tracker wait and inject the page mapping +SEEN_CTX=0 +for i in $(seq 1 24); do + sleep 10 + STATE=$(timeout 20 ./scripts/vgpu_vm.py exec --timeout 15 'tail -1 /tmp/cdt.out' 2>/dev/null | tail -1) + echo " [$((i*10))s] $STATE" + if echo "$STATE" | grep -q "cuCtxCreate"; then + SEEN_CTX=1 + if [ ! -f /tmp/opencode/injected_tracker ]; then + if /tmp/opencode/inject_tracker.sh >/tmp/opencode/inject.log 2>&1; then + touch /tmp/opencode/injected_tracker + echo " [*] tracker mapping injected: $(tail -1 /tmp/opencode/inject.log)" + fi + fi + fi + if echo "$STATE" | grep -qE "FAILED|ALL OK|cuMemAlloc"; then + echo "[*] cuCtxCreate finished: $STATE" + break + fi +done + +# if UVM's tracking semaphore lags, poke it forward +SEMAPA=$(sudo sh -c 'echo 0x121000010,1 > /sys/module/open_iov_vfio/parameters/sema_probe' 2>/dev/null; sleep 2; sudo dmesg -T | grep -A2 "sema-probe: va=0x121000010" | grep -oE "pa=0x[0-9a-f]+" | head -1 | cut -d= -f2) +if [ -n "$SEMAPA" ]; then + CUR=$(sudo dmesg -T | grep -A3 "sema-probe: va=0x121000010" | grep -oE "\[0\] 0x[0-9a-f]+" | tail -1 | awk '{print $2}') + echo "[*] UVM tracking sema pa=$SEMAPA value=$CUR" +fi diff --git a/tools/guest-diag/guest_pmap.py b/tools/guest-diag/guest_pmap.py new file mode 100644 index 0000000..96b7ddc --- /dev/null +++ b/tools/guest-diag/guest_pmap.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# guest-side helper: translate GPU/user VAs of a process through its pagemap. +# Usage: sudo python3 guest_pmap.py [hex-va ...] +import os +import sys + +pid = int(sys.argv[1]) +vas = [int(x, 16) for x in sys.argv[2:]] + +maps = [] +try: + with open('/proc/%d/maps' % pid) as f: + for line in f: + parts = line.split() + a, b = parts[0].split('-') + maps.append((int(a, 16), int(b, 16), line.strip())) +except OSError as e: + print('MAPS_ERROR: %s' % e) + +def in_maps(va): + for a, b, line in maps: + if a <= va < b: + return line + return None + +fd = os.open('/proc/%d/pagemap' % pid, os.O_RDONLY) +try: + for va in vas: + os.lseek(fd, (va // 4096) * 8, 0) + e = int.from_bytes(os.read(fd, 8), 'little') + present = (e >> 63) & 1 + pfn = e & ((1 << 55) - 1) + pa = (pfn << 12) | (va & 0xfff) + line = in_maps(va) + print('VA=0x%x present=%d PA=0x%x map=%s' % (va, present, pa, line if line else '-')) +finally: + os.close(fd) diff --git a/tools/guest-diag/inject_tracker.sh b/tools/guest-diag/inject_tracker.sh new file mode 100755 index 0000000..a563b0d --- /dev/null +++ b/tools/guest-diag/inject_tracker.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# inject_tracker.sh - find the libcuda tracker slot the stuck cdt is polling, +# translate its page through the guest pagemap, pin the mapping in the service. +set -u +cd "$(dirname "$(readlink -f "$0")")/../.." + +OUT=$(timeout 100 ./scripts/vgpu_vm.py exec --timeout 90 'PID=$(pgrep -x cdt | head -1); B=$(sudo grep libcuda.so.570.144 /proc/$PID/maps | head -1 | cut -d- -f1); sudo timeout -s KILL 70 gdb -p $PID -batch -ex "set pagination off" -ex "set confirm off" -ex "break *(long)(0x$B+0x3c7970)" -ex "continue" -ex "set \$p2 = *(unsigned long*)(\$rdi+0x20)" -ex "set \$vp = *(unsigned long*)(\$p2+0x10)" -ex "printf \"VALPTR=0x%lx EXPECTED=%lu PID=%d\n\", \$vp, \$rsi, $PID" -ex "detach" 2>&1; sudo kill -CONT $PID' 2>/dev/null) +LINE=$(echo "$OUT" | grep "VALPTR=" | head -1) +VALPTR_HEX=$(echo "$LINE" | sed -n 's/.*VALPTR=0x\([0-9a-f]*\).*/\1/p') +PID=$(echo "$LINE" | sed -n 's/.*PID=\([0-9]*\).*/\1/p') +[ -n "$VALPTR_HEX" ] || { echo "NO VALPTR FOUND"; echo "$OUT" | tail -3; exit 1; } +echo "valptr=0x$VALPTR_HEX pid=$PID" + +PA=$(timeout 40 ./scripts/vgpu_vm.py exec --timeout 35 "sudo python3 -c \" +import os +pid=$PID +va=0x$VALPTR_HEX +fd=os.open('/proc/%d/pagemap'%pid, os.O_RDONLY) +os.lseek(fd, (va//4096)*8, 0) +e=int.from_bytes(os.read(fd,8),'little') +os.close(fd) +pfn=e & ((1<<55)-1) +print('PA=0x%x'%((pfn<<12)|(va&0xfff))) +\"" 2>/dev/null | grep -o 'PA=0x[0-9a-f]*' | head -1 | sed 's/^PA=//') +[ -n "$PA" ] || { echo "NO PA FOUND"; exit 1; } +echo "target pa=$PA" + +sudo sh -c "echo 0x$VALPTR_HEX,$PA > /sys/module/open_iov_vfio/parameters/sema_map_add" || exit 1 +sleep 1 +sudo dmesg -T | grep "injected mapping" | tail -1 diff --git a/tools/guest-diag/pmap_va.sh b/tools/guest-diag/pmap_va.sh new file mode 100755 index 0000000..e5bcef4 --- /dev/null +++ b/tools/guest-diag/pmap_va.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# pmap_va.sh [hex-va ...] +# Push guest_pmap.py to the running VM and translate the VAs of the named +# process through the guest's own pagemap (root in the guest). +set -u +cd "$(dirname "$0")/../.." + +TARGET="$1"; shift +VAS="$*" + +B64=$(base64 -w0 tools/guest-diag/guest_pmap.py) +timeout 30 ./scripts/vgpu_vm.py exec --timeout 25 \ + "echo '$B64' | base64 -d > /tmp/pmap.py" >/dev/null 2>&1 + +if ! [[ "$TARGET" =~ ^[0-9]+$ ]]; then + PID=$(timeout 20 ./scripts/vgpu_vm.py exec --timeout 15 "pgrep -x $TARGET | head -1" 2>/dev/null | tail -1) + [ -n "$PID" ] && TARGET="$PID" +fi +[ -n "$TARGET" ] || { echo "no pid found"; exit 1; } + +timeout 60 ./scripts/vgpu_vm.py exec --timeout 55 "sudo python3 /tmp/pmap.py $TARGET $VAS" 2>/dev/null | grep -E '^VA=' diff --git a/tools/guest-diag/run_traced.sh b/tools/guest-diag/run_traced.sh new file mode 100755 index 0000000..e8d3a09 --- /dev/null +++ b/tools/guest-diag/run_traced.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# run_traced.sh - run cdt under the ioctl tracer, keep the libcuda tracker page +# mapped (pagemap + sema_map_add) and poke UVM's tracking semaphore forward as +# the sweep completes pushes, so the run reaches the rc=999 rollback where the +# tracer records the failing driver call. +set -u +cd "$(dirname "$(readlink -f "$0")")/../.." + +EXEC() { timeout 30 ./scripts/vgpu_vm.py exec --timeout 25 "$1" 2>/dev/null; } + +EXEC 'sudo pkill -9 -x cdt; sudo pkill -9 -x ptio; sleep 1; sudo rm -f /tmp/cdt.out /tmp/tr.log; sudo setsid /tmp/ptio /tmp/cdt > /tmp/cdt.out 2> /tmp/tr.log & sleep 3; pgrep -x cdt | head -1' + +for i in $(seq 1 40); do + sleep 10 + PID=$(EXEC 'pgrep -x cdt | head -1' | tail -1) + [ -n "$PID" ] || { echo "[*] cdt gone at $((i*10))s"; break; } + + # pin the tracker page when the guest has it mapped + PA=$(EXEC "sudo python3 -c \" +import os +fd=os.open('/proc/$PID/pagemap', os.O_RDONLY) +os.lseek(fd, (0x20480f000//4096)*8, 0) +e=int.from_bytes(os.read(fd,8),'little') +os.close(fd) +pfn=e & ((1<<55)-1) +print('PA=0x%x'%((pfn<<12)|0xeb0)) +\"" | grep -o 'PA=0x[0-9a-f]*' | head -1 | sed 's/PA=//') + if [ -n "$PA" ]; then + sudo sh -c "echo 0x20480feb0,$PA > /sys/module/open_iov_vfio/parameters/sema_map_add" 2>/dev/null + fi + + # nudge UVM's tracking semaphore forward when the sweep has stalled + sudo sh -c 'echo 0x121000010,1 > /sys/module/open_iov_vfio/parameters/sema_probe' 2>/dev/null + sleep 1 + VALS=$(sudo dmesg -T | grep -A4 "sema-probe: va=0x121000010" | tail -4) + SPA=$(echo "$VALS" | grep -oE "pa=0x[0-9a-f]+" | head -1 | cut -d= -f2) + CUR=$(echo "$VALS" | grep -oE "\[0\] 0x[0-9a-f]+" | head -1 | awk '{print $2}') + if [ -n "$SPA" ] && [ -n "$CUR" ]; then + NEXT=$(printf '0x%x' $(( CUR + 1 ))) + sudo sh -c "echo 0x$SPA,$NEXT > /sys/module/open_iov_vfio/parameters/poke_guest" 2>/dev/null + echo " [$((i*10))s] cdt=$PID sema $CUR -> $NEXT" + else + echo " [$((i*10))s] cdt=$PID (sema unresolved)" + fi + + STATE=$(EXEC 'tail -1 /tmp/cdt.out' | tail -1) + case "$STATE" in + *FAILED*|*"ALL OK"*|*cuMemAlloc*) echo "[*] finished: $STATE"; break ;; + esac +done + +echo "=== trace tail ===" +EXEC 'tail -25 /tmp/tr.log' diff --git a/tools/guest-diag/vmrestart.sh b/tools/guest-diag/vmrestart.sh new file mode 100755 index 0000000..dbe60ec --- /dev/null +++ b/tools/guest-diag/vmrestart.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# vmrestart.sh - stop QEMU, reload open-iov-vfio.ko from the tree, fresh mdev, +# relaunch the vGPU VM. Everything bounded so a stuck step cannot hang the +# caller. +set -u +cd "$(dirname "$(readlink -f "$0")")/../.." + +sudo pkill -f qemu-system-x86_64 2>/dev/null +sleep 4 +sudo rm -f /tmp/qemu-guest-agent.sock /tmp/qemu-qmp.sock + +for m in /sys/bus/mdev/devices/*; do + [ -d "$m" ] || continue + timeout 20 sudo sh -c "echo 1 > $m/remove" || echo "WARN: mdev remove timeout" +done + +timeout 30 sudo rmmod open_iov_vfio || { echo "ERROR: rmmod failed"; exit 1; } +sudo insmod kernel/open-iov-vfio.ko || { echo "ERROR: insmod failed"; exit 1; } + +U=$(uuidgen) +echo "$U" | sudo tee /sys/class/mdev_bus/0000:01:00.0/mdev_supported_types/nvidia-nvidia-vgpu-turing/create >/dev/null +sed -i "s|/sys/bus/mdev/devices/[0-9a-f-]*|/sys/bus/mdev/devices/$U|" tools/guest-diag/launch_qemu_rootport.run.sh +mkdir -p /tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06 + +(setsid nohup tools/guest-diag/launch_qemu_rootport.run.sh > tools/guest-diag/run/qemu.log 2>&1 &) +sleep 6 +if pgrep -f qemu-system-x86_64 >/dev/null; then + echo "VM up (mdev $U)" +else + echo "VM FAILED"; tail -5 tools/guest-diag/run/qemu.log; exit 1 +fi -- cgit v1.2.3