diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 1218 |
1 files changed, 1156 insertions, 62 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 2f453fa..10d6b80 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -43,6 +43,7 @@ #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" +#include "gsp_rm_emulation.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 @@ -82,6 +83,18 @@ #define HOST_SEM_OP_RELEASE 1 #define HOST_SEM_OP_REDUCTION 6 +/* + * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other + * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared + * by the TURING and later classes. RELEASE (0) ends the push. + */ +#define HOST_M_REPORT_SEM_A 0x1b00 +#define HOST_M_REPORT_SEM_B 0x1b04 +#define HOST_M_REPORT_SEM_C 0x1b08 +#define HOST_M_REPORT_SEM_D 0x1b0c +#define HOST_RS_OP_MASK 0x3 +#define HOST_RS_OP_RELEASE 0 + struct sema_map_entry { u64 gpu_page; /* GPU VA, page aligned */ u64 pa_page; /* guest PA, page aligned */ @@ -91,7 +104,7 @@ struct sema_map_entry { /* registered channels (GPFIFO-driven execution) */ /* ------------------------------------------------------------------ */ -#define SVC_MAX_CHANNELS 24 +#define SVC_MAX_CHANNELS 128 #define SVC_GPFIFO_ENTRY_SIZE 8 #define SVC_PUSH_MAX (256 * 1024) @@ -103,12 +116,17 @@ struct svc_channel { bool valid; u32 h_object; u32 engine_type; + u32 cid; /* RM channel id (doorbell token vector) */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ + bool userd_ok; /* userd_pa reads a plausible put/get */ + bool logged; /* one state log per channel */ u64 instance_pa; - u64 notifier_pa; + u64 notifier_pa; /* error notifier: NvNotification slots */ + u32 notifier_as; + u64 notified; /* work-submit-token notifications completed */ u32 last_get; /* next GPFIFO entry we have still to run */ u64 submissions; /* GPFIFO entries executed */ }; @@ -122,6 +140,37 @@ struct svc_channel { static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)]; static u32 svc_map_hash[SEMA_MAP_MAX * 2]; +/* VA spaces whose page directory the driver published / learned VA->PA pairs. */ +#define SVC_MAX_VASPACES 8 +#define SVC_LAT_MAX 65536 + +/* + * Pinned VA->PA translations: translations a session has verified by hand + * (debug injection) and that must not be evicted by the learned cache, which + * is content-addressed and gets thrashed by the lookup stream. + */ +#define SVC_PIN_SLOTS 4096 + +struct svc_pin { + u64 va_page; + u64 pa_page; +}; + +struct svc_vaspace { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; /* root entry count */ + int ap; /* SVC_APERTURE_* of the directory itself */ +}; + +struct svc_lat { + bool valid; + u64 va; + u64 addr; + int ap; /* SVC_APERTURE_* */ +}; + static void svc_mark_hot(u64 pa) { u64 page = pa >> SEMA_PAGE_SHIFT; @@ -133,6 +182,7 @@ static void svc_mark_hot(u64 pa) struct sema_svc { struct task_struct *task; bool stop; + struct completion stopped; u64 direct_base; u32 map_n; @@ -144,7 +194,17 @@ struct sema_svc { spinlock_t chan_lock; struct svc_channel chan[SVC_MAX_CHANNELS]; + /* VA spaces whose page directory the driver published */ + spinlock_t vas_lock; + struct svc_vaspace vas[SVC_MAX_VASPACES]; + struct svc_lat lat[SVC_LAT_MAX]; + struct svc_pin pin[SVC_PIN_SLOTS]; + u32 pin_n; + /* stats */ + u64 walks_ok; + u64 walks_miss; + u64 lat_hits; u64 pairs_seen; u64 pages_decoded; u64 pushes_seen; @@ -162,6 +222,12 @@ struct sema_svc { u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ u64 vidmem_writes; /* semaphores written in vidmem */ + u64 gpfifo_log; /* channel-state log budget */ + u64 gpfifo_entry_log; /* per-entry log budget */ + u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */ + u32 doorbell_token; /* last token written */ + u32 doorbell_log; /* token log budget */ + bool doorbell_pending; /* pump on next service pass */ }; static struct sema_svc svc; @@ -249,19 +315,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->valid = true; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; + ch->cid = desc->cid; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; + ch->notifier_as = desc->notifier_as ? desc->notifier_as + : SVC_APERTURE_SYS; } spin_unlock_irqrestore(&svc.chan_lock, flags); if (ch) - pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx\n", + pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, - desc->userd_pa); + desc->userd_pa, desc->engine_type, desc->cid, + desc->notifier_pa); + else + pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n", + desc->h_object, SVC_MAX_CHANNELS); } void gsp_sema_service_channel_del(u32 h_object) @@ -285,30 +358,604 @@ void gsp_sema_service_channel_del(u32 h_object) } /* + * The driver publishes the page directory of a GPU VA space it owns (it builds + * the page tables itself and hands their base address to the GPU). Store it so + * the service can translate that space's VAs directly. + */ +void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) +{ + struct svc_vaspace *vs = NULL; + unsigned long flags; + int i; + + if (!desc || !desc->pdb || !desc->num_entries) + return; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) { + vs = &svc.vas[i]; + break; + } + if (!vs && !svc.vas[i].valid) + vs = &svc.vas[i]; + } + if (vs) { + vs->valid = true; + vs->h_vaspace = desc->h_vaspace; + vs->pdb = desc->pdb; + vs->entries = desc->num_entries; + vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + /* A new directory invalidates what we learned from the old one. */ + memset(svc.lat, 0, sizeof(svc.lat)); + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + + if (vs) + pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n", + desc->h_vaspace, desc->pdb, desc->num_entries, + desc->b_vidmem ? "vidmem" : "sysmem"); +} + +void gsp_sema_service_vaspace_del(u32 h_vaspace) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) { + svc.vas[i].valid = false; + memset(svc.lat, 0, sizeof(svc.lat)); + pr_info("sema-svc: VA space 0x%x page directory dropped\n", + h_vaspace); + break; + } + } + spin_unlock_irqrestore(&svc.vas_lock, flags); +} + +/* * The client's vidmem VAs are its window base plus the vidmem offset, and the * backing store is the emulated BAR1 buffer. Everything else that resolves * goes through the discovered guest-RAM map. + * + * The window base is not 0x200000000: the observed layout places the first + * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame + * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000. + * Verified against the guest's own view of its GPFIFO rings and USERD. */ -#define SVC_VIDMEM_VA_BASE 0x200000000ULL +#define SVC_VIDMEM_VA_BASE 0x200080000ULL +#define SVC_VIDMEM_VA_LO 0x200000000ULL /* anything in the window below + * the base is still vidmem, but + * never maps into the buffer */ static int svc_map_lookup(u64 gpu_va, u64 *pa); -static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, - u32 *aperture) +/* ------------------------------------------------------------------ */ +/* client page-table walk */ +/* ------------------------------------------------------------------ */ +/* + * The driver builds its own page tables for the GPU VA spaces it owns and + * publishes each top-level directory through SET_PAGE_DIRECTORY. With those + * directories the service can translate the client's GPU VAs directly - the + * authoritative answer, and independent of any heuristic discovery. + * + * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere: + * + * PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21] + * dual entry (16 bytes): + * big half -> PT_BIG [20:16] + * small half -> PT_4K [20:12] + * + * A directory entry carries the aperture in bits 2:1 (1 = video memory, + * 2 = system coherent, 3 = system non-coherent) and the child address in bits + * 53:8 at 4 KB granularity (bits 53:4 at 256 B granularity in the big half of + * a dual entry). Bit 0 of a leaf entry is VALID; bit 0 of a directory entry is + * IS_PTE, so a set bit means the level terminates in a page here. + */ +#define SVC_MMU_VA_BITS 49 +#define SVC_MMU_AP_VID 1 /* in directory entries */ +#define SVC_MMU_AP_SYS_COH 2 +#define SVC_MMU_AP_SYS_NC 3 +#define SVC_MMU_PTE_AP_VID 0 /* in page table entries */ +#define SVC_MMU_PTE_AP_PEER 1 +#define SVC_MMU_PTE_AP_SYS_COH 2 +#define SVC_MMU_PTE_AP_SYS_NC 3 + +static u64 svc_pte_addr(u64 e, bool vidmem) { - if (!svc_map_lookup(va, addr)) { - *aperture = 0; /* guest RAM */ + if (vidmem) + return (e & 0x00000001FFFFFFFF00ULL); + return (e & 0x003FFFFFFFFF00ULL); +} + +/* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */ +static int svc_pte_aperture(u32 ap) +{ + if (ap == SVC_MMU_PTE_AP_VID) + return SVC_APERTURE_FB; + if (ap == SVC_MMU_PTE_AP_PEER) + return -ENODEV; + return SVC_APERTURE_SYS; +} + +static int svc_pde_aperture(u32 ap) +{ + if (ap == SVC_MMU_AP_VID) + return SVC_APERTURE_FB; + if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC) + return SVC_APERTURE_SYS; + return -ENODEV; +} + +/* + * A translated address is only usable if it lies inside the aperture that + * backs it: guest RAM up to the module's RAM limit for system memory, and the + * emulated vidmem buffer for video memory. Rejecting out-of-range results also + * stops a bogus page-table chain from being mistaken for a translation. + */ +static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap) +{ + if (ap == SVC_APERTURE_FB) + return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size; + return addr + 8 <= SEMA_PA_MAX; +} + +static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off, + int ap, u64 *out) +{ + if (!svc_ap_ok(vgpu, table + off, ap)) + return -ENOENT; + if (svc_read_ap(vgpu, table + off, + ap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + out, sizeof(*out))) + return -ENOENT; + return 0; +} + +/* + * Leaf lookup under a table: read the PTE at `index`, validate it and return + * the backing address plus its aperture. + */ +static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap, + u32 index, u64 *addr, int *ap) +{ + u64 e; + int eap; + + if (svc_read_ent(vgpu, table, 8 * index, tap, &e)) + return -ENOENT; + if (!(e & 1)) /* VALID */ + return -ENOENT; + eap = svc_pte_aperture((e >> 1) & 0x3); + if (eap < 0) + return -ENOENT; + *addr = svc_pte_addr(e, eap == SVC_APERTURE_FB); + if (!svc_ap_ok(vgpu, *addr, eap)) + return -ENOENT; + *ap = eap; + return 0; +} + +/* + * Walk the four-level page table rooted at `pdb` for one VA. Directory levels + * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry + * is the 16-byte dual entry (big half first, small half second). + */ +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace) +{ + u64 table = pdb; + u32 root_bits, idx, level; + u64 big, small; + int sap, bap; + + root_bits = ilog2(entries ? entries : 1); + if (root_bits < 1 || root_bits > 12) + return -ENOENT; + + if (!table) + return -ENOENT; + + for (level = 0; level < 3; level++) { + u64 e; + u32 bits, shift; + int eap; + + if (level == 0) { + bits = root_bits; + shift = SVC_MMU_VA_BITS - root_bits; + } else { + bits = 9; + shift = 29 + 9 * (2 - level); + } + idx = (u32)((va >> shift) & ((1u << bits) - 1)); + if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) { + if (trace) + pr_info("sema-walk: L%u read fail table=0x%llx idx=%u\n", + level, table, idx); + return -ENOENT; + } + if (trace) + pr_info("sema-walk: L%u e=0x%016llx idx=%u\n", + level, e, idx); + /* A page at a directory level would cover at least 2^38 bytes; + * the driver never maps those, so treat it as a false chain. */ + if (e & 1) + return -ENOENT; + eap = svc_pde_aperture((e >> 1) & 0x3); + if (eap < 0) + return -ENOENT; + tap = eap; + table = svc_pte_addr(e, eap == SVC_APERTURE_FB); + if (!svc_ap_ok(vgpu, table, eap)) + return -ENOENT; + } + + /* PD0: one dual entry per index (big half then small half). */ + idx = (va >> 21) & 0xff; + if (svc_read_ap(vgpu, table + 16 * idx, + tap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + &big, sizeof(big)) || + svc_read_ap(vgpu, table + 16 * idx + 8, + tap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + &small, sizeof(small))) + return -ENOENT; + + /* Small half: 4 KB page table under it. */ + sap = svc_pde_aperture(((small >> 1) & 0x3)); + if (sap >= 0 && !(small & 1)) { + u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB); + + if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff, + addr, ap)) { + *addr += va & 0xfff; + return 0; + } + } + + /* Big half: a 2 MB page here, or a large (64 KB) page table. */ + if (big & 1) { + bap = svc_pte_aperture((big >> 1) & 0x3); + if (bap >= 0) { + *addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) + + (va & ((1ULL << 21) - 1)); + *ap = bap; + return 0; + } + } + bap = svc_pde_aperture((big >> 1) & 0x3); + if (bap >= 0) { + u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB); + + if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f, + addr, ap)) { + *addr += va & 0xffff; + return 0; + } + } + return -ENOENT; +} + +/* + * Page directories the driver published (SET_PAGE_DIRECTORY). + */ +static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, + u64 *addr, int *ap) +{ + unsigned long flags; + int i, rc = -ENOENT; + bool trace = svc.walks_ok + svc.walks_miss < 24; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + int alt; + + if (!svc.vas[i].valid) + continue; + if (va >> SVC_MMU_VA_BITS) + break; + if (trace) + pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n", + i, va, svc.vas[i].pdb, svc.vas[i].entries); + /* + * Try the aperture the driver named first, then the other one: + * the aperture encoding has changed between driver generations + * and a wrong guess only costs a rejected walk. + */ + alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; + if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, + svc.vas[i].ap, addr, ap, trace) || + !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, + alt, addr, ap, trace)) { + svc.walks_ok++; + rc = 0; + break; + } + } + if (rc) + svc.walks_miss++; + spin_unlock_irqrestore(&svc.vas_lock, flags); + return rc; +} + +/* + * Self-discovering the page directory. + * + * The published directories are not always usable (a guest can publish a + * directory location the host never gets to see written), so as a fallback + * find the directory by verification: take pairs of GPU VAs whose backing + * guest page is already known (the discovered VA->PA map), and accept the + * first guest page that, used as a top-level directory, reproduces several of + * those translations exactly. A wrong candidate cannot pass that test, so no + * layout assumption is needed to trust the result; only the entry encoding + * (which the walk itself applies consistently) is shared. + */ +static u64 svc_pd_base; +static int svc_pd_ap; +static u32 svc_pd_entries; +static u8 svc_pd_tried; +static u8 svc_pd_found; +#define SVC_PD_SAMPLES 8 +#define SVC_PD_MIN_HITS 4 + +static u32 svc_pd_cands; +static u32 svc_pd_best; + +static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand, + int ap, const u64 *sva, const u64 *spa, + u32 nsamp) +{ + u32 i, hits = 0; + + svc_pd_cands++; + + for (i = 0; i < nsamp; i++) { + u64 a = 0; + int eap = 0; + + if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) && + (a & ~0xfffULL) == (spa[i] & ~0xfffULL)) + hits++; + } + if (hits > svc_pd_best) + svc_pd_best = hits; + if (hits >= SVC_PD_MIN_HITS) { + svc_pd_base = cand; + svc_pd_ap = ap; + svc_pd_entries = 4; + svc_pd_found = 1; + pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n", + cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram", + hits, nsamp); + } +} + +static void svc_pd_discover(struct nv_vgpu_device *vgpu) +{ + u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES]; + u32 i, n = 0, step; + u64 chunk = 64 * 1024; + u8 *buf; + int pass; + + if (svc_pd_tried) + return; + svc_pd_tried = 1; + + if (svc.map_n < SVC_PD_SAMPLES) + return; + step = svc.map_n / SVC_PD_SAMPLES; + for (i = 0; i < SVC_PD_SAMPLES; i++) { + sva[i] = svc.map[i * step].gpu_page; + spa[i] = svc.map[i * step].pa_page; + if (sva[i] >> SVC_MMU_VA_BITS) + return; /* outside the walkable VA range */ + n++; + } + if (!n) + return; + + /* Guest RAM first, then the emulated vidmem. Page directories are page + * aligned, and every VA here is below 2^47, so the root entry that + * matters is the first one of the page. */ + for (pass = 0; pass < 2 && !svc_pd_found; pass++) { + int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB; + u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size; + u8 *fb = (u8 *)vgpu->bars[1].vaddr; + u64 gpa; + + buf = kmalloc(chunk, GFP_KERNEL); + if (!buf) + return; + for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) { + u32 len = (u32)min_t(u64, chunk, limit - gpa); + u64 off; + + if (pass == 0) { + if (svc_read(vgpu, gpa, buf, len)) + continue; + } else { + if (!fb) + break; + memcpy(buf, fb + gpa, len); + } + for (off = 0; off + 8 <= len && !svc_pd_found; + off += SEMA_PAGE_SIZE) { + u64 e; + + memcpy(&e, buf + off, sizeof(e)); + if (!(e & 1) && ((e >> 1) & 0x3)) + svc_pd_try_candidate(vgpu, gpa + off, + ap, sva, spa, n); + } + } + kfree(buf); + } + pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n", + svc_pd_found ? "found" : "not found", svc_pd_base, + svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram", + svc_pd_cands, svc_pd_best, n); +} + +static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va, + u64 *addr, int *ap) +{ + if (!svc_pd_found) + return -ENOENT; + return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, + svc_pd_ap, addr, ap, false); +} + +/* Learned translations: a walk costs a handful of guest reads, so cache the + * successes and answer repeats without touching the page tables again. A cache + * line covers a whole page, so arrays of semaphores in one page all resolve. */ +static int svc_lat_lookup(u64 va, u64 *addr, int *ap) +{ + struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; + + if (l->valid && + (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (va & ~(u64)(SEMA_PAGE_SIZE - 1))) { + *addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + *ap = l->ap; + svc.lat_hits++; + return 0; + } + return -ENOENT; +} + +static void svc_lat_store(u64 va, u64 addr, int ap) +{ + struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; + + l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1); + l->ap = ap; + l->valid = true; +} + +static int svc_pin_lookup(u64 va, u64 *pa) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < 8; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page) + return -ENOENT; + if (p->va_page == page) { + *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); + return 0; + } + } + return -ENOENT; +} + +static void svc_pin_store(u64 va, u64 pa) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < 8; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page || p->va_page == page) { + if (!p->va_page) + svc.pin_n++; + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + return; + } + } +} + +/* + * Translate a GPU VA using trusted sources only: a pinned translation, the + * learned cache, the discovered map, the directories the driver published and + * the self-discovered directory. The final vidmem window convention is + * deliberately NOT part of this path: it answers with a vidmem offset for any + * VA above the window base, including sysmem-backed VAs, so callers that must + * not act on a guess (push execution) use this variant. + */ +static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + int *ap) +{ + u64 pa = 0; + + if (!svc_pin_lookup(va, addr)) { + *ap = SVC_APERTURE_SYS; + return 0; + } + if (!svc_lat_lookup(va, addr, ap)) + return 0; + if (!svc_map_lookup(va, &pa)) { + *addr = pa; + *ap = SVC_APERTURE_SYS; + svc_lat_store(va, pa, SVC_APERTURE_SYS); + return 0; + } + if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) { + *addr += va & 0xfff; + svc_lat_store(va, *addr, *ap); return 0; } + if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { + *addr += va & 0xfff; + svc_lat_store(va, *addr, *ap); + return 0; + } + if (!svc_pd_tried) { + svc_pd_discover(vgpu); + if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { + *addr += va & 0xfff; + svc_lat_store(va, *addr, *ap); + return 0; + } + } + return -ENOENT; +} + +/* + * Translate a GPU VA: the trusted sources, then the vidmem window convention + * (a structural guess that is correct for the driver's vidmem window). + */ +static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) +{ + if (!svc_lookup_trusted(vgpu, va, addr, ap)) + return 0; if (va >= SVC_VIDMEM_VA_BASE && va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) { *addr = va - SVC_VIDMEM_VA_BASE; - *aperture = 1; /* emulated vidmem */ + *ap = SVC_APERTURE_FB; + svc_lat_store(va, *addr, *ap); return 0; } return -ENOENT; } +static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + u32 *aperture) +{ + int ap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, addr, &ap)) + return -ENOENT; + *aperture = (ap == SVC_APERTURE_FB) ? 1 : 0; + return 0; +} + static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { u64 addr = 0; @@ -316,11 +963,9 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; - if (ap) { - memcpy(val, vgpu->bars[1].vaddr + addr, sizeof(*val)); - return 0; - } - return svc_read(vgpu, addr, val, sizeof(*val)); + return svc_read_ap(vgpu, addr, + ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, + val, sizeof(*val)); } static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) @@ -330,12 +975,11 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; - if (ap) { - memcpy(vgpu->bars[1].vaddr + addr, &val, sizeof(val)); + if (ap) svc.vidmem_writes++; - return 0; - } - return svc_write(vgpu, addr, &val, sizeof(val)); + return svc_write_ap(vgpu, addr, + ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, + &val, sizeof(val)); } /* ------------------------------------------------------------------ */ @@ -357,7 +1001,7 @@ struct base_candidate { * vas[CPU] followed by vas[GPU]), collected raw for pass 2. */ #define PAIR_MAX 32768 -struct raw_pair { u64 cpu_va; u64 gpu_va; }; +struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; }; static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, struct base_candidate *cands, u32 *ncand, @@ -404,8 +1048,32 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, b >= SEMA_VA_LO && b < SEMA_VA_HI && (a & 0xfff) == 0 && (b & 0xfff) == 0 && *npairs < PAIR_MAX) { + u8 ok = 0; + + /* + * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] + * (offset 48) directly before vas[UVM_ID_GPU0] + * (offset 56), so a pair taken from a live rm_mem + * is preceded by the header {u32 type; bitmap + * mapped_on}. Validate it when it fits in the same + * page: freed allocations and unrelated tables + * that happen to hold KVA/VA pairs do not match, + * and a stale entry for the same VA must not + * shadow the live one. + */ + if (off >= 48) { + u32 type; + u64 mapped_on; + + memcpy(&type, buf + off - 48, 4); + memcpy(&mapped_on, buf + off - 40, 8); + if ((type == 0 || type == 1) && + (mapped_on & 0x3) == 0x3) + ok = 1; + } pairs[*npairs].cpu_va = a; pairs[*npairs].gpu_va = b; + pairs[*npairs].valid_hdr = ok; (*npairs)++; } } @@ -424,6 +1092,22 @@ static void svc_add_map(u64 gpu_va, u64 pa) } } +/* Add one scanned {CPU KVA, GPU VA} pair to the map. */ +static void svc_add_pair(u64 base, const struct raw_pair *p) +{ + u64 pa; + + if (p->cpu_va < base) + return; + pa = p->cpu_va - base; + if (pa >= 0x140000000ULL || (pa & 0xfff)) + return; + if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI) + return; + svc_add_map(p->gpu_va, pa); + svc.pairs_seen++; +} + static int svc_map_cmp(const void *a, const void *b) { const struct sema_map_entry *x = a, *y = b; @@ -495,6 +1179,28 @@ static int svc_map_lookup(u64 gpu_va, u64 *pa) return -ENOENT; } +/* Binary search over a sorted sub-range (used while the tail is unsorted). */ +static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa) +{ + u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); + int lo = (int)from, hi = (int)to - 1; + + while (lo <= hi) { + int mid = (lo + hi) / 2; + + if (svc.map[mid].gpu_page == gpu_page) { + *pa = svc.map[mid].pa_page + + (gpu_va & (SEMA_PAGE_SIZE - 1)); + return 0; + } + if (svc.map[mid].gpu_page < gpu_page) + lo = mid + 1; + else + hi = mid - 1; + } + return -ENOENT; +} + static void svc_rebuild(struct nv_vgpu_device *vgpu) { static u8 *buf; @@ -521,32 +1227,47 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) } if (best_base && best_count >= 2) { + u32 n_valid; + svc.direct_base = best_base; + + /* + * Live rm_mem-style records first (dedup is first-wins), then + * headerless candidates only for VAs the validated set does not + * cover - a freed allocation or an unrelated table holding a + * stale pair for the same VA must not shadow the live record. + */ + for (i = 0; i < npairs; i++) { + if (pairs[i].valid_hdr) + svc_add_pair(best_base, &pairs[i]); + } + n_valid = svc.map_n; + sort(svc.map, n_valid, sizeof(svc.map[0]), svc_map_cmp, NULL); for (i = 0; i < npairs; i++) { - u64 cpu = pairs[i].cpu_va; - u64 pa; + u64 dummy; - if (cpu < best_base) - continue; - pa = cpu - best_base; - if (pa >= 0x140000000ULL || (pa & 0xfff)) + if (pairs[i].valid_hdr) continue; - if (pairs[i].gpu_va < SEMA_VA_LO || - pairs[i].gpu_va >= SEMA_VA_HI) + if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid, + &dummy)) continue; - svc_add_map(pairs[i].gpu_va, pa); - svc.pairs_seen++; + svc_add_pair(best_base, &pairs[i]); } } + /* Stale entries from a previous rebuild (wrong PA for a reused VA) + * must not outlive the map they were derived from. */ + memset(svc.lat, 0, sizeof(svc.lat)); + svc_map_finalize(); svc.builds++; svc.last_build = jiffies; - pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu\n", + pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu\n", svc.builds, best_base, best_count, npairs, svc.map_n, svc.releases_seen, svc.releases_applied, svc.releases_missed, - svc.releases_stale, svc.releases_jumped); + svc.releases_stale, svc.releases_jumped, + svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits); pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, @@ -631,6 +1352,8 @@ struct svc_push_state { u32 have_remap; u32 release_dma; u32 sem_inc; + u32 rs_upper, rs_lower, rs_payload; + u32 have_rs_addr, have_rs_payload; }; /* ------------------------------------------------------------------ */ @@ -925,11 +1648,73 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } - if (subch != 4) { + /* + * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of + * four words {offset_upper, offset_lower, payload, operation} + * or the same words written one at a time. A RELEASE operation + * ends the push: run the collected work, then write the + * payload to the semaphore (forward progress only, so recycled + * buffers stay harmless). + */ + if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) { + u32 vv[4] = { v0, v1, v2, v3 }; + u32 k, n = count > 4 ? 4 : count; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case HOST_M_REPORT_SEM_A: + st.rs_upper = vv[k] & 0xff; + st.have_rs_addr = 1; + break; + case HOST_M_REPORT_SEM_B: + st.rs_lower = vv[k]; + st.have_rs_addr = 1; + break; + case HOST_M_REPORT_SEM_C: + st.rs_payload = vv[k]; + st.have_rs_payload = 1; + break; + case HOST_M_REPORT_SEM_D: { + u64 va; + u32 cur = 0; + + if ((vv[k] & HOST_RS_OP_MASK) != + HOST_RS_OP_RELEASE || + !st.have_rs_addr || + !st.have_rs_payload || + !st.rs_payload) + break; + va = ((u64)st.rs_upper << 32) | + st.rs_lower; + if (va < SEMA_VA_LO || va >= SEMA_VA_HI) + break; + if (page_pa) + svc_mark_hot(page_pa); + svc.pushes_seen++; + if (svc_sem_read(vgpu, va, &cur)) + break; + if (st.rs_payload <= cur) + break; + svc.pushes_pending++; + svc_run_ops(vgpu, ops, n_ops); + n_ops = 0; + if (!svc_sem_write(vgpu, va, + st.rs_payload)) + svc.releases_applied++; + break; + } + default: + break; + } + } off += 4 + count * 4; continue; } + if (subch != 4) { + off += 4 + count * 4; + continue; + } switch (addr) { case CE_M_SET_SEMAPHORE_A: st.sem_va_hi = v0; @@ -1253,11 +2038,181 @@ static void svc_apply_mapped(struct nv_vgpu_device *vgpu) static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { - u64 pa; + u8 *p = buf; + + while (len) { + u64 pa = 0; + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); + int ap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa, + ap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * Read `len` bytes at a GPU VA through a trusted translation only. Push text + * must never be fetched through the vidmem window convention: a sysmem-backed + * pushbuffer VA above the window base answers with a vidmem offset and the + * caller would decode zeros and mark work done that never ran. + */ +static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + u64 pa = 0; + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); + int ap = SVC_APERTURE_SYS; - if (svc_map_lookup(va, &pa)) + if (svc_lookup_trusted(vgpu, va, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa, + ap == SVC_APERTURE_FB ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the + * values are plausible for a channel with `entries` GPFIFO slots. + */ +static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get) +{ + u32 p = 0, g = 0; + + if (!pa) + return -EINVAL; + if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p))) + return -ENOENT; + if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g))) return -ENOENT; - return svc_read(vgpu, pa, buf, len); + if (p > entries || g > p) + return -EINVAL; + if (!p && !g) + return -EAGAIN; /* plausible but idle: keep looking */ + *put = p; + *get = g; + return 0; +} + +/* + * Locate a channel's USERD. The allocation parameters carry an offset that is + * relative to a client pool the emulation cannot see, so it is only one + * candidate: for the combined GPFIFO+USERD allocations the control block sits + * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout). + * Whichever candidate reads a coherent get/put pair wins and is cached. An + * idle candidate (0/0) is not accepted until it shows activity, so a zeroed + * page is never mistaken for the control block. + */ +static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u64 ring_pa, int ring_ap, u32 *put, u32 *get) +{ + u64 cand[4]; + u32 as[4], nc = 0, i; + + if (ring_ap == SVC_APERTURE_FB && ring_pa) { + cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB; + cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB; + } + cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; + + for (i = 0; i < nc; i++) { + if (!cand[i]) + continue; + if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, + put, get)) { + ch->userd_pa = cand[i]; + ch->userd_as = as[i]; + ch->userd_ok = true; + return 0; + } + } + return -ENOENT; +} + +/* + * Complete a channel's work-submission notification the way the GPU does. + * + * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot + * in-progress (status 0xffff, info32 = work-submit token) and then polls the + * notification memory for it to come back cleared - the GPU clearing it is + * the acknowledgement that the submission was picked up. The emulated GPU + * never did that, so libcuda's context setup spun forever. Write the cleared + * notification when the slot is busy. + * + * NvNotification layout (16 bytes): timestamp[2], info32, info16, status. + */ +#define SVC_NOTIFY_SIZE 16 +#define SVC_NOTIFY_STATUS_OFF 14 +#define SVC_NOTIFY_INFO32_OFF 8 +#define SVC_NOTIFY_TOKEN_SLOT 1 /* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */ + +static void svc_notify_complete(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0}; + u64 base = ch->notifier_pa; + u32 status = 0, info32 = 0; + int slot, busy = 0; + + if (!ch->notifier_pa) + return; + + /* Both the error slot (0) and the work-submit-token slot (1) must read + * as completed; the client polls one of them per submission. */ + for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) { + u64 pa = base + slot * SVC_NOTIFY_SIZE; + + status = 0; + info32 = 0; + if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF, + ch->notifier_as, &status, 2) || + svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF, + ch->notifier_as, &info32, 4)) + continue; + if (status || info32) + busy = 1; + } + if (!busy) + return; + + /* + * Clear both slots: they are adjacent, and whichever one the client is + * polling must read as completed for a healthy channel. + */ + if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) { + ch->notified++; + if (ch->notified <= 4 || (ch->notified % 10000) == 0) + pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n", + ch->h_object, info32, status, ch->notified); + + /* + * The GPU consumed the submission: post the channel's event so + * the client's CPU-side RM signals the OS event the submitting + * thread waits on (a real GPU would raise an interrupt here). + */ + gsp_rm_post_channel_event(vgpu, ch->h_object); + } } /* @@ -1284,18 +2239,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + int rrc; u32 put = 0, get = 0, guard = 0; - if (!ch->userd_pa) - continue; + svc_notify_complete(vgpu, ch); - if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, - ch->userd_as, &put, sizeof(put))) { - svc.gpfifo_nomap++; + if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; + + rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); + + if (!ch->userd_ok) { + if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, + rrc ? -1 : ring_ap, &put, &get)) + continue; + } else { + if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, + ch->userd_as, &put, sizeof(put))) + continue; + (void)svc_read_ap(vgpu, + ch->userd_pa + SVC_USERD_GP_GET, + ch->userd_as, &get, sizeof(get)); + } + + if (!ch->logged) { + pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n", + ch->h_object, ch->engine_type, ch->cid, + ch->gpfifo_va, + ch->gpfifo_entries, ch->userd_pa, ch->userd_as, + put, get, rrc, ring_pa, ring_ap); + ch->logged = true; } - (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, - ch->userd_as, &get, sizeof(get)); if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; @@ -1305,6 +2281,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 va, idx = ch->last_get; u32 len, e0, e1, g; + if (rrc) + break; /* cannot read the ring at all */ + if (svc_read_va(vgpu, ch->gpfifo_va + idx * SVC_GPFIFO_ENTRY_SIZE, @@ -1320,17 +2299,29 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* GP_ENTRY1.LENGTH[30:10], in dwords */ len = ((e1 >> 10) & 0x1fffff) << 2; + if (svc.gpfifo_entry_log < 32) { + pr_info("sema-chan: entry[%llu] raw=%08x%08x va=0x%llx len=%u\n", + idx, e1, e0, va, len); + svc.gpfifo_entry_log++; + } + if (va && len && len <= SVC_PUSH_MAX) { u8 *pb = kmalloc(len, GFP_KERNEL); if (pb) { - if (!svc_read_va(vgpu, va, pb, len)) { + if (!svc_read_va_trusted(vgpu, va, pb, + len)) { svc_decode_stream(vgpu, pb, len, 0); ch->submissions++; svc.gpfifo_entries_done++; } else { + /* Keep the entry pending: the + * mapping may resolve once the + * page tables are filled. */ svc.gpfifo_nomap++; + kfree(pb); + break; } kfree(pb); } @@ -1348,6 +2339,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) for (k = 0; k < SVC_MAX_CHANNELS; k++) { if (svc.chan[k].valid && svc.chan[k].h_object == ch->h_object) { + svc.chan[k].userd_pa = ch->userd_pa; + svc.chan[k].userd_as = ch->userd_as; + svc.chan[k].userd_ok = ch->userd_ok; + svc.chan[k].logged = ch->logged; + svc.chan[k].notified = ch->notified; svc.chan[k].last_get = ch->last_get; svc.chan[k].submissions = ch->submissions; break; @@ -1383,19 +2379,30 @@ static void svc_handle_probe(struct nv_vgpu_device *vgpu) probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.direct_base, svc.map_n); - if (svc_map_lookup(probe_va, &pa)) { - pr_info("sema-probe: NO MAP ENTRY for page 0x%llx\n", - probe_va & ~(u64)(SEMA_PAGE_SIZE - 1)); - } else { - pr_info("sema-probe: pa=0x%llx\n", pa); - for (i = 0; i < probe_n; i++) { - u32 v = 0; + { + int ap = SVC_APERTURE_SYS; - if (svc_read(vgpu, pa + 4ULL * i, &v, sizeof(v))) { - pr_info("sema-probe: [%u] read failed\n", i); - break; + if (svc_lookup(vgpu, probe_va, &pa, &ap)) { + pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n", + probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), + svc.walks_ok, svc.walks_ok + svc.walks_miss); + } else { + pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa, + ap == SVC_APERTURE_FB ? "vidmem" : "ram"); + for (i = 0; i < probe_n; i++) { + u32 v = 0; + + if (svc_read_ap(vgpu, pa + 4ULL * i, + ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS, + &v, sizeof(v))) { + pr_info("sema-probe: [%u] read failed\n", + i); + break; + } + pr_info("sema-probe: [%u] 0x%08x\n", i, v); } - pr_info("sema-probe: [%u] 0x%08x\n", i, v); } } @@ -1422,6 +2429,59 @@ int gsp_sema_service_probe(u64 gpu_va, u32 count) return 0; } +/* + * Debug injection: teach the service one {GPU VA -> guest PA} translation. + * Some client mappings (user VA == GPU VA, sysmem backed) are not + * discoverable from the structures the service scans; this lets a session + * feed a pair that was verified from the guest and watch push execution use + * it. + */ +static struct completion map_add_done; +static u64 map_add_va; +static u64 map_add_pa; +static u64 map_add_len; +static bool map_add_pending; + +static void svc_handle_map_add(void) +{ + u64 off; + + if (!map_add_pending) + return; + + for (off = 0; off < (map_add_len ? map_add_len : 1); + off += SEMA_PAGE_SIZE) { + svc_pin_store(map_add_va + off, map_add_pa + off); + svc_lat_store(map_add_va + off, map_add_pa + off, + SVC_APERTURE_SYS); + } + pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (sysmem)\n", + map_add_va, map_add_pa, map_add_len); + map_add_pending = false; + complete(&map_add_done); +} + +int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len) +{ + if (!svc.task) + return -ENODEV; + + if (len > (64ULL << 20)) + return -EINVAL; + + init_completion(&map_add_done); + map_add_va = gpu_va; + map_add_pa = pa; + map_add_len = len; + map_add_pending = true; + + if (!wait_for_completion_timeout(&map_add_done, 3 * HZ)) { + map_add_pending = false; + return -ETIMEDOUT; + } + return 0; +} + static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; @@ -1449,6 +2509,7 @@ static int svc_thread(void *arg) } svc_handle_probe(vgpu); + svc_handle_map_add(); if (!svc.direct_base || time_after(jiffies, svc.last_build + 10 * HZ)) @@ -1458,8 +2519,10 @@ static int svc_thread(void *arg) * Channel-driven execution: notice submissions as they are made * and run the pushed work immediately, so the guest's UVM * progress waits complete in milliseconds rather than in the - * time it takes the fallback scan to find the push. + * time it takes the fallback scan to find the push. A doorbell + * token means a channel was just scheduled; run everything now. */ + svc.doorbell_pending = false; svc_pump_channels(vgpu); /* @@ -1484,15 +2547,39 @@ static int svc_thread(void *arg) pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); + complete(&svc.stopped); return 0; } +/* + * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0 + * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has + * written the puller GPFIFO entry, so a pending token means "run this + * channel's entries now". Called from the VFIO write path (atomic context): + * record the token and let the service thread do the work. + */ +void gsp_sema_service_doorbell(u32 token) +{ + svc.doorbell_token = token; + svc.doorbell_count++; + svc.doorbell_pending = true; + + if (svc.doorbell_log < 64) { + pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n", + token, (token >> 16) & 0x7f, token & 0xfff, + svc.doorbell_count); + svc.doorbell_log++; + } +} + int gsp_sema_service_start(void) { if (svc.task) return 0; svc.stop = false; + init_completion(&svc.stopped); spin_lock_init(&svc.chan_lock); + spin_lock_init(&svc.vas_lock); svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); @@ -1503,11 +2590,18 @@ int gsp_sema_service_start(void) return 0; } +/* + * Stop the service thread without kthread_stop(): the thread can also end on + * its own if it ever faults, and joining a task that is already gone faults in + * the caller (which wedges the whole module). The completion tells us the + * thread function has returned, whatever ended it. + */ void gsp_sema_service_stop(void) { - if (svc.task) { - svc.stop = true; - kthread_stop(svc.task); - svc.task = NULL; - } + if (!svc.task) + return; + svc.stop = true; + if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) + pr_err("sema-svc: thread did not stop within 5s\n"); + svc.task = NULL; } |