diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 104 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 7 | ||||
| -rw-r--r-- | kernel/main.c | 8 | ||||
| -rw-r--r-- | tools/guest-diag/memprobe.py | 20 |
4 files changed, 133 insertions, 6 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 4cfed0c..7f39209 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -125,6 +125,7 @@ struct svc_channel { u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ u8 ce_index; /* rank among the live CE channels (creation order) */ + u8 gr_index; /* rank among the live GR channels (creation order) */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; @@ -462,6 +463,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) svc.chan[k].h_object < desc->h_object) rank++; ch->ce_index = rank; + } else { + u8 rank = 0; + + for (k = 0; k < SVC_MAX_CHANNELS; k++) + if (svc.chan[k].valid && !svc.chan[k].engine_type && + svc.chan[k].h_object < desc->h_object) + rank++; + ch->gr_index = rank; } ch->valid = true; ch->h_object = desc->h_object; @@ -1893,7 +1902,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } - if (subch != 4) { + /* + * The CE semaphore/memset methods arrive on the copy subchannel + * (4) on a CE channel, but CUDA's push library also binds the + * copy class (0xc5b5) on subchannel 0 of a compute channel to + * carry the same method groups (the work-submission pushes that + * release the client tracker semaphores). Accept both. + */ + if (subch != 0 && subch != 4) { off += 4 + count * 4; continue; } @@ -2730,6 +2746,8 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, #define SVC_CE_RING_FB_STRIDE 0x20000ULL static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */ +static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ +static u64 svc_gr_ring_base; /* FB offset where GR chunks start */ /* Read and decode the channel's first GPFIFO entry (its first push). */ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, @@ -2747,6 +2765,84 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } +/* + * GR (compute) channel rings live in the same video-memory allocation as the + * CE rings, immediately after them: the CE region is 0x80000 + 0x20000 * (CE + * channel count) and each GR channel then occupies a 0x3000 chunk in + * registration order - the ring (0x2000, 1024 entries) followed by the USERD + * at +0x2000. The chunk is only accepted when entry 0 decodes to a 1 MB + * aligned push in the client VA window *and* the USERD reads a coherent + * put/get pair, so a layout change costs a failed pin, not a wrong one. + */ +#define SVC_GR_RING_STRIDE 0x3000ULL +#define SVC_GR_USERD_OFF 0x2000ULL + +bool gsp_sema_gr_rings = true; + +static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u64 pa; + int ap = SVC_APERTURE_SYS; + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, va, len, put = 0, get = 0; + unsigned long flags; + int k; + + if (!gsp_sema_gr_rings) + return; + if (ch->gr_index >= 32) + return; + if (test_bit(ch->gr_index, &svc_gr_ring_done)) + return; + if (!svc_pin_lookup(ch->gpfifo_va, &pa, &ap)) { + /* Already translated (bridge/manual pin): nothing to do. */ + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + return; + } + + if (!svc_gr_ring_base) { + u32 max_ce = 0; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (k = 0; k < SVC_MAX_CHANNELS; k++) + if (svc.chan[k].valid && svc.chan[k].engine_type && + svc.chan[k].ce_index + 1 > max_ce) + max_ce = svc.chan[k].ce_index + 1; + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!max_ce) + return; /* wait until the CE channels are known */ + svc_gr_ring_base = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * max_ce; + } + + pa = svc_gr_ring_base + SVC_GR_RING_STRIDE * ch->gr_index; + if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) + return; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || + len < 4 || len > 0x10000) + return; + if (svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF, SVC_APERTURE_FB, + ch->gpfifo_entries, &put, &get)) + return; + + svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); + ch->userd_pa = pa + SVC_GR_USERD_OFF; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, pa, va, len, put, + get); +} + static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -2755,7 +2851,11 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; - if (!ch->engine_type || ch->ce_index >= 32) + if (!ch->engine_type) { + svc_gr_ring_autopin(vgpu, ch); + return; + } + if (ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_ring_done)) return; diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index d5e41a6..f652ffd 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -61,4 +61,11 @@ int gsp_sema_service_dump_channels(void); int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap); void gsp_sema_service_doorbell(u32 token); +/* + * Discover and pin GR (compute) channel rings after the CE region of the + * channel video-memory allocation. Set false to leave GR rings untranslated + * (the pump then only runs the CE channels). + */ +extern bool gsp_sema_gr_rings; + #endif /* _GSP_SEMA_SERVICE_H_ */ diff --git a/kernel/main.c b/kernel/main.c index f23be27..4f7f9ee 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2265,6 +2265,14 @@ module_param_cb(chan_dump, &nv_vgpu_chan_dump_ops, NULL, 0200); MODULE_PARM_DESC(chan_dump, "Dump the sema service channel table (any value)"); /* + * GR ring discovery: pin the compute channels' rings/USERDs, which follow the + * CE rings in the channel video-memory allocation. Default on; set 0 to leave + * GR channels untranslated when debugging push-execution side effects. + */ +module_param_named(sema_gr_rings, gsp_sema_gr_rings, bool, 0644); +MODULE_PARM_DESC(sema_gr_rings, "Auto-discover+pin GR channel rings (default 1)"); + +/* * Debug: inject a verified {GPU VA -> guest PA} translation into the * semaphore service (used when the client's mapping cannot be discovered * from guest structures): diff --git a/tools/guest-diag/memprobe.py b/tools/guest-diag/memprobe.py index 96dd40b..c2bab9f 100644 --- a/tools/guest-diag/memprobe.py +++ b/tools/guest-diag/memprobe.py @@ -72,12 +72,24 @@ def scan_qwords(pid, fd, lo, hi, needle_lo=None, needle_hi=None, needle_set=None return hits def sem_objs(pid, fd, lo, hi): - # qword in [lo,hi) at address A -> a sem object holds it at +0x10 + # A real sem object stores its GPU VA three times, at so+0x08, so+0x10 and + # so+0x18. Requiring a duplicated neighbour keeps the scan selective: a + # bare qword in the window matches thousands of unrelated pointers once + # the ring/pushbuffer windows are part of the range. first = scan_qwords(pid, fd, None, None, needle_lo=lo, needle_hi=hi) - out = {} + cand = {} for addr, val in first: - so = addr - 0x10 - out.setdefault(val, set()).add(so) + cand.setdefault(val, set()).add(addr) + out = {} + for val, addrs in cand.items(): + for a in addrs: + buf = rd(fd, a - 8, 0x20) + if not buf or len(buf) < 0x20: + continue + q0, q1, q2, q3 = (r64(buf, i) for i in (0, 8, 0x10, 0x18)) + if q1 == val and (q0 == val or q2 == val) and \ + (q2 == val or q3 == val): + out.setdefault(val, set()).add(a - 0x10) return out def find_trackers(pid, fd, lo, hi, limit=8): |