summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_sema_service.c327
1 files changed, 316 insertions, 11 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 3b009ef..a67107b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -124,6 +124,7 @@ struct svc_channel {
u32 h_object;
u32 engine_type;
u32 cid; /* RM channel id (doorbell token vector) */
+ u8 ce_index; /* rank among the live CE channels (creation order) */
u64 gpfifo_va;
u32 gpfifo_entries;
u64 userd_pa;
@@ -156,7 +157,7 @@ static u32 svc_map_hash[SEMA_MAP_MAX * 2];
* (debug injection) and that must not be evicted by the learned cache, which
* is content-addressed and gets thrashed by the lookup stream.
*/
-#define SVC_PIN_SLOTS 4096
+#define SVC_PIN_SLOTS 8192
struct svc_pin {
u64 va_page;
@@ -187,6 +188,63 @@ static void svc_mark_hot(u64 pa)
set_bit(page, svc_hot);
}
+/*
+ * Push-buffer base pages found by content.
+ *
+ * Every channel's first push starts at its push-buffer base and is
+ * structurally identical across channels: a SET_OBJECT on subchannel 0, a
+ * SET_OBJECT on the compute subchannel (4), then the channel's tracking
+ * semaphore release at page offset 0x10 with payload 1. The release target's
+ * in-page offset selects the channel (0x00, 0x04, ... one slot per channel),
+ * so finding that group in a page names both the page (the channel's
+ * push-buffer base) and the channel (its slot).
+ */
+#define SVC_PB_MAX 32
+struct svc_pb_hit {
+ u64 pa_page;
+ u64 va; /* push-buffer base VA, once paired with a channel */
+ u32 slot;
+};
+
+static struct svc_pb_hit svc_pb[SVC_PB_MAX];
+static u32 svc_pb_n;
+static u32 svc_ring_n;
+static u8 svc_pb_passes;
+static u64 svc_pb_last;
+
+static bool svc_pb_first_push(const u8 *buf)
+{
+ u32 h0, h1;
+
+ memcpy(&h0, buf, 4);
+ memcpy(&h1, buf + 8, 4);
+ if (((h0 >> 29) & 0x7) != 1 || ((h0 >> 16) & 0xfff) != 1 ||
+ ((h0 >> 13) & 0x7) != 0 || (h0 & 0x1fff))
+ return false;
+ /* The second group is the compute class' SET_OBJECT: one method word
+ * to offset 0 of a non-zero subchannel. */
+ if (((h1 >> 29) & 0x7) != 1 || ((h1 >> 16) & 0xfff) != 1 ||
+ ((h1 >> 13) & 0x7) == 0 || (h1 & 0x1fff))
+ return false;
+ return true;
+}
+
+static void svc_pb_add(u64 pa_page, u32 slot)
+{
+ u32 i;
+
+ for (i = 0; i < svc_pb_n; i++)
+ if (svc_pb[i].slot == slot || svc_pb[i].pa_page == pa_page)
+ return;
+ if (svc_pb_n >= SVC_PB_MAX)
+ return;
+ svc_pb[svc_pb_n].pa_page = pa_page;
+ svc_pb[svc_pb_n].slot = slot;
+ svc_pb_n++;
+ pr_info("sema-svc: push-buffer base pa=0x%llx slot=%u (n=%u)\n",
+ pa_page, slot, svc_pb_n);
+}
+
struct sema_svc {
struct task_struct *task;
bool stop;
@@ -333,7 +391,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
ch = &svc.chan[i];
}
if (ch) {
+ int k;
+
memset(ch, 0, sizeof(*ch));
+ /*
+ * Rank among the live CE channels. The RM hands out object
+ * handles in creation order and the guest's UVM allocates one
+ * push buffer, ring and tracking semaphore per channel in that
+ * same order, so the rank is what ties a channel to its ring
+ * offset and its push-buffer base (see svc_ring_autopin /
+ * svc_pb_pair).
+ */
+ if (desc->engine_type) {
+ u8 rank = 0;
+
+ for (k = 0; k < SVC_MAX_CHANNELS; k++)
+ if (svc.chan[k].valid && svc.chan[k].engine_type &&
+ svc.chan[k].h_object < desc->h_object)
+ rank++;
+ ch->ce_index = rank;
+ }
ch->valid = true;
ch->h_object = desc->h_object;
ch->engine_type = desc->engine_type;
@@ -770,16 +847,34 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
return;
svc_pd_tried = 1;
- if (svc.map_n < SVC_PD_SAMPLES)
- return;
- step = svc.map_n / SVC_PD_SAMPLES;
- for (i = 0; i < SVC_PD_SAMPLES; i++) {
- sva[i] = svc.map[i * step].gpu_page;
- spa[i] = svc.map[i * step].pa_page;
- if (sva[i] >> SVC_MMU_VA_BITS)
- return; /* outside the walkable VA range */
+ /*
+ * Pinned push-buffer pairs are the strongest samples available: they
+ * are UVM-internal VAs whose translation was verified by content, and
+ * the directory that maps them is the one that translates the rest of
+ * the push-buffer pool. Fall back to the discovered VA->PA map while
+ * fewer than SVC_PD_MIN_HITS of them exist: that map also holds pairs
+ * from other VA spaces and no single directory reproduces a mixed set.
+ */
+ for (i = 0; i < svc_pb_n && n < SVC_PD_SAMPLES; i++) {
+ if (!svc_pb[i].va)
+ continue;
+ sva[n] = svc_pb[i].va;
+ spa[n] = svc_pb[i].pa_page;
n++;
}
+ if (n < SVC_PD_MIN_HITS) {
+ n = 0;
+ if (svc.map_n < SVC_PD_SAMPLES)
+ return;
+ step = svc.map_n / SVC_PD_SAMPLES;
+ for (i = 0; i < SVC_PD_SAMPLES; i++) {
+ sva[i] = svc.map[i * step].gpu_page;
+ spa[i] = svc.map[i * step].pa_page;
+ if (sva[i] >> SVC_MMU_VA_BITS)
+ return; /* outside the walkable VA range */
+ n++;
+ }
+ }
if (!n)
return;
@@ -1296,9 +1391,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
svc.memsets_fail);
- pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu\n",
+ pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u\n",
svc.gpfifo_entries_done, svc.gpfifo_nomap,
- svc.vidmem_writes);
+ svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes);
/* show a few entries so mappings can be verified from dmesg */
{
u32 shown;
@@ -1960,6 +2055,15 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
continue;
}
+ /*
+ * A release at page offset 0x10 with payload 1 in front of a
+ * channel's first push identifies the page as that channel's
+ * push-buffer base (see svc_pb_add).
+ */
+ if (off == 0x10 && hi == 1 && payload == 1 &&
+ (lo & 0xfff) < 0x20 && svc_pb_first_push(buf))
+ svc_pb_add(pa_page, lo & 0xfff);
+
if (svc_sem_write(vgpu, va, payload) == 0) {
svc.releases_applied++;
svc.post_hint_until = jiffies + 10 * HZ;
@@ -2062,6 +2166,75 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
}
+static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page)
+{
+ u32 hdr, hi, lo, payload;
+
+ (void)vgpu;
+ memcpy(&hdr, buf + 0x10, 4);
+ if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT ||
+ (hdr & 0xff) != CE_SEMA_HDR_ADDR)
+ return;
+ memcpy(&hi, buf + 0x14, 4);
+ memcpy(&lo, buf + 0x18, 4);
+ memcpy(&payload, buf + 0x1c, 4);
+ if (hi != 1 || payload != 1 || (lo & 0xfff) >= 0x20)
+ return;
+ if (!svc_pb_first_push(buf))
+ return;
+ svc_pb_add(pa_page, lo & 0xfff);
+ svc_mark_hot(pa_page);
+}
+
+/*
+ * Dedicated pass for the push-buffer base pages: a chunked walk of guest RAM
+ * completes in a couple of seconds, where the rolling sweep visits only 32 MB
+ * per pass. It runs while CE channels exist and stops once enough bases are
+ * known, so a page that appears while a client waits on it is pinned and
+ * pumped immediately.
+ */
+static void svc_pb_discover(struct nv_vgpu_device *vgpu)
+{
+ unsigned long flags;
+ u8 *buf;
+ u64 gpa;
+ bool ce = false;
+ int i;
+
+ if (svc_pb_passes >= 12 || svc_pb_n >= SVC_PB_MAX)
+ return;
+ if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ))
+ return;
+
+ spin_lock_irqsave(&svc.chan_lock, flags);
+ for (i = 0; i < SVC_MAX_CHANNELS; i++)
+ if (svc.chan[i].valid && svc.chan[i].engine_type) {
+ ce = true;
+ break;
+ }
+ spin_unlock_irqrestore(&svc.chan_lock, flags);
+ if (!ce)
+ return;
+
+ buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+ if (!buf)
+ return;
+
+ svc_pb_passes++;
+ svc_pb_last = jiffies;
+ for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+ u32 o;
+
+ if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+ continue;
+ for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE)
+ svc_pb_note(vgpu, buf + o, gpa + o);
+ }
+ kfree(buf);
+ pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u)\n",
+ svc_pb_passes, svc_pb_n, svc_ring_n);
+}
+
/*
* Targeted pass: run the work in the pages we have mapped. The map covers the
* allocations the guest has described to us - the pushbuffer pools among them
@@ -2315,6 +2488,125 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
* memsets, copies, semaphore releases) - this is what makes the guest's UVM
* progress waits complete. The GPU's own GPGet is updated as we go.
*/
+
+/*
+ * The guest's UVM allocates each CE channel's ring and USERD in one
+ * contiguous video-memory block, in channel creation order, and the emulation
+ * sees the ring only as a GPU VA. The block's layout is a function of the
+ * channel's rank (ring at 0x80000 + 0x20000 * rank, USERD at ring + 0x10000;
+ * observed for every CE channel across boots). The derived offset is verified
+ * against the ring's first GPFIFO entry, so a layout change costs a failed
+ * pin, not a wrong one.
+ */
+#define SVC_CE_RING_FB_BASE 0x80000ULL
+#define SVC_CE_RING_FB_STRIDE 0x20000ULL
+
+static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */
+
+/* Read and decode the channel's first GPFIFO entry (its first push). */
+static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+ u64 *va, u32 *len)
+{
+ u8 e[SVC_GPFIFO_ENTRY_SIZE];
+ u32 e0, e1;
+
+ if (svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e)))
+ return -ENOENT;
+ memcpy(&e0, e, 4);
+ memcpy(&e1, e + 4, 4);
+ *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+ *len = ((e1 >> 10) & 0x1fffff) << 2;
+ return 0;
+}
+
+static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch)
+{
+ u64 pa = 0, va = 0;
+ int ap = SVC_APERTURE_SYS;
+ u8 e[SVC_GPFIFO_ENTRY_SIZE];
+ u32 e0, e1, len;
+
+ if (!ch->engine_type || ch->ce_index >= 32)
+ return;
+ if (test_bit(ch->ce_index, &svc_ring_done))
+ return;
+ if (!svc_pin_lookup(ch->gpfifo_va, &pa, &ap)) {
+ set_bit(ch->ce_index, &svc_ring_done);
+ svc_ring_n++;
+ return;
+ }
+
+ pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index;
+ if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e)))
+ return;
+ memcpy(&e0, e, 4);
+ memcpy(&e1, e + 4, 4);
+ va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+ len = ((e1 >> 10) & 0x1fffff) << 2;
+ if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
+ len < 4 || len > 0x10000)
+ return;
+
+ svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+ svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+ set_bit(ch->ce_index, &svc_ring_done);
+ svc_ring_n++;
+ pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n",
+ ch->h_object, ch->ce_index, ch->gpfifo_va, pa, va, len);
+}
+
+/*
+ * Pair a channel's push buffer with its base page: the ring's first entry
+ * carries the push-buffer VA, the base page was found by content and its
+ * release names the channel's slot (4 bytes per rank).
+ */
+static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */
+
+static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
+{
+ u64 hit_pa = 0, va = 0;
+ u32 len = 0, i, hit_i = 0, want, off;
+
+ if (!ch->engine_type || ch->ce_index >= 32)
+ return;
+ if (test_bit(ch->ce_index, &svc_pb_done))
+ return;
+ want = 4 * ch->ce_index;
+ for (i = 0; i < svc_pb_n; i++)
+ if (svc_pb[i].slot == want) {
+ hit_pa = svc_pb[i].pa_page;
+ hit_i = i;
+ break;
+ }
+ if (!hit_pa)
+ return;
+ if (svc_chan_entry0(vgpu, ch, &va, &len))
+ return;
+ if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
+ len < 4 || len > 0x10000)
+ return;
+
+ for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
+ svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
+ svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
+ svc_mark_hot(hit_pa + off);
+ }
+ svc_pb[hit_i].va = va;
+ set_bit(ch->ce_index, &svc_pb_done);
+
+ /*
+ * The pinned pairs are verified samples of UVM's own page directory;
+ * let the directory discovery run again with them, so the walk can
+ * translate the parts of the pool no base page describes.
+ */
+ if (!svc_pd_found)
+ svc_pd_tried = 0;
+
+ pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n",
+ ch->h_object, ch->ce_index, want, va, hit_pa, len);
+}
+
static void svc_pump_channels(struct nv_vgpu_device *vgpu)
{
/* Only ever called from the service thread. */
@@ -2340,8 +2632,13 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
if (!ch->gpfifo_va || !ch->gpfifo_entries)
continue;
+ svc_ring_autopin(vgpu, ch);
+
rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap);
+ if (!rrc)
+ svc_pb_pair(vgpu, ch);
+
if (!ch->userd_ok) {
if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
rrc ? -1 : ring_ap, &put, &get))
@@ -2620,6 +2917,14 @@ static int svc_thread(void *arg)
svc_pump_channels(vgpu);
/*
+ * Push-buffer base discovery: a chunked walk of guest RAM
+ * finds the pages whose first push names a channel, so a
+ * channel that appears with work already waiting is pinned and
+ * pumped without waiting for the rolling sweep.
+ */
+ svc_pb_discover(vgpu);
+
+ /*
* Completion hint: re-post the events for a few seconds after a
* completion, so a client that starts waiting after the
* edge-triggered wake-up still observes it.
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 22:28:03 +0000