summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c418
1 files changed, 418 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
new file mode 100644
index 0000000..ad6c6cb
--- /dev/null
+++ b/kernel/gsp_sema_service.c
@@ -0,0 +1,418 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * gsp_sema_service.c - emulated GPU progress service.
+ *
+ * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory
+ * that end with a copy-engine semaphore release: a method group
+ * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the
+ * engine write a payload into a semaphore in system memory. UVM then waits for
+ * that value.
+ *
+ * A real GPU performs those writes. Our emulation must too, or every UVM tracker
+ * wait times out. This service does it without any cooperation from the guest
+ * driver:
+ *
+ * 1. Discover the GPU-VA -> guest-PA relationship from the guest's own
+ * bookkeeping. UVM/RM keep structures that record, for each allocation,
+ * both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map
+ * address). We first learn the direct-map base from self-referential
+ * structures in guest memory, then collect {cpu_va, gpu_va} pairs.
+ *
+ * 2. Scan the mapped pages for semaphore-release method groups and apply
+ * them: write the payload to the semaphore's guest-physical slot, in
+ * order (only when the slot currently holds payload-1).
+ *
+ * Everything runs from a kthread: the hypervisor's guest-DMA interface
+ * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the
+ * mapping's mm (the hypervisor process).
+ *
+ * This file is part of the host-side device emulation; see
+ * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md.
+ */
+
+#define pr_fmt(fmt) "gsp-resman-vfio: " fmt
+
+#include <linux/kthread.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/vfio.h>
+#include <linux/jiffies.h>
+#include <linux/mutex.h>
+
+#include "nvidia-vgpu-vfio.h"
+#include "gsp_sema_service.h"
+
+#define SEMA_MAP_MAX 16384
+#define SEMA_PAGE_SIZE 4096
+#define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */
+
+/* Plausible GPU VA window for the client's system-memory objects (Turing). */
+#define SEMA_VA_LO 0x100000000ULL
+#define SEMA_VA_HI 0x160000000ULL
+
+/* Canonical kernel virtual address range (x86_64). */
+#define KVA_LO 0xffff800000000000ULL
+#define KVA_HI 0xffffc00000000000ULL
+
+/* INC_METHOD header for the CE semaphore group: addr>>2 == 0x240, count == 3 */
+#define CE_SEMA_HDR_ADDR 0x90
+#define CE_SEMA_HDR_COUNT 3
+
+struct sema_map_entry {
+ u64 gpu_page; /* GPU VA, page aligned */
+ u64 pa_page; /* guest PA, page aligned */
+};
+
+struct sema_svc {
+ struct task_struct *task;
+ bool stop;
+
+ u64 direct_base;
+ u32 map_n;
+ struct sema_map_entry map[SEMA_MAP_MAX];
+ u64 last_build;
+
+ /* stats */
+ u64 pairs_seen;
+ u64 releases_seen;
+ u64 releases_applied;
+ u64 builds;
+};
+
+static struct sema_svc svc;
+
+/* ------------------------------------------------------------------ */
+/* guest memory helpers */
+/* ------------------------------------------------------------------ */
+
+static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
+{
+ return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
+}
+
+static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
+{
+ return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
+}
+
+/* ------------------------------------------------------------------ */
+/* discovery */
+/* ------------------------------------------------------------------ */
+
+struct base_candidate {
+ u64 base;
+ u32 count;
+};
+
+/*
+ * Pass 1 of a rebuild: walk guest RAM looking for
+ * (a) self-referential pointers: value V at offset `off` of page `gpa`
+ * such that V - (gpa + off) is a 1 GB aligned kernel address. This
+ * gives the direct-map base (page_offset_base), because such pointers
+ * are common (list heads, self-references) and consistent.
+ * (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style:
+ * vas[CPU] followed by vas[GPU]), collected raw for pass 2.
+ */
+#define PAIR_MAX 32768
+struct raw_pair { u64 cpu_va; u64 gpu_va; };
+
+static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
+ struct base_candidate *cands, u32 *ncand,
+ struct raw_pair *pairs, u32 *npairs)
+{
+ u64 gpa;
+ u32 nonz = 0;
+
+ for (gpa = 0; gpa < (0x140000000ULL); gpa += SEMA_PAGE_SIZE) {
+ u32 off;
+
+ if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
+ continue;
+
+ for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) {
+ u64 a, b;
+
+ memcpy(&a, buf + off, 8);
+ memcpy(&b, buf + off + 8, 8);
+
+ /* (a) self-pointer -> base candidate */
+ if (a >= KVA_LO && a < KVA_HI &&
+ (a & 0xfff) == ((gpa + off) & 0xfff)) {
+ u64 cb = a - gpa - off;
+ u32 i;
+
+ if ((cb & 0x3fffffffULL) == 0) {
+ for (i = 0; i < *ncand; i++) {
+ if (cands[i].base == cb) {
+ cands[i].count++;
+ break;
+ }
+ }
+ if (i == *ncand && *ncand < 8) {
+ cands[*ncand].base = cb;
+ cands[*ncand].count = 1;
+ (*ncand)++;
+ }
+ }
+ }
+
+ /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */
+ if (a >= KVA_LO && a < KVA_HI &&
+ b >= SEMA_VA_LO && b < SEMA_VA_HI &&
+ (a & 0xfff) == 0 && (b & 0xfff) == 0 &&
+ *npairs < PAIR_MAX) {
+ pairs[*npairs].cpu_va = a;
+ pairs[*npairs].gpu_va = b;
+ (*npairs)++;
+ }
+ }
+ nonz++;
+ }
+}
+
+static void svc_add_map(u64 gpu_va, u64 pa)
+{
+ u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+ u32 i;
+
+ for (i = 0; i < svc.map_n; i++) {
+ if (svc.map[i].gpu_page == gpu_page) {
+ svc.map[i].pa_page = pa_page;
+ return;
+ }
+ }
+ if (svc.map_n < SEMA_MAP_MAX) {
+ svc.map[svc.map_n].gpu_page = gpu_page;
+ svc.map[svc.map_n].pa_page = pa_page;
+ svc.map_n++;
+ }
+}
+
+static int svc_map_lookup(u64 gpu_va, u64 *pa)
+{
+ u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ u32 i;
+
+ for (i = 0; i < svc.map_n; i++) {
+ if (svc.map[i].gpu_page == gpu_page) {
+ *pa = svc.map[i].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1));
+ return 0;
+ }
+ }
+ return -ENOENT;
+}
+
+static void svc_rebuild(struct nv_vgpu_device *vgpu)
+{
+ static u8 *buf;
+ static struct base_candidate cands[8];
+ static struct raw_pair pairs[PAIR_MAX];
+ u32 ncand = 0, npairs = 0, i;
+ u64 best_base = 0;
+ u32 best_count = 0;
+
+ if (!buf) {
+ buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+ if (!buf)
+ return;
+ }
+
+ svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs);
+
+ for (i = 0; i < ncand; i++) {
+ if (cands[i].count > best_count) {
+ best_count = cands[i].count;
+ best_base = cands[i].base;
+ }
+ }
+
+ if (best_base && best_count >= 2) {
+ svc.direct_base = best_base;
+ for (i = 0; i < npairs; i++) {
+ u64 cpu = pairs[i].cpu_va;
+ u64 pa;
+
+ if (cpu < best_base)
+ continue;
+ pa = cpu - best_base;
+ if (pa >= 0x140000000ULL || (pa & 0xfff))
+ continue;
+ if (pairs[i].gpu_va < 0x100000000ULL ||
+ pairs[i].gpu_va >= 0x140000000ULL)
+ continue;
+ svc_add_map(pairs[i].gpu_va, pa);
+ svc.pairs_seen++;
+ }
+ }
+
+ svc.builds++;
+ svc.last_build = jiffies;
+
+ pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu\n",
+ svc.builds, best_base, best_count, npairs, svc.map_n,
+ svc.releases_seen, svc.releases_applied);
+ /* show a few entries so mappings can be verified from dmesg */
+ {
+ u32 shown;
+
+ for (shown = 0; shown < svc.map_n && shown < 8; shown++)
+ pr_info("sema-svc: map[%u] va=0x%llx -> pa=0x%llx\n",
+ shown, svc.map[shown].gpu_page,
+ svc.map[shown].pa_page);
+ }
+}
+
+/* ------------------------------------------------------------------ */
+/* push application */
+/* ------------------------------------------------------------------ */
+
+/*
+ * Look for CE semaphore-release method groups inside mapped pages:
+ *
+ * [INC_METHOD header (addr>>2 == 0x240, count == 3)]
+ * [upper 32 bits of semaphore VA]
+ * [lower 32 bits of semaphore VA]
+ * [payload]
+ * [... LAUNCH_DMA ...]
+ *
+ * Apply by writing payload to the semaphore slot, but only in order
+ * (current == payload - 1) so that UVM's tracker never sees a jump.
+ */
+static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
+{
+ u32 off, applied = 0;
+
+ if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
+ return 0;
+
+ for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) {
+ u32 hdr, hi, lo, payload;
+ u64 va, pa, cur;
+ u32 cur32;
+
+ memcpy(&hdr, buf + off, 4);
+ if ((hdr & 0xffff) != CE_SEMA_HDR_ADDR)
+ continue;
+ if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT)
+ continue;
+
+ memcpy(&hi, buf + off + 4, 4);
+ memcpy(&lo, buf + off + 8, 4);
+ memcpy(&payload, buf + off + 12, 4);
+
+ if (hi > 0xffff || payload == 0 || payload > 0x10000000)
+ continue;
+
+ va = ((u64)hi << 32) | lo;
+ if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
+ continue;
+
+ svc.releases_seen++;
+
+ if (svc_map_lookup(va, &pa))
+ continue;
+
+ cur = 0;
+ if (svc_read(vgpu, pa, &cur32, sizeof(cur32)))
+ continue;
+ cur = cur32;
+
+ /* in-order completion only */
+ if (cur + 1 != payload)
+ continue;
+
+ if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) {
+ svc.releases_applied++;
+ applied++;
+ if (svc.releases_applied <= 16)
+ pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n",
+ va, pa, payload);
+ }
+ }
+ return applied;
+}
+
+static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
+{
+ static u8 *buf;
+ u32 i;
+
+ if (!buf) {
+ buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+ if (!buf)
+ return;
+ }
+
+ for (i = 0; i < svc.map_n; i++)
+ svc_apply_page(vgpu, buf, svc.map[i].pa_page);
+}
+
+/* ------------------------------------------------------------------ */
+/* thread */
+/* ------------------------------------------------------------------ */
+
+static int svc_thread(void *arg)
+{
+ struct nv_vgpu_device *vgpu;
+
+ pr_info("sema-svc: thread started\n");
+
+ while (!kthread_should_stop() && !svc.stop) {
+ vgpu = NULL;
+ mutex_lock(&nv_vgpu_driver.lock);
+ {
+ struct nv_vgpu_device *d;
+
+ list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
+ if (atomic_read(&d->vfio_dev_open_count) > 0) {
+ vgpu = d;
+ break;
+ }
+ }
+ }
+ mutex_unlock(&nv_vgpu_driver.lock);
+
+ if (!vgpu) {
+ msleep(500);
+ continue;
+ }
+
+ if (!svc.direct_base ||
+ time_after(jiffies, svc.last_build + 10 * HZ))
+ svc_rebuild(vgpu);
+
+ if (svc.direct_base && svc.map_n)
+ svc_apply_pushes(vgpu);
+
+ msleep(100);
+ }
+
+ pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n",
+ svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n);
+ return 0;
+}
+
+int gsp_sema_service_start(void)
+{
+ if (svc.task)
+ return 0;
+ svc.stop = false;
+ svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
+ if (IS_ERR(svc.task)) {
+ int ret = PTR_ERR(svc.task);
+
+ svc.task = NULL;
+ return ret;
+ }
+ return 0;
+}
+
+void gsp_sema_service_stop(void)
+{
+ if (svc.task) {
+ svc.stop = true;
+ kthread_stop(svc.task);
+ svc.task = NULL;
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 09:57:15 +0000