summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c150
1 files changed, 136 insertions, 14 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index ad6c6cb..68a890c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -48,15 +48,23 @@
/* Plausible GPU VA window for the client's system-memory objects (Turing). */
#define SEMA_VA_LO 0x100000000ULL
-#define SEMA_VA_HI 0x160000000ULL
+#define SEMA_VA_HI 0x2000000000ULL
/* Canonical kernel virtual address range (x86_64). */
#define KVA_LO 0xffff800000000000ULL
#define KVA_HI 0xffffc00000000000ULL
-/* INC_METHOD header for the CE semaphore group: addr>>2 == 0x240, count == 3 */
+/*
+ * INC_METHOD header for the CE semaphore group. Observed value 0x20038090:
+ * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode
+ * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed
+ * header). Match on count == 3 and (address >> 2) low byte == 0x90.
+ */
#define CE_SEMA_HDR_ADDR 0x90
#define CE_SEMA_HDR_COUNT 3
+/* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */
+#define CE_DMA_HDR_ADDR 0xc0
+#define CE_DMA_HDR_COUNT 1
struct sema_map_entry {
u64 gpu_page; /* GPU VA, page aligned */
@@ -76,6 +84,9 @@ struct sema_svc {
u64 pairs_seen;
u64 releases_seen;
u64 releases_applied;
+ u64 releases_missed; /* no VA->PA entry */
+ u64 releases_stale; /* payload <= current */
+ u64 releases_jumped; /* applied with gap > 1 */
u64 builds;
};
@@ -249,9 +260,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
svc.builds++;
svc.last_build = jiffies;
- pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu\n",
+ pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu\n",
svc.builds, best_base, best_count, npairs, svc.map_n,
- svc.releases_seen, svc.releases_applied);
+ svc.releases_seen, svc.releases_applied, svc.releases_missed,
+ svc.releases_stale, svc.releases_jumped);
/* show a few entries so mappings can be verified from dmesg */
{
u32 shown;
@@ -292,15 +304,29 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
u32 cur32;
memcpy(&hdr, buf + off, 4);
- if ((hdr & 0xffff) != CE_SEMA_HDR_ADDR)
- continue;
+ /* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */
if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT)
continue;
+ if ((hdr & 0xff) != CE_SEMA_HDR_ADDR)
+ continue;
memcpy(&hi, buf + off + 4, 4);
memcpy(&lo, buf + off + 8, 4);
memcpy(&payload, buf + off + 12, 4);
+ /* Require the trailing LAUNCH_DMA group: dramatically reduces
+ * false positives when scanning unrelated memory. */
+ if (off + 20 <= SEMA_PAGE_SIZE) {
+ u32 dma;
+
+ memcpy(&dma, buf + off + 16, 4);
+ if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT ||
+ (dma & 0xff) != CE_DMA_HDR_ADDR)
+ continue;
+ } else {
+ continue;
+ }
+
if (hi > 0xffff || payload == 0 || payload > 0x10000000)
continue;
@@ -310,48 +336,142 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
svc.releases_seen++;
- if (svc_map_lookup(va, &pa))
+ if (svc_map_lookup(va, &pa)) {
+ svc.releases_missed++;
continue;
+ }
- cur = 0;
+ cur32 = 0;
if (svc_read(vgpu, pa, &cur32, sizeof(cur32)))
continue;
cur = cur32;
- /* in-order completion only */
- if (cur + 1 != payload)
+ /*
+ * Forward progress only. A payload at or below the slot's
+ * current value is a stale copy of an already-completed push.
+ * Any payload ahead of the slot is applied even across a gap:
+ * UVM tolerates forward jumps up to
+ * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and
+ * a gap only means we missed seeing an intermediate push (push
+ * buffers are recycled). Bound the jump to keep false
+ * positives harmless.
+ */
+ if (payload <= cur || payload - cur > 0x100000) {
+ svc.releases_stale++;
continue;
+ }
if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) {
svc.releases_applied++;
applied++;
- if (svc.releases_applied <= 16)
+ if (payload != cur + 1) {
+ svc.releases_jumped++;
+ pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n",
+ va, pa, payload, cur32);
+ } else if (svc.releases_applied <= 16) {
pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n",
va, pa, payload);
+ }
}
}
return applied;
}
+/*
+ * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for
+ * semaphore-release groups. Push buffers can live anywhere in the guest's
+ * system memory, including pages we have no VA mapping for; only the
+ * *semaphore* address inside a release needs to resolve through the map.
+ */
+#define SVC_SCAN_WINDOW (256ULL << 20) /* 256 MB per pass */
+static u64 svc_scan_cursor;
+
static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
{
static u8 *buf;
- u32 i;
+ u64 start = svc_scan_cursor;
+ u64 end = start + SVC_SCAN_WINDOW;
+ u64 gpa;
if (!buf) {
buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
if (!buf)
return;
}
+ if (end > 0x140000000ULL)
+ end = 0x140000000ULL;
+
+ for (gpa = start; gpa < end; gpa += SEMA_PAGE_SIZE)
+ svc_apply_page(vgpu, buf, gpa);
- for (i = 0; i < svc.map_n; i++)
- svc_apply_page(vgpu, buf, svc.map[i].pa_page);
+ svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end;
}
/* ------------------------------------------------------------------ */
/* thread */
/* ------------------------------------------------------------------ */
+/*
+ * Diagnostic probe: dump the service's view of one GPU VA (map entry and
+ * the current slot values). Runs in the service thread because guest-DMA
+ * reads require a kthread context; the sysfs writer waits for completion.
+ */
+static struct completion probe_done;
+static u64 probe_va;
+static u32 probe_n;
+static bool probe_pending;
+
+static void svc_handle_probe(struct nv_vgpu_device *vgpu)
+{
+ u64 pa = 0;
+ u32 i;
+
+ if (!probe_pending)
+ return;
+
+ pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n",
+ probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+ svc.direct_base, svc.map_n);
+
+ if (svc_map_lookup(probe_va, &pa)) {
+ pr_info("sema-probe: NO MAP ENTRY for page 0x%llx\n",
+ probe_va & ~(u64)(SEMA_PAGE_SIZE - 1));
+ } else {
+ pr_info("sema-probe: pa=0x%llx\n", pa);
+ for (i = 0; i < probe_n; i++) {
+ u32 v = 0;
+
+ if (svc_read(vgpu, pa + 4ULL * i, &v, sizeof(v))) {
+ pr_info("sema-probe: [%u] read failed\n", i);
+ break;
+ }
+ pr_info("sema-probe: [%u] 0x%08x\n", i, v);
+ }
+ }
+
+ probe_pending = false;
+ complete(&probe_done);
+}
+
+int gsp_sema_service_probe(u64 gpu_va, u32 count)
+{
+ if (!svc.task)
+ return -ENODEV;
+ if (!count || count > 64)
+ count = 8;
+
+ init_completion(&probe_done);
+ probe_va = gpu_va;
+ probe_n = count;
+ probe_pending = true;
+
+ if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) {
+ probe_pending = false;
+ return -ETIMEDOUT;
+ }
+ return 0;
+}
+
static int svc_thread(void *arg)
{
struct nv_vgpu_device *vgpu;
@@ -378,6 +498,8 @@ static int svc_thread(void *arg)
continue;
}
+ svc_handle_probe(vgpu);
+
if (!svc.direct_base ||
time_after(jiffies, svc.last_build + 10 * HZ))
svc_rebuild(vgpu);
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 12:05:26 +0000