From 17e1f522e4e9e04d13c4ca9495a2f4f1bdc58c36 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Mon, 28 Sep 2026 07:34:34 -0400 Subject: sema-svc: apply releases promptly and decode compute-push releases The content sweep now: - decodes a release whose method stream continues with any INC_METHOD, not only the CE LAUNCH_DMA group, so the releases at the head of a compute/GR channel's first push are applied too; - remembers releases whose target VA had no translation (deferred list) and retries them every service pass, so a VA->PA injection lands them immediately instead of after the next full sweep; - re-arms the fast sliced sweep for a few seconds after an injection; - treats a channel whose USERD/ring cannot be resolved as stalled, so the sweep keeps looking for the releases its unreadable pushes carry. tools: add per-cdt client-pin helper (sem pages + /dev/nvidia0 windows), sem-page scanner and memprobe fixes (short-read guard, --brief). Live-verified: injecting a client sem page applied 12 deferred releases within a second; the service kept applying (89 -> 202) while cdt ran. --- kernel/gsp_sema_service.c | 151 +++++++++++++++++++++++++++++++++--- tools/guest-diag/memprobe.py | 11 ++- tools/guest-diag/pin_client_sems.sh | 66 ++++++++++++++++ tools/guest-diag/sempages.py | 37 +++++++++ 4 files changed, 252 insertions(+), 13 deletions(-) create mode 100755 tools/guest-diag/pin_client_sems.sh create mode 100755 tools/guest-diag/sempages.py diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1f1222e..4cfed0c 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -270,6 +270,13 @@ struct sema_svc { u64 fast_passes; u64 fast_pass_applied; u64 fast_hold_until; + /* + * A VA->PA injection (or a batch of them) invalidates the "nothing to + * apply" assumption the fast sweep backs off on: releases that missed + * translation may be applicable now. Keep the fast sweep armed for a + * few seconds after any injection. + */ + u64 fast_force_until; /* * Completion hint window: after a channel completes work the client is @@ -2018,6 +2025,81 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* push application */ /* ------------------------------------------------------------------ */ +/* + * Deferred releases. + * + * A release group can be visible in guest RAM long before its target VA has a + * trusted translation: the page may be scanned before the client mapping is + * pinned, or before the pair scan learns the VA. Dropping such a release and + * waiting for the next full sweep costs minutes; instead remember the + * (VA, payload) and retry it on every service pass. As soon as a pin or a + * learned pair makes the VA translatable the release lands, which is exactly + * what a client waiting on that semaphore needs. + */ +#define SVC_PENDING_MAX 512 +struct svc_pending_release { + u64 va; + u32 payload; + u32 op; /* 0 = RELEASE, 1 = REDUCTION (add payload) */ +}; + +static struct svc_pending_release svc_pending[SVC_PENDING_MAX]; +static u32 svc_pending_n; +static u32 svc_pending_next; + +static void svc_pending_add(u64 va, u32 payload, u32 op) +{ + u32 i; + + if (!payload) + return; + for (i = 0; i < svc_pending_n; i++) + if (svc_pending[i].va == va && + svc_pending[i].payload >= payload) + return; + if (svc_pending_n < SVC_PENDING_MAX) { + i = svc_pending_n++; + } else { + i = svc_pending_next++ % SVC_PENDING_MAX; + } + svc_pending[i].va = va; + svc_pending[i].payload = payload; + svc_pending[i].op = op; +} + +static void svc_apply_pending(struct nv_vgpu_device *vgpu) +{ + u32 i = 0; + + while (i < svc_pending_n) { + struct svc_pending_release *p = &svc_pending[i]; + u32 cur = 0; + + if (svc_sem_read(vgpu, p->va, &cur)) { + i++; /* translation still missing: keep waiting */ + continue; + } + if (p->payload <= cur) { + svc_pending[i] = svc_pending[--svc_pending_n]; + continue; + } + if (p->payload - cur > 0x100000) { + svc_pending[i] = svc_pending[--svc_pending_n]; + continue; + } + if (!svc_sem_write(vgpu, p->va, + p->op ? cur + p->payload : p->payload)) { + svc.releases_applied++; + svc.post_hint_until = jiffies + 10 * HZ; + pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n", + p->va, p->payload, cur); + svc_pending[i] = svc_pending[--svc_pending_n]; + continue; + } + i++; /* unresolved: try again next pass */ + } +} + /* * Look for CE semaphore-release method groups inside mapped pages: * @@ -2025,7 +2107,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * [upper 32 bits of semaphore VA] * [lower 32 bits of semaphore VA] * [payload] - * [... LAUNCH_DMA ...] + * [method stream continues] * * Apply by writing payload to the semaphore slot, but only in order * (current == payload - 1) so that UVM's tracker never sees a jump. @@ -2058,15 +2140,29 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, memcpy(&lo, buf + off + 8, 4); memcpy(&payload, buf + off + 12, 4); - /* Require the trailing LAUNCH_DMA group: dramatically reduces - * false positives when scanning unrelated memory. */ + /* + * The word after the release must look like the continuation of + * a method stream: the CE LAUNCH_DMA this group normally + * precedes, or another INC_METHOD header. Compute/GR pushes put + * arbitrary methods after a release (channel setup, GPFIFO + * updates), so requiring LAUNCH_DMA alone silently drops their + * releases - the client tracker semaphores those pushes release + * are exactly the ones clients wait on. Random + * {0x20038090, hi, lo, payload} runs in unrelated data almost + * never satisfy either shape. + */ if (off + 20 <= SEMA_PAGE_SIZE) { u32 dma; memcpy(&dma, buf + off + 16, 4); if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT || - (dma & 0xff) != CE_DMA_HDR_ADDR) - continue; + (dma & 0xff) != CE_DMA_HDR_ADDR) { + u32 type = (dma >> 29) & 0x7; + u32 cnt = (dma >> 16) & 0xfff; + + if (type < 1 || type > 4 || !cnt || cnt > 16) + continue; + } } else { continue; } @@ -2083,6 +2179,7 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, cur32 = 0; if (svc_sem_read(vgpu, va, &cur32)) { svc.releases_missed++; + svc_pending_add(va, payload, 0); if (svc.releases_missed_log < 16 && va != svc.last_miss_va) { svc.last_miss_va = va; @@ -2251,10 +2348,12 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) /* * Back off after a full pass that applied nothing: a stalled channel * with no applicable release in RAM would otherwise keep a 6 GB pass - * running forever. + * running forever. A recent VA->PA injection clears the hold: the + * translation landscape changed, so the sweep can apply something now. */ if (start == 0) { - if (time_before(jiffies, (unsigned long)svc.fast_hold_until)) + if (time_before(jiffies, (unsigned long)svc.fast_hold_until) && + time_after(jiffies, (unsigned long)svc.fast_force_until)) return; svc.fast_pass_applied = svc.releases_applied; } @@ -2892,8 +2991,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, - rrc ? -1 : ring_ap, &put, &get)) + rrc ? -1 : ring_ap, &put, &get)) { + /* + * The channel's work cannot be run (USERD not + * found): its pushes stay pending, so the release + * it carries will never be applied by the pump. + * Treat as stalled so the content sweep looks for + * the releases in RAM. + */ + stalled = true; continue; + } } else { if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, ch->userd_as, &put, sizeof(put))) @@ -2926,8 +3034,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 va, idx = ch->last_get; u32 len, e0, e1, g; - if (rrc) - break; /* cannot read the ring at all */ + if (rrc) { + stalled = true; /* ring unreadable: sweep RAM */ + break; + } if (svc_read_va(vgpu, ch->gpfifo_va + @@ -3216,6 +3326,12 @@ static void svc_handle_map_add(void) svc_pin_store(map_add_va + off, map_add_pa + off, ap); svc_lat_store(map_add_va + off, map_add_pa + off, ap); } + /* + * Releases that missed translation before this injection may be + * applicable now: keep the fast full-RAM sweep armed for a few + * seconds so they land without waiting for an unrelated stall. + */ + svc.fast_force_until = jiffies + 10 * HZ; pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n", map_add_va, map_add_pa, map_add_len, ap == SVC_APERTURE_FB ? "vidmem" : "sysmem"); @@ -3289,6 +3405,21 @@ static int svc_thread(void *arg) svc.doorbell_pending = false; svc_pump_channels(vgpu); + /* + * Releases whose target VA was not translatable when the scan + * saw them: retry every pass, so a pin or a learned pair applies + * them immediately instead of on the next full sweep. + */ + svc_apply_pending(vgpu); + + /* + * A fresh VA->PA injection changes which releases are + * applicable: run the fast sliced sweep while the forced window + * is open (the pump's stall path also uses it). + */ + if (time_before(jiffies, (unsigned long)svc.fast_force_until)) + svc_apply_sweep_slice(vgpu); + /* * Push-buffer base discovery: a chunked walk of guest RAM * finds the pages whose first push names a channel, so a diff --git a/tools/guest-diag/memprobe.py b/tools/guest-diag/memprobe.py index 26ed0af..96dd40b 100644 --- a/tools/guest-diag/memprobe.py +++ b/tools/guest-diag/memprobe.py @@ -126,8 +126,10 @@ def dump_tracker(pid, fd, p): if sob: print(" sem_obj[0:0x40]=%s" % sob.hex()) mem = r64(sob, 0x10) - mvb = rd(fd, mem & ~0xfff, 0x40) - print(" sem_mem=0x%x val@+0x%x=0x%x" % (mem, mem & 0xfff, r32(mvb, mem & 0xfff) if mvb else -1)) + mvb = rd(fd, mem & ~0xfff, 0x1000) + off = mem & 0xfff + val = r32(mvb, off) if mvb and off + 4 <= len(mvb) else -1 + print(" sem_mem=0x%x val@+0x%x=0x%x" % (mem, off, val)) q = r64(rd(fd, p + 0x48, 8) or b"\0" * 8) print(" prog_obj=0x%x next=0x%x cblist=0x%x idx95bc=0x%x p34=0x%x" % (q, r64(rd(fd, p + 0x9470, 8) or b"\0" * 8), @@ -154,15 +156,18 @@ def main(): if mode == "scan": lo, hi = 0x204800000, 0x204c0000 limit = 8 + brief = False for i, a in enumerate(sys.argv[3:]): if a == "--min": lo = int(sys.argv[4 + i], 16) if a == "--max": hi = int(sys.argv[4 + i], 16) if a == "--limit": limit = int(sys.argv[4 + i]) + if a == "--brief": brief = True res = find_trackers(pid, fd, lo, hi, limit) for p, sub, last, cur, sem, mem in res: print("P=0x%x sub=0x%x last=0x%x cur=0x%x sem_obj=0x%x sem_mem=0x%x" % (p, sub, last, cur, sem, mem or 0)) - dump_tracker(pid, fd, p) + if not brief: + dump_tracker(pid, fd, p) elif mode == "dump": dump_tracker(pid, fd, int(sys.argv[3], 16)) elif mode == "hex": diff --git a/tools/guest-diag/pin_client_sems.sh b/tools/guest-diag/pin_client_sems.sh new file mode 100755 index 0000000..82d2789 --- /dev/null +++ b/tools/guest-diag/pin_client_sems.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# pin_client_sems.sh - bridge the CUDA client's device mappings into the +# emulation for the live cdt process. +# +# Two things are pinned, both per-process (a new cdt maps new physical pages +# at the same VAs, so re-run this for every cdt run): +# 1. the /dev/nvidia0 window: the driver's channel rings/USERDs (the GR +# channels' GPFIFO VAs live here), so the pump can run their pushes; +# 2. the semaphore pages libcuda's trackers poll (found by scanning the +# client's memory for sem references, then translated through the guest +# pagemap). +# +# Usage: tools/guest-diag/pin_client_sems.sh [iterations] [sleep-secs] +set -u +cd "$(dirname "$(readlink -f "$0")")/../.." +VMEXEC=./scripts/vgpu_vm.py +ITER=${1:-1} +SLEEP=${2:-10} + +PID=$(timeout 20 $VMEXEC exec --timeout 15 'pgrep -x cdt | head -1' 2>/dev/null | tail -1) +[ -n "$PID" ] || { echo "no cdt pid"; exit 1; } +echo "cdt pid=$PID" + +B64=$(gzip -9 < tools/guest-diag/sempages.py | base64 -w0) +timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d | gunzip > /tmp/sempages.py" >/dev/null 2>&1 +B64=$(gzip -9 < tools/guest-diag/memprobe.py | base64 -w0) +timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d | gunzip > /tmp/memprobe.py" >/dev/null 2>&1 +B64=$(base64 -w0 tools/guest-diag/guest_pmap_all.py) +timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d > /tmp/pmapall.py" >/dev/null 2>&1 + +pin_range() { # pin_range