From dd1135ced1b1e5390034239086784a44bd2fca20 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Tue, 29 Sep 2026 14:25:05 -0400 Subject: kernel: batch the map-add channel rewind; trust injected streams for ops Two fixes for the CE data path. (1) svc_handle_map_add rewound every channel by 8 ring entries on every injected mapping; a 16 MB push-buffer pin is thousands of page injections, so the pump's last_get was pinned in place and it re-read the same entries for the whole batch (3645 submissions, zero ops). The rewind now runs once per drained batch. (2) The push-start heuristic (op=INC_METHOD at address 0) rejects UVM's real CE memop pushes, which start with an IMM method at address 8 (observed 0x60020002, PTE data words) or a release method at 0x700 (0x200381c0); a buffer read through a host-injected (ipin) translation is now decoded fully trusted, ops included. Result on the next boot: memset=2718/0/0 ringops=1794 (first sustained CE work), false 12k->1.9k. semfix_watch: bulk-pin contiguous runs via sema_map_add len, read rm_mem size, 16 MB (4096 page) cap, non-blocking QMP read (the fixed 30 ms sleep made 4096 gva2gpa calls take two minutes), and KEEP_TRACE=1 to preserve the guest trace for post-mortems. --- kernel/gsp_sema_service.c | 70 +++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index ac1849f..b9e7db3 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2050,10 +2050,12 @@ static bool svc_method_stream_plausible(const u8 *buf, u32 len) * not match the channel rank). */ static u64 svc_rel_va_seen; +static u32 svc_dbg_streams; static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, - u32 ch_h, u32 ch_eng, u64 push_va) + u32 ch_h, u32 ch_eng, u64 push_va, + bool trusted) { struct svc_push_state st; /* @@ -2075,13 +2077,26 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * addr 0, sane count). Releases from such a page still apply, so a * genuinely shifted stream is not lost. */ - if (allow_ops && !svc_method_stream_plausible(buf, len)) { + if (allow_ops && !trusted && !svc_method_stream_plausible(buf, len)) { svc.ops_denied++; allow_ops = false; } memset(&st, 0, sizeof(st)); svc.pages_decoded++; + if (ch_eng && svc_dbg_streams < 24) { + svc_dbg_streams++; + pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n", + ch_h, ch_eng, push_va, len, allow_ops, + len >= 4 ? *(const u32 *)(buf + 0) : 0, + len >= 8 ? *(const u32 *)(buf + 4) : 0, + len >= 12 ? *(const u32 *)(buf + 8) : 0, + len >= 16 ? *(const u32 *)(buf + 12) : 0, + len >= 20 ? *(const u32 *)(buf + 16) : 0, + len >= 24 ? *(const u32 *)(buf + 20) : 0, + len >= 28 ? *(const u32 *)(buf + 24) : 0, + len >= 32 ? *(const u32 *)(buf + 28) : 0); + } while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; @@ -2595,7 +2610,8 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, * fabricated "pushes" executing memsets); work execution is limited * to pushes the pump read through a trusted ring translation. */ - svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page, false, 0, 0, 0); + svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page, false, 0, 0, 0, + false); for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; @@ -3898,9 +3914,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) &iap) == 0); if (!svc_method_stream_plausible(pb, len)) { svc.false_trans++; - if (injected) - allow_ops = false; - else + if (!injected) unreadable = true; } } @@ -3923,7 +3937,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) allow_ops, ch->h_object, ch->engine_type, - va); + va, + injected); if (svc_rel_va_seen != rel_before) ch->last_rel_va = svc_rel_va_seen; @@ -3990,7 +4005,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) false, ch->h_object, ch->engine_type, - va); + va, + false); if (svc_rel_va_seen != rel_before) ch->last_rel_va = svc_rel_va_seen; @@ -4380,10 +4396,12 @@ static DEFINE_SPINLOCK(svc_map_add_lock); static void svc_handle_map_add(void) { unsigned long flags; + bool rewound = false; + u32 i; for (;;) { struct svc_map_add_req req; - u32 ap, i; + u32 ap; u64 off; spin_lock_irqsave(&svc_map_add_lock, flags); @@ -4402,17 +4420,25 @@ static void svc_handle_map_add(void) svc_ipin_store(req.va + off, req.pa + off, ap); svc_lat_store(req.va + off, req.pa + off, ap); } - /* - * A translation can be replaced after the pump already - * consumed pushes that release through it (a new client maps - * new physical pages at the same VA, or the first translation - * was wrong): the releases resolved against the old page, - * compared as stale against its values, and were dropped. - * Rewind every channel a few ring entries so those pushes are - * re-read under the new translation; the release/fill - * application paths are forward-progress guarded, so - * re-executing them is idempotent. - */ + rewound = true; + pr_info_ratelimited("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n", + req.va, req.pa, req.len, + ap == SVC_APERTURE_FB ? "vidmem" : "sysmem"); + } + + /* + * Rewind once per drained batch, not once per mapping: a push-buffer + * pin injects thousands of single pages, and rewinding the channels + * on every one of them pinned the pump's last_get in place and + * re-read the same ring entries for the whole injection (observed: + * 3645 "submissions" with zero executed ops). A translation can be + * replaced after the pump consumed pushes that released through it + * (new client at the same VA, or a first translation that was + * wrong), so the batch still replays a few entries; release/fill + * application is forward-progress guarded, so re-executing is + * idempotent. + */ + if (rewound) { spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { struct svc_channel *ch = &svc.chan[i]; @@ -4440,10 +4466,6 @@ static void svc_handle_map_add(void) * stall. */ svc.fast_force_until = jiffies + 10 * HZ; - pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s), rewound channels by %u\n", - req.va, req.pa, req.len, - ap == SVC_APERTURE_FB ? "vidmem" : "sysmem", - SVC_PIN_REWIND); } } -- cgit v1.2.3