diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 82 |
1 files changed, 80 insertions, 2 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index ce2e0c9..ae8f7da 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -350,6 +350,7 @@ struct sema_svc { u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 releases_faked; /* skipped-entry fallback */ + u64 counter_rel_applied; /* releases to registered counters */ u64 builds; u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ @@ -1766,11 +1767,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.releases_stale, svc.releases_jumped, svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits, svc_scan_cursor); - pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu\n", + pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu crel=%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail, svc.ring_ops, svc.ops_denied, - svc.false_trans, svc.gpfifo_skipped, svc.releases_faked); + svc.false_trans, svc.gpfifo_skipped, svc.releases_faked, + svc.counter_rel_applied); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -2059,6 +2061,56 @@ static bool svc_method_stream_plausible(const u8 *buf, u32 len) static u64 svc_rel_va_seen; static u32 svc_dbg_streams; +/* + * Apply a decoded release directly to a channel's registered completion + * counter slot. + * + * GR pushes release the client's per-channel completion counter and the client + * tracks push completion by polling that u32. The release VA is known from the + * push, but its VA->PA translation may be missing (the counter page is a + * client UVM allocation that is not in any scanned window) or stale (recycled + * client), while the slot's guest PA is known exactly: it is registered + * through the sema_counter parameter from the client's own pagemap. Writing + * the payload there is what lets the client's wait observe the release + * regardless of the VA map. Forward-only, so a stale copy of an + * already-completed push cannot rewind the counter. + */ +static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, + u32 payload) +{ + unsigned long flags; + u64 pa = 0; + u32 cur = 0; + int i; + + if (!h || !payload) + return; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].h_object == h) { + pa = svc.chan[i].counter_pa; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!pa) + return; + + if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) + return; + if (payload <= cur || payload - cur > 0x100000) + return; + if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &payload, + sizeof(payload))) + return; + svc.releases_applied++; + svc.counter_rel_applied++; + svc.post_hint_until = jiffies + 10 * HZ; + pr_info_ratelimited("sema-chan: counter release h=0x%x pa=0x%llx val=%u (was %u)\n", + h, pa, payload, cur); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, u32 ch_h, u32 ch_eng, u64 push_va, @@ -2482,6 +2534,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.post_hint_until = jiffies + 10 * HZ; } } + + /* + * Exact-slot fallback: the releasing channel's + * completion counter was enumerated from the + * client's pagemap, so the release can be + * applied to the very slot the client polls + * even when the release VA has no (or a stale) + * translation. Same forward-only guard as the + * VA path. + */ + svc_counter_release_apply(vgpu, ch_h, + st.sem_payload); /* release ends the push */ n_ops = 0; } @@ -3741,6 +3805,7 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { u32 val; + u32 cur = 0; if (!ch->counter_pa) return; @@ -3748,6 +3813,19 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, val = put; if (val == ch->counter_last) return; + /* + * A decoded release (svc_counter_release_apply) may already have + * advanced the slot past the put position the client last submitted; + * the client polls this u32, so never write it backwards. + */ + if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur, + sizeof(cur))) + return; + if (val <= cur) { + if (cur > ch->counter_last) + ch->counter_last = cur; + return; + } if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &val, sizeof(val))) return; |