diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 180 |
1 files changed, 147 insertions, 33 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 83ca6bd..0605692 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -275,6 +275,13 @@ struct svc_channel { * counter page can land a moment after the * first push decode); flushed as soon as the * slot is registered. 0 = none */ + u32 pending_rel_inc; /* 1 = the queued release is an increment + * (add payload to the slot), 0 = absolute */ + u32 relearn_n; /* counter-slot re-registrations seen for this + * channel: a healthy GR setup re-learns each + * slot a bounded number of times, while a + * clobbered slot loops forever (storm + * detector) */ }; /* @@ -516,6 +523,9 @@ struct sema_svc { * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ u64 counter_rel_dbg; /* rel-apply diagnostic budget */ + u64 counter_noslot_dbg; /* "no slot registered" budget, + * separate so it cannot hide the + * written/skip lines */ u64 counter_pub_dbg; /* counter-publish diagnostic budget */ u64 counter_pub_calls; /* publishes that found a registered counter */ u64 gr_rel_applied; /* release writes into the GR counter page */ @@ -4238,14 +4248,20 @@ static inline bool svc_counter_va(u64 va) * through the sema_counter parameter from the client's own pagemap. Writing * the payload there is what lets the client's wait observe the release * regardless of the VA map. Forward-only, so a stale copy of an - * already-completed push cannot rewind the counter. + * already-completed push cannot rewind the counter. `inc` selects UVM's + * increment-release semantics (write cur + payload) instead of the absolute + * semantics (write payload); GR tracking-semaphore pushes release an + * increment, and an absolute-only guard drops every increment after the + * first one because payload == cur. */ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, - u64 sem_va, u32 payload) + u64 sem_va, u32 payload, bool inc) { unsigned long flags; u64 pa = 0; u32 cur = 0; + u32 nv; + bool skip; int i; u32 dbg = 0; @@ -4284,9 +4300,12 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, if (svc.chan[i].valid && svc.chan[i].h_object == h) { if (payload > - svc.chan[i].pending_rel_pay) + svc.chan[i].pending_rel_pay) { svc.chan[i].pending_rel_pay = payload; + svc.chan[i].pending_rel_inc = + inc ? 1 : 0; + } break; } } @@ -4296,8 +4315,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> slot pending\n", h, sem_va, payload); } - } else if (gsp_sema_dbg && svc.counter_rel_dbg < 96) { - svc.counter_rel_dbg++; + } else if (gsp_sema_dbg && svc.counter_noslot_dbg < 16) { + svc.counter_noslot_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> no slot registered\n", h, sem_va, payload); } @@ -4312,7 +4331,17 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, } return; } - if (payload <= cur || payload - cur > 0x100000) { + /* + * Absolute releases must be forward-only (a stale copy of a completed + * push must not rewind the counter); increment releases add the + * payload, like the generic LAUNCH path does. The jump guard keeps a + * bogus payload from teleporting the counter past the client's wait. + */ + if (inc) + skip = payload > 0x100000; + else + skip = payload <= cur || payload - cur > 0x100000; + if (skip) { if (gsp_sema_dbg && svc.counter_rel_dbg < 32) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> skip\n", @@ -4320,8 +4349,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, } return; } - if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &payload, - sizeof(payload))) + nv = inc ? cur + payload : payload; + if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv))) return; svc.releases_applied++; svc.counter_rel_applied++; @@ -4336,34 +4365,74 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, } /* + * Clear a channel's queued release in the authoritative array. The caller of + * svc_counter_flush_pending() usually works on a pass snapshot, so clearing + * only the snapshot would let the next pass flush the same payload again - + * harmless for an absolute release (its guard drops it), but an increment + * would advance the slot twice. + */ +static void svc_counter_pending_clear(u32 h) +{ + unsigned long flags; + int i; + + if (!h) + return; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].h_object == h) { + svc.chan[i].pending_rel_pay = 0; + svc.chan[i].pending_rel_inc = 0; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +/* * Deliver a release that was decoded before the channel's counter slot was * registered (see pending_rel_pay). Called whenever a slot becomes known and * from the per-pass publish path, so a queued payload lands as soon as the - * injected pin is visible to the service. + * injected pin is visible to the service. An increment-queued payload adds to + * the slot (UVM INC semantics), an absolute one writes its value. */ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u32 payload = ch->pending_rel_pay; + bool inc = ch->pending_rel_inc; u32 cur = 0; + u32 nv; if (!payload || !ch->counter_pa) return; if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) return; - if (payload <= cur || payload - cur > 0x100000) { + if (inc) { + if (payload > 0x100000) { + ch->pending_rel_pay = 0; + ch->pending_rel_inc = 0; + svc_counter_pending_clear(ch->h_object); + return; + } + } else if (payload <= cur || payload - cur > 0x100000) { ch->pending_rel_pay = 0; + ch->pending_rel_inc = 0; + svc_counter_pending_clear(ch->h_object); if (cur > ch->counter_last) ch->counter_last = cur; return; } - if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &payload, - sizeof(payload))) + nv = inc ? cur + payload : payload; + if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &nv, + sizeof(nv))) return; ch->pending_rel_pay = 0; - if (payload > ch->counter_last) - ch->counter_last = payload; + ch->pending_rel_inc = 0; + svc_counter_pending_clear(ch->h_object); + if (nv > ch->counter_last) + ch->counter_last = nv; svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; @@ -4494,33 +4563,61 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, return; derived = true; } - ch->counter_pa = pa & ~0x3ULL; + pa &= ~0x3ULL; /* - * Persist the registration into the authoritative channel array - * immediately, under the lock, instead of relying on the pump's - * end-of-pass write-back: the pump works on a snapshot and several - * early `continue`s (USERD not yet resolvable, ring not yet - * readable) skip that write-back. When that happened every pass - * re-registered the same slot - the "counter auto" line flooded the - * log ring (1.17M lines, noclaim27) and, worse, the publish path - * never saw a registered counter (`pub_calls=0`), so the client's - * tracker semaphores were never advanced and cudaMalloc hung. The - * authoritative store makes the registration durable and the log - * one-shot. + * Never install a slot another live channel already owns: the + * rank-derived formula is evaluated while channels are still + * registering and can name a slot a lower rank already registered + * (observed: rank3 -> rank0's slot ...ec0). The collision makes every + * pass flip the slot (svc_counter_learn writes the exact slot, the + * pass write-back restores the guess) and the client's tracker never + * completes; leave counter_pa zero for a later pass instead. */ { unsigned long flags; int j; + bool collision = false; spin_lock_irqsave(&svc.chan_lock, flags); for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && - svc.chan[j].h_object == ch->h_object) { - svc.chan[j].counter_pa = ch->counter_pa; + svc.chan[j].h_object != ch->h_object && + svc.chan[j].counter_pa == pa) { + collision = true; break; } } + if (!collision) { + /* + * Persist the registration into the authoritative + * channel array immediately, under the lock, instead + * of relying on the pump's end-of-pass write-back: + * the pump works on a snapshot and several early + * `continue`s (USERD not yet resolvable, ring not + * yet readable) skip that write-back. When that + * happened every pass re-registered the same slot + * (1.17M log lines, noclaim27) and, worse, the + * publish path never saw a registered counter + * (`pub_calls=0`), so the client's tracker + * semaphores were never advanced and cudaMalloc + * hung. The authoritative store makes the + * registration durable and the log one-shot. + */ + ch->counter_pa = pa; + for (j = 0; j < SVC_MAX_CHANNELS; j++) { + if (svc.chan[j].valid && + svc.chan[j].h_object == ch->h_object) { + svc.chan[j].counter_pa = pa; + break; + } + } + } spin_unlock_irqrestore(&svc.chan_lock, flags); + if (collision) { + pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n", + ch->h_object, ch->gr_index, pa); + return; + } } if (derived) pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", @@ -4570,6 +4667,11 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) if (svc.chan[i].valid && svc.chan[i].h_object == h) { if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) { svc.chan[i].counter_pa = pa & ~0x3ULL; + svc.chan[i].relearn_n++; + if (svc.chan[i].relearn_n == 64) + pr_warn_ratelimited("sema-chan: counter re-learn storm h=0x%x n=%u (slot clobbered; UVM will stall)\n", + h, + svc.chan[i].relearn_n); if (derived) pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n", h, pa & ~0x3ULL, sem_va); @@ -4600,7 +4702,8 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, if (ch->h_object && svc_counter_va(va)) { svc_counter_learn(vgpu, ch->h_object, va); - svc_counter_release_apply(vgpu, ch->h_object, va, payload); + svc_counter_release_apply(vgpu, ch->h_object, va, payload, + op == HOST_SEM_OP_REDUCTION); return false; } if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { @@ -4623,7 +4726,8 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, if (ch->h_object && svc_counter_va(va)) { svc_counter_learn(vgpu, ch->h_object, va); - svc_counter_release_apply(vgpu, ch->h_object, va, payload); + svc_counter_release_apply(vgpu, ch->h_object, va, payload, + false); return false; } if (svc_sem_read(vgpu, va, &cur)) { @@ -4668,7 +4772,8 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, } if (ch->engine_type == 0) svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); return false; } @@ -11226,9 +11331,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * snapshot: without this the next pass re-derives * it (with a different live GR count during * teardown) and the publishes target the wrong - * slot. + * slot. Never let the pass-start snapshot revert + * a slot learned mid-pass (svc_counter_learn) or + * registered from the injected pin: the snapshot + * may still hold the rank-derived autoreg guess + * (or 0), which made the registration + * non-durable (24,834 re-learn lines in one + * stalled run vs <=35 per channel in passing + * runs, 2026-10-06). Only fill a missing slot; a + * live authoritative value wins. */ - svc.chan[k].counter_pa = ch->counter_pa; + if (!svc.chan[k].counter_pa) + svc.chan[k].counter_pa = ch->counter_pa; break; } } |