diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 119 |
1 files changed, 117 insertions, 2 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index a203a66..ffc8ae2 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -138,6 +138,7 @@ struct svc_channel { u32 last_get; /* next GPFIFO entry we have still to run */ u64 submissions; /* GPFIFO entries executed */ u32 expected_sem_payload; /* next semaphore payload we expect for this channel's tracking sem */ + u32 logged_expected; /* last pool value logged (rate limit) */ }; /* @@ -264,6 +265,12 @@ struct sema_svc { u64 last_build; u64 last_scan; + /* full-RAM apply pass fallback for channels whose pushes cannot be read */ + u64 fast_cursor; + u64 fast_passes; + u64 fast_pass_applied; + u64 fast_hold_until; + /* * Completion hint window: after a channel completes work the client is * waiting on, keep re-posting the event for a few seconds. A client @@ -2213,6 +2220,81 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu) svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end; } +/* + * Fast full-RAM apply pass, sliced. + * + * The rolling sweep visits only 32 MB per service pass, so a push page that + * appears just after the cursor passed it waits a full cycle (minutes at the + * current rebuild cost) before its release is applied. When a channel has + * pending GPFIFO entries the pump cannot execute (missing or wrong VA + * translation), sweep guest RAM quickly instead: each service pass decodes + * one slice, so the client wait clears within a couple of seconds while the + * thread stays responsive to the sysfs control path. No separate rate limit: + * the slice cursor is the pacing, and a clean pass re-runs only while some + * channel is still stalled. + */ +#define SVC_FAST_SLICE (256ULL << 20) + +static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) +{ + static u8 *buf; + u64 start = svc.fast_cursor; + u64 end = start + SVC_FAST_SLICE; + u64 gpa; + u32 pages = 0; + bool complete; + + if (end > SEMA_PA_MAX) + end = SEMA_PA_MAX; + complete = (end >= SEMA_PA_MAX); + + /* + * Back off after a full pass that applied nothing: a stalled channel + * with no applicable release in RAM would otherwise keep a 6 GB pass + * running forever. + */ + if (start == 0) { + if (time_before(jiffies, (unsigned long)svc.fast_hold_until)) + return; + svc.fast_pass_applied = svc.releases_applied; + } + + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return; + } + + for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { + u32 o; + + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) + continue; + for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) { + u64 seen = svc.releases_seen; + + svc_apply_page_data(vgpu, buf + o, gpa + o); + if (svc.releases_seen != seen) { + svc_mark_hot(gpa + o); + pages++; + } + } + } + + svc.fast_cursor = complete ? 0 : end; + if (complete) { + svc.fast_passes++; + if (svc.releases_applied == svc.fast_pass_applied) + svc.fast_hold_until = jiffies + 3 * HZ; + if (svc.releases_applied != svc.fast_pass_applied || + svc.fast_passes <= 8) + pr_info("sema-svc: fast sweep #%llu done: applied=%llu (+%llu), pages=%u\n", + svc.fast_passes, svc.releases_applied, + svc.releases_applied - svc.fast_pass_applied, + pages); + } +} + static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 hdr, hi, lo, payload; @@ -2764,8 +2846,12 @@ static void svc_init_expected_payload(struct nv_vgpu_device *vgpu, if (cur > 0 && cur < 0x100000) { ch->expected_sem_payload = cur + 1; - pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n", - ch->h_object, ch->ce_index, pool_va, cur, ch->expected_sem_payload); + if (ch->logged_expected != cur) { + ch->logged_expected = cur; + pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n", + ch->h_object, ch->ce_index, pool_va, cur, + ch->expected_sem_payload); + } } } @@ -2775,6 +2861,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) static struct svc_channel local[SVC_MAX_CHANNELS]; unsigned long flags; int i, k, n = 0; + bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) @@ -2788,6 +2875,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) int ring_ap = SVC_APERTURE_SYS; int rrc; u32 put = 0, get = 0, guard = 0; + u32 get_before; + u64 applied_before; svc_notify_complete(vgpu, ch); @@ -2829,6 +2918,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; + get_before = ch->last_get; + applied_before = svc.releases_applied; + while (ch->last_get != put && guard++ < ch->gpfifo_entries) { u8 entry[SVC_GPFIFO_ENTRY_SIZE]; u64 va, idx = ch->last_get; @@ -2920,6 +3012,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; svc.chan[k].logged = ch->logged; + svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; svc.chan[k].last_get = ch->last_get; svc.chan[k].submissions = ch->submissions; @@ -2927,7 +3020,29 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } spin_unlock_irqrestore(&svc.chan_lock, flags); + + if (ch->last_get != put) + stalled = true; + else if (ch->last_get != get_before && + ch->expected_sem_payload && + svc.releases_applied == applied_before) + /* + * Entries were consumed but nothing was released: the + * translation points at a stale/wrong page, so the + * push's semaphore release was never performed. + */ + stalled = true; } + + /* + * A channel still has pending entries after the pass: its pushes either + * fail translation or decode to nothing (stale/wrong VA->PA pair). A + * fast content sweep applies the releases those pages hold, so the + * client's tracker wait clears now instead of after the slow rolling + * sweep wraps around. + */ + if (stalled) + svc_apply_sweep_slice(vgpu); } /* ------------------------------------------------------------------ */ |