diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 99 |
1 files changed, 96 insertions, 3 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index fd70869..5fd583a 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -716,6 +716,13 @@ struct sema_svc { u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; + /* + * GR launch completion releases (the QMD-level semaphore the physical + * engine authors on success) that the emulation had to write itself + * because the hardware copy did not deliver them (forward timeout, + * abort, or a release0 page the backend could not observe). + */ + u64 fwd_gr_sw_rel; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -3294,14 +3301,16 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.fwd_rel_deferred || svc.fwd_rel_applied || svc.fwd_rel_fallback || svc.fwd_rel_pending || svc.fwd_rel_overflow || svc.fwd_hold_engaged || - svc.fwd_hold_timeouts || svc.fwd_hold_canceled) - pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu\n", + svc.fwd_hold_timeouts || svc.fwd_hold_canceled || + svc.fwd_gr_sw_rel) + pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu gr_sw_rel=%llu\n", svc.fwd_rel_deferred, svc.fwd_rel_applied, svc.fwd_rel_fallback, svc.fwd_rel_pending, svc.fwd_rel_overflow, svc.fwd_hold_engaged, svc.fwd_hold_timeouts, svc.fwd_hold_canceled, svc.fwd_copy_skipped, svc.fwd_copy_soft, - svc.fwd_copy_fail, svc.rel_hold_direct); + svc.fwd_copy_fail, svc.rel_hold_direct, + svc.fwd_gr_sw_rel); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; @@ -10364,6 +10373,15 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) * failure / 500 ms timeout / abort, drop on a vanished channel. */ #define SVC_FWD_HOLD_TIMEOUT_MS 500u +/* + * A GR launch hold waits for the physical submission's bounded outcome + * (the backend's GR poll is 5 s) before its software fallback fires; the CE + * copy bound stays 500 ms. Expiring a GR hold while the request is still in + * flight would author the launch completion release before the engine has + * finished writing the guest pages, so the client could observe data the + * hardware has not produced yet. + */ +#define SVC_FWD_GR_HOLD_TIMEOUT_MS 12000u struct svc_fwd_hold { bool active; @@ -10537,8 +10555,54 @@ static void svc_fwd_hold_direct_note(void) svc_fwd_hold.rel[i].payload); } +/* + * Author a GR launch completion release (the QMD-level semaphore the + * physical engine writes when the launch completes) in software, + * forward-only. Used when a hold resolves without a successful hardware + * copy: the backend timed out or aborted, or release0 could not be observed + * on the guest page so only the module semaphore carried the outcome. + * Without this the guest tracker keeps polling a word the engine never + * wrote and the client wedges with the pump idle (ISSUE-0002: the second + * session's forwarded launch timed out and the client spun in the launch + * completion wait forever). A word already at/ahead of the payload is left + * alone - the engine, the CUDA bridge or an earlier fallback authored it. + */ +static void svc_gr_sw_release(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 payload) +{ + u32 cur = 0; + + if (!vgpu || !va || !payload) + return; + if (!svc_sem_read(vgpu, va, &cur) && cur >= payload) + return; + if (!svc_sem_write(vgpu, h, va, payload)) { + svc.fwd_gr_sw_rel++; + pr_info("vgsp-work: GR launch release software-applied h=0x%x va=0x%llx pay=0x%x (no successful hardware copy; forward-only)\n", + h, va, payload); + } +} + +/* + * The held GR request's QMD completion release, if any. The request slot is + * stable while a hold is active (svc_fwd_request refuses reuse), so the + * svc_fwd_req fields read here belong to this hold's submission. + */ +static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + if (!svc_fwd_req.gr) + return; + svc_gr_sw_release(vgpu, ch ? ch->h_object : 0, + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_payload); +} + static bool svc_fwd_hold_expired(void) { + if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) + return time_after(jiffies, svc_fwd_hold.since + + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS)); return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } @@ -10764,6 +10828,7 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); + svc_fwd_hold_gr_release(hv, &shim); } if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_direct_note(); @@ -10868,11 +10933,21 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); + svc_fwd_hold_gr_release(hv, ch); svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; if (!done) svc.fwd_hold_timeouts++; } + /* + * GR launch completion release: on a successful hardware copy the + * engine normally authored it; when release0 had no guest page to + * observe (module-semaphore fallback) the guest word is still behind, + * so author it forward-only here as well. No-op when the word already + * carries the payload (success path with a resolved release page). + */ + if (done && svc_fwd_req.hw_rc == 0) + svc_fwd_hold_gr_release(hv, ch); /* Mirror the normal consume tail for the held entry. */ svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; @@ -13059,6 +13134,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (svc_fwd_req.relay) svc.fwd_relay_fallback++; svc_fwd_req.relay = false; + /* + * A GR capture whose request + * did not queue (refused, + * backend busy, hold active) + * has no hardware copy coming: + * author its QMD completion + * release forward-only so the + * client's wait is not left to + * a launch that never ran. With + * the CUDA bridge enabled the + * bridge job owns the release. + */ + if (!ch->engine_type && + !gsp_cuda_bridge_enabled()) + svc_gr_sw_release(vgpu, + ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); } } /* |