diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 159 |
1 files changed, 149 insertions, 10 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f8925df..373da64 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -816,6 +816,13 @@ struct sema_svc { u64 fwd_gr_fault; u64 fwd_gr_bridge; /* + * m156 fail-closed releases: data-bearing forwarded GR launches whose + * completion release was withheld because no executor was witnessed + * (the engine acked but wrote nothing), with a channel error + * delivered instead. Counts the gate firing, not backend failures. + */ + u64 fwd_gr_failclosed; + /* * GR launch completion releases (the QMD-level semaphore the physical * engine authors on success) that the emulation had to write itself * because the hardware copy did not deliver them (forward timeout, @@ -3507,15 +3514,15 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.fwd_rel_fallback || svc.fwd_rel_pending || svc.fwd_rel_overflow || svc.fwd_hold_engaged || svc.fwd_hold_timeouts || svc.fwd_hold_canceled || - svc.fwd_gr_sw_rel) - pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu gr_sw_rel=%llu\n", + svc.fwd_gr_sw_rel || svc.fwd_gr_failclosed) + pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu gr_sw_rel=%llu gr_failclosed=%llu\n", svc.fwd_rel_deferred, svc.fwd_rel_applied, svc.fwd_rel_fallback, svc.fwd_rel_pending, svc.fwd_rel_overflow, svc.fwd_hold_engaged, svc.fwd_hold_timeouts, svc.fwd_hold_canceled, svc.fwd_copy_skipped, svc.fwd_copy_soft, svc.fwd_copy_fail, svc.rel_hold_direct, - svc.fwd_gr_sw_rel); + svc.fwd_gr_sw_rel, svc.fwd_gr_failclosed); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; @@ -10907,6 +10914,17 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) * hardware has not produced yet. */ #define SVC_FWD_GR_HOLD_TIMEOUT_MS 12000u +/* + * m156: a data-bearing GR launch under the fail-closed gate can legitimately + * spend the whole cold-context recovery chain before its outcome is + * published (5 s initial poll + re-arm + 5 s + up to three 2 s fresh-entry + * retries plus the retirement waits). Expiring the hold inside that window + * would fail a launch that is still retrying, so the gate raises the GR + * hold bound to this value; the backend is itself bounded, so the work + * item's done always arrives before it, and the value remains the + * last-resort watchdog. + */ +#define SVC_FWD_GR_HOLD_GATED_MS 40000u struct svc_fwd_hold { bool active; @@ -11416,11 +11434,98 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_rel_payload); } +/* + * m156 fail-closed witness. A data-bearing forwarded GR launch's completion + * release may only be authored when an executor witnessed the launch: the + * backend returns success only after the retirement wait observed engine + * writes (gsp_phys_rmcall_gr_failclosed), and the CUDA bridge owns a launch + * it executed. The engine's own ack (qmd_rel/ch_rel) is a front-end + * retirement, not an execution witness - r155 live: streams2's launch + * logged qmd_rel=6/ch_rel=0xc0ffee02 with "0/205 pages changed" on the + * initial attempt and all three outcome retries, the backend returned + * success, and the client read all-zero output (STREAMS bad=32768). A + * launch with no resolved data pages has no output to lose and keeps the + * historical release behavior. + */ +static bool svc_gr_rel_witnessed(bool done) +{ + if (!gsp_phys_rmcall_gr_failclosed_value()) + return true; /* gate disabled: historical behavior */ + if (!svc_fwd_req.gr || !svc_fwd_req.gr_n_data) + return true; + /* + * A delivered engine fault is itself a loud failure edge (notifier + + * channel event); the release then wakes the client's tracker so it + * observes the error instead of hanging. + */ + if (svc_fwd_hold.fault_delivered) + return true; + if (gsp_cuda_bridge_enabled() && svc_fwd_req.gr_bridge_done) + return true; + return done && svc_fwd_req.hw_rc == 0; +} + +/* + * m156 fail-closed delivery. When the release is withheld because no + * executor was witnessed, write the channel's error notifier and post the + * channel event so the guest driver observes a launch failure instead of + * waiting forever on a word nothing will write (the ISSUE-0002 wedge). The + * record shape and the notifier_err latch mirror svc_fwd_fault_deliver's + * notifier tail; there is no decoded fault record here, so info32 carries + * the host-detected failure marker 0x1. + */ +static void svc_gr_failclosed_deliver(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u32 info32 = 0x1u; + u16 info16, status = SVC_NOTIFY_ERROR_STATUS; + int ret; + + if (!vgpu || !ch || !ch->notifier_pa || ch->notifier_err) + return; + info16 = (u16)ch->engine_type; + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + ret = svc_write_ap(vgpu, ch->notifier_pa, ch->notifier_as, notif, + sizeof(notif)); + if (ret) { + pr_err_ratelimited("vgsp-gr: FAIL-CLOSED notifier write failed ch=0x%x pa=0x%llx as=%u ret=%d\n", + ch->h_object, ch->notifier_pa, + ch->notifier_as, ret); + return; + } + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + svc.fwd_gr_failclosed++; + pr_info("vgsp-gr: FAIL-CLOSED: completion release withheld for data-bearing launch h=0x%x (no executor witnessed; guest release va=0x%llx pay=0x%x not authored); channel error delivered\n", + ch->h_object, svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_payload); + ret = gsp_rm_post_channel_event(vgpu, ch->h_object); + if (ret) + pr_info("vgsp-gr: FAIL-CLOSED channel event post failed ch=0x%x ret=%d\n", + ch->h_object, ret); +} + static bool svc_fwd_hold_expired(void) { - if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) + if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) { + unsigned long ms = SVC_FWD_GR_HOLD_TIMEOUT_MS; + + /* + * m156: a data-bearing launch under the fail-closed gate must + * not have its hold resolved while the backend is still + * running its bounded recovery chain (the outcome decides + * whether the release is authored or withheld). + */ + if (svc_fwd_req.gr_n_data > 0 && + gsp_phys_rmcall_gr_failclosed_value()) + ms = SVC_FWD_GR_HOLD_GATED_MS; return time_after(jiffies, svc_fwd_hold.since + - msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS)); + msecs_to_jiffies(ms)); + } return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } @@ -11646,7 +11751,15 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); - svc_fwd_hold_gr_release(hv, &shim); + /* + * m156: a data-bearing launch whose hold resolved with no + * executor witness keeps its completion release withheld and + * gets a channel error instead (fail closed). + */ + if (svc_gr_rel_witnessed(done)) + svc_fwd_hold_gr_release(hv, &shim); + else + svc_gr_failclosed_deliver(hv, ch); } if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_direct_note(); @@ -11751,7 +11864,17 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); - svc_fwd_hold_gr_release(hv, ch); + /* + * m156 fail-closed: a data-bearing forwarded launch with no + * executor witness must not have its completion release + * authored here (the r150 big-copy / r155 streams2 class: + * the client would read stale/zero output as a successful + * kernel). Withhold the release and deliver a channel error. + */ + if (svc_gr_rel_witnessed(done)) + svc_fwd_hold_gr_release(hv, ch); + else + svc_gr_failclosed_deliver(hv, ch); svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; if (!done) @@ -14316,10 +14439,26 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * request leaves the * authorship to the * hardware (or its hold - * fallback). + * fallback). m156: a + * queued data-bearing GR + * launch under the + * fail-closed gate needs + * the hold even when this + * pass recorded no release + * of its own - the hold's + * success edge is the only + * author of the deferred + * completion release and + * its failure edge withholds + * it. */ - if (svc_fwd_defer_armed && - svc_fwd_rel_n > 0) { + if ((fwd_guest_queued && + !ch->engine_type && + gsp_phys_rmcall_gr_failclosed_value() && + svc_fwd_req.gr && + svc_fwd_req.gr_n_data > 0) || + (svc_fwd_defer_armed && + svc_fwd_rel_n > 0)) { if (fwd_guest_queued && !svc_fwd_hold.active) { svc_fwd_hold_engage(vgpu, |