diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 114 |
1 files changed, 111 insertions, 3 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index e86c463..a0cdb2b 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1117,6 +1117,11 @@ struct sema_svc { u64 fwd_gr_argdefer; /* r543: held for a partial * argument set (unresolved arg) */ u32 fwd_gr_argdefer_log; + u64 fwd_gr_cewm; /* r590: held on the CE ring's + * live submission watermark */ + u64 fwd_gr_cewm_cap; /* r590: watermark still + * backlogged at the defer cap */ + u32 fwd_gr_cewm_log; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -21099,6 +21104,51 @@ static bool svc_copy_pending(void) } /* + * r590: this pass's channel snapshot, so the CE watermark guard can read the + * pump's CURRENT consumption position (local.last_get) instead of the live + * table's previous-pass merge. The pump is single-threaded and walks CE + * channels before GR ones, so a GR decision always sees every CE channel's + * consumed position for this pass. + */ +static struct svc_channel *svc_pass_local; +static u32 svc_pass_local_n; + +/* + * r590: does any CE ring of this device still hold entries this pass's pump + * has not consumed? Reads the ring's live GP_PUT from its own USERD and + * compares it (modulo the ring size) with the snapshot's last_get. A + * mismatch means the client submitted copies/fills the pump has not decoded + * yet - exactly the stream-order backlog the r541/r568 guards cannot see. + * Channels whose USERD is unreadable are skipped (their work cannot be + * ordered by this mechanism anyway). + */ +static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) +{ + u32 k; + + if (!svc_pass_local || !svc_pass_local_n) + return false; + for (k = 0; k < svc_pass_local_n; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0; + + if (!c->valid || (vgpu && c->vgpu != vgpu) || + !c->engine_type) + continue; + if (!c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu ? vgpu : c->vgpu, + c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) + return true; + } + return false; +} + +/* * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget * per boot). Logged from svc_gr_request right after the fill, for launches * whose fill saw dropped pages / a partial argument set or whose decision @@ -21113,7 +21163,7 @@ static unsigned svc_gr_dec_log; static void svc_gr_dec_dump(const struct svc_channel *ch) { - bool dstpend, copypend; + bool dstpend, copypend, ceback; u32 i, n = 0; if (svc_gr_dec_log >= gsp_phys_gr_dec_log_max) @@ -21122,13 +21172,15 @@ static void svc_gr_dec_dump(const struct svc_channel *ch) dstpend = svc_gr_dstpend_overlap(); copypend = svc_copy_pending(); + ceback = svc_ce_stream_backlog(NULL); - pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d defer_exp=%d\n", + pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n", ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data, svc_fwd_req.gr_dropped, svc_fwd_req.gr_arg_partial ? 1 : 0, svc_gr_data_stalled ? 1 : 0, dstpend ? 1 : 0, copypend ? 1 : 0, + ceback ? 1 : 0, svc_gr_defer_expired(svc_gr_cand.push_va) ? 1 : 0); /* @@ -21266,7 +21318,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (gsp_phys_gr_dec_log_max && (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial || svc_gr_data_stalled || svc_gr_dstpend_overlap() || - svc_copy_pending())) + svc_copy_pending() || svc_ce_stream_backlog(vgpu))) svc_gr_dec_dump(ch); /* * r541: launch-ordering guard. The fill's data pages are what the @@ -21327,6 +21379,50 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * candidate is owned and sw-completed */ } /* + * r590: the CE submission watermark. A launch must not be forwarded + * while a CE ring still holds entries the pump has not consumed yet: + * those are the client's stream-ordered copies, and an entry the + * pump has not decoded cannot be seen by the r541 dstpend guard + * (decoded copies only) nor by the r568 opdef gate (decoded holds + * only). The r589 mail VADD drew exactly this: the h_b H2D entries + * sat behind a translation-held CE entry, the pump's last read of the + * ring's GP_PUT was 92 while the guest had already submitted 95, the + * GR channel was processed later in the same pass and the launch was + * forwarded with the b page still zero (c = a + 0). + * + * The watermark is read LIVE (USERD GP_PUT) against THIS pass's + * consumption position (the pump runs CE channels before GR ones, so + * local.last_get already includes everything this pass consumed). + * r588's proxy failed because it compared the live table's stale + * last_get against a put sampled in a different epoch; both sides of + * this comparison come from the same channel and the same pass. + * + * Bounded by the normal busy-defer window; at the absolute cap the + * historical forward stands (witnessed) - the guard delays a launch, + * it never refuses one, so it cannot recreate r588's mail refusal. + */ + if (svc_ce_stream_backlog(vgpu)) { + if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (CE ring has unconsumed entries; stream-order watermark)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + svc.fwd_gr_cewm_cap++; + if (svc.fwd_gr_cewm_log < 32) { + svc.fwd_gr_cewm_log++; + pr_warn("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (CE ring still has unconsumed entries; historical forward)\n", + ch->h_object, svc_gr_cand.push_va); + } + } + /* * r568: the coarse companion gate - any unexecuted CE write at all * (see svc_copy_pending). Covers the dropped-argument-base case the * page-level guard cannot see (the r571 mail b-copy). @@ -22161,6 +22257,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_drop_gone(vgpu, local, n); + /* + * r590: publish this pass's snapshot for the CE submission-watermark + * guard (svc_ce_stream_backlog). CE channels were copied first, so a + * GR launch's decision sees the pass-current consumed positions. + */ + svc_pass_local = local; + svc_pass_local_n = (u32)n; + for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; @@ -24745,6 +24849,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (stalled) svc_apply_sweep_slice(vgpu); + + /* r590: the pass snapshot is only valid while the loop runs. */ + svc_pass_local = NULL; + svc_pass_local_n = 0; } /* ------------------------------------------------------------------ */ |