diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 261 |
1 files changed, 247 insertions, 14 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 741fdc2..0f71e8b 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1111,9 +1111,17 @@ struct sema_svc { u64 fwd_gr_dstpend; /* r541: held for an unexecuted * device write on a data page */ u32 fwd_gr_dstpend_log; + u64 fwd_gr_closed; /* r583: fail-closed at the defer + * cap (dstpend still live) */ + u32 fwd_gr_closed_log; u64 fwd_gr_argdefer; /* r543: held for a partial * argument set (unresolved arg) */ u32 fwd_gr_argdefer_log; + u64 fwd_gr_cewm; /* r590: held on the CE ring's + * live submission watermark */ + u64 fwd_gr_cewm_cap; /* r590: watermark still + * backlogged at the defer cap */ + u32 fwd_gr_cewm_log; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -5445,7 +5453,34 @@ static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 ele u32 aper; u32 prov = 0; /* r384: 1 pin, 2 ledger, 3 walk/scan */ - if (svc_vas_has(addr)) { + /* + * r584: a VID-aperture fill whose destination is a vidmem + * WINDOW VA (the guest's 0x2xxxxxxxx family) is offset- + * mapped by construction: VA - SVC_VIDMEM_VA_BASE is the + * FB offset into the emulated BAR1 shadow (the svc_lookup + * window convention; see also the 0x200080000 rationale at + * SVC_VIDMEM_VA_BASE). The generic chain below (pin -> + * ledger -> translate) answered SYS for the r583 boot's CE + * channel-setup fill to 0x204820000 (a stale landing), the + * r369 below-floor guard refused it, the fill never ran, + * and its DSTPEND note stalled GR launches to the fail- + * closed cap. Window VAs resolve structurally - no guest + * page table can express an "engineering window" offset. + */ + if (aperture == CE_APERTURE_VID && + va0 >= SVC_VIDMEM_VA_BASE && + va0 - SVC_VIDMEM_VA_BASE + total <= + svc_fb_shadow_size(vgpu)) { + addr = va0 - SVC_VIDMEM_VA_BASE; + /* aperture stays CE_APERTURE_VID: the whole-range + * contiguity block below writes it into the shadow */ + prov = 4; /* structural window convention */ + if (svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill-window va=0x%llx -> fb offset 0x%llx (vidmem window convention) elem=%u count=%u value=0x%llx\n", + va0, addr, elem, count, value); + } + } else if (svc_vas_has(addr)) { shadow = true; } else if (!svc_pin_lookup_any(addr, &pa, &aper)) { u64 lpa = 0; @@ -20665,10 +20700,29 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u32 dl; svc_gr_drop_log_va = svc_fwd_req.gr_push_va; - for (dl = 0; dl < 4 && dl < dropped_pages; dl++) + for (dl = 0; dl < 4 && dl < dropped_pages; dl++) { + /* + * r585: DEMAND the dropped pages through the + * pin channel. The host pin stream parses the + * "stalled VA #N: 0x..." lines (svc_log_stalled_va) + * and pushes each VA to the guest scanner within + * ~3 s, whose pin then lands inside the launch's + * bounded argument-defer window - the r584 matmul + * draw dropped 18 staging pages that were NEVER + * demanded (the GRFILL-DROP line format is + * invisible to the host's demand regex), the r543 + * defer window expired, and the r564 degenerate + * refusal SW-completed a launch that wrote + * nothing (MATMUL bad=65536 csum=0). One demand + * per page per boot (svc_log_stalled_va dedups); + * one episode per launch push VA (the marker + * above). + */ + svc_log_stalled_va(drop_va[dl]); pr_info("vgsp-gr: GRFILL-DROP va=0x%llx kind=%c idx=%u\n", drop_va[dl], drop_kind[dl], drop_idx[dl]); + } } } if (svc_gr_data_stalled) @@ -21050,6 +21104,62 @@ static bool svc_copy_pending(void) } /* + * r590: this pass's channel snapshot, so the CE watermark guard can read the + * pump's CURRENT consumption position (local.last_get) instead of the live + * table's previous-pass merge. The pump is single-threaded and walks CE + * channels before GR ones, so a GR decision always sees every CE channel's + * consumed position for this pass. + */ +static struct svc_channel *svc_pass_local; +static u32 svc_pass_local_n; + +/* + * r590: does any ring of this device still hold entries this pass's pump has + * not consumed? Reads the ring's live GP_PUT from its own USERD and compares + * it (modulo the ring size) with the snapshot's last_get. A mismatch means + * the client submitted work the pump has not decoded yet - exactly the + * stream-order backlog the r541/r568 guards cannot see. + * + * r590b (r591 draw): the driver submits its relocation/H2D copies on a + * COMPUTE channel (h=0x5c000059 registers eng=0), not a dedicated CE + * channel, so the engine_type filter skipped the very backlog this guard + * exists for. Every other channel of the device is now considered; the + * launching channel itself is excluded because its own ring legitimately + * runs ahead (its next entries are futures, not dependencies) and holding + * on it would deadlock the launch against its own channel. + * + * Channels whose USERD is unreadable are skipped (their work cannot be + * ordered by this mechanism anyway). + */ +static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) +{ + u32 k; + + if (!svc_pass_local || !svc_pass_local_n) + return false; + for (k = 0; k < svc_pass_local_n; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0; + + if (!c->valid || (vgpu && c->vgpu != vgpu)) + continue; + if (self && c->h_object == self->h_object) + continue; + if (!c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu ? vgpu : c->vgpu, + c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) + return true; + } + return false; +} + +/* * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget * per boot). Logged from svc_gr_request right after the fill, for launches * whose fill saw dropped pages / a partial argument set or whose decision @@ -21064,7 +21174,7 @@ static unsigned svc_gr_dec_log; static void svc_gr_dec_dump(const struct svc_channel *ch) { - bool dstpend, copypend; + bool dstpend, copypend, ceback; u32 i, n = 0; if (svc_gr_dec_log >= gsp_phys_gr_dec_log_max) @@ -21073,13 +21183,15 @@ static void svc_gr_dec_dump(const struct svc_channel *ch) dstpend = svc_gr_dstpend_overlap(); copypend = svc_copy_pending(); + ceback = svc_ce_stream_backlog(NULL, ch); - pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d defer_exp=%d\n", + pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n", ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data, svc_fwd_req.gr_dropped, svc_fwd_req.gr_arg_partial ? 1 : 0, svc_gr_data_stalled ? 1 : 0, dstpend ? 1 : 0, copypend ? 1 : 0, + ceback ? 1 : 0, svc_gr_defer_expired(svc_gr_cand.push_va) ? 1 : 0); /* @@ -21217,7 +21329,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (gsp_phys_gr_dec_log_max && (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial || svc_gr_data_stalled || svc_gr_dstpend_overlap() || - svc_copy_pending())) + svc_copy_pending() || svc_ce_stream_backlog(vgpu, ch))) svc_gr_dec_dump(ch); /* * r541: launch-ordering guard. The fill's data pages are what the @@ -21232,19 +21344,94 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * busy-defer + gr_hold machinery as every other busy refusal, so it * can never hang the launch. */ - if (svc_gr_dstpend_overlap() && - !svc_gr_defer_expired(svc_gr_cand.push_va)) { + if (svc_gr_dstpend_overlap()) { + if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (data page has an unexecuted device write; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + /* + * r583: the absolute defer cap expired while a data page still + * overlaps a pending device write (r581 saxpy live: GR-DEC + * dstpend=1 defer_exp=1 -> forwarded anyway -> bad=524121 + * with dstpend ages 9.5-85 s). Never forward a KNOWN-stale + * operand: fail-closed - author the candidate's QMD + * completion in software (forward-only, latched so the + * consume tail cannot author it twice) and refuse the + * engine launch. The client completes; the engine stays + * away from a page whose writer never ran. Witnessed. + */ + svc.fwd_gr_refused++; + svc.fwd_gr_closed++; + if (svc_gr_cand.rel_va) { + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc_gr_cand.rel_done = true; + } + svc_gr_cand.offered = true; + svc_gr_cand.valid = false; + if (svc.fwd_gr_closed_log < 32) { + svc.fwd_gr_closed_log++; + pr_warn("vgsp-work: GR forward REFUSED at defer cap h=0x%x push=0x%llx (data page still has an unexecuted device write; fail-closed, QMD release authored in software)\n", + ch->h_object, svc_gr_cand.push_va); + } clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); - svc_gr_busy_defer = true; - svc.fwd_gr_refused++; - svc.fwd_gr_dstpend++; - if (svc.fwd_gr_dstpend_log < 32) { - svc.fwd_gr_dstpend_log++; - pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (data page has an unexecuted device write; bounded retry)\n", + return false; /* no busy-defer: the pump consumes; the + * candidate is owned and sw-completed */ + } + /* + * r590: the CE submission watermark. A launch must not be forwarded + * while a CE ring still holds entries the pump has not consumed yet: + * those are the client's stream-ordered copies, and an entry the + * pump has not decoded cannot be seen by the r541 dstpend guard + * (decoded copies only) nor by the r568 opdef gate (decoded holds + * only). The r589 mail VADD drew exactly this: the h_b H2D entries + * sat behind a translation-held CE entry, the pump's last read of the + * ring's GP_PUT was 92 while the guest had already submitted 95, the + * GR channel was processed later in the same pass and the launch was + * forwarded with the b page still zero (c = a + 0). + * + * The watermark is read LIVE (USERD GP_PUT) against THIS pass's + * consumption position (the pump runs CE channels before GR ones, so + * local.last_get already includes everything this pass consumed). + * r588's proxy failed because it compared the live table's stale + * last_get against a put sampled in a different epoch; both sides of + * this comparison come from the same channel and the same pass. + * + * Bounded by the normal busy-defer window; at the absolute cap the + * historical forward stands (witnessed) - the guard delays a launch, + * it never refuses one, so it cannot recreate r588's mail refusal. + */ + if (svc_ce_stream_backlog(vgpu, ch)) { + if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (CE ring has unconsumed entries; stream-order watermark)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + svc.fwd_gr_cewm_cap++; + if (svc.fwd_gr_cewm_log < 32) { + svc.fwd_gr_cewm_log++; + pr_warn("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (CE ring still has unconsumed entries; historical forward)\n", ch->h_object, svc_gr_cand.push_va); } - return false; } /* * r568: the coarse companion gate - any unexecuted CE write at all @@ -21295,6 +21482,40 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * r586: the defer cap expired with argument pages still unresolved - + * fail-closed like the r583 dstpend cap. The historical path + * forwarded the launch with the partial set and the engine read + * stale/zero operands (r585 boot saxpy: n_data=969 dropped=7 + * partial=1 -> forwarded at the cap -> bad=1047527 csum=130523290; + * r584 matmul: n_data=2 dropped=18 -> r564 degenerate refusal -> + * no-write). Refuse the forward, author the completion in software, + * consume; the dropped pages were already demanded through the pin + * channel (r585) so the retry path had its chance first. + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc.fwd_gr_refused++; + svc.fwd_gr_closed++; + if (svc_gr_cand.rel_va) { + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc_gr_cand.rel_done = true; + } + svc_gr_cand.offered = true; + svc_gr_cand.valid = false; + if (svc.fwd_gr_closed_log < 32) { + svc.fwd_gr_closed_log++; + pr_warn("vgsp-work: GR forward REFUSED at defer cap h=0x%x push=0x%llx (argument pages still unresolved: %u; fail-closed, QMD release authored in software)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped); + } + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + return false; /* no busy-defer: the pump consumes; the + * candidate is owned and sw-completed */ + } + /* * m150: the fill found high argument pointers that no source can * translate (the second-client zeros class). Do NOT forward a launch * whose kernel would dereference unresolved pointers: leave the @@ -22047,6 +22268,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_drop_gone(vgpu, local, n); + /* + * r590: publish this pass's snapshot for the CE submission-watermark + * guard (svc_ce_stream_backlog). CE channels were copied first, so a + * GR launch's decision sees the pass-current consumed positions. + */ + svc_pass_local = local; + svc_pass_local_n = (u32)n; + for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; @@ -24631,6 +24860,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (stalled) svc_apply_sweep_slice(vgpu); + + /* r590: the pass snapshot is only valid while the loop runs. */ + svc_pass_local = NULL; + svc_pass_local_n = 0; } /* ------------------------------------------------------------------ */ |