diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 91 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 9 | ||||
| -rw-r--r-- | kernel/main.c | 9 |
3 files changed, 81 insertions, 28 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 3a955df..a045f07 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -10443,6 +10443,19 @@ unsigned gsp_sema_gr_grace_ms = 8000; unsigned gsp_phys_gr_hold_max_ms = 3000; /* + * GRFIX FIX1e (r215): master gate for the GR fill hold on dropped operand + * pages. Default 0 = LOG-ONLY: svc_gr_data_fill never holds on dropped + * pages - the drops are logged (GRFILL-DROP) and the launch takes the + * historical forward path immediately (exactly the pre-8b058884 behaviour); + * the GRFILL-HOLD/GRFILL-EXPIRED machinery and the per-launch absolute + * bound are skipped entirely. When set to 1 the page-count hold + absolute + * bound apply (r213-r215 showed the bound never fired: the per-channel + * record fields are not in the svc_pump_channels snapshot write-back list, + * so they were zeroed every pass - see the GRFIX NOTE at that list). + */ +unsigned gsp_phys_gr_fill_hold; /* default 0: log-only, never hold */ + +/* * F1-b: GR release-word authorship gate (default 1). While a forwarded GR * launch is in flight, the RAM-content sweep must not author the very QMD * completion release the backend poll is waiting for: a sweep write could @@ -14834,31 +14847,32 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, (n_arg_hi > 0 && n_arg_hi_res == 0); svc_fwd_req.gr_dropped = dropped_pages; /* - * GRFIX FIX1c (r214): the DECISION POINT IS SELF-ENFORCING. The - * absolute per-launch bound is consulted here - at the only place - * the stalled verdict is produced, before the GRFILL-HOLD line can - * be emitted and before any caller can act on the verdict - so no - * re-drive pass can bypass it. r214: the same launch re-filled - * ~2/s for 220 s with GRFILL-EXPIRED=0 because the a5715df1 - * consume-tail resets wiped the per-launch record between passes - * while the entry index moved (107 -> 99) under pin-batch rewinds, - * so the bound svc_gr_request consulted was re-armed before it - * could pass. When the bound passes the launch PROCEEDS: the - * historical (pre-8b058884) forward path runs, exactly as if the - * fill had not stalled. + * GRFIX FIX1e (r215): gsp_phys_gr_fill_hold is the master gate. + * Default 0 = LOG-ONLY: the fill NEVER holds on dropped pages - the + * stalled verdict keeps only the historical pre-8b058884 aggregate + * rule (all high argument pointers unresolved) and the HOLD/EXPIRED + * machinery below is skipped entirely; the launch takes the forward + * path immediately. When 1 the page-count hold applies, bounded by + * the per-launch absolute deadline below. * - * GRFIX FIX1d (benign class): the observed drops (probe=none / - * walk rc=-2) are indistinguishable at fill time from a page - * whose pin simply has not landed yet - both have no source at - * this instant and the only discriminator is whether the pin - * arrives inside the window. A blanket "probe=none => never hold" - * would also disable the pin-late retry the hold exists for, so - * the absolute bound stays the only guard: benign drops cost at - * most gsp_phys_gr_hold_max_ms once per launch VA, then the - * launch proceeds. + * GRFIX FIX1c/FIX1d (live only with fill_hold=1): the decision point + * is self-enforcing - the absolute per-launch bound is consulted at + * the only place the stalled verdict is produced, before the + * GRFILL-HOLD line can be emitted. When the bound passes the launch + * PROCEEDS via the historical forward path. The benign class + * (probe=none / walk rc=-2) is indistinguishable at fill time from a + * page whose pin has not landed yet, so a "never hold" predicate + * would also disable the pin-late retry; the bound is the only + * guard. NOTE (r213-r215): the bound never fired in the live cycles + * because these per-channel record fields are not in the + * svc_pump_channels snapshot write-back list (see the GRFIX NOTE + * there) and were zeroed on every pass - which is why the gate now + * defaults to 0. */ - if (svc_gr_data_stalled && ch && - svc_gr_hold_bound_expired(ch, svc_fwd_req.gr_push_va)) { + if (!gsp_phys_gr_fill_hold) { + svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0); + } else if (svc_gr_data_stalled && ch && + svc_gr_hold_bound_expired(ch, svc_fwd_req.gr_push_va)) { svc.fwd_gr_hold_exp++; bound_hit = true; if (!ch->gr_hold_exp_logged) { @@ -14870,10 +14884,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, } svc_gr_data_stalled = false; /* proceed: forward */ } - if (dropped_pages > 0 && !bound_hit) { - pr_info_ratelimited("vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n", - svc_fwd_req.gr_push_va, dropped_pages, - svc_fwd_req.gr_n_data); + if (dropped_pages > 0) { + if (gsp_phys_gr_fill_hold && !bound_hit) + pr_info_ratelimited("vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n", + svc_fwd_req.gr_push_va, dropped_pages, + svc_fwd_req.gr_n_data); /* * GRFIX FIX1c: name the dropped pages once per launch * episode (the fill re-runs every pass for a held launch; @@ -14898,6 +14913,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, else if (bound_hit) pr_info_ratelimited("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u dropped page(s); hold bound expired -> launch proceeds\n", n_cand, svc_fwd_req.gr_n_data, dropped_pages); + else if (dropped_pages > 0 && !gsp_phys_gr_fill_hold) + pr_info_ratelimited("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u dropped page(s); fill hold disabled (log-only) -> launch proceeds\n", + n_cand, svc_fwd_req.gr_n_data, dropped_pages); else pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", n_cand, svc_fwd_req.gr_n_data, reloc_base); @@ -15153,7 +15171,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * the channel's gr_hold window at the call site. Once the window * expires the entry is consumed exactly as before. */ - if (svc_gr_data_stalled) { + if (svc_gr_data_stalled && gsp_phys_gr_fill_hold) { u64 lva = svc_fwd_req.gr_push_va; /* * The channel is the pump's mutable live bookkeeping object @@ -15174,6 +15192,9 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * on every re-decode (the same launch is replayed at * different GPFIFO slots after rewinds) and held one launch * for 216+ s until the client's 240 s timeout killed it. + * GRFIX FIX1e: gated by gsp_phys_gr_fill_hold (default 0) - + * with the gate off the fill never stalls on drops, so this + * block only ever sees the historical aggregate stall. */ if (svc_gr_hold_bound_expired(mch, lva)) { svc.fwd_gr_hold_exp++; @@ -17661,6 +17682,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc.chan[k].gr_hold_until = ch->gr_hold_until; svc.chan[k].gr_hold_idx = ch->gr_hold_idx; + /* + * GRFIX NOTE (r213-r215): ANY new per-channel + * record field MUST be added to this snapshot + * write-back list or it is zeroed on every pass + * (the pump works on the local[] snapshot copied + * at the top of svc_pump_channels). The GRFIX + * absolute-bound fields (gr_hold_dead_va / + * gr_hold_dead_since / gr_hold_deadline / + * gr_hold_exp_logged) are deliberately NOT + * written back here - the proven record + * destroyer - which is why + * gsp_phys_gr_fill_hold defaults to 0. If the + * hold is ever re-enabled, add those four fields + * here first. */ /* m164: the CLASS-2 no-capture window is snapshot * state too - without the write-back every pass * would re-arm it and rehold the entry forever diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 87b281b..26249fd 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -277,6 +277,15 @@ extern unsigned gsp_sema_gr_grace_ms; extern unsigned gsp_phys_gr_hold_max_ms; /* + * GRFIX FIX1e (r215): master gate for the GR fill hold (default 0 = + * log-only, never hold). At 0 the fill logs dropped operand pages + * (GRFILL-DROP) and takes the historical forward path immediately; the + * GRFILL-HOLD/GRFILL-EXPIRED machinery is skipped. At 1 the page-count + * hold and its absolute bound apply. + */ +extern unsigned gsp_phys_gr_fill_hold; + +/* * m164 CLASS-2: hold window (ms) for a GR launch push whose decode saw QMD * words but built no launch candidate; 0 disables the hold. */ diff --git a/kernel/main.c b/kernel/main.c index ae9f262..bda4f97 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2781,6 +2781,15 @@ module_param_named(gsp_phys_gr_hold_max_ms, gsp_phys_gr_hold_max_ms, uint, 0644) MODULE_PARM_DESC(gsp_phys_gr_hold_max_ms, "Absolute GR fill hold bound in ms, keyed by launch push VA (default 3000; launch proceeds when it passes)"); /* + * GRFIX FIX1e (r215): master gate for the GR fill hold. At 0 (default) the + * fill logs dropped operand pages and takes the historical forward path + * immediately - it can never hold or delay a launch. At 1 the page-count + * hold and the absolute bound apply. + */ +module_param_named(gsp_phys_gr_fill_hold, gsp_phys_gr_fill_hold, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_fill_hold, "GR fill hold on unresolved operand pages (default 0: log-only, never hold)"); + +/* * F1-b: while a forwarded GR launch is in flight, defer (do not apply) a * RAM-sweep release whose VA is that launch's QMD completion word, so the * backend poll can only break on an engine-authored ack. Deferred releases |