diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 15 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 21 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 431 |
3 files changed, 386 insertions, 81 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 3fee456..75cba95 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17661,7 +17661,20 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, *n_tracked = 0; if (!req->n_data) return 0; - n_all = gsp_phys_gr_scan_pages(req, csum_in, n_tracked, NULL); + /* + * r343: scan_pages RETURNS the changed count and reports the tracked + * total through the out-param. The old call put the return (changed) + * into n_all while the tracker total went to *n_tracked, so the + * loop's `changed >= n_all` was `changed >= changed` - always true + * after the first scan (observed: "1/1 pages changed after 4 ms"), + * and the `!n_all` early-return also skipped the no-write settle + * when zero pages changed. Net effect: the copy-out shipped the + * bounce mid-ramp, so a cold engine (slowest store ramp) produced + * all-bad first runs (the matmul cold flake) and late stores could + * be dropped. Fix: tracked total into n_all; the content-stable + * rule below becomes the real settle bound. + */ + (void)gsp_phys_gr_scan_pages(req, csum_in, &n_all, NULL); /* * RETIRE-WIT (r227): the r227 capture shows n_tracked far below the * copy-in/copy-out rec counts for the same launch (4 tracked vs 1482 diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index e845493..e896664 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -638,13 +638,20 @@ int gsp_phys_rmcall_gr_qmdvirt_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value); /* - * Explicit-PCAS gate (default 1). The rebuilt launch carries the vendor's - * turingPushPcas(CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by - * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE). The stock guest push does - * not emit these (it relies on the IQ2M window's implied copy PCAS); setting - * this to 0 emits the pure vendor window-only form for the A/B test. - */ -static int gsp_phys_rmcall_gr_pcas_param = 1; + * Explicit-PCAS gate (default 0 since r343 - the double-execution fix). + * The rebuilt launch used to carry the vendor's turingPushPcas + * (CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by + * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE) ADDED to the inline-QMD run + * that the stock guest push already carries. The QMD's + * REQUIRE_SCHEDULING_PCAS bit is clear, so the inline run schedules by + * itself - the explicit pair was a SECOND scheduling edge and the same QMD + * was dispatched twice (launch_count: one atomicAdd +100 landed as +200; + * victory_reduce exactly 2x; scale/saxpy/memset2d doubled). The stock guest + * push has NO PCAS pair (captured push dumps); default 0 restores that + * single-edge form. 1 = re-add the pair (A/B debug only - with it on, the + * grid executes twice). + */ +static int gsp_phys_rmcall_gr_pcas_param; module_param_named(gsp_phys_rmcall_gr_pcas, gsp_phys_rmcall_gr_pcas_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas, diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index fbb278d..8623f07 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3550,6 +3550,10 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) } static void svc_pending_add(u64 va, u32 payload, u32 op); +/* r34x: defined with the pending ring / opdef slots (later in the file); + * the acquire gate drains/appraises withheld releases before it holds. */ +static void svc_apply_pending(struct nv_vgpu_device *vgpu); +static bool svc_wait_pend_release(u64 va); /* * F1: GR release-word authorship checks. Defined next to the svc_fwd_req @@ -3965,22 +3969,59 @@ static u64 svc_ledger_diverges; static u64 svc_ledger_pinwins; /* r336: pin-backed divergence wins */ static u32 svc_d2hmap_log; /* D2H-MAP per-page destination budget */ /* - * r336: host-semaphore acquire (C56F SEM_EXECUTE op 0/2/3/4/5) wait state. - * The decoder records the last acquire of the current push; svc_run_ops - * honors it before running any of the push's work, so a copy cannot - * execute ahead of the dependency its push waits on (the streams2 - * readback-before-scale class). The re-decode re-parses the push, so the - * state is refreshed on every pass. - */ -static u64 svc_wait_va; -static u32 svc_wait_payload; -static u8 svc_wait_mode; /* 0 = equal, 1 = geq, 2 = circ geq */ -static u8 svc_wait_pending; + * r336/r34x: host-semaphore acquire (C56F SEM_EXECUTE op 0/2/3/4/5) wait + * state. The decoder captures EVERY acquire of the ring entry currently + * being decoded (svc_waits[], reset per entry at the top of + * svc_decode_stream), and svc_run_ops honors them before running any of + * the entry's work, so a copy cannot execute ahead of the dependency its + * push waits on (the streams2 readback-before-scale class). Per-entry + * capture is essential: the wait state used to leak across entries (a + * later launch's self-release wait, e.g. need=85 while the held push + * releases 84, gated an earlier entry that carries no acquire at all - + * the r340 over-hold). The re-decode re-parses the entry, so the state + * is refreshed on every pass. + * + * Semantics per op (class contract, see svc_wait_satisfied): + * 0 ACQUIRE = (v == p); 2 ACQ_STRICT_GEQ = v >= p; 3 ACQ_CIRC_GEQ = + * circular (s32)(v - p) >= 0; 4 ACQ_AND = (v & p) != 0; + * 5 ACQ_NOR = ~(v | p) != 0. + */ +#define SVC_WAIT_MAX 8 +#define SVC_WAIT_HOLD_MS 100u /* bounded acquire wait (fail-open) */ +struct svc_wait_op { + u64 va; + u32 payload; + u8 op; /* C56F SEM_EXECUTE operation */ + u32 off; /* method offset in the entry (order) */ +}; +static struct svc_wait_op svc_waits[SVC_WAIT_MAX]; +static u32 svc_wait_n; /* acquires captured this entry */ +static u32 svc_wait_ovf; /* acquires dropped past the array */ static u64 svc_wait_held; static u64 svc_wait_pass; +static u64 svc_wait_giveup; static u32 svc_wait_log; /* per-session held-row print budget */ -static u64 svc_wait_rel_va; /* current push's own release (mirror) */ +static u32 svc_wait_pass_log; /* per-session resolved-row budget */ +static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ +static u32 svc_fill_noremap; /* r343 FILL: fail-closed holds */ +static u64 svc_wait_rel_va; /* this entry's own release (mirror) */ static u32 svc_wait_rel_payload; +static u32 svc_wait_rel_off; /* its method offset (order check) */ +static bool svc_wait_rel_have; +/* + * Per-channel first-hold clock for the bounded wait: a gate evaluation + * that cannot satisfy the acquire arms a 100 ms budget per hold episode; + * expiry fails open (ops run) with a loud 'give-up' row. The table is + * tiny (a handful of channels), service-thread only, cleared per pass + * resolution / session flush. + */ +struct svc_wait_hold { + bool active; + u32 h; + unsigned long since; +}; +#define SVC_WAIT_HOLD_MAX 16 +static struct svc_wait_hold svc_wait_holds[SVC_WAIT_HOLD_MAX]; static u64 svc_ledger_stores; static u64 svc_ledger_evictions; static u64 svc_ledger_evict_since_reset; @@ -4108,11 +4149,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gr_nocap_held, svc.gr_nocap_consumed, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, svc.pool_alt_applied, svc.pool_residue, - svc_wait_held, svc_wait_pass); + svc_wait_held, svc_wait_pass, svc_wait_giveup, + svc_wait_ovf); if (svc.resc_run || svc_rescue_scan_capped) pr_info("sema-svc: rescue_run=%llu scan_capped=%llu (multi-page content pins)\n", svc.resc_run, svc_rescue_scan_capped); @@ -6278,14 +6320,60 @@ static void svc_handle_map_add(void); /* Execute a decoded operation list (fills and copies). `ops_done` (may be * NULL) is the per-entry bitmap of ops that already ran for the entry's * identity; `ops_base` is the sequence number of ops[0] within the entry. */ -/* r336: evaluate a C56F SEM_EXECUTE acquire against a candidate value. */ -static bool svc_wait_satisfied(u32 cur, u8 mode, u32 need) + +/* + * r34x: evaluate a C56F SEM_EXECUTE acquire against the semaphore word. + * The class defines the opcodes (clc56f.h NVC56F_SEM_EXECUTE_OPERATION: + * 0 ACQUIRE, 1 RELEASE, 2 ACQ_STRICT_GEQ, 3 ACQ_CIRC_GEQ, 4 ACQ_AND, + * 5 ACQ_NOR, 6 REDUCTION) and the CUDA driver documents the wait + * conditions each op must implement (cuda_device.h CUIsemaphoreAquire- + * Flags_enum: GEQ default = circular >=, EQUAL = ==, AND = (v & p) != 0, + * NOR = ~(v | p) != 0, STRICT_GEQ = >= with no wrap; ampere_semaphore.c + * maps those flags onto the opcodes: EQUAL->0, STRICT_GEQ->2, GEQ->3, + * AND->4, NOR->5). Note op 0 is EQUALITY, not a >=: only op 2/3 are + * ordering compares. 32-bit payloads: every observed CUDA acquire is + * PAYLOAD_SIZE 32BIT (SEM_PAYLOAD_HI == 0); a 64-bit acquire is captured + * and judged on the low word only (its high word is not modelled). + */ +static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) { - if (mode == 0) + switch (op) { + case 0: /* ACQUIRE: wait for v == p */ return cur == need; - if (mode == 2) + case 2: /* ACQ_STRICT_GEQ: v >= p, no wrap */ + return cur >= need; + case 3: /* ACQ_CIRC_GEQ: v in [p, p + 2^31 - 1] (wrap allowed) */ return (s32)(cur - need) >= 0; - return cur >= need; + case 4: /* ACQ_AND: (v & p) != 0 */ + return (cur & need) != 0; + case 5: /* ACQ_NOR: ~(v | p) != 0 */ + return ~(cur | need) != 0; + default: + return true; + } +} + +/* r34x: the per-channel first-hold clock for the bounded wait. */ +static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) +{ + struct svc_wait_hold *free_slot = NULL; + int i; + + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h) + return e; + if (!e->active && !free_slot) + free_slot = e; + } + if (!create || !free_slot) + return NULL; + memset(free_slot, 0, sizeof(*free_slot)); + free_slot->active = true; + free_slot->h = h; + free_slot->since = jiffies; + return free_slot; } static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, @@ -6307,16 +6395,34 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, */ svc_handle_map_add(); /* - * r336: honor the push's host-semaphore acquire (C56F SEM_EXECUTE - * op 0/2/3/4/5) before any of its work runs. The push's release is - * withheld through the op-pending machinery (bounded, loud), so the - * guest cannot observe completion of work that ran ahead of the - * dependency it waits on. Unreadable semaphore: fail-open (the - * pre-r336 behaviour). + * r336/r34x: honor every host-semaphore acquire (C56F SEM_EXECUTE + * op 0/2/3/4/5) captured for THIS ring entry before any of its work + * runs, so a copy cannot execute ahead of the dependency its push + * waits on (the streams2 readback-before-scale class). Decision + * ladder per wait: + * 1. saved value satisfies the op's condition -> pass; + * 2. the entry's own release, when its method offset precedes the + * acquire's (release-then-acquire, channel.c crossEngineSync), + * counts as the candidate value - the release method has + * already executed by the time the acquire is evaluated; + * an acquire that precedes its own trailing tracking release + * does NOT count it (nothing has installed that value yet); + * 3. drain the pending-translation release queue (releases OUR + * machinery is merely late about) and re-read once; + * 4. still unsatisfied -> hold the entry (release withheld through + * the op-pending machinery) for at most SVC_WAIT_HOLD_MS per + * episode, then fail open loudly ('give-up': the value cannot + * be installed by firmware alone - it sits behind an upstream + * opdef/operand hold - so the old pass-through behaviour is + * safer than parking the channel forever). Unreadable + * semaphore: fail-open (the pre-r336 behaviour). + * The pass/hold/give-up decisions are logged as 'sema-wait' rows + * (op, payload, observed value, decision). */ - if (svc_wait_pending && gsp_sema_wait_gate) { - u32 cur = 0; - bool ok = true; + if (svc_wait_n && gsp_sema_wait_gate) { + struct svc_wait_hold *gh; + bool hold = false; + u32 wi; /* * Kill switch: a wait that never satisfies (a counter our @@ -6334,33 +6440,80 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, svc_wait_pass++; goto svc_wait_done; } - if (!svc_sem_read(vgpu, svc_wait_va, &cur)) - ok = svc_wait_satisfied(cur, svc_wait_mode, - svc_wait_payload); - /* - * The driver's release(N)+acquire(N) pattern (channel.c - * crossEngineSync): the push's own release lands only after - * its ops run, so it cannot satisfy the wait through memory - * yet - treat its payload as applied. - */ - if (!ok && svc_wait_rel_va == svc_wait_va && - svc_wait_rel_payload) - ok = svc_wait_satisfied(svc_wait_rel_payload, - svc_wait_mode, - svc_wait_payload); - if (!ok) { + gh = svc_wait_hold_get(ch_h, true); + for (wi = 0; wi < svc_wait_n; wi++) { + const struct svc_wait_op *w = &svc_waits[wi]; + u32 cur = 0, waited_ms = 0; + bool ok = true, same_rel = false, pend = false; + + if (gh) + waited_ms = jiffies_to_msecs(jiffies - gh->since); + if (svc_sem_read(vgpu, w->va, &cur)) + continue; /* unreadable: fail-open */ + ok = svc_wait_satisfied(cur, w->op, w->payload); + if (!ok && svc_wait_rel_have && + svc_wait_rel_va == w->va && + svc_wait_rel_off < w->off) { + same_rel = true; + ok = svc_wait_satisfied(svc_wait_rel_payload, + w->op, w->payload); + } + if (!ok) { + svc_apply_pending(vgpu); + if (!svc_sem_read(vgpu, w->va, &cur)) + ok = svc_wait_satisfied(cur, w->op, + w->payload); + } + if (ok) { + if (svc_wait_pass_log < 256) { + svc_wait_pass_log++; + pr_info("sema-wait: pass ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u waited_ms=%u -> ops run this pass\n", + ch_h, w->va, w->op, w->payload, + cur, same_rel, waited_ms); + } + continue; + } + pend = svc_wait_pend_release(w->va); + if (waited_ms >= SVC_WAIT_HOLD_MS) { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; + } + hold = true; + if (svc_wait_log < 512) { + svc_wait_log++; + pr_info("sema-wait: hold ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u pend=%u waited_ms=%u -> entry held (acquire not satisfied)\n", + ch_h, w->va, w->op, w->payload, cur, + same_rel, pend, waited_ms); + } + break; /* one unsatisfied wait holds the entry */ + } + if (hold) { svc_wait_held++; - if (svc_wait_log < 512) - pr_info("sema-wait: held ch=0x%x r=%u va=0x%llx need=%u cur=%u mode=%u rel=0x%llx rpay=%u (acquire not satisfied)\n", - ch_h, svc_wait_log++, - svc_wait_va, svc_wait_payload, - cur, svc_wait_mode, - svc_wait_rel_va, - svc_wait_rel_payload); svc.op_tr_pending = 1; return; } svc_wait_pass++; + if (gh) + gh->active = false; } svc_wait_done: for (i = 0; i < n_ops; i++) { @@ -7687,6 +7840,25 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ svc.cur_ch = ch->h_object; /* + * r34x: the acquire-wait capture is PER RING ENTRY. It used to be + * reset only at SET_OBJECT, so a decode pass over a later entry (or + * a set of entries, since a held entry is re-decoded after the + * sweep walks ahead) leaked its acquire into an earlier entry's + * release-site gate: r340 held a D2H copy entry that carries no + * acquire at all on a later launch's 'wait for the next tracking + * value' (need=85 while the held entry releases 84) - a circular + * wait only our own withheld release could ever satisfy, so every + * hold expired via the 10 s fallback (wait=4542 etc.). Reset here, + * per entry; SET_OBJECT still resets at the push boundary, and the + * re-decode re-captures the same waits, so held entries re-evaluate + * identically. + */ + svc_wait_n = 0; + svc_wait_rel_va = 0; + svc_wait_rel_payload = 0; + svc_wait_rel_off = 0; + svc_wait_rel_have = false; + /* * Static for the same reason as svc_handle_chan_dump: a 32-entry ops * array is a ~2 KB frame, and this is called from the single service * thread only. @@ -8118,9 +8290,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); - svc_wait_pending = 0; /* r336: new push, new wait */ + /* r336/r34x: new push, new acquire set + release mirror */ + svc_wait_n = 0; svc_wait_rel_va = 0; svc_wait_rel_payload = 0; + svc_wait_rel_off = 0; + svc_wait_rel_have = false; off += 4 + count * 4; continue; } @@ -8141,6 +8316,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.rel_payload = payload; svc_wait_rel_va = va; svc_wait_rel_payload = payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -8242,24 +8419,39 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } else if (op != HOST_SEM_OP_RELEASE && va >= SEMA_VA_LO && va < SEMA_VA_HI) { /* - * r336: C56F SEM_EXECUTE acquire ops (0 = - * acquire/equal, 2 = strict GEQ, 3 = circular - * GEQ, 4/5 = AND/NOR). Core CUDA starts its - * compute-channel pushes with one of these to - * wait for a progress counter (observed: + * r336/r34x: C56F SEM_EXECUTE acquire ops + * (0 = ACQUIRE/equal, 2 = strict GEQ, + * 3 = circular GEQ, 4 = AND, 5 = NOR). + * Core CUDA starts its compute-channel + * pushes with one of these to wait for a + * progress counter (observed: * [LO=0x0400ffd0][HI=2][pay=4][0][0x1003] on - * the D2H readback push - ACQ_CIRC_GEQ on the - * channel counter 0x20400ffd0). svc_run_ops - * honors the wait before running the push's - * work, so a copy cannot execute ahead of the + * the D2H readback push - ACQ_CIRC_GEQ on + * the channel counter 0x20400ffd0). + * svc_run_ops honors every captured acquire + * of this entry before running its work, so + * a copy cannot execute ahead of the * dependency it waits on (the streams2 - * readback-before-scale class). + * readback-before-scale class). Capturing + * into the per-entry list (instead of a + * single last-wait global) is what stops a + * later entry's acquire from gating an + * earlier entry (the r340 over-hold). + * PAYLOAD_SIZE 64BIT (execute bit 24) is + * captured on the low word only; every + * observed CUDA acquire is 32-bit. */ - svc_wait_va = va; - svc_wait_payload = payload; - svc_wait_mode = (op == 3) ? 2 : - (op == 0) ? 0 : 1; - svc_wait_pending = 1; + if (svc_wait_n < SVC_WAIT_MAX) { + struct svc_wait_op *w = + &svc_waits[svc_wait_n++]; + + w->va = va; + w->payload = payload; + w->op = (u8)op; + w->off = off; + } else { + svc_wait_ovf++; + } } off += 4 + count * 4; continue; @@ -8311,6 +8503,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.rel_payload = st.rs_payload; svc_wait_rel_va = va; svc_wait_rel_payload = st.rs_payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -8488,6 +8682,24 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: + /* + * r343 FILL instrument: a fill-shaped LAUNCH_DMA + * (REMAP_ENABLE + destination PITCH, OFFSET_OUT and + * no OFFSET_IN) needs this constant to build its op. + * The 8MB-memset class (memcpy_roundtrip all-bad, + * victory_memset2d) shows fill-shaped launches with + * have_remap=0 that decode nothing and are consumed + * silently. Log every decode (bounded) so one run + * captures the real run shape even when the count + * guard below cannot use it. + */ + if (svc_remapconst_log < 32) { + svc_remapconst_log++; + pr_info("sema-remapconst: h=0x%x push=0x%llx off=0x%x count=%u v0=0x%x v1=0x%x v2=0x%x trusted=%d allow=%u\n", + ch->h_object, push_va, off, count, + v0, count >= 2 ? v1 : 0u, + count >= 3 ? v2 : 0u, trusted, allow_ops); + } if (count >= 3) { st.remap_a = v0; st.remap_b = v1; @@ -8630,6 +8842,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* + * r343 FILL diagnostic (LOG-ONLY, no hold): a + * fill-shaped transfer (REMAP_ENABLE + destination + * PITCH, OFFSET_OUT and no OFFSET_IN) whose value + * constant did not decode currently decodes no op + * and is consumed silently - the guest's cudaMemset + * then reports success with zero bytes written + * (r341 memcpy_roundtrip: 8MB memset readback + * all-bad; victory_memset2d). An op_tr_pending hold + * here WEDGED client init (r343: first cudaMalloc + * retried forever - driver init issues fill-shaped + * CE pushes, the hold re-armed every decode pass + * and never expired). So: log loudly only; the + * sema-remapconst rows above capture the real + * decode shape to build the proper fill decoder. + */ + if (allow_ops && transfer != 0 && + st.line_length >= 1024 && st.have_out && + !st.have_in && DMA_REMAP_ENABLE(dma) && + DMA_DST_MEMORY_LAYOUT_PITCH(dma) && + !st.have_remap && n_ops == n_ops_before && + svc_fill_noremap < 8) { + svc_fill_noremap++; + pr_err("vgsp: FILL-NOREMAP h=0x%x push=0x%llx dma=0x%08x line=%u out=0x%llx: fill-shaped launch, value source not decoded (log-only diagnostic)\n", + ch->h_object, push_va, dma, + st.line_length, st.offset_out); + } + + /* * fail4: name a transfer-shaped launch that decoded NO * op because the pass was denied (allow_ops=0) - the * op-chain above (and its dma-skip dump) is skipped @@ -8866,6 +9106,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.rel_payload = st.sem_payload; svc_wait_rel_va = sem_va; svc_wait_rel_payload = st.sem_payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; if (st.have_sem && st.sem_payload && @@ -9607,6 +9849,12 @@ static void svc_session_flush(void) svc.dmatrace_log = 0; svc.copyent_log = 0; svc_wait_log = 0; + svc_wait_pass_log = 0; + svc_remapconst_log = 0; + svc_fill_noremap = 0; + svc_wait_n = 0; + svc_wait_rel_have = false; + memset(svc_wait_holds, 0, sizeof(svc_wait_holds)); svc.bigrun_log = 0; svc.lid_hdr_log = 0; svc.skiprun_log = 0; @@ -11823,13 +12071,19 @@ bool gsp_sema_pool_hold_fresh = 1; */ bool gsp_sema_copy_dst_pin_guard = 1; /* - * r336 wait gate: honor the C56F host-semaphore acquire before running a - * push's work. DEFAULT OFF: the acquire's tracking-semaphore values are - * not yet fully modelled (channel-first pushes over-hold: the wait needs a - * counter value only the withheld release itself would install), so the - * gate stays an experiment until the counter semantics are finished. The - * streams2 readback-before-scale class is root-caused in the handoff; - * with the gate off the behaviour is the pre-r336 one. + * r336/r34x wait gate: honor the C56F host-semaphore acquire ops captured + * for a ring entry before running its work. Still DEFAULT OFF (opt-in + * experiment; the module param sema_wait_gate=1 enables it), but the + * semantics are now the class contract's: op 0 = equal, 2 = strict GEQ, + * 3 = circular GEQ, 4 = AND, 5 = NOR; the wait set is captured PER ENTRY + * (the old single-last-wait global leaked a later entry's acquire into an + * earlier entry - the r340 over-hold); the entry's own release counts only + * when its method order precedes the acquire; pending releases our + * machinery is merely late about are drained first; and every hold is + * bounded at SVC_WAIT_HOLD_MS (100 ms) then fails open loudly ('give-up') + * so a value that can only be installed behind an upstream opdef hold + * cannot park a channel. Diagnostics: 'sema-wait: pass/hold/give-up' rows + * carry op, payload, observed value and decision. */ bool gsp_sema_wait_gate; @@ -14130,6 +14384,37 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu, } /* + * r34x: is a release to this VA currently withheld by our own machinery? + * True when the pending-translation ring holds one (translation late) or + * an opdef slot has one recorded (operand hold). The acquire gate calls + * this to classify an unsatisfied wait in its diagnostics: pend=1 names + * the 'value sits behind our own withhold' case - the bounded 100 ms + * give-up is exactly for it (waiting longer cannot install the value + * until the upstream entry's ops complete). Never forces an opdef release + * here: those releases are withheld because their ops have NOT completed, + * and delivering early re-opens the work-before-fence hole they close. + */ +static bool svc_wait_pend_release(u64 va) +{ + u32 i; + + for (i = 0; i < svc_pending_n; i++) + if (svc_pending[i].va == va) + return true; + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + u8 k; + + if (!e->active) + continue; + for (k = 0; k < e->n; k++) + if (e->rel[k].va == va) + return true; + } + return false; +} + +/* * STALL FIX (r186 entry=82/pay=84): called from svc_replay_note() on every * entry-consume path. If a release set was withheld for this entry while an * operand was pending and the entry is now leaving the decode path (so the |