diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 445 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 21 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 31 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 10 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 265 | ||||
| -rw-r--r-- | kernel/main.c | 1 |
6 files changed, 719 insertions, 54 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index b5898b6..bbb6d52 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -9631,9 +9631,38 @@ static u64 gsp_phys_rmcall_pde(u64 pa) return 0xeull | ((pa >> 12) << 8); } +/* + * r431 COHERENCY ROOT-CAUSE CANDIDATE: the leaf-PTE aperture for every host + * page this backend maps (the ladder windows, the win0 shadow pool, the + * bounce/data regions, the release/scratch shadows) was hardcoded to + * aperture 3 = SYSTEM_NON_COHERENT_MEMORY. The reference RM picks the + * aperture from the allocation's CPU cache attribute + * (fabric_vaspace.c:1123-1134): NV_MEMORY_CACHED -> SYSTEM_COHERENT (2), + * otherwise NON_COHERENT (3). Every page here is dma_alloc_coherent (CPU + * cached/coherent by construction), so 3 is wrong: the GPU's L2 does not + * snoop the CPU, which makes the engine's stores linger in L2 (the host's + * retire scan reads stale memory = the "no-execute wedge") and the host's + * re-staged copy-in content invisible to the engine (the retries re-read + * stale lines). gsp_phys_rmcall_pte_coh selects aperture 2 (default 1) for + * a one-flag A/B against the historical 3 (0); the PTEs are planted before + * the engine's first fetch, so the flag is boot-time only. + */ +static bool gsp_phys_rmcall_pte_coh_param; +module_param_named(gsp_phys_rmcall_pte_coh, gsp_phys_rmcall_pte_coh_param, + bool, 0444); +MODULE_PARM_DESC(gsp_phys_rmcall_pte_coh, + "r431: sysmem leaf-PTE aperture: 1 = SYSTEM_COHERENT (correct for dma_alloc_coherent host pages), 0 = SYSTEM_NON_COHERENT (historical)"); + static u64 gsp_phys_rmcall_pte(u64 pa) { - return 0xfull | ((pa >> 12) << 8); + /* + * Low nibble: VALID (1) | APERTURE<<1 | VOL (8). Aperture 2 = + * SYSTEM_COHERENT_MEMORY -> 1|4|8 = 0xd; aperture 3 = NON_COHERENT + * -> 1|6|8 = 0xf. + */ + u64 base = gsp_phys_rmcall_pte_coh_param ? 0xdull : 0xfull; + + return base | ((pa >> 12) << 8); } /* @@ -16303,6 +16332,94 @@ static u64 gsp_phys_gr_sem_wedged; */ static u64 gsp_phys_gr_stall_wait_max; static u64 gsp_phys_gr_stall_retry; +/* + * r410 wedge quiet window state. nw_streak counts consecutive accepted + * zero-write launches with tracked data (reset by any launch that wrote); + * the sticky wedge is the state where this grows without bound while the + * lane is kept busy. quiet counts the quiet holds issued; heals counts the + * quiet retries after which the engine wrote. Single-inflight submit path, + * so plain statics. + */ +static u32 gsp_phys_gr_nw_streak; +static u64 gsp_phys_gr_wedge_quiet; +static u64 gsp_phys_gr_wedge_heals; +/* + * r411: once a quiet heal has succeeded on this boot, the lane is in a + * wedge-prone episode. r418 showed the wedge can alternate one-no-write / + * one-heal, and the healed launch resets nw_streak, so a streak-only trigger + * quieted only every second wedged launch and the other was accepted. The + * heal count makes the episode sticky for the boot: any later zero-write + * launch gets the quiet path on its first retry, so every wedged launch + * either heals (engine writes, client data correct) or is loudly classified. + */ +static bool gsp_phys_gr_wedge_episode; +/* + * r420: true exactly while a WEDGE-QUIET hold sleeps with NO submission + * outstanding. The hold is a deliberate, bounded wait for the sticky wedge to + * clear (r410/r411); during it the in-flight forwarded request is NOT done. + * The sema service's completion deadlines - the 12 s GR release hold + * (SVC_FWD_GR_HOLD_TIMEOUT_MS) and the 3 s per-launch consumption bound + * (gsp_phys_gr_hold_max_ms) - are shorter than this hold, so without the + * publish they fire UNDER it and author the launch's release word before the + * post-hold copy-out: the client then verifies stale data (r419 soak: + * `victory_matmul bad=65536` while its launch was held; the authoring paths + * are GR-STALL-fwd-hold / the busy-defer inline software release). The + * accessor lets both deadlines pause instead; the flag is bounded by the + * msleep itself and by the pause caps on the sema side. Single-inflight + * submit path, so a plain static. + */ +static bool gsp_phys_gr_quiet_hold_on; + +bool gsp_phys_gr_quiet_hold_active(void) +{ + return READ_ONCE(gsp_phys_gr_quiet_hold_on); +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_quiet_hold_active); + +/* + * r425 experiment (the second-launch wedge): the stock guest driver emits the + * per-channel compute init block ONCE at channel creation (turing_init.c), + * long before any launch, while this backend re-emits it as the preamble of + * every forwarded launch. The wedge strikes the SECOND launch of a channel, so + * test whether the re-emission is the trigger: with gsp_phys_rmcall_gr_init_once + * set, the init preamble is emitted only on the first forwarded launch after + * the module load (the engine's compute-class/CWD state persists in the + * context; every launch still carries its QMD + report semaphore). 0 keeps the + * historical per-launch emission (default; runtime-writable A/B). + */ +static bool gsp_phys_rmcall_gr_init_once_param; +module_param_named(gsp_phys_rmcall_gr_init_once, gsp_phys_rmcall_gr_init_once_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_init_once, + "r425: emit the GR compute init preamble only on the first forwarded launch (0 = every launch, default)"); +static bool gsp_phys_rmcall_gr_init_emitted; + +/* + * r425 stale-QMD witness: the first engine-changed page of the LAST completed + * launch (bounce index + post-copy-out checksum). At a zero-write accept the + * page is re-scanned; a change proves a late store (a stuck/queued grid or a + * stale-QMD execution) landed after its launch was shipped. + */ +static u32 gsp_phys_gr_prev_out_bi; +static u32 gsp_phys_gr_prev_out_csum; +static bool gsp_phys_gr_prev_out_valid; + +/* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged + * comparison baseline lives in the same log). */ +static void gsp_phys_gr_sked_wit(void __iomem *bar0, const char *tag) +{ + if (!bar0) + return; + pr_info("vgsp-gr: SKED-WIT %s cwd0=%08x cwd1=%08x cwd2=%08x cwd3=%08x info1=%08x act=%08x grfifo=%08x fecs=%08x new=%08x ctxsw1=%08x qmdvirt0=%08x tpcm0=%08x\n", + tag, + readl(bar0 + 0x00407060u), readl(bar0 + 0x00407064u), + readl(bar0 + 0x00407068u), readl(bar0 + 0x0040706cu), + readl(bar0 + 0x00407028u), readl(bar0 + 0x00407054u), + readl(bar0 + 0x00400504u), readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u), readl(bar0 + 0x00409400u), + readl(bar0 + 0x0040a500u), readl(bar0 + 0x0040a680u)); +} + /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; static u32 gsp_phys_gr_override_len; @@ -16432,7 +16549,17 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value(); int two_step = gsp_phys_rmcall_gr_pcas_two_step_value(); - p = gsp_phys_rmcall_build_gr_init(p); + /* + * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble + * is emitted only on the first forwarded launch (the stock driver's + * per-channel shape); subsequent launches start at the inline-QMD + * window. + */ + if (!READ_ONCE(gsp_phys_rmcall_gr_init_once_param) || + !READ_ONCE(gsp_phys_rmcall_gr_init_emitted)) { + p = gsp_phys_rmcall_build_gr_init(p); + WRITE_ONCE(gsp_phys_rmcall_gr_init_emitted, true); + } if (qmdvirt) p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, @@ -17815,6 +17942,82 @@ static bool gsp_phys_gr_qmd_fetched(const u8 *qmd_mem) } /* + * r427 idempotent fast-accept (CONT.86 ยง6.1). A zero-write launch is either + * the no-execute wedge OR a legitimate idempotent re-launch (the same QMD + * content with the same staged inputs, whose result is already in place) - + * the host cannot tell them apart, and both burn the 40 s + 80 s quiet holds. + * The launch identity below (QMD minus the per-launch release0 address and + * payload, folded with the staged content checksums of every tracked page) is + * recorded for every launch that WROTE; a zero-write launch whose identity + * matches an entry accepts immediately, with no retries and no quiet window. + * Safe by construction: identical code + identical staged content + a prior + * execution that produced this launch's own footprint means the correct + * result is already in the buffers; a side-effecting kernel's re-launch is + * equally wrong under the current accept (the client sees the stale result + * either way) - this only removes the stall. + */ +#define GSP_PHYS_GR_IDEM_SLOTS 64u +static u64 gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS]; +static u32 gsp_phys_gr_idem_n; + +static u64 gsp_phys_gr_launch_fp(const u32 *qmd, + const struct gsp_phys_gr_submit *req, + const u32 *csum_in) +{ + u64 h = 1469598103934665603ULL; + u32 i; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS; i++) { + /* Skip release0's address (w23/w24) and payload (w25): the + * driver allocates a fresh release slot per launch, so they + * differ even for an identical re-launch. */ + u32 w; + + if (i == 23u || i == 24u || i == 25u) + continue; + w = qmd[i]; + h ^= w; + h *= 1099511628211ULL; + } + for (i = 0; i < req->n_data; i++) { + u32 bi = gsp_phys_gr_data_bi(req->data[i].reloc_va); + + if (bi == GSP_PHYS_GR_DATA_BI_NONE) + continue; + h ^= csum_in[bi]; + h *= 1099511628211ULL; + } + return h; +} + +static bool gsp_phys_gr_idem_match(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return true; + return false; +} + +static void gsp_phys_gr_idem_record(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return; + if (gsp_phys_gr_idem_n < GSP_PHYS_GR_IDEM_SLOTS) { + gsp_phys_gr_idem_fp[gsp_phys_gr_idem_n++] = fp; + } else { + /* ring: drop the oldest */ + memmove(&gsp_phys_gr_idem_fp[0], &gsp_phys_gr_idem_fp[1], + (GSP_PHYS_GR_IDEM_SLOTS - 1u) * sizeof(u64)); + gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS - 1u] = fp; + } +} + +/* * COPYAUDIT (2026-10-18 copy-completeness audit): per-copy sighting of the * GR operand staging copies. Every staged page is spot-checked (16 bytes * at the start and end of the page, source vs destination, right after the @@ -18098,6 +18301,8 @@ gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, w0, want_qmd, w1, (u32)ktime_to_us(ktime_get()) - since_us, usr_gpget, sub->gpget, sub->gpput); + /* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */ + gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll"); } /* @@ -18558,6 +18763,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; + bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; u64 qmd_dst = GSP_PHYS_RMCALL_VA_SRC; @@ -18815,6 +19021,15 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ qmd_rel_hpa = 0; qmd_rel_off = 0; + /* + * r420b: the ENGINE will not write the guest's QMD + * release word this launch (rel_mod, or no targetable + * guest release page). Tell the caller so it authors + * the word in software on every outcome - otherwise a + * launch without deferred release records is never + * completed on the guest word (the r414 saxpy hang). + */ + qmd_rel_redirected = true; } } @@ -19600,6 +19815,20 @@ gr_page_unmapped: pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); + /* + * r430: full 64-word deposit dump for the wedge's healthy-vs-wedged + * QMD diff (hardware-only fields w50/w51 INNER_GET/PUT, w52 span-list, + * w53 next-QMD pointer are not in the partial line above). Gated by + * the skedrec param (default on); one long line per launch. + */ + if (gsp_phys_rmcall_gr_skedrec_value()) { + char qb[64 * 9 + 1]; + int qo = 0; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", qmd[i]); + pr_info("vgsp-gr: QMD64 %s\n", qb); + } /* m120: dump the main PT page slots 0x100..0x11f so the PTE content * the walker would read (win0 + bounce window) is visible in the log. */ { @@ -19924,8 +20153,14 @@ gr_page_unmapped: * cancel or dispatch sequence of the launch window is on the * record even when the poll times out. */ - if (gsp_phys_rmcall_gr_skedrec_value() && bar0 && - (++sked_tick & 7u) == 0u) { + if (gsp_phys_rmcall_gr_skedrec_value() && bar0) { + /* + * r430: sample EVERY poll iteration (not every 8th): + * the wedge's release word lands ~7 ms after the + * doorbell, so the old 8-iteration gate saw at most one + * sample of the dispatch window. Changes only. + */ + sked_tick++; sked_sample[0] = ioread32(bar0 + 0x00407060u); sked_sample[1] = ioread32(bar0 + 0x00407064u); sked_sample[2] = ioread32(bar0 + 0x00407068u); @@ -19941,7 +20176,7 @@ gr_page_unmapped: memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", - (sked_tick / 8u) * + sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], sked_sample[2], sked_sample[3], @@ -20477,6 +20712,7 @@ gr_page_unmapped: if (req->n_data && gsp_phys_gr_data_chunk()->pages) { u32 n_tracked = 0; u32 nw_attempts = 0; + bool idem = false; u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, &n_tracked); @@ -20496,12 +20732,55 @@ gr_page_unmapped: ff ? "OK" : "NONE", ff ? "overwritten" : "intact", n_tracked); } - if (wrote == 0 && n_tracked && + /* + * r427 idempotent fast-accept: a zero-write launch whose identity + * (QMD minus release0 + staged content checksums) matches an + * earlier WRITING launch of this boot is a legitimate idempotent + * re-launch - the result is already in place. Accept immediately: + * no retries, no quiet holds. (See gsp_phys_gr_launch_fp.) + */ + if (wrote == 0 && n_tracked) { + u64 fp = gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in); + + if (gsp_phys_gr_idem_match(fp)) { + pr_warn("vgsp-gr: IDEMPOTENT-ACCEPT h=0x%x push=0x%llx fp=0x%llx tracked=%u (same QMD+content as an earlier writing launch; no retries, no hold)\n", + req->h_object, req->push_va, fp, + n_tracked); + idem = true; + } + } + if (wrote == 0 && n_tracked && !idem && gsp_phys_rmcall_gr_nowrite_retry_value()) { u32 nw_retries = (u32)gsp_phys_gr_nowrite_retries_value(); + /* + * r410: the sticky per-lane wedge (previous launch + * also no-wrote while the lane stays busy) is only + * cleared by idleness (r409 ~37 s, r416 ~52 s), so a + * back-to-back retry is futile and keeps it alive. + * When the streak says the lane is wedged, hold a + * quiet window with NO submissions before the retry + * (base on the first retry, doubled on the second), + * then let the normal fresh-bank + resched retry run. + * The quiet path relaxes the whole-launch budget + * (its own hold is the bound: 2 holds max) - the + * alternative is accepting every launch silently. + */ + u32 quiet_used = 0; while (wrote == 0 && nw_attempts < nw_retries) { + u32 quiet_ms = 0; + + if ((READ_ONCE(gsp_phys_gr_nw_streak) > 0 || + READ_ONCE(gsp_phys_gr_wedge_episode)) && + nw_attempts <= 1) { + quiet_ms = + gsp_phys_gr_wedge_quiet_ms_value(); + if (quiet_ms && nw_attempts) + quiet_ms <<= 1; + if (quiet_ms > 240000u) + quiet_ms = 240000u; + } /* * STALLFREE: the same whole-launch budget caps * the retirement retry chain. When it is crossed @@ -20509,9 +20788,13 @@ gr_page_unmapped: * its current outcome (the zero-write accept * below restores the release words and logs * loudly), so a chain of slow dud retries can - * never hold the client past the budget. + * never hold the client past the budget. The + * r410 quiet path is exempt: it is entered only + * for a launch whose previous launch also + * no-wrote, and its own holds are bounded. */ - if (gsp_phys_gr_wait_budget_over(start)) { + if (!quiet_ms && + gsp_phys_gr_wait_budget_over(start)) { gsp_phys_gr_stall_retry++; gsp_phys_gr_stall_log("retry", req->push_va, start, @@ -20522,6 +20805,32 @@ gr_page_unmapped: pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u tracked=%u): refill + re-publish\n", nw_attempts, nw_retries, wrote, n_tracked); + if (quiet_ms) { + gsp_phys_gr_wedge_quiet++; + pr_warn("vgsp-gr: WEDGE-QUIET hold %u ms before outcome retry %u (push=0x%llx; %u consecutive zero-write launch(es); no submissions in between)\n", + quiet_ms, nw_attempts, + req->push_va, + READ_ONCE(gsp_phys_gr_nw_streak)); + /* + * r420: publish the deliberate hold so + * the sema service's completion + * deadlines (the 12 s GR release hold + * and the 3 s per-launch consumption + * bound) pause instead of authoring + * this launch's release word under it - + * the client must not be released + * before the post-hold copy-out + * (gsp_phys_gr_quiet_hold_active). + */ + WRITE_ONCE(gsp_phys_gr_quiet_hold_on, + true); + msleep(quiet_ms); + WRITE_ONCE(gsp_phys_gr_quiet_hold_on, + false); + quiet_used = quiet_ms; + pr_warn("vgsp-gr: WEDGE-QUIET hold done (push=0x%llx); lanes re-armed by the retry below\n", + req->push_va); + } /* * m156 (E5): on the first zero-write retry, push the * proven stage-4 GR semaphore block before re-publishing. @@ -20778,6 +21087,22 @@ gr_page_unmapped: req, bounce_csum_in, &n_tracked); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", nw_attempts, wrote); + /* r410: the quiet-window verdict. */ + if (quiet_used) { + if (wrote) { + gsp_phys_gr_wedge_heals++; + WRITE_ONCE(gsp_phys_gr_wedge_episode, + true); + pr_warn("vgsp-gr: WEDGE-QUIET healed: engine wrote %u page(s) after the %u ms hold (streak was %u)\n", + wrote, quiet_used, + READ_ONCE( + gsp_phys_gr_nw_streak)); + } else { + pr_warn("vgsp-gr: WEDGE-QUIET no heal: %u ms quiet and retry %u still wrote 0 page(s)\n", + quiet_used, nw_attempts); + } + quiet_used = 0; + } } } /* @@ -20790,7 +21115,7 @@ gr_page_unmapped: * loop is disabled (gsp_phys_rmcall_gr_nowrite_retry=0) so a * zero-write outcome is never silent. */ - if (wrote == 0 && n_tracked) { + if (wrote == 0 && n_tracked && !idem) { bool restored = false; bool fetched = gsp_phys_gr_qmd_fetched(qmd_dst_mem); @@ -20808,11 +21133,14 @@ gr_page_unmapped: * may also be a legitimately read-only kernel * (AMBIGUOUS). */ - pr_warn("vgsp-gr: NOWRITE-%s h=0x%x push=0x%llx n_data=%u tracked=%u wrote=0 retries=%u qmd_rel=0x%x (engine fetch %s)\n", + pr_warn("vgsp-gr: NOWRITE-%s h=0x%x push=0x%llx n_data=%u tracked=%u wrote=0 retries=%u qmd_rel=0x%x (engine fetch %s) consec=%u quiet=%llu heals=%llu\n", fetched ? "AMBIGUOUS" : "CONFIRMED", req->h_object, req->push_va, req->n_data, n_tracked, nw_attempts, want_qmd, - fetched ? "OK" : "NONE"); + fetched ? "OK" : "NONE", + READ_ONCE(gsp_phys_gr_nw_streak) + 1u, + gsp_phys_gr_wedge_quiet, + gsp_phys_gr_wedge_heals); /* * m180 (lottery discriminator, agent-order H1/H2). @@ -20879,6 +21207,34 @@ gr_page_unmapped: readl(bar0 + 0x00409008u), readl(bar0 + 0x00409b00u), readl(bar0 + 0x00409b04u)); + /* r425: the wedged-launch SKED/FECS baseline. */ + gsp_phys_gr_sked_wit(bar0, "accept"); + /* + * r425 stale-store witness: re-scan the last completed + * launch's first engine-changed page. A change since its + * copy-out proves a late store landed after the launch + * was shipped (a stuck/queued grid or a stale-QMD + * execution), the mechanism behind the "1 page" heals. + */ + if (gsp_phys_gr_prev_out_valid && + bounce_staged_hpa[gsp_phys_gr_prev_out_bi] == 0 && + gsp_phys_gr_data_chunk()->pages) { + const u8 *pp = gsp_phys_chunk_page_va( + gsp_phys_gr_data_chunk(), + gsp_phys_gr_prev_out_bi); + u32 pc = 0, pi; + + if (pp) { + for (pi = 0; pi < GSP_PHYS_PAGE_SIZE; + pi += 4) + pc += get_unaligned_le32(pp + pi); + if (pc != gsp_phys_gr_prev_out_csum) + pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u csum %08x -> %08x (late store after its launch shipped)\n", + gsp_phys_gr_prev_out_bi, + gsp_phys_gr_prev_out_csum, + pc); + } + } /* * (c) Late-store check: a store ramp that retires after * the host's window is indistinguishable from a lost @@ -20928,6 +21284,31 @@ gr_page_unmapped: pr_warn("vgsp-gr: no-write accept: release words restored (qmd=0x%08x ch=0x%08x); client's own check decides\n", want_qmd, want_ch); } + /* + * r427: record the launch identity of every launch that WROTE, + * so a later idempotent re-launch (zero writes, same identity) + * can accept immediately (see gsp_phys_gr_launch_fp). + */ + if (wrote && n_tracked) + gsp_phys_gr_idem_record( + gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in)); + /* + * r410: streak bookkeeping for the quiet-window trigger. A + * launch with tracked data that wrote nothing extends the + * zero-write streak; any launch that wrote resets it. Runs + * even when the retry loop is disabled so the streak always + * reflects the accepted-outcome history. + */ + if (n_tracked) { + WRITE_ONCE(gsp_phys_gr_nw_streak, + wrote ? 0u : + READ_ONCE(gsp_phys_gr_nw_streak) + 1u); + /* r411: arm the wedge episode on any accepted + * zero-write launch, so the next zero-write launch + * quiets even after a heal reset the streak. */ + if (!wrote) + WRITE_ONCE(gsp_phys_gr_wedge_episode, true); + } } /* * m114: the kernel ran against the pre-commit bounce window; copy the @@ -21077,6 +21458,16 @@ gr_page_unmapped: } if (changed < 8) chg[changed] = bi; + /* + * r425: remember the first engine-changed page + * (post-copy-out content) for the next zero-write + * accept's late-store witness. + */ + if (changed == 0) { + gsp_phys_gr_prev_out_bi = bi; + gsp_phys_gr_prev_out_csum = csum; + gsp_phys_gr_prev_out_valid = true; + } changed++; } /* @@ -21296,6 +21687,7 @@ out: out->elapsed_ms = jiffies_to_msecs(jiffies - start); out->rc = ret; out->fault = fault; + out->qmd_rel_redirected = qmd_rel_redirected; } return ret; } @@ -22399,7 +22791,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused, &b_timeout, &b_pool); seq_printf(m, - "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu stall_wait_max=%llu stall_retry=%llu wait_max_ms=%u\n", + "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu stall_wait_max=%llu stall_retry=%llu wait_max_ms=%u nw_streak=%u wedge_quiet=%llu wedge_heals=%llu\n", gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout, gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool, gsp_phys_gr_forward_enabled(), @@ -22407,7 +22799,10 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) READ_ONCE(gsp_phys_gr_sem_wedged), READ_ONCE(gsp_phys_gr_stall_wait_max), READ_ONCE(gsp_phys_gr_stall_retry), - gsp_phys_gr_wait_max_ms_value()); + gsp_phys_gr_wait_max_ms_value(), + READ_ONCE(gsp_phys_gr_nw_streak), + READ_ONCE(gsp_phys_gr_wedge_quiet), + READ_ONCE(gsp_phys_gr_wedge_heals)); } { /* @@ -23386,6 +23781,30 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) sked_cwd1 = gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_1", 0x00407064u); + /* r425: the wedge diagnostics - the rest of the SKED CWD table, the + * per-8-group GR TPC masks (a zero compute mask would dispatch no + * TPCs), and the FE QMD-virtualization control for VEID 0. */ + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_2", + 0x00407068u); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_3", + 0x0040706cu); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_INFO_1", + 0x00407028u); + { + u32 gi; + + for (gi = 0; gi < 8u; gi++) { + gsp_phys_reg_probe_line(m, bar0, + "FE_TPCM_GR_TPC_MASK", + 0x0040a680u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_LAST_GR_TPC_MASK", + 0x0040a6a0u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_CWD_OWN_TPC_MASK", + 0x0040a6c0u + gi * 4u); + } + } fecs_irq = gsp_phys_reg_probe_line(m, bar0, "FECS_IRQSTAT", 0x00409008u); fecs_cpuctl = gsp_phys_reg_probe_line(m, bar0, "FECS_CPUCTL", diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index d0c5762..80beb54 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -275,6 +275,17 @@ struct gsp_phys_backend_copy_result { u32 elapsed_ms; /* return-to-caller elapsed milliseconds */ int rc; /* duplicated function return code */ struct gsp_phys_fault_info fault; /* decoded engine fault, if any */ + /* + * r420b: the forwarded launch's QMD release0 was redirected to the + * module semaphore (gsp_phys_rmcall_gr_rel_mod=1, or an unresolved + * guest release page), so the ENGINE will never write the guest's own + * QMD release word. The caller must author it in software on every + * outcome (the same write the hold's success edge performs); without + * this a rel_mod=1 launch with no deferred release records is never + * completed on the guest word and the client hangs (the r414 saxpy + * f0 cur=5 stall). + */ + bool qmd_rel_redirected; }; int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa, @@ -436,6 +447,16 @@ struct gsp_phys_gr_submit { * whose boot submission proved out (rmcall_gr_exec_ok). */ bool gsp_phys_backend_gr_ready(void); +/* + * r420: true exactly while the GR submit path sleeps in a WEDGE-QUIET hold + * (a deliberate, bounded wait for the sticky wedge to clear). The in-flight + * forwarded request is NOT done during it, so the sema service's completion + * deadlines (the 12 s GR release hold and the 3 s per-launch consumption + * bound) pause while this returns true instead of authoring the launch's + * release word under the hold (the r419 mid-boot stale-data escape). + */ +bool gsp_phys_gr_quiet_hold_active(void); + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out); diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index e896664..b93c254 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1124,6 +1124,37 @@ int gsp_phys_gr_nowrite_retries_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_nowrite_retries_value); /* + * r410 wedge quiet window (gsp_phys_gr_wedge_quiet_ms; default 40000, clamped + * [0,120000]; 0 disables). The sticky per-lane no-execute wedge (100% of + * launches no-write while kept busy) is proven to CLEAR when the lane stays + * quiet for tens of seconds (r409: a ~37 s client gap healed a wedged run; + * r416: 58 consecutive no-write launches during a 126 s client run, first + * launches green again after the lane sat ~52 s idle; r417: a 20 s host hold + * did not heal, a 40 s hold did - "WEDGE-QUIET healed: engine wrote 1 + * page(s) after the 40000 ms hold"). Immediate retries and + * re-arms do not heal it (r409/r154). So when a zero-write outcome follows a + * previous accepted zero-write launch, the retry loop first holds this many ms + * with NO submissions to the engine, then re-arms (lane enable + schedules) + * and re-publishes once. The value doubles per further quiet retry. The + * quiet window is recovery-only: a launch that wrote anything never enters it. + */ +static unsigned int gsp_phys_gr_wedge_quiet_ms_param = 40000u; +module_param_named(gsp_phys_gr_wedge_quiet_ms, + gsp_phys_gr_wedge_quiet_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wedge_quiet_ms, + "Quiet-window ms before retrying a launch on a lane whose previous launch also no-wrote (0=disable, default 40000; r417: a 20 s hold did not heal, 40 s healed)"); + +unsigned int gsp_phys_gr_wedge_quiet_ms_value(void) +{ + unsigned int v = gsp_phys_gr_wedge_quiet_ms_param; + + if (v > 120000u) + v = 120000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_quiet_ms_value); + +/* * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When * set, the forwarded GR launch copies the QMD's program/param pages into the * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index c664e12..e02d6d8 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -399,6 +399,16 @@ int gsp_phys_gr_nowrite_retries_value(void); * and the sema service authors the QMD completion release in software. */ unsigned int gsp_phys_gr_wait_max_ms_value(void); +/* r410 wedge quiet window: when a launch's zero-write outcome follows a + * previous accepted zero-write launch (the sticky per-lane no-execute wedge), + * hold the retry for this many ms with NO submissions to the engine before + * re-publishing. The wedge is proven to clear when the lane stays quiet for + * tens of seconds (r409 ~37 s, r416 ~52 s); immediate retries keep it alive. + * Base value for the first quiet retry, doubled per further attempt, clamped + * [0,120000]; 0 disables the quiet path (retries run back-to-back as before). + * Default 20000. */ +unsigned int gsp_phys_gr_wedge_quiet_ms_value(void); + /* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */ int gsp_phys_rmcall_gr_inst_sys_value(void); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 53e2140..931669c 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -10000,18 +10000,45 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * r402: a 2D fill (cudaMemset2D): * LINE_COUNT rows of * LINE_LENGTH_IN bytes at - * PITCH_OUT stride. Only a pitch - * WIDER than the row selects row - * mode; a flat fill carries - * pitch == row length (or no - * pitch method) and keeps the - * historical single-run path. + * PITCH_OUT stride. A pitch WIDER + * than the row selects row mode; a + * flat fill carries pitch == row + * length (or no pitch method) and + * keeps the historical single-run + * path. + * + * r420c: LINE_COUNT is + * authoritative for the flat case + * too - cudaMemset2D with pitch == + * width (victory_memset2d, + * pitch=4096/w=4096) is CONTIGUOUS + * rows, so the single run must + * cover rows*line bytes. The + * historical path filled one row + * and the readback lost the rest + * (bad ~= 12.5M of 12.58M). */ - if (st.have_pitch_out && - st.line_count > 1 && - st.pitch_out > st.line_length) { - m->rows = st.line_count; - m->dpitch = st.pitch_out; + if (st.line_count > 1) { + u32 fpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + + if (fpitch > st.line_length) { + m->rows = + st.line_count; + m->dpitch = fpitch; + } else if (st.line_length % + m->elem) { + m->elem = 1; + m->count = + st.line_length * + st.line_count; + } else { + m->count = + (st.line_length / + m->elem) * + st.line_count; + } } if (m->rows > 1 && svc_pitch_log < 32) { svc_pitch_log++; @@ -10053,20 +10080,41 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * LINE_COUNT rows of * LINE_LENGTH_IN bytes with * PITCH_IN/PITCH_OUT strides. - * Row mode only when a pitch is - * wider than the row (a flat 1D - * copy keeps pitch == length). + * Row mode when a pitch is + * wider than the row. + * + * r420c: pitch == row length + * (or no pitch method) makes the + * rows CONTIGUOUS, so the single + * flat run must cover + * rows*line bytes - the + * historical path copied one row + * (victory_memset2d's pitched + * D2H/H2D at pitch 4096 = width + * 4096 lost all but the first + * row). */ - if (st.line_count > 1 && - st.have_pitch_out && - st.pitch_out > st.line_length) { - m->rows = st.line_count; - m->dpitch = st.pitch_out; - m->spitch = st.have_pitch_in ? + if (st.line_count > 1) { + u32 dpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + u32 spitch = st.have_pitch_in ? (st.pitch_in ? st.pitch_in : - st.pitch_out) : - st.pitch_out; + dpitch) : + dpitch; + + if (dpitch == st.line_length && + spitch == st.line_length) { + m->bytes = + st.line_length * + st.line_count; + } else { + m->rows = + st.line_count; + m->dpitch = dpitch; + m->spitch = spitch; + } } n_ops++; ops_seq++; @@ -10376,6 +10424,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, lm->elem, lm->count, lm->aperture, lm->src_aperture, lm->phys, lm->src_phys); + /* + * r420d: 2D geometry of every copy op + * (the memset2d readback forensics - + * the driver decomposes cudaMemcpy2D + * into per-push ops whose LINE_COUNT/ + * PITCH decode must be visible to + * tell "one row per op" from "N rows + * in one op"). Bounded. + */ + if (lm->type == 1) + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n", + ch->h_object, lm->dst, + lm->src, lm->bytes, + lm->rows, lm->dpitch, + lm->spitch, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in); } } @@ -13396,20 +13465,28 @@ bool gsp_sema_pool_hold_fresh = 1; bool gsp_sema_copy_dst_pin_guard = 1; /* * r336/r34x wait gate: honor the C56F host-semaphore acquire ops captured - * for a ring entry before running its work. Still DEFAULT OFF (opt-in - * experiment; the module param sema_wait_gate=1 enables it), but the - * semantics are now the class contract's: op 0 = equal, 2 = strict GEQ, - * 3 = circular GEQ, 4 = AND, 5 = NOR; the wait set is captured PER ENTRY - * (the old single-last-wait global leaked a later entry's acquire into an - * earlier entry - the r340 over-hold); the entry's own release counts only - * when its method order precedes the acquire; pending releases our - * machinery is merely late about are drained first; and every hold is - * bounded at SVC_WAIT_HOLD_MS (100 ms) then fails open loudly ('give-up') - * so a value that can only be installed behind an upstream opdef hold - * cannot park a channel. Diagnostics: 'sema-wait: pass/hold/give-up' rows - * carry op, payload, observed value and decision. - */ -bool gsp_sema_wait_gate; + * for a ring entry before running its work. DEFAULT ON since r408: the + * r408 streams2d forensics reproduced the class the gate was built for + * with the gate off - the c2 readback (C56F ACQ_CIRC_GEQ pay=4 on the + * scale channel's progress counter 0x20400ffd0) ran at 106617.279 with + * cur=2, i.e. BEFORE the c2-add launch was even submitted (106617.411), + * read the not-yet-produced buffer (zeros) and the client failed with + * 'c2 all zeros'. With the gate on the same readback holds (sema-wait: + * hold ... cur=2 waited_ms=353) until the counter reaches 4 (pass ... + * cur=4 at 106812.259) and the ordering is correct. The semantics are + * the class contract's: op 0 = equal, 2 = strict GEQ, 3 = circular GEQ, + * 4 = AND, 5 = NOR; the wait set is captured PER ENTRY (the old + * single-last-wait global leaked a later entry's acquire into an earlier + * entry - the r340 over-hold); the entry's own release counts only when + * its method order precedes the acquire; pending releases our machinery + * is merely late about are drained first; and every hold is bounded at + * gsp_sema_wait_hold_ms (default 4000 ms, r401) then fails open loudly + * ('give-up') so a value that can only be installed behind an upstream + * opdef hold cannot park a channel. Diagnostics: 'sema-wait: + * pass/hold/give-up' rows carry op, payload, observed value and decision. + * sema_wait_gate=0 restores the pre-r336 pass-through for an A/B. + */ +bool gsp_sema_wait_gate = 1; /* * m169b: the guard above must never deadlock the boot's first client. The @@ -15428,6 +15505,19 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) * hardware has not produced yet. */ #define SVC_FWD_GR_HOLD_TIMEOUT_MS 12000u +/* + * r420: hard cap on the WEDGE-QUIET deferral of the GR hold deadline. The + * backend's quiet hold is a bounded msleep (its own caps: <=240 s per hold, + * two holds per launch), so the pause can only last a few minutes; the cap is + * a defence-in-depth bound so a stuck quiet-hold flag can never defer the + * client's completion indefinitely. + */ +#define SVC_FWD_GR_HOLD_PAUSE_MAX_MS 600000u +/* + * r420: same defence for the per-launch consumption bound pause in + * svc_gr_hold_bound_expired(). + */ +#define SVC_GR_QUIET_PAUSE_MAX_MS 600000u struct svc_fwd_hold { bool active; @@ -15463,9 +15553,9 @@ struct svc_fwd_hold { /* * P6c completion relay: the request selected record relay_index and * the hardware copy was asked to author it into the guest-visible - * semaphore page. On the success edge the hold re-reads the guest - * word and, when it carries the payload, skips that one record's - * software apply. Cleared with the hold. + * semaphore page. On the success edge the hold re-reads the guest word + * and, when it carries the payload, skips that one record's software + * apply. Cleared with the hold. */ bool relay_armed; u8 relay_index; @@ -15478,6 +15568,18 @@ struct svc_fwd_hold { struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; + /* + * r420: WEDGE-QUIET deferral state. While the backend deliberately + * holds the in-flight launch in its bounded quiet window + * (gsp_phys_gr_quiet_hold_active), the GR deadline is pushed forward + * (paused_until) so the hold cannot author the launch's release word + * under the hold - the client would then verify data the post-hold + * copy-out has not written yet (the r419 mid-boot escape). The + * deferral is itself bounded by SVC_FWD_GR_HOLD_PAUSE_MAX_MS from + * `since` and is announced once per hold. + */ + unsigned long paused_until; + bool pause_logged; }; static struct svc_fwd_hold svc_fwd_hold; @@ -16280,9 +16382,38 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) static bool svc_fwd_hold_expired(void) { - if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) + if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) { + /* + * r420: while the backend deliberately holds the in-flight + * launch in its bounded WEDGE-QUIET window, the GR deadline + * must not fire - expiring here authors the launch's release + * word (svc_fwd_hold_gr_release) and applies the recorded + * releases before the post-hold copy-out, so the client + * verifies stale data (the r419 soak's mid-boot + * "matmul fails after time"). Pause instead, bounded by + * SVC_FWD_GR_HOLD_PAUSE_MAX_MS from the hold's start; the + * pause extends the deadline a full GR window past the last + * quiet-hold pass, so the backend's retry/accept can finish + * before the fallback would author. Announced once per hold. + */ + if (gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, svc_fwd_hold.since + + msecs_to_jiffies(SVC_FWD_GR_HOLD_PAUSE_MAX_MS))) { + if (!svc_fwd_hold.pause_logged) { + svc_fwd_hold.pause_logged = true; + pr_warn("vgsp-gr: GR-HOLD-PAUSE va=0x%llx (backend wedge-quiet hold active; release authorship deferred under the bounded hold)\n", + svc_fwd_req.gr_push_va); + } + svc_fwd_hold.paused_until = jiffies + + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS); + return false; + } + if (svc_fwd_hold.paused_until && + time_before(jiffies, svc_fwd_hold.paused_until)) + return false; return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS)); + } return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } @@ -16316,6 +16447,9 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -16342,6 +16476,8 @@ static void svc_fwd_hold_clear(void) svc_fwd_hold.relay_armed = false; svc_fwd_hold.relay_index = 0; svc_fwd_hold.fault_delivered = false; + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_rel_pending = 0; } @@ -17106,6 +17242,17 @@ static void svc_fwd_gr_work(void) ga_va, ga_used, ga_led); hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + /* + * r420b: the backend redirected the launch's QMD release0 to the + * module semaphore (rel_mod=1 or no targetable guest release page), so + * the ENGINE never writes the guest's own QMD release word. Treat the + * word as unobserved so the bounded outcome check (and the hold's + * edges) author it in software on every outcome; without this a + * rel_mod=1 launch with no deferred release records is never completed + * on the guest word and the client hangs (the r414 saxpy stall). + */ + if (cp.qmd_rel_redirected) + svc_fwd_req.gr_rel_unobserved = true; if (!hw_rc) { svc.fwd_gr_ok++; } else { @@ -18055,6 +18202,27 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); ch->gr_hold_exp_logged = false; } + /* + * r420: pause the per-launch consumption bound while the backend + * deliberately holds the in-flight launch in its bounded WEDGE-QUIET + * window. At the bound the pump falls through to the historical + * consume, which authors the capture's QMD completion release in + * software (GR-STALL-hold) - but the backend is still going to + * submit/accept this very launch after the hold and copy its data + * out; authoring now releases the client onto stale data (the r419 + * mid-boot escape). The quiet hold is a bounded msleep (the + * backend's own caps), and the deferral here is bounded by + * SVC_GR_QUIET_PAUSE_MAX_MS from the bound's arm time as + * defence-in-depth. Rate-limited announcement. + */ + if (ch->gr_hold_dead_va && gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, ch->gr_hold_dead_since + + msecs_to_jiffies(SVC_GR_QUIET_PAUSE_MAX_MS))) { + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + pr_info_ratelimited("vgsp-gr: GR-HOLD-BOUND-PAUSE launch=0x%llx (backend wedge-quiet hold active; consumption bound deferred)\n", + launch_va); + return false; + } return time_after_eq(jiffies, ch->gr_hold_deadline); } @@ -20771,7 +20939,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, - ch->gr_hold_until))) { + ch->gr_hold_until) || + /* + * r420: the backend's + * bounded WEDGE-QUIET + * hold outlives this + * one-shot window; keep + * re-holding (never + * consume-with-sw-release) + * while it is active, so + * the launch is still + * forwarded and copied out + * after the hold instead of + * being released onto stale + * data. + */ + gsp_phys_gr_quiet_hold_active())) { u32 hold_ms = gsp_sema_gr_grace_ms ? gsp_sema_gr_grace_ms : 8000; diff --git a/kernel/main.c b/kernel/main.c index d2a44d6..0110372 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2829,6 +2829,7 @@ MODULE_PARM_DESC(sema_pool_hold_fresh, "apply pool releases at fresh-pin time in */ module_param_named(sema_copy_dst_pin_guard, gsp_sema_copy_dst_pin_guard, bool, 0644); module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644); +MODULE_PARM_DESC(sema_wait_gate, "honor C56F host-semaphore acquires before running a ring entry's work (default 1 since r408; 0 restores pass-through)"); module_param_named(sema_wait_hold_ms, gsp_sema_wait_hold_ms, uint, 0644); MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, ms (r401, default 4000; must exceed the emulated launch latency)"); MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)"); |