diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 175 |
1 files changed, 173 insertions, 2 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index c50832e..13ad648 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -23671,6 +23671,165 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, put, get, ch->submissions, waited, reason); } +/* + * --------------------------------------------------------------------- + * SESSION 126 - FENCE FOLD: channel-anchored sweep of a GR-class + * channel's PRE-BUILT ring continuation. + * + * The Vulkan client's fence waits never clear unaided because the + * driver's kick boundary (GP_PUT) stops at the first batch while the ring + * already carries the continuation entries it pre-built for the next + * submits - and the fence's own INC-5 RELEASE group (observed + * [2005c017][0x45ba090][0][payload 1][0][0x02100001]) lives in one of + * those entries, so the pump's GP_PUT-bounded walk never decodes it + * (r715/r716: put=11, entries 12..47 carried content, no dmesg row ever + * mentioned 0x45ba). The host-side fenceword_assist wrote the wait word + * directly; this fold makes the service author it from the driver's own + * push bytes. + * + * Doctrine: unlike the disabled page-content sweep (svc_apply_page_data, + * off since the false-positive corruption incidents), this sweep is + * CHANNEL-ANCHORED: it only walks entries reachable through the ring's + * validated GPFIFO translation, only accepts entries whose bytes pass the + * strict method-stream gate, and only applies the semaphore release + * groups it finds - forward-only, through the standard pending ledger + * (svc_pending_add). No consumption bookkeeping is touched (last_get, + * put_prev, slot_done_epoch and the replay state stay as the pump left + * them), so a later real kick of the same entries decodes them normally; + * repeated sweeps are idempotent (the pending ledger dedups (va,payload) + * and the apply path is forward-only). + * + * Gated by the same live-epoch arm as the counter publish + * (ch->gr_pub_armed) so a residual ring from a previous session cannot + * author releases before the session switch is proven. + * --------------------------------------------------------------------- + */ +unsigned gsp_sema_gr_ahead_max = 64; +module_param_named(sema_gr_ahead, gsp_sema_gr_ahead_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_ahead, + "GR-class channels: max GPFIFO entries beyond GP_PUT whose channel-anchored pushes are swept for semaphore releases (0=off)"); + +static u32 svc_gr_ahead_log; +static u64 svc_gr_ahead_entries; +static u64 svc_gr_ahead_rels; +static u64 svc_gr_ahead_skips; + +/* + * Extract the semaphore release groups from one coherent push and queue + * them forward-only. Mirrors the decode sites' group shape exactly: + * addr 0x5c, count >= 5, op RELEASE (1) or REDUCTION (6); the address is + * w0/w1 as a 40-bit VA and the payload is w2. + */ +static u32 svc_gr_ahead_push(struct svc_channel *ch, const u8 *buf, u32 len) +{ + u32 off = 0, nrel = 0; + + while (off + 4 <= len) { + u32 hdr, op, addr, count; + + memcpy(&hdr, buf + off, 4); + if (!hdr) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + addr = (hdr & 0x1fff) << 2; + count = (hdr >> 16) & 0xfff; + if (op == 0 || op == 2 || op == 4 || op == 7) { + off += 4; + continue; + } + if (op != 1 && op != 3 && op != 5) { + off += 4; + continue; + } + if (!count || count > 0xfff || off + 4 + count * 4 > len) { + off += 4; + continue; + } + if (addr == HOST_M_SEM_ADDR_LO && count >= 5) { + u32 w0, w1, w2, w4; + u64 va; + + memcpy(&w0, buf + off + 4, 4); + memcpy(&w1, buf + off + 8, 4); + memcpy(&w2, buf + off + 12, 4); + memcpy(&w4, buf + off + 20, 4); + va = ((u64)(w1 & 0xff) << 32) | + (u64)(w0 & 0xfffffffc); + op = w4 & 0x7; + if ((op == HOST_SEM_OP_RELEASE || + op == HOST_SEM_OP_REDUCTION) && w2 && + va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_HI) { + svc_pending_add(va, w2, + op == HOST_SEM_OP_REDUCTION ? 1 : 0); + nrel++; + } + } + off += 4 + count * 4; + } + (void)ch; + return nrel; +} + +static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put) +{ + static u8 abuf[2048]; + u32 i, maxent, nrel = 0, swept = 0, nskip = 0; + + if (!put || !ch->gpfifo_entries) + return; + maxent = min_t(u32, gsp_sema_gr_ahead_max, 512); + for (i = 0; i < maxent; i++) { + u32 idx = (put + i) % ch->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen; + u64 va; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + break; /* unwritten slot: end of the continuation */ + if (!va || !len || len > SVC_PUSH_MAX) + break; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + break; + rlen = min_t(u32, len, sizeof(abuf)); + if (svc_read_va_push(vgpu, ch, va, abuf, rlen)) { + /* + * Unreadable here (pre-anchor translation) is a + * reason to stop this pass: the continuation is not + * proven coherent, so nothing is authored. + */ + nskip++; + break; + } + if (!svc_stream_full_ok(abuf, rlen)) { + nskip++; + continue; + } + swept++; + nrel += svc_gr_ahead_push(ch, abuf, rlen); + } + svc_gr_ahead_entries += swept; + svc_gr_ahead_rels += nrel; + svc_gr_ahead_skips += nskip; + if (swept && (nrel || svc_gr_ahead_log < 64)) { + svc_gr_ahead_log++; + pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u skipped=%u (total rels=%llu)\n", + ch->h_object, put, swept, nrel, nskip, + svc_gr_ahead_rels); + } +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { if (svc_pdb_disc_pending) { @@ -24198,6 +24357,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->put_prev = put; /* + * SESSION 126 fence fold: sweep the ring continuation beyond + * GP_PUT for semaphore releases (see svc_gr_ahead_sweep). + * GR-class channels only, live-armed epochs only, and only + * with the ring translation proven through the channel + * object (rrc == 0). + */ + if (!ch->engine_type && ch->gr_pub_armed && + gsp_sema_gr_ahead_max && !rrc && ring_pa) + svc_gr_ahead_sweep(vgpu, ch, put); + + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. * Runs before the entry loop so a wedged session epoch is * recovered (and logged) instead of spinning: see @@ -24695,7 +24865,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * Budget: 512 rows per client (reset in * svc_session_flush). */ - if (gsp_sema_dbg && svc_pushdec_log < 512) { + if (gsp_sema_dbg && svc_pushdec_log < 2048) { svc_pushdec_log++; pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n", ch->h_object, va, len, @@ -26847,12 +27017,13 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) svc_read_ap(vgpu, ch->notifier_pa + 16 + 8, ch->notifier_as, &info1, 4); } - pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", + pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx armed=%d notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", ch->h_object, ch->engine_type, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, ch->userd_ok, ch->last_get, ch->submissions, ch->expected_sem_payload, ch->last_rel_va, ch->counter_pa, + ch->gr_pub_armed, ch->notifier_pa, ch->notifier_as, info0, st0, info1, st1, ch->notified); |