diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 83 |
1 files changed, 70 insertions, 13 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index e8a6a14..fbb278d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3978,6 +3978,9 @@ static u8 svc_wait_mode; /* 0 = equal, 1 = geq, 2 = circ geq */ static u8 svc_wait_pending; static u64 svc_wait_held; static u64 svc_wait_pass; +static u32 svc_wait_log; /* per-session held-row print budget */ +static u64 svc_wait_rel_va; /* current push's own release (mirror) */ +static u32 svc_wait_rel_payload; static u64 svc_ledger_stores; static u64 svc_ledger_evictions; static u64 svc_ledger_evict_since_reset; @@ -6275,6 +6278,16 @@ static void svc_handle_map_add(void); /* Execute a decoded operation list (fills and copies). `ops_done` (may be * NULL) is the per-entry bitmap of ops that already ran for the entry's * identity; `ops_base` is the sequence number of ops[0] within the entry. */ +/* r336: evaluate a C56F SEM_EXECUTE acquire against a candidate value. */ +static bool svc_wait_satisfied(u32 cur, u8 mode, u32 need) +{ + if (mode == 0) + return cur == need; + if (mode == 2) + return (s32)(cur - need) >= 0; + return cur >= need; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base) { @@ -6301,30 +6314,55 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * dependency it waits on. Unreadable semaphore: fail-open (the * pre-r336 behaviour). */ - if (svc_wait_pending) { + if (svc_wait_pending && gsp_sema_wait_gate) { u32 cur = 0; bool ok = true; - if (!svc_sem_read(vgpu, svc_wait_va, &cur)) { - if (svc_wait_mode == 0) - ok = (cur == svc_wait_payload); - else if (svc_wait_mode == 2) - ok = (s32)(cur - svc_wait_payload) >= 0; - else - ok = cur >= svc_wait_payload; + /* + * Kill switch: a wait that never satisfies (a counter our + * emulation does not advance the way the driver expects) + * would otherwise park entries until the sweep expires each + * one. After 2048 holds in a boot the gate fails open + * permanently (the pre-r336 behaviour) and says so once. + */ + if (svc_wait_held > 2048) { + if (svc_wait_log < 4096) { + svc_wait_log++; + pr_info("sema-wait: gate disabled after %llu holds (fail-open)\n", + svc_wait_held); + } + svc_wait_pass++; + goto svc_wait_done; } + if (!svc_sem_read(vgpu, svc_wait_va, &cur)) + ok = svc_wait_satisfied(cur, svc_wait_mode, + svc_wait_payload); + /* + * The driver's release(N)+acquire(N) pattern (channel.c + * crossEngineSync): the push's own release lands only after + * its ops run, so it cannot satisfy the wait through memory + * yet - treat its payload as applied. + */ + if (!ok && svc_wait_rel_va == svc_wait_va && + svc_wait_rel_payload) + ok = svc_wait_satisfied(svc_wait_rel_payload, + svc_wait_mode, + svc_wait_payload); if (!ok) { svc_wait_held++; - if (svc_wait_held <= 32) - pr_info("sema-wait: held ch=0x%x va=0x%llx need=%u cur=%u mode=%u (acquire not satisfied)\n", - ch_h, svc_wait_va, - svc_wait_payload, cur, - svc_wait_mode); + if (svc_wait_log < 512) + pr_info("sema-wait: held ch=0x%x r=%u va=0x%llx need=%u cur=%u mode=%u rel=0x%llx rpay=%u (acquire not satisfied)\n", + ch_h, svc_wait_log++, + svc_wait_va, svc_wait_payload, + cur, svc_wait_mode, + svc_wait_rel_va, + svc_wait_rel_payload); svc.op_tr_pending = 1; return; } svc_wait_pass++; } +svc_wait_done: for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; u32 seq = ops_base + i; @@ -8081,6 +8119,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, n_ops = 0; memset(&st, 0, sizeof(st)); svc_wait_pending = 0; /* r336: new push, new wait */ + svc_wait_rel_va = 0; + svc_wait_rel_payload = 0; off += 4 + count * 4; continue; } @@ -8099,6 +8139,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + svc_wait_rel_va = va; + svc_wait_rel_payload = payload; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -8267,6 +8309,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = st.rs_payload; + svc_wait_rel_va = va; + svc_wait_rel_payload = st.rs_payload; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -8820,6 +8864,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.sem_va_lo; st.rel_va = sem_va; st.rel_payload = st.sem_payload; + svc_wait_rel_va = sem_va; + svc_wait_rel_payload = st.sem_payload; st.have_rel = 1; if (st.have_sem && st.sem_payload && @@ -9560,6 +9606,7 @@ static void svc_session_flush(void) svc.opadd_log = 0; svc.dmatrace_log = 0; svc.copyent_log = 0; + svc_wait_log = 0; svc.bigrun_log = 0; svc.lid_hdr_log = 0; svc.skiprun_log = 0; @@ -11775,6 +11822,16 @@ bool gsp_sema_pool_hold_fresh = 1; * 0 restores the historical immediate-write behavior. */ bool gsp_sema_copy_dst_pin_guard = 1; +/* + * r336 wait gate: honor the C56F host-semaphore acquire before running a + * push's work. DEFAULT OFF: the acquire's tracking-semaphore values are + * not yet fully modelled (channel-first pushes over-hold: the wait needs a + * counter value only the withheld release itself would install), so the + * gate stays an experiment until the counter semantics are finished. The + * streams2 readback-before-scale class is root-caused in the handoff; + * with the gate off the behaviour is the pre-r336 one. + */ +bool gsp_sema_wait_gate; /* * m169b: the guard above must never deadlock the boot's first client. The |