diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 73 |
1 files changed, 71 insertions, 2 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 871e1af..e8a6a14 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3964,6 +3964,20 @@ static u32 svc_ledger_n; static u64 svc_ledger_diverges; static u64 svc_ledger_pinwins; /* r336: pin-backed divergence wins */ static u32 svc_d2hmap_log; /* D2H-MAP per-page destination budget */ +/* + * r336: host-semaphore acquire (C56F SEM_EXECUTE op 0/2/3/4/5) wait state. + * The decoder records the last acquire of the current push; svc_run_ops + * honors it before running any of the push's work, so a copy cannot + * execute ahead of the dependency its push waits on (the streams2 + * readback-before-scale class). The re-decode re-parses the push, so the + * state is refreshed on every pass. + */ +static u64 svc_wait_va; +static u32 svc_wait_payload; +static u8 svc_wait_mode; /* 0 = equal, 1 = geq, 2 = circ geq */ +static u8 svc_wait_pending; +static u64 svc_wait_held; +static u64 svc_wait_pass; static u64 svc_ledger_stores; static u64 svc_ledger_evictions; static u64 svc_ledger_evict_since_reset; @@ -4091,10 +4105,11 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gr_nocap_held, svc.gr_nocap_consumed, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, - svc.pool_alt_applied, svc.pool_residue); + svc.pool_alt_applied, svc.pool_residue, + svc_wait_held, svc_wait_pass); if (svc.resc_run || svc_rescue_scan_capped) pr_info("sema-svc: rescue_run=%llu scan_capped=%llu (multi-page content pins)\n", svc.resc_run, svc_rescue_scan_capped); @@ -6278,6 +6293,38 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * output was a_stale + b_fresh). */ svc_handle_map_add(); + /* + * r336: honor the push's host-semaphore acquire (C56F SEM_EXECUTE + * op 0/2/3/4/5) before any of its work runs. The push's release is + * withheld through the op-pending machinery (bounded, loud), so the + * guest cannot observe completion of work that ran ahead of the + * dependency it waits on. Unreadable semaphore: fail-open (the + * pre-r336 behaviour). + */ + if (svc_wait_pending) { + u32 cur = 0; + bool ok = true; + + if (!svc_sem_read(vgpu, svc_wait_va, &cur)) { + if (svc_wait_mode == 0) + ok = (cur == svc_wait_payload); + else if (svc_wait_mode == 2) + ok = (s32)(cur - svc_wait_payload) >= 0; + else + ok = cur >= svc_wait_payload; + } + if (!ok) { + svc_wait_held++; + if (svc_wait_held <= 32) + pr_info("sema-wait: held ch=0x%x va=0x%llx need=%u cur=%u mode=%u (acquire not satisfied)\n", + ch_h, svc_wait_va, + svc_wait_payload, cur, + svc_wait_mode); + svc.op_tr_pending = 1; + return; + } + svc_wait_pass++; + } for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; u32 seq = ops_base + i; @@ -8033,6 +8080,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); + svc_wait_pending = 0; /* r336: new push, new wait */ off += 4 + count * 4; continue; } @@ -8149,6 +8197,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, va, payload, op); } } + } else if (op != HOST_SEM_OP_RELEASE && + va >= SEMA_VA_LO && va < SEMA_VA_HI) { + /* + * r336: C56F SEM_EXECUTE acquire ops (0 = + * acquire/equal, 2 = strict GEQ, 3 = circular + * GEQ, 4/5 = AND/NOR). Core CUDA starts its + * compute-channel pushes with one of these to + * wait for a progress counter (observed: + * [LO=0x0400ffd0][HI=2][pay=4][0][0x1003] on + * the D2H readback push - ACQ_CIRC_GEQ on the + * channel counter 0x20400ffd0). svc_run_ops + * honors the wait before running the push's + * work, so a copy cannot execute ahead of the + * dependency it waits on (the streams2 + * readback-before-scale class). + */ + svc_wait_va = va; + svc_wait_payload = payload; + svc_wait_mode = (op == 3) ? 2 : + (op == 0) ? 0 : 1; + svc_wait_pending = 1; } off += 4 + count * 4; continue; |