diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 93 |
1 files changed, 92 insertions, 1 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 7426811..5a30b1d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1141,6 +1141,13 @@ struct sema_svc { u64 fwd_gr_cewm_cap; /* r590: watermark still * backlogged at the defer cap */ u32 fwd_gr_cewm_log; + u64 ce_mutual_skips; /* r638: watermark + * skipped a channel whose + * head entry is wait-gate + * held (a consumer-side + * mutual citation, not a + * producer backlog) */ + u32 ce_mutual_log; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -4738,6 +4745,10 @@ static bool svc_wait_rel_have; struct svc_wait_hold { bool active; u32 h; + u32 idx; /* r638: GPFIFO entry index this hold owns + * (0xffffffff when unset); the CE stream-order + * watermark uses it to detect a mutual citation + * (a held consumer entry, not a producer backlog) */ unsigned long since; }; #define SVC_WAIT_HOLD_MAX 16 @@ -8142,6 +8153,30 @@ static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) } } +/* + * r638: is (h, idx) currently owned by the bounded wait gate? The CE + * stream-order watermark consults this to tell a producer backlog (a CE + * copy the client ordered before the launch - the reason the watermark + * exists) from a consumer-side wait the forward itself will unblock (the + * mutual citation measured on r612/r636: the deferred launch's release is + * the only value that can satisfy the held copy's acquire, and the copy's + * unconsumed entry is the only reason the launch is deferred). + */ +static bool svc_wait_held_entry(u32 h, u32 idx) +{ + int i; + + if (idx == 0xffffffffu) + return false; + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + const struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h && e->idx == idx) + return true; + } + return false; +} + /* r34x: the per-channel first-hold clock for the bounded wait. */ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) { @@ -8161,6 +8196,7 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) memset(free_slot, 0, sizeof(*free_slot)); free_slot->active = true; free_slot->h = h; + free_slot->idx = 0xffffffffu; /* r638: unset until a hold lands */ free_slot->since = jiffies; return free_slot; } @@ -8539,6 +8575,10 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, svc_wait_held++; svc_wait_hold_h = ch_h; svc_wait_hold_idx = entry_idx; + /* r638: record the held entry for the CE watermark's + * mutual-cycle check. */ + if (gh) + gh->idx = entry_idx; svc.op_tr_pending = 1; return; } @@ -18559,6 +18599,19 @@ module_param_named(sema_refusal_fault, gsp_sema_refusal_fault, bool, 0644); MODULE_PARM_DESC(sema_refusal_fault, "Deliver a synthetic channel error edge on a failed/refused GR forward (default 1; 0 = legacy silent sw release)"); +/* + * r638: break the CE-watermark/wait-gate mutual citation (see + * svc_wait_held_entry and svc_ce_stream_backlog): a channel whose + * unconsumed head entry is wait-gate held is a consumer-side wait, not a + * producer backlog, and is skipped while the candidate's operands are fully + * resolved and no decoded device write is pending on a data page. Rollback + * to the r636 behaviour: sema_mutual_skip=0. + */ +bool gsp_sema_mutual_skip = true; +module_param_named(sema_mutual_skip, gsp_sema_mutual_skip, bool, 0644); +MODULE_PARM_DESC(sema_mutual_skip, + "Skip a wait-held consumer head in the CE stream-order watermark (default 1; 0 = r636 behaviour)"); + static void svc_gr_fault_synth(struct nv_vgpu_device *vgpu, u32 h_object) { u8 notif[SVC_NOTIFY_SIZE] = {0}; @@ -22173,8 +22226,46 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu, c->userd_as, &put, sizeof(put))) continue; put %= c->gpfifo_entries; - if (put != ((u32)c->last_get % c->gpfifo_entries)) + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* + * r638 mutual-cycle gate. The r612/r636 captures + * showed a provable deadlock: a launch L is deferred + * by this watermark because channel X holds an + * unconsumed entry; X's head entry is held by the + * bounded wait gate because its acquire's release is + * authored by L itself, which cannot forward until + * this watermark clears - both halves cite each + * other, and only the 15 s defer cap / the wait's + * fail-open break it (the copy runs ~20 s late; the + * launch finds its operand pages empty). Such a head + * is a consumer-side wait, not a producer backlog: + * skip it and let the forward install the release. + * Bounded by the same guards the watermark's + * consumers already trust: skip only while the + * candidate's operands are fully resolved (no + * stalled fill pages - a launch that still needs + * producer data stays protected) and no decoded + * device write is pending on a data page. Rollback: + * sema_mutual_skip=0. + */ + if (gsp_sema_mutual_skip && + svc_gr_forward_pending() && + !svc_gr_data_stalled && + !svc_gr_dstpend_overlap() && + svc_wait_held_entry(c->h_object, + (u32)c->last_get % c->gpfifo_entries)) { + svc.ce_mutual_skips++; + if (svc.ce_mutual_log < 32) { + svc.ce_mutual_log++; + pr_info("vgsp-work: watermark skip h=0x%x head=%u (head is wait-gate held; forward unblocks its release)\n", + c->h_object, + (u32)c->last_get % + c->gpfifo_entries); + } + continue; + } return true; + } } return false; } |