diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 97 |
1 files changed, 93 insertions, 4 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1c6c41a..a09a3c5 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -473,6 +473,17 @@ struct sema_svc { /* full-RAM apply pass fallback for channels whose pushes cannot be read */ u64 fast_cursor; u64 fast_passes; + /* + * jiffies of the most recent channel registration. A new client's + * channel appears before the external VA->PA stream has noticed + * the pid switch, so pool-VA translations can still point at the + * PREVIOUS client's page for the first seconds; a release applied + * through such a stale pin is absorbed there and the word the new + * client polls stays untouched. Pool-VA applies are held for a + * short window after registration so the retry pass uses the + * fresh pins. + */ + unsigned long last_chan_reg; u64 fast_pass_applied; u64 fast_hold_until; /* @@ -970,11 +981,13 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } spin_unlock_irqrestore(&svc.chan_lock, flags); - if (ch) + if (ch) { + svc.last_chan_reg = jiffies; pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, desc->notifier_pa); + } else pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n", desc->h_object, SVC_MAX_CHANNELS); @@ -2232,6 +2245,12 @@ static inline bool svc_pool_sem_va(u64 va) return va >= SVC_POOL_SEM_VA_LO && va < SVC_POOL_SEM_VA_HI; } +static inline bool svc_pool_hold_active(void) +{ + return svc.last_chan_reg && + time_before(jiffies, svc.last_chan_reg + 5 * HZ); +} + /* * Verified pool-semaphore translations, keyed by page VA. An entry is only * created from an injected pin or from a targeted rm_mem resolve whose page @@ -4660,6 +4679,18 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, if (svc_sem_read(vgpu, sem_va, &pcur)) return; + /* + * Zero = the guest's base write has not landed. + * An absolute payload write now is absorbed by + * it. Queue instead; the decode already queued + * this VA with op=1 for an increment and + * svc_pending_add() keeps that first entry. + */ + if (!pcur || svc_pool_hold_active()) { + svc_pending_add(sem_va, payload, + inc ? 1 : 0); + return; + } if (payload <= pcur) return; wrc = svc_sem_write(vgpu, h, sem_va, payload); @@ -6153,6 +6184,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, rrc2 = svc_sem_read(vgpu, sem_va, &cur); } + if (svc_pool_sem_va(sem_va) && gsp_sema_dbg && + svc.counter_rel_dbg < 256) { + svc.counter_rel_dbg++; + pr_info("sema-chan: pool-rel cand ch=0x%x va=0x%llx pay=%u inc=%d rrc=%d cur=%u idx=%d\n", + ch ? ch->h_object : 0, sem_va, + st.sem_payload, + (DMA_SEM_REDUCTION_EN(dma) && + DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC) ? 1 : 0, + rrc2, cur, (int)entry_idx); + } if (rrc2) { svc.pushes_nomap++; /* @@ -6171,6 +6212,33 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, (DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC) ? 1 : 0); + } else if (svc_pool_sem_va(sem_va) && + (svc_pool_hold_active() || !cur)) { + /* + * A readable pool word of 0 + * means the guest has not + * written the tracking + * semaphore's base value (1) + * yet; an apply now is + * absorbed by that write (the + * observed second-session + * cur=0 -> val=1 line). + * Defer with the increment + * flag preserved; the retry + * pass lands it after the + * base. Count it as handled so + * the pump consumes the entry + * instead of fake-authoring a + * pre-base write. + */ + svc.pushes_nomap++; + svc_pending_add(sem_va, + st.sem_payload, + (DMA_SEM_REDUCTION_EN(dma) && + DMA_SEM_REDUCTION(dma) == + SEM_REDUCTION_INC) ? 1 : 0); + svc_rel_va_seen = sem_va; + svc_rel_count++; } else { have_cur = true; if (st.sem_payload > cur) @@ -6555,11 +6623,32 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) i++; continue; } - if (p->payload <= cur) { - svc_pending[i] = svc_pending[--svc_pending_n]; + /* + * An increment entry adds to the word, so payload <= cur + * is not stale (the base write can equal the payload). For + * a pool VA the base may not have landed: applying while + * the word is still 0 is absorbed, so hold and retry. + */ + if (svc_pool_sem_va(p->va) && svc_pool_hold_active()) { + p->tries++; + svc_pending_note_starved(p); + i++; continue; } - if (p->payload - cur > 0x100000) { + if (p->op) { + if (svc_pool_sem_va(p->va) && !cur) { + p->tries++; + svc_pending_note_starved(p); + i++; + continue; + } + if (p->payload > 0x100000) { + svc_pending[i] = + svc_pending[--svc_pending_n]; + continue; + } + } else if (p->payload <= cur || + p->payload - cur > 0x100000) { svc_pending[i] = svc_pending[--svc_pending_n]; continue; } |