diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 53 |
1 files changed, 42 insertions, 11 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 34d928f..734fb1d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -312,6 +312,7 @@ struct sema_svc { u64 memsets_nomap; u64 memsets_fail; u64 ring_ops; /* op runs from ring-decoded pushes */ + u64 lda_log; /* LAUNCH_DMA inventory budget */ u64 releases_seen; u64 releases_applied; u64 releases_missed; /* no VA->PA entry */ @@ -1713,14 +1714,13 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * Push buffers are written back to back and pushes can span page * boundaries, so there is no reliable "push start" marker to anchor on: * decode the whole page as a method stream, carrying decoder state across - * the methods. Collected memory operations are executed only when the - * push's tracking semaphore release is reached and is still pending - * (payload ahead of the semaphore's current value) - recycled buffers - * replay old pushes whose semaphore is already satisfied, and those are - * skipped, so their stale memsets never overwrite newer state. + * the methods. Work decoded from pages read through a trusted ring + * translation is executed at the push's release method (see below); pages + * decoded from raw sweeps never execute work at all. */ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, - u32 len, u64 page_pa, bool allow_ops) + u32 len, u64 page_pa, bool allow_ops, + u32 ch_h, u32 ch_eng) { struct svc_push_state st; struct svc_memset_op ops[SVC_MAX_MEMSETS]; @@ -1979,6 +1979,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); + if (ch_h && svc.lda_log < 48) { + svc.lda_log++; + pr_info("sema-lda: h=0x%x eng=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", + ch_h, ch_eng, dma, transfer, + DMA_REMAP_ENABLE(dma), + DMA_DST_MEMORY_LAYOUT_PITCH(dma), + DMA_DST_TYPE_PHYSICAL(dma), + DMA_SRC_TYPE_PHYSICAL(dma), + DMA_SEMAPHORE_TYPE(dma), + st.offset_out, st.offset_in, + st.line_length, st.have_out, st.have_in, + st.have_remap, st.have_sem, n_ops); + } + if (allow_ops && transfer != 0 && st.line_length && st.have_out) { if (DMA_REMAP_ENABLE(dma) && st.have_remap && @@ -2039,10 +2053,23 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, pending = 1; } + /* + * The push's collected work always runs on real + * hardware; the release only orders observers. + * Execute it here, independently of whether the + * semaphore is translatable or still behind - + * gating the work on the semaphore drops the + * page-table writes that create those very + * translations (UVM's CE memsets into vidmem). + * Ops are only ever collected from pushes read + * through a trusted ring translation. + */ + if (allow_ops && n_ops) { + svc_run_ops(vgpu, ops, n_ops); + svc.ring_ops++; + } if (pending) { svc.pushes_pending++; - if (allow_ops) - svc_run_ops(vgpu, ops, n_ops); if (DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC) @@ -2173,7 +2200,7 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, * fabricated "pushes" executing memsets); work execution is limited * to pushes the pump read through a trusted ring translation. */ - svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page, false); + svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page, false, 0, 0); for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; @@ -3231,7 +3258,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_decode_stream(vgpu, pb, len, 0, - ch->engine_type != 0); + ch->engine_type != 0, + ch->h_object, + ch->engine_type); ch->submissions++; svc.gpfifo_entries_done++; /* Advance expected payload for this channel's tracking semaphore. */ @@ -3251,7 +3280,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) len)) { svc_decode_stream(vgpu, pb, len, 0, - ch->engine_type != 0); + ch->engine_type != 0, + ch->h_object, + ch->engine_type); ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) |