diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 125 |
1 files changed, 102 insertions, 23 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b367dcf..1add702 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -197,6 +197,21 @@ struct svc_channel { * vadd input). Releases still re-apply; only * op execution is once per entry identity. * 0 = no entry executed from this slot yet. */ + u64 slot_ops_done[SVC_GPFIFO_MAX_BITS]; + /* Bitmap (operations 0..63) of decoded ops + * whose work already ran for the identity in + * slot_ops_ident. An entry held because one + * operand's targeted scan was still running + * is decoded again on retry; without this, + * the successful ops re-run against a + * since-refilled staging buffer (observed: + * the second vadd input half written into + * the first via a retried HtoD copy). */ + u8 slot_ops_complete[SVC_GPFIFO_MAX_BITS]; + /* The pass that recorded the identity ran to + * the end with no operand left pending, so a + * replay suppresses every op. When clear, the + * done bitmap decides per op (retry path). */ u32 put_prev; /* put value at the previous pass (wrap detect) */ u64 last_rel_va; /* last tracking-sem VA seen in a readable push */ u32 logged_expected; /* last pool value logged (rate limit) */ @@ -445,6 +460,7 @@ struct sema_svc { unsigned long op_trace_until; u32 op_trace_log; u64 ring_ops; /* op runs from ring-decoded pushes */ + u64 ops_once_skipped; /* ops skipped by the done bitmap on a retried entry */ u64 lda_log; /* LAUNCH_DMA inventory budget */ u64 rescue_rej_log; /* rescue rejection budget */ u64 sem256_log; /* 256-byte op trace budget */ @@ -2707,9 +2723,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, - svc.releases_faked, svc.releases_recovered); + svc.releases_faked, svc.releases_recovered, + svc.ops_once_skipped); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback); @@ -3653,9 +3670,11 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return 0; } -/* Execute a decoded operation list (fills and copies). */ +/* Execute a decoded operation list (fills and copies). `ops_done` (may be + * NULL) is the per-entry bitmap of ops that already ran for the entry's + * identity; `ops_base` is the sequence number of ops[0] within the entry. */ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, - u32 n_ops, u32 ch_h) + u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base) { u32 i; static u64 bigop_log; @@ -3663,8 +3682,21 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, svc.cur_ch = ch_h; for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; + u32 seq = ops_base + i; int rc; + /* + * Once-only work per entry identity: this entry was held + * (e.g. an operand's targeted scan was still in progress) + * and is being decoded again. An op whose bit is set already + * ran; re-running a CE copy would read a staging buffer the + * client has since refilled and overwrite finished data. + */ + if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { + svc.ops_once_skipped++; + continue; + } + if (m->type == 1) rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, @@ -3672,6 +3704,9 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture); + if ((rc == 0 || rc == SVC_COPY_FORWARDED) && + ops_done && seq < 64) + *ops_done |= 1ULL << seq; /* * Large-copy proof (unconditional, bounded): the multi-MB @@ -4506,7 +4541,7 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, - u64 push_va, bool trusted) + u64 push_va, bool trusted, u64 *ops_done) { struct svc_push_state st; /* @@ -4517,6 +4552,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, static struct svc_memset_op ops[SVC_MAX_MEMSETS]; u32 n_ops = 0; u32 ops_runs = 0; + u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; /* @@ -4800,7 +4836,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* The push's work always runs; see the * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { - svc_run_ops(vgpu, ops, n_ops, ch->h_object); + svc_run_ops(vgpu, ops, n_ops, ch->h_object, + ops_done, ops_seq - n_ops); svc.ring_ops++; ops_runs++; } @@ -4924,7 +4961,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * cannot run swept-RAM false positives. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { - svc_run_ops(vgpu, ops, n_ops, ch->h_object); + svc_run_ops(vgpu, ops, n_ops, ch->h_object, + ops_done, ops_seq - n_ops); svc.ring_ops++; ops_runs++; } @@ -5164,6 +5202,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; + ops_seq++; } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { @@ -5181,6 +5220,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; n_ops++; + ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { /* * A launch that looks like a transfer but @@ -5302,7 +5342,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, allow_ops, n_ops, gsp_sema_skip_ops); } - svc_run_ops(vgpu, ops, n_ops, ch->h_object); + svc_run_ops(vgpu, ops, n_ops, ch->h_object, + ops_done, ops_seq - n_ops); svc.ring_ops++; ops_runs++; } @@ -9096,6 +9137,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 va, idx = ch->last_get; u32 len, e0, e1, g; u64 entry_ident = 0; + u64 entry_done = 0; bool entry_replay = false; /* @@ -9175,11 +9217,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool replay_entry = ident && + bool same = ident && ch->slot_ops_ident[idx] == ident; entry_ident = ident; - entry_replay = replay_entry; + /* + * A complete prior pass suppresses every op. + * A partial pass (an operand's targeted scan + * was still running when the entry was held) + * leaves the per-op done bitmap to decide: + * work that already ran must not run again, + * the pending op must get its retry. + */ + entry_replay = same && + ch->slot_ops_complete[idx]; + if (same) + entry_done = ch->slot_ops_done[idx]; } if (gsp_sema_dbg && svc.gpfifo_entry_log < 160) { @@ -9359,7 +9412,33 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch, idx, va, - injected); + injected, + entry_replay ? + NULL : + &entry_done); + /* + * Record what ran for this entry + * identity before any hold path can + * break out: a held entry is decoded + * again on retry, and the successful + * ops (CE copies of a staging buffer + * the client may have refilled) must + * not run twice. The complete flag + * tells the retry whether anything + * is still owed; when an operand's + * targeted scan is pending it is, + * and the done bitmap covers the + * rest. + */ + if (entry_ident) { + ch->slot_ops_ident[idx] = + entry_ident; + ch->slot_ops_done[idx] = + entry_done; + ch->slot_ops_complete[idx] = + svc.op_tr_pending ? + 0 : 1; + } /* * An operand of this push * failed to translate but a @@ -9400,15 +9479,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) break; } ch->op_retry = 0; - /* - * Ops (if any) have now run for - * this entry identity; remember it - * so a rewind replay cannot run - * them a second time. - */ - if (entry_ident) - ch->slot_ops_ident[idx] = - entry_ident; rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -9547,7 +9617,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !svc_read_va_push(vgpu, ch, va, pb, len)) svc_decode_stream(vgpu, pb, len, 0, false, - ch, idx, va, true); + ch, idx, va, true, + NULL); rel_seen = (svc_rel_count != rel_n2); /* @@ -9572,7 +9643,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !svc_read_va_push(vgpu, ch, va, pb, len)) svc_decode_stream(vgpu, pb, len, 0, false, - ch, idx, va, true); + ch, idx, va, true, + NULL); rel_seen = (svc_rel_count != rel_n2); } @@ -9800,7 +9872,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch, SVC_ENTRY_NONE, va, - false); + false, + NULL); if (svc_rel_count != rel_n_before) ch->last_rel_va = svc_rel_va_seen; @@ -10100,6 +10173,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(svc.chan[k].slot_ops_ident, ch->slot_ops_ident, sizeof(ch->slot_ops_ident)); + memcpy(svc.chan[k].slot_ops_done, + ch->slot_ops_done, + sizeof(ch->slot_ops_done)); + memcpy(svc.chan[k].slot_ops_complete, + ch->slot_ops_complete, + sizeof(ch->slot_ops_complete)); /* * Persist the counter slot autoreg resolved on the * snapshot: without this the next pass re-derives |