diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 44 |
1 files changed, 43 insertions, 1 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 663e1ba..a5f58db 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2925,6 +2925,8 @@ struct svc_push_state { u32 qmd_have; /* A/B seen bitmap */ u32 qmd_n; /* QMD words collected so far */ u64 qmd_bits; /* received-word bitmap */ + u32 qmd_hdr; /* push offset of the 0x318 run header */ + u32 qmd_end; /* push offset just past that run */ u64 push_va; /* push VA, for launch logging */ u32 qmd[SVC_QMD_WORDS]; }; @@ -3189,6 +3191,7 @@ static struct svc_fwd_cand svc_fwd_cand; #define SVC_GR_INLINE_BYTES 64 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 +#define SVC_GR_PUSH_WORDS 256 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -3216,6 +3219,10 @@ struct svc_gr_cand { u64 push_va; u32 push_len; u64 push_hash; + u32 push_n; /* raw tail words kept in raw[] */ + u32 push_qmd_hdr; /* raw[] offset of the 0x318 run header */ + u32 push_qmd_end; /* raw[] offset just past that run */ + u32 raw[SVC_GR_PUSH_WORDS]; u32 qmd[SVC_GR_QMD_WORDS]; u64 qmd_dev_va; u32 n_inline; @@ -3236,6 +3243,8 @@ static struct svc_gr_cand svc_gr_cand; static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, "the sema GR QMD word count must match the backend ABI"); +static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, + "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); static_assert(sizeof(struct svc_gr_inline) == 80 && @@ -3312,6 +3321,20 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.push_len = push_len; if (push && push_len) svc_gr_cand.push_hash = svc_push_hash(push, push_len); + if (push && push_len >= 4) { + u32 total = push_len / 4; + u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS; + u32 base = total - n; + u32 hdr = st->qmd_hdr / 4; + u32 end = st->qmd_end / 4; + + memcpy(svc_gr_cand.raw, push + base * 4, n * 4); + svc_gr_cand.push_n = n; + if (end > hdr && hdr >= base && end - base <= n) { + svc_gr_cand.push_qmd_hdr = hdr - base; + svc_gr_cand.push_qmd_end = end - base; + } + } memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd)); svc_gr_cand.qmd_dev_va = st->qmd_dev_va; n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX; @@ -4804,6 +4827,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; + st->qmd_hdr = 0; + st->qmd_end = 0; } } @@ -5007,9 +5032,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, (gsp_cuda_bridge_enabled() || gsp_phys_gr_forward_enabled()) && addr >= SVC_QMD_ADDR_A && - addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && (st.qmd_have || (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) { + if (!st.qmd_end) { + st.qmd_hdr = off; + st.qmd_end = off + 4 + count * 4; + } svc_qmd_accum(vgpu, ch, &st, addr, count, buf + off + 4, buf, len, trusted, replay); off += 4 + count * 4; @@ -8004,6 +8033,10 @@ struct svc_fwd_req { u32 gr_h_object; u64 gr_push_va; u32 gr_push_len; + u32 gr_push_n; + u32 gr_push_qmd_hdr; + u32 gr_push_qmd_end; + u32 gr_raw[SVC_GR_PUSH_WORDS]; u64 gr_qmd_dev_va; /* * The QMD buffer page (the inline-QMD destination the CUDA driver @@ -8926,6 +8959,10 @@ static void svc_fwd_gr_work(void) gs.sem_payload = 0; gs.push_va = svc_fwd_req.gr_push_va; gs.push_len = svc_fwd_req.gr_push_len; + gs.raw = svc_fwd_req.gr_raw; + gs.raw_words = svc_fwd_req.gr_push_n; + gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; + gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end; gs.h_object = svc_fwd_req.gr_h_object; hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); @@ -9267,6 +9304,11 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_h_object = ch->h_object; svc_fwd_req.gr_push_va = svc_gr_cand.push_va; svc_fwd_req.gr_push_len = svc_gr_cand.push_len; + svc_fwd_req.gr_push_n = svc_gr_cand.push_n; + svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr; + svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end; + memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw, + sizeof(svc_fwd_req.gr_raw)); svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; /* * Resolve the QMD buffer page now, on the service thread. A miss |