From 896bd9f744cadfd8af2dd03c79ac31114fb37dba Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Tue, 6 Oct 2026 07:48:07 -0400 Subject: vgsp: verbatim replay of the captured GR launch prelude+window --- kernel/gsp_phys_booter.c | 69 ++++++++++++++++++++++++++++++++++++++--------- kernel/gsp_phys_booter.h | 12 +++++++++ kernel/gsp_sema_service.c | 44 +++++++++++++++++++++++++++++- 3 files changed, 112 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 2c0fdc0..badf14f 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -12765,6 +12765,12 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_GR_LOAD_INLINE_QMD_DATA 0x320u /* Header count: ADDRESS_A + ADDRESS_B + 64 QMD words. */ #define GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS 66u +/* Whole run: one method header + the inline-QMD data words. */ +#define GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS 67u +/* Dwords the per-channel compute init emits (see build_gr_init). */ +#define GSP_PHYS_RMCALL_GR_INIT_WORDS 152u +/* PCAS(4) + REPORT_SEM(8). */ +#define GSP_PHYS_RMCALL_GR_TAIL_WORDS 12u #define GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD GSP_PHYS_RMCALL_SEM_PAYLOAD #define GSP_PHYS_RMCALL_GR_SEM_PAYLOAD 0xc0ffee02u /* @@ -12874,21 +12880,56 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) * channel long before the launch), so the replay must not either. * Returns the block size in dwords. */ -static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const u32 *qmd, - u64 qmd_va, u64 sem_va, - u32 sem_payload) +static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, + const struct gsp_phys_gr_submit *req, + const u32 *qmd, u64 qmd_va, + u64 sem_va, u32 sem_payload) { u32 *p = pb; p = gsp_phys_rmcall_build_gr_init(p); - *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | - (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) | - ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) | - (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2); - *p++ = (u32)(qmd_va >> 40); - *p++ = (u32)((qmd_va >> 8) & 0xffffffffu); - memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4); - p += GSP_PHYS_GR_QMD_WORDS; + /* + * Verbatim replay: copy the captured GRCE prelude and the guest's + * own inline-QMD run, keeping the method headers, and relocate only + * the ADDRESS_A/B data words and the 64 QMD words exactly like the + * rebuilt window. Only a contiguous 0x318 run is replayed; anything + * else falls back to the rebuilt window. + */ + if (req && req->raw && + req->raw_words >= + (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS && + req->raw_qmd_end == + (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS && + GSP_PHYS_RMCALL_GR_INIT_WORDS + + (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS + + GSP_PHYS_RMCALL_GR_TAIL_WORDS <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + u32 pre = (u32)req->raw_qmd_hdr + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS; + + memcpy(p, req->raw, pre * 4); + p[req->raw_qmd_hdr + 1] = (u32)(qmd_va >> 40); + p[req->raw_qmd_hdr + 2] = + (u32)((qmd_va >> 8) & 0xffffffffu); + memcpy(p + req->raw_qmd_hdr + 3, qmd, + GSP_PHYS_GR_QMD_WORDS * 4); + p += pre; + pr_info("vgsp-gr: verbatim replay prelude+window words=%u qmd_hdr=%u\n", + pre, req->raw_qmd_hdr); + } else { + if (req && req->raw) + pr_info("vgsp-gr: raw prelude unusable n=%u hdr=%u end=%u; rebuilt window\n", + req->raw_words, req->raw_qmd_hdr, + req->raw_qmd_end); + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) | + ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) | + (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2); + *p++ = (u32)(qmd_va >> 40); + *p++ = (u32)((qmd_va >> 8) & 0xffffffffu); + memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4); + p += GSP_PHYS_GR_QMD_WORDS; + } p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SEND_PCAS_A, (u32)(qmd_va >> 8)); p = gsp_phys_rmcall_push_gr(p, @@ -13554,7 +13595,7 @@ gr_page_unmapped: qmd_dst, GSP_PHYS_GR_QMD_WORDS); } - words = gsp_phys_rmcall_build_gr_launch(methods, qmd, + words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, ch_sem_va, want_ch); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); @@ -16029,6 +16070,10 @@ static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); +static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS + + GSP_PHYS_RMCALL_GR_TAIL_WORDS <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS, + "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound"); static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= GSP_PHYS_RMCALL_COPY_SIZE, "the GR launch method block must fit one module pushbuffer page"); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 42bf1ff..9a0331f 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -336,6 +336,7 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, #define GSP_PHYS_GR_INLINE_BYTES 64u #define GSP_PHYS_GR_QMD_WORDS 64u #define GSP_PHYS_GR_QMD_WORD_NONE 0xffffffffu +#define GSP_PHYS_GR_RAW_WORDS 256u struct gsp_phys_gr_page { u64 guest_va; /* guest VA (page aligned) */ @@ -353,6 +354,17 @@ struct gsp_phys_gr_inline { struct gsp_phys_gr_submit { const u32 *qmd; /* 64 rewritten-QMD source words */ u32 qmd_words; /* must be GSP_PHYS_GR_QMD_WORDS */ + /* + * Verbatim replay: the captured tail of the guest push (up to + * GSP_PHYS_GR_RAW_WORDS dwords) and the dword offsets of its + * inline-QMD run inside that tail. A contiguous 0x318 run + * (header + A/B + 64 words) is replayed with the same data-field + * relocations; anything else leaves the rebuilt window in use. + */ + const u32 *raw; + u32 raw_words; + u32 raw_qmd_hdr; + u32 raw_qmd_end; u64 qmd_va; /* guest VA of the QMD buffer */ u64 qmd_hpa; /* guest QMD buffer page HPA, or 0 */ const struct gsp_phys_gr_page *pages; diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 663e1ba..a5f58db 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2925,6 +2925,8 @@ struct svc_push_state { u32 qmd_have; /* A/B seen bitmap */ u32 qmd_n; /* QMD words collected so far */ u64 qmd_bits; /* received-word bitmap */ + u32 qmd_hdr; /* push offset of the 0x318 run header */ + u32 qmd_end; /* push offset just past that run */ u64 push_va; /* push VA, for launch logging */ u32 qmd[SVC_QMD_WORDS]; }; @@ -3189,6 +3191,7 @@ static struct svc_fwd_cand svc_fwd_cand; #define SVC_GR_INLINE_BYTES 64 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 +#define SVC_GR_PUSH_WORDS 256 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -3216,6 +3219,10 @@ struct svc_gr_cand { u64 push_va; u32 push_len; u64 push_hash; + u32 push_n; /* raw tail words kept in raw[] */ + u32 push_qmd_hdr; /* raw[] offset of the 0x318 run header */ + u32 push_qmd_end; /* raw[] offset just past that run */ + u32 raw[SVC_GR_PUSH_WORDS]; u32 qmd[SVC_GR_QMD_WORDS]; u64 qmd_dev_va; u32 n_inline; @@ -3236,6 +3243,8 @@ static struct svc_gr_cand svc_gr_cand; static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, "the sema GR QMD word count must match the backend ABI"); +static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, + "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); static_assert(sizeof(struct svc_gr_inline) == 80 && @@ -3312,6 +3321,20 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.push_len = push_len; if (push && push_len) svc_gr_cand.push_hash = svc_push_hash(push, push_len); + if (push && push_len >= 4) { + u32 total = push_len / 4; + u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS; + u32 base = total - n; + u32 hdr = st->qmd_hdr / 4; + u32 end = st->qmd_end / 4; + + memcpy(svc_gr_cand.raw, push + base * 4, n * 4); + svc_gr_cand.push_n = n; + if (end > hdr && hdr >= base && end - base <= n) { + svc_gr_cand.push_qmd_hdr = hdr - base; + svc_gr_cand.push_qmd_end = end - base; + } + } memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd)); svc_gr_cand.qmd_dev_va = st->qmd_dev_va; n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX; @@ -4804,6 +4827,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; + st->qmd_hdr = 0; + st->qmd_end = 0; } } @@ -5007,9 +5032,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, (gsp_cuda_bridge_enabled() || gsp_phys_gr_forward_enabled()) && addr >= SVC_QMD_ADDR_A && - addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && (st.qmd_have || (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) { + if (!st.qmd_end) { + st.qmd_hdr = off; + st.qmd_end = off + 4 + count * 4; + } svc_qmd_accum(vgpu, ch, &st, addr, count, buf + off + 4, buf, len, trusted, replay); off += 4 + count * 4; @@ -8004,6 +8033,10 @@ struct svc_fwd_req { u32 gr_h_object; u64 gr_push_va; u32 gr_push_len; + u32 gr_push_n; + u32 gr_push_qmd_hdr; + u32 gr_push_qmd_end; + u32 gr_raw[SVC_GR_PUSH_WORDS]; u64 gr_qmd_dev_va; /* * The QMD buffer page (the inline-QMD destination the CUDA driver @@ -8926,6 +8959,10 @@ static void svc_fwd_gr_work(void) gs.sem_payload = 0; gs.push_va = svc_fwd_req.gr_push_va; gs.push_len = svc_fwd_req.gr_push_len; + gs.raw = svc_fwd_req.gr_raw; + gs.raw_words = svc_fwd_req.gr_push_n; + gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; + gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end; gs.h_object = svc_fwd_req.gr_h_object; hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); @@ -9267,6 +9304,11 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_h_object = ch->h_object; svc_fwd_req.gr_push_va = svc_gr_cand.push_va; svc_fwd_req.gr_push_len = svc_gr_cand.push_len; + svc_fwd_req.gr_push_n = svc_gr_cand.push_n; + svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr; + svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end; + memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw, + sizeof(svc_fwd_req.gr_raw)); svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; /* * Resolve the QMD buffer page now, on the service thread. A miss -- cgit v1.2.3