diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 161 |
1 files changed, 134 insertions, 27 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 3cc9936..13d99cc 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -6637,6 +6637,9 @@ static_assert(sizeof(struct svc_gr_page) == 24, static u64 svc_push_hash(const u8 *p, u32 len); /* SESSION 149 R15: exactly-once guard for the hash-verified replay capture. */ static bool svc_gr_cand_push_seen(u64 push_va, u64 hash); +/* SESSION 149 R16 (M3): trusted-walk VA read, defined below the decode. */ +static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); /* QMD field decode: the exact shape the host CUDA bridge uses. */ static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c) @@ -6826,6 +6829,7 @@ struct svc_gl_sec { }; static struct svc_gl_sec svc_gl; /* service thread only */ static u32 svc_gl_rep_log; /* bounded replay-already-captured witness */ +static u32 svc_gl_prog_log; /* R16 bounded program-read witness */ static void svc_gl_add_ptr(u64 va) { @@ -6896,14 +6900,26 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) * data words ARE the observed upload payload. Keep the * FIRST run's bytes for the M2-S1 inline attachment. * Observed: a004006c -> [00000041 00000670 00000690 - * 000006b0]. + * 000006b0] = LAUNCH_DMA cfg + 3 LID words = 12 B. + * SESSION 149 R16 (M3): keep only the LID payload (the + * cfg word is not code) and bound the copy by the run's + * OWN count - dlen reaches the end of the push, so the + * R15 copy over-read the run (blob=64 = 12 real bytes + + * 52 B of the following method words). These bytes are + * only the fallback: svc_gl_flush reads the real + * program bytes from prog_va via the trusted walk. */ - if (!svc_gl.blob_len && dlen) { - u32 n = dlen < sizeof(svc_gl.blob) ? - dlen : (u32)sizeof(svc_gl.blob); - - memcpy(svc_gl.blob, data, n); - svc_gl.blob_len = n; + if (!svc_gl.blob_len && count > 1 && dlen > 4) { + u32 n = (count - 1) * 4; + + if (n > dlen - 4) + n = dlen - 4; + if (n > sizeof(svc_gl.blob)) + n = (u32)sizeof(svc_gl.blob); + if (n) { + memcpy(svc_gl.blob, data + 4, n); + svc_gl.blob_len = n; + } } return; default: @@ -6920,7 +6936,8 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) * (synthesized - no QMD buffer exists in guest memory) so the backend uses * its qmdwin/scratch path. */ -static void svc_gl_flush(struct svc_channel *ch, +static void svc_gl_flush(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, const struct svc_push_state *st, const u8 *buf, u32 len, u64 push_va, bool trusted, bool replay, u32 at_off) @@ -6928,6 +6945,16 @@ static void svc_gl_flush(struct svc_channel *ch, struct svc_push_state s2; struct gsp_bridge_inline pr; u32 n_recs = 0, k; + /* + * SESSION 149 R16 (M3): the inline attachment's source. Priority: + * the REAL program bytes read from prog_va through the trusted walk + * (the GL program page), else the observed LID payload bounded to its + * own run. r888's blob=64 over-read the 12 B LID run. + */ + u8 blob[SVC_GR_INLINE_BYTES]; + u32 blob_n = 0; + int blob_rc = -1; + bool blob_trusted = false; if (!svc_gl.seen || !svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va || !trusted) @@ -6965,6 +6992,48 @@ static void svc_gl_flush(struct svc_channel *ch, * sema service authors the guest word (set it once rel_hpa resolving * is wanted). qmd[24] b26 (REDUCTION) likewise stays clear. */ + /* + * SESSION 149 R16 (M3): the scheduling fields the r888 engine + * refused to schedule without. Word/field offsets from the public + * QMDV02_03 class header (nvidia/inc/class/clc5c0qmd.h; the booter's + * own 48/49 program pair matches its QMDV02_03 offsets, so the + * rest of the layout is the same class): + * w12 CTA_RASTER_WIDTH MW(415:384) + * w13 CTA_RASTER_HEIGHT MW(431:416) + * w14 CTA_RASTER_DEPTH MW(463:448) + * w18 QMD_VERSION MW(579:576), QMD_MAJOR_VERSION MW(583:580), + * CTA_THREAD_DIMENSION0 MW(607:592) -> w18 hi word + * w19 CTA_THREAD_DIMENSION1 MW(623:608), + * CTA_THREAD_DIMENSION2 MW(639:624) + * w20 REGISTER_COUNT_V MW(656:648) <- SKEDCHECK06 field + * w29 BARRIER_COUNT MW(959:955) + * w31 SASS_VERSION MW(1023:1016) + * SKEDCHECK16 fired on CTA_THREAD_DIMENSION0==0 and SKEDCHECK06 on + * REGISTER_COUNT_V==0. The register count is the stream's own value: + * REGISTER_COUNT(1) = 0x1a = 26 (method 0x200c + 1*0x40 = 0x204c, + * words [0x1a, 0]) - the same number the driver's C5C0 path writes + * ("a NULL QMD will result in SKEDCHECK06_REGISTER_COUNT", cui + * turing_launch.c:781). CTA dims: the 0xC597 stream carries no grid / + * CTA operands we can locate (the 3D-class runs are format/viewport/ + * shader config; the C597 SHADER_CONFIG "CTA raster" words are not in + * this push), so a minimal 1x1x1 grid of 1x1x1 CTAs is used and + * documented as the choice: the checks only reject ZERO dims and a + * zero total-thread count, both of which a 1x1x1 launch satisfies. + * SHARED_MEMORY_SIZE (w17) stays 0 = "no shared memory" (valid and + * NOT among the failed checks); the m72 local-memory prelude already + * covers SKEDCHECK05. The driver's cache-control block (w5 top bits) + * is set as every captured guest QMD carries it; the backend turns it + * into its icache-invalidate bit for the shadow line. + */ + s2.qmd[12] = 1; /* CTA raster width (CTAs) */ + s2.qmd[13] = 1; /* CTA raster height (CTAs) */ + s2.qmd[14] = 1; /* CTA raster depth (CTAs) */ + s2.qmd[18] = 0x23u | (1u << 16); /* QMD 2.3, CTA dim0 = 1 */ + s2.qmd[19] = 1u | (1u << 16); /* CTA dim1 = 1, dim2 = 1 */ + s2.qmd[20] = 0x1au << 8; /* REGISTER_COUNT_V = 26 */ + s2.qmd[29] |= 1u << 27; /* BARRIER_COUNT = 1 */ + s2.qmd[31] |= 0x75u << 24; /* SASS_VERSION 7.5 (sm_75) */ + s2.qmd[5] |= 0x3c000000u; /* driver cache-control block */ for (k = 0; k < svc_gl.n_ptr && k < SVC_GR_POINTER_MAX - 1; k++) { s2.qmd[34 + 2 * k] = (u32)svc_gl.ptr[k]; s2.qmd[35 + 2 * k] = ((u32)(svc_gl.ptr[k] >> 32)) & 0x1ffff; @@ -6974,21 +7043,42 @@ static void svc_gl_flush(struct svc_channel *ch, s2.qmd_end = 0; s2.qmd_have = 0; /* - * SESSION 149 R15 (M2-S1): attach the observed LAUNCH_DMA / - * LOAD_INLINE_DATA payload as an inline record at the synthesized - * program VA. The GL program page (0x121201b00) is NOT among the - * section's own upload destinations, so without this the backend's - * w48 code-source screen sees an empty page, withdraws to the guest - * VA and the launch dies at FAULT_PDE; with it the proven win0 - * shadow copy + w48/49 rewrite line runs (the short blob is - * expected to fault in the SM later - that is the next wall, not - * this one). - */ - if (svc_gl.blob_len) { + * SESSION 149 R15 (M2-S1) + R16 (M3): attach program-page content as + * an inline record at the synthesized program VA. The GL program page + * (0x121201b00) is NOT among the section's own upload destinations, + * so without this the backend's w48 code-source screen sees an empty + * page, withdraws to the guest VA and the launch dies at FAULT_PDE; + * with it the proven win0 shadow copy + w48/49 rewrite line runs. + * + * R16 source order: the REAL program bytes read from prog_va through + * the trusted walk (the read machinery translates GL VAs now; src=4 + * answered this channel's ring/pushes in r888), else the observed + * LID payload bounded to its own run (12/24/28 B - "materialize + * exactly that"). A trusted read that lands on an all-zero page is + * treated as "not real" and falls through to the LID bytes. + */ + memset(blob, 0, sizeof(blob)); + blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, + sizeof(blob)); + if (!blob_rc) { + u32 z = 0; + + for (k = 0; k < 16; k++) + z |= blob[k]; + if (z) + blob_trusted = true; + } + if (blob_trusted) { + blob_n = sizeof(blob); + } else if (svc_gl.blob_len) { + blob_n = svc_gl.blob_len; + memcpy(blob, svc_gl.blob, blob_n); + } + if (blob_n) { memset(&pr, 0, sizeof(pr)); pr.dst_va = svc_gl.prog_va; - pr.len = svc_gl.blob_len; - memcpy(pr.data, svc_gl.blob, pr.len); + pr.len = blob_n; + memcpy(pr.data, blob, blob_n); n_recs = 1; } pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u\n", @@ -6996,6 +7086,22 @@ static void svc_gl_flush(struct svc_channel *ch, svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, replay ? 1u : 0u); + /* R16 bounded witness: what the synthesized QMD now schedules and + * where the code bytes came from. */ + if (gsp_sema_dbg && svc_gl_prog_log < 8) { + svc_gl_prog_log++; + pr_info("sema-gl: M3 sched w12=%u w13=%u w14=%u w18=0x%08x w19=0x%08x w20=0x%08x w29=0x%08x w31=0x%08x w5=0x%08x\n", + s2.qmd[12], s2.qmd[13], s2.qmd[14], s2.qmd[18], + s2.qmd[19], s2.qmd[20], s2.qmd[29], s2.qmd[31], + s2.qmd[5]); + pr_info("sema-gl: M3 code src=%s read_rc=%d len=%u lid=%u first=%08x %08x %08x %08x\n", + blob_trusted ? "trusted-prog" : "lid-fallback", + blob_rc, blob_n, svc_gl.blob_len, + get_unaligned_le32(blob), + get_unaligned_le32(blob + 4), + get_unaligned_le32(blob + 8), + get_unaligned_le32(blob + 12)); + } svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len, push_va); out: @@ -11739,17 +11845,17 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 keep_pay = svc_gl.rel_payload; if (svc_gl.seen) - svc_gl_flush(ch, &st, buf, len, - push_va, trusted, replay, - off); + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); memset(&svc_gl, 0, sizeof(svc_gl)); svc_gl.seen = true; svc_gl.subch = subch; svc_gl.rel_va = keep_rel; svc_gl.rel_payload = keep_pay; } else if (svc_gl.seen) { - svc_gl_flush(ch, &st, buf, len, push_va, - trusted, replay, off); + svc_gl_flush(vgpu, ch, &st, buf, len, + push_va, trusted, replay, off); } } else if (gsp_sema_gl_xlate && svc_gl.seen && subch == svc_gl.subch) { @@ -13635,7 +13741,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * this is the normal capture point (one section per large GL push). */ if (gsp_sema_gl_xlate && svc_gl.seen) - svc_gl_flush(ch, &st, buf, len, push_va, trusted, replay, off); + svc_gl_flush(vgpu, ch, &st, buf, len, push_va, trusted, + replay, off); if (gsp_sema_dbg && ch->h_object && svc.entry_log < 16384) { |