diff options
| -rw-r--r-- | kernel/gsp_cuda_bridge.h | 2 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.c | 82 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 2 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 227 |
4 files changed, 279 insertions, 34 deletions
diff --git a/kernel/gsp_cuda_bridge.h b/kernel/gsp_cuda_bridge.h index 2fb63d7..243392e 100644 --- a/kernel/gsp_cuda_bridge.h +++ b/kernel/gsp_cuda_bridge.h @@ -15,7 +15,7 @@ struct nv_vgpu_device; * targeted resolve). */ #define GSP_BRIDGE_INLINE_MAX 16 -#define GSP_BRIDGE_INLINE_BYTES 64 +#define GSP_BRIDGE_INLINE_BYTES 256 struct gsp_bridge_inline { u64 dst_va; diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 6228d28..8c9a2eb 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -20223,6 +20223,25 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, (inl->dst_va & (GSP_PHYS_PAGE_SIZE - 1)), inl->data, len); + /* + * SESSION 153-CONTINUED-11: bounded witness of + * every inline record applied (which program + * page received which image), so the SPH + * delivery is auditable from the log. + */ + { + static u32 inl_apply_log; + + if (inl_apply_log < 24) { + inl_apply_log++; + pr_info("vgsp-gr: INLINE-APPLY dst=0x%llx len=%u pg_va=0x%llx guest=%u first=%08x %08x\n", + inl->dst_va, len, pg_va, + stream3d && pg->hpa ? + 1u : 0u, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u); + } + } copied++; } /* @@ -20654,6 +20673,34 @@ gr_page_unmapped: win3d_n = req->stream_n; pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u)\n", nrel, win3d_n, n_shadow); + /* + * SESSION 153-CONTINUED-11: bounded raw window dump - the + * exact C597 method stream the engine walks (program binds + * 0x2014/0x2018+j*0x40, SET_PIPELINE_SHADER 0x2000+j*0x40, + * draws 0x1618/0xd74). Two windows per boot. + */ + { + static u32 gr3d_win_log; + u32 wi; + + if (gr3d_win_log < 4) { + gr3d_win_log++; + pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, win3d_n); + for (wi = 0; wi < win3d_n && wi < 384; + wi += 16) + pr_info("vgsp-gr: 3D win[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + win3d[wi + 0], win3d[wi + 1], + win3d[wi + 2], win3d[wi + 3], + win3d[wi + 4], win3d[wi + 5], + win3d[wi + 6], win3d[wi + 7], + win3d[wi + 8], win3d[wi + 9], + win3d[wi + 10], win3d[wi + 11], + win3d[wi + 12], win3d[wi + 13], + win3d[wi + 14], win3d[wi + 15]); + } + } } words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, @@ -24716,6 +24763,36 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) 0x00404194u); sked_hww_esr = gsp_phys_reg_probe_line(m, bar0, "SKED_HWW_ESR", 0x00407020u); + /* + * SESSION 153-CONTINUED-11: the shader-program-header fault window. + * DS_HWW_ESR bit n = "the SPH with index n failed validation" (hwref: + * NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR, ESR_2 for 24..30); + * the SPH_INFO registers snapshot the START address of each stage's + * faulting SPH, so a post-fault read names the exact VA the engine + * fetched. + */ + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR", + 0x00405840u); + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR_2", + 0x00405848u); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A", + 0x00405c1cu); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_B", + 0x00405c20u); + { + u32 si; + + for (si = 0; si < 6u; si++) { + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START", + 0x00405c00u + si * 4u); + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START_UPPER", + 0x00405c24u + si * 4u); + } + } sked_activity = gsp_phys_reg_probe_line(m, bar0, "SKED_ACTIVITY", 0x00407054u); sked_cwd0 = gsp_phys_reg_probe_line(m, bar0, @@ -25428,8 +25505,9 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + static_assert(sizeof(struct gsp_phys_gr_page) == 24, "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes"); static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && - sizeof(struct gsp_phys_gr_inline) == 80, - "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); + sizeof(struct gsp_phys_gr_inline) >= + 12 + GSP_PHYS_GR_INLINE_BYTES, + "the GR forwarding inline record must stay {u64,u32,u8[N]}"); static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index dfdf1d9..b6c3930 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -354,7 +354,7 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, */ #define GSP_PHYS_GR_MAX_PAGES 16u #define GSP_PHYS_GR_MAX_INLINE 8u -#define GSP_PHYS_GR_INLINE_BYTES 64u +#define GSP_PHYS_GR_INLINE_BYTES 256u #define GSP_PHYS_GR_QMD_WORDS 64u /* * IQ2M sentinel: written to the inline-QMD destination's word 63 after the diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 9d7f8bd..ce008d6 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -6568,7 +6568,7 @@ static struct svc_fwd_cand svc_fwd_cand; */ #define SVC_GR_QMD_WORDS 64 #define SVC_GR_INLINE_MAX 8 -#define SVC_GR_INLINE_BYTES 64 +#define SVC_GR_INLINE_BYTES 256 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* @@ -6821,8 +6821,8 @@ static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); -static_assert(sizeof(struct svc_gr_inline) == 80 && - offsetof(struct svc_gr_inline, data) == 12, +static_assert(offsetof(struct svc_gr_inline, data) == 12 && + sizeof(struct svc_gr_inline) >= 12 + SVC_GR_INLINE_BYTES, "the sema GR inline record must stay layout-compatible with the backend"); static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && offsetof(struct svc_gr_inline, dst_va) == @@ -7071,6 +7071,7 @@ struct svc_gl_up { u8 data[SVC_GR_INLINE_BYTES]; }; +#define SVC_GL_PROG_MAX 8 struct svc_gl_sec { bool seen; u32 subch; @@ -7080,6 +7081,18 @@ struct svc_gl_sec { * host-sem/2D ops whose VAs are not mapped in the * replay channel and FAULT the method walk) */ u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ + /* + * SESSION 153-CONTINUED-11: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the window binds (the engine validates the SPH at EACH bound + * program VA when a draw issues - r944/r946 faults on the draw's + * program pages), with the pipeline slot's shader type collected + * from SET_PIPELINE_SHADER(j) (0x2000+j*0x40, TYPE bits 7:4; 0xff = + * unknown). The per-VA inline records author the program bytes the + * driver's upload never delivered. + */ + u64 progs[SVC_GL_PROG_MAX]; + u8 prog_slot[SVC_GL_PROG_MAX]; + u32 n_progs; u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ u64 rel_va; /* last VALID release decoded on the channel */ @@ -7112,6 +7125,17 @@ struct svc_gl_sec { }; static struct svc_gl_sec svc_gl; /* service thread only */ /* + * SESSION 153-CONTINUED-11: the channel's sticky per-slot shader types + * (SET_PIPELINE_SHADER(j) TYPE field). Sticky like the C597 bind: the init + * push binds the programs once; later draw pushes reuse the state. 0xff = + * never seen. File-static so a section reset cannot wipe it. + */ +static u8 svc_gl_shader_type[32] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +/* * SESSION 153-CONTINUED-10 (task 1): the C597 class ATTACHMENT. * * A SET_OBJECT bind is sticky on real hardware: the class stays bound to @@ -7264,39 +7288,74 @@ static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ static char *sema_gl_prog_hex; module_param_named(sema_gl_prog_hex, sema_gl_prog_hex, charp, 0644); MODULE_PARM_DESC(sema_gl_prog_hex, - "explicit program bytes (hex) for the synthesized GL launch (<=64 B)"); + "explicit program bytes (hex) for the synthesized GL launch (<=256 B)"); +/* + * SESSION 153-CONTINUED-11: per-stage program images. A Turing graphics + * program is [SPH V4 (128 B)][SASS]; the SPH tells the engine the stage + * (VTG vs PS) and the I/O mapping. The engine validates the SPH of every + * bound pipeline program at draw time, so every program VA the window + * binds needs bytes. sema_gl_vs_hex feeds VTG-typed slots, sema_gl_ps_hex + * pixel-typed slots, and sema_gl_prog_hex is the generic fallback (also + * used for the single-record compute path). All three re-parse whenever + * their string changes, so a running module can be iterated against the + * "Shader Program Header N Error" fault oracle without a reload. + */ +static char *sema_gl_vs_hex; +module_param_named(sema_gl_vs_hex, sema_gl_vs_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_vs_hex, + "explicit VTG program image (SPH+SASS hex, <=256 B) for vertex slots"); +static char *sema_gl_ps_hex; +module_param_named(sema_gl_ps_hex, sema_gl_ps_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_ps_hex, + "explicit PS program image (SPH+SASS hex, <=256 B) for pixel slots"); static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES]; static u32 svc_gl_prog_len; -static bool svc_gl_prog_parsed; +static u8 svc_gl_vs[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_vs_len; +static u8 svc_gl_ps[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_ps_len; +static char svc_gl_prog_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_vs_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_ps_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; -static void svc_gl_prog_apply_param(void) +static void svc_gl_hex_parse(const char *h, const char *tag, u8 *dst, + u32 *dst_len, char *seen) { - const char *h = sema_gl_prog_hex; u32 n, i; - if (svc_gl_prog_parsed) - return; - svc_gl_prog_parsed = true; + if (!strcmp(seen, h ? h : "")) + return; /* unchanged since the last parse */ + strscpy(seen, h ? h : "", GSP_BRIDGE_INLINE_BYTES * 2 + 1); + *dst_len = 0; if (!h || !h[0]) return; n = (u32)strlen(h); if (n % 2 || n / 2 > GSP_BRIDGE_INLINE_BYTES) { - pr_warn("sema-gl: prog-hex param ignored (bad length %u)\n", - n); + pr_warn("sema-gl: %s param ignored (bad length %u)\n", + tag, n); return; } for (i = 0; i < n; i += 2) { unsigned int b; if (sscanf(h + i, "%2x", &b) != 1) { - pr_warn("sema-gl: prog-hex param bad at %u\n", i); + pr_warn("sema-gl: %s param bad at %u\n", tag, i); return; } - svc_gl_prog[i / 2] = (u8)b; + dst[i / 2] = (u8)b; } - svc_gl_prog_len = n / 2; - pr_info("sema-gl: prog-hex param: %u bytes stored\n", - svc_gl_prog_len); + *dst_len = n / 2; + pr_info("sema-gl: %s param: %u bytes stored\n", tag, *dst_len); +} + +static void svc_gl_prog_apply_param(void) +{ + svc_gl_hex_parse(sema_gl_prog_hex, "prog-hex", svc_gl_prog, + &svc_gl_prog_len, svc_gl_prog_seen); + svc_gl_hex_parse(sema_gl_vs_hex, "vs-hex", svc_gl_vs, + &svc_gl_vs_len, svc_gl_vs_seen); + svc_gl_hex_parse(sema_gl_ps_hex, "ps-hex", svc_gl_ps, + &svc_gl_ps_len, svc_gl_ps_seen); } /* @@ -7436,13 +7495,46 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) break; memcpy(&v[k], data + 4 * k, 4); } + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) = 0x2000+j*0x40, + * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA + * can be typed (VTG vs PS) for the program-bytes delivery. + */ + if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32 && count >= 1) + svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); + return; + } /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. + * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine + * validates each bound program's SPH at draw time) and the program + * page joins the candidate's page set so the per-VA inline records + * can land there. */ if (addr >= 0x2014 && addr < 0x4000 && (addr & 0x3f) == 0x14 && - count >= 2 && !svc_gl.prog_va) { - svc_gl.prog_va = ((u64)(v[0] & 0xff) << 32) | - (u64)(v[1] & 0xfffffffc); + count >= 2) { + u64 pva = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + u32 j = (addr - 0x2014) / 0x40, k; + + if (!svc_gl.prog_va) + svc_gl.prog_va = pva; + if (pva && j < 32) { + bool dup = false; + + for (k = 0; k < svc_gl.n_progs; k++) + if (svc_gl.progs[k] == pva) + dup = true; + if (!dup && svc_gl.n_progs < SVC_GL_PROG_MAX) { + svc_gl.progs[svc_gl.n_progs] = pva; + svc_gl.prog_slot[svc_gl.n_progs] = (u8)j; + svc_gl.n_progs++; + } + svc_gl_add_ptr(pva); + } return; } switch (addr) { @@ -7612,7 +7704,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, u32 at_off) { struct svc_push_state s2; - struct gsp_bridge_inline pr; + struct gsp_bridge_inline prs[SVC_GL_PROG_MAX]; u32 n_recs = 0, k; /* * SESSION 149 R16 (M3): the inline attachment's source. Priority: @@ -7871,12 +7963,69 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * uploads/binds, so the synthesized compute attachments (the inline * program record, and with it the prog-hex fallback) do not apply. * Draw-synth/hold captures keep the old compute path unchanged. - */ - if (blob_n && op_src[0] != 's') { - memset(&pr, 0, sizeof(pr)); - pr.dst_va = svc_gl.prog_va; - pr.len = blob_n; - memcpy(pr.data, blob, blob_n); + * + * SESSION 153-CONTINUED-11: exception - the GL program-bytes + * delivery. The engine validates the SPH of EVERY pipeline program + * the channel state binds when a draw issues, and the driver's + * upload of those bytes never ran (the program pages read empty). + * For a window that binds program slots (the 0x2014/0x2018 pairs + * collected above) attach one inline record per program VA with the + * stage-typed image (vs-hex for VTG slots, ps-hex for pixel slots, + * prog-hex fallback) - this authors the missing upload. Windows + * without program pairs keep the historical single-record path. + */ + if (svc_gl.n_progs) { + u32 pi; + + for (pi = 0; pi < svc_gl.n_progs && n_recs < GSP_BRIDGE_INLINE_MAX; + pi++) { + const u8 *b = NULL; + u32 bl = 0; + u8 t = svc_gl_shader_type[svc_gl.prog_slot[pi] < 32 ? + svc_gl.prog_slot[pi] : 0]; + + if (t == 5 && svc_gl_ps_len) { + b = svc_gl_ps; + bl = svc_gl_ps_len; + } else if (t != 5 && t != 0xff && svc_gl_vs_len) { + b = svc_gl_vs; + bl = svc_gl_vs_len; + } else if (svc_gl_prog_len) { + b = svc_gl_prog; + bl = svc_gl_prog_len; + } else if (blob_n) { + b = blob; + bl = blob_n; + } + if (!b || !bl) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = svc_gl.progs[pi]; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + n_recs++; + } + if (n_recs && gsp_sema_dbg && svc_gl_prog_log < 64) { + u32 pi2; + + svc_gl_prog_log++; + pr_info("sema-gl: prog-images h=0x%x push=0x%llx recs=%u n_progs=%u vs=%u ps=%u gen=%u\n", + ch->h_object, push_va, n_recs, + svc_gl.n_progs, + svc_gl_vs_len, svc_gl_ps_len, svc_gl_prog_len); + for (pi2 = 0; pi2 < svc_gl.n_progs && pi2 < 8; + pi2++) + pr_info("sema-gl: prog-slot[%u] va=0x%llx slot=%u type=%02x\n", + pi2, svc_gl.progs[pi2], + svc_gl.prog_slot[pi2], + svc_gl_shader_type[svc_gl.prog_slot[pi2] < 32 ? + svc_gl.prog_slot[pi2] : 0]); + } + } else if (blob_n && op_src[0] != 's') { + memset(&prs[0], 0, sizeof(prs[0])); + prs[0].dst_va = svc_gl.prog_va; + prs[0].len = blob_n; + memcpy(prs[0].data, blob, blob_n); n_recs = 1; } /* @@ -7885,11 +8034,11 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * replays of the same (push_va, content hash). */ svc_gl_cap_store(push_va, svc_push_hash(buf, len)); - pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s\n", + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s nprog=%u recs=%u\n", ch->h_object, push_va, len, at_off, svc_gl.prog_va, svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, - replay ? 1u : 0u, op_src); + replay ? 1u : 0u, op_src, svc_gl.n_progs, n_recs); /* * SESSION 153: remember real operands for this channel so a later * draw-only section on the same channel can still capture. A @@ -7915,7 +8064,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, get_unaligned_le32(blob + 8), get_unaligned_le32(blob + 12)); } - svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len, + svc_gr_cand_capture(ch, &s2, n_recs ? prs : NULL, n_recs, buf, len, push_va); /* * r929 (Path A): a stream-derived capture (op_src=stream: the section @@ -12852,6 +13001,24 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) as an IMMD + * (op 4) method - the GL driver emits it as a 1U method, so + * the TYPE field rides in the header's immediate (bits 27:16, + * value = [ENABLE(0) | TYPE(7:4)]). Sticky per slot, like the + * run-shaped case in svc_gl_field. + */ + if (gsp_sema_gl_xlate && op == 4 && addr >= 0x2000 && + addr < 0x2800 && (addr & 0x3f) == 0 && + ((svc_gl.seen && subch == svc_gl.subch) || + (svc_gl_att_h == ch->h_object && + subch == svc_gl_att_subch))) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32) + svc_gl_shader_type[j] = + (u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf); + } if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ |
