diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 210 |
1 files changed, 191 insertions, 19 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index d099a7e..ed1cb0f 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -1761,6 +1761,19 @@ static inline void gsp_phys_rmcall_wait_quantum(void) #define GSP_PHYS_RMCALL_CE_SUBCHANNEL 4u /* + * SESSION 153-CONTINUED-68: SEC_OP_ONE_INC = 5 (class/cl906f.h:181 + * NV906F_DMA_SEC_OP_ONE_INC, NVC06F_DMA_ONEINCR_* field layout). The + * client emits its MME IR uploads (method 0x114) as op=5 runs whose first + * data word is the IR instruction address and whose remaining words feed + * the IR data port (the r1099/r1116 captures: hdr 0xa0100045 = op 5, + * cnt 16, method 0x114, data [pc, code...]). op=1 (INC_METHOD) walks the + * METHOD address per data word instead (0x114 -> 0x118 -> 0x11c -> + * 0x120), so an IR upload emitted with op=1 writes one stray IR word and + * then clobbers the SA pointer/data registers. + */ +#define GSP_PHYS_RMCALL_METHOD_ONEINC 5u + +/* * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248, * LAUNCH_DMA 0x300, OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/ @@ -17324,12 +17337,38 @@ MODULE_PARM_DESC(sema_gl_mme_prelude, * (the r1116 0x238c poison). Default 1; A/B with 0. */ #define GSP_PHYS_RMCALL_GR_MME_STUB_PC 0xC0u +/* + * SESSION 153-CONTINUED-68: the stub body. The r1134/r1136 builds used a + * bare [0, 0, 1] (end_next with every other bit zero) and the engine still + * raised ILLEGAL_OPCODE with pc_valid=0 - a decode failure at the macro's + * FIRST instruction, i.e. the bare END is not an encoding the MME accepts. + * This body is a byte-exact copy of a real compiler-emitted instruction + * from the client's own captured program (macro(0) instruction 3, + * 20208c00 38c40046 038c4301): both ALUs write the discarded ZERO + * register (no side effects), no outputs are emitted, end_next=1 - a + * known-good macro terminator. + */ +static const u32 gsp_phys_gr_mme_stub_code[3] = { + 0x20208c00u, 0x38c40046u, 0x038c4301u +}; static bool gsp_phys_rmcall_gr_sa_stub = true; module_param_named(sema_gl_mme_sa_stub, gsp_phys_rmcall_gr_sa_stub, bool, 0644); MODULE_PARM_DESC(sema_gl_mme_sa_stub, "Load a harmless end-noop stub into every MME macro slot the window calls but the captured START-ADDRESS RAM table does not cover (default 1; the r1116 0x238c lane poison class)"); +/* + * SESSION 153-CONTINUED-68: stub EVERY called slot, covered or not + * (attribution: with all calls no-op'd the window either still faults + * (the fault is in non-call content) or faults differently (the fault is + * macro execution). Default 0; A/B knob. + */ +static bool gsp_phys_rmcall_gr_sa_stub_all; +module_param_named(sema_gl_mme_sa_stub_all, gsp_phys_rmcall_gr_sa_stub_all, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_stub_all, + "Stub every macro slot the window calls, including slots the captured SA table covers (attribution knob; default 0)"); + static inline bool gsp_sema_gr_sa_stub_value(void) { return READ_ONCE(gsp_phys_rmcall_gr_sa_stub); @@ -17345,8 +17384,16 @@ static inline bool gsp_sema_gr_sa_stub_value(void) * window replay ran its macros against an unloaded MME (mme_pc=513 class * stall, no retire). The recovery clears the latch so the next launch * re-preloads the context. + * + * SESSION 153-CONTINUED-68: the latch is keyed by (generation, channel). + * A bare generation latch is SHARED across channels: the X channel's + * first post-maturity launch consumed the emission for the whole system + * and every beef launch then shipped without the program (r1140: only + * h=0x1000013 logged "MME-PRELUDE emitted gen=3", every beef launch + * words=400 prelude-less). Re-emitting per channel is idempotent. */ static u32 gsp_phys_gr_mme_prelude_gen; +static u32 gsp_phys_gr_mme_prelude_h; /* * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091 @@ -17515,14 +17562,17 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, */ u32 sa_ent[64]; u64 sa_mask = 0; - u32 sa_stub[24]; - u32 sa_nstub = 0, sa_block = 0; + u32 sa_stub[40]; + u32 sa_stub_ent[40]; + u32 sa_cov[40]; + u32 sa_cov_ent[40]; + u32 sa_nstub = 0, sa_ncov = 0, sa_block = 0; if (gsp_sema_gr_sa_stub_value() && win3d && win3d_n) { u32 so = 0; sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); - while (so < win3d_n && sa_nstub < 24u) { + while (so < win3d_n && (sa_nstub < 40u || sa_ncov < 40u)) { u32 h = win3d[so]; u32 sop = (h >> 29) & 7u; u32 scnt = (h >> 16) & 0x1fffu; @@ -17533,7 +17583,17 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, scnt && so + 1u + scnt <= win3d_n) { sspan = 1u + scnt; slo = 0; - shi = scnt - 1u; + /* + * SESSION 153-CONTINUED-68: ONE_INC + * (op 5) writes only data[0] to the + * header method; the rest go to + * method+4 (the data port, see + * nv3dLoadConstantsHeader). Walking + * mthd+4k for op=5 invented ghost + * CALL_MME_MACRO slots (r1134/1135 + * list). + */ + shi = (sop == 5u) ? 0u : scnt - 1u; } else if (sop == 4u) { slo = 0; shi = 0; @@ -17541,7 +17601,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, so++; continue; } - for (k = slo; k <= shi && sa_nstub < 24u; k++) { + for (k = slo; k <= shi; k++) { u32 mthd = ((h & 0xfffu) << 2) + 4u * k; u32 j, t; bool have = false; @@ -17552,13 +17612,43 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, j = (mthd - 0x3800u) / 8u; if (j >= 64u) continue; - if (sa_mask & (1ull << j)) - continue; + /* + * SESSION 153-CONTINUED-68: a slot is only + * usable when the capture holds its SA entry + * AND the IR instruction that entry names + * actually entered the replay (see + * gsp_sema_mme_ir_covered); a last-wins/ + * truncated mismatch makes the engine fetch + * macro data that is not there + * (MISSING_MACRO_DATA). Called-and-covered + * slots are logged too (r1134 ILLEGAL_OPCODE + * attribution: which macros actually run). + */ + if (sa_mask & (1ull << j)) { + if (!READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all) && + gsp_sema_mme_ir_covered(sa_ent[j])) { + for (t = 0; t < sa_ncov; t++) + if (sa_cov[t] == j) + have = true; + if (!have && sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + sa_ent[j]; + sa_ncov++; + } + continue; + } + } for (t = 0; t < sa_nstub; t++) if (sa_stub[t] == j) have = true; - if (!have) - sa_stub[sa_nstub++] = j; + if (!have && sa_nstub < 40u) { + sa_stub[sa_nstub] = j; + sa_stub_ent[sa_nstub] = + (sa_mask & (1ull << j)) ? + sa_ent[j] : 0u; + sa_nstub++; + } } so += sspan; } @@ -17566,10 +17656,50 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, sa_block = 5u + 3u * sa_nstub; } + /* + * SESSION 153-CONTINUED-68: full run-header census of the 3D + * window being replayed (bounded; first 4 windows). The + * r1134 ILLEGAL_OPCODE attribution needs the real call + * structure: which methods the window writes, how many data + * words each run carries, and the CALL_MME_MACRO/DATA + * pattern (op 1 INC vs op 5 ONE_INC) around them. + */ + { + static u32 win_dump_n; + + if (win3d && win3d_n && win_dump_n < 4u) { + u32 wo = 0, wr = 0; + + win_dump_n++; + pr_info("vgsp-gr: MME-WINRUN dump h=0x%x push=0x%llx window=%u\n", + req->h_object, req->push_va, win3d_n); + while (wo < win3d_n && wr < 200u) { + u32 h = win3d[wo]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 addr = (h & 0xfffu) << 2; + u32 span = 1u; + u32 d0 = (wo + 1u < win3d_n) ? + win3d[wo + 1u] : 0u; + u32 d1 = (wo + 2u < win3d_n) ? + win3d[wo + 2u] : 0u; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && wo + 1u + cnt <= win3d_n) + span = 1u + cnt; + pr_info("vgsp-gr: MME-WINRUN off=%x op=%u cnt=%u mthd=%x d0=%x d1=%x\n", + wo * 4u, op, cnt, addr, d0, d1); + wo += span; + wr++; + } + } + } + if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) { pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen); if (pl && pl_n && pl_gen && - pl_gen != READ_ONCE(gsp_phys_gr_mme_prelude_gen)) { + (pl_gen != READ_ONCE(gsp_phys_gr_mme_prelude_gen) || + req->h_object != READ_ONCE(gsp_phys_gr_mme_prelude_h))) { if (win3d_n + 2u + 8u + pl_n + sa_block <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { emit_prelude = true; @@ -17627,6 +17757,8 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, gsp_sema_mme_prelude_get(NULL, &g1); if (g1 == g0) { WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, g0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_h, + req->h_object); pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx sdm_drop=%u\n", g0, pwords, win3d_n, req->h_object, req->push_va, sdrop); @@ -17645,14 +17777,29 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, * and the context keeps it for every later window. */ if (emit_prelude && sa_block) { - u32 t; + u32 t, stub_pc = gsp_sema_mme_ir_end_get(); - *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + /* + * SESSION 153-CONTINUED-68: the stub PC must be past the + * replayed program's last instruction - the window also + * calls COVERED slots, and a fixed PC inside a captured + * macro body would clobber its code. Fall back to 0xC0 + * when the capture reports nothing yet. + */ + if (stub_pc < GSP_PHYS_RMCALL_GR_MME_STUB_PC) + stub_pc = GSP_PHYS_RMCALL_GR_MME_STUB_PC; + /* + * The IR upload MUST use the client's own SEC_OP (5, + * ONE_INC): with op=1 the method auto-increment walked + * the data words onto 0x11c/0x120 and clobbered SA[0] + * while writing no usable stub code (r1131). + */ + *p++ = (GSP_PHYS_RMCALL_METHOD_ONEINC << 29) | (4u << 16) | (0x114u >> 2); - *p++ = GSP_PHYS_RMCALL_GR_MME_STUB_PC; - *p++ = 0u; - *p++ = 0u; - *p++ = 1u; /* end_next, no emits (harmless) */ + *p++ = stub_pc; + *p++ = gsp_phys_gr_mme_stub_code[0]; + *p++ = gsp_phys_gr_mme_stub_code[1]; + *p++ = gsp_phys_gr_mme_stub_code[2]; /* * Each [slot, stub] pair is its own INC 0x11c run: the * pair methods are re-written per slot (a single longer @@ -17663,11 +17810,36 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (2u << 16) | (0x11cu >> 2); *p++ = sa_stub[t]; - *p++ = GSP_PHYS_RMCALL_GR_MME_STUB_PC; + *p++ = stub_pc; } - pr_info("vgsp-gr: MME-SA-STUB slots=%u words=%u h=0x%x push=0x%llx\n", - sa_nstub, sa_block, req->h_object, + pr_info("vgsp-gr: MME-SA-STUB slots=%u words=%u pc=0x%x h=0x%x push=0x%llx\n", + sa_nstub, sa_block, stub_pc, req->h_object, req->push_va); + { + char sb[192]; + u32 k, o = 0; + + sb[0] = '\0'; + for (k = 0; k < sa_nstub && o + 8u < sizeof(sb); k++) + o += scnprintf(sb + o, sizeof(sb) - o, + "%s%u:%x", k ? "," : "", + sa_stub[k], + sa_stub_ent[k]); + pr_info("vgsp-gr: MME-SA-STUB list=[%s]\n", sb); + } + { + char sb[192]; + u32 k, o = 0; + + sb[0] = '\0'; + for (k = 0; k < sa_ncov && o + 8u < sizeof(sb); k++) + o += scnprintf(sb + o, sizeof(sb) - o, + "%s%u:%x", k ? "," : "", + sa_cov[k], + sa_cov_ent[k]); + pr_info("vgsp-gr: MME-SA-COVERED list=[%s]\n", + sb); + } } /* * SESSION 153-CONTINUED-57: attr/pipeline state prelude (see the |