diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 318 |
1 files changed, 318 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index bd87d4c..b7897d8 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -13986,6 +13986,16 @@ static u64 svc_mme_prelude_sigs[SVC_MME_PRELUDE_SIGS]; static u32 svc_mme_prelude_sig_n; static u32 svc_mme_prelude_dump_n; static u32 svc_mme_prelude_flag = 1; +/* + * SESSION 153-CONTINUED-57: macro-program payload words seen by the + * capture (sum of the 0x114 LOAD_MME_INSTRUCTION_RAM run counts). The + * r1094 failure mode was a launch emitted with the capture still at 9 + * words (scratch only, zero code words): the MME executed macro(2) with + * pc=0 -> ILLEGAL_OPCODE -> poison 0.6 s before the client's real setup + * push became readable. gsp_sema_mme_prelude_mature() lets the forward + * path gate the first beef launch on the program actually being present. + */ +static u32 svc_mme_prelude_code_words; module_param_named(sema_mme_prelude_cap, svc_mme_prelude_flag, uint, 0644); MODULE_PARM_DESC(sema_mme_prelude_cap, "Capture the client's MME context-setup method runs (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc) from trusted beef pushes for the GR launch builder's one-time preplay (default 1)"); @@ -14069,6 +14079,14 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, } memcpy(&svc_mme_prelude_buf[svc_mme_prelude_n], buf + off, span * 4u); + /* + * SESSION 153-CONTINUED-57: count the macro-program payload - + * the 0x114 instruction-RAM uploads are the only runs that + * make the prelude functionally complete; the maturity gate + * keys on them. + */ + if (addr == 0x114u && span > 1u) + svc_mme_prelude_code_words += span - 1u; svc_mme_prelude_n += span; svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig; added++; @@ -14102,6 +14120,194 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, } } +/* + * SESSION 153-CONTINUED-57: is the captured MME setup complete enough to + * launch against? The r1091 (works) / r1094 (poisons) A/B: a mature capture + * holds the client's macro program (>= 64 code words = at least four + * 0x114 uploads); the premature r1094 capture held 9 scratch-only words + * and the launch raised MME ILLEGAL_OPCODE within milliseconds. The gate + * in svc_gr_request defers beef forwards until this reads true (bounded by + * sema_gr_prelude_gate_ms so a client that never uploads cannot livelock + * the forward path). + */ +bool gsp_sema_mme_prelude_mature(void) +{ + return READ_ONCE(svc_mme_prelude_code_words) >= 64u; +} + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. + * + * The full-MME-prelude boot (r1091) cleared the MME ILLEGAL_OPCODE but the + * first beef window then faulted the draw shader: NV_PGRAPH_PRI_DS_HWW_ESR + * (0x405840) latched 0xa2040a00 = SPH9|SPH11|SPH18 pending with + * M_PIPE_STAGE = VSB|PS, i.e. draws launched against shader-program-header + * slots whose SET_PIPELINE_PROGRAM_ADDRESS binds live in windows that were + * NEVER replayed (mid-stream state cut), and the same window then raised a + * class error at SET_VERTEX_ATTRIBUTE_A(2) (0x1168, INVALID_ENUM) - the + * window rebinds SOME attribute/pipeline state (byte-faithful, verified) + * but the surrounding record the client set earlier is missing. This + * buffer captures every trusted beef push's method runs in the C597 + * attribute/pipeline families, first-encounter order: + * + * 0x1100..0x11fc attribute formats, skip masks, vertex attribute A/B, + * anti-alias sample positions + * 0x2000..0x205c SET_PIPELINE_SHADER/REGISTER_COUNT/PROGRAM_ADDRESS/ + * BINDING - the per-stage pipeline record + * + * The launch builder preplays the captured runs (run-aligned, relocation + * applied exactly like the window itself) ahead of the first 3D window + * replay of a capture generation. The SET_PIPELINE_PROGRAM_ADDRESS_A/B + * operand VAs the runs bind are also recorded (svc_attr_prelude_ptr) so + * svc_gr_req_fill can put their pages into the launch page set - without + * them the shadow relocation MISSes the pair, the emitted address stays a + * raw client VA, and the DS cannot fetch the SPH through it. Identical + * re-uploads are deduped by an exact-run signature table (same scheme as + * the MME prelude). + */ +#define SVC_ATTR_PRELUDE_WORDS 512u +#define SVC_ATTR_PRELUDE_SIGS 256u +#define SVC_ATTR_PRELUDE_PTRS 8u +static u32 svc_attr_prelude_buf[SVC_ATTR_PRELUDE_WORDS]; +static u32 svc_attr_prelude_n; +static u32 svc_attr_prelude_gen; +static u64 svc_attr_prelude_sigs[SVC_ATTR_PRELUDE_SIGS]; +static u32 svc_attr_prelude_sig_n; +static u32 svc_attr_prelude_dump_n; +static u32 svc_attr_prelude_flag = 1; +static u64 svc_attr_prelude_ptr[SVC_ATTR_PRELUDE_PTRS]; +static u32 svc_attr_prelude_ptr_n; +module_param_named(sema_attr_prelude_cap, svc_attr_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_attr_prelude_cap, + "Capture the client's attr/pipeline method runs (0x1100..0x11fc, 0x2000..0x205c) from trusted beef pushes for the GR launch builder's preplay ahead of the first 3D window (default 1)"); + +static bool svc_attr_setup_run_addr(u32 addr) +{ + return (addr >= 0x1100u && addr < 0x1200u) || + (addr >= 0x2000u && addr < 0x2060u); +} + +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_n); + if (gen) + *gen = READ_ONCE(svc_attr_prelude_gen); + return svc_attr_prelude_buf; +} + +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_ptr_n); + return svc_attr_prelude_ptr; +} + +static void svc_attr_prelude_note_ptr(u64 va) +{ + u32 i; + + if (!va || svc_attr_prelude_ptr_n >= SVC_ATTR_PRELUDE_PTRS) + return; + for (i = 0; i < svc_attr_prelude_ptr_n; i++) + if (svc_attr_prelude_ptr[i] == va || + (svc_attr_prelude_ptr[i] & ~(u64)0xfffu) == + (va & ~(u64)0xfffu)) + return; + svc_attr_prelude_ptr[svc_attr_prelude_ptr_n++] = va; +} + +static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_attr_prelude_n; + + if (!READ_ONCE(svc_attr_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (!svc_attr_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + /* + * SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) pair [upper(7:0), + * lower(31:0)]: remember the program VA so its page can join + * the launch page set (see the header comment). + */ + if (addr >= 0x2014u && addr < 0x2060u && + (addr & 0x3fu) == 0x14u && span >= 3u) { + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_attr_prelude_note_ptr(((u64)(v0 & 0xffu) << 32) | + (u64)(v1 & 0xfffffffcu)); + } + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_attr_prelude_sig_n; i++) { + if (svc_attr_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_attr_prelude_n + span > SVC_ATTR_PRELUDE_WORDS || + svc_attr_prelude_sig_n >= SVC_ATTR_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_attr_prelude_buf[svc_attr_prelude_n], buf + off, + span * 4u); + svc_attr_prelude_n += span; + svc_attr_prelude_sigs[svc_attr_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_attr_prelude_gen, svc_attr_prelude_gen + 1); + if (svc_attr_prelude_dump_n < 4) { + u32 k; + + svc_attr_prelude_dump_n++; + pr_info("sema-attr-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u ptr_n=%u (was %u)\n", + h_object, push_va, len, added, + svc_attr_prelude_n, svc_attr_prelude_gen, + svc_attr_prelude_ptr_n, words0); + for (k = 0; k + 4u <= svc_attr_prelude_n && + k < 512u; k += 4u) + pr_info("sema-attr-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_attr_prelude_buf[k], + svc_attr_prelude_buf[k + 1], + svc_attr_prelude_buf[k + 2], + svc_attr_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-attr-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_attr_prelude_n, dups); + } +} + static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, u32 h_object) { @@ -14839,6 +15045,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ if (ch && trusted) svc_mme_prelude_capture(buf, len, push_va, ch->h_object); + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude capture (same + * trusted-only rule; the runs are replayed into a hardware context). + */ + if (ch && trusted) + svc_attr_prelude_capture(buf, len, push_va, ch->h_object); memset(&st, 0, sizeof(st)); st.push_va = push_va; @@ -30716,6 +30928,39 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, SVC_GR_QMD_WORD_NONE); + /* + * SESSION 153-CONTINUED-57: attr-prelude operand pages. The captured + * attr/pipeline prelude (see svc_attr_prelude_capture) binds + * SET_PIPELINE_PROGRAM_ADDRESS pairs whose VAs were set in earlier + * windows; the launch builder relocates the emitted pairs to + * shadow-pool VAs - which requires each page to be IN this page set, + * or gsp_phys_rmcall_gr_3d_reloc MISSes and the emitted address stays + * a raw client VA the DS cannot fetch an SPH through. Append them + * (deduped by page inside svc_gr_page_note), soft: an unresolved or + * overflowing page is rolled back so the fill's ENOENT/overflow + * checks cannot refuse the launch over a prelude nicety. + */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) { + u32 apn = 0, ai; + const u64 *aps = gsp_sema_attr_prelude_ptrs(&apn); + static u32 svc_attr_ptr_log; + + for (ai = 0; ai < apn; ai++) { + u32 before = svc_fwd_req.gr_n_pages; + + if (svc_fwd_req.gr_n_pages + 2u >= SVC_GR_PAGES_MAX) + break; + svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE); + if (svc_fwd_req.gr_n_pages > before && + !svc_fwd_req.gr_pages[svc_fwd_req.gr_n_pages - 1].pa) + svc_fwd_req.gr_n_pages = before; + else if (svc_attr_ptr_log < 16) { + svc_attr_ptr_log++; + pr_info("sema-gr: attr-prelude page va=0x%llx noted (n_pages=%u)\n", + aps[ai], svc_fwd_req.gr_n_pages); + } + } + } if (gsp_sema_dbg && svc_gr_pages_log < 24) { svc_gr_pages_log++; pr_info("sema-gr: cand h=0x%x param=0x%llx prog=0x%llx n_ptr=%u inline=%u n_pages=%u push=0x%llx\n", @@ -30889,6 +31134,57 @@ static bool svc_gr_defer_expired(u64 push_va) } /* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate. + * + * r1094's second failure mode: the FIRST beef launch of the boot ran with + * `MME-PRELUDE emitted gen=2 words=9` (the client's setup push had not + * become readable yet; the capture matured to 443 words only ~90 s later, + * after the premature launch had already poisoned the lane with MME + * ILLEGAL_OPCODE). The 5 ms sked sampler recorded `mme_esr=80000004 + * mme_mac=0x30` at +5 ms. The mirror of that boot (r1091) read the setup + * push BEFORE the first draw and launched with words=512: MME clean. + * + * This gate withholds a beef forward while the captured MME prelude is + * immature (no macro program yet), so the first window that reaches the + * engine carries the client's real setup. The pump's busy-defer machinery + * holds the entry and retries; once the setup push lands (the setup-hold + * demand/pin path is already armed for it) the capture matures and the + * launch proceeds. Bounded: after sema_gr_prelude_gate_ms the gate stops + * firing for the boot (fail-open, one loud log) so a client that never + * uploads a macro program cannot livelock the forward path. + */ +static unsigned gsp_sema_gr_prelude_gate_ms = 60000u; +module_param_named(sema_gr_prelude_gate_ms, gsp_sema_gr_prelude_gate_ms, + uint, 0644); +MODULE_PARM_DESC(sema_gr_prelude_gate_ms, + "Withhold the first beef GR forwards while the captured MME prelude is immature (no 0x114 macro program yet); fail-open after this many ms (default 60000, 0 = off)"); +static unsigned long svc_gr_prelude_gate_j; +static u64 svc_gr_prelude_gate_defer; +static u64 svc_gr_prelude_gate_exp; + +static bool svc_gr_prelude_gate_active(const struct svc_channel *ch) +{ + if (!READ_ONCE(svc_mme_prelude_flag) || !gsp_sema_gr_prelude_gate_ms) + return false; + if ((ch->h_object & 0xfffff000u) != 0xbeef0000u) + return false; + if (gsp_sema_mme_prelude_mature()) + return false; + if (!svc_gr_prelude_gate_j) + svc_gr_prelude_gate_j = jiffies; + if (time_after(jiffies, svc_gr_prelude_gate_j + + msecs_to_jiffies(gsp_sema_gr_prelude_gate_ms))) { + if (!svc_gr_prelude_gate_exp) { + svc_gr_prelude_gate_exp = 1; + pr_warn("vgsp-work: prelude gate EXPIRED h=0x%x (MME prelude still immature after %u ms; forwarding anyway)\n", + ch->h_object, gsp_sema_gr_prelude_gate_ms); + } + return false; + } + return true; +} + +/* * r604: the r543 arg-defer latch and the r556 absolute defer cap are keyed * by push VA ALONE, and the UVM pool reuses the same launch push VAs for * every client of a boot - the exact lifetime bug the session flush already @@ -31410,6 +31706,28 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate (see + * svc_gr_prelude_gate_active). Defer the forward while the captured + * MME setup is incomplete: the candidate survives (busy-defer), the + * pump re-decodes the entry and retries, and the launch goes out + * with the client's real macro program instead of poisoning the + * lane. Skipped for every non-beef channel and after the bounded + * fail-open window. + */ + if (svc_gr_prelude_gate_active(ch)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 16 || + (svc_gr_prelude_gate_defer & 0x3F) == 1) + pr_info("vgsp-work: prelude gate h=0x%x push=0x%llx deferred (MME prelude immature; capture will gate this launch until the setup push lands) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc_gr_prelude_gate_defer); + return false; + } + /* * m143: drain the host injection queue before resolving the launch's * pages. The watcher's pins for the client's a/b/c buffers can land * between the service loop's map_add drain and this capture; the fill |