diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 177 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 318 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 22 | ||||
| -rw-r--r-- | tools/guest-diag/pin_gr_stream.py | 27 |
4 files changed, 529 insertions, 15 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 3bbcf45..42fcdb1 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17130,6 +17130,43 @@ MODULE_PARM_DESC(sema_gl_mme_prelude, "Preplay the captured client MME context-setup runs once per capture generation ahead of the first 3D window replay (default 1)"); /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091 + * full-MME-prelude boot cleared MME (mme_esr=80000000) but the first beef + * window then latched a draw-shader program-header error + * (NV_PGRAPH_PRI_DS_HWW_ESR 0x405840 = 0xa2040a00: SPH9|SPH11|SPH18, + * M_PIPE_STAGE = VSB|PS) and a class error at SET_VERTEX_ATTRIBUTE_A(2) + * (0x1168, INVALID_ENUM): the replayed window is a mid-stream state cut - + * the attribute/pipeline record the client set in EARLIER windows (the + * missing stages' SET_PIPELINE_PROGRAM_ADDRESS binds and the attribute + * format state) was never replayed, so the DS validated program headers + * that were never bound. With this knob the launch builder preplays the + * captured runs from the service's attr prelude buffer (see + * svc_attr_prelude_capture), relocated exactly like the window itself + * (the patched copy is computed by the caller - it needs the launch page + * set and the live shadow counter), run-aligned, drained across the first + * launches of a capture generation. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_attr_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_attr_prelude = true; +module_param_named(sema_gl_attr_prelude, gsp_phys_rmcall_gr_attr_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_attr_prelude, + "Preplay the captured client attr/pipeline runs (0x1100..0x11fc, 0x2000..0x205c, relocation applied) ahead of the first 3D window replays of a capture generation (default 1)"); + +/* + * The patched (relocation-applied) attr prelude for the current capture + * generation, built by the submit path just before build_gr_launch. Single + * caller (the ordered single-inflight sema work item), so plain statics. + */ +#define GSP_PHYS_GR_ATTR_PRE_WORDS 512u +static u32 gsp_phys_gr_attr_pre[GSP_PHYS_GR_ATTR_PRE_WORDS]; +static u32 gsp_phys_gr_attr_pre_n; +static u32 gsp_phys_gr_attr_pre_gen; /* capture gen the patch is from */ +static u32 gsp_phys_gr_attr_emit_gen; /* gen whose runs are draining */ +static u32 gsp_phys_gr_attr_emit_off; /* word cursor into attr_pre */ + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -17140,7 +17177,9 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_va, u64 sem_va, u32 sem_payload, - const u32 *win3d, u32 win3d_n) + const u32 *win3d, u32 win3d_n, + const u32 *attr_pre, + u32 attr_pre_n) { u32 *p = pb; int prelude = gsp_phys_rmcall_gr_prelude_value(); @@ -17214,6 +17253,66 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, g0, g1, pl_n); } } + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude (see the + * sema_gl_attr_prelude knob above). Emit the patched captured + * runs after the MME prelude and ahead of the window, + * run-aligned, draining across launches when the remaining + * block budget cannot take the whole remainder. The cursor + * resets when a new capture generation is patched. A window + * with no room at all defers the whole step to the next launch + * (logged by the patch site). + */ + if (attr_pre && attr_pre_n && + READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + u32 avail = GSP_PHYS_RMCALL_GR_METHOD_WORDS - + (u32)(p - pb) - (win3d_n + 2u + 8u); + u32 off, copied = 0; + + if (gsp_phys_gr_attr_emit_gen != + READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + WRITE_ONCE(gsp_phys_gr_attr_emit_gen, + READ_ONCE(gsp_phys_gr_attr_pre_gen)); + WRITE_ONCE(gsp_phys_gr_attr_emit_off, 0); + } + off = READ_ONCE(gsp_phys_gr_attr_emit_off); + while (off < attr_pre_n) { + u32 hdr = attr_pre[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt) ? + 1u + cnt : 1u; + + if (off + span > attr_pre_n) + span = attr_pre_n - off; + if (copied + span > avail) + break; + memcpy(&p[copied], &attr_pre[off], + (size_t)span * 4u); + copied += span; + off += span; + } + if (copied) { + p += copied; + WRITE_ONCE(gsp_phys_gr_attr_emit_off, off); + pr_info("vgsp-gr: ATTR-PRELUDE emitted gen=%u words=%u/%u window=%u h=0x%x push=0x%llx%s\n", + READ_ONCE(gsp_phys_gr_attr_pre_gen), + off, attr_pre_n, win3d_n, + req->h_object, req->push_va, + off < attr_pre_n ? + " (continuation next launch)" : ""); + } else if (off < attr_pre_n) { + static u32 attr_defer_log; + + if (attr_defer_log < 8) { + attr_defer_log++; + pr_info("vgsp-gr: ATTR-PRELUDE deferred (no room: window=%u avail=%u) h=0x%x\n", + win3d_n, avail, + req->h_object); + } + } + } memcpy(p, win3d, (size_t)win3d_n * 4); p += win3d_n; p = gsp_phys_rmcall_push_gr(p, @@ -20629,8 +20728,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; bool sem_gated = false; - static u32 sked_prev[20]; - static u32 sked_sample[20]; + static u32 sked_prev[24]; + static u32 sked_sample[24]; static u32 sked_tick; static bool sked_first; /* @@ -21865,6 +21964,41 @@ gr_page_unmapped: pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u)\n", nrel, win3d_n, n_shadow, ndrop, gsp_phys_rmcall_gr_3d_last_subdrop); + /* + * SESSION 153-CONTINUED-57: patch the attr/pipeline prelude for + * this launch (relocation exactly like the window: the + * SET_PIPELINE_PROGRAM_ADDRESS pairs and any other recognised + * operand land in engine-visible shadow VAs; sub!=0 and + * win_drop runs are removed by the same pass). Only done when + * the capture generation moved - the shadow VAs are stable + * per HPA for the boot, so an older patch stays valid while it + * drains across launches. See gsp_phys_gr_attr_pre*. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + const u32 *ap; + u32 apn = 0, apg = 0; + + ap = gsp_sema_attr_prelude_get(&apn, &apg); + if (ap && apn && apg && + apg != READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + u32 adrop = 0, an = apn; + + if (an > GSP_PHYS_GR_ATTR_PRE_WORDS) + an = GSP_PHYS_GR_ATTR_PRE_WORDS; + gsp_phys_rmcall_gr_3d_patch_window(ap, an, + gsp_phys_gr_attr_pre, + &gsp_phys_gr_attr_pre_n, + &adrop, req, &n_shadow); + if (gsp_phys_gr_attr_pre_n) { + WRITE_ONCE(gsp_phys_gr_attr_pre_gen, + apg); + pr_info("vgsp-gr: ATTR-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u h=0x%x push=0x%llx\n", + apg, an, gsp_phys_gr_attr_pre_n, + adrop, req->h_object, + req->push_va); + } + } + } if (ndrop) { static u32 gr3d_drop_log; @@ -21950,7 +22084,9 @@ gr_page_unmapped: words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, ch_sem_va, want_ch, - win3d, win3d_n); + win3d, win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); /* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */ WRITE_ONCE(gsp_phys_gr_last_words, words); @@ -22427,10 +22563,28 @@ gr_page_unmapped: */ sked_sample[18] = ioread32(bar0 + 0x004000c0u); sked_sample[19] = ioread32(bar0 + 0x004000c4u); + /* + * SESSION 153-CONTINUED-57: the class-error trapped + * register set (gr_error_gk104.c grGetExceptionChannel): + * NV_PGRAPH_TRAPPED_ADDR 0x40704 (MTHD offset + SUBCH + + * INVALID status bit), TRAPPED_DATA_LOW 0x40708 (the + * class error's "Data" field), TRAPPED_DATA_HIGH + * 0x4070c, and TRAPPED_DATA_MME 0x40710 (PC 11:0 + + * MACRO_ADDRESS 19:12) - the last one names whether the + * offending write came from MME macro execution. The + * real RM clears these within ~ms of the fault; a + * per-tick sample catches the first change. The + * persistent 0x1168/Data=0 class error (r1091/r1098) + * needs exactly this decode. + */ + sked_sample[20] = ioread32(bar0 + 0x00400704u); + sked_sample[21] = ioread32(bar0 + 0x00400708u); + sked_sample[22] = ioread32(bar0 + 0x0040070cu); + sked_sample[23] = ioread32(bar0 + 0x00400710u); if (sked_first || memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { - pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x ds_esr=%08x mme_esr=%08x mme_pc=%u mme_mac=0x%x mme_ir=%08x mth0=%08x mth1=%08x(pbaddr=0x%x sub=%u incr=%u)\n", + pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x ds_esr=%08x mme_esr=%08x mme_pc=%u mme_mac=0x%x mme_ir=%08x mth0=%08x mth1=%08x(pbaddr=0x%x sub=%u incr=%u) trap=%08x(data_lo=%08x hi=%08x sub=%u mmegen=%u dhv=%u) mmepc=%u mmemac=0x%x\n", sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], @@ -22447,7 +22601,14 @@ gr_page_unmapped: sked_sample[18], sked_sample[19], (sked_sample[18] >> 2) << 2, (sked_sample[18] >> 16) & 7u, - sked_sample[18] & 1u); + sked_sample[18] & 1u, + sked_sample[20], sked_sample[21], + sked_sample[22], + (sked_sample[20] >> 16) & 7u, + (sked_sample[20] >> 20) & 1u, + (sked_sample[20] >> 24) & 1u, + sked_sample[23] & 0xfffu, + (sked_sample[23] >> 12) & 0xffu); memcpy(sked_prev, sked_sample, sizeof(sked_sample)); sked_first = false; @@ -23372,7 +23533,9 @@ gr_page_unmapped: methods, req, qmd, qmd_dst, ch_sem_va, want_ch, win3d, - win3d_n); + win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); } diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index bd87d4c..b7897d8 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -13986,6 +13986,16 @@ static u64 svc_mme_prelude_sigs[SVC_MME_PRELUDE_SIGS]; static u32 svc_mme_prelude_sig_n; static u32 svc_mme_prelude_dump_n; static u32 svc_mme_prelude_flag = 1; +/* + * SESSION 153-CONTINUED-57: macro-program payload words seen by the + * capture (sum of the 0x114 LOAD_MME_INSTRUCTION_RAM run counts). The + * r1094 failure mode was a launch emitted with the capture still at 9 + * words (scratch only, zero code words): the MME executed macro(2) with + * pc=0 -> ILLEGAL_OPCODE -> poison 0.6 s before the client's real setup + * push became readable. gsp_sema_mme_prelude_mature() lets the forward + * path gate the first beef launch on the program actually being present. + */ +static u32 svc_mme_prelude_code_words; module_param_named(sema_mme_prelude_cap, svc_mme_prelude_flag, uint, 0644); MODULE_PARM_DESC(sema_mme_prelude_cap, "Capture the client's MME context-setup method runs (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc) from trusted beef pushes for the GR launch builder's one-time preplay (default 1)"); @@ -14069,6 +14079,14 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, } memcpy(&svc_mme_prelude_buf[svc_mme_prelude_n], buf + off, span * 4u); + /* + * SESSION 153-CONTINUED-57: count the macro-program payload - + * the 0x114 instruction-RAM uploads are the only runs that + * make the prelude functionally complete; the maturity gate + * keys on them. + */ + if (addr == 0x114u && span > 1u) + svc_mme_prelude_code_words += span - 1u; svc_mme_prelude_n += span; svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig; added++; @@ -14102,6 +14120,194 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, } } +/* + * SESSION 153-CONTINUED-57: is the captured MME setup complete enough to + * launch against? The r1091 (works) / r1094 (poisons) A/B: a mature capture + * holds the client's macro program (>= 64 code words = at least four + * 0x114 uploads); the premature r1094 capture held 9 scratch-only words + * and the launch raised MME ILLEGAL_OPCODE within milliseconds. The gate + * in svc_gr_request defers beef forwards until this reads true (bounded by + * sema_gr_prelude_gate_ms so a client that never uploads cannot livelock + * the forward path). + */ +bool gsp_sema_mme_prelude_mature(void) +{ + return READ_ONCE(svc_mme_prelude_code_words) >= 64u; +} + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. + * + * The full-MME-prelude boot (r1091) cleared the MME ILLEGAL_OPCODE but the + * first beef window then faulted the draw shader: NV_PGRAPH_PRI_DS_HWW_ESR + * (0x405840) latched 0xa2040a00 = SPH9|SPH11|SPH18 pending with + * M_PIPE_STAGE = VSB|PS, i.e. draws launched against shader-program-header + * slots whose SET_PIPELINE_PROGRAM_ADDRESS binds live in windows that were + * NEVER replayed (mid-stream state cut), and the same window then raised a + * class error at SET_VERTEX_ATTRIBUTE_A(2) (0x1168, INVALID_ENUM) - the + * window rebinds SOME attribute/pipeline state (byte-faithful, verified) + * but the surrounding record the client set earlier is missing. This + * buffer captures every trusted beef push's method runs in the C597 + * attribute/pipeline families, first-encounter order: + * + * 0x1100..0x11fc attribute formats, skip masks, vertex attribute A/B, + * anti-alias sample positions + * 0x2000..0x205c SET_PIPELINE_SHADER/REGISTER_COUNT/PROGRAM_ADDRESS/ + * BINDING - the per-stage pipeline record + * + * The launch builder preplays the captured runs (run-aligned, relocation + * applied exactly like the window itself) ahead of the first 3D window + * replay of a capture generation. The SET_PIPELINE_PROGRAM_ADDRESS_A/B + * operand VAs the runs bind are also recorded (svc_attr_prelude_ptr) so + * svc_gr_req_fill can put their pages into the launch page set - without + * them the shadow relocation MISSes the pair, the emitted address stays a + * raw client VA, and the DS cannot fetch the SPH through it. Identical + * re-uploads are deduped by an exact-run signature table (same scheme as + * the MME prelude). + */ +#define SVC_ATTR_PRELUDE_WORDS 512u +#define SVC_ATTR_PRELUDE_SIGS 256u +#define SVC_ATTR_PRELUDE_PTRS 8u +static u32 svc_attr_prelude_buf[SVC_ATTR_PRELUDE_WORDS]; +static u32 svc_attr_prelude_n; +static u32 svc_attr_prelude_gen; +static u64 svc_attr_prelude_sigs[SVC_ATTR_PRELUDE_SIGS]; +static u32 svc_attr_prelude_sig_n; +static u32 svc_attr_prelude_dump_n; +static u32 svc_attr_prelude_flag = 1; +static u64 svc_attr_prelude_ptr[SVC_ATTR_PRELUDE_PTRS]; +static u32 svc_attr_prelude_ptr_n; +module_param_named(sema_attr_prelude_cap, svc_attr_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_attr_prelude_cap, + "Capture the client's attr/pipeline method runs (0x1100..0x11fc, 0x2000..0x205c) from trusted beef pushes for the GR launch builder's preplay ahead of the first 3D window (default 1)"); + +static bool svc_attr_setup_run_addr(u32 addr) +{ + return (addr >= 0x1100u && addr < 0x1200u) || + (addr >= 0x2000u && addr < 0x2060u); +} + +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_n); + if (gen) + *gen = READ_ONCE(svc_attr_prelude_gen); + return svc_attr_prelude_buf; +} + +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_ptr_n); + return svc_attr_prelude_ptr; +} + +static void svc_attr_prelude_note_ptr(u64 va) +{ + u32 i; + + if (!va || svc_attr_prelude_ptr_n >= SVC_ATTR_PRELUDE_PTRS) + return; + for (i = 0; i < svc_attr_prelude_ptr_n; i++) + if (svc_attr_prelude_ptr[i] == va || + (svc_attr_prelude_ptr[i] & ~(u64)0xfffu) == + (va & ~(u64)0xfffu)) + return; + svc_attr_prelude_ptr[svc_attr_prelude_ptr_n++] = va; +} + +static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_attr_prelude_n; + + if (!READ_ONCE(svc_attr_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (!svc_attr_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + /* + * SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) pair [upper(7:0), + * lower(31:0)]: remember the program VA so its page can join + * the launch page set (see the header comment). + */ + if (addr >= 0x2014u && addr < 0x2060u && + (addr & 0x3fu) == 0x14u && span >= 3u) { + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_attr_prelude_note_ptr(((u64)(v0 & 0xffu) << 32) | + (u64)(v1 & 0xfffffffcu)); + } + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_attr_prelude_sig_n; i++) { + if (svc_attr_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_attr_prelude_n + span > SVC_ATTR_PRELUDE_WORDS || + svc_attr_prelude_sig_n >= SVC_ATTR_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_attr_prelude_buf[svc_attr_prelude_n], buf + off, + span * 4u); + svc_attr_prelude_n += span; + svc_attr_prelude_sigs[svc_attr_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_attr_prelude_gen, svc_attr_prelude_gen + 1); + if (svc_attr_prelude_dump_n < 4) { + u32 k; + + svc_attr_prelude_dump_n++; + pr_info("sema-attr-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u ptr_n=%u (was %u)\n", + h_object, push_va, len, added, + svc_attr_prelude_n, svc_attr_prelude_gen, + svc_attr_prelude_ptr_n, words0); + for (k = 0; k + 4u <= svc_attr_prelude_n && + k < 512u; k += 4u) + pr_info("sema-attr-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_attr_prelude_buf[k], + svc_attr_prelude_buf[k + 1], + svc_attr_prelude_buf[k + 2], + svc_attr_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-attr-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_attr_prelude_n, dups); + } +} + static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, u32 h_object) { @@ -14839,6 +15045,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ if (ch && trusted) svc_mme_prelude_capture(buf, len, push_va, ch->h_object); + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude capture (same + * trusted-only rule; the runs are replayed into a hardware context). + */ + if (ch && trusted) + svc_attr_prelude_capture(buf, len, push_va, ch->h_object); memset(&st, 0, sizeof(st)); st.push_va = push_va; @@ -30716,6 +30928,39 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, SVC_GR_QMD_WORD_NONE); + /* + * SESSION 153-CONTINUED-57: attr-prelude operand pages. The captured + * attr/pipeline prelude (see svc_attr_prelude_capture) binds + * SET_PIPELINE_PROGRAM_ADDRESS pairs whose VAs were set in earlier + * windows; the launch builder relocates the emitted pairs to + * shadow-pool VAs - which requires each page to be IN this page set, + * or gsp_phys_rmcall_gr_3d_reloc MISSes and the emitted address stays + * a raw client VA the DS cannot fetch an SPH through. Append them + * (deduped by page inside svc_gr_page_note), soft: an unresolved or + * overflowing page is rolled back so the fill's ENOENT/overflow + * checks cannot refuse the launch over a prelude nicety. + */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) { + u32 apn = 0, ai; + const u64 *aps = gsp_sema_attr_prelude_ptrs(&apn); + static u32 svc_attr_ptr_log; + + for (ai = 0; ai < apn; ai++) { + u32 before = svc_fwd_req.gr_n_pages; + + if (svc_fwd_req.gr_n_pages + 2u >= SVC_GR_PAGES_MAX) + break; + svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE); + if (svc_fwd_req.gr_n_pages > before && + !svc_fwd_req.gr_pages[svc_fwd_req.gr_n_pages - 1].pa) + svc_fwd_req.gr_n_pages = before; + else if (svc_attr_ptr_log < 16) { + svc_attr_ptr_log++; + pr_info("sema-gr: attr-prelude page va=0x%llx noted (n_pages=%u)\n", + aps[ai], svc_fwd_req.gr_n_pages); + } + } + } if (gsp_sema_dbg && svc_gr_pages_log < 24) { svc_gr_pages_log++; pr_info("sema-gr: cand h=0x%x param=0x%llx prog=0x%llx n_ptr=%u inline=%u n_pages=%u push=0x%llx\n", @@ -30889,6 +31134,57 @@ static bool svc_gr_defer_expired(u64 push_va) } /* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate. + * + * r1094's second failure mode: the FIRST beef launch of the boot ran with + * `MME-PRELUDE emitted gen=2 words=9` (the client's setup push had not + * become readable yet; the capture matured to 443 words only ~90 s later, + * after the premature launch had already poisoned the lane with MME + * ILLEGAL_OPCODE). The 5 ms sked sampler recorded `mme_esr=80000004 + * mme_mac=0x30` at +5 ms. The mirror of that boot (r1091) read the setup + * push BEFORE the first draw and launched with words=512: MME clean. + * + * This gate withholds a beef forward while the captured MME prelude is + * immature (no macro program yet), so the first window that reaches the + * engine carries the client's real setup. The pump's busy-defer machinery + * holds the entry and retries; once the setup push lands (the setup-hold + * demand/pin path is already armed for it) the capture matures and the + * launch proceeds. Bounded: after sema_gr_prelude_gate_ms the gate stops + * firing for the boot (fail-open, one loud log) so a client that never + * uploads a macro program cannot livelock the forward path. + */ +static unsigned gsp_sema_gr_prelude_gate_ms = 60000u; +module_param_named(sema_gr_prelude_gate_ms, gsp_sema_gr_prelude_gate_ms, + uint, 0644); +MODULE_PARM_DESC(sema_gr_prelude_gate_ms, + "Withhold the first beef GR forwards while the captured MME prelude is immature (no 0x114 macro program yet); fail-open after this many ms (default 60000, 0 = off)"); +static unsigned long svc_gr_prelude_gate_j; +static u64 svc_gr_prelude_gate_defer; +static u64 svc_gr_prelude_gate_exp; + +static bool svc_gr_prelude_gate_active(const struct svc_channel *ch) +{ + if (!READ_ONCE(svc_mme_prelude_flag) || !gsp_sema_gr_prelude_gate_ms) + return false; + if ((ch->h_object & 0xfffff000u) != 0xbeef0000u) + return false; + if (gsp_sema_mme_prelude_mature()) + return false; + if (!svc_gr_prelude_gate_j) + svc_gr_prelude_gate_j = jiffies; + if (time_after(jiffies, svc_gr_prelude_gate_j + + msecs_to_jiffies(gsp_sema_gr_prelude_gate_ms))) { + if (!svc_gr_prelude_gate_exp) { + svc_gr_prelude_gate_exp = 1; + pr_warn("vgsp-work: prelude gate EXPIRED h=0x%x (MME prelude still immature after %u ms; forwarding anyway)\n", + ch->h_object, gsp_sema_gr_prelude_gate_ms); + } + return false; + } + return true; +} + +/* * r604: the r543 arg-defer latch and the r556 absolute defer cap are keyed * by push VA ALONE, and the UVM pool reuses the same launch push VAs for * every client of a boot - the exact lifetime bug the session flush already @@ -31410,6 +31706,28 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate (see + * svc_gr_prelude_gate_active). Defer the forward while the captured + * MME setup is incomplete: the candidate survives (busy-defer), the + * pump re-decodes the entry and retries, and the launch goes out + * with the client's real macro program instead of poisoning the + * lane. Skipped for every non-beef channel and after the bounded + * fail-open window. + */ + if (svc_gr_prelude_gate_active(ch)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 16 || + (svc_gr_prelude_gate_defer & 0x3F) == 1) + pr_info("vgsp-work: prelude gate h=0x%x push=0x%llx deferred (MME prelude immature; capture will gate this launch until the setup push lands) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc_gr_prelude_gate_defer); + return false; + } + /* * m143: drain the host injection queue before resolving the launch's * pages. The watcher's pins for the client's a/b/c buffers can land * between the service loop's map_add drain and this capture; the fill diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index e47e2e8..7a9feb7 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -464,6 +464,28 @@ int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out); const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen); /* + * SESSION 153-CONTINUED-57: is the captured MME prelude mature enough to + * launch against - i.e. does it hold the client's macro program (>= 64 + * captured 0x114 code words)? The r1094 premature-emission poison + * (MME ILLEGAL_OPCODE at +5 ms, capture still at 9 scratch words) is + * gated on this: the sema forward path withholds beef launches while it + * reads false, bounded by sema_gr_prelude_gate_ms. + */ +bool gsp_sema_mme_prelude_mature(void); + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude accessor (C597 + * families 0x1100..0x11fc and 0x2000..0x205c, verbatim, first-encounter + * order, deduped). Same contract as gsp_sema_mme_prelude_get: preplay once + * per generation ahead of the first 3D window replay. The runs carry the + * pipeline program binds whose SPH the draw shader validates, so the + * builder also registers the program VAs (gsp_sema_attr_prelude_ptrs) in + * the launch page set to make the address relocation land. + */ +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen); +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n); + +/* * r378 write guards. Both re-validate a guest-memory write target that was * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's * per-page resolution) against the CURRENT resolution before the bytes land, diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py index 1acef4b..7a9bb66 100644 --- a/tools/guest-diag/pin_gr_stream.py +++ b/tools/guest-diag/pin_gr_stream.py @@ -230,13 +230,24 @@ def _demand_loop(): expiry) as the stalled-VA class. """ track = {} # va -> [first_seen, last_sent] + it = 0 while True: - time.sleep(3) - try: - r = subprocess.run(["sudo", "-n", "dmesg"], - capture_output=True, text=True, timeout=20) - except Exception: - continue + # SESSION 153-CONTINUED-57: the 3 s pass was the losing half of the + # r1096 race - the beef setup-hold's demanded pages resolved in the + # guest ~2-4 s after the arm, while the pump re-read the entry + # every ~110 ms and moved on. Poll fast (0.75 s) so a newly armed + # demand goes to the guest within a second; the expensive dmesg + # harvest stays on a slower cadence (every 4th pass) because the + # debugfs setup-demand set is the time-critical source. + time.sleep(0.75) + it += 1 + r = None + if it % 4 == 1: + try: + r = subprocess.run(["sudo", "-n", "dmesg"], + capture_output=True, text=True, timeout=20) + except Exception: + continue now = time.time() for m in re.finditer( r"(?:stalled VA #\d+[:=]\s*" @@ -258,7 +269,7 @@ def _demand_loop(): # cannot, the guest content resolver is the last source - # harvest the VA so it resolves and injects the real page. r"|sema-pushzero: h=0x[0-9a-f]+ va=(0x[0-9a-f]+)", - (r.stdout or "")[-2000000:]): + (r.stdout or "")[-2000000:] if r else ""): va = int(m.group(1) or m.group(2) or m.group(3), 16) track.setdefault(va, [now, 0.0]) # SESSION 153-CONTINUED-51: the pushzero line now carries the @@ -268,7 +279,7 @@ def _demand_loop(): # whole push at once (bulk demand instead of one page per cycle). for m in re.finditer( r"sema-pushzero: h=0x[0-9a-f]+ va=(0x[0-9a-f]+) len=(\d+)", - (r.stdout or "")[-2000000:]): + (r.stdout or "")[-2000000:] if r else ""): zva = int(m.group(1), 16) zlen = int(m.group(2)) for zoff in range(0, min(zlen, 0x8000), 0x1000): |