diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 405 |
1 files changed, 403 insertions, 2 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index a94bbef..5bb1fec 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -5132,6 +5132,55 @@ static void gsp_phys_mme_regs_log(void __iomem *bar0, const char *tag) } /* + * SESSION 153-CONTINUED-107: MME IRAM debug port readback. + * + * CONT-106 found the debug port at 0x4044d0 host-side; the definitive field + * spec (tu104 dev_graphics_nobundle.h:2166 NV_PGRAPH_PRI_MME_DEBUG) is + * bit 0 STATUS (R; 0=RUNNING, 1=STOPPED) + * bit 1 ENABLE (RW; init DISABLED) + * bit 2 IRAM_RD (write 1 = TRIGGER, reads 0 = DONE) + * bits 26:15 IRAM_PC (RW, 12 bits = the IRAM word address) + * and the fetched group lands in NV_PGRAPH_PRI_MME_HWW_ESR_INFO4/3/2 + * (0x4044ac/a8/9c, field IR 31:0 each). CONT-106's r1364 read at PC 0 + * returned the reconstructed image's first three words + * (0x00003463,0x190d82f0,0x39804300), so the data path is the INFO + * triplet. The read is a two-step hardware handshake: set ENABLE, write + * the PC with IRAM_RD=1 (trigger), WAIT for IRAM_RD to read back 0 + * (DONE), then read the group. `a` is an IRAM WORD address (callers pass + * instruction * 3, matching gsp_phys_gr_mme_ir_pack[] indexing). + * Read-only diagnostic; bounded by its call sites. + */ +static void gsp_phys_mme_iram_read3(void __iomem *bar0, u32 a, u32 out[3]) +{ + u32 i; + + iowrite32(0x2u, bar0 + 0x004044d0u); /* ENABLE */ + iowrite32(((a & 0xfffu) << 15) | 6u, bar0 + 0x004044d0u); + for (i = 0; i < 1000u; i++) { + if (!(ioread32(bar0 + 0x004044d0u) & 0x4u)) + break; + cpu_relax(); + } + out[0] = ioread32(bar0 + 0x004044acu); + out[1] = ioread32(bar0 + 0x004044a8u); + out[2] = ioread32(bar0 + 0x0040449cu); +} + +/* + * Defined next to the packed-image array (gsp_phys_gr_mme_ir_pack); forward + * declaration so the post-doorbell forensics spin can call it. + */ +static void gsp_phys_mme_iram_probe(void __iomem *bar0, const char *tag, + u32 h_object); + +/* + * SESSION 153-CONTINUED-107: probe enable (see the sema_gl_mme_iram_probe + * knob beside sema_gl_mme_iram_gate). Declared here for the forensics spin + * above; the matching module_param sits with the gate knob. + */ +static bool gsp_phys_rmcall_gr_mme_iram_probe; + +/* * SESSION 153-CONTINUED-69: pre-reset MME fault capture. Everything sampled * so far was post-mortem: by the time the 5 ms launch poll ticks, the * GSP-RM's FECS exception handler has already read NV_PGRAPH_PRI_MME_HWW_ESR @@ -5319,6 +5368,16 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, fed1_p = ioread32(bar0 + 0x0040a634u); pr_info("vgsp-gr: MME-FAST begin %s h=0x%x esr=%08x ds=%08x grfifo=%08x trap=%08x fe0=%08x fe1=%08x (spin %u ms)\n", tag, h_object, esr_p, ds_p, gf_p, tr_p, fed0_p, fed1_p, ms); + /* + * SESSION 153-CONTINUED-107: IRAM chronology probe (see the + * gsp_phys_mme_iram_probe comment). Bounded to the first 8 fast + * samples of the boot so the launch train cannot flood; knob-gated + * because the debug-port writes disturb a live launch (see the + * sema_gl_mme_iram_probe comment). + */ + if (fast_boot_n <= 8u && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "begin", h_object); t_end = ktime_get() + (s64)ms * 1000000ll; for (;;) { u32 esr = ioread32(bar0 + 0x00404490u); @@ -5387,6 +5446,16 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, mmemac & 0xfffu, (mmemac >> 12) & 0xffu, (mmemac >> 20) & 1u); + /* + * SESSION 153-CONTINUED-107: sample the live + * IRAM at every fault-register transition, so + * the chronology (words loaded early vs wiped + * by the poison/DROP machinery) is on record + * for the same instant the ESR changes. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "trans", + h_object); } /* * SESSION 153-CONTINUED-106: when the DS latch CHANGES @@ -5483,6 +5552,14 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n", tag, h_object, n, poisoned ? " (poison)" : ""); /* + * SESSION 153-CONTINUED-107: post-spin IRAM sample - the state left + * when the poison (if any) has just cleared, before the poll/retire + * work touches anything. + */ + if (fast_boot_n <= 8u && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "end", h_object); + /* * SESSION 153-CONTINUED-83 store-side emission forensics: right after * the launch's retire spin, snapshot the DS shader-program-header * latches (which pipeline stages failed SPH validation), the MME @@ -18218,6 +18295,20 @@ MODULE_PARM_DESC(sema_gl_mme_hold_calls, "Hold (elide) window CALL_MME_MACRO/DATA runs until the MME prelude latches for the current generation+channel (default 1; 0 = legacy, real calls run against a partial prelude)"); /* + * SESSION 153-CONTINUED-107: separately control the CONT-104 FE-idle clause + * of the hold (see the call site). With the IRAM gate proving residency the + * clause's only value is the r1359 race, and r1373 showed the FE wedged at + * 0x07071602 for minutes while every calls-kept window was held. Default 1 + * (pre-CONT-107 behavior); 0 releases the calls as soon as the prelude + * latch + IRAM gate allow. + */ +static bool gsp_phys_rmcall_gr_mme_hold_fe_idle = true; +module_param_named(sema_gl_mme_hold_fe_idle, + gsp_phys_rmcall_gr_mme_hold_fe_idle, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_fe_idle, + "Also hold the real CALL runs while the front-end is not settled (0x400504 != 1; default 1)"); + +/* * SESSION 153-CONTINUED-84: emit NVC597_SET_MME_VERSION=1 (0x234) with the * prelude step. The channel-init defaults the client pushes once per channel * (Turing+) are never replayed, so the replayed context's MME stays in its @@ -18287,6 +18378,173 @@ MODULE_PARM_DESC(sema_gl_mme_prelude_pack, static u32 gsp_phys_gr_mme_ir_pack[4096]; /* + * SESSION 153-CONTINUED-107: IRAM chronology + load-gate helpers. + * + * CONT-106 proved the trap-time fetch at PC 0x364 (macro 57 entry) reads + * ZERO IR words while the host-side pack is verbatim (CONT-105), and the + * IRAM debug port at 0x4044d0 also reads zero at 0x364 on that boot while + * the same port read the golden first words at pc=0 on r1364. The one + * unproven link is the LOAD: did the emitted 0x114 pack writes land in the + * MME's IRAM before the macro executed? These helpers read the live IRAM + * back and (a) log the timeline around each launch's fault-register + * transitions, (b) gate the real CALL runs on the readback matching the + * packed image at every covered SA entry. + * + * The probe reads the macro-57 entry group (0x364) under BOTH port address + * conventions - instruction base (write 0x364) and word base (write + * 3*0x364 = 0xA2C) - plus word 0, and prints the live words next to the + * pack-image expectation, so one line answers "is the program resident + * right now and which address unit does the port use". + */ +static void gsp_phys_mme_iram_probe(void __iomem *bar0, const char *tag, + u32 h_object) +{ + u32 a[3], bi[3], bw[3]; + + if (!bar0) + return; + gsp_phys_mme_iram_read3(bar0, 0u, a); + gsp_phys_mme_iram_read3(bar0, 0x364u, bi); + gsp_phys_mme_iram_read3(bar0, 3u * 0x364u, bw); + pr_info("vgsp-gr: MME-IRAM %s h=0x%x w0=%08x,%08x,%08x pack0=%08x,%08x,%08x i364=%08x,%08x,%08x w364=%08x,%08x,%08x pack364=%08x,%08x,%08x\n", + tag, h_object, a[0], a[1], a[2], + gsp_phys_gr_mme_ir_pack[0], + gsp_phys_gr_mme_ir_pack[1], + gsp_phys_gr_mme_ir_pack[2], + bi[0], bi[1], bi[2], + bw[0], bw[1], bw[2], + gsp_phys_gr_mme_ir_pack[3u * 0x364u], + gsp_phys_gr_mme_ir_pack[3u * 0x364u + 1u], + gsp_phys_gr_mme_ir_pack[3u * 0x364u + 2u]); +} + +/* + * SESSION 153-CONTINUED-107 gate knob (see the helper comment above). + * Default 0: A/B flip - with the gate on, a launch whose readback shows the + * IRAM not holding the packed image holds its real CALL runs and owes a + * full re-emission (prelude latch cleared; pack/drain cursors rewound), so + * the calls only ever execute against a verified-resident program. + */ +static bool gsp_phys_rmcall_gr_mme_iram_gate; +module_param_named(sema_gl_mme_iram_gate, gsp_phys_rmcall_gr_mme_iram_gate, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate, + "Read the MME IRAM back (0x4044d0 port) and hold/re-emit the packed prelude until it reads back the packed image at the SA entries (default 0; the CONT-106 all-zero fetch)"); + +/* + * SESSION 153-CONTINUED-107: separate knob for the per-launch chronology + * probe lines. They write the MME debug port DURING a live launch, and the + * first three probe-bearing cycles (r1369..r1371) all parked the engine on + * the first X-channel launches while the last probe-free cycle (r1368) ran + * the full lane; until the port's side effects are cleared, keep the probe + * off by default and enable it only on a known-good boot (A/B). + */ +module_param_named(sema_gl_mme_iram_probe, gsp_phys_rmcall_gr_mme_iram_probe, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_probe, + "Per-launch MME-IRAM chronology probe lines in the post-doorbell spin (writes the debug port during a live launch; default 0)"); +MODULE_PARM_DESC(sema_gl_mme_iram_probe, + "Per-launch MME-IRAM chronology probe lines in the post-doorbell spin (writes the debug port during a live launch; default 0)"); + +/* + * Read every covered SA entry from the live IRAM and compare with the packed + * image. Auto-calibrates the port's address unit once (word base 3*entry vs + * instruction base entry - the r1364 validation could only settle address 0). + * Returns true when every checked entry matches (or there is nothing to + * check); false on any mismatch (all-zero included). Bounded logging. + */ +static bool gsp_phys_gr_mme_iram_gate(void __iomem *bar0, u32 h_object, + u32 gen, const char *tag) +{ + static u32 gate_run_n, gate_fail_n, gate_pass_n; + static int base_sel = -1; /* -1 unresolved; 0 word; 1 instruction */ + u32 sa_ent[64], i; + u32 checked = 0, match = 0, zero = 0; + u32 bad_slot = 0xffffffffu, bad_entry = 0, checked_to_bad = 0; + u32 got0 = 0, got1 = 0, got2 = 0, exp0 = 0, exp1 = 0, exp2 = 0; + u64 sa_mask; + + if (!bar0) + return true; + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + for (i = 0; i < 64u; i++) { + u32 entry, base, g0, g1, g2, got[3]; + + if (!(sa_mask & (1ull << i)) || !sa_ent[i]) + continue; + entry = sa_ent[i]; + if (entry * 3u + 2u >= 4096u) + continue; + g0 = gsp_phys_gr_mme_ir_pack[entry * 3u]; + g1 = gsp_phys_gr_mme_ir_pack[entry * 3u + 1u]; + g2 = gsp_phys_gr_mme_ir_pack[entry * 3u + 2u]; + /* + * Zero-gold entries are stub-covered (no captured code) or + * not part of the packed image; they carry nothing to gate. + */ + if (!g0 && !g1 && !g2) + continue; + if (base_sel < 0 && entry) { + /* + * One-time unit calibration on the first non-zero + * entry: try the word base, then the instruction + * base. Leaving it unresolved simply keeps the word + * base (the documented form). + */ + gsp_phys_mme_iram_read3(bar0, 3u * entry, got); + if (got[0] == g0 && got[1] == g1 && got[2] == g2) { + base_sel = 0; + pr_info("vgsp-gr: MME-IRAM-BASE resolved=word (3*entry) at entry=0x%x\n", + entry); + } else { + gsp_phys_mme_iram_read3(bar0, entry, got); + if (got[0] == g0 && got[1] == g1 && + got[2] == g2) { + base_sel = 1; + pr_info("vgsp-gr: MME-IRAM-BASE resolved=instruction (entry) at entry=0x%x\n", + entry); + } + } + } + base = (base_sel == 1) ? entry : 3u * entry; + gsp_phys_mme_iram_read3(bar0, base, got); + checked++; + if (got[0] == g0 && got[1] == g1 && got[2] == g2) { + match++; + continue; + } + if (!got[0] && !got[1] && !got[2]) + zero++; + if (bad_slot == 0xffffffffu) { + bad_slot = i; + bad_entry = entry; + checked_to_bad = checked; + got0 = got[0]; got1 = got[1]; got2 = got[2]; + exp0 = g0; exp1 = g1; exp2 = g2; + } + } + if (!checked || match == checked) { + if (gate_pass_n < 8u) { + gate_pass_n++; + pr_info("vgsp-gr: MME-IRAM-GATE pass %s h=0x%x gen=%u base=%d checked=%u run=%u\n", + tag, h_object, gen, base_sel, checked, + gate_run_n); + } + gate_run_n++; + return true; + } + gate_run_n++; + if (gate_fail_n < 24u) { + gate_fail_n++; + pr_warn("vgsp-gr: MME-IRAM-GATE FAIL %s h=0x%x gen=%u base=%d checked=%u match=%u zero=%u bad slot=%u(entry=0x%x @%u) got=%08x,%08x,%08x want=%08x,%08x,%08x run=%u\n", + tag, h_object, gen, base_sel, checked, match, zero, + bad_slot, bad_entry, checked_to_bad, + got0, got1, got2, exp0, exp1, exp2, gate_run_n); + } + return false; +} + +/* * SESSION 153-CONTINUED-86: the missing SPH (shader-program-header) state. * * The executed beef launches fault `Graphics Exception: Shader Program @@ -18765,6 +19023,45 @@ static u32 gsp_phys_gr_mme_prelude_h; static u32 gsp_phys_gr_mme_pre_full_dump; /* + * SESSION 153-CONTINUED-107: readback-failure counter + give-up bound for + * the load-only starvation (see the mme_load_only comment). 0 = unlimited; + * default 24. + */ +static u32 gsp_phys_gr_mme_iram_gate_fails; +static u32 gsp_phys_rmcall_gr_mme_iram_gate_max = 24u; + +/* + * SESSION 153-CONTINUED-107: gate decision shared by both IRAM-gate call + * sites (see gsp_phys_gr_mme_iram_gate). The readback failed, so hold the + * calls and owe a full re-emission. Clearing the prelude latch makes the + * next launch emit again (the builder's emit condition); rewinding the pack + * and drain cursors makes that re-emission restart from the first word. The + * pending-gen latch is cleared so the poll break cannot promote the + * generation to "resident". + */ +static void gsp_phys_gr_mme_iram_gate_fail(void) +{ + gsp_phys_gr_mme_iram_gate_fails++; + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); +} + +module_param_named(sema_gl_mme_iram_gate_max, + gsp_phys_rmcall_gr_mme_iram_gate_max, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate_max, + "Give up the IRAM-gate load-only starvation after this many readback failures (0 = never; default 24)"); + +static bool gsp_phys_gr_mme_iram_gate_giveup(void) +{ + u32 m = READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate_max); + + return m && READ_ONCE(gsp_phys_gr_mme_iram_gate_fails) >= m; +} + +/* * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091 * full-MME-prelude boot cleared MME (mme_esr=80000000) but the first beef * window then latched a draw-shader program-header error @@ -18947,6 +19244,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, bool pre_done = false; bool mme_hold_calls = false; bool mme_hold_fe_busy = false; + bool mme_load_only = false; bool latch_before = false; u32 pre_words_out = 0; const u32 *pl = NULL; @@ -19896,6 +20194,43 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, } } /* + * SESSION 153-CONTINUED-107: prelude-only LOAD carriers while + * the IRAM gate has not verified the packed program. + * + * r1372's first instruction (decisive): the calls-held pack + * carrier still poisoned (`DS-FAST ... ds a2040a00->badf1201 + * ... MME-FAST end launch events=1 (poison)` -> WEDGE-RECOVER + * wipes the context) because the block also carried the WINDOW + * - the window's own SPH state trips the DS validation and the + * FECS poison episode interrupts the 0x114/0x118 IRAM load + * mid-flight. Holding the calls is not enough; the LOAD must + * ride a carrier whose methods cannot fault, so while the + * generation is not verified live (live_gen != pl_gen) emit + * the pack/SA/release WITHOUT the window. The carrier retires, + * the poll-break gate reads the IRAM back; on a pass live_gen + * is promoted and the very next launch carries the window + * again (latch && live). Bounded by + * sema_gl_mme_iram_gate_max so a broken readback cannot starve + * the lane forever. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls) && + pl && pl_n && pl_gen && + !gsp_phys_gr_mme_iram_gate_giveup()) { + bool latch_now = + (pl_gen == + READ_ONCE(gsp_phys_gr_mme_prelude_gen) && + req->h_object == + READ_ONCE(gsp_phys_gr_mme_prelude_h)); + bool live_now = + (READ_ONCE(gsp_phys_rmcall_gr_mme_live_skip) && + READ_ONCE(gsp_phys_gr_mme_live_gen) == pl_gen); + + if (!(latch_now && live_now)) + mme_load_only = true; + } + /* * SESSION 153-CONTINUED-82: hold real CALL runs until the * prelude latch for this generation+channel is set (see the * sema_gl_mme_hold_calls comment). Evaluated AFTER the @@ -19929,13 +20264,46 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, * settled (the r1359 MME ILLEGAL_OPCODE race; see * gsp_phys_gr_fe_idle). The held launch retires and * a later window retries once 0x400504 reads idle. + * SESSION 153-CONTINUED-107: separately knob-gated - + * with the IRAM gate proving residency, the residual + * value of this clause is only the r1359 race; r1373 + * left the FE at 0x07071602 for minutes and starved + * every calls-kept window, so allow an explicit A/B + * (sema_gl_mme_hold_fe_idle=0). */ - if (!mme_hold_calls && !gsp_phys_gr_fe_idle()) { + if (!mme_hold_calls && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_fe_idle) && + !gsp_phys_gr_fe_idle()) { mme_hold_calls = true; mme_hold_fe_busy = true; } } /* + * SESSION 153-CONTINUED-107: IRAM readback gate (see the + * gsp_phys_gr_mme_iram_gate comment). The calls would run this + * launch; verify the live IRAM actually holds the packed image + * at the SA entries first. On a mismatch (the CONT-106 + * all-zero fetch) hold the calls and owe a full re-emission - + * the r1368 evidence says the real CALL runs against a + * non-resident program poison the lane (ILLEGAL_OPCODE at the + * macro entry) instead of drawing. Only meaningful on the + * packed path (the pack image is the comparator). + */ + if (!mme_hold_calls && pl && pl_n && pl_gen && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate)) { + void __iomem *gbar0 = gsp_phys_gpu_bar0(); + + if (gbar0 && + !gsp_phys_gr_mme_iram_gate(gbar0, req->h_object, + pl_gen, "build")) { + mme_hold_calls = true; + gsp_phys_gr_mme_iram_gate_fail(); + } + } + /* * SESSION 153-CONTINUED-63: emit the SA-RAM completion computed * above - a one-instruction end-noop stub in the IR plus one * ONE_INC 0x11c run loading [slot, stub] for every called slot @@ -20174,7 +20542,23 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, pr_info("vgsp-gr: MME-SA-REARM %u slot(s) re-pointed before the window (h=0x%x push=0x%llx)\n", rearm, req->h_object, req->push_va); } - if (mme_hold_calls) { + if (mme_load_only) { + /* + * SESSION 153-CONTINUED-107: no window on a LOAD + * carrier (see the mme_load_only comment). The block + * is SET_OBJECT + prelude/SA/pack + release only; + * the engine can retire it without any DS/shader + * state to validate, so the IRAM load cannot be + * interrupted by the SPH poison. + */ + static u32 load_only_log; + + load_only_log++; + if (load_only_log <= 24u) + pr_info("vgsp-gr: MME-LOAD-ONLY carrier h=0x%x gen=%u window=%u dropped until the IRAM gate verifies (log %u) push=0x%llx\n", + req->h_object, pl_gen, win3d_n, + load_only_log, req->push_va); + } else if (mme_hold_calls) { /* * SESSION 153-CONTINUED-82: the prelude latch for * this generation+channel is not set - elide every @@ -27623,6 +28007,23 @@ gr_page_unmapped: else w1 = READ_ONCE(*(u32 *)(sem_va + 4)); if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) { + /* + * SESSION 153-CONTINUED-107: if THIS launch carried + * the final packed prelude part, gate its residency on + * the IRAM readback BEFORE the break log promotes the + * generation to live (see gsp_phys_gr_mme_iram_gate). + * A failed readback clears the pending gen, the + * prelude latch and both emission cursors: the calls + * stay held and the next launch re-emits from word 0. + */ + if (bar0 && + READ_ONCE(gsp_phys_gr_mme_pending_gen) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) && + !gsp_phys_gr_mme_iram_gate(bar0, req->h_object, + READ_ONCE(gsp_phys_gr_mme_pending_gen), + "break")) + gsp_phys_gr_mme_iram_gate_fail(); /* F4-a: first poll break + USERD witnesses. */ gsp_phys_gr_poll_break_log(w0, want_qmd, w1, poll_start_us, &sub, |