diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 287 |
1 files changed, 281 insertions, 6 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 9d5fb71..b9e247c 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -18412,6 +18412,179 @@ MODULE_PARM_DESC(sema_gl_mme_nop_pair, "Elide program-address pair runs (0x2054/0x2094/0x2154 families) from a calls-kept 3D window (default 0; the engine traps 0x2058 on the relocated shadow operand)"); /* + * r1453 PAIR-CHAIN (sema_gl_pair_chain): the offline EC 0x0d decode + * (BEGIN_END_ACTIVE, nv50.c:173) and the r1452b window dumps show the + * guest's draws arriving as back-to-back normal/normal BEGIN (0x1618) + * ... END (0x1614) pairs with no WAIT_FOR_IDLE (0x110) / SEND_GO_IDLE + * (0x13c) drain between them, and the FE then refuses the closing END + * (trap 0x80001614, Data 0, EC 0x0d) - and every later state write + * (0x2058 PROGRAM_ADDRESS_B) while a pair stays active. + * + * The guest's BEGIN data word is carried VERBATIM end to end (the + * window is a raw-push slice: svc_gl_flush -> req->stream -> + * gsp_phys_rmcall_gr_3d_patch_window memcpy). Nothing in the kernel + * tree constructs or normalises a 0x586/0x585 word: the only 32-bit + * BEGIN data form is a count-carrying run at method 0x1618 whose first + * payload word is the full data (SPLIT_MODE is bits 30:29 of THAT + * word); the 13-bit IMMD header form (op 4, data 8/7) cannot express + * the split bits and is what the observed raw pairs carry. This pass + * therefore never rebuilds a run: it (a) receipts the full BEGIN data + * word it carries, (b) inserts ONE WAIT_FOR_IDLE<-0 (INC 0x110, 1 data + * word) ahead of a following normal-begin pair so each END closes a + * drained pair, and (c) leaves split-mode chains (BEGIN 1/2/3) alone. + * 0 = legacy bytes (pass not run; held-call elision unchanged). + */ +static bool gsp_phys_rmcall_gr_pair_chain = true; +module_param_named(sema_gl_pair_chain, gsp_phys_rmcall_gr_pair_chain, + bool, 0644); +MODULE_PARM_DESC(sema_gl_pair_chain, + "One logical BEGIN..END chain per launch: carry the full 32-bit BEGIN data (SPLIT_MODE 30:29), insert WAIT_FOR_IDLE(0x110) drains between back-to-back normal/normal pairs, and extend pair elision to held-call windows (default 1; 0 = legacy bytes)"); +static bool gsp_phys_gr_mme_pairs_emitted; /* launch emitted a pair run */ +static u32 gsp_phys_gr_mme_pairs_gen; /* gen whose pairs retired OK */ +static u32 gsp_phys_gr_mme_launch_gen; /* last built launch's gen */ + +/* + * True when a method run in [w, w+n) writes NVC597_BEGIN (0x1618) or + * NVC597_END (0x1614) - the per-launch proof the pair-elision extension + * gates on (pairs must have reached the FE once for the generation). + */ +static bool gsp_phys_gr_pair_present(const u32 *w, u32 n) +{ + u32 r = 0; + + while (r < n) { + u32 h = w[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u; + u32 m = (h & 0xfffu) << 2; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + if ((op == 1u || op == 3u || op == 4u || op == 5u) && + (m == 0x1618u || m == 0x1614u)) + return true; + r += span; + } + return false; +} + +/* + * Apply the pair-chain rule to the final emitted window (prepends + * included). Returns the number of inserted drains; *np is updated. + * The scan is method-run aligned and insertion copies whole runs, so a + * run is never split and the BEGIN data word is never rewritten. + */ +static u32 gsp_phys_gr_pair_chain_fix(u32 *w, u32 *np, u32 cap) +{ + u32 n = *np, r = 0, begins = 0, ends = 0, drains = 0; + u32 chains = 0, orphans = 0, nested = 0, badv = 0; + bool open = false, seen = false, begin_logged = false; + u32 open_split = 0; + + while (r < n) { + u32 h = w[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u; + u32 m = (h & 0xfffu) << 2; + bool is_begin = false, is_end = false; + u32 data = 0, split = 0, ev = 0; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + if (op == 4u) { + if (m == 0x1618u) { + is_begin = true; + data = (h >> 16) & 0x1fffu; + } else if (m == 0x1614u) { + is_end = true; + ev = (h >> 16) & 1u; + } + } else if ((op == 1u || op == 3u || op == 5u) && cnt && + m == 0x1618u) { + is_begin = true; + data = w[r + 1u]; + split = (data >> 29) & 3u; + } else if ((op == 1u || op == 3u || op == 5u) && cnt && + m == 0x1614u) { + is_end = true; + ev = w[r + 1u] & 1u; + } + if (is_begin) { + if (!begin_logged) { + begin_logged = true; + pr_info_ratelimited("vgsp-gr: PAIR-BEGIN-DATA off=%u data=%08x split=%u form=%s (full 32-bit data carried verbatim)\n", + r, data, split, + (op == 4u) ? "immd13(no-split)" : + "run32"); + } + if (!open && seen && split == 0u) { + /* + * EC 0x0d: a normal/normal pair begun while + * the previous pair is not drained is refused + * at its END. Insert the class's own stall + * (WAIT_FOR_IDLE <- 0) ahead of it. + */ + if (n + 2u <= cap) { + memmove(&w[r + 2u], &w[r], + (size_t)(n - r) * 4u); + w[r] = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | (1u << 16) | + (0x110u >> 2); + w[r + 1u] = 0u; + n += 2u; + r += 2u; + drains++; + pr_info_ratelimited("vgsp-gr: PAIR-DRAIN inserted WAIT_FOR_IDLE(0x110)<-0 before pair #%u at off=%u window=%u\n", + begins + 1u, r, n); + } else { + pr_warn_ratelimited("vgsp-gr: PAIR-DRAIN skipped (window=%u cap=%u): back-to-back pairs can still raise EC0x0d\n", + n, cap); + } + } + if (open) { + /* + * Legal only as a split chain (the previous + * BEGIN left its end open). + */ + if (open_split == 1u || open_split == 2u) + chains++; + else + nested++; + } else { + open = true; + seen = true; + } + open_split = split; + begins++; + } else if (is_end) { + if (!open) { + orphans++; + } else { + u32 expect = ((open_split == 1u || + open_split == 2u) ? 1u : 0u); + + if (ev != expect) + badv++; + open = false; + ends++; + } + } + r += span; + } + if (begins > 1u || drains || chains || orphans || nested || badv || + open) + pr_info_ratelimited("vgsp-gr: PAIR-CHAIN begin=%u end=%u drains=%u chain=%u orphan_end=%u nested=%u endv_mismatch=%u open_at_end=%u window=%u\n", + begins, ends, drains, chains, orphans, nested, + badv, open ? 1u : 0u, n); + *np = n; + return drains; +} + +/* * SESSION 153-CONTINUED-79: parameter-conserving NOP for kept parking slots. * The r1177 bisect: keep={0,2,57} (full 3089-word program resident, split * emission) parks the PBDMA on the emitted block (MME-FAST events=0, grfifo @@ -19775,6 +19948,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, bool mme_load_only = false; bool latch_before = false; u32 pre_words_out = 0; + u32 win_emit_off = 0; const u32 *pl = NULL; u32 pl_n = 0, pl_gen = 0, pl_room = 0; u32 block_cap = READ_ONCE(gsp_phys_rmcall_gr_block_cap); @@ -20017,6 +20191,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) { pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen); /* + * r1453 PAIR-CHAIN: remember the generation this + * launch was built against; the submit's success + * edge promotes it to gsp_phys_gr_mme_pairs_gen + * once a pair-bearing window retires (the held + * elision extension's install proof). + */ + gsp_phys_gr_mme_launch_gen = pl_gen; + /* * SESSION 153-CONTINUED-72: with keep_mask bits set a * kept macro can execute, and it needs the MME program * resident. The r1152 evidence: the prelude was carried @@ -21577,6 +21759,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); mme_hold_window = false; } + /* + * r1453 PAIR-CHAIN: the window was already drained in the + * submit path (after the SPH-BIND/SURF-PRELUDE prepends, + * once per launch - see gsp_phys_gr_pair_chain_fix). The + * emitted-pair proof is recomputed from the final branch + * output below. + */ + win_emit_off = (u32)(p - pb); if (mme_load_only || mme_hold_window) { if (mme_hold_window && !mme_load_only) { pr_info_ratelimited("vgsp-gr: MME-WINDOW-HOLD gen=%u h=%#x window=%u dropped (r1447 arming hold) push=0x%llx\n", @@ -21611,21 +21801,46 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, * the MME macro method space 0x3800..0x39fc are the * ones that execute a macro; the drop decision is * the same walk gsp_phys_gr_mme_drop_calls uses. + * + * r1453 PAIR-CHAIN: the held window is the 0x2058 + * "other writer" (r1453 strategy ยง3.2) - it copies + * everything except CALL runs, including the + * program-address pair runs, and the r1452b #33 + * launch then traps 0x2058 at Data 0x19b0900 / + * EC 0x0d. Extend the sema_gl_mme_nop_pair elision + * here ONLY once a pair-bearing window of this + * same generation has retired (pairs_gen == pl_gen, + * set on the submit success edge): eliding before + * that starves the binds (r1452 D1, HUBCLIENT_PD + * @ 0x0). The elision still requires the nop_pair + * opt-in, so the default held shape is unchanged. */ - u32 r = 0, w = 0, held = 0; + u32 r = 0, w = 0, held = 0, pelided = 0; + bool pair_installed = + READ_ONCE(gsp_phys_rmcall_gr_mme_nop_pair) && + READ_ONCE(gsp_phys_rmcall_gr_pair_chain) && + pl_gen && READ_ONCE(gsp_phys_gr_mme_pairs_gen) == + pl_gen; while (r < win3d_n) { u32 h = win3d[r]; u32 op = (h >> 29) & 7u; u32 cnt = (h >> 16) & 0xfffu; u32 span = 1u; - bool iscall = false; - u32 k; + bool iscall = false, ispair = false; + u32 k, m0, mend; if ((op == 1u || op == 3u || op == 5u) && cnt && r + 1u + cnt <= win3d_n) span = 1u + cnt; - for (k = 0; k < span; k++) { + m0 = (h & 0xfffu) << 2; + mend = m0 + 4u * (span - 1u); + if (pair_installed && + (op == 0u || op == 1u || op == 3u || + op == 4u || op == 5u) && + m0 <= 0x2158u && mend >= 0x2054u) + ispair = true; + for (k = 0; k < span && !ispair; k++) { u32 m = ((h & 0xfffu) << 2) + 4u * k; @@ -21636,8 +21851,11 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, break; } } - if (iscall) { - held += span; + if (iscall || ispair) { + if (iscall) + held += span; + if (ispair) + pelided += span; r += span; continue; } @@ -21651,6 +21869,10 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, win3d_n, w, req->push_va, mme_hold_fe_busy ? " [FE-not-idle (CONT-104)]" : ""); + if (pelided) + pr_info_ratelimited("vgsp-gr: PAIR-CHAIN held-elide %u word(s) of program-pair runs dropped (pairs installed gen=%u h=0x%x; 0x2058 other-writer) push=0x%llx\n", + pelided, pl_gen, req->h_object, + req->push_va); p += w; } else if (READ_ONCE(gsp_phys_rmcall_gr_mme_nop_pair)) { /* @@ -21705,6 +21927,18 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, p += win3d_n; } /* + * r1453 PAIR-CHAIN: the install proof for the held-call + * elision extension. Recomputed from the FINAL emitted window + * (after the CALL/pair elision branches above), so a launch + * that dropped every pair never marks its generation as + * pair-installed. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_pair_chain) && + (u32)(p - pb) > win_emit_off) + gsp_phys_gr_mme_pairs_emitted = + gsp_phys_gr_pair_present(&pb[win_emit_off], + (u32)(p - pb) - win_emit_off); + /* * SESSION 153-CONTINUED-84: the SA re-arm block used to be * emitted HERE (after the window). It was moved ahead of the * window above; see that block's comment for the r1236 @@ -26554,6 +26788,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, fast_rearmed = false; deposit_rb_done = false; + gsp_phys_gr_mme_pairs_emitted = false; /* r1453 PAIR-CHAIN */ + gsp_phys_gr_mme_launch_gen = 0; gsp_phys_gr_codehash_done = false; /* CODEHASH: once per launch */ /* * M91 FAULT-CAP: snapshot the fault epoch per launch, so the capture @@ -28356,6 +28592,21 @@ gr_page_unmapped: } } + /* + * r1453 PAIR-CHAIN: drain the final window ONCE per launch, after + * every prepend (SPH-BIND, SURF-PRELUDE) and the relocation pass, + * so the same bytes feed both the first build and the outcome + * retry. The pass only inserts WAIT_FOR_IDLE(0x110) runs ahead of + * back-to-back normal/normal BEGIN pairs; the guest's BEGIN data + * words are carried verbatim. Knob 0 = no pass (legacy bytes). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_pair_chain) && win3d && win3d_n) { + u32 pn = win3d_n; + + (void)gsp_phys_gr_pair_chain_fix((u32 *)win3d, &pn, + GSP_PHYS_RMCALL_GR_METHOD_WORDS); + win3d_n = pn; + } words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, ch_sem_va, want_ch, @@ -29019,6 +29270,9 @@ gr_page_unmapped: WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); + /* r1453 PAIR-CHAIN: context wipe voids the + * pair-install proof; re-earn it. */ + WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0); /* * r1447: the wiped context voids the arming * window-hold and the strict grace budget too. @@ -29155,6 +29409,21 @@ gr_page_unmapped: w1 = READ_ONCE(*(u32 *)(sem_va + 4)); if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) { /* + * r1453 PAIR-CHAIN: this launch retired. If its + * emitted window carried BEGIN/END runs, the pair + * runs are installed on the channel for this + * prelude generation - the proof the held-call + * elision extension waits for. A context wipe + * (WEDGE-RECOVER / park recovery) clears this so a + * later generation re-earns it from a drain-carrying + * window. + */ + if (gsp_phys_gr_mme_pairs_emitted && + gsp_phys_gr_mme_launch_gen && + READ_ONCE(gsp_phys_rmcall_gr_pair_chain)) + WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, + gsp_phys_gr_mme_launch_gen); + /* * SESSION 153-CONTINUED-107: if THIS launch carried * the final packed prelude part, gate its residency on * the IRAM readback BEFORE the break log promotes the @@ -29364,6 +29633,12 @@ gr_page_unmapped: WRITE_ONCE( gsp_phys_gr_mme_release_h, 0); + /* r1453 PAIR-CHAIN: same + * context wipe - the pair + * install proof is void. */ + WRITE_ONCE( + gsp_phys_gr_mme_pairs_gen, + 0); /* * r1447: same context wipe - * void the arming window-hold |