diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 611 |
1 files changed, 606 insertions, 5 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index e43ba23..babd3c0 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -47,6 +47,7 @@ #include <linux/acpi.h> #include <linux/mm.h> #include <linux/vmalloc.h> +#include <linux/spinlock.h> #include <asm/barrier.h> #include <asm/unaligned.h> @@ -5382,6 +5383,18 @@ static void gsp_phys_store_side_log(void __iomem *bar0, const char *tag, gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x1618u >> 2)); } +/* + * eng52 (DS stage-image gate): census of the six stage start VAs latched by + * the DS at a fault transition. Each nonzero START (with its UPPER byte) is + * resolved through the program-pair cache to the engine-visible page, then + * the first SPH word and the first code word (+0x80) are read back through + * the host direct map. One line per DS-FAST launch names exactly what each + * stage would fetch, so "is the authored image engine-visible" is provable + * from the log. Defined after the pair cache below. + */ +static void gsp_phys_ds_image_census(const char *tag, u32 h_object, + const u32 s[6], const u32 u[6]); + static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, u32 h_object, u32 ms_override) { @@ -5571,6 +5584,24 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, rr_a, rr_b, ioread32(bar0 + 0x00404490u)); /* + * eng52: name what each failing stage would + * fetch at its latched start VA (resolved + * through the program-pair cache) - the + * engine-visible proof for the family+0x900 + * VTG authoring. + */ + { + const u32 ds_s[6] = { + s0, s1, s2, s3, s4, s5 + }; + const u32 ds_u[6] = { + u0, u1, u2, u3, u4, u5 + }; + + gsp_phys_ds_image_census(tag, h_object, + ds_s, ds_u); + } + /* * SESSION 153-CONTINUED-106: the pending DS * report poisons the whole PRI block * (every register reads 0xbadf1201 - the @@ -18542,6 +18573,270 @@ static u32 gsp_phys_gr_mme_pack_done_gen; static u32 gsp_phys_gr_mme_pack_done_ch; /* + * c727 HOST-INITIATED PACK-CONTINUATION CARRIER (sema_gl_mme_pack_carry). + * + * The split packed emission only advances when a client launch carries the + * prelude: the final part is emitted by whatever 3D window fits it. c727 + * stalled at `MME-PRELUDE pack-part ir_words=1512/2799 sa=54 split=1` + + * `MME-CALL-HOLD 68` and no later client window ever completed the pack, so + * the calls stayed held for the rest of the boot. c716 had already proven + * the carrier shape: the launch that completed a pack was a final-part, + * window-dropped LOAD-ONLY launch. This state lets the service emit that + * same shape itself: when a partial part was just received and the packed + * program is the deterministic forced-golden image, arm one continuation + * launch for the same beef channel after a short settle delay. The launch + * carries the same golden pack (the builder reads the live cursor) and no + * window, no client pages and no release, so it cannot run a draw or an + * unheld CALL. Bounded: one carrier per partial emission, at most two per + * boot, no retries; every context wipe / generation rewind / load or handle + * change cancels it. Default off (sema_gl_mme_pack_carry=0) for the A/B. + */ +struct gsp_phys_gr_mme_carry_state { + bool armed; + bool fired; + u32 h_object; + u32 gen; + u32 off; + u32 maxa; + u64 origin_push; + unsigned long due_j; + u32 boot_n; +}; + +static struct gsp_phys_gr_mme_carry_state gsp_phys_gr_mme_carry; +static DEFINE_SPINLOCK(gsp_phys_gr_mme_carry_lock); +static bool gsp_phys_rmcall_gr_mme_pack_carry; +module_param_named(sema_gl_mme_pack_carry, gsp_phys_rmcall_gr_mme_pack_carry, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_carry, + "Host-initiated MME pack-continuation carrier: after a partial packed-prelude part, emit the remaining (forced-golden) pack on a windowless LOAD-ONLY launch of the same beef channel when no client window arrives (default 0 = off)"); +static u32 gsp_phys_rmcall_gr_mme_pack_carry_ms = 100u; +module_param_named(sema_gl_mme_pack_carry_ms, + gsp_phys_rmcall_gr_mme_pack_carry_ms, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_carry_ms, + "Settle delay before the host-initiated pack-continuation carrier fires (clamped 20..5000 ms; default 100)"); + +static void gsp_phys_gr_mme_iram_gate_fail(void); + +static u32 gsp_phys_gr_mme_carry_due_ms(void) +{ + u32 ms = READ_ONCE(gsp_phys_rmcall_gr_mme_pack_carry_ms); + + if (ms < 20u) + ms = 20u; + else if (ms > 5000u) + ms = 5000u; + return ms; +} + +/* + * Arm at the partial receipt (the caller logs `MME-PRELUDE pack-part` first). + * Only the deterministic forced-golden image is eligible: the pack cursor is + * global and generation-kept, so the continuation carries the same bytes on + * any launch. A fired carrier whose own emission was still partial is + * re-armed here (that emission is the next partial); the boot-wide budget of + * two fires is what bounds the chain. + */ +static void gsp_phys_gr_mme_carry_arm(bool split, u32 poff, u32 maxa, + u32 gen, u32 h_object, u64 push_va) +{ + struct gsp_phys_gr_mme_carry_state old; + unsigned long flags; + bool superseded = false; + u32 ms; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_carry) || !split || + poff >= maxa || (h_object & 0xfffff000u) != 0xbeef0000u || + READ_ONCE(gsp_phys_gr_mme_pack_load_h) != h_object || + !gsp_phys_gr_golden_ready()) + return; + memset(&old, 0, sizeof(old)); + ms = gsp_phys_gr_mme_carry_due_ms(); + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + /* + * A carrier that already fired and then saw another partial part has + * delivered its launch without completing the pack: this partial is + * that carrier's continuation, so clear the fired mark and arm again + * (bounded by boot_n below). + */ + if (gsp_phys_gr_mme_carry.fired) + gsp_phys_gr_mme_carry.fired = false; + if (gsp_phys_gr_mme_carry.armed) { + old = gsp_phys_gr_mme_carry; + gsp_phys_gr_mme_carry.armed = false; + superseded = true; + } + if (gsp_phys_gr_mme_carry.boot_n >= 2u) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (superseded) + pr_warn("vgsp-gr: MME-CARRY-CANCEL reason=budget h=0x%x gen=%u off=%u/%u (no carrier left this boot)\n", + old.h_object, old.gen, old.off, old.maxa); + return; + } + gsp_phys_gr_mme_carry.armed = true; + gsp_phys_gr_mme_carry.h_object = h_object; + gsp_phys_gr_mme_carry.gen = gen; + gsp_phys_gr_mme_carry.off = poff; + gsp_phys_gr_mme_carry.maxa = maxa; + gsp_phys_gr_mme_carry.origin_push = push_va; + gsp_phys_gr_mme_carry.due_j = jiffies + msecs_to_jiffies(ms); + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (superseded) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=superseded h=0x%x gen=%u off=%u/%u\n", + old.h_object, old.gen, old.off, old.maxa); + pr_info("vgsp-gr: MME-CARRY-ARM h=0x%x gen=%u off=%u/%u push=0x%llx due_ms=%u\n", + h_object, gen, poff, maxa, (unsigned long long)push_va, ms); +} + +/* Due and still armed? Read-only hint for the service pump. */ +bool gsp_phys_gr_mme_carry_peek(u32 *h_object) +{ + unsigned long flags; + bool due; + + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + due = gsp_phys_gr_mme_carry.armed && + time_after_eq(jiffies, gsp_phys_gr_mme_carry.due_j); + if (due && h_object) + *h_object = gsp_phys_gr_mme_carry.h_object; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + return due; +} + +/* + * Claim the armed carrier for one launch. The live state is re-validated + * here, not at arm time: the pack must still be partial for this channel, + * still loaded contiguously from word 0 on it, and the golden overlay must + * still be forced. Any mismatch cancels (never fires a stale carrier). + */ +bool gsp_phys_gr_mme_carry_take(u32 h_object, u32 *gen, u32 *off, u32 *maxa) +{ + struct gsp_phys_gr_mme_carry_state old; + const char *reason = NULL; + unsigned long flags; + u32 live_off; + + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + if (!gsp_phys_gr_mme_carry.armed || + !time_after_eq(jiffies, gsp_phys_gr_mme_carry.due_j)) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + return false; + } + live_off = READ_ONCE(gsp_phys_gr_mme_pack_off); + if (gsp_phys_gr_mme_carry.h_object != h_object || + (h_object & 0xfffff000u) != 0xbeef0000u) + reason = "h-mismatch"; + else if (!gsp_phys_gr_golden_ready()) + reason = "golden-lost"; + else if (READ_ONCE(gsp_phys_gr_mme_pack_load_h) != h_object) + reason = "load-mismatch"; + else if (live_off >= gsp_phys_gr_mme_carry.maxa) + reason = "pack-complete"; + if (reason) { + old = gsp_phys_gr_mme_carry; + gsp_phys_gr_mme_carry.armed = false; + gsp_phys_gr_mme_carry.fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s h=0x%x gen=%u off=%u/%u\n", + reason, old.h_object, old.gen, old.off, old.maxa); + return false; + } + gsp_phys_gr_mme_carry.armed = false; + gsp_phys_gr_mme_carry.fired = true; + gsp_phys_gr_mme_carry.boot_n++; + if (gen) + *gen = gsp_phys_gr_mme_carry.gen; + if (off) + *off = live_off; + if (maxa) + *maxa = gsp_phys_gr_mme_carry.maxa; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + return true; +} + +/* Pack completion: the carrier's proof (or an unrelated lane completed it). */ +static void gsp_phys_gr_mme_carry_complete(u32 h_object, u32 gen, u32 maxa) +{ + struct gsp_phys_gr_mme_carry_state old; + unsigned long flags; + bool fired, armed; + + memset(&old, 0, sizeof(old)); + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + old = gsp_phys_gr_mme_carry; + fired = gsp_phys_gr_mme_carry.fired; + armed = gsp_phys_gr_mme_carry.armed; + gsp_phys_gr_mme_carry.fired = false; + gsp_phys_gr_mme_carry.armed = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (fired && old.h_object == h_object) + pr_info("vgsp-gr: MME-CARRY-COMPLETE h=0x%x gen=%u off=%u/%u push=0x%llx\n", + h_object, gen, maxa, maxa, + (unsigned long long)old.origin_push); + else if (armed) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=pack-complete h=0x%x gen=%u off=%u/%u\n", + old.h_object, old.gen, old.off, old.maxa); + else if (fired) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=cross-lane-complete h=0x%x gen=%u off=%u/%u\n", + old.h_object, old.gen, old.off, old.maxa); +} + +/* Context wipes / generation rewinds / re-registers void the carrier. */ +void gsp_phys_gr_mme_carry_cancel(const char *reason) +{ + struct gsp_phys_gr_mme_carry_state old; + unsigned long flags; + bool had; + + memset(&old, 0, sizeof(old)); + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + had = gsp_phys_gr_mme_carry.armed || gsp_phys_gr_mme_carry.fired; + old = gsp_phys_gr_mme_carry; + gsp_phys_gr_mme_carry.armed = false; + gsp_phys_gr_mme_carry.fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (had) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s h=0x%x gen=%u off=%u/%u\n", + reason ? reason : "unspecified", old.h_object, old.gen, + old.off, old.maxa); +} + +/* + * Launch failure: the carrier never reached (or never completed on) the + * engine. Void it and owe a full re-emission - the pack cursors rewind so the + * next client window re-emits from word 0 and never leaves a false + * completion behind. + */ +void gsp_phys_gr_mme_carry_failed(int rc) +{ + struct gsp_phys_gr_mme_carry_state old; + unsigned long flags; + bool had; + + memset(&old, 0, sizeof(old)); + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + had = gsp_phys_gr_mme_carry.armed || gsp_phys_gr_mme_carry.fired; + old = gsp_phys_gr_mme_carry; + gsp_phys_gr_mme_carry.armed = false; + gsp_phys_gr_mme_carry.fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (!had) + return; + pr_warn("vgsp-gr: MME-CARRY-ABORT reason=hw-error rc=%d h=0x%x gen=%u off=%u/%u\n", + rc, old.h_object, old.gen, old.off, old.maxa); + gsp_phys_gr_mme_iram_gate_fail(); +} + +/* + * The carrier's windowless launch body: WAIT_FOR_IDLE(0x110) <- 0. Filled + * into the submit path's per-launch stream_patch scratch (writable: the + * pair-chain pass may prepend a closing END if a pair is carried open, and + * the window is dropped either way by the forced LOAD-ONLY decision). + */ +#define GSP_PHYS_GR_MME_CARRIER_WAIT_HDR 0x20010044u +#define GSP_PHYS_GR_MME_CARRIER_WAIT_DATA 0x00000000u + +/* * CONT-103: prelude residency witness + calls-kept block shrink. * The CONT-79 split-emission re-carries the whole prelude prefix on EVERY * launch while keep_mask is set (idempotent re-emission, CONT-72), so the @@ -19986,6 +20281,28 @@ module_param_named(sema_gl_sph_vtg_j1, gsp_phys_rmcall_gr_sph_vtg_j1, MODULE_PARM_DESC(sema_gl_sph_vtg_j1, "Also point the window's own slot-1 (VTG) bind at family+0x600 instead of the client's captured bind (default 0; c718 poison A/B)"); +/* + * eng52 (DS stage-image gate): the live direct-GL VTG SPH image is delivered + * by the client as an inline record (`req->inl[]`, 640 B, first word + * 0x020a0481, w6=0x0000f00f, w13=0x0000f000, w17=0x000f0000). At DS time + * the client's own program copy has overwritten the bind page (family+0x600 + * in the c716 class) with SASS, so binding that VA poisons the DS + * (c718/c720 badf1201). family+0x900 inside the same family page is zero + * and unused, and the engine can reach it through the same relocated page. + * Author the captured VTG image there, then point BOTH the synthesized VSA + * (j0) and the window's own VSB (j1) at it: j0 and j1 fetch a real + * V4T1 SHADER_TYPE=1 image while j2 stays at +0x200, the GS image at + * +0xe80 and the pixel image at +0x000. The copy is last (after the + * page-loop inline replay) so no later replay can overwrite it, and it is + * refused with a named receipt if another inline record already targets the + * destination. A/B with 0 (legacy +0x900 zero page / +0x600 selection). + */ +static bool gsp_phys_rmcall_gr_sph_vtg_auth = true; +module_param_named(sema_gl_sph_vtg_auth, gsp_phys_rmcall_gr_sph_vtg_auth, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_vtg_auth, + "Author the client's live VTG SPH image at family+0x900 and bind the synthesized VSA (j0) and window VSB (j1) there (default 1)"); + bool gsp_phys_rmcall_gr_sph_bind_value(void) { return READ_ONCE(gsp_phys_rmcall_gr_sph_bind); @@ -20689,6 +21006,7 @@ static u32 gsp_phys_rmcall_gr_mme_iram_gate_max = 24u; static void gsp_phys_gr_mme_iram_gate_fail(void) { gsp_phys_gr_mme_iram_gate_fails++; + gsp_phys_gr_mme_carry_cancel("iram-gate-fail"); WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); @@ -20926,6 +21244,7 @@ static u32 gsp_phys_gr_fe_status(void) */ static void gsp_phys_gr_ctx_wipe_latches(void) { + gsp_phys_gr_mme_carry_cancel("ctx-wipe"); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); @@ -20978,6 +21297,7 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) if (h != READ_ONCE(gsp_phys_gr_mme_pack_load_h) && h != READ_ONCE(gsp_phys_gr_mme_pack_done_h)) return; + gsp_phys_gr_mme_carry_cancel("chan-ctx-reset"); /* * eng43: sema_gl_mme_keep_done_across_recycle=1 - the re-register * recycles the client handle but leaves the GR engine context (and @@ -21346,6 +21666,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, bool mme_load_only = false; bool latch_before = false; bool stream_damaged = false; + bool carrier = req && req->mme_carrier; u32 pre_words_out = 0; u32 win_emit_off = 0; const u32 *pl = NULL; @@ -21710,6 +22031,7 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, gsp_phys_rmcall_gr_mme_pack_keep_gen) && READ_ONCE( gsp_phys_rmcall_gr_mme_golden_force))) { + gsp_phys_gr_mme_carry_cancel("gen-rewind"); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); @@ -21757,6 +22079,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, } } } + /* + * Host-initiated carrier: it exists only to carry the next pack + * part, so never let the latch/room heuristics defer the + * emission (the arm-time checks already proved the pack is + * partial for this channel and generation). + */ + if (carrier && pl && pl_n && pl_gen) + emit_prelude = true; *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | (0u << 13) | @@ -22793,6 +23123,9 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, packed = true; if (poff >= maxa) { doff = pl_n; + gsp_phys_gr_mme_carry_complete( + req->h_object, + pl_gen, maxa); pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", maxa, nruns, npair, @@ -22805,6 +23138,20 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, split ? 1u : 0u, req->h_object, req->push_va); + /* + * c727: the part left + * the pack partial. Arm + * one host-initiated + * continuation for this + * beef channel (bounded + * and default-off; see + * sema_gl_mme_pack_carry). + */ + gsp_phys_gr_mme_carry_arm( + split, poff, maxa, + pl_gen, + req->h_object, + req->push_va); } } else if (split && maxa >= 3u) { /* @@ -22977,6 +23324,15 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, mme_load_only = true; } /* + * Host-initiated carrier: windowless LOAD-ONLY by construction + * (no client window, no pages, no release). The stub window + * the submit path hands the builder is never emitted; it only + * keeps the launch in this 3D branch and gives the block + * arithmetic a nonzero window size. + */ + if (carrier) + mme_load_only = true; + /* * SESSION 153-CONTINUED-82: hold real CALL runs until the * prelude latch for this generation+channel is set (see the * sema_gl_mme_hold_calls comment). Evaluated AFTER the @@ -23147,6 +23503,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, } } /* + * Host-initiated carrier: it carries no draw at all, so it can + * never run the window's CALL runs - force the held shape even + * if the latch/release heuristics would release them on this + * launch. + */ + if (carrier) + mme_hold_calls = true; + /* * eng40/eng42: post-completion re-arm. Once the packed * program completed (deterministic golden image), the engine * context keeps it until a wipe. Every later frame bumps the @@ -26863,6 +27227,53 @@ static u64 gsp_phys_gr_pair_va[GSP_PHYS_RMCALL_GR_PAIR_CACHE]; static u32 gsp_phys_gr_pair_n; /* + * eng52 (DS stage-image gate): census of the six stage start VAs latched by + * the DS at a fault transition (see the forward declaration). A stage START + * register holds the low 32 bits (including the in-page offset) and the + * UPPER register bits 39:32, so the VA is (upper & 0xff) << 32 | start. + * Resolve it through the program-pair cache (the same 8K line the + * relocation pass uses for the family page), then read the first SPH word + * and the first code word (+0x80) through the host direct map of the + * engine-visible HPA. Zero when the VA is unset or not in the pair cache. + * Diagnostic only; the pair arrays are written under gsp_phys_rm_lock, so + * READ_ONCE keeps the census from tearing a line during a live submit. + */ +static void gsp_phys_ds_image_census(const char *tag, u32 h_object, + const u32 s[6], const u32 u[6]) +{ + u32 w0[6] = { 0 }, code[6] = { 0 }; + u32 si, pi, n = READ_ONCE(gsp_phys_gr_pair_n); + + for (si = 0; si < 6u; si++) { + u64 va = ((u64)(u[si] & 0xffu) << 32) | (u64)s[si]; + + if (!va) + continue; + for (pi = 0; + pi < n && pi < GSP_PHYS_RMCALL_GR_PAIR_CACHE; pi++) { + u64 pv = READ_ONCE(gsp_phys_gr_pair_va[pi]); + u64 ph; + + if (!pv || va < pv || + va >= pv + 2u * GSP_PHYS_PAGE_SIZE) + continue; + ph = READ_ONCE(gsp_phys_gr_pair_hpa[pi]) + (va - pv); + if (pfn_valid((unsigned long)(ph >> PAGE_SHIFT))) { + w0[si] = get_unaligned_le32( + (const u8 *)__va(ph)); + code[si] = get_unaligned_le32( + (const u8 *)__va(ph) + 0x80u); + } + break; + } + } + pr_info_ratelimited("vgsp-gr: DS-IMAGE %s h=0x%x st0 w0=%08x code=%08x st1 w0=%08x code=%08x st2 w0=%08x code=%08x st3 w0=%08x code=%08x st4 w0=%08x code=%08x st5 w0=%08x code=%08x\n", + tag, h_object, + w0[0], code[0], w0[1], code[1], w0[2], code[2], + w0[3], code[3], w0[4], code[4], w0[5], code[5]); +} + +/* * Resolve the HPA of `page0 + 4K` through the same sources the relocation * chain uses: the launch's own page set (the pages the copy-in authored), * the tracked data recs, the sema service's trusted resolver, then the @@ -29075,6 +29486,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n; /* + * Host-initiated MME pack-continuation carrier: no guest window and no + * guest memory. The window is a static WAIT_FOR_IDLE stub (never + * emitted - the builder forces LOAD-ONLY) and every capture/relocation + * prepend below is skipped. + */ + bool carrier = req && req->mme_carrier; + /* * CONT-73: fresh direct-write witness for this submission. The * window/attr patch passes below record every relocated operand * destination's guest page; the pre-doorbell capture and the retire @@ -29266,7 +29684,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * into the same software-completion fallback without touching the * engine, and cannot wedge it for the next client. */ - if (req->n_data < gsp_phys_gr_min_data_value()) { + if (!carrier && req->n_data < gsp_phys_gr_min_data_value()) { gsp_phys_gr_refused++; pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", req->n_data, @@ -30610,7 +31028,21 @@ gr_page_unmapped: * FAULT_PDE on a Tier-1-mapped guest VA). Runs with the rm lock * held, before the method block is built. */ - if (stream3d) { + if (stream3d && carrier) { + /* + * Host-initiated pack-continuation carrier: the request has no + * guest window at all. Hand the builder a WAIT_FOR_IDLE stub + * (in the per-launch writable patch scratch) so it stays on + * the 3D path; the builder forces LOAD-ONLY (window dropped, + * CALL runs held), so these two words are never executed. No + * capture relocation, no attr/SURF prepend, no SPH-BIND/ + * SPH-AUTHOR: there is no guest memory in this launch. + */ + stream_patch[0] = GSP_PHYS_GR_MME_CARRIER_WAIT_HDR; + stream_patch[1] = GSP_PHYS_GR_MME_CARRIER_WAIT_DATA; + win3d = stream_patch; + win3d_n = 2; + } else if (stream3d) { u32 win_n = req->stream_n; u32 ndrop = 0; u32 nrel; @@ -30689,11 +31121,175 @@ gr_page_unmapped: static u32 sph_bind_log; u64 fam = req->prog_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); - u64 vtg_va = READ_ONCE(gsp_phys_rmcall_gr_sph_vtg600) ? - (fam + 0x600u) : (fam + 0x900u); + u64 vtg_va; + bool vtg_auth = false; u32 in[96], out[96]; u32 nfin = 0, nrun = 0, on = 0, odrop = 0, orel; u32 bi; + /* + * eng52 (DS stage-image gate): author the client's live + * VTG SPH image at family+0x900. The image arrives as an + * inline record on the family page (first word 0x020a0481 + * = V4T1 SHADER_TYPE=1), but the client's own program copy + * overwrites the bind page (family+0x600 in the c716 + * class) with SASS by DS time, so binding that VA poisons + * the DS. family+0x900 is zero and unused; write the + * captured image there through the same mapped-HPA path + * the page-loop inline replay uses (pg->hpa is the + * direct-mappable host page for both AP=FB shadow and + * sysmem), then bind j0/j1 to it below. The copy is last, + * after the page-loop replay, so nothing overwrites it. + * Refuse with a named receipt if another inline record + * already targets the destination (never corrupt it). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_vtg_auth) && fam) { + static u32 sph_auth_log; + const struct gsp_phys_gr_inline *vinl = NULL; + const struct gsp_phys_gr_page *vpg = NULL; + const u8 *vsrc = NULL; + u32 vlen = 0, vj = 0, vpi; + bool vbuiltin = false; + u32 vj2; + + for (vj2 = 0; vj2 < req->n_inl; vj2++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[vj2]; + u32 w0; + + if ((inl->dst_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + fam) + continue; + if (inl->len < 4u || + inl->len > GSP_PHYS_GR_INLINE_BYTES) + continue; + w0 = get_unaligned_le32(inl->data); + if ((w0 & 0x1fu) != 1u || + ((w0 >> 10) & 0xfu) != 1u) + continue; + vinl = inl; + vj = vj2; + } + if (vinl) { + vsrc = vinl->data; + vlen = vinl->len; + } else { + vsrc = gsp_sema_gl_vs_img(&vlen); + vbuiltin = true; + } + if (vlen > GSP_PHYS_PAGE_SIZE - 0x900u) + vlen = GSP_PHYS_PAGE_SIZE - 0x900u; + for (vpi = 0; vpi < req->n_pages; vpi++) { + if ((req->pages[vpi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + fam || + !req->pages[vpi].hpa) + continue; + vpg = &req->pages[vpi]; + break; + } + if (!vsrc || !vlen || !vpg) { + if (sph_auth_log < 64u) { + sph_auth_log++; + pr_info("vgsp-gr: SPH-AUTHOR skip=%s fam=0x%llx h=0x%x push=0x%llx\n", + !vsrc ? "no-src" : + !vlen ? "no-len" : + "no-page", fam, + req->h_object, + req->push_va); + } + } else { + bool voverlap = false; + + for (vj2 = 0; vj2 < req->n_inl; vj2++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[vj2]; + u64 io, ie; + + if ((inl->dst_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + fam) + continue; + io = inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1); + ie = io + inl->len; + if (io < 0x900u + vlen && + 0x900u < ie) + voverlap = true; + } + if (voverlap) { + if (sph_auth_log < 64u) { + sph_auth_log++; + pr_info("vgsp-gr: SPH-AUTHOR skip=overlap fam=0x%llx dst=0x%llx len=%u h=0x%x push=0x%llx\n", + fam, fam + 0x900u, + vlen, + req->h_object, + req->push_va); + } + } else if (!pfn_valid((unsigned long) + (vpg->hpa >> + PAGE_SHIFT))) { + if (sph_auth_log < 64u) { + sph_auth_log++; + pr_info("vgsp-gr: SPH-AUTHOR skip=no-hpa fam=0x%llx dst=0x%llx len=%u hpa=0x%llx h=0x%x push=0x%llx\n", + fam, fam + 0x900u, + vlen, vpg->hpa, + req->h_object, + req->push_va); + } + } else { + u32 vfirst = get_unaligned_le32(vsrc); + u32 vw6 = vlen >= 28u ? + get_unaligned_le32(vsrc + 24u) : 0u; + u32 vw13 = vlen >= 56u ? + get_unaligned_le32(vsrc + 52u) : 0u; + u32 vw17 = vlen >= 72u ? + get_unaligned_le32(vsrc + 68u) : 0u; + + memcpy((u8 *)__va(vpg->hpa) + + 0x900u, + vsrc, vlen); + dma_wmb(); + vtg_auth = true; + if (sph_auth_log < 64u) { + sph_auth_log++; + if (vbuiltin) + pr_info("vgsp-gr: SPH-AUTHOR fam=0x%llx dst=0x%llx len=%u src=builtin first=%08x w6=%08x w13=%08x w17=%08x h=0x%x push=0x%llx\n", + fam, fam + 0x900u, + vlen, vfirst, vw6, + vw13, vw17, + req->h_object, + req->push_va); + else + pr_info("vgsp-gr: SPH-AUTHOR fam=0x%llx dst=0x%llx len=%u src=inl[%u] first=%08x w6=%08x w13=%08x w17=%08x h=0x%x push=0x%llx\n", + fam, fam + 0x900u, + vlen, vj, vfirst, + vw6, vw13, vw17, + req->h_object, + req->push_va); + } + /* + * Read the authored words back + * through the direct map (shadow + * is not chosen until the + * relocation pass, hence 0). + */ + gsp_phys_gr_sph_alias_receipt( + fam, fam + 0x900u, 0, + vpg->hpa, "auth"); + } + } + } + /* + * Keep the historical +0x600 selection only when the + * authoring knob is off (or the copy was refused): a + * successful authoring lives at +0x900 and both j0 and + * j1 must bind there. + */ + vtg_va = (vtg_auth || + !READ_ONCE(gsp_phys_rmcall_gr_sph_vtg600)) ? + (fam + 0x900u) : (fam + 0x600u); + bool bind_binding = READ_ONCE(gsp_phys_rmcall_gr_sph_bind_binding); bool pair_safe = @@ -30997,9 +31593,13 @@ gr_page_unmapped: * 0x0d0d7623); the VTG SPH image is at * family+0x600. Point the VTG slot at the * real image when the knob is on. + * eng52: an authored VTG image at +0x900 is + * always the better operand, so the override + * activates whenever the authoring succeeded. */ if (sl == 1u && - READ_ONCE(gsp_phys_rmcall_gr_sph_vtg_j1)) + (READ_ONCE(gsp_phys_rmcall_gr_sph_vtg_j1) || + vtg_auth)) va = vtg_va; in[nfin++] = 0x20000000u | (2u << 16) | ((0x2014u + sl * 0x40u) >> 2); @@ -32853,6 +33453,7 @@ gr_page_unmapped: mutex_unlock(&gsp_phys_rm_lock); park_rec_n++; if (!sp_ret) { + gsp_phys_gr_mme_carry_cancel("park-recover"); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); |