diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 611 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 25 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 537 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 9 |
4 files changed, 1155 insertions, 27 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index e43ba23..babd3c0 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -47,6 +47,7 @@ #include <linux/acpi.h> #include <linux/mm.h> #include <linux/vmalloc.h> +#include <linux/spinlock.h> #include <asm/barrier.h> #include <asm/unaligned.h> @@ -5382,6 +5383,18 @@ static void gsp_phys_store_side_log(void __iomem *bar0, const char *tag, gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x1618u >> 2)); } +/* + * eng52 (DS stage-image gate): census of the six stage start VAs latched by + * the DS at a fault transition. Each nonzero START (with its UPPER byte) is + * resolved through the program-pair cache to the engine-visible page, then + * the first SPH word and the first code word (+0x80) are read back through + * the host direct map. One line per DS-FAST launch names exactly what each + * stage would fetch, so "is the authored image engine-visible" is provable + * from the log. Defined after the pair cache below. + */ +static void gsp_phys_ds_image_census(const char *tag, u32 h_object, + const u32 s[6], const u32 u[6]); + static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, u32 h_object, u32 ms_override) { @@ -5571,6 +5584,24 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, rr_a, rr_b, ioread32(bar0 + 0x00404490u)); /* + * eng52: name what each failing stage would + * fetch at its latched start VA (resolved + * through the program-pair cache) - the + * engine-visible proof for the family+0x900 + * VTG authoring. + */ + { + const u32 ds_s[6] = { + s0, s1, s2, s3, s4, s5 + }; + const u32 ds_u[6] = { + u0, u1, u2, u3, u4, u5 + }; + + gsp_phys_ds_image_census(tag, h_object, + ds_s, ds_u); + } + /* * SESSION 153-CONTINUED-106: the pending DS * report poisons the whole PRI block * (every register reads 0xbadf1201 - the @@ -18542,6 +18573,270 @@ static u32 gsp_phys_gr_mme_pack_done_gen; static u32 gsp_phys_gr_mme_pack_done_ch; /* + * c727 HOST-INITIATED PACK-CONTINUATION CARRIER (sema_gl_mme_pack_carry). + * + * The split packed emission only advances when a client launch carries the + * prelude: the final part is emitted by whatever 3D window fits it. c727 + * stalled at `MME-PRELUDE pack-part ir_words=1512/2799 sa=54 split=1` + + * `MME-CALL-HOLD 68` and no later client window ever completed the pack, so + * the calls stayed held for the rest of the boot. c716 had already proven + * the carrier shape: the launch that completed a pack was a final-part, + * window-dropped LOAD-ONLY launch. This state lets the service emit that + * same shape itself: when a partial part was just received and the packed + * program is the deterministic forced-golden image, arm one continuation + * launch for the same beef channel after a short settle delay. The launch + * carries the same golden pack (the builder reads the live cursor) and no + * window, no client pages and no release, so it cannot run a draw or an + * unheld CALL. Bounded: one carrier per partial emission, at most two per + * boot, no retries; every context wipe / generation rewind / load or handle + * change cancels it. Default off (sema_gl_mme_pack_carry=0) for the A/B. + */ +struct gsp_phys_gr_mme_carry_state { + bool armed; + bool fired; + u32 h_object; + u32 gen; + u32 off; + u32 maxa; + u64 origin_push; + unsigned long due_j; + u32 boot_n; +}; + +static struct gsp_phys_gr_mme_carry_state gsp_phys_gr_mme_carry; +static DEFINE_SPINLOCK(gsp_phys_gr_mme_carry_lock); +static bool gsp_phys_rmcall_gr_mme_pack_carry; +module_param_named(sema_gl_mme_pack_carry, gsp_phys_rmcall_gr_mme_pack_carry, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_carry, + "Host-initiated MME pack-continuation carrier: after a partial packed-prelude part, emit the remaining (forced-golden) pack on a windowless LOAD-ONLY launch of the same beef channel when no client window arrives (default 0 = off)"); +static u32 gsp_phys_rmcall_gr_mme_pack_carry_ms = 100u; +module_param_named(sema_gl_mme_pack_carry_ms, + gsp_phys_rmcall_gr_mme_pack_carry_ms, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_carry_ms, + "Settle delay before the host-initiated pack-continuation carrier fires (clamped 20..5000 ms; default 100)"); + +static void gsp_phys_gr_mme_iram_gate_fail(void); + +static u32 gsp_phys_gr_mme_carry_due_ms(void) +{ + u32 ms = READ_ONCE(gsp_phys_rmcall_gr_mme_pack_carry_ms); + + if (ms < 20u) + ms = 20u; + else if (ms > 5000u) + ms = 5000u; + return ms; +} + +/* + * Arm at the partial receipt (the caller logs `MME-PRELUDE pack-part` first). + * Only the deterministic forced-golden image is eligible: the pack cursor is + * global and generation-kept, so the continuation carries the same bytes on + * any launch. A fired carrier whose own emission was still partial is + * re-armed here (that emission is the next partial); the boot-wide budget of + * two fires is what bounds the chain. + */ +static void gsp_phys_gr_mme_carry_arm(bool split, u32 poff, u32 maxa, + u32 gen, u32 h_object, u64 push_va) +{ + struct gsp_phys_gr_mme_carry_state old; + unsigned long flags; + bool superseded = false; + u32 ms; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_carry) || !split || + poff >= maxa || (h_object & 0xfffff000u) != 0xbeef0000u || + READ_ONCE(gsp_phys_gr_mme_pack_load_h) != h_object || + !gsp_phys_gr_golden_ready()) + return; + memset(&old, 0, sizeof(old)); + ms = gsp_phys_gr_mme_carry_due_ms(); + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + /* + * A carrier that already fired and then saw another partial part has + * delivered its launch without completing the pack: this partial is + * that carrier's continuation, so clear the fired mark and arm again + * (bounded by boot_n below). + */ + if (gsp_phys_gr_mme_carry.fired) + gsp_phys_gr_mme_carry.fired = false; + if (gsp_phys_gr_mme_carry.armed) { + old = gsp_phys_gr_mme_carry; + gsp_phys_gr_mme_carry.armed = false; + superseded = true; + } + if (gsp_phys_gr_mme_carry.boot_n >= 2u) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (superseded) + pr_warn("vgsp-gr: MME-CARRY-CANCEL reason=budget h=0x%x gen=%u off=%u/%u (no carrier left this boot)\n", + old.h_object, old.gen, old.off, old.maxa); + return; + } + gsp_phys_gr_mme_carry.armed = true; + gsp_phys_gr_mme_carry.h_object = h_object; + gsp_phys_gr_mme_carry.gen = gen; + gsp_phys_gr_mme_carry.off = poff; + gsp_phys_gr_mme_carry.maxa = maxa; + gsp_phys_gr_mme_carry.origin_push = push_va; + gsp_phys_gr_mme_carry.due_j = jiffies + msecs_to_jiffies(ms); + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (superseded) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=superseded h=0x%x gen=%u off=%u/%u\n", + old.h_object, old.gen, old.off, old.maxa); + pr_info("vgsp-gr: MME-CARRY-ARM h=0x%x gen=%u off=%u/%u push=0x%llx due_ms=%u\n", + h_object, gen, poff, maxa, (unsigned long long)push_va, ms); +} + +/* Due and still armed? Read-only hint for the service pump. */ +bool gsp_phys_gr_mme_carry_peek(u32 *h_object) +{ + unsigned long flags; + bool due; + + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + due = gsp_phys_gr_mme_carry.armed && + time_after_eq(jiffies, gsp_phys_gr_mme_carry.due_j); + if (due && h_object) + *h_object = gsp_phys_gr_mme_carry.h_object; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + return due; +} + +/* + * Claim the armed carrier for one launch. The live state is re-validated + * here, not at arm time: the pack must still be partial for this channel, + * still loaded contiguously from word 0 on it, and the golden overlay must + * still be forced. Any mismatch cancels (never fires a stale carrier). + */ +bool gsp_phys_gr_mme_carry_take(u32 h_object, u32 *gen, u32 *off, u32 *maxa) +{ + struct gsp_phys_gr_mme_carry_state old; + const char *reason = NULL; + unsigned long flags; + u32 live_off; + + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + if (!gsp_phys_gr_mme_carry.armed || + !time_after_eq(jiffies, gsp_phys_gr_mme_carry.due_j)) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + return false; + } + live_off = READ_ONCE(gsp_phys_gr_mme_pack_off); + if (gsp_phys_gr_mme_carry.h_object != h_object || + (h_object & 0xfffff000u) != 0xbeef0000u) + reason = "h-mismatch"; + else if (!gsp_phys_gr_golden_ready()) + reason = "golden-lost"; + else if (READ_ONCE(gsp_phys_gr_mme_pack_load_h) != h_object) + reason = "load-mismatch"; + else if (live_off >= gsp_phys_gr_mme_carry.maxa) + reason = "pack-complete"; + if (reason) { + old = gsp_phys_gr_mme_carry; + gsp_phys_gr_mme_carry.armed = false; + gsp_phys_gr_mme_carry.fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s h=0x%x gen=%u off=%u/%u\n", + reason, old.h_object, old.gen, old.off, old.maxa); + return false; + } + gsp_phys_gr_mme_carry.armed = false; + gsp_phys_gr_mme_carry.fired = true; + gsp_phys_gr_mme_carry.boot_n++; + if (gen) + *gen = gsp_phys_gr_mme_carry.gen; + if (off) + *off = live_off; + if (maxa) + *maxa = gsp_phys_gr_mme_carry.maxa; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + return true; +} + +/* Pack completion: the carrier's proof (or an unrelated lane completed it). */ +static void gsp_phys_gr_mme_carry_complete(u32 h_object, u32 gen, u32 maxa) +{ + struct gsp_phys_gr_mme_carry_state old; + unsigned long flags; + bool fired, armed; + + memset(&old, 0, sizeof(old)); + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + old = gsp_phys_gr_mme_carry; + fired = gsp_phys_gr_mme_carry.fired; + armed = gsp_phys_gr_mme_carry.armed; + gsp_phys_gr_mme_carry.fired = false; + gsp_phys_gr_mme_carry.armed = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (fired && old.h_object == h_object) + pr_info("vgsp-gr: MME-CARRY-COMPLETE h=0x%x gen=%u off=%u/%u push=0x%llx\n", + h_object, gen, maxa, maxa, + (unsigned long long)old.origin_push); + else if (armed) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=pack-complete h=0x%x gen=%u off=%u/%u\n", + old.h_object, old.gen, old.off, old.maxa); + else if (fired) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=cross-lane-complete h=0x%x gen=%u off=%u/%u\n", + old.h_object, old.gen, old.off, old.maxa); +} + +/* Context wipes / generation rewinds / re-registers void the carrier. */ +void gsp_phys_gr_mme_carry_cancel(const char *reason) +{ + struct gsp_phys_gr_mme_carry_state old; + unsigned long flags; + bool had; + + memset(&old, 0, sizeof(old)); + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + had = gsp_phys_gr_mme_carry.armed || gsp_phys_gr_mme_carry.fired; + old = gsp_phys_gr_mme_carry; + gsp_phys_gr_mme_carry.armed = false; + gsp_phys_gr_mme_carry.fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (had) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s h=0x%x gen=%u off=%u/%u\n", + reason ? reason : "unspecified", old.h_object, old.gen, + old.off, old.maxa); +} + +/* + * Launch failure: the carrier never reached (or never completed on) the + * engine. Void it and owe a full re-emission - the pack cursors rewind so the + * next client window re-emits from word 0 and never leaves a false + * completion behind. + */ +void gsp_phys_gr_mme_carry_failed(int rc) +{ + struct gsp_phys_gr_mme_carry_state old; + unsigned long flags; + bool had; + + memset(&old, 0, sizeof(old)); + spin_lock_irqsave(&gsp_phys_gr_mme_carry_lock, flags); + had = gsp_phys_gr_mme_carry.armed || gsp_phys_gr_mme_carry.fired; + old = gsp_phys_gr_mme_carry; + gsp_phys_gr_mme_carry.armed = false; + gsp_phys_gr_mme_carry.fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_carry_lock, flags); + if (!had) + return; + pr_warn("vgsp-gr: MME-CARRY-ABORT reason=hw-error rc=%d h=0x%x gen=%u off=%u/%u\n", + rc, old.h_object, old.gen, old.off, old.maxa); + gsp_phys_gr_mme_iram_gate_fail(); +} + +/* + * The carrier's windowless launch body: WAIT_FOR_IDLE(0x110) <- 0. Filled + * into the submit path's per-launch stream_patch scratch (writable: the + * pair-chain pass may prepend a closing END if a pair is carried open, and + * the window is dropped either way by the forced LOAD-ONLY decision). + */ +#define GSP_PHYS_GR_MME_CARRIER_WAIT_HDR 0x20010044u +#define GSP_PHYS_GR_MME_CARRIER_WAIT_DATA 0x00000000u + +/* * CONT-103: prelude residency witness + calls-kept block shrink. * The CONT-79 split-emission re-carries the whole prelude prefix on EVERY * launch while keep_mask is set (idempotent re-emission, CONT-72), so the @@ -19986,6 +20281,28 @@ module_param_named(sema_gl_sph_vtg_j1, gsp_phys_rmcall_gr_sph_vtg_j1, MODULE_PARM_DESC(sema_gl_sph_vtg_j1, "Also point the window's own slot-1 (VTG) bind at family+0x600 instead of the client's captured bind (default 0; c718 poison A/B)"); +/* + * eng52 (DS stage-image gate): the live direct-GL VTG SPH image is delivered + * by the client as an inline record (`req->inl[]`, 640 B, first word + * 0x020a0481, w6=0x0000f00f, w13=0x0000f000, w17=0x000f0000). At DS time + * the client's own program copy has overwritten the bind page (family+0x600 + * in the c716 class) with SASS, so binding that VA poisons the DS + * (c718/c720 badf1201). family+0x900 inside the same family page is zero + * and unused, and the engine can reach it through the same relocated page. + * Author the captured VTG image there, then point BOTH the synthesized VSA + * (j0) and the window's own VSB (j1) at it: j0 and j1 fetch a real + * V4T1 SHADER_TYPE=1 image while j2 stays at +0x200, the GS image at + * +0xe80 and the pixel image at +0x000. The copy is last (after the + * page-loop inline replay) so no later replay can overwrite it, and it is + * refused with a named receipt if another inline record already targets the + * destination. A/B with 0 (legacy +0x900 zero page / +0x600 selection). + */ +static bool gsp_phys_rmcall_gr_sph_vtg_auth = true; +module_param_named(sema_gl_sph_vtg_auth, gsp_phys_rmcall_gr_sph_vtg_auth, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_vtg_auth, + "Author the client's live VTG SPH image at family+0x900 and bind the synthesized VSA (j0) and window VSB (j1) there (default 1)"); + bool gsp_phys_rmcall_gr_sph_bind_value(void) { return READ_ONCE(gsp_phys_rmcall_gr_sph_bind); @@ -20689,6 +21006,7 @@ static u32 gsp_phys_rmcall_gr_mme_iram_gate_max = 24u; static void gsp_phys_gr_mme_iram_gate_fail(void) { gsp_phys_gr_mme_iram_gate_fails++; + gsp_phys_gr_mme_carry_cancel("iram-gate-fail"); WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); @@ -20926,6 +21244,7 @@ static u32 gsp_phys_gr_fe_status(void) */ static void gsp_phys_gr_ctx_wipe_latches(void) { + gsp_phys_gr_mme_carry_cancel("ctx-wipe"); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); @@ -20978,6 +21297,7 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) if (h != READ_ONCE(gsp_phys_gr_mme_pack_load_h) && h != READ_ONCE(gsp_phys_gr_mme_pack_done_h)) return; + gsp_phys_gr_mme_carry_cancel("chan-ctx-reset"); /* * eng43: sema_gl_mme_keep_done_across_recycle=1 - the re-register * recycles the client handle but leaves the GR engine context (and @@ -21346,6 +21666,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, bool mme_load_only = false; bool latch_before = false; bool stream_damaged = false; + bool carrier = req && req->mme_carrier; u32 pre_words_out = 0; u32 win_emit_off = 0; const u32 *pl = NULL; @@ -21710,6 +22031,7 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, gsp_phys_rmcall_gr_mme_pack_keep_gen) && READ_ONCE( gsp_phys_rmcall_gr_mme_golden_force))) { + gsp_phys_gr_mme_carry_cancel("gen-rewind"); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); @@ -21757,6 +22079,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, } } } + /* + * Host-initiated carrier: it exists only to carry the next pack + * part, so never let the latch/room heuristics defer the + * emission (the arm-time checks already proved the pack is + * partial for this channel and generation). + */ + if (carrier && pl && pl_n && pl_gen) + emit_prelude = true; *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | (0u << 13) | @@ -22793,6 +23123,9 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, packed = true; if (poff >= maxa) { doff = pl_n; + gsp_phys_gr_mme_carry_complete( + req->h_object, + pl_gen, maxa); pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", maxa, nruns, npair, @@ -22805,6 +23138,20 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, split ? 1u : 0u, req->h_object, req->push_va); + /* + * c727: the part left + * the pack partial. Arm + * one host-initiated + * continuation for this + * beef channel (bounded + * and default-off; see + * sema_gl_mme_pack_carry). + */ + gsp_phys_gr_mme_carry_arm( + split, poff, maxa, + pl_gen, + req->h_object, + req->push_va); } } else if (split && maxa >= 3u) { /* @@ -22977,6 +23324,15 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, mme_load_only = true; } /* + * Host-initiated carrier: windowless LOAD-ONLY by construction + * (no client window, no pages, no release). The stub window + * the submit path hands the builder is never emitted; it only + * keeps the launch in this 3D branch and gives the block + * arithmetic a nonzero window size. + */ + if (carrier) + mme_load_only = true; + /* * SESSION 153-CONTINUED-82: hold real CALL runs until the * prelude latch for this generation+channel is set (see the * sema_gl_mme_hold_calls comment). Evaluated AFTER the @@ -23147,6 +23503,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, } } /* + * Host-initiated carrier: it carries no draw at all, so it can + * never run the window's CALL runs - force the held shape even + * if the latch/release heuristics would release them on this + * launch. + */ + if (carrier) + mme_hold_calls = true; + /* * eng40/eng42: post-completion re-arm. Once the packed * program completed (deterministic golden image), the engine * context keeps it until a wipe. Every later frame bumps the @@ -26863,6 +27227,53 @@ static u64 gsp_phys_gr_pair_va[GSP_PHYS_RMCALL_GR_PAIR_CACHE]; static u32 gsp_phys_gr_pair_n; /* + * eng52 (DS stage-image gate): census of the six stage start VAs latched by + * the DS at a fault transition (see the forward declaration). A stage START + * register holds the low 32 bits (including the in-page offset) and the + * UPPER register bits 39:32, so the VA is (upper & 0xff) << 32 | start. + * Resolve it through the program-pair cache (the same 8K line the + * relocation pass uses for the family page), then read the first SPH word + * and the first code word (+0x80) through the host direct map of the + * engine-visible HPA. Zero when the VA is unset or not in the pair cache. + * Diagnostic only; the pair arrays are written under gsp_phys_rm_lock, so + * READ_ONCE keeps the census from tearing a line during a live submit. + */ +static void gsp_phys_ds_image_census(const char *tag, u32 h_object, + const u32 s[6], const u32 u[6]) +{ + u32 w0[6] = { 0 }, code[6] = { 0 }; + u32 si, pi, n = READ_ONCE(gsp_phys_gr_pair_n); + + for (si = 0; si < 6u; si++) { + u64 va = ((u64)(u[si] & 0xffu) << 32) | (u64)s[si]; + + if (!va) + continue; + for (pi = 0; + pi < n && pi < GSP_PHYS_RMCALL_GR_PAIR_CACHE; pi++) { + u64 pv = READ_ONCE(gsp_phys_gr_pair_va[pi]); + u64 ph; + + if (!pv || va < pv || + va >= pv + 2u * GSP_PHYS_PAGE_SIZE) + continue; + ph = READ_ONCE(gsp_phys_gr_pair_hpa[pi]) + (va - pv); + if (pfn_valid((unsigned long)(ph >> PAGE_SHIFT))) { + w0[si] = get_unaligned_le32( + (const u8 *)__va(ph)); + code[si] = get_unaligned_le32( + (const u8 *)__va(ph) + 0x80u); + } + break; + } + } + pr_info_ratelimited("vgsp-gr: DS-IMAGE %s h=0x%x st0 w0=%08x code=%08x st1 w0=%08x code=%08x st2 w0=%08x code=%08x st3 w0=%08x code=%08x st4 w0=%08x code=%08x st5 w0=%08x code=%08x\n", + tag, h_object, + w0[0], code[0], w0[1], code[1], w0[2], code[2], + w0[3], code[3], w0[4], code[4], w0[5], code[5]); +} + +/* * Resolve the HPA of `page0 + 4K` through the same sources the relocation * chain uses: the launch's own page set (the pages the copy-in authored), * the tracked data recs, the sema service's trusted resolver, then the @@ -29075,6 +29486,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n; /* + * Host-initiated MME pack-continuation carrier: no guest window and no + * guest memory. The window is a static WAIT_FOR_IDLE stub (never + * emitted - the builder forces LOAD-ONLY) and every capture/relocation + * prepend below is skipped. + */ + bool carrier = req && req->mme_carrier; + /* * CONT-73: fresh direct-write witness for this submission. The * window/attr patch passes below record every relocated operand * destination's guest page; the pre-doorbell capture and the retire @@ -29266,7 +29684,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * into the same software-completion fallback without touching the * engine, and cannot wedge it for the next client. */ - if (req->n_data < gsp_phys_gr_min_data_value()) { + if (!carrier && req->n_data < gsp_phys_gr_min_data_value()) { gsp_phys_gr_refused++; pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", req->n_data, @@ -30610,7 +31028,21 @@ gr_page_unmapped: * FAULT_PDE on a Tier-1-mapped guest VA). Runs with the rm lock * held, before the method block is built. */ - if (stream3d) { + if (stream3d && carrier) { + /* + * Host-initiated pack-continuation carrier: the request has no + * guest window at all. Hand the builder a WAIT_FOR_IDLE stub + * (in the per-launch writable patch scratch) so it stays on + * the 3D path; the builder forces LOAD-ONLY (window dropped, + * CALL runs held), so these two words are never executed. No + * capture relocation, no attr/SURF prepend, no SPH-BIND/ + * SPH-AUTHOR: there is no guest memory in this launch. + */ + stream_patch[0] = GSP_PHYS_GR_MME_CARRIER_WAIT_HDR; + stream_patch[1] = GSP_PHYS_GR_MME_CARRIER_WAIT_DATA; + win3d = stream_patch; + win3d_n = 2; + } else if (stream3d) { u32 win_n = req->stream_n; u32 ndrop = 0; u32 nrel; @@ -30689,11 +31121,175 @@ gr_page_unmapped: static u32 sph_bind_log; u64 fam = req->prog_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); - u64 vtg_va = READ_ONCE(gsp_phys_rmcall_gr_sph_vtg600) ? - (fam + 0x600u) : (fam + 0x900u); + u64 vtg_va; + bool vtg_auth = false; u32 in[96], out[96]; u32 nfin = 0, nrun = 0, on = 0, odrop = 0, orel; u32 bi; + /* + * eng52 (DS stage-image gate): author the client's live + * VTG SPH image at family+0x900. The image arrives as an + * inline record on the family page (first word 0x020a0481 + * = V4T1 SHADER_TYPE=1), but the client's own program copy + * overwrites the bind page (family+0x600 in the c716 + * class) with SASS by DS time, so binding that VA poisons + * the DS. family+0x900 is zero and unused; write the + * captured image there through the same mapped-HPA path + * the page-loop inline replay uses (pg->hpa is the + * direct-mappable host page for both AP=FB shadow and + * sysmem), then bind j0/j1 to it below. The copy is last, + * after the page-loop replay, so nothing overwrites it. + * Refuse with a named receipt if another inline record + * already targets the destination (never corrupt it). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_vtg_auth) && fam) { + static u32 sph_auth_log; + const struct gsp_phys_gr_inline *vinl = NULL; + const struct gsp_phys_gr_page *vpg = NULL; + const u8 *vsrc = NULL; + u32 vlen = 0, vj = 0, vpi; + bool vbuiltin = false; + u32 vj2; + + for (vj2 = 0; vj2 < req->n_inl; vj2++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[vj2]; + u32 w0; + + if ((inl->dst_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + fam) + continue; + if (inl->len < 4u || + inl->len > GSP_PHYS_GR_INLINE_BYTES) + continue; + w0 = get_unaligned_le32(inl->data); + if ((w0 & 0x1fu) != 1u || + ((w0 >> 10) & 0xfu) != 1u) + continue; + vinl = inl; + vj = vj2; + } + if (vinl) { + vsrc = vinl->data; + vlen = vinl->len; + } else { + vsrc = gsp_sema_gl_vs_img(&vlen); + vbuiltin = true; + } + if (vlen > GSP_PHYS_PAGE_SIZE - 0x900u) + vlen = GSP_PHYS_PAGE_SIZE - 0x900u; + for (vpi = 0; vpi < req->n_pages; vpi++) { + if ((req->pages[vpi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + fam || + !req->pages[vpi].hpa) + continue; + vpg = &req->pages[vpi]; + break; + } + if (!vsrc || !vlen || !vpg) { + if (sph_auth_log < 64u) { + sph_auth_log++; + pr_info("vgsp-gr: SPH-AUTHOR skip=%s fam=0x%llx h=0x%x push=0x%llx\n", + !vsrc ? "no-src" : + !vlen ? "no-len" : + "no-page", fam, + req->h_object, + req->push_va); + } + } else { + bool voverlap = false; + + for (vj2 = 0; vj2 < req->n_inl; vj2++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[vj2]; + u64 io, ie; + + if ((inl->dst_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + fam) + continue; + io = inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1); + ie = io + inl->len; + if (io < 0x900u + vlen && + 0x900u < ie) + voverlap = true; + } + if (voverlap) { + if (sph_auth_log < 64u) { + sph_auth_log++; + pr_info("vgsp-gr: SPH-AUTHOR skip=overlap fam=0x%llx dst=0x%llx len=%u h=0x%x push=0x%llx\n", + fam, fam + 0x900u, + vlen, + req->h_object, + req->push_va); + } + } else if (!pfn_valid((unsigned long) + (vpg->hpa >> + PAGE_SHIFT))) { + if (sph_auth_log < 64u) { + sph_auth_log++; + pr_info("vgsp-gr: SPH-AUTHOR skip=no-hpa fam=0x%llx dst=0x%llx len=%u hpa=0x%llx h=0x%x push=0x%llx\n", + fam, fam + 0x900u, + vlen, vpg->hpa, + req->h_object, + req->push_va); + } + } else { + u32 vfirst = get_unaligned_le32(vsrc); + u32 vw6 = vlen >= 28u ? + get_unaligned_le32(vsrc + 24u) : 0u; + u32 vw13 = vlen >= 56u ? + get_unaligned_le32(vsrc + 52u) : 0u; + u32 vw17 = vlen >= 72u ? + get_unaligned_le32(vsrc + 68u) : 0u; + + memcpy((u8 *)__va(vpg->hpa) + + 0x900u, + vsrc, vlen); + dma_wmb(); + vtg_auth = true; + if (sph_auth_log < 64u) { + sph_auth_log++; + if (vbuiltin) + pr_info("vgsp-gr: SPH-AUTHOR fam=0x%llx dst=0x%llx len=%u src=builtin first=%08x w6=%08x w13=%08x w17=%08x h=0x%x push=0x%llx\n", + fam, fam + 0x900u, + vlen, vfirst, vw6, + vw13, vw17, + req->h_object, + req->push_va); + else + pr_info("vgsp-gr: SPH-AUTHOR fam=0x%llx dst=0x%llx len=%u src=inl[%u] first=%08x w6=%08x w13=%08x w17=%08x h=0x%x push=0x%llx\n", + fam, fam + 0x900u, + vlen, vj, vfirst, + vw6, vw13, vw17, + req->h_object, + req->push_va); + } + /* + * Read the authored words back + * through the direct map (shadow + * is not chosen until the + * relocation pass, hence 0). + */ + gsp_phys_gr_sph_alias_receipt( + fam, fam + 0x900u, 0, + vpg->hpa, "auth"); + } + } + } + /* + * Keep the historical +0x600 selection only when the + * authoring knob is off (or the copy was refused): a + * successful authoring lives at +0x900 and both j0 and + * j1 must bind there. + */ + vtg_va = (vtg_auth || + !READ_ONCE(gsp_phys_rmcall_gr_sph_vtg600)) ? + (fam + 0x900u) : (fam + 0x600u); + bool bind_binding = READ_ONCE(gsp_phys_rmcall_gr_sph_bind_binding); bool pair_safe = @@ -30997,9 +31593,13 @@ gr_page_unmapped: * 0x0d0d7623); the VTG SPH image is at * family+0x600. Point the VTG slot at the * real image when the knob is on. + * eng52: an authored VTG image at +0x900 is + * always the better operand, so the override + * activates whenever the authoring succeeded. */ if (sl == 1u && - READ_ONCE(gsp_phys_rmcall_gr_sph_vtg_j1)) + (READ_ONCE(gsp_phys_rmcall_gr_sph_vtg_j1) || + vtg_auth)) va = vtg_va; in[nfin++] = 0x20000000u | (2u << 16) | ((0x2014u + sl * 0x40u) >> 2); @@ -32853,6 +33453,7 @@ gr_page_unmapped: mutex_unlock(&gsp_phys_rm_lock); park_rec_n++; if (!sp_ret) { + gsp_phys_gr_mme_carry_cancel("park-recover"); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 92c8e69..dd7e64a 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -505,6 +505,16 @@ struct gsp_phys_gr_submit { */ u32 stream_rel; /* + * Host-initiated MME pack-continuation carrier (sema_gl_mme_pack_carry): + * the caller is the service's synthetic request, not a guest push. + * The submit path then ignores req->stream/pages/inl/data/releases, + * emits the builder's static WAIT_FOR_IDLE stub window (never + * executed: the launch is forced LOAD-ONLY with the CALL runs held) + * and lets the builder carry the next packed-prelude part from its + * own live cursor. No guest memory is read or written. + */ + bool mme_carrier; + /* * SESSION 153-CONTINUED-87: the first program VA the captured * window binds (SET_PIPELINE_PROGRAM_ADDRESS pair; 0 when none). * The launch builder uses its 4K page as the program family for @@ -550,6 +560,21 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ bool gsp_phys_gr_mme_fe_defer_wanted(u32 h_object, u32 win3d_n); +/* + * Host-initiated MME pack-continuation carrier (sema_gl_mme_pack_carry). + * The service pump peeks for a due carrier, then takes it (which re-validates + * the live pack cursor/handle/golden state) and submits the synthetic + * load-only request. On launch failure the service calls carry_failed so the + * next client window re-emits from word 0; carry_cancel drops a carrier on a + * non-launch path (queue failure etc.). All three are no-ops when no carrier + * is armed. + */ +bool gsp_phys_gr_mme_carry_peek(u32 *h_object); +bool gsp_phys_gr_mme_carry_take(u32 h_object, u32 *gen, u32 *off, + u32 *maxa); +void gsp_phys_gr_mme_carry_cancel(const char *reason); +void gsp_phys_gr_mme_carry_failed(int rc); + /* GR forwarding counters (submitted/verified/cancelled), plain reads. */ void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused, u64 *timed_out, u64 *pool_exhausted); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 96fabcf..4655e77 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -47,6 +47,7 @@ #include <linux/debugfs.h> #include <linux/seq_file.h> #include <linux/io.h> +#include <linux/preempt.h> #include <asm/unaligned.h> #include "nvidia-vgpu-vfio.h" @@ -251,14 +252,24 @@ struct sema_map_entry { * The module parameter is the per-cycle guard: shrink it to the boot's real * RAM top. * - * r933: this parameter is now the single live PA acceptance bound for the - * whole service (svc_pa_max), not just the ring scan - the walk leaves in - * the 6-8 GiB band were exactly what the old separate 6 GB SEMA_PA_MAX - * rejected (the GL client's kick pages). SEMA_PA_MAX remains the - * compile-time maximum that sizes fixed tables; the param clamps it. - */ -unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL; + * X-DMA-FIX (2026-11-15): the campaign guest is 12 GiB (e820 top + * 0x300000000), so the old 0x280000000 default stopped every content scan + * 2 GiB below the guest's own allocations ("RAM scan found no candidate + * ... top 0x280000000"). The default now covers the whole 12 GiB band; + * a smaller guest still fails reads past its mapped RAM fast. + * + * r933: this parameter is the live PA acceptance bound for the whole + * service (svc_pa_max), not just the ring scan - the walk leaves in the + * 6-8 GiB band were exactly what the old separate 6 GB SEMA_PA_MAX + * rejected (the GL client's kick pages). svc_pa_max() clamps a larger + * param to the compile-time SEMA_PA_MAX before extending it to the + * registered guest-RAM top (CONT.478), so the param sizes the scans + * while registration provides the acceptance truth. + */ +unsigned long long gsp_sema_gr_scan_top = 0x300000000ULL; module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644); +MODULE_PARM_DESC(sema_gr_scan_top, + "Guest-RAM ring/USERD content-scan top, guest-physical bytes (default 0x300000000 = 12 GiB); shrink to the boot's RAM top"); /* * r1780 (CONT.430b): keep the direct-GL (beef GR) ring content scan alive @@ -888,6 +899,23 @@ struct svc_channel { * write-back list. */ u64 notify2d_va; + /* + * X-DMA tracker fix (sema_ddx_tracker_auth): a re-created DDX + * channel loses its USERD latch, so the pump refuses to walk the + * ring and the client's own release push (C36F/SEM_EXECUTE) is + * never decoded. svc_ddx_head_decode reads the head entries + * directly instead. ddx_head_at paces that scan to one attempt per + * 100 ms per channel (persisted immediately - the stalled branch + * skips the end-of-pass write-back); ddx_cache_* hold the last + * decoded release for the mode-2 forward-only diagnostic re-apply. + * All four are zeroed by the registration memset, so a + * re-registration or a generation change never inherits a dead + * session's release. + */ + unsigned long ddx_head_at; + u64 ddx_cache_va; + u32 ddx_cache_payload; + bool ddx_cache_valid; /* r829: bounded count of zero-entry (bad-translation) holds; see the * zero-entry hold in svc_pump_channels. Persists via the write-back. */ u32 zero_held; @@ -2129,11 +2157,16 @@ static void svc_retry_enqueue(u64 va, u32 len, u32 want) static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { + /* vfio_dma_rw() may sleep on the type1 iommu mutex. */ + if (WARN_ON_ONCE(in_atomic())) + return -EINVAL; return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { + if (WARN_ON_ONCE(in_atomic())) + return -EINVAL; return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } @@ -3686,6 +3719,17 @@ static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, { int i; + /* + * The walk reads guest page tables through vfio_dma_rw(), which + * takes the type1 iommu mutex and may sleep. The r707e every-pass + * verify originally ran under svc.chan_lock and produced the + * "scheduling while atomic" class (PART 43 / PART 57) again in + * c741; this guard keeps a future locked call site from silently + * reintroducing it. + */ + if (WARN_ON_ONCE(in_atomic())) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { u64 wpa = 0; int wap = 0; @@ -10492,6 +10536,20 @@ const u8 *gsp_sema_gl_prog_img(u64 va, u32 *len_out) } /* + * eng52 (DS stage-image gate): the published VTG image without the + * stage-typed selection above. The booter's family+0x900 authoring must + * always copy the VERTEX image; gsp_sema_gl_prog_img can legitimately + * answer the PIXEL image for an unregistered page, which would poison the + * VSA/VSB stages it is bound to. + */ +const u8 *gsp_sema_gl_vs_img(u32 *len_out) +{ + if (len_out) + *len_out = svc_gl_vs_len; + return svc_gl_vs_len ? svc_gl_vs : NULL; +} + +/* * SESSION 153 (GL client draw sections): the GL client's own per-frame * pushes bind TURING_A (0xC597) exactly like the X server's one-shot * section, but carry no program/param/release operands at all (measured: @@ -23244,6 +23302,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + /* + * X-DMA tracker fix: keep this guest-authored + * release for svc_ddx_head_decode's mode-2 + * diagnostic re-apply (forward-only). The + * snapshot write-back persists it. + */ + ch->ddx_cache_va = va; + ch->ddx_cache_payload = payload; + ch->ddx_cache_valid = true; /* SESSION 149 R14: keep the last VALID release * decoded on this channel for the GL section * synthesizer (the 24 B release pushes bracket @@ -23458,6 +23525,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = st.rs_payload; + /* X-DMA tracker fix: cache the + * validated REPORT_SEM release for the + * head-decode mode-2 diagnostic. */ + ch->ddx_cache_va = va; + ch->ddx_cache_payload = st.rs_payload; + ch->ddx_cache_valid = true; svc_wait_rel_va = va; svc_wait_rel_payload = st.rs_payload; svc_wait_rel_off = off; @@ -24770,6 +24843,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) { int rrc2; + /* X-DMA tracker fix: cache only the + * validated release (a missing + * semaphore word would cache a + * fabricated target). */ + ch->ddx_cache_va = sem_va; + ch->ddx_cache_payload = st.sem_payload; + ch->ddx_cache_valid = true; if (svc_pool_sem_va(sem_va)) { u64 ppa = 0; int pap = SVC_APERTURE_SYS; @@ -27806,6 +27886,199 @@ static void svc_kick_capture_arm(struct nv_vgpu_device *vgpu, } /* + * X-DMA tracker fix (DDX ring-head tracker decode). + * + * The guest X DDX's channel (0x01000013 and the 0xbfef00xx kick family) is + * re-created between push generations. Its USERD is client-allocated in + * guest RAM and the latch can go missing/reset, so the pump refuses to walk + * the ring (!userd_ok) and the client's own release push - measured as + * va=0x1200a0000 len=24 targeting rel=0x120001000, the nvPush progress + * tracker - is never decoded. The tracker word then never advances and the + * DDX reports "Failed to idle DMA". + * + * This decode walks the ring head directly - no USERD, no GP_GET advance, no + * userd_ok latch - and runs the normal release/decode path with method ops + * disabled: the guest-authored payload is applied verbatim through + * svc_rel_apply_906f/svc_sem_write, never synthesized. The + * 0x120000000-0x121000000 window is the driver's push window (the X ring, + * its pushes and the tracker word all live there). + * + * sema_ddx_tracker_auth: 0 = off (default), 1 = head decode, 2 = also + * re-apply the last decoded release forward-only when the head itself is + * unreadable (diagnostic only; never fabricates a payload). + */ +static uint gsp_sema_ddx_tracker_auth; +module_param_named(sema_ddx_tracker_auth, gsp_sema_ddx_tracker_auth, uint, + 0644); +MODULE_PARM_DESC(sema_ddx_tracker_auth, + "DDX ring-head tracker decode when the USERD latch is missing: 0=off, 1=decode head entries, 2=also re-apply the last decoded release forward-only"); + +/* + * Persist the head-decode pacing/cache immediately: the !userd_ok path + * `continue`s past the end-of-pass snapshot write-back, so the 100 ms rate + * limit and the cached release must survive on their own. + */ +static void svc_ddx_state_store(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].ddx_head_at = ch->ddx_head_at; + svc.chan[i].ddx_cache_va = ch->ddx_cache_va; + svc.chan[i].ddx_cache_payload = + ch->ddx_cache_payload; + svc.chan[i].ddx_cache_valid = ch->ddx_cache_valid; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +static void svc_ddx_head_decode(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, + int ring_ap) +{ + static u8 *buf; + const char *reason = NULL; + u32 entries = ch->gpfifo_entries; + u32 start = (u32)ch->last_get; + u32 maxn, idx = 0, slot = 0; + u64 va = 0; + u32 len = 0; + u64 rel_va = 0; + u32 rel_pay = 0; + bool released = false; + bool head_unreadable = false; + + if (!gsp_sema_ddx_tracker_auth) + return; + if (!svc_userd_ddx_family(ch)) + return; + if (ch->ddx_head_at && time_before(jiffies, ch->ddx_head_at)) { + reason = "rate"; + goto miss; + } + ch->ddx_head_at = jiffies + msecs_to_jiffies(100); + svc_ddx_state_store(vgpu, ch); + if (!ring_pa || ring_ap < 0) { + reason = "no-ring"; + head_unreadable = true; + goto miss; + } + if (!entries || start >= entries) { + reason = "no-entry"; + head_unreadable = true; + goto miss; + } + + maxn = min_t(u32, 8u, entries - start); + for (idx = 0; idx < maxn; idx++) { + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, mark; + + slot = (start + idx) % entries; + if (svc_read_ap(vgpu, + ring_pa + (u64)slot * SVC_GPFIFO_ENTRY_SIZE, + ring_ap, ent, sizeof(ent))) { + reason = "unreadable"; + head_unreadable = true; + break; + } + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + if (!e0 && !e1) { + reason = "no-entry"; + head_unreadable = true; + break; + } + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (va < 0x120000000ULL || va >= 0x121000000ULL) { + reason = "bad-va"; + break; + } + if (len < 4 || len > 0x10000) { + reason = "bad-len"; + break; + } + if (!buf) { + buf = kmalloc(0x10000, GFP_KERNEL); + if (!buf) { + reason = "unreadable"; + head_unreadable = true; + break; + } + } + if (svc_read_va_push(vgpu, ch, va, buf, len)) { + reason = "unreadable"; + head_unreadable = true; + break; + } + if (!memchr_inv(buf, 0, len)) { + /* + * The push read succeeded but answered an all-zero + * page (a stale/zero translation): there is no + * release to decode and no value to apply. + */ + reason = "zero-payload"; + head_unreadable = true; + break; + } + mark = svc_rel_count; + /* + * No ops (allow_ops=false): the release apply and the + * trusted-decode side effects stay, method execution does + * not. page_pa only feeds svc_mark_hot; use the page the + * push read actually answered from. + */ + svc_decode_stream(vgpu, buf, len, + svc_read_last_pa & + ~(u64)(SEMA_PAGE_SIZE - 1), + false, ch, slot, va, true, false, NULL); + if (svc_rel_count != mark && ch->ddx_cache_valid) { + released = true; + rel_va = ch->ddx_cache_va; + rel_pay = ch->ddx_cache_payload; + break; + } + } + svc_ddx_state_store(vgpu, ch); + if (released) { + pr_info_ratelimited("sema-ddx: head-decode h=0x%x idx=%u va=0x%llx len=%u rel=0x%llx pay=0x%x\n", + ch->h_object, slot, va, len, rel_va, rel_pay); + return; + } + if (!reason) + reason = "no-release"; +miss: + pr_info_ratelimited("sema-ddx: head-decode MISS h=0x%x idx=%u reason=%s\n", + ch->h_object, slot, reason); + /* + * Mode 2 diagnostic: the ring head itself is unreadable and the + * channel already decoded a real release. Re-apply that exact + * (va, payload) forward-only - never a synthesized value, never in + * mode 1. + */ + if (gsp_sema_ddx_tracker_auth == 2 && head_unreadable && + ch->ddx_cache_valid) { + u32 cur = 0; + + (void)svc_sem_read(vgpu, ch->ddx_cache_va, &cur); + if (svc_rel_apply_906f(vgpu, ch, ch->ddx_cache_va, + ch->ddx_cache_payload, + HOST_SEM_OP_RELEASE)) + pr_info_ratelimited("sema-ddx: cached-apply h=0x%x rel=0x%llx pay=0x%x cur=0x%x\n", + ch->h_object, ch->ddx_cache_va, + ch->ddx_cache_payload, cur); + } +} + +/* * r1659: direct-GL CE USERD adoption. The beef0101/0102 channels declare a * real FB control page (userdMem.base + userdOffset: 0x244f800 / 0x244f400) * while their GPFIFO ring is mapped in guest RAM. svc_chan_userd only @@ -35747,6 +36020,17 @@ struct svc_fwd_req { * exactly-once latch for the work-item CUDA bridge fallback. */ bool gr; + /* + * Host-initiated MME pack-continuation carrier (sema_gl_mme_pack_carry): + * gr_synthetic marks the service-authored request that carries the next + * packed-prelude part for an already-armed beef channel. It has no guest + * push, pages, inline records, data records, releases or QMD completion: + * the backend emits a windowless LOAD-ONLY launch from its own pack + * cursor. Its outcome never authorises a guest release and never raises + * a synthetic fault - on launch failure the booter's carrier failure + * path rewinds the pack so the next client window re-emits from word 0. + */ + bool gr_synthetic; bool gr_bridge_done; bool gr_page_overflow; /* @@ -37578,6 +37862,13 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) bool done = smp_load_acquire(&svc_fwd_req.done); unsigned long waited; + /* + * Host-initiated carrier: its failure path is the backend call's own + * (gsp_phys_gr_mme_carry_failed); it carries no guest release and no + * QMD word, so the stall-completion path must never touch it. + */ + if (svc_fwd_req.gr_synthetic) + return; if (!svc_fwd_req.gr || svc_fwd_req.vgpu != vgpu || svc_fwd_req.gr_swrel_done || svc_fwd_hold.active || !done) return; @@ -38709,6 +39000,8 @@ static void svc_fwd_gr_work(void) } } gs.h_object = svc_fwd_req.gr_h_object; + /* Host-initiated carrier: no guest window/pages/releases. */ + gs.mme_carrier = svc_fwd_req.gr_synthetic; /* * GRVAUDIT: one line per launch (n_data matches the backend @@ -38742,12 +39035,23 @@ static void svc_fwd_gr_work(void) if (cp.fault.valid) svc.fwd_gr_fault++; } - if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) - pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n", - hw_rc, svc_fwd_req.gr_h_object, - svc_fwd_req.gr_push_va, svc.fwd_gr_req, - svc.fwd_gr_ok, svc.fwd_gr_fallback); - svc_gr_req_bridge_fallback(svc_fwd_req.vgpu); + if (svc_fwd_req.gr_synthetic) { + /* + * The carrier never reached the engine (or the engine + * rejected the load-only block). Never fabricate a + * completion: void the carrier and rewind the pack so + * the next client window re-emits from word 0. The + * failure path logs MME-CARRY-ABORT. + */ + gsp_phys_gr_mme_carry_failed(hw_rc); + } else { + if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n", + hw_rc, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, svc.fwd_gr_req, + svc.fwd_gr_ok, svc.fwd_gr_fallback); + svc_gr_req_bridge_fallback(svc_fwd_req.vgpu); + } } svc_fwd_req.fault = cp.fault; @@ -41643,6 +41947,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault)); svc_fwd_req.guest = false; svc_fwd_req.gr = true; + svc_fwd_req.gr_synthetic = false; svc_fwd_req.gr_bridge_done = false; /* STALLFREE: fresh request - re-arm the software-completion latch and * stamp the fill time for the GR-STALL waited_ms accounting. */ @@ -43166,6 +43471,112 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } /* + * Host-initiated MME pack-continuation carrier (sema_gl_mme_pack_carry). + * + * c727: a split packed emission stalled at part 1 of 3 (1512/2799 words) and + * no later client launch carried the remaining parts, so the CALL runs stayed + * held. c716 proved the completing launch shape: a final-part, window-dropped + * LOAD-ONLY launch. The service therefore asks the booter for a due carrier + * (armed at a partial receipt for a beef channel on which the forced-golden + * pack is contiguously loaded from word 0) and queues it as a synthetic GR + * request. The request carries no guest push/pages/inline/data/releases, so + * the backend can only emit the next golden pack part on a windowless + * LOAD-ONLY block: no draw runs, no unheld CALL runs, no guest memory is read + * or written. The booter re-validates the live cursor in carry_take(); on any + * mismatch it cancels and this function is a no-op. + */ +#define SVC_GR_CARRIER_PUSH_VA 0x6d6d6500ULL /* unique synthetic id */ +#define SVC_GR_CARRIER_STUB_WAIT 0x20010044u /* INC 0x110 WAIT_FOR_IDLE */ + +static bool svc_gr_cand_unoffered_for(u32 h_object) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + struct svc_gr_cand_rec *c = &svc_gr_cand_ring[ri]; + + if (c->valid && !c->offered && c->h_object == h_object) + return true; + } + return false; +} + +static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 h_object = ch->h_object; + u32 gen = 0, off = 0, maxa = 0; + + if (!gsp_phys_gr_forward_enabled() || + !gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) + return false; + /* One launch owner per channel: never compete with a real candidate. */ + if (svc_gr_cand_unoffered_for(h_object) || svc_fwd_hold.active) + return false; + if (!gsp_phys_gr_mme_carry_take(h_object, &gen, &off, &maxa)) + return false; + pr_info("vgsp-gr: MME-CARRY-FIRE h=0x%x gen=%u off=%u/%u push=0x%llx\n", + h_object, gen, off, maxa, + (unsigned long long)SVC_GR_CARRIER_PUSH_VA); + mutex_lock(&svc_fwd_lock); + if (svc_fwd_paused || + test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { + mutex_unlock(&svc_fwd_lock); + /* + * Nothing was queued and nothing was emitted: drop the carrier + * without rewinding the pack (the next client window continues + * from the current cursor). + */ + gsp_phys_gr_mme_carry_cancel("request-busy"); + return false; + } + memset(&svc_fwd_req, 0, sizeof(svc_fwd_req)); + svc_fwd_req.done = false; + svc_fwd_req.hw_rc = 0; + svc_fwd_req.vgpu = vgpu; + svc_fwd_req.gr = true; + svc_fwd_req.gr_synthetic = true; + svc_fwd_req.gr_h_object = h_object; + svc_fwd_req.gr_push_va = SVC_GR_CARRIER_PUSH_VA; + svc_fwd_req.gr_fill_jiffies = jiffies; + /* + * The backend's 3D branch is entered by a non-empty stream; the + * two-word stub here is only the request shape (the booter replaces it + * with its own WAIT_FOR_IDLE stub and forces LOAD-ONLY), so this body + * is never executed. No pages, inline records, data records, windows, + * program binds, sem fields or release fields are filled - the memset + * above zeroed them and qmd[] is the all-zero 64-word source the + * submit requires. + */ + svc_fwd_req.gr_stream_3d = true; + svc_fwd_req.gr_stream_off = 0; + svc_fwd_req.gr_stream_n = 2; + svc_fwd_req.gr_stream_rel = 0; + svc_fwd_req.gr_raw[0] = SVC_GR_CARRIER_STUB_WAIT; + svc_fwd_req.gr_raw[1] = 0; + svc_fwd_requests++; + if (!svc_fwd_wq) { + svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); + if (!svc_fwd_wq) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + gsp_phys_gr_mme_carry_cancel("queue-alloc"); + return false; + } + } + if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + gsp_phys_gr_mme_carry_cancel("queue-work"); + return false; + } + mutex_unlock(&svc_fwd_lock); + return true; +} + +/* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold * the entry for its deferred releases. All wave-1 gates are unchanged. @@ -43234,6 +43645,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, * return value below would report a queue that never happened). */ svc_fwd_req.gr = false; + svc_fwd_req.gr_synthetic = false; svc_fwd_req.gr_bridge_done = false; svc_fwd_req.guest = svc_fwd_cand.valid && gsp_phys_guest_mem_enabled(); @@ -45249,6 +45661,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) int i, k, n = 0, n_gr = 0; bool stalled = false; + /* + * r707e: every-pass candidate verify with the last-good ring pair. + * Cheap (<=16 parked candidates, one walk each) and idempotent; the + * first pass after the user-VAS candidate parks registers it, so the + * client's very first push reads resolve. + * + * c741: this walk calls svc_read_ap() -> vfio_dma_rw(), which takes + * the type1 iommu mutex and may sleep. It must run unlocked: the + * original placement under svc.chan_lock produced the re-entered + * "scheduling while atomic" class (PART 43 / PART 57) and stalled + * the pump (the pending draw-window launch was never issued). + * Hoisting it here keeps the same-pass ordering (it still runs + * before the per-channel loop) without the atomic section. + */ + if (svc_lastgood_ring_pa) + svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va, + svc_lastgood_ring_pa); + spin_lock_irqsave(&svc.chan_lock, flags); if (svc_rewind_pending) { svc_rewind_pending = false; @@ -45273,15 +45703,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } /* - * r707e: every-pass candidate verify with the last-good ring pair. - * Cheap (<=16 parked candidates, one walk each) and idempotent; the - * first pass after the user-VAS candidate parks registers it, so the - * client's very first push reads resolve. - */ - if (svc_lastgood_ring_pa) - svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va, - svc_lastgood_ring_pa); - /* * r507o stream-order: walk the CE channels BEFORE the GR channels. * The client's stream order places its H2D copies before the kernel * that consumes them, but the channels register in handle order (the @@ -45839,6 +46260,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) (void)svc_notify2d_write(vgpu, ch, true); } + /* + * X-DMA tracker fix: the USERD latch is gone, + * so the put-bounded walk below never runs and + * the client's own release push (the nvPush + * progress tracker, measured va=0x1200a0000 + * len=24 rel=0x120001000) is never decoded. + * Decode the ring head directly and apply that + * guest-authored payload; no USERD is + * fabricated and last_get/GP_GET stay put. + * Gated/rate-limited inside. + */ + svc_ddx_head_decode(vgpu, ch, rrc ? 0 : ring_pa, + rrc ? -1 : ring_ap); stalled = true; continue; } @@ -45849,6 +46283,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &get, sizeof(get)); + /* + * X-DMA tracker fix: a GP_PUT reset to 0 on a + * channel that had consumed entries means the DDX + * re-created its ring. The stale USERD block can + * still read coherent, so the normal walk does not + * run for the fresh ring; decode its head directly + * (same gated/rate-limited path as the missing- + * USERD case above). + */ + if (!put && ch->last_get) + svc_ddx_head_decode(vgpu, ch, + rrc ? 0 : ring_pa, + rrc ? -1 : ring_ap); } /* @@ -49249,6 +49696,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].fb_userd_scan_n = ch->fb_userd_scan_n; svc.chan[k].notify2d_va = ch->notify2d_va; + /* X-DMA tracker fix: head-decode pacing and + * the last decoded release are snapshot state + * too (the helper also stores them immediately + * on the stalled path, which skips this + * write-back). */ + svc.chan[k].ddx_head_at = ch->ddx_head_at; + svc.chan[k].ddx_cache_va = ch->ddx_cache_va; + svc.chan[k].ddx_cache_payload = + ch->ddx_cache_payload; + svc.chan[k].ddx_cache_valid = + ch->ddx_cache_valid; svc.chan[k].zero_held = ch->zero_held; svc.chan[k].userd_reg = ch->userd_reg; svc.chan[k].logged = ch->logged; @@ -49460,6 +49918,41 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * c727 host-initiated MME pack continuation. After the channel loop (so + * a client window decoded in this pass can complete the pack first), + * emit the next golden pack part for the armed beef channel. The gates + * mirror the request path: the channel must still be in this pass's + * snapshot, no unoffered client candidate may exist for it, no hold or + * in-flight request may own the slot, and the FE-busy defer must not + * veto a load-only carrier. gsp_phys_gr_mme_carry_take() re-validates + * the live pack cursor honestly; a mismatch cancels (never fires a + * stale carrier). + */ + { + u32 carry_h = 0; + + if (gsp_phys_gr_mme_carry_peek(&carry_h)) { + int ci; + + for (ci = 0; ci < n; ci++) { + struct svc_channel *c = &local[ci]; + + if (c->h_object != carry_h || c->engine_type) + continue; + if (svc_gr_cand_unoffered_for(carry_h)) + break; + if (svc_fwd_hold.active || + test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) + break; + if (gsp_phys_gr_mme_fe_defer_wanted(carry_h, 2)) + break; + (void)svc_gr_carrier_request(vgpu, c); + break; + } + } + } + + /* * A channel still has pending entries after the pass: its pushes either * fail translation or decode to nothing (stale/wrong VA->PA pair). A * fast content sweep applies the releases those pages hold, so the diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index f2a0777..ba1da96 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -686,6 +686,15 @@ int gsp_sema_gl_sph_blob_version(void); const u8 *gsp_sema_gl_prog_img(u64 va, u32 *len_out); /* + * eng52 (DS stage-image gate): the published VTG (vertex) SPH+SASS image, + * independent of the stage-typed gsp_sema_gl_prog_img fallback. The GR + * booter uses it to author the live VTG stage image at family+0x900 when + * the client's own VTG inline record is absent from the launch. NULL/0 + * when no image is loaded. + */ +const u8 *gsp_sema_gl_vs_img(u32 *len_out); + +/* * SESSION 153-CONTINUED-68: furthest IR instruction covered by the CAPTURED * 0x114 uploads (pc + ceil(code words / 3)). The GR launch builder places * its end-noop macro stub at this address (aligned past the replayed |