From 78cf0b1694dfc92b75b6e52e43e391b14cdaaf01 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sat, 24 Oct 2026 20:26:43 -0400 Subject: vgsp-sema r635: host-side operand resolution when the scanner misses - per-shadow-page REAL-content tracking (r->content set on every landed write) anchors a client's own H2D-shadowed pages and arms a launch-level content anchor, so an answerless output candidate (contiguous tail of a shadowed input, or a non-contiguous allocation) is served from a bounded synthetic page set (1024/candidate, 4096/launch, 3/4-table guard) instead of failing the launch closed; synthetic pages can never re-anchor (the first draft self-fed on its own minted pages and filled the shared table to the guard - 6144/8192 measured on a green boot - content gating restored 0 minted pages there and the historical green shape); trusted pin/walk/ledger answers keep the r633 anchored budgets and no longer arm the cross mint; fresh r636 boot: best-equal battery 9 OK / 2 clean FAIL / 1 HANG (streams2 bad=65534 - its H2D dsts are replay-denied, the GR forward defers on the CE stream-order watermark, degenerate refusal faults cleanly; streams_events hang->clean CUDA_ERROR via the auto-fault; smiley first-wait crawl) --- kernel/gsp_sema_service.c | 260 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 233 insertions(+), 27 deletions(-) diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index ec3764f..7426811 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2656,6 +2656,16 @@ static void svc_lat_store(u64 va, u64 addr, int ap) struct svc_vas_rec { u64 va_page; /* 0 = empty */ u8 *data; + /* + * r635: the page holds bytes a client/host WRITE actually landed + * (an H2D shadow-lane write, an inline write, a bridge write, an + * engine writeback). Synthetic pages minted by the GR fill serve + * zeros and must never be mistaken for the client's own content: + * only real content anchors a span or arms the launch-level anchor + * (without this flag the fill would re-anchor on its own minted + * pages and mint without bound across retry passes). + */ + u8 content; }; static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS]; @@ -2691,6 +2701,14 @@ static bool svc_vas_has(u64 va) return svc_vas_find(va) != NULL; } +/* r635: the page holds REAL written content (not a synthetic zero page). */ +static bool svc_vas_has_content(u64 va) +{ + struct svc_vas_rec *r = svc_vas_find(va); + + return r && r->content; +} + /* * A VA may receive a new shadow record only when it is engine-only device * memory: high in the client VA space, where the guest's own sysmem objects @@ -2764,6 +2782,7 @@ static int svc_vas_write(u64 va, const u8 *buf, size_t len) if (!r) return -ENOMEM; memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk); + r->content = 1; /* r635: real bytes landed here */ svc.vas_writes += chunk; buf += chunk; va += chunk; @@ -20646,6 +20665,12 @@ static struct svc_gr_synth_ep { u64 push_va; u64 base; /* span base; 0 = the launch-level total record */ u32 n; + /* + * r635: the cross-mint counter (launch-level anchor, see + * svc_gr_shadow_fill_serve). Tracked separately from n so a cross + * mint can never consume the anchored budgets and vice versa. + */ + u32 cross; } svc_gr_synth_ep[SVC_GR_SYNTH_EP_SLOTS]; static u32 svc_gr_synth_ep_pos; @@ -20687,28 +20712,123 @@ static void svc_gr_synth_ep_reset(void) svc_gr_synth_ep_pos = 0; } -static bool svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool anchored, - u64 push_va, u64 base) +/* + * r635 (host-side operand resolution without the scanner): the r634 red + * draws measured the remaining shape - a client whose INPUT buffers are + * shadow-CONTENT served (its own H2D bytes, the copy took the shadow lane) + * while its OUTPUT buffer (an argument candidate the client never wrote) + * still failed closed, because the r633 anchor gate only counted a page + * answered through ipin/resolve/ledger and every page of the output span + * was answerless. Two additions, both host-side (no scanner/pin needed): + * + * - a CONTENT-served page anchors its span exactly like a trusted answer: + * the client's own H2D bytes are first-hand truth, and the output tail + * of a contiguous allocation (matmul d_a|d_b|d_c, streams2's buffers) + * then mints synthetic pages through the unchanged per-candidate / + * per-launch budgets instead of dropping; + * - a LAUNCH-LEVEL anchor (cross): once any argument page of the launch + * is served from content or trust, an answerless ARG candidate whose + * span holds nothing at all (a non-contiguous output allocation) may + * still mint a strictly smaller cross-budget page set. This is what + * makes output buffers resolvable regardless of candidate order or + * allocation placement. + * + * Guards kept from r632/r633: only argument candidates mint, budgets are + * per launch episode, and every synthetic mint additionally respects + * SVC_GR_SHADOW_TABLE_GUARD - once the shared session shadow table is + * three-quarters full, no guess may displace the headroom real client + * content (H2D writes) still needs; content serves unconditionally. + */ +#define SVC_GR_SHADOW_NONE 0 +#define SVC_GR_SHADOW_CONTENT 1 +#define SVC_GR_SHADOW_SYNTH 2 +#define SVC_GR_SHADOW_SYNTH_CROSS 3 +/* + * Cross budgets: the r633 proven caps (1024 per candidate / 4096 per + * launch). They apply to CONTENT-anchored spans; the flood class they + * must never re-open (r632: 8192/8192 synthetic pages from answerless + * junk spans) is excluded by the r635 content gate - a green boot has + * no written shadow content at all, so the cross mint is off there, and + * an answerless junk span on a red boot has no head to anchor it. + */ +#define SVC_GR_SHADOW_CROSS_PER_CAND 1024u +#define SVC_GR_SHADOW_CROSS_MAX 4096u +#define SVC_GR_SHADOW_TABLE_GUARD (SVC_VAS_SLOTS * 3u / 4u) + +static int svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool trusted, + bool span_content, bool launch_content, + u64 push_va, u64 base, bool *real_content) { struct svc_gr_synth_ep *ep = NULL; bool have; + if (real_content) + *real_content = false; if (!svc_vas_can_shadow(va)) - return false; + return SVC_GR_SHADOW_NONE; have = svc_vas_has(va); if (!have) { - if (!is_arg || !anchored) - return false; - ep = svc_gr_synth_ep_get(push_va, base, true); - if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND) - return false; - if (svc_gr_synth_ep_launch_n(push_va) >= - SVC_GR_SHADOW_SYNTH_MAX) - return false; + if (!is_arg) + return SVC_GR_SHADOW_NONE; + if (trusted) { + /* + * r633 path: a trusted answer (ipin/resolver/ledger) + * anchors the span; the large r633 budgets apply. + */ + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND) + return SVC_GR_SHADOW_NONE; + if (svc_gr_synth_ep_launch_n(push_va) >= + SVC_GR_SHADOW_SYNTH_MAX) + return SVC_GR_SHADOW_NONE; + } else if (span_content || launch_content) { + /* + * r635 path: the anchor is the client's own written + * content (its H2D took the shadow lane), not a + * resolver answer. The cross budgets apply - smaller, + * because this serves the output tail of a red-draw + * client, not a trusted span. + */ + struct svc_gr_synth_ep *lep; + + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (ep && ep->cross >= SVC_GR_SHADOW_CROSS_PER_CAND) + return SVC_GR_SHADOW_NONE; + lep = svc_gr_synth_ep_get(push_va, 0, false); + if (lep && lep->cross >= SVC_GR_SHADOW_CROSS_MAX) + return SVC_GR_SHADOW_NONE; + } else { + return SVC_GR_SHADOW_NONE; + } } - if (!svc_vas_get(va, true)) - return false; - if (!have) { + { + struct svc_vas_rec *r = svc_vas_get(va, true); + + if (!r) + return SVC_GR_SHADOW_NONE; + if (have) { + /* + * r635: real_content tells the caller whether these + * bytes are the client's own WRITES (anchoring + * material) or a synthetic zero page minted by an + * earlier pass (servable, but never an anchor - the + * guard against the self-feeding mint loop). + */ + if (real_content) + *real_content = r->content != 0; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content%s)\n", + va, r->content ? "" : ", synthetic"); + } + return SVC_GR_SHADOW_CONTENT; + } + } + if (trusted) { struct svc_gr_synth_ep *lep = svc_gr_synth_ep_get(push_va, 0, true); @@ -20721,12 +20841,23 @@ static bool svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool anchored, va, ep->n, svc_gr_synth_ep_launch_n(push_va)); } - } else if (svc_gr_shadow_fill_log < 32) { - svc_gr_shadow_fill_log++; - pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content)\n", - va); + return SVC_GR_SHADOW_SYNTH; } - return true; + { + struct svc_gr_synth_ep *lep = svc_gr_synth_ep_get(push_va, 0, + true); + + if (ep) + ep->cross++; + if (lep) + lep->cross++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic cross; cand=%u launch=%u)\n", + va, ep ? ep->cross : 0, lep ? lep->cross : 0); + } + } + return SVC_GR_SHADOW_SYNTH_CROSS; } static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, @@ -20734,6 +20865,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, { u64 wins[SVC_GR_CAND_MAX]; bool is_arg[SVC_GR_CAND_MAX]; /* m150: candidate came from the arg scan */ + /* + * r635: launch-level CONTENT anchor for the host-side operand + * fallback (see the pre-pass and svc_gr_shadow_fill_serve). Armed + * only by the client's own WRITTEN shadow content (H2D bytes that + * took the shadow lane) - a resolver/pin/ledger answer is NOT + * enough, so a green boot cannot enable the cross mint; consumed + * by the cross-mint for answerless argument candidates. + */ + bool launch_arg_content = false; u32 n_cand; u32 off = 0, i, w; u8 kind; @@ -20958,6 +21098,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + /* + * r635: the launch-level CONTENT anchor for the host-side fallback, + * computed BEFORE the span loop so candidate order cannot decide. + * An argument candidate whose span already holds the client's own + * WRITTEN shadow bytes (r->content; a synthetic zero page does not + * count) marks the launch's operands as host-visible; an answerless + * ARG candidate (the output buffer the client never wrote, possibly + * not contiguous with the inputs) may then mint a strictly bounded + * cross-budget page set instead of failing the whole launch closed. + * Only the first SVC_GR_CROSS_ANCHOR_PROBE pages of each span are + * consulted - a real H2D'd buffer always has its head written, and + * the probe is a pure hash lookup. Trusted answers (pin/walk/ledger) + * do NOT arm this: on a green boot the cross mint stays off. + */ +#define SVC_GR_CROSS_ANCHOR_PROBE 64u + { + bool found = false; + + for (w = 0; w < n_cand && !found; w++) { + u32 p; + if (!is_arg[w] || wins[w] < 0x40000000000ULL) + continue; + for (p = 0; p < SVC_GR_CROSS_ANCHOR_PROBE; p++) { + if (svc_vas_has_content(wins[w] + + (u64)p * SEMA_PAGE_SIZE)) { + found = true; + break; + } + } + } + launch_arg_content = found; + } /* m110/B-2: one contiguous run of pages per candidate pointer, * relocated into the active engine-visible data region (grown region * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the @@ -20970,8 +21142,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, bool cand_anchored = false; /* r633: >= 1 page of this span * resolved through a trusted * source (ipin/resolve/ledger); - * the synthetic shadow lane only - * extends an anchored span */ + * the r633 large synthetic + * budgets only apply here */ + bool cand_content = false; /* r635: >= 1 page of this span + * holds the client's own + * WRITTEN shadow bytes; the + * cross budgets extend it */ bool first_unresolved = false; /* m170: span's first page failed */ u64 pba = 0; u32 pba_ap = 0; @@ -21046,6 +21222,8 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u32 ap = 0, x; struct svc_gr_data *rec; bool shadow_serve = false; + int serve = SVC_GR_SHADOW_NONE; + bool real_content = false; /* r635 */ if (va >= (1ULL << 47)) break; @@ -21090,11 +21268,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, pa = lpa; ap = lap; cand_anchored = true; - } else if (svc_gr_shadow_fill_serve(va, - is_arg[w], - cand_anchored, - svc_gr_cand.push_va, - wins[w])) { + } else if ((serve = svc_gr_shadow_fill_serve( + va, is_arg[w], + cand_anchored, + cand_content, + launch_arg_content, + svc_gr_cand.push_va, + wins[w], + &real_content)) != + SVC_GR_SHADOW_NONE) { /* * r632/r633: no resolver, no * landing - serve the page @@ -21116,6 +21298,27 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * resolved page. */ shadow_serve = true; + /* + * r635: REAL content (the + * client's own written H2D + * bytes, r->content) anchors + * the span for the cross + * budgets and arms the + * launch-level content + * anchor. A synthetic zero + * page served from an + * earlier pass is NOT an + * anchor (the self-feeding + * mint guard). + */ + if (serve == + SVC_GR_SHADOW_CONTENT && + real_content) { + cand_content = true; + if (is_arg[w]) + launch_arg_content = + true; + } } else { /* * m170 (R4a): the very first @@ -21190,7 +21393,10 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, } } else { /* r633: the trusted resolver answered - * this page - the span is anchored. */ + * this page - the span is anchored + * (the r633 large budgets apply; + * the r635 cross anchor stays + * content-only). */ cand_anchored = true; } } -- cgit v1.2.3