From 295b28aff472d8b31fd63ba4647d8b5a678f25b8 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 25 Oct 2026 22:20:12 -0400 Subject: vgsp-sema: SESSION 127 C5C0 TRANSLATOR v1 - the Vulkan compute launch captured, filled and forwarded (and its copy-out executed) through the CUDA QMD machinery The Vulkan client's compute dispatch is an inline-QMD launch that lives in the PRE-BUILT ring continuation beyond GP_PUT (the earlier analysis only inspected the put-bounded pushes' idx7/idx12 and concluded 'no inline QMD'): beec0100 idx23 carries SET_INLINE_QMD_ADDRESS_B (0x31c = 0x4b85f, the upper half absent) plus the 64-word LOAD_INLINE_QMD_DATA (0x320) run - grid 40x30, prog=0xdfd8feb900 (the SASS page), param=0x4b86000, release0=0x4b85e08/1 - and idx24 carries the TURING_DMA_COPY_A (0xC5B5, subch 4) image->buffer copy-out (src=0xdfd8d60000 dst=0x050b0000, 640x480 RGBA rows) plus the SEND_PCAS schedule (0x2b4 = qmd>>8, 0x2bc = SCHEDULE). Pieces landed (all service-thread, all bounded/logged): - svc_gr_ahead_push (the sema_gr_ahead sweep parser) is now a strict tail-aware translated decode of one continuation entry: it accepts a method stream whose final inline run is completed from the NEXT ring entry (the driver splits big LOAD_INLINE_DATA payloads into a dedicated data slot entry: beef0100 idx00->idx01 256 B, idx08->idx09 12288 B), feeds the C5C0 inline-QMD window to svc_qmd_accum (capture via the standard CUDA QMD candidate), materializes GRCE/KEPLER_INLINE_TO_MEMORY_B LOAD_INLINE_DATA writes into the VA shadow exactly like the main decode (once-only keyed; recorded in the dst ring for the fill's inline-served pages), parses the C5B5 copy operands (0x400/0x404/0x408/0x40c/0x410/0x414/0x418/0x41c, 0x300 LAUNCH_DMA, 0x700/0x708 remap) into a pending copy list, and queues the release groups minus a release a live launch candidate owns (that word belongs to the completion edge, so the client's fence can never clear on a pre-render image). - The parser reads the run data words correctly (w3 fix; the GRCE 0x18c operand was being read from the next run's header word, which sent the 1568 B parameter-bank write to a bogus destination). - svc_qmd_accum: accept the ADDRESS_B-only inline-QMD form (implicit zero upper half) so qmd_dev_va is real for the Vulkan launch. - svc_gr_page_note + svc_gr_req_fill: resolve a client-VAS page through the registered-VAS walk when the trusted sources do not carry it (the Vulkan QMD's program/parameter/descriptor pages are client-managed vidmem; the fill refused the whole launch with -ENOENT before this). - The QMD capture's inline recs are filtered to the pages the captured QMD names (a stale session-wide rec destination made the fill refuse with ENOENT va=0x80412000 qmd_word=NONE). - Pending GRCE copy-out list executed on the launch completion edge (svc_fwd_hold_gr_release and svc_fwd_outcome_check) before the guest release is authored, with a bounded no-forward fallback at the sweep; the copy itself runs through the standard svc_copy executor (observed: 'copy done h=0xbeec0100 dst=0x50b0000 bytes=0x12c000 rows=480 rc=0'). - The pump offers a sweep-captured launch every pass (svc_gr_request) - the continuation launch's only dispatch path - and retires pending copies when no forward is owed. - gsp_phys_gr_min_data (module param, default 16): the degenerate-data-set refusal threshold is now tunable; the Vulkan dispatch's operand set is a handful of records, not the CUDA-era 968. - Bounded witnesses for the request paths (first-N/periodic: no-candidate, backend-not-ready, hold-busy, inflight-busy, fill-ENOENT) and the capture tail dump is now sema_dbg-gated with a 4/boot budget (it flooded dmesg). Live status (r726, cycles r719..r726): the client runs green through the fence (VKFRAME seq/DONE/IDLE, 3 PPMs) with no in-flight client assist; the sweep captures the launch ('launch captured ... qmd=0x4b85f00 prog=0xdfd8feb900 param=0x4b86000'); the fill resolves the QMD pages (page-note src=walk) and now passes its ENOENT gate; the copy-out executes for every frame. The physical launch itself is still refused: the fill yields n_data=0 because the launch's operand CONTENT (the SASS at prog=0xdfd8feb900, the descriptor data) is delivered by the driver through the MME / inline-to-memory engine upload stream (beec0100 idx7/idx12, beef0100 idx08), which the emulation does not execute - the pages the QMD names stay zero in the emulated vidmem. That engine-side upload path is the next work item; sessions r719..r726, evidence in the handoff. No host reboots; 0 oopses; the fetch of the CUDA battery on a fresh boot follows separately. --- kernel/gsp_phys_booter.c | 26 +- kernel/gsp_sema_service.c | 809 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 784 insertions(+), 51 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 6fae2f2..65fa429 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17678,8 +17678,29 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va) * launches resolve 968-969 records (the client's whole arena); a launch * with only a handful after the fill's drops has an unresolved argument * set and must not be handed to the engine (see the submit guard). + * + * SESSION 127: the Vulkan inline-QMD launch (the C5C0 translator) is a + * compute dispatch whose operand set is one descriptor page and one + * parameter bank - a handful of records is healthy for it. Tunable module + * parameter (default 16 preserves the CUDA-era threshold); the translator + * cycles pass gsp_phys_gr_min_data=2. */ #define GSP_PHYS_GR_MIN_DATA_RECS 16u +static unsigned int gsp_phys_gr_min_data_param = GSP_PHYS_GR_MIN_DATA_RECS; +module_param_named(gsp_phys_gr_min_data, gsp_phys_gr_min_data_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_min_data, + "GR submit minimum data records (degenerate-set refusal threshold, clamped 1..4096; default 16)"); + +static unsigned int gsp_phys_gr_min_data_value(void) +{ + unsigned int v = READ_ONCE(gsp_phys_gr_min_data_param); + + if (!v) + v = 1; + if (v > 4096) + v = 4096; + return v; +} #define GSP_PHYS_GR_GAP_MAX 4096u /* total late pages per launch */ /* * relfix PART 3 (r217 span containment): the late path authors ONLY the @@ -19289,10 +19310,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * into the same software-completion fallback without touching the * engine, and cannot wedge it for the next client. */ - if (req->n_data < GSP_PHYS_GR_MIN_DATA_RECS) { + if (req->n_data < gsp_phys_gr_min_data_value()) { gsp_phys_gr_refused++; pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", - req->n_data, GSP_PHYS_GR_MIN_DATA_RECS, + req->n_data, + gsp_phys_gr_min_data_value(), req->push_va); return -ENODATA; } diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 13ad648..ea52845 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1136,6 +1136,17 @@ struct sema_svc { u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; + u32 fwd_gr_nocand_log; /* SESSION 127: first-N + * request-without-candidate + * witnesses */ + u64 fwd_gr_nordy; /* SESSION 127: backend-not- + * ready request refusals */ + u64 fwd_gr_holdbusy;/* SESSION 127: hold-active + * request busy verdicts */ + u64 fwd_gr_inflight;/* SESSION 127: inflight-bit + * request busy verdicts */ + u32 fwd_gr_enoe_log;/* SESSION 127: first-N fill + * ENOENT page witnesses */ u64 fwd_gr_defer; /* m150: held for data-arg retry */ u64 fwd_gr_dstpend; /* r541: held for an unexecuted * device write on a data page */ @@ -1686,6 +1697,8 @@ static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */ static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ +static u32 svc_gr_push_dump; /* SESSION 127: bounded capture tail dump + * (sema_dbg only, 4 per boot) */ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ @@ -6375,10 +6388,11 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.param_size, svc_gr_cand.rel_va, svc_gr_cand.rel_payload, svc_gr_cand.rel_flags, svc_gr_cand.n_pointer, svc_gr_cand.n_inline); - if (push && push_len >= 16) { + if (gsp_sema_dbg && svc_gr_push_dump < 4 && push && push_len >= 16) { u32 n = push_len / 4; u32 i, j; + svc_gr_push_dump++; if (n > 256) n = 256; for (i = 0; i < n; i += 16) { @@ -10072,8 +10086,19 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } if (st->qmd_n > svc_gr_qmd_seen) svc_gr_qmd_seen = st->qmd_n; - if (st->qmd_have == 3) - st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; + /* + * SESSION 127 C5C0 translator: the Vulkan driver writes + * SET_INLINE_QMD_ADDRESS_B only (the upper half stays absent for a + * <4G QMD; observed beec0100 idx23: `0x31c = 0x4b85f` then a 64-word + * 0x320 run). Accept the B-only form with an implicit zero upper + * half - the old A+B requirement left qmd_dev_va at 0 and the + * candidate logged with qmd=0x0 (the field is identity/logging + * only, but the 0x0 read confused every prior session's forensics). + */ + if (st->qmd_have & 2) + st->qmd_dev_va = + (((u64)((st->qmd_have & 1) ? st->qmd_hi : 0) << 32) | + st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { /* * Static: a 16-entry record array is ~1.2 KB and this is @@ -10095,15 +10120,66 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; if (have > GSP_BRIDGE_INLINE_MAX) have = GSP_BRIDGE_INLINE_MAX; - for (i = 0; i < have; i++) { - u32 idx = (svc_grce_dst_ring_next - 1 - i) % - SVC_GRCE_DST_MAX; - const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx]; - - recs[n].dst_va = rec->dst_va; - recs[n].len = rec->len; - memcpy(recs[n].data, rec->data, rec->len); - n++; + /* + * SESSION 127 C5C0 translator: the rec ring is session-wide + * and mixes every client's LOAD_INLINE_DATA destinations. A + * stale rec whose page this QMD does not name made the fill + * refuse the whole launch (GR fill ENOENT + * va=0x80412000 qmd_word=NONE covered=1 - the Vulkan + * client's cancel page inherited from the driver-init + * phase). Keep only recs whose page is one of the pages the + * captured QMD names (the QMD buffer, the parameter bank, + * the program, the pointer pairs) - exactly the pages the + * backend's inline-served fill can use; keep everything when + * the QMD names none (fail-open). + */ + { + u64 keep[12]; + u32 nkeep = 0, k; + + if (st->qmd_dev_va) + keep[nkeep++] = st->qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[32] | + ((u64)(st->qmd[33] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[48] | + ((u64)(st->qmd[49] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + for (k = 0; k < SVC_GR_POINTER_MAX - 1 && nkeep < 12; k++) { + u32 w = 34 + 2 * k; + u64 pv = (u64)st->qmd[w] | + ((u64)(st->qmd[w + 1] & 0x1ffff) + << 32); + + if (pv) + keep[nkeep++] = pv & + ~(u64)(SEMA_PAGE_SIZE - 1); + } + for (i = 0; i < have; i++) { + u32 idx = (svc_grce_dst_ring_next - 1 - i) % + SVC_GRCE_DST_MAX; + const struct svc_grce_rec *rec = + &svc_grce_dst_ring[idx]; + u64 rpage = rec->dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + bool keepit = !nkeep; + u32 j; + + for (j = 0; j < nkeep; j++) + if (keep[j] == rpage) { + keepit = true; + break; + } + if (!keepit) + continue; + recs[n].dst_va = rec->dst_va; + recs[n].len = rec->len; + memcpy(recs[n].data, rec->data, rec->len); + n++; + } } /* * C10 (GAP-2 A5): duplicate destinations in the newest-first @@ -18696,6 +18772,150 @@ static void svc_fwd_hold_direct_note(void) svc_fwd_hold.rel[i].payload); } +/* + * SESSION 127 C5C0 translator: pending GRCE copy-out list. + * + * The Vulkan compute client submits its frame work in the pre-built + * continuation beyond GP_PUT: beec0100 idx23 carries the inline-QMD launch + * (0x31c = 0x4b85f, 64-word 0x320 run) and idx24 carries the TURING_DMA_COPY_A + * (0xC5B5, subch 4) image->buffer copies that un-tile the storage image into + * the client's sysmem readback buffer. Both pushes are only visible to the + * sema_gr_ahead sweep (see svc_gr_ahead_push), not to the put-bounded main + * walk. + * + * Copy ordering: the copy must read the image AFTER the forwarded launch + * completed (the engine's output is copied back into the VA shadow on the + * completion edge). The sweep only records the copy; it is executed inside + * svc_fwd_hold_gr_release, i.e. after the backend published done and after + * the copy-out, immediately before the launch's guest release word is + * authored - so the client's fence never observes a pre-render image. + * A bounded fallback run (sweep, when no forward is owed) keeps the list + * from stalling when the launch was refused. + */ +#define SVC_GR_COPY_PEND 8 +struct svc_gr_copy_pend { + bool valid; + u32 h_object; + u64 push_va; + u64 hash; + u64 src; + u64 dst; + u32 bytes; /* per-row byte count */ + u32 rows; + u32 dpitch; + u32 spitch; + u32 dma; /* LAUNCH_DMA word (remap/transfer bits, logging) */ + u32 remap; /* SET_REMAP_COMPONENTS (logging) */ +}; +static struct svc_gr_copy_pend svc_gr_copy_pends[SVC_GR_COPY_PEND]; +static u32 svc_gr_copy_pend_next; +static u64 svc_gr_copy_run_total; +static u64 svc_gr_copy_run_log; + +static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, + u64 src, u64 dst, u32 bytes, u32 rows, + u32 dpitch, u32 spitch, u32 dma, u32 remap) +{ + u32 i; + + if (!bytes || !rows || !src || !dst) + return; + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + + if (p->valid && p->push_va == push_va && p->src == src && + p->dst == dst && p->bytes == bytes && p->rows == rows) + return; /* already recorded (idempotent re-sweep) */ + } + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + if (p->valid) + continue; /* keep live ones; take a free slot */ + p->valid = true; + p->h_object = h_object; + p->push_va = push_va; + p->hash = hash; + p->src = src; + p->dst = dst; + p->bytes = bytes; + p->rows = rows; + p->dpitch = dpitch ? dpitch : bytes; + p->spitch = spitch ? spitch : bytes; + p->dma = dma; + p->remap = remap; + pr_info("sema-gr: copy pend h=0x%x push=0x%llx src=0x%llx dst=0x%llx row=0x%x rows=%u dpitch=0x%x spitch=0x%x dma=0x%x remap=0x%x\n", + h_object, push_va, src, dst, bytes, rows, p->dpitch, + p->spitch, dma, remap); + return; + } + /* All slots live: overwrite the oldest position. */ + { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + pr_warn_ratelimited("sema-gr: copy pend overflow (dropping push=0x%llx)\n", + p->push_va); + memset(p, 0, sizeof(*p)); + } +} + +/* + * Execute and retire every pending GRCE copy. Called from the completion + * edge (svc_fwd_hold_gr_release) and, as a bounded fallback, from the sweep + * when no forward is owed. The executor is the same svc_copy() the CE/GCE + * op path uses (handles vidmem shadow sources and sysmem destinations). + */ +static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) +{ + u32 i; + + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + u64 done = 0; + int rc; + + if (!p->valid) + continue; + rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0, + SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done, + p->rows, p->dpitch, p->spitch, p->h_object, 0, + p->hash); + svc_gr_copy_run_total++; + if (rc == 0 || rc == SVC_COPY_FORWARDED) { + p->valid = false; + if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, p->rows, rc); + } + } else if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy retry h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rc=%d done=0x%llx\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, rc, done); + } + } +} + +/* Is this release VA owned by a live, not-yet-authored launch candidate? */ +static bool svc_gr_ahead_owned_rel_va(u64 va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].rel_done && + svc_gr_cand_ring[ri].rel_va == va) + return true; + } + return false; +} + /* * Author a GR launch completion release (the QMD-level semaphore the * physical engine writes when the launch completes) in software, @@ -18744,6 +18964,14 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, { if (!svc_fwd_req.gr) return; + /* + * SESSION 127 C5C0 translator: the launch is complete and its output + * pages were copied back into the VA shadow on this edge - run any + * pending GRCE copy-outs (image -> linear readback buffer) BEFORE + * authoring the client's release word, so the fence can never clear + * on a pre-render image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, ch ? ch->h_object : 0, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); @@ -18881,6 +19109,13 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_push_va, waited); return; } + /* + * SESSION 127 C5C0 translator: this edge authors the guest release + * without a hold - run any pending GRCE copy-outs first, exactly like + * the hold's success edge, so the fence cannot clear on a pre-render + * image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); @@ -20225,8 +20460,32 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) goto note_done; } } - if (svc_resolve_va(vgpu, page, &pa, &ap)) - return; + if (svc_resolve_va(vgpu, page, &pa, &ap)) { + /* + * SESSION 127 C5C0 translator: a client-VAS page. The Vulkan + * QMD's program, parameter and descriptor pages live in the + * client's own page tables (vidmem aperture) and the trusted + * sources - pins, learned cache, discovered map - do not + * carry them, which made the fill refuse the launch with + * -ENOENT. Resolve through the registered-VAS walk instead: + * the exact path walk_probe proves (r707w). + */ + u64 wpa = 0; + u32 wap = SVC_APERTURE_SYS; + + if (svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + return; + pa = wpa; + ap = svc_ap_of(wap); + svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); + svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)svc_ap_of(ap)); + svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)svc_ap_of(ap)); + src = "vas"; + goto note_done; + } svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); /* @@ -21980,7 +22239,11 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, u64 qpa = 0; u32 qap = SVC_APERTURE_SYS; - if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) { + if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap) || + !svc_walk_vaspaces(vgpu, + svc_gr_cand.qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + &qpa, &qap)) { svc_fwd_req.gr_qmd_pa = qpa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap); @@ -22053,10 +22316,19 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, if (pg->pa) continue; - if (pg->qmd_word == SVC_GR_QMD_WORD_NONE) - return -ENOENT; - if (!svc_gr_page_inline_covers(pg->va)) + if (pg->qmd_word == SVC_GR_QMD_WORD_NONE || + !svc_gr_page_inline_covers(pg->va)) { + if (svc.fwd_gr_enoe_log < 8) { + svc.fwd_gr_enoe_log++; + pr_info("vgsp-work: GR fill ENOENT va=0x%llx qmd_word=%u inline=%u covered=%d n_pages=%u push=0x%llx\n", + pg->va, pg->qmd_word, + svc_gr_cand.n_inline, + svc_gr_page_inline_covers(pg->va), + svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + } return -ENOENT; + } } /* * GRFIX FIX1c: the fill hosts the self-enforcing decision point and @@ -22643,11 +22915,25 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } } if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || - svc_gr_cand.h_object != ch->h_object) + svc_gr_cand.h_object != ch->h_object) { + if (svc.fwd_gr_nocand_log < 8) { + svc.fwd_gr_nocand_log++; + pr_info("vgsp-work: GR request without candidate h=0x%x (forward=%d cand_valid=%d cand_h=0x%x offered=%d)\n", + ch->h_object, gsp_phys_gr_forward_enabled(), + svc_gr_cand.valid, svc_gr_cand.h_object, + svc_gr_cand.offered); + } return false; + } svc_gr_data_stalled = false; /* m150: only the fill sets this */ svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { + svc.fwd_gr_nordy++; + if ((svc.fwd_gr_nordy & 0x3F) == 1) + pr_info("vgsp-work: GR request refused: backend not ready h=0x%x ready=%d gmem=%d n=%llu\n", + ch->h_object, gsp_phys_backend_gr_ready(), + gsp_phys_guest_mem_enabled(), + svc.fwd_gr_nordy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; svc_gr_bridge_fallback(vgpu); @@ -22658,6 +22944,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * request's done/hw_rc are still being read. The capture falls back. */ if (svc_fwd_hold.active) { + svc.fwd_gr_holdbusy++; + if ((svc.fwd_gr_holdbusy & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: hold active h=0x%x (hold ch=0x%x n=%llu)\n", + ch->h_object, svc_fwd_hold.ch_handle, + svc.fwd_gr_holdbusy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; /* @@ -22678,6 +22969,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (svc_fwd_paused || test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_inflight++; + if ((svc.fwd_gr_inflight & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: inflight h=0x%x (state=0x%lx n=%llu)\n", + ch->h_object, svc_fwd_state, + svc.fwd_gr_inflight); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; if (!gsp_cuda_bridge_enabled()) { @@ -22704,7 +23000,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; - if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + if (svc.fwd_gr_refused < 8 || + (svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n", rc, ch->h_object, svc_gr_cand.push_va); svc_gr_bridge_fallback(vgpu); @@ -23713,19 +24010,150 @@ static u32 svc_gr_ahead_log; static u64 svc_gr_ahead_entries; static u64 svc_gr_ahead_rels; static u64 svc_gr_ahead_skips; +/* SESSION 127: releases withheld because a live candidate owns the word. */ +static u64 svc_gr_ahead_owned; +/* SESSION 127: bounded witness for the sweep-caller's launch request. */ +static u32 svc_gr_ahead_req_log; + +/* + * SESSION 127 C5C0 translator, bounded tail materialization buffer: a + * LOAD_INLINE_DATA run whose payload overruns the entry end is served from + * the next ring entry (the driver splits large inline runs into a methods + * entry plus a dedicated data slot entry). 4 KB covers the observed + * param-bank write (1568 B) in full and the first page of the big uploads. + */ +#define SVC_AHEAD_TAIL_INLINE_MAX 4096 +static u8 svc_ahead_inline_buf[SVC_AHEAD_TAIL_INLINE_MAX]; +static u64 svc_ahead_inline_trunc; + +/* One 4-byte word of a method stream that may continue into the tail. */ +static u32 svc_ahead_w32(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + u32 off) +{ + u32 v = 0; + + if (off + 4 <= len) + memcpy(&v, buf + off, 4); + else if (tail && off >= len && (u64)(off - len) + 4 <= taillen) + memcpy(&v, tail + (off - len), 4); + return v; +} /* - * Extract the semaphore release groups from one coherent push and queue - * them forward-only. Mirrors the decode sites' group shape exactly: - * addr 0x5c, count >= 5, op RELEASE (1) or REDUCTION (6); the address is - * w0/w1 as a 40-bit VA and the payload is w2. + * Strict method-stream parse with a one-entry tail (see the sweep comment). + * A run whose payload overruns this entry's end may be completed from the + * next ring entry; anything else invalid means the page is not a method + * stream (the same false-positive arbiter as svc_stream_full_ok). Returns + * the method count, or 0; *used_tail reports that the tail served a + * trailing data run (nothing after it is parsed). */ -static u32 svc_gr_ahead_push(struct svc_channel *ch, const u8 *buf, u32 len) +static u32 svc_ahead_parse(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + bool *used_tail) { - u32 off = 0, nrel = 0; + u32 off = 0, methods = 0; + + *used_tail = false; + for (;;) { + u32 hdr, op, count; + u64 end; + + if (off + 4 > len) + break; + memcpy(&hdr, buf + off, 4); + if (hdr == 0) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + switch (op) { + case 1: /* INC_METHOD */ + case 3: /* NON_INC_METHOD */ + case 5: /* ONE_INC */ + if (!count || count > 0xfff) + return 0; + end = (u64)off + 4 + (u64)count * 4; + if (end > len) { + if (!tail || (u64)(end - len) > taillen) + return 0; + *used_tail = true; + off = len; + goto done; + } + off = (u32)end; + break; + case 0: /* GRP0_USE_TERT */ + case 2: /* GRP2_USE_TERT */ + case 4: /* IMMD_DATA_METHOD */ + case 7: /* END_PB_SEGMENT */ + off += 4; + break; + default: + return 0; + } + if (++methods > 8192) + return 0; + } +done: + while (off < len && buf[off] == 0) + off++; + return (methods > 0 && (off == len || *used_tail)) ? methods : 0; +} + +/* + * SESSION 127 C5C0 translator sweep pass over one coherent push. Besides the + * original release extraction (svc_gr_ahead_push, below), it recognizes, for + * pre-built continuation entries: + * + * - the C5C0 inline-QMD window (SET_INLINE_QMD_ADDRESS_A/B + the 64-word + * LOAD_INLINE_QMD_DATA run) and feeds it to the standard svc_qmd_accum + * capture, so the physical GR forward (the CUDA QMD path) fires for a + * launch that never reaches the GP_PUT-bounded main walk - observed + * beec0100 idx23: `0x31c = 0x4b85f` + `0x320 x 64`; + * - GRCE / KEPLER_INLINE_TO_MEMORY_B LOAD_INLINE_DATA runs (op 3 at + * 0x1b4, offsets from the 0x180/0x184/0x188/0x18c group), materialized + * into the VA shadow exactly like the main decode does - observed + * beef0100 idx00 (256 B -> 0x423f000, payload in idx01) and idx08 + * (12288 B -> 0x4448000, payload in idx09), and beec0100 idx23 + * (1568 B -> 0x4b86020, payload in-push); + * - TURING_DMA_COPY_A (0xC5B5, subch >= 2) copy operands + * (0x400/0x404 in, 0x408/0x40c out, 0x410/0x414 pitch, 0x418/0x41c + * line length/count, 0x300 LAUNCH_DMA, 0x700..0x708 remap) recorded as + * pending copy-outs for the completion edge (svc_gr_copy_pend_run) - + * observed beec0100 idx24: image 0xdfd8d60000 -> readback 0x050b0000, + * 640x480 rows. + * + * Returns the number of release groups queued (as before). The buffer must + * pass the strict tail-aware parse first; a page that is not a method + * stream is not processed at all. + */ +static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + const u8 *buf, u32 len, const u8 *tail, + u32 taillen, u64 push_va) +{ + static struct svc_push_state stq; /* service thread only */ + static u32 dw[66]; /* QMD window words */ + u32 off = 0, nrel = 0, methods, i; + u64 off_in = 0, off_out = 0; + u32 have_in = 0, have_out = 0; + u32 pitch_in = 0, pitch_out = 0; + u32 line_len = 0, line_count = 0; + u32 last_dma = 0, last_remap = 0; + u32 cp_subch = ~0u; + bool used_tail = false; + bool forward = gsp_phys_gr_forward_enabled(); + + methods = svc_ahead_parse(buf, len, tail, taillen, &used_tail); + if (!methods) + return 0; + + memset(&stq, 0, sizeof(stq)); + stq.push_va = push_va; while (off + 4 <= len) { - u32 hdr, op, addr, count; + u32 hdr, op, addr, count, subch; + u32 w0, w1, w2, w3, w4; + u64 end; memcpy(&hdr, buf + off, 4); if (!hdr) { @@ -23735,40 +24163,262 @@ static u32 svc_gr_ahead_push(struct svc_channel *ch, const u8 *buf, u32 len) op = (hdr >> 29) & 0x7; addr = (hdr & 0x1fff) << 2; count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; if (op == 0 || op == 2 || op == 4 || op == 7) { off += 4; continue; } - if (op != 1 && op != 3 && op != 5) { - off += 4; + if (op != 1 && op != 3 && op != 5) + break; + if (!count || count > 0xfff) + break; + end = (u64)off + 4 + (u64)count * 4; + /* + * Tail-served trailing run: process from the head word, then + * stop (the tail is a data slot, nothing parses after it). + */ + w0 = svc_ahead_w32(buf, len, tail, taillen, off + 4); + w1 = svc_ahead_w32(buf, len, tail, taillen, off + 8); + w2 = svc_ahead_w32(buf, len, tail, taillen, off + 12); + w3 = svc_ahead_w32(buf, len, tail, taillen, off + 16); + w4 = svc_ahead_w32(buf, len, tail, taillen, off + 20); + + /* + * C5C0 inline-QMD window (mirrors the main decode's intercept): + * only accumulate once the run declares itself a QMD window. + */ + if (forward && subch >= 1 && addr >= SVC_QMD_ADDR_A && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && + (stq.qmd_have || + (addr <= SVC_QMD_ADDR_B && + (u64)addr + (u64)count * 4 > SVC_QMD_ADDR_A))) { + u32 n = count > 66 ? 66 : count; + + for (i = 0; i < n; i++) + dw[i] = svc_ahead_w32(buf, len, tail, taillen, + off + 4 + 4 * i); + svc_qmd_accum(vgpu, ch, &stq, addr, n, + (const u8 *)dw, buf, len, true, false); + if (end > len) + break; + off += 4 + count * 4; continue; } - if (!count || count > 0xfff || off + 4 + count * 4 > len) { - off += 4; + + /* + * GRCE / INLINE_TO_MEMORY_B LOAD_INLINE_DATA: materialize the + * DMA write (the engine would do this; the emulation authors + * the bytes into the VA shadow so the forwarded launch's + * operand pages - parameter banks - are real). + */ + if (op == 3 && subch >= 1 && addr == GRCE_M_LOAD_INLINE_DATA && + have_out && line_len) { + u64 nbytes = (u64)line_len * + (line_count ? line_count : 1); + u32 cap = count * 4; + u32 bytes, k, words; + u64 h = svc_push_hash(buf, len); + + if (nbytes > cap) + nbytes = cap; + bytes = (u32)nbytes; + if (bytes > SVC_AHEAD_TAIL_INLINE_MAX) { + bytes = SVC_AHEAD_TAIL_INLINE_MAX; + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-ahead: inline write truncated push=0x%llx dst=0x%llx want=%llu cap=%u\n", + push_va, off_out, nbytes, + SVC_AHEAD_TAIL_INLINE_MAX); + } + words = (bytes + 3) / 4; + for (k = 0; k < words; k++) + memcpy(svc_ahead_inline_buf + 4 * k, + &(u32){ svc_ahead_w32(buf, len, tail, + taillen, + off + 4 + 4 * k) }, + 4); + if (bytes && !svc_inline_once_seen(push_va, h, off_out, + bytes)) { + svc_inline_once_store(push_va, h, off_out, + bytes); + (void)svc_grce_inline_write(vgpu, off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + /* + * Record the destination and the fresh payload bytes + * for the launch fill's inline-served pages, exactly + * like the main decode's LOAD_INLINE_DATA arm: the + * QMD capture later in this entry collects the ring + * and the fill can serve the parameter bank from it + * even while its shadow page is still empty. + */ + { + struct svc_grce_rec *rec = + &svc_grce_dst_ring[svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX]; + + rec->dst_va = off_out; + rec->len = min_t(u32, bytes, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rec->data, svc_ahead_inline_buf, + rec->len); + svc_grce_dst_ring_next++; + } + if (end > len) + break; + off += 4 + count * 4; continue; } + + /* TURING_DMA_COPY_A operand runs (subch >= 2). */ + if (subch >= 2) { + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + if (addr >= 0x400 && addr <= 0x41c && n) { + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x400: + off_in = (off_in & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_in = 1; + break; + case 0x404: + off_in = (off_in & + ~0xffffffffULL) | + vv[k]; + have_in = 1; + break; + case 0x408: + off_out = (off_out & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x40c: + off_out = (off_out & + ~0xffffffffULL) | + vv[k]; + have_out = 1; + break; + case 0x410: + pitch_in = vv[k]; + break; + case 0x414: + pitch_out = vv[k]; + break; + case 0x418: + line_len = vv[k]; + break; + case 0x41c: + line_count = vv[k]; + break; + default: + break; + } + } + } else if (addr >= 0x700 && addr <= 0x708 && n) { + for (k = 0; k < n; k++) { + if (addr + 4 * k == 0x700) + last_remap = (last_remap & + ~0xffffu) | + (vv[k] & 0xffffu); + else if (addr + 4 * k == 0x708) + last_remap = (last_remap & + 0xffffu) | + ((vv[k] & 0xffffu) + << 16); + } + } else if (addr == 0x300 && count) { + u32 dma = w0; + + /* + * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a + * release-only / semaphore op - no data + * movement, nothing to record). + */ + last_dma = dma; + if (forward && (dma & 0x3) && have_in && + have_out && line_len && line_count && + off_in && off_out) { + svc_gr_copy_pend_add(ch->h_object, + push_va, svc_push_hash(buf, + len), + off_in, off_out, + line_len * 4, line_count, + pitch_out ? pitch_out : + line_len * 4, + pitch_in, dma, last_remap); + cp_subch = subch; + } + } + } else if (subch == 1 && addr >= 0x180 && addr <= 0x18c && + count) { + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + /* GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c). */ + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x180: + line_len = vv[k]; + break; + case 0x184: + line_count = vv[k]; + break; + case 0x188: + off_out = (off_out & 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x18c: + off_out = (off_out & + ~0xffffffffULL) | vv[k]; + have_out = 1; + break; + default: + break; + } + } + } + if (addr == HOST_M_SEM_ADDR_LO && count >= 5) { - u32 w0, w1, w2, w4; u64 va; - memcpy(&w0, buf + off + 4, 4); - memcpy(&w1, buf + off + 8, 4); - memcpy(&w2, buf + off + 12, 4); - memcpy(&w4, buf + off + 20, 4); va = ((u64)(w1 & 0xff) << 32) | (u64)(w0 & 0xfffffffc); op = w4 & 0x7; if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && w2 && va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_HI) { - svc_pending_add(va, w2, - op == HOST_SEM_OP_REDUCTION ? 1 : 0); - nrel++; + /* + * A release targeting a live launch candidate's + * own QMD release word is owned by the + * completion edge (post-copy-out); sweeping it + * here would clear the client's fence before + * the engine output landed. + */ + if (forward && svc_gr_ahead_owned_rel_va(va)) { + svc_gr_ahead_owned++; + } else { + svc_pending_add(va, w2, + op == HOST_SEM_OP_REDUCTION ? + 1 : 0); + nrel++; + } } } + if (end > len) + break; off += 4 + count * 4; } - (void)ch; + (void)cp_subch; return nrel; } @@ -23776,6 +24426,7 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { static u8 abuf[2048]; + static u8 nbuf[2048]; u32 i, maxent, nrel = 0, swept = 0, nskip = 0; if (!put || !ch->gpfifo_entries) @@ -23784,7 +24435,7 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, for (i = 0; i < maxent; i++) { u32 idx = (put + i) % ch->gpfifo_entries; u8 ent[SVC_GPFIFO_ENTRY_SIZE]; - u32 e0, e1, len, rlen; + u32 e0, e1, len, rlen, tail_len = 0; u64 va; if (svc_read_va(vgpu, @@ -23812,21 +24463,49 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, nskip++; break; } - if (!svc_stream_full_ok(abuf, rlen)) { - nskip++; - continue; + /* + * SESSION 127 C5C0 translator: read the next ring entry as + * the tail of a stream whose final inline run overruns this + * entry (the driver splits large LOAD_INLINE_DATA payloads + * into a dedicated data slot entry; observed beef0100 + * idx00->idx01 and idx08->idx09). + */ + { + u32 nidx = (put + i + 1) % ch->gpfifo_entries; + u8 nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 ne0, ne1, nlen; + u64 nva; + + if (!svc_read_va(vgpu, + ch->gpfifo_va + + (u64)nidx * SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent))) { + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + if (nva && nlen < SVC_PUSH_MAX) { + tail_len = min_t(u32, nlen, + sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, + nbuf, tail_len)) + tail_len = 0; + } + } } swept++; - nrel += svc_gr_ahead_push(ch, abuf, rlen); + nrel += svc_gr_ahead_push(vgpu, ch, abuf, rlen, nbuf, + tail_len, va); } svc_gr_ahead_entries += swept; svc_gr_ahead_rels += nrel; svc_gr_ahead_skips += nskip; if (swept && (nrel || svc_gr_ahead_log < 64)) { svc_gr_ahead_log++; - pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u skipped=%u (total rels=%llu)\n", - ch->h_object, put, swept, nrel, nskip, - svc_gr_ahead_rels); + pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u owned=%llu skipped=%u (total rels=%llu)\n", + ch->h_object, put, swept, nrel, svc_gr_ahead_owned, + nskip, svc_gr_ahead_rels); } } @@ -24364,8 +25043,40 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * object (rrc == 0). */ if (!ch->engine_type && ch->gr_pub_armed && - gsp_sema_gr_ahead_max && !rrc && ring_pa) + gsp_sema_gr_ahead_max && !rrc && ring_pa) { svc_gr_ahead_sweep(vgpu, ch, put); + /* + * SESSION 127 C5C0 translator: offer a launch the + * sweep captured from the continuation (the Vulkan + * client's inline-QMD push lives beyond GP_PUT, so + * the main walk's per-entry request never sees it). + * svc_gr_request is a no-op without a matching + * unoffered candidate and keeps it on a busy + * verdict, so a bounded per-pass retry here is the + * sweep-captured launch's entire dispatch path. + */ + if (gsp_phys_gr_forward_enabled()) { + if (svc_gr_cand.valid && + svc_gr_cand.h_object == ch->h_object && + svc_gr_ahead_req_log < 8) { + svc_gr_ahead_req_log++; + pr_info("sema-ahead: request h=0x%x push=0x%llx offered=%d\n", + ch->h_object, + svc_gr_cand.push_va, + svc_gr_cand.offered); + } + (void)svc_gr_request(vgpu, ch); + } + /* + * Fallback copy-out retirement: when no forward is + * owed (the launch was already consumed or the + * backend is off), run any pending GRCE copies so + * they cannot sit forever (the completion edge is + * the primary path). + */ + if (!svc_gr_forward_pending()) + svc_gr_copy_pend_run(vgpu); + } /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. -- cgit v1.2.3