From 6e7bad0791d9ff64531e1d670bfafa59a4c7fb63 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 2 Oct 2026 05:04:44 -0400 Subject: gsp-phys: P6b wave 1 - shadow-forward guest CE copies to the physical engine - the sema service captures the first eligible copy per push from the software resolver's own operands (the real push shape mixes guest RAM with emulated-VRAM shadow addresses, including physical destinations) - HPA resolution: guest RAM through the shared guest-RAM file page cache, emulated VRAM through the mapping of its kernel shadow - new scratch page-table pool: 16 zeroed page-table pages with their page-directory entries built before any engine access, handing out one whole 128-byte GMMU PTE fetch line per copy pair (16 PTEs, 512 pairs per boot, never reused; a PTE written into an already-fetched line is not observed and there is no invalidate path) - the backend submits the copy on the module-owned channel and verifies the result host-side; every miss falls back to the ladder shadow copy, and the software pump remains authoritative - live (full smoke, both parameters on): req=42 ok=42 verify_ok=42 verify_fail=0 fallback=0, guest payload roundtrip ALL OK 64/64, clean latched release; defaults stay off --- kernel/gsp_phys_booter.c | 399 +++++++++++++++++++++++++++++++++++++++++++++- kernel/gsp_phys_booter.h | 30 ++++ kernel/gsp_sema_service.c | 362 +++++++++++++++++++++++++++++++++++++++-- kernel/gsp_sema_service.h | 17 ++ 4 files changed, 791 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 765232c..90a0a27 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -52,6 +52,7 @@ #include "gsp_phys_falcon.h" #include "gsp_phys_fwsec.h" #include "gsp_phys_gpu.h" +#include "gsp_sema_service.h" #include "gsp_phys_booter_load_img.h" #include "gsp_phys_gsp_rm_boot_img.h" #include "gsp_phys_registry_img.h" @@ -336,6 +337,35 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_VA_GUEST_S (GSP_PHYS_RMCALL_VA_BASE + 0x83000) +/* + * P6b wave-1 guest-operand scratch window. The engine's GMMU caches PTE + * fetches at 128-byte (16 PTE) granularity with no invalidate path, so a + * runtime guest mapping must land on a full 128-byte line the engine has + * never fetched. A 64-byte half-line is not enough: two adjacent copy pairs + * share one 128-byte fetch line and, once the engine has cached it, the + * second pair's PTE writes are invisible (live P6b evidence: the first copy + * of a pair verified, every following copy on the same fetch line timed out + * with sem=0). This window is a pool of GSP_PHYS_RMCALL_GUEST_PT_PAGES fresh + * 2 MB PD0 slots (indices 1..16, right above the ladder's PD0 slot 0) each + * backed by its own 4 KB small-page PT; every runtime guest-HPA copy pair + * consumes one whole never-used 128-byte line, writing the source PTE at + * small-PT slot 16*line and the destination PTE at 16*line+1 within the page + * (a line holds GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES = 16 PTEs, so one copy + * pair owns the fetch line and the remaining 14 slots are never written). + * Lines are allocated monotonically and never reused, so a cached line can + * never shadow a later mapping. All PD0 entries are built once in + * gsp_phys_rmcall_pdb_build (before any engine access) and every PTE page + * starts zeroed (invalid). 16 pages x 32 lines = 512 guest copies per boot + * with no reuse. + */ +#define GSP_PHYS_RMCALL_VA_GUEST_PT_BASE 0x00200000ULL +#define GSP_PHYS_RMCALL_GUEST_PT_PAGES 16u +/* One GMMU PTE fetch line is 128 B = 16 x 8 B PTEs; a pair consumes a line. */ +#define GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES 16u +/* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */ +#define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE \ + ((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) + /* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 @@ -420,6 +450,28 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_PD1, GSP_PHYS_RMCALL_BUF_PD0, GSP_PHYS_RMCALL_BUF_PT, + /* + * P6b wave-1 guest scratch page tables: GSP_PHYS_RMCALL_GUEST_PT_PAGES + * fresh 4 KB PT pages behind PD0 entries 1..16 (never-reused operand + * VA lines). Every proven allocation is a single-page memdesc, so each + * PD0 slot gets its own chunk instead of sharing one multi-page chunk. + */ + GSP_PHYS_RMCALL_BUF_GUEST_PT0, + GSP_PHYS_RMCALL_BUF_GUEST_PT1, + GSP_PHYS_RMCALL_BUF_GUEST_PT2, + GSP_PHYS_RMCALL_BUF_GUEST_PT3, + GSP_PHYS_RMCALL_BUF_GUEST_PT4, + GSP_PHYS_RMCALL_BUF_GUEST_PT5, + GSP_PHYS_RMCALL_BUF_GUEST_PT6, + GSP_PHYS_RMCALL_BUF_GUEST_PT7, + GSP_PHYS_RMCALL_BUF_GUEST_PT8, + GSP_PHYS_RMCALL_BUF_GUEST_PT9, + GSP_PHYS_RMCALL_BUF_GUEST_PT10, + GSP_PHYS_RMCALL_BUF_GUEST_PT11, + GSP_PHYS_RMCALL_BUF_GUEST_PT12, + GSP_PHYS_RMCALL_BUF_GUEST_PT13, + GSP_PHYS_RMCALL_BUF_GUEST_PT14, + GSP_PHYS_RMCALL_BUF_GUEST_PT15, GSP_PHYS_RMCALL_BUF_COUNT }; @@ -993,6 +1045,16 @@ static struct gsp_phys_boot_stage { bool rmcall_guest_r_ok; /* trigger 7: guest page read proved */ bool rmcall_guest_w2g_ok; /* trigger 6: guest page write proved */ bool rmcall_guest_selftest_ok; /* trigger 8: fresh-line PTE write proved */ + /* + * P6b wave-1 guest forwarding pool. Line index L maps to scratch PT + * page L / LINES_PER_PAGE and 128-byte line-in-page L % LINES_PER_PAGE; + * each line holds one copy pair (src PTE then dst PTE, 14 PTEs spare). + * rmcall_guest_pt_next_line is monotonic for the life of a primed + * staging: a consumed line is never reused, so no engine-cached 128 B + * PTE fetch line can alias a later guest mapping. + */ + u64 rmcall_va_guest_pt; /* guest scratch-PT VA window base */ + u32 rmcall_guest_pt_next_line; /* next never-used 128 B PTE line */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -1545,6 +1607,8 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_guest_r_ok = false; boot_stage.rmcall_guest_w2g_ok = false; boot_stage.rmcall_guest_selftest_ok = false; + boot_stage.rmcall_va_guest_pt = 0; + boot_stage.rmcall_guest_pt_next_line = 0; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -5496,7 +5560,12 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0) return 0; } -/* One 4 KB coherent chunk per ladder buffer; all freed by rm-call fini. */ +/* + * One coherent chunk per ladder buffer; all freed by rm-call fini. Every + * buffer is one 4 KB page. The P6b guest scratch pool is + * GSP_PHYS_RMCALL_GUEST_PT_PAGES separate single-page chunks (one per PD0 + * slot), which is the only shape the server accepts for client memdescs. + */ static int gsp_phys_rmcall_alloc_chunks(void) { u32 i; @@ -5511,7 +5580,7 @@ static int gsp_phys_rmcall_alloc_chunks(void) i); return -EINVAL; } - if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1)) { + if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1u)) { pr_err("gsp-phys-boot: rm-call: buffer %u allocation failed\n", i); return -ENOMEM; @@ -5552,7 +5621,7 @@ static int gsp_phys_rmcall_stage2(void __iomem *bar0) } } - pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt zeroed)\n", + pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt/guest_pt0..15 zeroed)\n", GSP_PHYS_RMCALL_BUF_COUNT); return 0; } @@ -5783,6 +5852,7 @@ static void gsp_phys_rmcall_pdb_build(void) u32 src_idx = (u32)((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ffu); u32 dst_idx = (u32)((GSP_PHYS_RMCALL_VA_DST >> 12) & 0x1ffu); u32 sem_idx = (u32)((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ffu); + u32 i; u64 root_ent, pd2_ent, pd1_ent, pd0_small; u64 pt_gpfifo, pt_pb, pt_src, pt_dst, pt_sem; @@ -5870,6 +5940,29 @@ static void gsp_phys_rmcall_pdb_build(void) boot_stage.rmcall_guest_gpa, gpa_ret); } } + /* + * P6b wave-1 scratch PT hierarchy: PD0 entries 1..16 (the 2 MB slots + * right above the ladder window) carry the GSP_PHYS_RMCALL_GUEST_PT_PAGES + * single-page PT chunks (GUEST_PT0..GUEST_PT15). Only the small half of + * each dual entry is used. Writing them here, before any engine access, + * means the first fetch of any 128 B operand line already loads a valid + * hierarchy; the leaf PTEs stay zero (invalid) until the runtime + * allocator consumes a line, and the monotonic allocator never rewrites + * a consumed line. + */ + for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_PAGES; i++) { + u64 guest_pt_pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT0 + i], 0)); + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)(1 + i) * 16); + put_unaligned_le64(guest_pt_pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)(1 + i) * 16 + 8); + } dma_wmb(); boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE; @@ -5878,6 +5971,7 @@ static void gsp_phys_rmcall_pdb_build(void) boot_stage.rmcall_va_src = GSP_PHYS_RMCALL_VA_SRC; boot_stage.rmcall_va_dst = GSP_PHYS_RMCALL_VA_DST; boot_stage.rmcall_va_sem = GSP_PHYS_RMCALL_VA_SEM; + boot_stage.rmcall_va_guest_pt = GSP_PHYS_RMCALL_VA_GUEST_PT_BASE; boot_stage.rmcall_pdb_root = root_ent; boot_stage.rmcall_pdb_pd2 = pd2_ent; boot_stage.rmcall_pdb_pd1 = pd1_ent; @@ -5909,6 +6003,11 @@ static void gsp_phys_rmcall_pdb_build(void) src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0), dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0), sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0)); + pr_info("gsp-phys-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n", + GSP_PHYS_RMCALL_GUEST_PT_PAGES, + GSP_PHYS_RMCALL_VA_GUEST_PT_BASE, + GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); } /* @@ -6464,6 +6563,246 @@ void gsp_phys_backend_work_counters(u64 *submitted, u64 *released, } EXPORT_SYMBOL_GPL(gsp_phys_backend_work_counters); +/* + * P6b wave-1 guest-operand forwarding API (requires gsp_phys_work_forward=1 + * AND gsp_phys_guest_mem=1; both default off). It runs one bounded CE copy + * on the ladder channel with both operands mapped to real guest RAM by HPA, + * consuming one whole fresh, never-reused 128 B PTE fetch line per copy pair + * from the guest scratch PT pool: PD0 entries 1..16 -> 16 small-PT pages, + * line L = page L / 32, small-PT index 16 * (L % 32) for src and +1 for dst + * (a full GMMU fetch line per pair; 64 B half-lines shared by two pairs go + * stale because the GMMU caches at 128 B). No guest VA is ever submitted: the + * caller passes translated HPAs only. + * + * Shadow mode: the software pump has already applied the same bytes, so a + * successful hardware submission is proven by the translated-HPA submission + * being accepted, the module semaphore releasing, and the direct-map verify + * comparing source and destination (identity domain + pfn_valid make __va + * safe). A late hardware write can theoretically race a guest reuse of the + * page; the window is one ordered workqueue hop plus the bounded poll + * (<= 200 ms) and wave 2 removes it by making hardware primary. + * + * Counters: _fwd_ok counts verified copies, _verify_ok/_verify_fail the + * direct-map comparison, _pool_exhausted exhausted line allocations and + * _refused every attempt that produced no verified copy for another reason + * (gate, structural, submit or poll timeout). Plain increments from the + * sema service work item; only logging and the debugfs state file read them. + */ +static u64 gsp_phys_guest_fwd_ok; +static u64 gsp_phys_guest_verify_ok; +static u64 gsp_phys_guest_verify_fail; +static u64 gsp_phys_guest_pool_exhausted; +static u64 gsp_phys_guest_refused; + +void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail, + u64 *pool_exhausted, u64 *refused) +{ + if (ok) + *ok = gsp_phys_guest_fwd_ok; + if (verify_ok) + *verify_ok = gsp_phys_guest_verify_ok; + if (verify_fail) + *verify_fail = gsp_phys_guest_verify_fail; + if (pool_exhausted) + *pool_exhausted = gsp_phys_guest_pool_exhausted; + if (refused) + *refused = gsp_phys_guest_refused; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters); + +/* + * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool + * and write both operand PTEs into it (src at 16*line_in_page, dst at + * 16*line_in_page + 1). Called with gsp_phys_rm_lock held, so the monotonic + * line index and the PTE writes are serialised. A whole 128 B line is + * consumed per copy pair and never reused: the GMMU caches PTE fetches at + * 128 B granularity and there is no invalidate, so a later write into an + * already-fetched line is invisible - sharing a 128 B line between copy pairs + * (one 64 B half each) fails exactly this way in live P6b runs, where only + * the first copy of the shared line took effect and every following copy + * timed out with sem=0. The scratch PT pages were zeroed at stage 2, so every + * unconsumed slot holds an invalid PTE. Returns -ENOSPC when the pool is + * exhausted and -EIO when the scratch PT pages are not allocated. + */ +static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, + u64 *src_va, u64 *dst_va) +{ + struct gsp_phys_boot_chunk *guest_pt_chunk; + u32 line = boot_stage.rmcall_guest_pt_next_line; + u32 page, line_in_page; + u64 va; + u8 *pt; + + if (!boot_stage.rmcall_va_guest_pt) + return -EIO; + if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + return -ENOSPC; + page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + guest_pt_chunk = &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; + if (!guest_pt_chunk->pages) + return -EIO; + /* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */ + va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + ((u64)line_in_page << 16); + pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0); + if (!pt) + return -EIO; + put_unaligned_le64(gsp_phys_rmcall_pte(src_hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page) * 8); + put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page + 1) * 8); + dma_wmb(); + boot_stage.rmcall_guest_pt_next_line = line + 1; + + *src_va = va; + *dst_va = va + GSP_PHYS_PAGE_SIZE; + return 0; +} + +int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, + u32 dst_off, u32 bytes, u32 poll_ms) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + void __iomem *bar0; + unsigned long deadline; + u64 src_va = 0, dst_va = 0; + u32 poll = 0; + int ret; + + /* + * Every gate is fail-closed: the sema service's ladder fallback keeps + * the software release authoritative, so a refusal never loses work. + */ + if (!gsp_phys_backend_work_ready()) { + gsp_phys_guest_refused++; + return -EAGAIN; + } + if (!gsp_phys_guest_mem_enabled() || + gsp_phys_dma_domain_identity() != 1 || + !gsp_phys_guest_ram_active()) { + gsp_phys_guest_refused++; + return -EAGAIN; + } + if (!src_hpa || !dst_hpa || + !IS_ALIGNED(src_hpa, GSP_PHYS_PAGE_SIZE) || + !IS_ALIGNED(dst_hpa, GSP_PHYS_PAGE_SIZE) || + bytes < 4 || bytes > GSP_PHYS_RMCALL_COPY_SIZE || (bytes & 3) || + (u64)src_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE || + (u64)dst_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE) { + gsp_phys_guest_refused++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n", + src_hpa, src_off, dst_hpa, dst_off, bytes, + GSP_PHYS_RMCALL_COPY_SIZE); + return -EINVAL; + } + if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS) + poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS; + + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) { + gsp_phys_guest_refused++; + return -EAGAIN; + } + + mutex_lock(&gsp_phys_rm_lock); + /* + * Re-check under the lock like the P5 path: the release may have + * committed between the readiness gate and the lock. No device write + * happens on any of these paths. + */ + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel || + boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_guest_refused++; + return -EAGAIN; + } + + ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, &src_va, + &dst_va); + if (ret) { + if (ret == -ENOSPC) { + gsp_phys_guest_pool_exhausted++; + if ((gsp_phys_guest_pool_exhausted & 63) == 1) + pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", + gsp_phys_guest_pool_exhausted, + GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + } else { + gsp_phys_guest_refused++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n", + ret, src_hpa, dst_hpa); + } + mutex_unlock(&gsp_phys_rm_lock); + return ret; + } + + /* + * The scratch PTEs map whole pages, so fold each operand's in-page + * offset into the submitted VAs; the verify below uses the same + * offsets, keeping the transfer and the check consistent. + */ + ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va + src_off, + dst_va + dst_off, + GSP_PHYS_RMCALL_VA_SEM, bytes, + NULL); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + gsp_phys_guest_refused++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + ret, src_hpa, dst_hpa, bytes); + return ret; + } + + /* + * Poll the module semaphore WITHOUT gsp_phys_rm_lock (same discipline + * as the P5 path): the release is written by the engine, so no host + * path can race it and holding the lock would stall the RPC path. + */ + deadline = jiffies + msecs_to_jiffies(poll_ms); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + break; + if (time_after_eq(jiffies, deadline)) { + gsp_phys_guest_refused++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, + src_hpa, dst_hpa, bytes); + return -ETIMEDOUT; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + + /* + * Direct-map verify: safe only because the physical device is in an + * identity domain (gated above) and gsp_phys_gpa_to_hpa() validated + * the RAM PFN behind each HPA. + */ + if (memcmp((u8 *)__va(dst_hpa) + dst_off, + (u8 *)__va(src_hpa) + src_off, bytes) != 0) { + gsp_phys_guest_verify_fail++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", + poll, dst_hpa, dst_off, src_hpa, src_off, bytes); + return -EIO; + } + + gsp_phys_guest_verify_ok++; + gsp_phys_guest_fwd_ok++; + if (gsp_phys_guest_fwd_ok == 1 || (gsp_phys_guest_fwd_ok & 63) == 0) + pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n", + src_hpa, dst_hpa, bytes, poll, + gsp_phys_guest_fwd_ok); + return 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); + /* * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE -> * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> @@ -7226,6 +7565,36 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_guest_selftest_ok, boot_stage.rmcall_guest_gpa_valid, gsp_phys_dma_domain_identity()); + { + /* + * P6b wave-1 forwarding: req/fallback/ok are the sema service's + * request/fallback/verified counters, cand_* and resolve_fail + * its per-request candidate accounting, the rest are the + * booter guest-copy counters and the scratch-PT pool position. + */ + u64 fwd_req = 0, fwd_fallback = 0; + u64 guest_ok = 0, guest_verify_ok = 0, guest_verify_fail = 0; + u64 guest_pool = 0, guest_refused = 0; + u64 fwd_cand_none = 0, fwd_cand_sys = 0, fwd_cand_mixed = 0; + u64 fwd_cand_vid = 0, fwd_resolve_fail = 0; + + gsp_sema_service_fwd_guest_stats(&fwd_req, &fwd_fallback, NULL); + gsp_sema_service_fwd_cand_stats(&fwd_cand_none, &fwd_cand_sys, + &fwd_cand_mixed, &fwd_cand_vid, + &fwd_resolve_fail); + gsp_phys_backend_guest_counters(&guest_ok, &guest_verify_ok, + &guest_verify_fail, + &guest_pool, &guest_refused); + seq_printf(m, + "work_guest: req=%llu fallback=%llu ok=%llu verify_ok=%llu verify_fail=%llu pool_exhausted=%llu refused=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu next_line=%u capacity=%u\n", + fwd_req, fwd_fallback, guest_ok, guest_verify_ok, + guest_verify_fail, guest_pool, guest_refused, + fwd_cand_none, fwd_cand_sys, fwd_cand_mixed, + fwd_cand_vid, fwd_resolve_fail, + boot_stage.rmcall_guest_pt_next_line, + GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + } { char guest_ram[512]; @@ -8388,8 +8757,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 14, - "the rm-call ladder registers exactly fourteen DMA buffers"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 30, + "the rm-call ladder registers exactly thirty DMA buffers (14 ladder + 16 single-page guest scratch PTs)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, @@ -8408,6 +8777,26 @@ static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) != (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) == ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff), "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104"); +static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 && + ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) == 1 && + ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) + + GSP_PHYS_RMCALL_GUEST_PT_PAGES <= 256, + "the guest scratch PT window must use PD0 entries 1..16 under the same root/PD2/PD1 chain as the ladder"); +static_assert(GSP_PHYS_RMCALL_VA_GUEST_PT_BASE + + (u64)GSP_PHYS_RMCALL_GUEST_PT_PAGES * (2ULL << 20) <= + GSP_PHYS_RMCALL_VAS_SIZE, + "the guest scratch PT window must stay below the VAS vaSize limit"); +static_assert(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE <= + (GSP_PHYS_PAGE_SIZE / 8u), + "each full 128 B guest scratch PT line (16 small-PTEs) must fit " + "one 4 KB PT page"); +static_assert(GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE == + 512, + "the guest scratch PT pool must hold 512 one-pair fetch lines (16 pages x 32 lines)"); /* * GspSystemInfo wire ABI, pinned against the reference header compile diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index b44d079..cc3b935 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -211,6 +211,36 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, void gsp_phys_backend_work_counters(u64 *submitted, u64 *released, u64 *timed_out, u64 *refused); +/* + * P6b wave-1 guest-operand forwarding (gsp_phys_work_forward=1 AND + * gsp_phys_guest_mem=1; both default off, either off keeps the P5 ladder + * shadow exactly as it was). + * + * gsp_phys_backend_guest_ce_copy() runs one bounded CE copy on the ladder + * channel with both operands mapped to real guest RAM by translated host PA: + * it re-checks the READY backend, the ladder channel, work forwarding, guest + * memory, the identity DMA domain and the active guest-RAM file; consumes one + * whole fresh, never-reused 128 B PTE fetch line per copy pair from the guest + * scratch PT pool built at stage 3 (no line reuse: the GMMU caches PTE + * fetches at 128 B granularity and there is no invalidate, so two pairs + * sharing a line would leave the second pair's PTEs stale); then submits + * src_off..+bytes from src_hpa to dst_hpa with the module semaphore, polls + * it lock-free for up to poll_ms (clamped to 200 ms) and verifies the copy + * with a direct-map memcmp (safe: identity domain and pfn_valid HPAs). Returns 0 on a verified copy, + * -ENOSPC when the scratch-PT pool is exhausted, -ETIMEDOUT on the poll + * bound, -EAGAIN when a gate is closed and -EINVAL/-EIO on operand/structural + * failures. No guest VA is ever submitted; the sema service falls back to its + * ladder copy on every failure, so the software release never depends on it. + * + * gsp_phys_backend_guest_counters() reads the booter-side guest counters + * (verified copies, direct-map verifies, verify failures, pool exhaustion, + * refusals including poll timeouts) for logging and the state file. + */ +int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, + u32 dst_off, u32 bytes, u32 poll_ms); +void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail, + u64 *pool_exhausted, u64 *refused); + /* * Command-queue TX: build a msgq record with the 570.144 element/rpc header * layout, splitting it into 1..16 4 KB elements when it does not fit one diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 25a3cdb..51c51b2 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -42,6 +42,8 @@ #include #include #include +#include +#include #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" @@ -420,6 +422,30 @@ struct sema_svc { u32 doorbell_log; /* token log budget */ bool doorbell_pending; /* pump on next service pass */ + /* + * P6b wave-1 guest forwarding (sema-service side): eligible requests + * marked guest, requests that fell back to the ladder copy, verified + * hardware copies. The booter keeps its own submit/verify/pool + * counters; gsp_sema_service_fwd_guest_stats() exposes this triplet + * for the gsp-phys-state work_guest line. + */ + u64 fwd_guest_req; + u64 fwd_guest_fallback; + u64 fwd_guest_ok; + /* + * P6b wave-1 candidate accounting, one increment per forwarding + * request attempt once the pump is enabled: no eligible candidate was + * captured, or the captured pair's aperture mix. fwd_guest_resolve_fail + * counts guest-work resolution failures that fell back to the ladder + * copy. All zero while the feature is off, so the summary stays silent. + */ + u64 fwd_cand_none; + u64 fwd_cand_sys_sys; + u64 fwd_cand_mixed; + u64 fwd_cand_vid_vid; + u64 fwd_guest_resolve_fail; + u64 fwd_guest_resolve_log; /* first-16 diagnostic budget */ + /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; @@ -1877,6 +1903,19 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop); + /* + * P6b wave 1: guest-forwarding summary. Printed whenever any counter + * moved (not only on requests) so a run that captured no candidate is + * still diagnosable from the log. + */ + if (svc.fwd_guest_req || svc.fwd_cand_none || svc.fwd_cand_sys_sys || + svc.fwd_cand_mixed || svc.fwd_cand_vid_vid || + svc.fwd_guest_resolve_fail) + pr_info("sema-svc: work_guest req=%llu ok=%llu fallback=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu\n", + svc.fwd_guest_req, svc.fwd_guest_ok, + svc.fwd_guest_fallback, svc.fwd_cand_none, + svc.fwd_cand_sys_sys, svc.fwd_cand_mixed, + svc.fwd_cand_vid_vid, svc.fwd_guest_resolve_fail); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; @@ -2158,6 +2197,30 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, return 0; } +/* + * P6b wave-1 guest-forwarding candidate: the FIRST linear CE copy in the + * decoded push whose operands the software path resolved successfully. + * Captured by svc_copy() while the push's work runs, so the candidate is + * exactly the copy the software path executed; one capture per decode pass, + * consumed by svc_fwd_request() on the same service thread. Each operand + * records its aperture kind and the resolver's page/offset: SYS holds a + * page-aligned guest GPA, VID holds a page-aligned byte offset into the + * emulated BAR1 vidmem shadow. The work item later turns a SYS operand into + * an HPA through the guest-RAM backing file and a VID operand into an HPA + * through the vmalloc backing of the shadow; no address is ever guessed. + * Service-thread only, like the rest of the pump state. + */ +struct svc_fwd_cand { + bool valid; + u8 src_kind; /* SVC_APERTURE_SYS (GPA) / SVC_APERTURE_FB (shadow) */ + u8 dst_kind; + u64 src_page, dst_page; /* resolver page: GPA / shadow byte offset */ + u32 src_off, dst_off; /* in-page offsets */ + u32 bytes; +}; + +static struct svc_fwd_cand svc_fwd_cand; + /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) @@ -2199,6 +2262,51 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, src_aperture = aper; } + /* + * P6b wave 1: the software path has now resolved both operands, so + * this successful CE copy is the guest-forwarding candidate. Physical + * operands are the norm here (the guest pushes raw vidmem offsets as + * physical destinations, with virtual sources); after the block above, + * a virtual side already holds its resolved pa + aperture, while a + * physical side carries its raw address + the push's aperture, so the + * same page/offset/kind handling covers both. Eligibility stays + * conservative: one page per side, 4-aligned, 1..4096 bytes, and no + * same-page overlap within one aperture; anything doubtful is left to + * the ladder submission. Only the first eligible copy of the push is + * kept (svc_fwd_cand is reset per decode pass). + */ + if (!svc_fwd_cand.valid && + total >= 1 && total <= SEMA_PAGE_SIZE && !(total & 3)) { + u64 src_page = src & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 dst_page = dst & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 src_off = (u32)(src & (SEMA_PAGE_SIZE - 1)); + u32 dst_off = (u32)(dst & (SEMA_PAGE_SIZE - 1)); + u8 src_kind = (src_aperture == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + u8 dst_kind = (dst_aperture == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + /* + * Same-domain operands inside the same page must not overlap: + * the engine's copy direction would be undefined. Different + * domains (SYS vs VID) are disjoint by construction. + */ + bool overlap = src_kind == dst_kind && src_page == dst_page && + src_off < dst_off + total && + dst_off < src_off + total; + + if (!overlap && src_off + total <= SEMA_PAGE_SIZE && + dst_off + total <= SEMA_PAGE_SIZE) { + svc_fwd_cand.valid = true; + svc_fwd_cand.src_kind = src_kind; + svc_fwd_cand.dst_kind = dst_kind; + svc_fwd_cand.src_page = src_page; + svc_fwd_cand.dst_page = dst_page; + svc_fwd_cand.src_off = src_off; + svc_fwd_cand.dst_off = dst_off; + svc_fwd_cand.bytes = (u32)total; + } + } + /* * Data-path watch: for copies of exactly sema_copy_watch bytes, log * the first 8 source and destination bytes so a broken link in the @@ -2826,6 +2934,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } memset(&st, 0, sizeof(st)); + /* + * P6b wave 1: clear the guest-forwarding candidate at the start of + * every decode pass; the first eligible copy captured below is the one + * svc_fwd_request() sees when this pass ends. + */ + svc_fwd_cand.valid = false; svc.pages_decoded++; if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) { svc_dbg_streams++; @@ -5020,7 +5134,8 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) } /* ------------------------------------------------------------------ */ -/* P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1) */ +/* P5/P6b wave-1 shadow work forwarding */ +/* (gsp_phys_work_forward=1, + gsp_phys_guest_mem=1 for guest data) */ /* ------------------------------------------------------------------ */ /* @@ -5029,12 +5144,46 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) * request that arrives while one is in flight (or before the backend/ladder * is ready) is dropped and counted. The work item runs the ladder's * module-owned copy and observes the real semaphore; the software release - * path is unchanged and stays authoritative. No guest VA or guest PA ever - * reaches the engine in this wave. + * path is unchanged and stays authoritative. + * + * P6b wave 1: when svc_copy() captured an eligible linear copy (virtual + * operands, one page, resolved by the software path) and gsp_phys_guest_mem=1, + * the work item instead resolves each operand to a host PA and runs one real + * CE copy against guest memory or the emulated vidmem shadow on a fresh + * scratch-PT line: a SYS operand resolves through the guest-RAM backing file, + * a VID operand through the vmalloc backing of BAR1. Any failure (translation, + * out-of-range shadow, missing vmalloc page, pool exhaustion, submit, timeout, + * verify) falls back to the unchanged ladder copy, so the software pump is + * never gated by the hardware path. Shadow-mode limit: the software copy + * already applied the same bytes, so hardware execution is proven by + * acceptance of the translated-HPA submission plus the semaphore release and + * the direct-map consistency check; a late hardware write can theoretically + * race a guest reuse of the page. The window is one ordered workqueue hop plus + * the bounded poll (<= 200 ms); wave 2 removes it by making hardware primary. */ #define SVC_FWD_INFLIGHT 0 #define SVC_FWD_LOG_MASK 63 +/* + * One pending request, written by the service thread under svc_fwd_lock + * before queue_work and read by the work item without that lock (quiesce + * holds that lock across cancel_work_sync, so the work item must never take + * it). The SVC_FWD_INFLIGHT bit serialises writers and queue_work orders the + * slot write before the work item runs. vgpu is carried for the VID operand + * math only; it is stable because the request is single-inflight and quiesce + * quiesces the work item before the backend/device state is released. + */ +struct svc_fwd_req { + bool guest; + struct nv_vgpu_device *vgpu; + u8 src_kind, dst_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ + u64 src_page, dst_page; /* SYS: page GPA; VID: shadow byte offset */ + u32 src_off, dst_off; + u32 bytes; +}; + +static struct svc_fwd_req svc_fwd_req; + static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; @@ -5049,13 +5198,128 @@ static u64 svc_fwd_dropped; static DEFINE_MUTEX(svc_fwd_lock); static bool svc_fwd_paused; +/* + * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve + * the page-aligned GPA through the guest-RAM backing file; VID operands + * resolve the shadow byte offset through the vmalloc backing of BAR1 (the + * same buffer the AP helpers use). The resolver's in-page part, if any, is + * folded into the offset. Returns 0 on success; -EINVAL for an unknown kind, + * -ERANGE when the FB base is missing or the range is out of bounds, -EFAULT + * when vmalloc_to_page() fails, and otherwise gsp_phys_gpa_to_hpa()'s errno. + * The caller falls back to the ladder copy on any failure. + */ +static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, + u64 pa, u32 off, u32 bytes, u64 *hpa, + u32 *out_off) +{ + u64 page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 page_off = (u32)(pa & (SEMA_PAGE_SIZE - 1)) + off; + int ret; + + if (kind == SVC_APERTURE_FB) { + u8 *base = vgpu ? vgpu->bars[1].vaddr : NULL; + struct page *pg; + + if (!base || page > vgpu->bars[1].size || + (u64)page_off + bytes > vgpu->bars[1].size - page) + return -ERANGE; + pg = vmalloc_to_page(base + page); + if (!pg) + return -EFAULT; + *hpa = (u64)page_to_phys(pg); + *out_off = page_off; + return 0; + } + if (kind != SVC_APERTURE_SYS) + return -EINVAL; + ret = gsp_phys_gpa_to_hpa(page, hpa); + if (ret) + return ret; + *out_off = page_off; + return 0; +} + +/* Classify the pass's captured candidate into the forwarding counters. */ +static void svc_fwd_cand_note(void) +{ + if (!svc_fwd_cand.valid) { + svc.fwd_cand_none++; + return; + } + if (svc_fwd_cand.src_kind == SVC_APERTURE_FB && + svc_fwd_cand.dst_kind == SVC_APERTURE_FB) + svc.fwd_cand_vid_vid++; + else if (svc_fwd_cand.src_kind == SVC_APERTURE_SYS && + svc_fwd_cand.dst_kind == SVC_APERTURE_SYS) + svc.fwd_cand_sys_sys++; + else + svc.fwd_cand_mixed++; +} + static void svc_fwd_work_fn(struct work_struct *work) { + bool guest = svc_fwd_req.guest; u64 src_va = 0, dst_va = 0, sem_va = 0; u32 bytes = 0; unsigned long start = jiffies; int ret; + /* + * P6b wave 1: try the guest hardware copy first when the pump marked + * this request guest. Every failure falls through to the unchanged + * ladder submission below. The request slot is stable here: the + * INFLIGHT bit is held, so only this reader touches it. + */ + if (guest) { + u64 src_hpa = 0, dst_hpa = 0; + u32 src_off = 0, dst_off = 0; + int src_rc, dst_rc; + bool resolved; + + src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, + svc_fwd_req.src_kind, + svc_fwd_req.src_page, + svc_fwd_req.src_off, + svc_fwd_req.bytes, &src_hpa, + &src_off); + dst_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, + svc_fwd_req.dst_kind, + svc_fwd_req.dst_page, + svc_fwd_req.dst_off, + svc_fwd_req.bytes, &dst_hpa, + &dst_off); + resolved = !src_rc && !dst_rc; + if (resolved && + !gsp_phys_backend_guest_ce_copy(src_hpa, src_off, + dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS)) { + svc.fwd_guest_ok++; + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + return; + } + if (!resolved) { + svc.fwd_guest_resolve_fail++; + svc.fwd_guest_resolve_log++; + if (svc.fwd_guest_resolve_log <= 16) + pr_info("gsp-phys-work: guest candidate resolve failed: src kind=%u pa=0x%llx off=%u rc=%d dst kind=%u pa=0x%llx off=%u rc=%d vgpu=%d bytes=%u\n", + svc_fwd_req.src_kind, + svc_fwd_req.src_page, + svc_fwd_req.src_off, src_rc, + svc_fwd_req.dst_kind, + svc_fwd_req.dst_page, + svc_fwd_req.dst_off, dst_rc, + svc_fwd_req.vgpu ? 1 : 0, + svc_fwd_req.bytes); + } + svc.fwd_guest_fallback++; + if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: guest copy fell back to the ladder copy (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n", + svc.fwd_guest_req, svc.fwd_guest_ok, + svc.fwd_guest_fallback, + svc.fwd_guest_resolve_fail); + } + /* * Re-resolve the ladder operands on the work item: the backend may * have gone away between the pump's gate and this run. A false return @@ -5086,12 +5350,15 @@ static void svc_fwd_work_fn(struct work_struct *work) static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); /* Called only from the service thread (the pump). */ -static void svc_fwd_request(const struct svc_channel *ch) +static void svc_fwd_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) { if (!ch->engine_type) return; if (!gsp_phys_work_forward_enabled()) return; + /* Classify this decode pass's candidate for the run summary. */ + svc_fwd_cand_note(); if (!gsp_phys_backend_work_ready()) { svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) @@ -5116,6 +5383,34 @@ static void svc_fwd_request(const struct svc_channel *ch) svc_fwd_dropped); return; } + /* + * Fill the request slot before queueing (INFLIGHT is held, so no + * other writer can race it). A captured candidate runs on the guest + * path only when gsp_phys_guest_mem=1; otherwise, and with no + * candidate, the request stays the unchanged ladder shadow. + */ + svc_fwd_req.guest = svc_fwd_cand.valid && + gsp_phys_guest_mem_enabled(); + if (svc_fwd_req.guest) { + svc_fwd_req.vgpu = vgpu; + svc_fwd_req.src_kind = svc_fwd_cand.src_kind; + svc_fwd_req.dst_kind = svc_fwd_cand.dst_kind; + svc_fwd_req.src_page = svc_fwd_cand.src_page; + svc_fwd_req.dst_page = svc_fwd_cand.dst_page; + svc_fwd_req.src_off = svc_fwd_cand.src_off; + svc_fwd_req.dst_off = svc_fwd_cand.dst_off; + svc_fwd_req.bytes = svc_fwd_cand.bytes; + svc.fwd_guest_req++; + } else { + svc_fwd_req.vgpu = NULL; + svc_fwd_req.src_kind = 0; + svc_fwd_req.dst_kind = 0; + svc_fwd_req.src_page = 0; + svc_fwd_req.dst_page = 0; + svc_fwd_req.src_off = 0; + svc_fwd_req.dst_off = 0; + svc_fwd_req.bytes = 0; + } svc_fwd_requests++; if (!svc_fwd_wq) { /* @@ -5158,6 +5453,41 @@ void gsp_sema_service_work_resume(void) mutex_unlock(&svc_fwd_lock); } +/* + * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest + * line (see gsp_sema_service.h). Plain reads: the service thread and the + * work item are the only writers and the values are monotonic. + */ +void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok) +{ + if (req) + *req = svc.fwd_guest_req; + if (fallback) + *fallback = svc.fwd_guest_fallback; + if (ok) + *ok = svc.fwd_guest_ok; +} + +/* + * P6b wave-1 candidate accounting (see gsp_sema_service.h): per-request + * classification of the captured operand mix plus guest-work resolution + * failures. Same plain-read discipline as the stats above. + */ +void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed, + u64 *vid_vid, u64 *resolve_fail) +{ + if (none) + *none = svc.fwd_cand_none; + if (sys_sys) + *sys_sys = svc.fwd_cand_sys_sys; + if (mixed) + *mixed = svc.fwd_cand_mixed; + if (vid_vid) + *vid_vid = svc.fwd_cand_vid_vid; + if (resolve_fail) + *resolve_fail = svc.fwd_guest_resolve_fail; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -5404,16 +5734,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->op_retry = 0; rel_seen = (svc_rel_count != rel_n_before); /* - * P5 wave 1: shadow this - * CE push on the ladder - * channel. The software - * release above is already + * P5 wave 1 / P6b: + * shadow this CE push + * on the ladder + * channel, or run it + * against guest memory + * when the decode + * captured a trusted + * sysmem copy and + * gsp_phys_guest_mem=1. + * The software release + * above is already * applied and stays - * authoritative; the work - * item only observes the - * real semaphore. + * authoritative; every + * hardware failure + * falls back to the + * ladder copy. */ - svc_fwd_request(ch); + svc_fwd_request(vgpu, ch); /* * A CE push that should carry this * channel's next tracking release but diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index ae9b7da..9638dde 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -76,6 +76,23 @@ void gsp_sema_service_doorbell(u32 token); void gsp_sema_service_work_quiesce(void); void gsp_sema_service_work_resume(void); +/* + * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest + * line: guest-eligible requests seen by the pump, requests that fell back to + * the ladder copy, and verified hardware copies. Plain reads of the service + * counters; safe from the debugfs show path. + */ +void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok); + +/* + * P6b wave-1 candidate accounting for the same work_guest line: how many + * forwarding requests found no eligible captured copy, and of those that did, + * the operand aperture mix (SYS/SYS, mixed, VID/VID), plus guest-work + * resolution failures on the work item. All zero while the feature is off. + */ +void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed, + u64 *vid_vid, u64 *resolve_fail); + /* * Discover and pin GR (compute) channel rings after the CE region of the * channel video-memory allocation. Set false to leave GR rings untranslated -- cgit v1.2.3