diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 348 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 36 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 25 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 15 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 387 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 14 |
6 files changed, 750 insertions, 75 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 94b0cae..5c36c35 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -6163,10 +6163,13 @@ static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data) * (channel_utils.c:812-820). Returns the block size in dwords (13 method * pairs = 26 dwords, <= GPFIFO entry LENGTH field, bits 30:10). The ladder * stage-4 call passes GSP_PHYS_RMCALL_COPY_SIZE; the P5 shadow submit passes - * the caller's size (both within one 4 KB ladder page). + * the caller's size (both within one 4 KB ladder page). P6c: sem_payload is + * the value the release writes; every legacy call site passes + * GSP_PHYS_RMCALL_SEM_PAYLOAD so the block is byte-identical to before. */ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, - u64 dst_va, u32 bytes) + u64 dst_va, u32 bytes, + u32 sem_payload) { u32 *p = pb; @@ -6177,7 +6180,7 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B, (u32)sem_va); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD, - GSP_PHYS_RMCALL_SEM_PAYLOAD); + sem_payload); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER, (u32)(dst_va >> 32)); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER, @@ -6225,9 +6228,14 @@ struct gsp_phys_rmcall_ce_submit { * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the * token check. The semaphore is cleared before the doorbell so a stale release * from a previous run cannot be observed as this submission's completion. + * P6c: sem_payload is the release value the method block carries; legacy + * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is + * still cleared even when the caller polls a guest page: the clear only + * guards the module-sem poll path and is a harmless non-guest write. */ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = @@ -6276,7 +6284,7 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, wmb(); words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, - bytes); + bytes, sem_payload); gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); @@ -6376,7 +6384,9 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, - bytes, &sub); + bytes, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + &sub); mutex_unlock(&gsp_phys_rm_lock); if (ret) return ret; @@ -6510,7 +6520,9 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, return -EAGAIN; } ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, - bytes, &sub); + bytes, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + &sub); mutex_unlock(&gsp_phys_rm_lock); if (ret) { gsp_phys_work_refused++; @@ -6625,6 +6637,39 @@ void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail, EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters); /* + * P6c relay-path counters. Only a copy submitted with a nonzero sem_hpa (the + * completion relay) moves these; every legacy call keeps moving the guest + * counters above only, so with gsp_phys_relay=0 they stay zero and the legacy + * lines are identical. Plain increments from the sema service work item; only + * logging and the debugfs state file read them. + */ +static u64 gsp_phys_relay_requested; +static u64 gsp_phys_relay_ok; +static u64 gsp_phys_relay_verify_fail; +static u64 gsp_phys_relay_timed_out; +static u64 gsp_phys_relay_refused; +static u64 gsp_phys_relay_pool_exhausted; + +void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, + u64 *timeouts, u64 *refused, + u64 *pool_exhausted) +{ + if (requested) + *requested = gsp_phys_relay_requested; + if (ok) + *ok = gsp_phys_relay_ok; + if (verify_fail) + *verify_fail = gsp_phys_relay_verify_fail; + if (timeouts) + *timeouts = gsp_phys_relay_timed_out; + if (refused) + *refused = gsp_phys_relay_refused; + if (pool_exhausted) + *pool_exhausted = gsp_phys_relay_pool_exhausted; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_relay_counters); + +/* * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool * and write both operand PTEs into it (src at 16*line_in_page, dst at * 16*line_in_page + 1). Called with gsp_phys_rm_lock held, so the monotonic @@ -6637,9 +6682,15 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters); * timed out with sem=0. The scratch PT pages were zeroed at stage 2, so every * unconsumed slot holds an invalid PTE. Returns -ENOSPC when the pool is * exhausted and -EIO when the scratch PT pages are not allocated. + * + * P6c relay: a nonzero sem_hpa additionally writes a third PTE at + * 16*line_in_page + 2 and returns sem_va = line_va + 2*PAGE_SIZE; sem_hpa==0 + * writes nothing and returns sem_va=0. One line per relay copy either way, so + * the pool capacity (and its exhaustion behavior) is unchanged. */ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, - u64 *src_va, u64 *dst_va) + u64 sem_hpa, u64 *src_va, + u64 *dst_va, u64 *sem_va) { struct gsp_phys_boot_chunk *guest_pt_chunk; u32 line = boot_stage.rmcall_guest_pt_next_line; @@ -6670,11 +6721,21 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + 1) * 8); + /* + * P6c relay: the guest-visible semaphore page gets the third slot of + * the same never-reused line; the write is ordered with the other two + * by the dma_wmb() below, before any doorbell. + */ + if (sem_hpa) + put_unaligned_le64(gsp_phys_rmcall_pte(sem_hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page + 2) * 8); dma_wmb(); boot_stage.rmcall_guest_pt_next_line = line + 1; *src_va = va; *dst_va = va + GSP_PHYS_PAGE_SIZE; + *sem_va = sem_hpa ? va + 2 * GSP_PHYS_PAGE_SIZE : 0; return 0; } @@ -6688,39 +6749,73 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, * gsp_phys_work_forward requirement, so the P6a triggers can share the same * fresh-line copy path with forwarding. *sem_out receives the last * semaphore word seen on success or timeout. + * + * P6c completion relay: a nonzero sem_hpa adds a third PTE on the same fresh + * line for the guest-visible semaphore page (pinned with pfn_valid -> + * get_page/put_page for the duration), points the CE release methods at + * sem_va + sem_off with sem_payload and polls the guest-visible word at + * __va(sem_hpa) + sem_off for the exact payload instead of the module + * semaphore. A zero sem_hpa keeps the module-semaphore behavior + * byte-identical (same gates, same counters, same logs). Relay refusals, + * timeouts and verify failures move the relay counters and return an errno + * the caller falls back to, so no release can be lost. *out (if given) + * always receives the last semaphore word, the elapsed milliseconds and the + * return code. */ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms, - bool need_work_forward, u32 *sem_out) + bool need_work_forward, + u64 sem_hpa, u32 sem_off, + u32 sem_payload, + struct gsp_phys_backend_copy_result *out, + u32 *sem_out) { struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; void __iomem *bar0; + struct page *sem_page = NULL; + unsigned long start = jiffies; unsigned long deadline; - u64 src_va = 0, dst_va = 0; + u64 src_va = 0, dst_va = 0, sem_va = 0; u32 poll = 0; + bool relay = sem_hpa != 0; int ret; + if (relay) + gsp_phys_relay_requested++; + /* * Every gate is fail-closed: the sema service's ladder fallback keeps * the software release authoritative, so a refusal never loses work. */ if (need_work_forward) { if (!gsp_phys_backend_work_ready()) { - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } } else { if (!gsp_phys_backend_ladder_ready()) { - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } } if (!gsp_phys_guest_mem_enabled() || gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } if (!src_hpa || !dst_hpa || !IS_ALIGNED(src_hpa, GSP_PHYS_PAGE_SIZE) || @@ -6732,15 +6827,42 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, pr_err("gsp-phys-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n", src_hpa, src_off, dst_hpa, dst_off, bytes, GSP_PHYS_RMCALL_COPY_SIZE); - return -EINVAL; + ret = -EINVAL; + goto out; + } + if (relay && + (!IS_ALIGNED(sem_hpa, GSP_PHYS_PAGE_SIZE) || + (u64)sem_off + 4 > bytes || !sem_payload)) { + gsp_phys_relay_refused++; + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: invalid semaphore relay sem_hpa=0x%llx sem_off=%u sem_payload=0x%08x bytes=%u (want a page-aligned HPA, sem_off+4 <= copy size, nonzero payload)\n", + sem_hpa, sem_off, sem_payload, bytes); + ret = -EINVAL; + goto out; + } + if (relay) { + unsigned long pfn = sem_hpa >> PAGE_SHIFT; + + if (!pfn_valid(pfn)) { + gsp_phys_relay_refused++; + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore HPA 0x%llx is not a valid RAM page\n", + sem_hpa); + ret = -EINVAL; + goto out; + } + sem_page = pfn_to_page(pfn); + get_page(sem_page); } if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS) poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS; bar0 = gsp_phys_gpu_bar0(); if (!bar0) { - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } mutex_lock(&gsp_phys_rm_lock); @@ -6753,64 +6875,104 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) { mutex_unlock(&gsp_phys_rm_lock); - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } - ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, &src_va, - &dst_va); + ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, + relay ? sem_hpa : 0, &src_va, + &dst_va, &sem_va); if (ret) { if (ret == -ENOSPC) { - gsp_phys_guest_pool_exhausted++; - if ((gsp_phys_guest_pool_exhausted & 63) == 1) - pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", - gsp_phys_guest_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + if (relay) { + gsp_phys_relay_pool_exhausted++; + if ((gsp_phys_relay_pool_exhausted & 63) == 1) + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n", + gsp_phys_relay_pool_exhausted, + GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + } else { + gsp_phys_guest_pool_exhausted++; + if ((gsp_phys_guest_pool_exhausted & 63) == 1) + pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", + gsp_phys_guest_pool_exhausted, + GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + } } else { - gsp_phys_guest_refused++; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n", ret, src_hpa, dst_hpa); } mutex_unlock(&gsp_phys_rm_lock); - return ret; + goto out; } /* * The scratch PTEs map whole pages, so fold each operand's in-page * offset into the submitted VAs; the verify below uses the same - * offsets, keeping the transfer and the check consistent. + * offsets, keeping the transfer and the check consistent. The relay + * points the release at the guest semaphore PTE at sem_va + sem_off + * and carries the caller's payload. */ ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va + src_off, dst_va + dst_off, - GSP_PHYS_RMCALL_VA_SEM, bytes, + relay ? sem_va + sem_off : + GSP_PHYS_RMCALL_VA_SEM, + bytes, + relay ? sem_payload : + GSP_PHYS_RMCALL_SEM_PAYLOAD, NULL); mutex_unlock(&gsp_phys_rm_lock); if (ret) { - gsp_phys_guest_refused++; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", ret, src_hpa, dst_hpa, bytes); - return ret; + goto out; } /* - * Poll the module semaphore WITHOUT gsp_phys_rm_lock (same discipline - * as the P5 path): the release is written by the engine, so no host - * path can race it and holding the lock would stall the RPC path. + * Poll the release WITHOUT gsp_phys_rm_lock (same discipline as the + * P5 path): the release is written by the engine, so no host path can + * race it and holding the lock would stall the RPC path. The relay + * watches the guest-visible word through its pin/direct map; the + * legacy path polls the module ladder semaphore exactly as before. */ deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { - poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); - if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + if (relay) + poll = READ_ONCE(*(u32 *)((u8 *)__va(sem_hpa) + sem_off)); + else + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == (relay ? sem_payload : GSP_PHYS_RMCALL_SEM_PAYLOAD)) break; if (time_after_eq(jiffies, deadline)) { - gsp_phys_guest_refused++; + if (relay) { + gsp_phys_relay_timed_out++; + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: timeout after %u ms (guest sem=0x%08x want=0x%08x) sem_hpa=0x%llx+%u src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + poll_ms, poll, sem_payload, sem_hpa, + sem_off, src_hpa, dst_hpa, bytes); + } else { + gsp_phys_guest_refused++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + poll_ms, poll, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + src_hpa, dst_hpa, bytes); + } if (sem_out) *sem_out = poll; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", - poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, - src_hpa, dst_hpa, bytes); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto out; } msleep(GSP_PHYS_RMCALL_POLL_MS); } @@ -6822,28 +6984,70 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, */ if (memcmp((u8 *)__va(dst_hpa) + dst_off, (u8 *)__va(src_hpa) + src_off, bytes) != 0) { - gsp_phys_guest_verify_fail++; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", - poll, dst_hpa, dst_off, src_hpa, src_off, bytes); - return -EIO; + if (relay) { + gsp_phys_relay_verify_fail++; + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", + poll, dst_hpa, dst_off, src_hpa, src_off, + bytes); + } else { + gsp_phys_guest_verify_fail++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", + poll, dst_hpa, dst_off, src_hpa, src_off, + bytes); + } + ret = -EIO; + goto out; } - gsp_phys_guest_verify_ok++; - gsp_phys_guest_fwd_ok++; if (sem_out) *sem_out = poll; - if (gsp_phys_guest_fwd_ok == 1 || (gsp_phys_guest_fwd_ok & 63) == 0) - pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n", - src_hpa, dst_hpa, bytes, poll, - gsp_phys_guest_fwd_ok); - return 0; + if (relay) { + gsp_phys_relay_ok++; + if (gsp_phys_relay_ok == 1 || (gsp_phys_relay_ok & 63) == 0) + pr_info("gsp-phys-work: GUEST CE RELAY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem_hpa=0x%llx+%u sem=0x%08x relays=%llu\n", + src_hpa, dst_hpa, bytes, sem_hpa, sem_off, + poll, gsp_phys_relay_ok); + } else { + gsp_phys_guest_verify_ok++; + gsp_phys_guest_fwd_ok++; + if (gsp_phys_guest_fwd_ok == 1 || + (gsp_phys_guest_fwd_ok & 63) == 0) + pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n", + src_hpa, dst_hpa, bytes, poll, + gsp_phys_guest_fwd_ok); + } + ret = 0; + +out: + if (sem_page) + put_page(sem_page); + if (out) { + out->sem_value = poll; + out->elapsed_ms = jiffies_to_msecs(jiffies - start); + out->rc = ret; + } + return ret; +} + +int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa, + u32 dst_off, u32 bytes, u32 poll_ms, + u64 sem_hpa, u32 sem_off, + u32 sem_payload, + struct gsp_phys_backend_copy_result *out) +{ + return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off, + bytes, poll_ms, true, + sem_hpa, sem_off, sem_payload, + out, NULL); } +EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy_ex); int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms) { - return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off, - bytes, poll_ms, true, NULL); + return gsp_phys_backend_guest_ce_copy_ex(src_hpa, src_off, dst_hpa, + dst_off, bytes, poll_ms, + 0, 0, 0, NULL); } EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); @@ -6861,7 +7065,8 @@ static int gsp_phys_backend_guest_ce_copy_trigger(u64 src_hpa, u32 src_off, u32 *sem_out) { return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off, - bytes, poll_ms, false, sem_out); + bytes, poll_ms, false, + 0, 0, 0, NULL, sem_out); } /* @@ -7043,7 +7248,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, - GSP_PHYS_RMCALL_COPY_SIZE, &sub); + GSP_PHYS_RMCALL_COPY_SIZE, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + &sub); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n", ret); @@ -7691,6 +7898,29 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) gsp_phys_work_hw_primary_enabled()); } { + /* + * P6c completion relay: the sema service's relay-selection and + * hold-verification counters (requested/ok/fallback/skipped_sw/ + * verify_fail/resolve_fail) plus the booter relay copy + * counters (timeouts/pool_exhausted/refused) and the parameter + * state. All zero while the relay is off. + */ + u64 r_req = 0, r_ok = 0, r_fallback = 0, r_skipped = 0; + u64 r_verify_fail = 0, r_resolve_fail = 0; + u64 r_timeouts = 0, r_pool = 0, r_refused = 0; + + gsp_sema_service_fwd_relay_stats(&r_req, &r_ok, &r_fallback, + &r_skipped, &r_verify_fail, + &r_resolve_fail); + gsp_phys_backend_relay_counters(NULL, NULL, NULL, &r_timeouts, + &r_refused, &r_pool); + seq_printf(m, + "work_relay: requested=%llu ok=%llu fallback=%llu skipped_sw=%llu verify_fail=%llu resolve_fail=%llu timeouts=%llu pool_exhausted=%llu refused=%llu relay_param=%u\n", + r_req, r_ok, r_fallback, r_skipped, r_verify_fail, + r_resolve_fail, r_timeouts, r_pool, r_refused, + gsp_phys_relay_enabled()); + } + { char guest_ram[512]; gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram)); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index ada3280..408d380 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -235,12 +235,48 @@ void gsp_phys_backend_work_counters(u64 *submitted, u64 *released, * gsp_phys_backend_guest_counters() reads the booter-side guest counters * (verified copies, direct-map verifies, verify failures, pool exhaustion, * refusals including poll timeouts) for logging and the state file. + * + * P6c completion relay (gsp_phys_relay=1; default off): + * gsp_phys_backend_guest_ce_copy_ex() is the same copy with an optional + * guest-visible release relay: a nonzero sem_hpa (page aligned) makes the + * submission map that page as a third PTE on the same fresh 128 B line and + * point the CE release methods at sem_va + sem_off with sem_payload; the + * caller's page is pinned for the duration (pfn_valid -> get_page/put_page) + * and the poll watches __va(sem_hpa) + sem_off for the exact payload instead + * of the module ladder semaphore. *out (if given) always carries the last + * semaphore word observed, the elapsed milliseconds and the return code. A + * zero sem_hpa keeps the module-semaphore behavior byte-identical. + * gsp_phys_backend_guest_ce_copy() is a wrapper for the zero-relay form. */ int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms); void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail, u64 *pool_exhausted, u64 *refused); +/* Result of one guest CE copy (see the P6c note above). */ +struct gsp_phys_backend_copy_result { + u32 sem_value; /* last word observed at the polled semaphore */ + u32 elapsed_ms; /* return-to-caller elapsed milliseconds */ + int rc; /* duplicated function return code */ +}; + +int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa, + u32 dst_off, u32 bytes, u32 poll_ms, + u64 sem_hpa, u32 sem_off, + u32 sem_payload, + struct gsp_phys_backend_copy_result *out); + +/* + * gsp_phys_backend_relay_counters() reads the P6c relay-path counters + * (requested relay copies, verified relays, data verify failures, guest + * semaphore poll timeouts, gate/structural refusals and scratch-pool + * exhaustion) for logging and the gsp-phys-state work_relay line. All zero + * unless gsp_phys_relay=1 copies ran. + */ +void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, + u64 *timeouts, u64 *refused, + u64 *pool_exhausted); + /* * Command-queue TX: build a msgq record with the 570.144 element/rpc header * layout, splitting it into 1..16 4 KB elements when it does not fit one diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 512d30b..fc84c1d 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -248,6 +248,31 @@ bool gsp_phys_work_hw_primary_enabled(void) EXPORT_SYMBOL_GPL(gsp_phys_work_hw_primary_enabled); /* + * P6c completion relay (default 0). When set together with + * gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and + * gsp_phys_work_defer_release=1, a forwarded guest CE copy additionally maps + * the guest-visible semaphore page into the CE channel page tables and lets + * the engine author one selected one-word release into it (custom payload). + * The hold then verifies the guest word carries the payload and skips the + * software apply of that one record; every refusal, hardware failure, + * timeout, abort or failed verification leaves the unchanged software apply + * in charge, so the release stays exactly-once. With 0, every behavior is + * byte-identical to wave 2 (the module semaphore is polled; no guest + * semaphore page is ever mapped or pinned). One-line rollback: + * gsp_phys_relay=0. + */ +static bool gsp_phys_relay_param; +module_param_named(gsp_phys_relay, gsp_phys_relay_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_relay, + "Let the forwarded guest CE copy author the selected guest-visible semaphore release into the guest semaphore page (default 0: software release stays the sole author; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); + +bool gsp_phys_relay_enabled(void) +{ + return gsp_phys_relay_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_relay_enabled); + +/* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 70b98e4..6bfe3f0 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -124,6 +124,21 @@ bool gsp_phys_work_defer_release_enabled(void); bool gsp_phys_work_hw_primary_enabled(void); /* + * True when the gsp_phys_relay module parameter is set (default 0: a + * forwarded guest CE copy polls the module-owned ladder semaphore and the + * software release path stays the sole author of the guest-visible release). + * When set with gsp_phys_work_forward, gsp_phys_guest_mem and + * gsp_phys_work_defer_release, one selected one-word release of a forwarded + * guest push is authored by the real CE engine directly into the + * guest-visible semaphore page (a third scratch PTE maps that page) and the + * service thread skips the software apply of that record once the guest word + * carries the payload; every failure, timeout, abort or failed verification + * falls back to the unchanged software apply, so a release is never lost or + * duplicated. One-line rollback: gsp_phys_relay=0. + */ +bool gsp_phys_relay_enabled(void); + +/* * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder * support). The mdev is registered with vfio_register_emulated_iommu_dev(): * its VFIO type-1 container keeps the guest DMA mappings privately, so diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1f599fc..dea8249 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -470,6 +470,22 @@ struct sema_svc { u64 fwd_hold_engaged; u64 fwd_hold_timeouts; u64 fwd_hold_canceled; + /* + * P6c completion relay (gsp_phys_relay=1 only): relay selections armed + * on a deferred guest request (fwd_relay_requested), relays whose + * engine-authored guest semaphore was verified with the software apply + * skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to + * the software apply (fwd_relay_fallback), hold re-reads that did not + * see the payload (fwd_relay_verify_fail) and work-item semaphore HPA + * resolution failures (fwd_relay_resolve_fail). All zero while the + * parameter is off. + */ + u64 fwd_relay_requested; + u64 fwd_relay_ok; + u64 fwd_relay_fallback; + u64 fwd_relay_skipped_sw; + u64 fwd_relay_verify_fail; + u64 fwd_relay_resolve_fail; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -2308,6 +2324,18 @@ struct svc_fwd_rel { u8 kind; /* SVC_FWD_REL_* */ u8 op; /* 906f: host sem op / launch: pending flag */ u8 inc_claimed; /* launch: one-shot inc release claimed */ + /* + * P6c completion relay: the release target resolved once at record + * time by svc_fwd_relay_note() (only while gsp_phys_relay is on and + * every relay guard passes). tgt_page is a page-aligned GPA (SYS) or + * BAR1 shadow byte offset (FB); tgt_off is the in-page offset; a + * false tgt_ok means the record is software-applied and never + * relayed. + */ + u8 tgt_ok; + u8 tgt_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ + u64 tgt_page; + u32 tgt_off; u64 va; u32 payload; u32 entry_idx; @@ -3133,6 +3161,67 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, return false; } +/* + * P6c completion relay: resolve one deferred release record's target once at + * record time so the request can hand the engine a page/offset it may + * program. Fail-open by construction: every miss (parameter off, non-CE + * channel, GR counter VA, a non-one-word or reduction LAUNCH release, an + * unresolvable VA, an unreadable word, or a word already at/ahead of the + * payload) leaves tgt_ok false and the record is software-applied exactly as + * in wave 1.5. The resolution uses the same svc_resolve_va() the software + * apply path uses and pins the page via svc_pin_store(), so the work item's + * later svc_fwd_operand_hpa() resolves the same page. Service-thread only. + */ +static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + struct svc_fwd_rel *r, u32 launch_dma, + bool is_launch) +{ + u64 addr = 0; + u32 ap = 0; + u32 cur = 0; + + r->tgt_ok = false; + r->tgt_kind = 0; + r->tgt_page = 0; + r->tgt_off = 0; + + if (!gsp_phys_relay_enabled() || !ch || !ch->engine_type) + return; + if (svc_counter_va(r->va)) + return; + /* + * The relay always authors an absolute one-word release (the CE block + * writes SET_SEMAPHORE_PAYLOAD and LAUNCH_DMA RELEASE_ONE_WORD), so a + * record whose software apply is additive must never be relayed: the + * engine would leave the guest word short of the value it waits on. + * The LAUNCH increment is excluded by the dma check below; the 906F + * REDUCTION op is excluded here. + */ + if (!is_launch && r->kind == SVC_FWD_REL_906F && + r->op == HOST_SEM_OP_REDUCTION) + return; + if (is_launch) { + if (DMA_SEMAPHORE_TYPE(launch_dma) != + SEM_RELEASE_ONE_WORD) + return; + if (DMA_SEM_REDUCTION_EN(launch_dma) && + DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC) + return; + } + if (svc_resolve_va(vgpu, r->va, &addr, &ap)) + return; + if (svc_sem_read(vgpu, r->va, &cur)) + return; + /* Forward-only pre-check: only a word behind the payload is relayed. */ + if (cur >= r->payload) + return; + r->tgt_ok = true; + r->tgt_kind = ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1); + r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1)); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -3399,6 +3488,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = payload; r->entry_idx = entry_idx; r->dma = 0; + svc_fwd_relay_note(vgpu, ch, r, + 0, false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; } else { @@ -3512,6 +3603,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = st.rs_payload; r->entry_idx = entry_idx; r->dma = 0; + svc_fwd_relay_note(vgpu, ch, + r, 0, + false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; break; @@ -3844,6 +3938,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = st.sem_payload; r->entry_idx = entry_idx; r->dma = dma; + svc_fwd_relay_note(vgpu, ch, r, + dma, true); /* * P6b wave 1.5: do NOT bump * svc.pushes_pending here - @@ -5486,6 +5582,25 @@ struct svc_fwd_req { u64 src_page, dst_page; /* SYS: page GPA; VID: shadow byte offset */ u32 src_off, dst_off; u32 bytes; + /* + * P6c completion relay: when relay is true this request's hardware + * copy additionally authors the guest-visible release of the selected + * deferred record (relay_index into svc_fwd_rel[]): relay_kind/page/ + * off identify the resolved semaphore target and relay_payload is the + * value the engine writes. relay_done/relay_sem/relay_elapsed_ms are + * published by the work item before its done release, for the hold's + * verification/logging. Cleared with the rest of the slot whenever the + * releases do not qualify. + */ + bool relay; + u8 relay_index; + u8 relay_kind; + u64 relay_page; + u32 relay_off; + u32 relay_payload; + bool relay_done; + u32 relay_sem; + u32 relay_elapsed_ms; }; static struct svc_fwd_req svc_fwd_req; @@ -5669,6 +5784,15 @@ struct svc_fwd_hold { u64 gpfifo_va; u32 entry_idx; u8 nrel; + /* + * P6c completion relay: the request selected record relay_index and + * the hardware copy was asked to author it into the guest-visible + * semaphore page. On the success edge the hold re-reads the guest + * word and, when it carries the payload, skips that one record's + * software apply. Cleared with the hold. + */ + bool relay_armed; + u8 relay_index; struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; @@ -5733,14 +5857,26 @@ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu, inc, r->inc_claimed); } -/* Apply the recorded releases of one deferred push (service thread). */ -static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, - const struct svc_channel *ch, - const struct svc_fwd_rel *rel, u8 n) +/* Skip value that never matches a record index. */ +#define SVC_FWD_REL_SKIP_NONE 0xff + +/* + * Apply the recorded releases of one deferred push (service thread), except + * the record at `skip` (SVC_FWD_REL_SKIP_NONE applies all). P6c uses the skip + * on the relay verification success edge: the engine already authored that + * one release into the guest-visible semaphore, so re-applying it in software + * is unnecessary (and would count a second release). + */ +static void svc_fwd_rel_apply_all_ex(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + const struct svc_fwd_rel *rel, u8 n, + u8 skip) { u8 i; for (i = 0; i < n; i++) { + if (i == skip) + continue; switch (rel[i].kind) { case SVC_FWD_REL_906F: svc_rel_apply_906f(vgpu, ch, rel[i].va, @@ -5759,6 +5895,14 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, } } +/* Apply every recorded release (no relay skip). */ +static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + const struct svc_fwd_rel *rel, u8 n) +{ + svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE); +} + static bool svc_fwd_hold_expired(void) { return time_after(jiffies, svc_fwd_hold.since + @@ -5790,6 +5934,8 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.nrel = n; svc_fwd_hold.last_rel_va = svc_rel_va_seen; svc_fwd_hold.since = jiffies; + svc_fwd_hold.relay_armed = svc_fwd_req.relay; + svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -5813,6 +5959,8 @@ static void svc_fwd_hold_clear(void) memset(&svc_fwd_hold.cand, 0, sizeof(svc_fwd_hold.cand)); svc_fwd_hold.vgpu = NULL; svc_fwd_hold.nrel = 0; + svc_fwd_hold.relay_armed = false; + svc_fwd_hold.relay_index = 0; svc.fwd_rel_pending = 0; } @@ -5861,6 +6009,13 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, if (!done) svc.fwd_hold_timeouts++; } + /* + * P6b/P6c: the timeout path never reaches the relay verification + * edge, so a relayed set is software-applied whole (forward-only, so + * the engine's write cannot be duplicated). + */ + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -5883,16 +6038,63 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { - svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, - svc_fwd_hold.nrel); - svc.fwd_rel_applied++; + /* + * P6c: on the relay success edge, re-read the guest-visible + * semaphore through the software view. Only a word that + * actually carries the payload lets the one relayed record be + * skipped; anything else (unreadable, behind) is a relay + * verification failure and the software applies the whole set + * (the forward-only apply makes the relayed write idempotent, + * so the release is never duplicated or lost). + */ + if (svc_fwd_hold.relay_armed && + svc_fwd_hold.relay_index < svc_fwd_hold.nrel) { + struct svc_fwd_rel *rr = + &svc_fwd_hold.rel[svc_fwd_hold.relay_index]; + u32 cur = 0; + + if (!svc_sem_read(hv, rr->va, &cur) && + cur >= rr->payload) { + svc_fwd_rel_apply_all_ex(hv, ch, + svc_fwd_hold.rel, + svc_fwd_hold.nrel, + svc_fwd_hold.relay_index); + svc.fwd_relay_ok++; + svc.fwd_relay_skipped_sw++; + svc.fwd_rel_applied++; + /* + * The hold's post is the single completion + * edge for this channel: the pump's per-pass + * notifier was suppressed while the relay hold + * was active. + */ + svc_notify_complete(hv, ch); + } else { + svc.fwd_relay_verify_fail++; + svc.fwd_relay_fallback++; + svc_fwd_rel_apply_all(hv, ch, + svc_fwd_hold.rel, + svc_fwd_hold.nrel); + svc.fwd_rel_applied++; + if ((svc.fwd_relay_verify_fail & 15) == 1) + pr_err("gsp-phys-work: RELAY VERIFY FAILED: guest sem va=0x%llx cur=0x%08x payload=0x%08x; software applied all releases\n", + rr->va, cur, rr->payload); + } + } else { + svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + svc.fwd_rel_applied++; + } } else { /* * P6b wave 2: failure/timeout/abort fallback - author the * skipped data once, then apply the releases exactly as - * wave 1.5 does. + * wave 1.5 does. P6c: a relayed set is applied whole here + * (the engine did not deliver it). */ svc_fwd_hold_soft_copy(hv); + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc.fwd_rel_fallback++; @@ -5973,6 +6175,13 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) svc_fwd_hold_soft_copy(hv); svc.fwd_hold_canceled++; } + /* + * P6c: a vanished channel cannot run the relay verification edge; a + * relayed set falls back to the software path (which is exactly-once + * against the engine's forward write). + */ + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -6006,6 +6215,8 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) svc.fwd_rel_applied++; else svc.fwd_rel_fallback++; + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -6029,9 +6240,10 @@ static void svc_fwd_work_fn(struct work_struct *work) * while a hold is active. */ if (guest) { - u64 src_hpa = 0, dst_hpa = 0; - u32 src_off = 0, dst_off = 0; - int src_rc, dst_rc; + u64 src_hpa = 0, dst_hpa = 0, sem_hpa = 0; + u32 src_off = 0, dst_off = 0, sem_off = 0; + struct gsp_phys_backend_copy_result cp = { 0 }; + int src_rc, dst_rc, sem_rc = 0; bool resolved; src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, @@ -6047,11 +6259,43 @@ static void svc_fwd_work_fn(struct work_struct *work) svc_fwd_req.bytes, &dst_hpa, &dst_off); resolved = !src_rc && !dst_rc; + /* + * P6c: resolve the selected relay target to an HPA with the + * same operand resolver. A failure is not fatal - hw_rc is + * forced to an error so the hold applies the software + * release exactly as wave 1.5 (never lost, never duplicated). + */ + if (svc_fwd_req.relay) + sem_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, + svc_fwd_req.relay_kind, + svc_fwd_req.relay_page, + svc_fwd_req.relay_off, 4, + &sem_hpa, &sem_off); if (resolved) { - hw_rc = gsp_phys_backend_guest_ce_copy(src_hpa, src_off, - dst_hpa, dst_off, - svc_fwd_req.bytes, - GSP_PHYS_BACKEND_WORK_POLL_MS); + if (svc_fwd_req.relay && !sem_rc) { + hw_rc = gsp_phys_backend_guest_ce_copy_ex( + src_hpa, src_off, dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS, + sem_hpa, sem_off, + svc_fwd_req.relay_payload, &cp); + svc_fwd_req.relay_sem = cp.sem_value; + svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms; + svc_fwd_req.relay_done = (hw_rc == 0); + } else if (svc_fwd_req.relay) { + hw_rc = -EINVAL; + svc.fwd_relay_resolve_fail++; + if (svc.fwd_relay_resolve_fail <= 16) + pr_info("gsp-phys-work: relay semaphore resolve failed: kind=%u page=0x%llx off=%u rc=%d; software release stays authoritative\n", + svc_fwd_req.relay_kind, + svc_fwd_req.relay_page, + svc_fwd_req.relay_off, sem_rc); + } else { + hw_rc = gsp_phys_backend_guest_ce_copy( + src_hpa, src_off, dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS); + } if (!hw_rc) { svc.fwd_guest_ok++; if (!svc_fwd_req.defer) { @@ -6145,6 +6389,56 @@ static void svc_fwd_work_fn(struct work_struct *work) static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); /* + * P6c completion relay: pick the ONE deferred release record this request's + * hardware copy may author. Only a deferred guest request whose records were + * armed can relay, and only a record whose record-time resolution succeeded + * (tgt_ok) qualifies; the push-ending LAUNCH record is preferred (it is the + * completion edge the guest tracker waits on), else the last eligible record. + * Fail-open: no eligible record leaves req->relay false and the whole release + * set stays software-applied. Service-thread only; called with the request + * slot's INFLIGHT bit held, before queue_work. + */ +static void svc_fwd_relay_select(struct svc_fwd_req *req) +{ + int i; + + req->relay = false; + req->relay_index = 0; + req->relay_kind = 0; + req->relay_page = 0; + req->relay_off = 0; + req->relay_payload = 0; + + if (!req->defer || !gsp_phys_relay_enabled()) + return; + for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { + if (svc_fwd_rel[i].kind == SVC_FWD_REL_LAUNCH && + svc_fwd_rel[i].tgt_ok) { + req->relay = true; + req->relay_index = (u8)i; + req->relay_kind = svc_fwd_rel[i].tgt_kind; + req->relay_page = svc_fwd_rel[i].tgt_page; + req->relay_off = svc_fwd_rel[i].tgt_off; + req->relay_payload = svc_fwd_rel[i].payload; + svc.fwd_relay_requested++; + return; + } + } + for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { + if (svc_fwd_rel[i].tgt_ok) { + req->relay = true; + req->relay_index = (u8)i; + req->relay_kind = svc_fwd_rel[i].tgt_kind; + req->relay_page = svc_fwd_rel[i].tgt_page; + req->relay_off = svc_fwd_rel[i].tgt_off; + req->relay_payload = svc_fwd_rel[i].payload; + svc.fwd_relay_requested++; + return; + } + } +} + +/* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold * the entry for its deferred releases. All wave-1 gates are unchanged. @@ -6209,6 +6503,12 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, gsp_phys_guest_mem_enabled(); svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed && svc_fwd_rel_n > 0; + /* + * P6c: choose the one relayed record for this request (if any). The + * selection is reset with the rest of the slot every time the releases + * do not qualify, so a later pass can never replay a stale pick. + */ + svc_fwd_relay_select(&svc_fwd_req); if (svc_fwd_req.guest) { svc_fwd_req.vgpu = vgpu; svc_fwd_req.src_kind = svc_fwd_cand.src_kind; @@ -6360,6 +6660,42 @@ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft, *canceled = svc.fwd_hold_canceled; } +/* + * P6c completion-relay counters (see gsp_sema_service.h). Same plain-read + * discipline as the wave-1/1.5 stats above. + */ +void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback, + u64 *skipped_sw, u64 *verify_fail, + u64 *resolve_fail) +{ + if (requested) + *requested = svc.fwd_relay_requested; + if (ok) + *ok = svc.fwd_relay_ok; + if (fallback) + *fallback = svc.fwd_relay_fallback; + if (skipped_sw) + *skipped_sw = svc.fwd_relay_skipped_sw; + if (verify_fail) + *verify_fail = svc.fwd_relay_verify_fail; + if (resolve_fail) + *resolve_fail = svc.fwd_relay_resolve_fail; +} + +/* + * P6c: true while this channel owns an active RELAY hold. The pump's + * per-pass notifier is suppressed for such a channel so the hold's verified + * success edge (or the next pass after a failure resolution) is the single + * completion post rather than a premature one before the guest release lands. + */ +static bool svc_fwd_relay_hold_active(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + return svc_fwd_hold.active && svc_fwd_hold.relay_armed && + svc_fwd_hold.vgpu == vgpu && + svc_fwd_hold.ch_handle == ch->h_object; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -6414,7 +6750,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 applied_before; bool held = false; - svc_notify_complete(vgpu, ch); + /* + * P6c: while a relay hold owns this channel, the hold's + * verified success edge is the single completion post; skip + * the per-pass notifier so it cannot fire before the guest + * release is authored. + */ + if (!svc_fwd_relay_hold_active(vgpu, ch)) + svc_notify_complete(vgpu, ch); /* * P6b wave 1.5: enforce the hold cap on EVERY pass for the held @@ -6729,6 +7072,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); + /* + * P6c: a + * selected + * relay whose + * request did + * not queue is + * software-applied + * here. + */ + if (svc_fwd_req.relay) + svc.fwd_relay_fallback++; + svc_fwd_req.relay = false; } } /* diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 35d1c0f..ffb7df6 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -118,6 +118,20 @@ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft, u64 *canceled); /* + * P6c completion-relay counters for the gsp-phys-state work_relay line: + * requested relay selections armed on a deferred guest request, relays whose + * engine-authored guest semaphore was verified (ok) with their software + * applies skipped (skipped_sw), relays that fell back to the software apply + * (fallback: engine failure/timeout/abort or a word that did not carry the + * payload), hold re-reads that did not see the payload (verify_fail) and + * work-item semaphore HPA resolution failures (resolve_fail). All zero while + * gsp_phys_relay=0. Plain reads, safe from the debugfs show path. + */ +void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback, + u64 *skipped_sw, u64 *verify_fail, + u64 *resolve_fail); + +/* * Discover and pin GR (compute) channel rings after the CE region of the * channel video-memory allocation. Set false to leave GR rings untranslated * (the pump then only runs the CE channels). |