From 0a9887b1ff8c5546d6b2b33bd84248a9e02c8b6b Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 2 Oct 2026 07:43:05 -0400 Subject: gsp-phys: P6c C0+C1 - engine-authored guest CE semaphore (default off) New default-off gsp_phys_relay: for a forwarded guest CE copy whose push ends in a one-word release, the release page is resolved at forward time, mapped on a fresh scratch line, and the real engine writes the payload directly into the guest-visible semaphore page; the deferred-release hold then verifies that page and skips only that record's software apply. Any failure (unresolvable target, refused kind, pool exhaustion, poll timeout, data mismatch, channel teardown) falls back to applying every release in software, so a release is never lost or duplicated. - new exported gsp_phys_backend_guest_ce_copy_ex() + relay counters; gsp_phys_backend_guest_ce_copy() behavior unchanged (wrapper) - new work_relay: counters line in gsp-phys-state - relay limited to CE channels and one-word absolute releases; counter and reduction releases stay on the existing paths Live: param-off regression matched the wave-2 baseline exactly (req=101 ok=101 verify_ok=101, copy_skipped=119, smoke matched=64/64, relay zeros). With the parameter on: requested=68 ok=67 skipped_sw=67, verify_fail=0, resolve_fail=0, timeouts=0, smoke 64/64, and the engine-authored guest semaphore was observed directly (sem=0x00000003 / 0x0000007e at the guest page). One copy in 68 released its semaphore but failed the host-side data compare; it fell back to the software apply (fallback accounting applied+fallback==engaged) with the smoke green - residual to triage, which is why the parameter stays default-off. A forced-unresolvable run (requested=103, fallback=105, no relay submissions) kept the smoke green. Every cycle: 5 power-domain locks held, driver rebound, WPR2 down, no rail cycles, pstore empty, same boot. --- kernel/gsp_phys_booter.c | 348 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 289 insertions(+), 59 deletions(-) (limited to 'kernel/gsp_phys_booter.c') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 94b0cae..5c36c35 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -6163,10 +6163,13 @@ static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data) * (channel_utils.c:812-820). Returns the block size in dwords (13 method * pairs = 26 dwords, <= GPFIFO entry LENGTH field, bits 30:10). The ladder * stage-4 call passes GSP_PHYS_RMCALL_COPY_SIZE; the P5 shadow submit passes - * the caller's size (both within one 4 KB ladder page). + * the caller's size (both within one 4 KB ladder page). P6c: sem_payload is + * the value the release writes; every legacy call site passes + * GSP_PHYS_RMCALL_SEM_PAYLOAD so the block is byte-identical to before. */ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, - u64 dst_va, u32 bytes) + u64 dst_va, u32 bytes, + u32 sem_payload) { u32 *p = pb; @@ -6177,7 +6180,7 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B, (u32)sem_va); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD, - GSP_PHYS_RMCALL_SEM_PAYLOAD); + sem_payload); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER, (u32)(dst_va >> 32)); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER, @@ -6225,9 +6228,14 @@ struct gsp_phys_rmcall_ce_submit { * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the * token check. The semaphore is cleared before the doorbell so a stale release * from a previous run cannot be observed as this submission's completion. + * P6c: sem_payload is the release value the method block carries; legacy + * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is + * still cleared even when the caller polls a guest page: the clear only + * guards the module-sem poll path and is a harmless non-guest write. */ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = @@ -6276,7 +6284,7 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, wmb(); words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, - bytes); + bytes, sem_payload); gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); @@ -6376,7 +6384,9 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, - bytes, &sub); + bytes, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + &sub); mutex_unlock(&gsp_phys_rm_lock); if (ret) return ret; @@ -6510,7 +6520,9 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, return -EAGAIN; } ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, - bytes, &sub); + bytes, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + &sub); mutex_unlock(&gsp_phys_rm_lock); if (ret) { gsp_phys_work_refused++; @@ -6624,6 +6636,39 @@ void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail, } EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters); +/* + * P6c relay-path counters. Only a copy submitted with a nonzero sem_hpa (the + * completion relay) moves these; every legacy call keeps moving the guest + * counters above only, so with gsp_phys_relay=0 they stay zero and the legacy + * lines are identical. Plain increments from the sema service work item; only + * logging and the debugfs state file read them. + */ +static u64 gsp_phys_relay_requested; +static u64 gsp_phys_relay_ok; +static u64 gsp_phys_relay_verify_fail; +static u64 gsp_phys_relay_timed_out; +static u64 gsp_phys_relay_refused; +static u64 gsp_phys_relay_pool_exhausted; + +void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, + u64 *timeouts, u64 *refused, + u64 *pool_exhausted) +{ + if (requested) + *requested = gsp_phys_relay_requested; + if (ok) + *ok = gsp_phys_relay_ok; + if (verify_fail) + *verify_fail = gsp_phys_relay_verify_fail; + if (timeouts) + *timeouts = gsp_phys_relay_timed_out; + if (refused) + *refused = gsp_phys_relay_refused; + if (pool_exhausted) + *pool_exhausted = gsp_phys_relay_pool_exhausted; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_relay_counters); + /* * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool * and write both operand PTEs into it (src at 16*line_in_page, dst at @@ -6637,9 +6682,15 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters); * timed out with sem=0. The scratch PT pages were zeroed at stage 2, so every * unconsumed slot holds an invalid PTE. Returns -ENOSPC when the pool is * exhausted and -EIO when the scratch PT pages are not allocated. + * + * P6c relay: a nonzero sem_hpa additionally writes a third PTE at + * 16*line_in_page + 2 and returns sem_va = line_va + 2*PAGE_SIZE; sem_hpa==0 + * writes nothing and returns sem_va=0. One line per relay copy either way, so + * the pool capacity (and its exhaustion behavior) is unchanged. */ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, - u64 *src_va, u64 *dst_va) + u64 sem_hpa, u64 *src_va, + u64 *dst_va, u64 *sem_va) { struct gsp_phys_boot_chunk *guest_pt_chunk; u32 line = boot_stage.rmcall_guest_pt_next_line; @@ -6670,11 +6721,21 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + 1) * 8); + /* + * P6c relay: the guest-visible semaphore page gets the third slot of + * the same never-reused line; the write is ordered with the other two + * by the dma_wmb() below, before any doorbell. + */ + if (sem_hpa) + put_unaligned_le64(gsp_phys_rmcall_pte(sem_hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page + 2) * 8); dma_wmb(); boot_stage.rmcall_guest_pt_next_line = line + 1; *src_va = va; *dst_va = va + GSP_PHYS_PAGE_SIZE; + *sem_va = sem_hpa ? va + 2 * GSP_PHYS_PAGE_SIZE : 0; return 0; } @@ -6688,39 +6749,73 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, * gsp_phys_work_forward requirement, so the P6a triggers can share the same * fresh-line copy path with forwarding. *sem_out receives the last * semaphore word seen on success or timeout. + * + * P6c completion relay: a nonzero sem_hpa adds a third PTE on the same fresh + * line for the guest-visible semaphore page (pinned with pfn_valid -> + * get_page/put_page for the duration), points the CE release methods at + * sem_va + sem_off with sem_payload and polls the guest-visible word at + * __va(sem_hpa) + sem_off for the exact payload instead of the module + * semaphore. A zero sem_hpa keeps the module-semaphore behavior + * byte-identical (same gates, same counters, same logs). Relay refusals, + * timeouts and verify failures move the relay counters and return an errno + * the caller falls back to, so no release can be lost. *out (if given) + * always receives the last semaphore word, the elapsed milliseconds and the + * return code. */ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms, - bool need_work_forward, u32 *sem_out) + bool need_work_forward, + u64 sem_hpa, u32 sem_off, + u32 sem_payload, + struct gsp_phys_backend_copy_result *out, + u32 *sem_out) { struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; void __iomem *bar0; + struct page *sem_page = NULL; + unsigned long start = jiffies; unsigned long deadline; - u64 src_va = 0, dst_va = 0; + u64 src_va = 0, dst_va = 0, sem_va = 0; u32 poll = 0; + bool relay = sem_hpa != 0; int ret; + if (relay) + gsp_phys_relay_requested++; + /* * Every gate is fail-closed: the sema service's ladder fallback keeps * the software release authoritative, so a refusal never loses work. */ if (need_work_forward) { if (!gsp_phys_backend_work_ready()) { - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } } else { if (!gsp_phys_backend_ladder_ready()) { - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } } if (!gsp_phys_guest_mem_enabled() || gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } if (!src_hpa || !dst_hpa || !IS_ALIGNED(src_hpa, GSP_PHYS_PAGE_SIZE) || @@ -6732,15 +6827,42 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, pr_err("gsp-phys-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n", src_hpa, src_off, dst_hpa, dst_off, bytes, GSP_PHYS_RMCALL_COPY_SIZE); - return -EINVAL; + ret = -EINVAL; + goto out; + } + if (relay && + (!IS_ALIGNED(sem_hpa, GSP_PHYS_PAGE_SIZE) || + (u64)sem_off + 4 > bytes || !sem_payload)) { + gsp_phys_relay_refused++; + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: invalid semaphore relay sem_hpa=0x%llx sem_off=%u sem_payload=0x%08x bytes=%u (want a page-aligned HPA, sem_off+4 <= copy size, nonzero payload)\n", + sem_hpa, sem_off, sem_payload, bytes); + ret = -EINVAL; + goto out; + } + if (relay) { + unsigned long pfn = sem_hpa >> PAGE_SHIFT; + + if (!pfn_valid(pfn)) { + gsp_phys_relay_refused++; + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore HPA 0x%llx is not a valid RAM page\n", + sem_hpa); + ret = -EINVAL; + goto out; + } + sem_page = pfn_to_page(pfn); + get_page(sem_page); } if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS) poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS; bar0 = gsp_phys_gpu_bar0(); if (!bar0) { - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } mutex_lock(&gsp_phys_rm_lock); @@ -6753,64 +6875,104 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) { mutex_unlock(&gsp_phys_rm_lock); - gsp_phys_guest_refused++; - return -EAGAIN; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; + ret = -EAGAIN; + goto out; } - ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, &src_va, - &dst_va); + ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, + relay ? sem_hpa : 0, &src_va, + &dst_va, &sem_va); if (ret) { if (ret == -ENOSPC) { - gsp_phys_guest_pool_exhausted++; - if ((gsp_phys_guest_pool_exhausted & 63) == 1) - pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", - gsp_phys_guest_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + if (relay) { + gsp_phys_relay_pool_exhausted++; + if ((gsp_phys_relay_pool_exhausted & 63) == 1) + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n", + gsp_phys_relay_pool_exhausted, + GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + } else { + gsp_phys_guest_pool_exhausted++; + if ((gsp_phys_guest_pool_exhausted & 63) == 1) + pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", + gsp_phys_guest_pool_exhausted, + GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + } } else { - gsp_phys_guest_refused++; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n", ret, src_hpa, dst_hpa); } mutex_unlock(&gsp_phys_rm_lock); - return ret; + goto out; } /* * The scratch PTEs map whole pages, so fold each operand's in-page * offset into the submitted VAs; the verify below uses the same - * offsets, keeping the transfer and the check consistent. + * offsets, keeping the transfer and the check consistent. The relay + * points the release at the guest semaphore PTE at sem_va + sem_off + * and carries the caller's payload. */ ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va + src_off, dst_va + dst_off, - GSP_PHYS_RMCALL_VA_SEM, bytes, + relay ? sem_va + sem_off : + GSP_PHYS_RMCALL_VA_SEM, + bytes, + relay ? sem_payload : + GSP_PHYS_RMCALL_SEM_PAYLOAD, NULL); mutex_unlock(&gsp_phys_rm_lock); if (ret) { - gsp_phys_guest_refused++; + if (relay) + gsp_phys_relay_refused++; + else + gsp_phys_guest_refused++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", ret, src_hpa, dst_hpa, bytes); - return ret; + goto out; } /* - * Poll the module semaphore WITHOUT gsp_phys_rm_lock (same discipline - * as the P5 path): the release is written by the engine, so no host - * path can race it and holding the lock would stall the RPC path. + * Poll the release WITHOUT gsp_phys_rm_lock (same discipline as the + * P5 path): the release is written by the engine, so no host path can + * race it and holding the lock would stall the RPC path. The relay + * watches the guest-visible word through its pin/direct map; the + * legacy path polls the module ladder semaphore exactly as before. */ deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { - poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); - if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + if (relay) + poll = READ_ONCE(*(u32 *)((u8 *)__va(sem_hpa) + sem_off)); + else + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == (relay ? sem_payload : GSP_PHYS_RMCALL_SEM_PAYLOAD)) break; if (time_after_eq(jiffies, deadline)) { - gsp_phys_guest_refused++; + if (relay) { + gsp_phys_relay_timed_out++; + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: timeout after %u ms (guest sem=0x%08x want=0x%08x) sem_hpa=0x%llx+%u src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + poll_ms, poll, sem_payload, sem_hpa, + sem_off, src_hpa, dst_hpa, bytes); + } else { + gsp_phys_guest_refused++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + poll_ms, poll, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + src_hpa, dst_hpa, bytes); + } if (sem_out) *sem_out = poll; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", - poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, - src_hpa, dst_hpa, bytes); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto out; } msleep(GSP_PHYS_RMCALL_POLL_MS); } @@ -6822,28 +6984,70 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, */ if (memcmp((u8 *)__va(dst_hpa) + dst_off, (u8 *)__va(src_hpa) + src_off, bytes) != 0) { - gsp_phys_guest_verify_fail++; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", - poll, dst_hpa, dst_off, src_hpa, src_off, bytes); - return -EIO; + if (relay) { + gsp_phys_relay_verify_fail++; + pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", + poll, dst_hpa, dst_off, src_hpa, src_off, + bytes); + } else { + gsp_phys_guest_verify_fail++; + pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", + poll, dst_hpa, dst_off, src_hpa, src_off, + bytes); + } + ret = -EIO; + goto out; } - gsp_phys_guest_verify_ok++; - gsp_phys_guest_fwd_ok++; if (sem_out) *sem_out = poll; - if (gsp_phys_guest_fwd_ok == 1 || (gsp_phys_guest_fwd_ok & 63) == 0) - pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n", - src_hpa, dst_hpa, bytes, poll, - gsp_phys_guest_fwd_ok); - return 0; + if (relay) { + gsp_phys_relay_ok++; + if (gsp_phys_relay_ok == 1 || (gsp_phys_relay_ok & 63) == 0) + pr_info("gsp-phys-work: GUEST CE RELAY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem_hpa=0x%llx+%u sem=0x%08x relays=%llu\n", + src_hpa, dst_hpa, bytes, sem_hpa, sem_off, + poll, gsp_phys_relay_ok); + } else { + gsp_phys_guest_verify_ok++; + gsp_phys_guest_fwd_ok++; + if (gsp_phys_guest_fwd_ok == 1 || + (gsp_phys_guest_fwd_ok & 63) == 0) + pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n", + src_hpa, dst_hpa, bytes, poll, + gsp_phys_guest_fwd_ok); + } + ret = 0; + +out: + if (sem_page) + put_page(sem_page); + if (out) { + out->sem_value = poll; + out->elapsed_ms = jiffies_to_msecs(jiffies - start); + out->rc = ret; + } + return ret; +} + +int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa, + u32 dst_off, u32 bytes, u32 poll_ms, + u64 sem_hpa, u32 sem_off, + u32 sem_payload, + struct gsp_phys_backend_copy_result *out) +{ + return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off, + bytes, poll_ms, true, + sem_hpa, sem_off, sem_payload, + out, NULL); } +EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy_ex); int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms) { - return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off, - bytes, poll_ms, true, NULL); + return gsp_phys_backend_guest_ce_copy_ex(src_hpa, src_off, dst_hpa, + dst_off, bytes, poll_ms, + 0, 0, 0, NULL); } EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); @@ -6861,7 +7065,8 @@ static int gsp_phys_backend_guest_ce_copy_trigger(u64 src_hpa, u32 src_off, u32 *sem_out) { return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off, - bytes, poll_ms, false, sem_out); + bytes, poll_ms, false, + 0, 0, 0, NULL, sem_out); } /* @@ -7043,7 +7248,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, - GSP_PHYS_RMCALL_COPY_SIZE, &sub); + GSP_PHYS_RMCALL_COPY_SIZE, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + &sub); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n", ret); @@ -7690,6 +7897,29 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) g_copy_soft, g_copy_fail, gsp_phys_work_hw_primary_enabled()); } + { + /* + * P6c completion relay: the sema service's relay-selection and + * hold-verification counters (requested/ok/fallback/skipped_sw/ + * verify_fail/resolve_fail) plus the booter relay copy + * counters (timeouts/pool_exhausted/refused) and the parameter + * state. All zero while the relay is off. + */ + u64 r_req = 0, r_ok = 0, r_fallback = 0, r_skipped = 0; + u64 r_verify_fail = 0, r_resolve_fail = 0; + u64 r_timeouts = 0, r_pool = 0, r_refused = 0; + + gsp_sema_service_fwd_relay_stats(&r_req, &r_ok, &r_fallback, + &r_skipped, &r_verify_fail, + &r_resolve_fail); + gsp_phys_backend_relay_counters(NULL, NULL, NULL, &r_timeouts, + &r_refused, &r_pool); + seq_printf(m, + "work_relay: requested=%llu ok=%llu fallback=%llu skipped_sw=%llu verify_fail=%llu resolve_fail=%llu timeouts=%llu pool_exhausted=%llu refused=%llu relay_param=%u\n", + r_req, r_ok, r_fallback, r_skipped, r_verify_fail, + r_resolve_fail, r_timeouts, r_pool, r_refused, + gsp_phys_relay_enabled()); + } { char guest_ram[512]; -- cgit v1.2.3