summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c123
-rw-r--r--kernel/gsp_phys_booter.h17
-rw-r--r--kernel/gsp_sema_service.c30
-rw-r--r--kernel/gsp_sema_service.h8
4 files changed, 144 insertions, 34 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 4409a29..2199cae 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -7345,18 +7345,22 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters);
* completion relay) moves these; every legacy call keeps moving the guest
* counters above only, so with gsp_phys_relay=0 they stay zero and the legacy
* lines are identical. Plain increments from the sema service work item; only
- * logging and the debugfs state file read them.
+ * logging and the debugfs state file read them. verify_transient counts data
+ * mismatches that a second read 2 ms later resolved (live guest page mutated
+ * under the first compare); verify_fail counts final mismatches only, after
+ * that one retry.
*/
static u64 gsp_phys_relay_requested;
static u64 gsp_phys_relay_ok;
static u64 gsp_phys_relay_verify_fail;
+static u64 gsp_phys_relay_verify_transient;
static u64 gsp_phys_relay_timed_out;
static u64 gsp_phys_relay_refused;
static u64 gsp_phys_relay_pool_exhausted;
void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
- u64 *timeouts, u64 *refused,
- u64 *pool_exhausted)
+ u64 *verify_transient, u64 *timeouts,
+ u64 *refused, u64 *pool_exhausted)
{
if (requested)
*requested = gsp_phys_relay_requested;
@@ -7364,6 +7368,8 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
*ok = gsp_phys_relay_ok;
if (verify_fail)
*verify_fail = gsp_phys_relay_verify_fail;
+ if (verify_transient)
+ *verify_transient = gsp_phys_relay_verify_transient;
if (timeouts)
*timeouts = gsp_phys_relay_timed_out;
if (refused)
@@ -7517,7 +7523,11 @@ static bool gsp_phys_fault_relay_active(void)
* semaphore. A zero sem_hpa keeps the module-semaphore behavior
* byte-identical (same gates, same counters, same logs). Relay refusals,
* timeouts and verify failures move the relay counters and return an errno
- * the caller falls back to, so no release can be lost. *out (if given)
+ * the caller falls back to, so no release can be lost. A relay data mismatch
+ * is retried once after 2 ms (the pages are live guest-writable RAM and the
+ * engine publishes the semaphore before the host reads the data back): a
+ * mismatch that clears counts verify_transient and stays a verified copy, a
+ * persistent one counts verify_fail and fails closed. *out (if given)
* always receives the last semaphore word, the elapsed milliseconds and the
* return code.
*/
@@ -7533,6 +7543,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
void __iomem *bar0;
struct page *sem_page = NULL;
+ u8 *snap = NULL;
unsigned long start = jiffies;
unsigned long deadline;
u64 src_va = 0, dst_va = 0, sem_va = 0;
@@ -7625,6 +7636,19 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
goto out;
}
+ /*
+ * P6c hardening: snapshot the source window before the doorbell so a
+ * final verify mismatch can show whether the live source changed under
+ * the host (mutation detected) or the copy itself landed wrong. Relay
+ * only, so the gsp_phys_relay=0 path allocates nothing. bytes <= 4 KB
+ * (GSP_PHYS_RMCALL_COPY_SIZE, enforced above).
+ */
+ if (relay) {
+ snap = kmalloc(bytes, GFP_KERNEL);
+ if (snap)
+ memcpy(snap, (u8 *)__va(src_hpa) + src_off, bytes);
+ }
+
mutex_lock(&gsp_phys_rm_lock);
/*
* Re-check under the lock like the P5 path: the release may have
@@ -7768,19 +7792,71 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
*/
if (memcmp((u8 *)__va(dst_hpa) + dst_off,
(u8 *)__va(src_hpa) + src_off, bytes) != 0) {
- if (relay) {
- gsp_phys_relay_verify_fail++;
- pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
- poll, dst_hpa, dst_off, src_hpa, src_off,
- bytes);
- } else {
+ if (!relay) {
gsp_phys_guest_verify_fail++;
pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
poll, dst_hpa, dst_off, src_hpa, src_off,
bytes);
+ ret = -EIO;
+ goto out;
}
- ret = -EIO;
- goto out;
+ /*
+ * P6c hardening: the engine publishes the guest semaphore
+ * before the host reads the data back and the guest can mutate
+ * either live page inside that window, so the first mismatch
+ * can be a transient torn read. This is a process-context work
+ * item (the poll above already msleeps), so sleep briefly and
+ * re-read both pages once; a retry that matches proves the
+ * copy landed and keeps the verified-copy semantics, only a
+ * persistent mismatch falls back as before.
+ */
+ msleep(2);
+ if (memcmp((u8 *)__va(dst_hpa) + dst_off,
+ (u8 *)__va(src_hpa) + src_off, bytes) != 0) {
+ gsp_phys_relay_verify_fail++;
+ if ((gsp_phys_relay_verify_fail & 15) == 1) {
+ const u8 *dp = (const u8 *)__va(dst_hpa) +
+ dst_off;
+ const u8 *sp = (const u8 *)__va(src_hpa) +
+ src_off;
+ const char *snap_str = "none";
+ char dbuf[33], sbuf[33], nbuf[33];
+ u32 diff, dlen;
+ int src_mutated = 0, snap_dst_diff = 0;
+
+ for (diff = 0; diff < bytes; diff++)
+ if (dp[diff] != sp[diff])
+ break;
+ dlen = bytes - diff;
+ if (dlen > 16)
+ dlen = 16;
+ bin2hex(dbuf, dp + diff, dlen);
+ bin2hex(sbuf, sp + diff, dlen);
+ if (snap) {
+ bin2hex(nbuf, snap + diff, dlen);
+ snap_str = nbuf;
+ src_mutated =
+ memcmp(sp, snap, bytes) != 0;
+ snap_dst_diff =
+ memcmp(dp, snap, bytes) != 0;
+ }
+ pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes (retried after 2 ms) diff@%u dst=%s src=%s snap=%s src_mutated=%u snap_vs_dst=%u sem_hpa=0x%llx+%u payload=0x%08x poll=0x%08x elapsed_ms=%u verify_fail=%llu verify_transient=%llu\n",
+ poll, dst_hpa, dst_off, src_hpa,
+ src_off, bytes, diff, dbuf, sbuf,
+ snap_str, src_mutated, snap_dst_diff,
+ sem_hpa, sem_off, sem_payload, poll,
+ jiffies_to_msecs(jiffies - start),
+ gsp_phys_relay_verify_fail,
+ gsp_phys_relay_verify_transient);
+ }
+ ret = -EIO;
+ goto out;
+ }
+ gsp_phys_relay_verify_transient++;
+ if ((gsp_phys_relay_verify_transient & 63) == 1)
+ pr_info("gsp-phys-work: GUEST CE RELAY transient data mismatch resolved on retry src_hpa=0x%llx dst_hpa=0x%llx bytes=%u transient=%llu\n",
+ src_hpa, dst_hpa, bytes,
+ gsp_phys_relay_verify_transient);
}
if (sem_out)
@@ -7805,6 +7881,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
out:
if (sem_page)
put_page(sem_page);
+ kfree(snap);
if (out) {
out->sem_value = poll;
out->elapsed_ms = jiffies_to_msecs(jiffies - start);
@@ -8741,22 +8818,30 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
/*
* P6c completion relay: the sema service's relay-selection and
* hold-verification counters (requested/ok/fallback/skipped_sw/
- * verify_fail/resolve_fail) plus the booter relay copy
- * counters (timeouts/pool_exhausted/refused) and the parameter
- * state. All zero while the relay is off.
+ * hold_verify_fail/resolve_fail) plus the booter relay copy
+ * counters (verify_fail/verify_transient/timeouts/
+ * pool_exhausted/refused) and the parameter state. The
+ * backend verify_fail is the data-compare result after the one
+ * retry; hold_verify_fail is the sema service's guest-sem
+ * re-read miss. All zero while the relay is off.
*/
u64 r_req = 0, r_ok = 0, r_fallback = 0, r_skipped = 0;
- u64 r_verify_fail = 0, r_resolve_fail = 0;
+ u64 r_hold_verify_fail = 0, r_resolve_fail = 0;
+ u64 r_verify_fail = 0, r_verify_transient = 0;
u64 r_timeouts = 0, r_pool = 0, r_refused = 0;
gsp_sema_service_fwd_relay_stats(&r_req, &r_ok, &r_fallback,
- &r_skipped, &r_verify_fail,
+ &r_skipped,
+ &r_hold_verify_fail,
&r_resolve_fail);
- gsp_phys_backend_relay_counters(NULL, NULL, NULL, &r_timeouts,
+ gsp_phys_backend_relay_counters(NULL, NULL, &r_verify_fail,
+ &r_verify_transient,
+ &r_timeouts,
&r_refused, &r_pool);
seq_printf(m,
- "work_relay: requested=%llu ok=%llu fallback=%llu skipped_sw=%llu verify_fail=%llu resolve_fail=%llu timeouts=%llu pool_exhausted=%llu refused=%llu relay_param=%u\n",
+ "work_relay: requested=%llu ok=%llu fallback=%llu skipped_sw=%llu verify_fail=%llu verify_transient=%llu hold_verify_fail=%llu resolve_fail=%llu timeouts=%llu pool_exhausted=%llu refused=%llu relay_param=%u\n",
r_req, r_ok, r_fallback, r_skipped, r_verify_fail,
+ r_verify_transient, r_hold_verify_fail,
r_resolve_fail, r_timeouts, r_pool, r_refused,
gsp_phys_relay_enabled());
}
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 52d9c43..81f1803 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -247,6 +247,10 @@ void gsp_phys_backend_work_counters(u64 *submitted, u64 *released,
* semaphore word observed, the elapsed milliseconds and the return code. A
* zero sem_hpa keeps the module-semaphore behavior byte-identical.
* gsp_phys_backend_guest_ce_copy() is a wrapper for the zero-relay form.
+ * A relay data-verify mismatch sleeps 2 ms and re-reads both pages once
+ * (live guest-writable RAM): a mismatch that clears counts as a transient and
+ * stays a verified copy, a persistent one counts as verify_fail and fails
+ * closed to the caller's software release.
*/
int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
u32 dst_off, u32 bytes, u32 poll_ms);
@@ -281,14 +285,15 @@ int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa,
/*
* gsp_phys_backend_relay_counters() reads the P6c relay-path counters
- * (requested relay copies, verified relays, data verify failures, guest
- * semaphore poll timeouts, gate/structural refusals and scratch-pool
- * exhaustion) for logging and the gsp-phys-state work_relay line. All zero
- * unless gsp_phys_relay=1 copies ran.
+ * (requested relay copies, verified relays, final data verify failures,
+ * transient verify mismatches resolved by the one retry, guest semaphore poll
+ * timeouts, gate/structural refusals and scratch-pool exhaustion) for logging
+ * and the gsp-phys-state work_relay line. All zero unless gsp_phys_relay=1
+ * copies ran.
*/
void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
- u64 *timeouts, u64 *refused,
- u64 *pool_exhausted);
+ u64 *verify_transient, u64 *timeouts,
+ u64 *refused, u64 *pool_exhausted);
/*
* P6c C3 fault observation/relay (gsp_phys_fault_relay=1; default off).
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 2b52c0e..d42edab 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -486,9 +486,10 @@ struct sema_svc {
* engine-authored guest semaphore was verified with the software apply
* skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to
* the software apply (fwd_relay_fallback), hold re-reads that did not
- * see the payload (fwd_relay_verify_fail) and work-item semaphore HPA
- * resolution failures (fwd_relay_resolve_fail). All zero while the
- * parameter is off.
+ * see the payload (fwd_relay_verify_fail, shown as hold_verify_fail in
+ * the state line to keep it distinct from the booter compare's
+ * verify_fail) and work-item semaphore HPA resolution failures
+ * (fwd_relay_resolve_fail). All zero while the parameter is off.
*/
u64 fwd_relay_requested;
u64 fwd_relay_ok;
@@ -6121,11 +6122,28 @@ static void svc_fwd_fault_deliver(struct nv_vgpu_device *vgpu,
pr_info("gsp-phys-fault: FAULT RELAY ch=0x%x hwchid=%u rl=%u except=0x%x engine=%u delivered=%llu\n",
ch->h_object, f->chid, f->runlist_id, f->except_type,
engine, svc.fwd_fault_delivered);
- if (gsp_rm_post_channel_event(vgpu, ch->h_object)) {
+ /*
+ * C3 hardening: decode the helper's failure at the call site. -ENODEV
+ * is a missing RM context / channel handle; -ENOENT is the helper's
+ * post-attempt path finding no postable event at all (it already
+ * retried every valid registered event as a fallback), so the exact
+ * "no event for this handle" reason is not recoverable from the
+ * return code alone.
+ */
+ ret = gsp_rm_post_channel_event(vgpu, ch->h_object);
+ if (ret) {
+ const char *why;
+
+ if (ret == -ENODEV)
+ why = "no-rm-context-or-handle";
+ else if (ret == -ENOENT)
+ why = "no-postable-event";
+ else
+ why = "event-post-send-failed";
svc.fwd_fault_no_event++;
- pr_info("gsp-phys-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x no_event=%llu\n",
+ pr_info("gsp-phys-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x reason=%s ret=%d no_event=%llu\n",
ch->h_object, f->chid, f->runlist_id, f->except_type,
- svc.fwd_fault_no_event);
+ why, ret, svc.fwd_fault_no_event);
}
}
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 4b429ea..d5a62e9 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -123,9 +123,11 @@ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft,
* engine-authored guest semaphore was verified (ok) with their software
* applies skipped (skipped_sw), relays that fell back to the software apply
* (fallback: engine failure/timeout/abort or a word that did not carry the
- * payload), hold re-reads that did not see the payload (verify_fail) and
- * work-item semaphore HPA resolution failures (resolve_fail). All zero while
- * gsp_phys_relay=0. Plain reads, safe from the debugfs show path.
+ * payload), hold re-reads that did not see the payload (verify_fail, shown as
+ * hold_verify_fail in the state line, distinct from the backend compare's
+ * verify_fail) and work-item semaphore HPA resolution failures (resolve_fail).
+ * All zero while gsp_phys_relay=0. Plain reads, safe from the debugfs show
+ * path.
*/
void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback,
u64 *skipped_sw, u64 *verify_fail,
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 09:13:08 +0000