summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c163
1 files changed, 131 insertions, 32 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 91a128f..8ae11ec 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3866,6 +3866,23 @@ static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa)
static void svc_session_flush(void); /* defined next to the pending queue */
+/*
+ * Ledger sizing + counters hoisted above svc_rebuild so the periodic diag
+ * can report the table's fill state: a full table degrades the write/read
+ * authority silently (home replacement evicts another VA's landing), which
+ * is exactly the D2H "reads back prior content" window.
+ */
+#define SVC_LEDGER_MAX (1u << 14)
+#define SVC_LEDGER_MASK (SVC_LEDGER_MAX - 1u)
+#define SVC_LEDGER_PROBE 16u
+static u32 svc_ledger_n;
+static u64 svc_ledger_diverges;
+static u64 svc_ledger_stores;
+static u64 svc_ledger_evictions;
+static u64 svc_ledger_rewrites;
+static u64 svc_ledger_reloc_diverges;
+static u32 svc_ledger_store_log;
+
static void svc_rebuild(struct nv_vgpu_device *vgpu)
{
static u8 *buf;
@@ -3990,6 +4007,17 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
svc_vas_n, SVC_VAS_SLOTS, svc.vas_writes, svc.vas_reads,
svc.vas_shadowed);
/*
+ * D2H-source authority visibility (r220): a full ledger degrades to
+ * home-slot replacement, which silently evicts another VA's landing
+ * and lets readers fall back to the resolver - the "reads back prior
+ * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a
+ * client's window is the tell.
+ */
+ pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu diverges=%llu reloc_div=%llu rewrites=%llu\n",
+ svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores,
+ svc_ledger_evictions, svc_ledger_diverges,
+ svc_ledger_reloc_diverges, svc_ledger_rewrites);
+ /*
* P6b wave 1: guest-forwarding summary. Printed whenever any counter
* moved (not only on requests) so a run that captured no candidate is
* still diagnosable from the log.
@@ -4298,7 +4326,9 @@ static bool svc_dst_watched(u64 dst)
*/
#define SVC_COPY_AUDIT_MIN (32u << 10) /* >32 KB copies audited */
#define SVC_COPY_AUDIT_MAX 2048u /* per-boot line budget */
+#define SVC_CEAUDIT_MAX 4096u /* per-boot CEAUDIT budget */
static u64 svc_copy_audit_n;
+static u64 svc_ceaudit_n;
/* COPYAUDIT for a finished fill: log want/done and check the first 16
* destination bytes against the repeated pattern. The readback uses the
@@ -4346,7 +4376,7 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined belo
static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */
static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
- u32 count, u32 phys, u32 aperture)
+ u32 count, u32 phys, u32 aperture, u64 *done_out)
{
u8 pat[8];
u8 bounce[256];
@@ -4355,6 +4385,8 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
bool shadow = false;
u64 va0 = addr; /* original destination VA (fills, C2) */
+ if (done_out)
+ *done_out = 0;
if (elem != 1 && elem != 2 && elem != 4 && elem != 8)
return -EINVAL;
if (!count || total > (256ULL << 20))
@@ -4401,8 +4433,13 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
svc.op_tr_src, elem, pb, svc.cur_ch);
}
- if (shadow)
- return svc_vas_fill(addr, pat, elem, total);
+ if (shadow) {
+ int rc = svc_vas_fill(addr, pat, elem, total);
+
+ if (!rc && done_out)
+ *done_out = (u64)total;
+ return rc;
+ }
/*
* C2 (data class) + m180 unification: a large virtual fill is a
@@ -4430,6 +4467,8 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
memcpy(base + addr + o, pat, 8);
if (o < total)
memcpy(base + addr + o, pat, total - o);
+ if (done_out)
+ *done_out = (u64)total;
svc_fill_audit(vgpu, va0, addr, aperture, total, pat, elem);
return 0;
}
@@ -4446,6 +4485,8 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
if (i < chunk)
memcpy(bounce + i, pat, chunk - i);
if (svc_write(vgpu, addr + o, bounce, chunk)) {
+ if (done_out)
+ *done_out = (u64)o;
if (total >= SVC_COPY_AUDIT_MIN)
pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=sema-fill rc=%d\n",
va0, (u64)total, (u64)o, -EIO);
@@ -4453,6 +4494,8 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
}
o += chunk;
}
+ if (done_out)
+ *done_out = (u64)total;
svc_fill_audit(vgpu, va0, addr, aperture, total, pat, elem);
return 0;
}
@@ -4874,17 +4917,12 @@ struct svc_ledger_ent {
* (8 MB in + 8 MB out + engine records ~ 12 K pages); hash collisions
* only occur between VA pages 64 MiB apart or across clients whose pages
* do not co-reside in time.
+ *
+ * COPYAUDIT r220: SVC_LEDGER_MAX/MASK/PROBE and the counters live above
+ * svc_rebuild (hoisted for the diag line); only the table and the home
+ * helper stay here.
*/
-#define SVC_LEDGER_MAX (1u << 14)
-#define SVC_LEDGER_MASK (SVC_LEDGER_MAX - 1u)
-#define SVC_LEDGER_PROBE 16u
static struct svc_ledger_ent svc_ledger[SVC_LEDGER_MAX];
-static u32 svc_ledger_n;
-static u64 svc_ledger_diverges;
-static u64 svc_ledger_stores;
-static u64 svc_ledger_rewrites;
-static u64 svc_ledger_reloc_diverges;
-static u32 svc_ledger_store_log;
static inline u32 svc_ledger_home(u64 page)
{
@@ -4937,8 +4975,15 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap)
/*
* Cluster full (16 same-home VA pages co-resident): replace the home
* slot. Bounded, rare, and only affects VAs whose pages hash to the
- * identical home slot.
+ * identical home slot. r220: this is a SILENT loss of the evicted
+ * VA's landing authority (its later readers fall back to the
+ * resolver, possibly stale), so the first occurrences are logged and
+ * counted into the diag line.
*/
+ svc_ledger_evictions++;
+ if (svc_ledger_evictions <= 32)
+ pr_warn("sema-ledger: EVICT home=%u old_va=0x%llx new_va=0x%llx (cluster full; old landing authority lost until rewritten)\n",
+ home, (u64)svc_ledger[home].va_page, page);
svc_ledger[home].va_page = page;
svc_ledger[home].pa_page = pa_page;
svc_ledger[home].ap = ap;
@@ -4986,30 +5031,38 @@ static void svc_ledger_clear(void)
*/
static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va,
u64 o, const u8 *bounce, size_t chunk,
- bool shadow, u64 pa_eff, u32 aper)
+ bool shadow, u64 pa_eff, u32 aper, bool tail)
{
size_t n = chunk < 16 ? chunk : 16;
+ size_t so = tail ? chunk - n : 0; /* true end sample */
u8 rb[16] = {0};
if (shadow) {
- if (svc_vas_read(dst_va + o, rb, n))
+ if (svc_vas_read(dst_va + o + so, rb, n))
return;
} else if (aper == CE_APERTURE_VID) {
u8 *base = vgpu->bars[1].vaddr;
- if (!base || pa_eff + n > svc_fb_shadow_size(vgpu))
+ if (!base || pa_eff + so + n > svc_fb_shadow_size(vgpu))
return;
- memcpy(rb, base + pa_eff, n);
- } else if (svc_read(vgpu, pa_eff, rb, n)) {
+ memcpy(rb, base + pa_eff + so, n);
+ } else if (svc_read(vgpu, pa_eff + so, rb, n)) {
return;
}
- if (memcmp(rb, bounce, n))
+ if (memcmp(rb, bounce + so, n)) {
pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x%llx path=ce-soft-copy want=%*phN got=%*phN\n",
- dst_va + o, o, (int)n, bounce, (int)n, rb);
+ dst_va + o + so, o + so, (int)n,
+ bounce + so, (int)n, rb);
+ pr_info_ratelimited("vgsp: CEAUDIT-MISMATCH op=copy va=0x%llx off=0x%llx path=ce-soft-copy want=%*phN got=%*phN\n",
+ dst_va + o + so, o + so, (int)n,
+ bounce + so, (int)n, rb);
+ }
}
#define SVC_COPY_TRUNC_RET(rc_) \
do { \
+ if (done_out) \
+ *done_out = (u64)o; \
if (total >= SVC_COPY_AUDIT_MIN) \
pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=ce-soft-copy rc=%d\n", \
dst_orig, (u64)total, (u64)o, (rc_)); \
@@ -5017,7 +5070,8 @@ static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va,
} while (0)
static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
- u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture)
+ u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture,
+ u64 *done_out)
{
u8 bounce[256];
size_t o;
@@ -5030,6 +5084,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
bool audit = total >= SVC_COPY_AUDIT_MIN;
static u64 bigcopy_log;
+ if (done_out)
+ *done_out = 0; /* 0 until the first byte lands */
if (!total || total > (256ULL << 20))
return -EINVAL;
@@ -5439,7 +5495,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
SVC_COPY_TRUNC_RET(-ENOMEM);
if (audit && (!o || o + chunk == total))
svc_copy_audit_chunk(vgpu, dst_orig, o, bounce,
- chunk, true, 0, 0);
+ chunk, true, 0, 0,
+ (o != 0));
} else if (dst_virt) {
u64 pa = 0;
u32 aper = 0;
@@ -5489,7 +5546,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
}
if (audit && (!o || o + chunk == total))
svc_copy_audit_chunk(vgpu, dst_orig, o, bounce,
- chunk, false, pa, aper);
+ chunk, false, pa, aper,
+ (o != 0));
/*
* m171/m180: a large device-dest write records its
* landing page - SYS and VID alike, because the
@@ -5514,14 +5572,16 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
if (audit && (!o || o + chunk == total))
svc_copy_audit_chunk(vgpu, dst_orig, o, bounce,
chunk, false, dst + o,
- CE_APERTURE_VID);
+ CE_APERTURE_VID,
+ (o != 0));
} else {
if (svc_write(vgpu, dst + o, bounce, chunk))
SVC_COPY_TRUNC_RET(-EIO);
if (audit && (!o || o + chunk == total))
svc_copy_audit_chunk(vgpu, dst_orig, o, bounce,
chunk, false, dst + o,
- CE_APERTURE_SYS);
+ CE_APERTURE_SYS,
+ (o != 0));
}
(void)i;
o += chunk;
@@ -5588,6 +5648,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
pr_info("vgsp: COPYAUDIT path=ce-soft-copy va=0x%llx len=0x%llx done=0x%llx\n",
dst_orig, (u64)total, (u64)o);
}
+ if (done_out)
+ *done_out = (u64)o; /* o == total on the success path */
return 0;
}
@@ -5765,13 +5827,50 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
continue;
}
- if (m->type == 1)
- rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
- m->phys, m->aperture,
- m->src_phys, m->src_aperture);
- else
- rc = svc_fill(vgpu, m->dst, m->value, m->elem,
- m->count, m->phys, m->aperture);
+ {
+ u64 ce_len = (m->type == 1) ? m->bytes :
+ (u64)m->elem * m->count;
+ u64 ce_done = 0;
+
+ if (m->type == 1)
+ rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
+ m->phys, m->aperture,
+ m->src_phys, m->src_aperture,
+ &ce_done);
+ else
+ rc = svc_fill(vgpu, m->dst, m->value, m->elem,
+ m->count, m->phys, m->aperture,
+ &ce_done);
+ /*
+ * CEAUDIT (2026-10-18 r220): per-op audit of the CE
+ * data path - the software executor of every guest
+ * cudaMemcpy - at/above SVC_COPY_AUDIT_MIN, including
+ * the rc != 0 attempts the copy-level COPYAUDIT cannot
+ * see (the pre-loop resolution misses return before
+ * any byte moves: done=0x0). src/dst are the guest
+ * VAs of the op; done is the byte count that landed.
+ * Bounded per boot.
+ */
+ if (ce_len >= SVC_COPY_AUDIT_MIN &&
+ svc_ceaudit_n < SVC_CEAUDIT_MAX) {
+ svc_ceaudit_n++;
+ /*
+ * A failing op is retried by the hold and would
+ * spam: throttle the rc != 0 repeats; successful
+ * big ops log unconditionally (budget-bounded).
+ */
+ if (rc)
+ pr_info_ratelimited("vgsp: CEAUDIT op=%s src=0x%llx dst=0x%llx len=0x%llx done=0x%llx rc=%d\n",
+ m->type == 1 ? "copy" : "fill",
+ m->type == 1 ? m->src : 0ull,
+ m->dst, ce_len, ce_done, rc);
+ else
+ pr_info("vgsp: CEAUDIT op=%s src=0x%llx dst=0x%llx len=0x%llx done=0x%llx rc=%d\n",
+ m->type == 1 ? "copy" : "fill",
+ m->type == 1 ? m->src : 0ull,
+ m->dst, ce_len, ce_done, rc);
+ }
+ }
if ((rc == 0 || rc == SVC_COPY_FORWARDED) &&
ops_done && seq < 64)
*ops_done |= 1ULL << seq;
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-19 16:24:33 +0000