diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 57 |
1 files changed, 56 insertions, 1 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index e57bcac..9de9038 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3104,6 +3104,8 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa) static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */ static bool svc_dst_guard_hold(u64 va); /* m169b */ +static bool svc_dst_guard_pending(u64 va); /* r231 */ +static u32 svc_read_order_log; /* r231 budget */ static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap) { @@ -5241,6 +5243,24 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, dst_orig, dst_prov); return -ENOENT; } + /* + * r231 READ-ORDER guard/witness: a device-src copy whose source VA + * has a PENDING device-dst write hold reads the PRE-write content - + * the writer's bytes land only when the guard resolves while the + * reader executes immediately (the ctaid_probe RAWA round trip is + * the reproducer). Witnessed always (bounded); deferred only with + * sema_read_order_guard, through the same bounded-hold re-decode. + */ + if (src_orig >= SVC_VAS_FLOOR && svc_dst_guard_pending(src_orig)) { + if (svc_read_order_log < 16) { + svc_read_order_log++; + pr_info("sema-svc: READ-ORDER-WIT device-src copy va=0x%llx reads while a device-dst write to the same VA is still held%s\n", + src_orig, gsp_sema_read_order_guard ? + " (deferring)" : " (guard off)"); + } + if (gsp_sema_read_order_guard) + return -ENOENT; + } if (!src_phys) { u64 pa; u32 aper; @@ -5285,7 +5305,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * 16 bytes on each side before the copy. The vadd data path is exactly * these copies; this separates a wrong resolution from a wrong copy. */ - if ((total >= (1u << 20) || + if ((total >= SVC_COPY_AUDIT_MIN || (dst_virt && dst_orig >= SVC_VAS_FLOOR)) && bigcopy_log < gsp_sema_bigcopy_log_max) { u8 sb[16] = {0}, db[16] = {0}; @@ -10862,6 +10882,21 @@ bool gsp_sema_copy_dst_pin_guard = 1; unsigned gsp_sema_copy_dst_guard_ms = 3000; /* + * r231 READ-ORDER guard (default 0: witness only). A device-src copy whose + * source VA has a PENDING device-dst write hold (the m169 guard window is + * still open) reads the PRE-write content: the writer's bytes land only + * when the guard resolves, while the reader can execute immediately. The + * ctaid_probe RAWA round trip is the reproducer - upload a fresh 64 KB + * buffer, read it straight back with no kernel - and the readback came + * back as foreign deterministic content. With this on, such a read is + * deferred through the existing bounded-hold machinery (-ENOENT) so it + * re-decodes after the write resolves; with it off the condition is + * logged (READ-ORDER-WIT) so the next capture proves the coincidence. + * Opt-in until an A/B proves no green regression. + */ +bool gsp_sema_read_order_guard; + +/* * COPYWIT fail4: per-boot budget for the sema-bigcopy resolution witnesses * (svc_copy's large-copy proof: both operands' VA -> pa/aperture/provenance * plus the first 16 bytes each side). The shipped default kept the boot's @@ -10911,6 +10946,26 @@ static bool svc_dst_guard_hold(u64 va) } /* + * r231 READ-ORDER guard: true while a held device-dst write for this VA + * page is still inside its bounded window (the writer's bytes have not + * landed yet). See the gsp_sema_read_order_guard declaration. + */ +static bool svc_dst_guard_pending(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + unsigned long now = jiffies; + u32 i; + + for (i = 0; i < SVC_DST_GUARD_SLOTS; i++) + if (svc_dst_guard[i].va_page == page && + svc_dst_guard[i].first && + time_before(now, svc_dst_guard[i].first + + msecs_to_jiffies(gsp_sema_copy_dst_guard_ms))) + return true; + return false; +} + +/* * CE release fallback: when a push carrying a tracking-semaphore release * cannot be read (and the content rescue failed), advance the channel's * tracking semaphore to the expected payload directly. The push's data |