From 0bb52e0e3cdc86b31dce2daa23eafd099461604a Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sat, 17 Oct 2026 01:15:00 -0400 Subject: vgsp: m171 event completion fixes (minimal cudaEventSynchronize repro now passes) + AGENT-F R2 write/read consistency ledger - gsp_rm_emulation: the event burst budget becomes a time-windowed rate (1 s) with completion posts allowed headroom over hint re-posts; the periodic hint dedups events posted within 250 ms (strictly below its own 500 ms cadence - an equal window races the boundary and skips forever); force_post skips retired slots (rc=22 spam was an artifact). Verified: the minimal cudaEventCreate/Record/Synchronize repro and a second repro with cross-stream waits and elapsed-time now complete on a live boot; previously cudaEventSynchronize hung forever. - gsp_sema_service: write/read consistency ledger (AGENT-F R2). Large device-destination writes record {va_page -> pa, aperture}; the launch data fill and device-source reads prefer the recorded page whenever their own resolution disagrees (sema-ledger DIVERGE log). Session scoped, zero cost while empty. --- kernel/gsp_rm_emulation.c | 118 +++++++++++++++++++++++++++++++++++++++----- kernel/gsp_rm_emulation.h | 4 +- kernel/gsp_sema_service.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 0afb013..f203826 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -452,8 +452,26 @@ static u32 gsp_rm_build_record(struct gsp_rm_emu *rm, u8 *buf, u32 buf_size, /* * Write a status queue record (1+ elements) and advance the writePtr. */ -static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, - u32 sequence, u32 result, const void *data, u32 data_len) +/* + * m171: the event burst budget is a rate over a short time window, not a + * monotonic session counter. Without the time reset a flood left the counter + * pinned at the cap and silently dropped the completion post of a client + * that only started waiting later - a minimal cudaEventSynchronize repro sat + * forever while its registrations were valid and 'completion hint rc=-22' + * repeated. Channel-completion posts get headroom over hint re-posts: a hint + * can be retried a moment later, a completion wake-up cannot. + */ +#define GSPRM_EVENT_BURST_CRIT_MAX 48 +#define GSPRM_EVENT_BURST_WINDOW (HZ) +/* Strictly shorter than the service's 0.5 s hint cadence: an equal window + * races the boundary and every later hint skips forever (observed r181: + * 'completion hint rc=-2' forever after one success and the stream client + * still stalled - the late-waiter wake-up never fired again). */ +#define GSPRM_EVENT_HINT_DEDUP (HZ / 4) + +static int gsp_rm_send_flags(struct gsp_emu_context *ctx, u32 function, + u32 sequence, u32 result, const void *data, + u32 data_len, bool critical) { struct gsp_rm_emu *rm = ctx->rm_emu; u8 *buf; @@ -489,13 +507,22 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, * newer record and the client aborts with "Bad sequence number". */ if (function == GSPRM_RPC_EVENT_POST_EVENT) { - if (rm->event_burst >= GSPRM_EVENT_BURST_MAX) { + u32 cap = critical ? GSPRM_EVENT_BURST_CRIT_MAX + : GSPRM_EVENT_BURST_MAX; + + /* m171: expire the budget by wall time so a long-ago + * flood can never starve a fresh completion post. */ + if (time_after(jiffies, rm->event_burst_at + + GSPRM_EVENT_BURST_WINDOW)) + rm->event_burst = 0; + rm->event_burst_at = jiffies; + if (rm->event_burst >= cap) { rm->events_dropped++; if (rm->events_dropped <= 8 || (rm->events_dropped % 1000) == 0) nv_vgpu_info(ctx->vgpu, - "vgsp-rm: event burst capped (%u), dropped total=%llu", - GSPRM_EVENT_BURST_MAX, + "vgsp-rm: event burst capped (%u/%u), dropped total=%llu", + rm->event_burst, cap, rm->events_dropped); mutex_unlock(&rm->stat_lock); kfree(buf); @@ -504,6 +531,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, rm->event_burst++; } else { rm->event_burst = 0; + rm->event_burst_at = jiffies; } record_size = gsp_rm_build_record(rm, buf, buf_size, function, @@ -589,6 +617,14 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, return 0; } +/* Non-event sends: the historical entry point, budget untouched. */ +static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, + u32 sequence, u32 result, const void *data, u32 data_len) +{ + return gsp_rm_send_flags(ctx, function, sequence, result, data, + data_len, false); +} + /* * Post the GPU event that tells the guest's CPU-side RM a channel's * submitted work is complete. A real GPU raises an interrupt, the host GSP @@ -627,9 +663,10 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src) memcpy(payload + 8, &rm->events[i].notify_index, 4); memcpy(payload + 12, &rm->events[i].data, 4); /* info16@16, status@20, eventDataSize@24, bNotifyList@28 = 0 */ - if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, - payload, sizeof(payload))) + if (gsp_rm_send_flags(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, + payload, sizeof(payload), true)) continue; + rm->events[i].last_post = jiffies; posted++; } @@ -648,9 +685,11 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src) memcpy(payload + 4, &rm->events[i].h_event, 4); memcpy(payload + 8, &rm->events[i].notify_index, 4); memcpy(payload + 12, &rm->events[i].data, 4); - if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, - payload, sizeof(payload))) + if (gsp_rm_send_flags(ctx, GSPRM_RPC_EVENT_POST_EVENT, + 0, 0, payload, + sizeof(payload), true)) continue; + rm->events[i].last_post = jiffies; posted++; } } @@ -675,6 +714,7 @@ static int gsp_rm_post_event_idx(struct gsp_emu_context *ctx, u32 idx) { struct gsp_rm_emu *rm = ctx->rm_emu; u8 payload[32]; + int rc; if (idx >= rm->events_n || !rm->events[idx].valid) return -EINVAL; @@ -685,8 +725,11 @@ static int gsp_rm_post_event_idx(struct gsp_emu_context *ctx, u32 idx) memcpy(payload + 8, &rm->events[idx].notify_index, 4); memcpy(payload + 12, &rm->events[idx].data, 4); - return gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, - payload, sizeof(payload)); + rc = gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, + payload, sizeof(payload)); + if (!rc) + rm->events[idx].last_post = jiffies; + return rc; } int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx) @@ -709,8 +752,14 @@ int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx) } for (i = 0; i < rm->events_n; i++) { - int r = gsp_rm_post_event_idx(ctx, i); + int r; + /* m171: retired slots are not errors - skipping them + * keeps the hint's rc meaningful (a valid table must not + * report EINVAL because the last allocated slot is dead). */ + if (!rm->events[i].valid) + continue; + r = gsp_rm_post_event_idx(ctx, i); if (r) last = r; else @@ -725,10 +774,53 @@ int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx) * guest allocates carry no source object, so a per-channel association is * not available in the ABI). Used by the service's completion hint so a * client that starts waiting after the event fired still gets a wake-up. + * + * m171: this is a HINT, and hints repeat every 0.5 s while the window is + * open. Re-posting an event that a real completion post already woke moments + * ago burns the burst budget and floods the status ring without adding a + * wake-up, so skip events posted within GSPRM_EVENT_HINT_DEDUP. A waiter + * that arrives after the edge still gets a re-post within half a second. */ int gsp_rm_post_all_events(struct nv_vgpu_device *vgpu) { - return gsp_rm_force_post(vgpu, 0xffffffffu); + struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL; + struct gsp_rm_emu *rm; + u8 payload[32]; + u32 i; + int n = 0, skipped = 0; + + if (!ctx) + return -ENODEV; + rm = ctx->rm_emu; + if (!rm) + return -ENODEV; + + for (i = 0; i < rm->events_n; i++) { + if (!rm->events[i].valid) + continue; + if (time_before(jiffies, rm->events[i].last_post + + GSPRM_EVENT_HINT_DEDUP)) { + skipped++; + continue; + } + + memset(payload, 0, sizeof(payload)); + memcpy(payload + 0, &rm->events[i].h_client, 4); + memcpy(payload + 4, &rm->events[i].h_event, 4); + memcpy(payload + 8, &rm->events[i].notify_index, 4); + memcpy(payload + 12, &rm->events[i].data, 4); + if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, + payload, sizeof(payload))) + continue; + rm->events[i].last_post = jiffies; + n++; + } + + if (n) + return 0; + /* -EAGAIN: valid events existed but every one was dedup-skipped + * (normal during an active post burst); -ENOENT: nothing to post. */ + return skipped ? -EAGAIN : -ENOENT; } /* diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index e464809..96ea16a 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -197,11 +197,13 @@ struct gsp_rm_emu { u32 notify_index; u32 data; bool valid; + unsigned long last_post; /* m171: jiffies of the last posted record */ } events[GSPRM_MAX_EVENTS]; u32 events_n; u64 events_posted; u64 events_dropped; - u32 event_burst; /* event records written since the last reply */ + u32 event_burst; /* event records written in the current window */ + unsigned long event_burst_at; /* m171: burst window start (time-based reset) */ }; int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 0217594..95134bd 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -4519,6 +4519,74 @@ static bool svc_fwd_defer_armed; #define SVC_COPY_FORWARDED 1 /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ +/* + * m171 (AGENT-F R2): write/read consistency ledger. A large H2D write to a + * device VA records {va_page -> pa, aperture} for every page it wrote; the + * later read of the same VA (the launch data fill, or a D2H copy) prefers + * the recorded page whenever its own resolution disagrees. The pipeline + * then reads exactly what was written regardless of which of the two + * resolutions is stale - the r174/r175 first-page divergence (the write + * landed on pa X, the fill read a stale page Y, and the engine computed on + * old bytes) can no longer split a write/read pair. Session-scoped: cleared + * with the flush; zero cost while empty. Apertures are stored in the SVC_ + * encoding, the canonical space for the svc accessors. + */ +struct svc_ledger_ent { + u64 va_page; + u64 pa_page; + u32 ap; +}; +#define SVC_LEDGER_MAX 256 +static struct svc_ledger_ent svc_ledger[SVC_LEDGER_MAX]; +static u32 svc_ledger_n; +static u32 svc_ledger_next; +static u64 svc_ledger_diverges; + +static void svc_ledger_store(u64 va, u64 pa, u32 ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_ledger_n; i++) { + if (svc_ledger[i].va_page == page) { + svc_ledger[i].pa_page = pa_page; + svc_ledger[i].ap = ap; + return; + } + } + if (svc_ledger_n < SVC_LEDGER_MAX) + i = svc_ledger_n++; + else + i = svc_ledger_next++ % SVC_LEDGER_MAX; + svc_ledger[i].va_page = page; + svc_ledger[i].pa_page = pa_page; + svc_ledger[i].ap = ap; +} + +static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_ledger_n; i++) { + if (svc_ledger[i].va_page == page) { + *pa = svc_ledger[i].pa_page | + (va & (SEMA_PAGE_SIZE - 1)); + *ap = svc_ledger[i].ap; + return 0; + } + } + return -ENOENT; +} + +static void svc_ledger_clear(void) +{ + memset(svc_ledger, 0, sizeof(svc_ledger)); + svc_ledger_n = 0; + svc_ledger_next = 0; +} + static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) { @@ -4830,9 +4898,28 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, } else if (src_virt) { u64 pa = 0; u32 aper = 0; + u64 lpa = 0; + u32 lap = 0; if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) return -ENOENT; + /* + * m171 (AGENT-F R2): read-after-write - when a large + * H2D wrote this device VA page through a different + * translation, read the page the write landed on. + */ + if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src_orig + o, &lpa, &lap) && + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) { + svc_ledger_diverges++; + if (svc_ledger_diverges <= 32) + pr_info("sema-ledger: DIVERGE read va=0x%llx wrote=0x%llx read=0x%llx\n", + src_orig + o, lpa, pa); + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } if (aper == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; @@ -4869,6 +4956,16 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (!base || pa + chunk > svc_fb_shadow_size(vgpu)) return -EINVAL; memcpy(base + pa, bounce, chunk); + /* + * m171 (AGENT-F R2): a large device-dest write + * records its landing page so the later kernel + * fill / D2H read of the same VA uses it even + * if its own resolution is stale. + */ + if (dst_orig >= SVC_VAS_FLOOR && + total >= (64u << 10)) + svc_ledger_store(dst_orig + o, pa, + SVC_APERTURE_FB); } else if (svc_write(vgpu, pa, bounce, chunk)) { return -EIO; } @@ -7647,6 +7744,8 @@ static void svc_session_flush(void) } svc_vas_n = 0; memset(svc.lat, 0, sizeof(svc.lat)); + /* m171 (AGENT-F R2): the write/read ledger is per-session. */ + svc_ledger_clear(); spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) svc.vas[i].valid = false; @@ -13308,6 +13407,28 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) break; } } + /* + * m171 (AGENT-F R2): prefer the write/read ledger's + * landing page for this operand VA when it disagrees + * with the resolution - the kernel then reads exactly + * the page a large H2D wrote (r174: write landed on + * pa X while the fill read a stale page Y). + */ + if (va >= SVC_VAS_FLOOR) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(va, &lpa, &lap) && + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) { + svc_ledger_diverges++; + if (svc_ledger_diverges <= 32) + pr_info("sema-ledger: DIVERGE fill va=0x%llx wrote=0x%llx read=0x%llx\n", + va, lpa, pa); + pa = lpa; + ap = lap; + } + } if (svc_fwd_req.gr_n_data >= bounce_max) { /* * B-2 fail-closed cap: the active region is -- cgit v1.2.3