diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 91 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 72 |
2 files changed, 106 insertions, 57 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 9c19efb..9eb1179 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -16741,6 +16741,24 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) u32 runlist_id, ch_id, doorbell; int ret; + /* + * m155 (HF-3 extended re-arm): after the channel has been idle long + * enough for the FECS to save/unload the context, re-issuing only the + * two GPFIFO_SCHEDULE verbs does not revive dispatch - r154: matmul's + * post-smiley launches acked (qmd_rel/ch_rel) but SKED stayed idle + * (cwd0=0x2086/fecs_cur=0) for every attempt including outcome + * retries, while the same channel executed mail/smiley earlier in the + * boot. Replay the mode-35 lane-enable tail (CILP preempt mode, + * timeslice, device sched) that stage 4 ran when the lane first went + * live - it is proven accepted and is the design's extended re-arm. + * The poison variants (explicit PREEMPT/tail, BIND, PROMOTE) stay out. + */ + ret = gsp_phys_rmcall_user_lane_lane_enable(bar0); + if (ret) + pr_info("vgsp-gr: rearm lane-enable rc=%d (continuing to schedules)\n", + ret); + rpc_result = 0; + status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, sched_params, sizeof(sched_params), 16, @@ -18351,72 +18369,35 @@ gr_page_unmapped: * gsp_phys_gr_output_retire_wait(). * * m153: outcome-verifying retry. The intermittent per-boot - * mis-execution classes (engine wrote nothing at all; or it wrote - * into a lower operand window while the highest - output - operand - * stayed untouched) both complete with qmd_rel/ch_rel satisfied, so - * they cannot be caught by the completion poll. The retirement wait - * is the only execution witness. When it saw zero writes, or writes - * that skipped the highest-reloc operand (n_data >= 2), refill the - * bounce window from the guest inputs, re-publish the same method - * block on a fresh ring entry, re-issue the schedule and re-poll, - * then re-wait - bounded by the same attempt budget as the timeout - * path. After the budget the launch proceeds to copy-out with - * whatever the engine produced (a genuinely read-only kernel is - * indistinguishable here and simply pays the bounded extra polls). + * no-execution class (qmd_rel/ch_rel satisfied but the engine wrote + * nothing at all) cannot be caught by the completion poll; the + * retirement wait is the only execution witness. On zero writes, + * refill the bounce window from the guest inputs, re-publish the + * same method block on a fresh ring entry, re-issue the schedule and + * re-poll, then re-wait - bounded by the same attempt budget as the + * timeout path. After the budget the launch proceeds to copy-out + * with whatever the engine produced: a genuinely read-only kernel is + * indistinguishable from a no-execution here and simply pays the + * bounded extra polls (safe to re-execute by construction - a kernel + * that writes nothing has no side effects to duplicate). A + * wrong-place write detector was tried and removed: the highest-reloc + * data entry is often a UVM-internal buffer, not the output, so it + * false-fired on correct launches and would re-execute + * side-effecting kernels (atomics) up to 3x. */ if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in); if (gsp_phys_rmcall_gr_nowrite_retry_value()) { - u64 top_reloc = 0; - u32 top_bi = 0xffffffffu, di2; - - for (di2 = 0; di2 < req->n_data; di2++) { - u64 rv = req->data[di2].reloc_va; - u32 bi2; - - if (!rv || rv <= top_reloc) - continue; - bi2 = (u32)((rv - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); - if (bi2 >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) - continue; - top_reloc = rv; - top_bi = bi2; - } while (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { - bool top_changed = false; - - if (top_bi != 0xffffffffu) { - const u8 *tp = - gsp_phys_chunk_page_va( - &boot_stage - .rmcall_gr_bounce, - top_bi); - - if (tp) { - u32 ti, tc = 0; - - for (ti = 0; - ti < GSP_PHYS_PAGE_SIZE; - ti += 4) - tc += get_unaligned_le32( - tp + ti); - top_changed = - (tc != - bounce_csum_in[top_bi]); - } - } - if (wrote > 0 && - (req->n_data < 2 || top_changed)) + if (wrote > 0) break; attempts++; if (attempts > GSP_PHYS_RMCALL_GR_ATTEMPTS) break; - pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u top_bi=%u top_changed=%u): refill + re-publish\n", + pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u): refill + re-publish\n", attempts, GSP_PHYS_RMCALL_GR_ATTEMPTS, - wrote, top_bi, top_changed); + wrote); gsp_phys_gr_fill_bounce(req, bounce_csum_in); mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_publish_locked( diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index d2d1462..f8925df 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -989,6 +989,13 @@ static bool svc_gr_content_rearm; static u32 svc_cepin_log; /* + * m154: injected pins dropped because they collide with a live channel's + * GPFIFO ring range (see svc_handle_map_add). Counted + logged so the + * defensive drop is visible in the boot log. + */ +static u32 svc_ring_pin_skips; + +/* * m148 (second-client wedge): re-arm the per-rank one-shot ring/push-buffer * latches for a (re-)registered channel. The guest driver rebuilds the GSP * session for every client context and RM reuses the same object handles and @@ -1034,6 +1041,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; + svc_ring_pin_skips = 0; } /* @@ -15404,6 +15412,41 @@ static u32 svc_map_add_head; /* producer */ static u32 svc_map_add_tail; /* consumer (service thread) */ static DEFINE_SPINLOCK(svc_map_add_lock); +/* + * m154: does this injected-pin VA fall inside any live channel's GPFIFO ring + * range? The ring VA translation must come from the channel's own discovery + * path (channel object / push-buffer scan); a scanner run that happens to + * span the ring VA is enumerating a different mapping and its translation is + * always wrong for the ring (r153: an all-zero read consumed a ring entry + * and wedged the client for the boot). Called on the service thread; takes + * chan_lock like the rewind below. + */ +static bool svc_va_hits_chan_ring(u64 va) +{ + unsigned long flags; + u32 i; + bool hit = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + u64 lo, hi; + + if (!ch->valid || !ch->gpfifo_va || !ch->gpfifo_entries) + continue; + lo = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); + hi = (ch->gpfifo_va + + (u64)ch->gpfifo_entries * 8 + SEMA_PAGE_SIZE - 1) & + ~(u64)(SEMA_PAGE_SIZE - 1); + if (va >= lo && va < hi) { + hit = true; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return hit; +} + static void svc_handle_map_add(void) { unsigned long flags; @@ -15428,8 +15471,33 @@ static void svc_handle_map_add(void) ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { - svc_ipin_store(req.va + off, req.pa + off, ap); - svc_lat_store(req.va + off, req.pa + off, ap); + u64 va = req.va + off; + + /* + * m154: never let an injected pin shadow a live + * channel's GPFIFO ring. r153: the scanner's merged + * CE-window run (va=0x121000000 len=0x40000, the + * length varies per session) covered a CE channel's + * ring VA 0x121010000; the injected translation took + * precedence in svc_ipin_lookup/read, the ring then + * read the guest page at 0x102e02000 - an all-zero + * entry was silently consumed, the pool release was + * never authored, and the client spun in userspace + * for the rest of the boot (pushes frozen). The ring + * pages already have their own discovery path (the + * channel object / push-buffer scan); a colliding + * injection is always the wrong translation, so drop + * it and leave the ring to that path. + */ + if (svc_va_hits_chan_ring(va)) { + svc_ring_pin_skips++; + if (svc_ring_pin_skips <= 8) + pr_info("sema-svc: injected pin va=0x%llx DROPPED (live channel ring range; ring reads stay on the channel's own translation)\n", + va); + continue; + } + svc_ipin_store(va, req.pa + off, ap); + svc_lat_store(va, req.pa + off, ap); } /* * m149: bounded per-session evidence that the CE push-buffer |