diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 134 |
1 files changed, 123 insertions, 11 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 856947f..11549ce 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1545,6 +1545,19 @@ struct sema_svc { */ #define SVC_SETUP_DEMAND_MAX 512 u64 setup_demand_vas[SVC_SETUP_DEMAND_MAX]; + /* + * SESSION 153-CONTINUED-64: the held entry's exact read window. + * The armed setup hold records its entry page with (off = entry va's + * in-page offset, len = the GPFIFO entry length); every other page + * keeps len = 0. The guest pin resolver uses the pair to validate a + * candidate page RANGE against the exact bytes the host will read + * [va, va+len) with the host's own full method-stream parse, so a pin + * is only injected when the host read is guaranteed to parse (kills + * the content-heuristic family learn that served a wrong area in + * r1120: `first8=00040000 00001000 ... plausible=0`). + */ + u8 setup_demand_off[SVC_SETUP_DEMAND_MAX]; + u32 setup_demand_len[SVC_SETUP_DEMAND_MAX]; u32 setup_demand_count; u32 setup_demand_head; /* ring write cursor (valid once full) */ u64 setup_demand_total; /* notes accepted (dedup hits excluded) */ @@ -1594,17 +1607,33 @@ static void svc_log_stalled_va(u64 va) * aligned u64 on a 64-bit kernel, so the in-place overwrite cannot tear for * the seq reader (worst case it sees the previous resident VA once more). */ -static void svc_setup_demand_record(u64 va) +static void svc_setup_demand_record_ex(u64 va, u32 off, u32 len) { u32 i, n = svc.setup_demand_count; if (!va) return; for (i = 0; i < n; i++) - if (svc.setup_demand_vas[i] == va) + if (svc.setup_demand_vas[i] == va) { + /* + * SESSION 153-CONTINUED-64: upgrade a plain entry to the + * anchored (off,len) window when the hold re-records it. + * off/len publish order keeps the reader consistent for + * new slots; an in-place upgrade may be seen one pass + * torn (off new, len old) - the guest then validates the + * wrong window once and retries, which is harmless. + */ + if (len && !svc.setup_demand_len[i]) { + svc.setup_demand_off[i] = (u8)off; + smp_wmb(); + svc.setup_demand_len[i] = len; + } return; + } if (n < SVC_SETUP_DEMAND_MAX) { svc.setup_demand_vas[n] = va; + svc.setup_demand_off[n] = (u8)off; + svc.setup_demand_len[n] = len; smp_wmb(); svc.setup_demand_count = n + 1; } else { @@ -1612,6 +1641,8 @@ static void svc_setup_demand_record(u64 va) u32 w = svc.setup_demand_head; svc.setup_demand_vas[w] = va; + svc.setup_demand_off[w] = (u8)off; + svc.setup_demand_len[w] = len; svc.setup_demand_head = (w + 1u) % SVC_SETUP_DEMAND_MAX; svc.setup_demand_evicted++; if (ring_log < 4u || !(svc.setup_demand_evicted % 4096u)) { @@ -1624,6 +1655,11 @@ static void svc_setup_demand_record(u64 va) svc.setup_demand_total++; } +static void svc_setup_demand_record(u64 va) +{ + svc_setup_demand_record_ex(va, 0, 0); +} + /* * SESSION 153-CONTINUED-59: cross-file entry for the GR launch builder (see * gsp_sema_service.h). Records the page of a VA whose window/attr operand @@ -1642,9 +1678,12 @@ void gsp_sema_demand_note(u64 va) * Read-only view of the setup-hold demand set for the pin tool: * # sema setup-hold demand VAs (page-aligned): N total=T evicted=E * 0x4021000 + * 0x4001004 0x4 60216 <- anchored entry: entry va, in-page offset, len * ... * `total` counts every accepted note (dedup hits excluded); `evicted` counts - * ring overwrites of the oldest slot (SESSION 153-CONTINUED-60). + * ring overwrites of the oldest slot (SESSION 153-CONTINUED-60). An anchored + * line carries the exact [va, va+len) read window of the held GPFIFO entry + * (SESSION 153-CONTINUED-64). */ static int svc_setup_demand_show(struct seq_file *m, void *v) { @@ -1653,9 +1692,18 @@ static int svc_setup_demand_show(struct seq_file *m, void *v) seq_printf(m, "# sema setup-hold demand VAs (page-aligned): %u total=%llu evicted=%llu\n", n, (unsigned long long)svc.setup_demand_total, (unsigned long long)svc.setup_demand_evicted); - for (i = 0; i < n && i < SVC_SETUP_DEMAND_MAX; i++) - seq_printf(m, "0x%llx\n", - (unsigned long long)svc.setup_demand_vas[i]); + for (i = 0; i < n && i < SVC_SETUP_DEMAND_MAX; i++) { + u32 dl = svc.setup_demand_len[i]; + + if (dl) + seq_printf(m, "0x%llx 0x%x %u\n", + (unsigned long long)svc.setup_demand_vas[i] + + svc.setup_demand_off[i], + (unsigned)svc.setup_demand_off[i], dl); + else + seq_printf(m, "0x%llx\n", + (unsigned long long)svc.setup_demand_vas[i]); + } return 0; } @@ -3725,6 +3773,37 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, (*drops)++; } +/* + * SESSION 153-CONTINUED-64d: same store, but reports a probe-window + * exhaustion back to the caller so it can evict instead of losing the pin + * (see svc_ipin_store). Returns true when the slot was NOT stored. + */ +static bool pin_store_tab_nodrop(struct svc_pin *tab, u32 slots, + u32 probe_max, u32 *n, u64 *drops, u64 va, + u64 pa, int ap, u8 win) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = slots - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < probe_max; probe++) { + struct svc_pin *p = &tab[(h + probe) & mask]; + + if (!p->va_page || p->va_page == page) { + if (!p->va_page) + (*n)++; + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + p->ap = ap; + p->win = win; + return false; + } + } + (*drops)++; + return true; +} + static int svc_pin_lookup(u64 va, u64 *pa, int *ap) { return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, @@ -3939,8 +4018,32 @@ static void svc_ipin_store(u64 va, u64 pa, int ap) (old_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) != (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) svc_ipin_alt_store(va, old_pa, old_ap); - pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, - &svc.ipin_drop, va, pa, ap, 0); + /* + * SESSION 153-CONTINUED-64d: the table SATURATES under client map + * churn (r1122: `ipins=32768/148777` - the old store DROPPED every + * pin past the 32768 slots, so the demand stream's setup-push pages + * could never land and the held read never flipped). Slots: 1<<15. + * When the probe window is exhausted, evict the home slot's + * occupant instead of dropping: the evicted pair is preserved in + * the alt table (where a reader may still fall back to it) and both + * the scanner's periodic re-assert and the demand resend cadences + * restore any evicted entry that is still needed. A dropped demand + * pin, by contrast, is unrecoverable within the hold window. + */ + if (pin_store_tab_nodrop(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, + &svc.ipin_n, &svc.ipin_drop, va, pa, ap, 0)) { + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_IPIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + struct svc_pin *p = &svc.ipin[h]; + + if (p->va_page && p->va_page != page) + svc_ipin_alt_store(p->va_page, p->pa_page, p->ap); + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + p->ap = ap; + p->win = 0; + } /* * A fresh injection supersedes every older translation for this * page: drop the verified pool cache entry so a subsequent pool @@ -14029,8 +14132,8 @@ static void svc_mme_census(const u8 *buf, u32 len, u64 push_va, u32 h_object) * appends strictly in order; the generation only increments after a * capture that appended, giving the booter a seqcheck against torn views. */ -#define SVC_MME_PRELUDE_WORDS 512u -#define SVC_MME_PRELUDE_SIGS 256u +#define SVC_MME_PRELUDE_WORDS 1024u +#define SVC_MME_PRELUDE_SIGS 512u static u32 svc_mme_prelude_buf[SVC_MME_PRELUDE_WORDS]; static u32 svc_mme_prelude_n; static u32 svc_mme_prelude_gen; @@ -22711,8 +22814,17 @@ static void svc_gr_setup_hold_arm(struct svc_channel *ch, u64 va, u32 idx, * debugfs demand set the pin tool reads directly, so the * demand no longer depends on the dmesg row surviving the * ring rotation under vgsp-sema spam. + * + * SESSION 153-CONTINUED-64: the entry's own page (poff == 0) + * additionally carries (off, len) - the exact [va, va+len) + * window the host read will parse - so the guest resolver + * can validate a candidate page range with the host's own + * full method-stream parse before injecting the pin. */ - svc_setup_demand_record(page + poff); + svc_setup_demand_record_ex(page + poff, + poff == 0 ? + (u32)(va & (SEMA_PAGE_SIZE - 1)) : 0u, + poff == 0 ? len : 0u); } } |