diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 153 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 10 | ||||
| -rw-r--r-- | kernel/main.c | 12 |
3 files changed, 175 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 3c837b3..d02b789 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -4613,6 +4613,109 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined belo static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */ static void svc_ledger_clear(void); /* STALLFREE: cap-and-degrade reset */ +/* + * r384 (the guest boot-corruption lottery, root-caused 2026-10-20): + * page-table pages of the guest kernel's direct map were found ZEROED + * (two adjacent 4 KiB PTE pages at gpa 0x12d3c2000/0x3000 => two 2 MiB + * holes in the guest physmap, kernel objects inside them unreachable, + * multi-oops panic). The only emulation writers of page-sized ZERO runs + * are the virtual FILL executor (svc_fill / CE memsets), whose large + * >= SVC_VAS_FLOOR destinations resolve through the walk/scan when no + * pin and no ledger answer exists - and that resolver is demonstrably + * non-deterministic (four executions of the same 8 KiB engine-window + * clear resolved to four different guest pages, all refused by the + * below-floor guard). + * + * Two nets, mirroring the m169/r378 copy guards: + * 1. svc_fill_guard_hold(): a LARGE (>= 64 KiB) virtual fill whose only + * authority is the walk/scan (no pin, no ledger) is held through a + * short bounded window and its VA pushed to the pin-demand channel + * (the userspace pin stream injects the guest-pagemap truth), so the + * retried decode resolves through a real pin. Fail-open after the + * bound with the historical behaviour, like the copy guard. + * 2. svc_fill_target_is_pt(): a ZERO fill about to land in guest RAM on + * a page whose first 64 qwords are all PTE-shaped (present, NX, + * physical address in range) is REFUSED loudly. A page table page is + * never a legitimate zero-fill destination; this directly blocks the + * observed corruption shape even when some other resolver approved + * the target. Pin-backed resolutions are exempt (the guest's own + * pagemap is truth - if it says the VA maps the page, obey it). + */ +unsigned gsp_sema_fill_guard_ms = 800; +bool gsp_sema_fill_guard = true; +#define SVC_FILL_GUARD_SLOTS 16 +static struct { + u64 va_page; + unsigned long first; +} svc_fill_guard[SVC_FILL_GUARD_SLOTS]; +static u32 svc_fill_guard_next; +static u32 svc_fill_held; +static u32 svc_fill_pt_refused; + +/* true = the guard may hold this fill; false = fail-open, run it. */ +static bool svc_fill_guard_hold(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + unsigned long now = jiffies; + u32 i, slot = SVC_FILL_GUARD_SLOTS; + + for (i = 0; i < SVC_FILL_GUARD_SLOTS; i++) { + if (svc_fill_guard[i].va_page == page) { + if (time_before(now, svc_fill_guard[i].first + + msecs_to_jiffies( + gsp_sema_fill_guard_ms))) + return true; + if (svc_fill_guard[i].first) { + svc_fill_guard[i].first = 0; + pr_info("sema-svc: fill guard released without a pin va=0x%llx (using the walk translation)\n", + page); + } + return false; + } + if (!svc_fill_guard[i].va_page) + slot = i; + } + if (slot == SVC_FILL_GUARD_SLOTS) + slot = svc_fill_guard_next++ % SVC_FILL_GUARD_SLOTS; + svc_fill_guard[slot].va_page = page; + svc_fill_guard[slot].first = now; + return true; +} + +/* + * A guest-kernel page table page: the first SVC_PT_SCAN qwords are all + * present PTEs (bit 0), non-executable (bit 63 - kernel data mappings), + * with a physical address inside the 34-bit guest range. Random payload + * qwords pass all three conditions with probability far below the 75% + * threshold at SVC_PT_SCAN=64; a genuine table passes at 100%. + */ +#define SVC_PT_SCAN 64u +#define SVC_PT_MIN 48u +static bool svc_fill_target_is_pt(struct nv_vgpu_device *vgpu, u64 addr) +{ + u64 ents[SVC_PT_SCAN]; + u32 i, hits = 0; + + if (addr & (SEMA_PAGE_SIZE - 1)) + return false; /* only a page-aligned table shape */ + if (svc_read(vgpu, addr, ents, sizeof(ents))) + return false; + for (i = 0; i < SVC_PT_SCAN; i++) { + u64 e = ents[i]; + + if (!(e & 1ull)) + continue; + if (!(e & (1ull << 63))) + continue; + if ((e & 0x000ffffffffff000ull) == 0) + continue; + if ((e & 0x000ffffffffff000ull) >= (1ull << 34)) + continue; + hits++; + } + return hits >= SVC_PT_MIN; +} + static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture, u64 *done_out) { @@ -4643,6 +4746,7 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, * pinned map and the VA->PA map. */ u64 pa; u32 aper; + u32 prov = 0; /* r384: 1 pin, 2 ledger, 3 walk/scan */ if (svc_vas_has(addr)) { shadow = true; @@ -4650,6 +4754,7 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u64 lpa = 0; u32 lap = 0; + prov = 1; /* * r372 pin-wins: repair the landing table when it * still names the pre-re-pin page, so every later @@ -4673,6 +4778,7 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, aperture = (aper == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; } else if (!svc_ledger_lookup(addr, &pa, &aper)) { + prov = 2; addr = pa; aperture = (aper == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; @@ -4682,6 +4788,7 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, shadow = true; svc.vas_shadowed++; } else { + prov = 3; addr = pa; aperture = aper; } @@ -4715,6 +4822,52 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, va0, addr, aperture, elem, count); return -ENOENT; } + /* + * r384 net 1 (pin-demand hold for unproven large fills): a + * >= 64 KiB virtual fill whose only authority is the + * walk/scan resolver (no pin, no landing-table entry) is the + * class that zeroed the guest's page-table pages (see the + * guard commentary above svc_fill_guard_hold). Demand the VA + * through the pin channel and hold the op through a short + * bounded window so the retried decode resolves through the + * guest's own pagemap; fail open with the historical + * behaviour after the bound. + */ + if (gsp_sema_fill_guard && va0 >= SVC_VAS_FLOOR && !shadow && + total >= (64u << 10) && prov >= 3) { + svc_log_stalled_va(va0); + if (svc_fill_guard_hold(va0)) { + svc_fill_held++; + if (svc_fill_held <= 32) + pr_info("sema-svc: FILL-HOLD va=0x%llx resolved=0x%llx ap=%u total=%zu value=0x%llx (unproven large fill; pin demanded)\n", + va0, addr, aperture, total, + value); + return -ENOENT; + } + } + /* + * r384 net 2 (page-table-shape refusal): a ZERO fill about + * to land in guest RAM on a page whose first 64 qwords are + * all present+NX+in-range PTEs is refused loudly - a page + * table page is never a legitimate zero-fill destination. + * This blocks the observed corruption shape (two adjacent + * PTE pages zeroed => two 2 MiB holes in the guest physmap) + * even when some other resolver approved the target, and + * irrespective of how wrong the resolution is. Pin-backed + * resolutions are exempt: the guest's own pagemap is truth + * there. + */ + if (gsp_sema_fill_guard && va0 >= SVC_VAS_FLOOR && !shadow && + value == 0 && total >= SEMA_PAGE_SIZE && + aperture == CE_APERTURE_SYS && prov != 1 && + svc_fill_target_is_pt(vgpu, addr)) { + svc_fill_pt_refused++; + if (svc_fill_pt_refused <= 32) + pr_err("vgsp-sema: FILL-PT-REFUSED va=0x%llx pa=0x%llx prov=%u total=%zu (zero fill target is a page table)\n", + va0, addr, prov, total); + svc_log_stalled_va(va0); + return -ENOENT; + } } if (gsp_sema_dbg && !shadow && svc_dst_watched(addr) && diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 0f47244..063ed5c 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -312,6 +312,16 @@ extern bool gsp_sema_copy_dst_pin_guard; extern bool gsp_sema_wait_gate; /* + * r384 (guest boot-corruption lottery): fill-destination guards (default + * on). Large virtual fills whose only authority is the walk/scan resolver + * are held through a short bounded window while their VA is demanded + * through the pin channel (fail-open after gsp_sema_fill_guard_ms); zero + * fills targeting a page-table-shaped guest page are refused loudly. + */ +extern bool gsp_sema_fill_guard; +extern unsigned gsp_sema_fill_guard_ms; + +/* * r231 READ-ORDER guard (default 0: witness only). Defer a device-src copy * whose source VA has a pending device-dst write hold (READ-ORDER-WIT) so * the read re-decodes after the write resolves. Opt-in pending an A/B. diff --git a/kernel/main.c b/kernel/main.c index be5ff5e..92754d1 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2832,6 +2832,18 @@ module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644); MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)"); /* + * r384 (guest boot-corruption lottery): fill-destination guards. A large + * virtual fill whose only authority is the walk/scan resolver is held for + * gsp_sema_fill_guard_ms while its VA is demanded through the pin channel + * (fail-open, mirrors sema_copy_dst_pin_guard); a zero fill targeting a + * page-table-shaped guest page is refused loudly (FILL-PT-REFUSED). + */ +module_param_named(sema_fill_guard, gsp_sema_fill_guard, bool, 0644); +MODULE_PARM_DESC(sema_fill_guard, "guard virtual fill destinations (pin-demand hold + page-table-shape refusal, default 1)"); +module_param_named(sema_fill_guard_ms, gsp_sema_fill_guard_ms, uint, 0644); +MODULE_PARM_DESC(sema_fill_guard_ms, "fill pin-demand hold window in ms (default 800)"); + +/* * r231 READ-ORDER guard: defer a device-src copy whose source VA has a * pending device-dst write hold so the read re-decodes after the write * resolves (fixes the ctaid_probe RAWA upload/readback inversion). |