diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 152 |
1 files changed, 151 insertions, 1 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8d223d8..e69c46c 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -529,6 +529,8 @@ struct sema_svc { u64 gpfifo_skipped; /* stale entries consumed */ u64 gpfifo_rescued; /* unreadable push made readable * by the content rescue */ + u64 resc_run; /* pushes pinned by the multi-page + * content rescue run */ u64 resc_resync; /* expectation re-synced down to * the content watermark */ u64 releases_seen; @@ -2916,6 +2918,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback); + if (svc.resc_run) + pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n", + svc.resc_run); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -7927,6 +7932,114 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } /* + * Multi-page content rescue run (ISSUE-0001 durable direction). + * + * svc_rescue_try() substitutes exactly one candidate page and requires every + * other page of the entry to resolve through svc_lookup_trusted(). For a + * multi-page push whose pages have no translations yet that precondition can + * never hold: the stalled-boot class reads the CE entry at va=0x120021db8 + * (len=12816, four pages, release #39 at the last page) through a window + * guess that is wrong for the entry page while the three continuation pages + * are missing from every trusted source; the content scan finds the + * release-group page (the matcher's candidate) but every single-substitution + * candidate is rejected because the untranslated pages are exactly the ones + * that need a candidate. The userspace pin stream was the only supplier of + * those translations (SA-STALL 3.2/3.4; 17-host-m84-r1.txt:841-842). + * + * A pushbuffer allocation is physically contiguous, so once content has + * anchored ONE page of the entry, the remaining pages sit at the same signed + * page delta from the anchor in one of the two directions. Observed both + * directions in the evidence: rank-2's consecutive release pages descend by + * 0x1000 per VA page ascending (17-host-m84-r1.txt:3-13: 0x1232b8000 -> + * 0x1232b7000 -> ... -> 0x1232b0000), i.e. dir=-1; other boots pair the + * window base forward (svc_pb_pair's convention), i.e. dir=+1. Try both: + * dir=+1: VA page anchor_va + k*PAGE -> PA anchor_pa + k*PAGE + * dir=-1: VA page anchor_va + k*PAGE -> PA anchor_pa - k*PAGE + * Read every page of the entry directly from the candidate PAs, validate the + * assembled buffer exactly as svc_rescue_try() does (full method-stream parse + * + this channel's exact next release), and only then pin every page of the + * run. A failed direction writes nothing, so a wrong anchor or direction + * cannot poison the pin table; a passed validation is content proof for + * every page, strictly stronger than the watcher pin it replaces. + * + * Bounded: entry_len <= SVC_PUSH_MAX, at most SVC_RESCUE_RUN_MAX pages + * (128 KB pushes fit), one direct read per page. Service-thread only. + */ +#define SVC_RESCUE_RUN_MAX 32 /* pages one rescue run may pin */ + +static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 entry_va, + u32 entry_len, u64 anchor_pa, + u64 anchor_va_page, int dir) +{ + u64 pin_va[SVC_RESCUE_RUN_MAX]; + u64 pin_pa[SVC_RESCUE_RUN_MAX]; + u8 *full; + u32 npages = 0, off = 0; + int rc = 0; + + if (!entry_len || (dir != 1 && dir != -1)) + return -EINVAL; + + full = kmalloc(entry_len, GFP_KERNEL); + if (!full) + return -ENOMEM; + + while (off < entry_len) { + u64 va = entry_va + off; + u64 va_page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, entry_len - off, + SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); + s64 delta = (s64)(va_page - anchor_va_page) * dir; + u64 pa; + + if (npages >= SVC_RESCUE_RUN_MAX) { + rc = -E2BIG; + break; + } + if (delta < 0 && (u64)(-delta) > anchor_pa) { + rc = -EINVAL; + break; + } + pa = (u64)((s64)anchor_pa + delta); + if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) { + rc = -EINVAL; + break; + } + if (svc_read(vgpu, pa | (va & (SEMA_PAGE_SIZE - 1)), + full + off, chunk)) { + rc = -EIO; + break; + } + pin_va[npages] = va_page; + pin_pa[npages] = pa; + npages++; + off += chunk; + } + + if (!rc && svc_stream_full_ok(full, entry_len) && + svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload, + full, entry_len)) { + u32 i; + + for (i = 0; i < npages; i++) { + svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); + svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); + svc_mark_hot(pin_pa[i]); + } + svc.resc_run++; + pr_info("sema-svc: content rescue run ch=0x%x rank=%u entry_va=0x%llx len=%u want=%u dir=%d anchor=0x%llx pages=%u (pinned from content)\n", + ch->h_object, ch->ce_index, entry_va, entry_len, + ch->expected_sem_payload, dir, anchor_pa, npages); + rc = 0; + } else if (!rc) { + rc = -ENOENT; + } + kfree(full); + return rc; +} + +/* * Scan a guest-RAM range descending for a page that validates as this entry's * push (method stream + expected release payload), pinning it and returning 0 * on the first hit. `hi` and `lo` are inclusive chunk start addresses. @@ -7955,6 +8068,21 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, &ch->content_max); total += ncand; for (n = 0; n < ncand; n++) { + /* + * Run first: when the entry spans pages with + * no translations yet, only the multi-page + * assembly can validate the candidate (see + * svc_rescue_run_try); the single-page try + * below stays as the fallback for entries + * whose other pages are already trusted. + */ + if (!svc_rescue_run_try(vgpu, ch, entry_va, + entry_len, cand[n], + cand_va[n], 1) || + !svc_rescue_run_try(vgpu, ch, entry_va, + entry_len, cand[n], + cand_va[n], -1)) + return 0; if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], n, ncand)) @@ -8020,6 +8148,22 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, entry_va < svc_pb[i].va) continue; cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va); + /* + * The window base is guest-truth only for its first page; a + * multi-page entry may continue in either physical direction + * (observed both). Try the whole run first: it pins every page + * from content and needs no pre-existing translations, which + * the single-page try below does. + */ + if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len, + cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1), + entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), + 1) || + !svc_rescue_run_try(vgpu, ch, entry_va, entry_len, + cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1), + entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), + -1)) + return 0; if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand_pa, entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), i, svc_pb_n)) @@ -8055,7 +8199,13 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, cand_va, 0, &ch->content_max); for (n = 0; n < ncand; n++) { - if (!svc_rescue_try(vgpu, ch, entry_va, + if (!svc_rescue_run_try(vgpu, ch, entry_va, + entry_len, cand[n], + cand_va[n], 1) || + !svc_rescue_run_try(vgpu, ch, entry_va, + entry_len, cand[n], + cand_va[n], -1) || + !svc_rescue_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], n, ncand)) { kfree(buf); |