summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c152
1 files changed, 151 insertions, 1 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 8d223d8..e69c46c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -529,6 +529,8 @@ struct sema_svc {
u64 gpfifo_skipped; /* stale entries consumed */
u64 gpfifo_rescued; /* unreadable push made readable
* by the content rescue */
+ u64 resc_run; /* pushes pinned by the multi-page
+ * content rescue run */
u64 resc_resync; /* expectation re-synced down to
* the content watermark */
u64 releases_seen;
@@ -2916,6 +2918,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n",
svc.counter_pub_calls, svc.gr_rel_applied,
svc.sem_write_reresolve, svc.pushpg_fallback);
+ if (svc.resc_run)
+ pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n",
+ svc.resc_run);
pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n",
svc.gpfifo_entries_done, svc.gpfifo_nomap,
svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
@@ -7927,6 +7932,114 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
}
/*
+ * Multi-page content rescue run (ISSUE-0001 durable direction).
+ *
+ * svc_rescue_try() substitutes exactly one candidate page and requires every
+ * other page of the entry to resolve through svc_lookup_trusted(). For a
+ * multi-page push whose pages have no translations yet that precondition can
+ * never hold: the stalled-boot class reads the CE entry at va=0x120021db8
+ * (len=12816, four pages, release #39 at the last page) through a window
+ * guess that is wrong for the entry page while the three continuation pages
+ * are missing from every trusted source; the content scan finds the
+ * release-group page (the matcher's candidate) but every single-substitution
+ * candidate is rejected because the untranslated pages are exactly the ones
+ * that need a candidate. The userspace pin stream was the only supplier of
+ * those translations (SA-STALL 3.2/3.4; 17-host-m84-r1.txt:841-842).
+ *
+ * A pushbuffer allocation is physically contiguous, so once content has
+ * anchored ONE page of the entry, the remaining pages sit at the same signed
+ * page delta from the anchor in one of the two directions. Observed both
+ * directions in the evidence: rank-2's consecutive release pages descend by
+ * 0x1000 per VA page ascending (17-host-m84-r1.txt:3-13: 0x1232b8000 ->
+ * 0x1232b7000 -> ... -> 0x1232b0000), i.e. dir=-1; other boots pair the
+ * window base forward (svc_pb_pair's convention), i.e. dir=+1. Try both:
+ * dir=+1: VA page anchor_va + k*PAGE -> PA anchor_pa + k*PAGE
+ * dir=-1: VA page anchor_va + k*PAGE -> PA anchor_pa - k*PAGE
+ * Read every page of the entry directly from the candidate PAs, validate the
+ * assembled buffer exactly as svc_rescue_try() does (full method-stream parse
+ * + this channel's exact next release), and only then pin every page of the
+ * run. A failed direction writes nothing, so a wrong anchor or direction
+ * cannot poison the pin table; a passed validation is content proof for
+ * every page, strictly stronger than the watcher pin it replaces.
+ *
+ * Bounded: entry_len <= SVC_PUSH_MAX, at most SVC_RESCUE_RUN_MAX pages
+ * (128 KB pushes fit), one direct read per page. Service-thread only.
+ */
+#define SVC_RESCUE_RUN_MAX 32 /* pages one rescue run may pin */
+
+static int svc_rescue_run_try(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch, u64 entry_va,
+ u32 entry_len, u64 anchor_pa,
+ u64 anchor_va_page, int dir)
+{
+ u64 pin_va[SVC_RESCUE_RUN_MAX];
+ u64 pin_pa[SVC_RESCUE_RUN_MAX];
+ u8 *full;
+ u32 npages = 0, off = 0;
+ int rc = 0;
+
+ if (!entry_len || (dir != 1 && dir != -1))
+ return -EINVAL;
+
+ full = kmalloc(entry_len, GFP_KERNEL);
+ if (!full)
+ return -ENOMEM;
+
+ while (off < entry_len) {
+ u64 va = entry_va + off;
+ u64 va_page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ size_t chunk = min_t(size_t, entry_len - off,
+ SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
+ s64 delta = (s64)(va_page - anchor_va_page) * dir;
+ u64 pa;
+
+ if (npages >= SVC_RESCUE_RUN_MAX) {
+ rc = -E2BIG;
+ break;
+ }
+ if (delta < 0 && (u64)(-delta) > anchor_pa) {
+ rc = -EINVAL;
+ break;
+ }
+ pa = (u64)((s64)anchor_pa + delta);
+ if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) {
+ rc = -EINVAL;
+ break;
+ }
+ if (svc_read(vgpu, pa | (va & (SEMA_PAGE_SIZE - 1)),
+ full + off, chunk)) {
+ rc = -EIO;
+ break;
+ }
+ pin_va[npages] = va_page;
+ pin_pa[npages] = pa;
+ npages++;
+ off += chunk;
+ }
+
+ if (!rc && svc_stream_full_ok(full, entry_len) &&
+ svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload,
+ full, entry_len)) {
+ u32 i;
+
+ for (i = 0; i < npages; i++) {
+ svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS);
+ svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS);
+ svc_mark_hot(pin_pa[i]);
+ }
+ svc.resc_run++;
+ pr_info("sema-svc: content rescue run ch=0x%x rank=%u entry_va=0x%llx len=%u want=%u dir=%d anchor=0x%llx pages=%u (pinned from content)\n",
+ ch->h_object, ch->ce_index, entry_va, entry_len,
+ ch->expected_sem_payload, dir, anchor_pa, npages);
+ rc = 0;
+ } else if (!rc) {
+ rc = -ENOENT;
+ }
+ kfree(full);
+ return rc;
+}
+
+/*
* Scan a guest-RAM range descending for a page that validates as this entry's
* push (method stream + expected release payload), pinning it and returning 0
* on the first hit. `hi` and `lo` are inclusive chunk start addresses.
@@ -7955,6 +8068,21 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu,
&ch->content_max);
total += ncand;
for (n = 0; n < ncand; n++) {
+ /*
+ * Run first: when the entry spans pages with
+ * no translations yet, only the multi-page
+ * assembly can validate the candidate (see
+ * svc_rescue_run_try); the single-page try
+ * below stays as the fallback for entries
+ * whose other pages are already trusted.
+ */
+ if (!svc_rescue_run_try(vgpu, ch, entry_va,
+ entry_len, cand[n],
+ cand_va[n], 1) ||
+ !svc_rescue_run_try(vgpu, ch, entry_va,
+ entry_len, cand[n],
+ cand_va[n], -1))
+ return 0;
if (!svc_rescue_try(vgpu, ch, entry_va,
entry_len, cand[n],
cand_va[n], n, ncand))
@@ -8020,6 +8148,22 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
entry_va < svc_pb[i].va)
continue;
cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va);
+ /*
+ * The window base is guest-truth only for its first page; a
+ * multi-page entry may continue in either physical direction
+ * (observed both). Try the whole run first: it pins every page
+ * from content and needs no pre-existing translations, which
+ * the single-page try below does.
+ */
+ if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len,
+ cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+ entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+ 1) ||
+ !svc_rescue_run_try(vgpu, ch, entry_va, entry_len,
+ cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+ entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+ -1))
+ return 0;
if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand_pa,
entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
i, svc_pb_n))
@@ -8055,7 +8199,13 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
cand_va, 0,
&ch->content_max);
for (n = 0; n < ncand; n++) {
- if (!svc_rescue_try(vgpu, ch, entry_va,
+ if (!svc_rescue_run_try(vgpu, ch, entry_va,
+ entry_len, cand[n],
+ cand_va[n], 1) ||
+ !svc_rescue_run_try(vgpu, ch, entry_va,
+ entry_len, cand[n],
+ cand_va[n], -1) ||
+ !svc_rescue_try(vgpu, ch, entry_va,
entry_len, cand[n],
cand_va[n], n, ncand)) {
kfree(buf);
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 08:16:59 +0000