summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_sema_service.c123
1 files changed, 71 insertions, 52 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index e6f4a62..7681ef6 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -140,6 +140,7 @@ struct svc_channel {
u64 submissions; /* GPFIFO entries executed */
u32 expected_sem_payload; /* next semaphore payload we expect for this channel's tracking sem */
u32 logged_expected; /* last pool value logged (rate limit) */
+ unsigned long rescue_last; /* rate limit for the full-RAM rescue scan */
};
/*
@@ -2395,6 +2396,7 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
*/
#define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */
#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */
+#define SVC_MBUF_MAX 6 /* rescue pin candidates */
/*
* Rescan the pages that have carried release groups before. Pushbuffer pools
@@ -3100,90 +3102,107 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
u64 gpa;
u32 slot = 4 * ch->ce_index;
u32 want_payload = ch->expected_sem_payload;
- int found = 0;
- u64 hit_pa = 0;
+ u64 cand[SVC_MBUF_MAX];
+ u32 ncand = 0, i;
if (!ch->engine_type || ch->ce_index >= 32 || !want_payload)
return -ENOENT;
+ if ((entry_va & 0xfffff) || entry_va < SEMA_VA_LO ||
+ entry_va >= SEMA_VA_HI)
+ return -EINVAL;
+ if (time_before(jiffies, ch->rescue_last + 5 * HZ))
+ return -EAGAIN;
+ ch->rescue_last = jiffies;
buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
if (!buf)
return -ENOMEM;
/*
- * Scan guest RAM for a page that starts with the known 40-byte release
- * push shape:
- * +0x00 SET_OBJECT (class bind)
- * +0x08 SET_OBJECT (class bind, address 0)
- * +0x10 SET_SEMAPHORE_A run of 3 (hdr, hi==1, lo=slot)
- * +0x1c payload == this channel's next expected value
- * +0x20 LAUNCH_DMA with semaphore-release transfer type
- * The old matcher checked only the release group and matched GSP
- * firmware pages; requiring the full push shape around it does not.
+ * Scan guest RAM for the exact release group at any 4-byte offset:
+ * +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4)
+ * +0x04 hi == 1
+ * +0x08 lo & 0xfff == this channel's pool slot
+ * +0x0c payload (any small value; the tracked expectation can lag)
+ * +0x10 LAUNCH_DMA semaphore-release follower (type 0)
+ * The header is checked first so the inner loop is one compare for
+ * almost every offset; the remaining exact fields make false
+ * positives (the firmware-page matches of the old matcher)
+ * negligible.
+ *
+ * Each match yields two candidate pin bases: the page the group sits
+ * in (correct when the matched group is in the slot's first page,
+ * e.g. a stale release push) and the page implied by treating the
+ * group as the end of THIS entry's push (correct when the push starts
+ * at the slot base and is longer than one page). Each candidate is
+ * validated by pinning and reading the entry VA back; the first that
+ * parses as a push wins.
*/
- for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+ for (gpa = 0; gpa < SEMA_PA_MAX && ncand < SVC_MBUF_MAX; gpa += SVC_SCAN_CHUNK) {
u32 o;
if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
continue;
- for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
- u32 w0, w2, hdr, hi, lo, payload, w5, v5;
-
- memcpy(&w0, buf + o, 4);
- memcpy(&w2, buf + o + 8, 4);
- memcpy(&hdr, buf + o + 0x10, 4);
- memcpy(&hi, buf + o + 0x14, 4);
- memcpy(&lo, buf + o + 0x18, 4);
- memcpy(&payload, buf + o + 0x1c, 4);
- memcpy(&w5, buf + o + 0x20, 4);
- memcpy(&v5, buf + o + 0x24, 4);
- if (w0 != 0x20010000u)
- continue;
- if ((w2 >> 29) != 1 || ((w2 >> 16) & 0xfff) != 1 ||
- ((w2 & 0x1fff) << 2) != 0)
- continue;
+ for (o = 0; o + 24 <= SVC_SCAN_CHUNK && ncand < SVC_MBUF_MAX; o += 4) {
+ u32 hdr, hi, lo, payload, w5, v5;
+ u64 rel_pa, s;
+
+ memcpy(&hdr, buf + o, 4);
if (hdr != 0x20038090u)
continue;
+ memcpy(&hi, buf + o + 4, 4);
if (hi != 1)
continue;
+ memcpy(&lo, buf + o + 8, 4);
if ((lo & 0xfff) != slot)
continue;
- if (payload != want_payload)
+ memcpy(&payload, buf + o + 12, 4);
+ if (!payload || payload >= (1u << 24))
continue;
+ memcpy(&w5, buf + o + 16, 4);
+ memcpy(&v5, buf + o + 20, 4);
if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 ||
((w5 & 0x1fff) << 2) != 0x300 ||
(v5 & 0x3) != 0)
continue;
- hit_pa = (gpa + o) & ~(SEMA_PAGE_SIZE - 1);
- found = 1;
- break;
+ rel_pa = gpa + o;
+ cand[ncand++] = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+ s = (rel_pa - (entry_len >= 24 ? entry_len - 24 : 0)) &
+ ~(u64)(SEMA_PAGE_SIZE - 1);
+ if (ncand < SVC_MBUF_MAX &&
+ (ncand == 0 || cand[ncand - 1] != s))
+ cand[ncand++] = s;
}
- if (found)
- break;
}
kfree(buf);
- if (!found)
- return -ENOENT;
-
- /*
- * Pin the entire 1 MB push-buffer slot (16 pages) at the entry VA.
- * The entry_va is 1 MB aligned (push-buffer pool VAs are 1 MB aligned).
- */
- if ((entry_va & 0xfffff) || entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI)
- return -EINVAL;
-
- for (u32 off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
- svc_pin_store(entry_va + off, hit_pa + off, SVC_APERTURE_SYS);
- svc_lat_store(entry_va + off, hit_pa + off, SVC_APERTURE_SYS);
- svc_mark_hot(hit_pa + off);
+ for (i = 0; i < ncand; i++) {
+ u8 probe[64];
+ u32 j;
+
+ for (j = 0; j < 0x100000; j += SEMA_PAGE_SIZE) {
+ svc_pin_store(entry_va + j, cand[i] + j,
+ SVC_APERTURE_SYS);
+ svc_lat_store(entry_va + j, cand[i] + j,
+ SVC_APERTURE_SYS);
+ }
+ if (!svc_read_va_trusted(vgpu, entry_va, probe,
+ sizeof(probe)) &&
+ svc_method_stream_plausible(probe, sizeof(probe))) {
+ for (j = 0; j < 0x100000; j += SEMA_PAGE_SIZE)
+ svc_mark_hot(cand[i] + j);
+ pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (cand %u/%u)\n",
+ ch->h_object, ch->ce_index, slot, entry_va,
+ entry_len, cand[i], i + 1, ncand);
+ return 0;
+ }
}
- pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (payload=%u)\n",
- ch->h_object, ch->ce_index, slot, entry_va, entry_len, hit_pa, want_payload);
-
- return 0;
+ if (ncand)
+ pr_info("sema-svc: content rescue ch=0x%x entry_va=0x%llx len=%u: %u candidate(s) did not validate\n",
+ ch->h_object, entry_va, entry_len, ncand);
+ return -ENOENT;
}
/*
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 14:18:08 +0000