summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c224
-rw-r--r--kernel/gsp_sema_service.c241
-rw-r--r--kernel/gsp_sema_service.h10
3 files changed, 460 insertions, 15 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index e7879cc..b20e43e 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -5216,6 +5216,9 @@ static bool gsp_phys_rmcall_gr_mme_iram_probe;
* launch thread's time and stops early once the poison is seen).
*/
static u32 gsp_phys_mme_fast_ms = 400u;
+/* r1497: defined with the emission knob below; used by the post-launch
+ * DS ESR clear. */
+static bool gsp_phys_rmcall_gr_ds_mask_clear;
module_param_named(sema_mme_fast_ms, gsp_phys_mme_fast_ms, uint, 0644);
MODULE_PARM_DESC(sema_mme_fast_ms,
"Spin-sample the MME fault registers right after each GR launch doorbell for this many ms, logging every transition before the GSP-RM clears them (0 = off; default 400)");
@@ -5618,6 +5621,28 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n",
tag, h_object, n, poisoned ? " (poison)" : "");
/*
+ * r1497: clear the DS HWW ESR latch (0x405840, W1C) after the
+ * launch as well as before it. The pre-clear makes each window's
+ * validation set readable; a draw that latches a6040e24 otherwise
+ * leaves the pending report to poison the next window's entry state
+ * (r1496: ds a6040e24 read at the following begin while poison was
+ * otherwise low). The clear is W1C and host-only, like the mask
+ * write above, so the A/B knob covers both sites.
+ */
+ if (READ_ONCE(gsp_phys_rmcall_gr_ds_mask_clear) && bar0) {
+ static u32 ds_post_log;
+ u32 esr = ioread32(bar0 + 0x00405840u);
+
+ if (esr) {
+ iowrite32(esr, bar0 + 0x00405840u);
+ if (ds_post_log < 24u) {
+ ds_post_log++;
+ pr_info("vgsp-gr: DS-ESR post-clear=%08x h=0x%x tag=%s\n",
+ esr, h_object, tag);
+ }
+ }
+ }
+ /*
* SESSION 153-CONTINUED-107: post-spin IRAM sample - the state left
* when the poison (if any) has just cleared, before the poll/retire
* work touches anything.
@@ -10729,6 +10754,61 @@ static u64 gsp_phys_rmcall_gr_pte_leaf(u64 hpa)
}
/*
+ * r1497: a VID-aperture leaf pointing at the page's real frame-buffer
+ * offset. The guest's own page tables encode exactly this
+ * (VALID=1, APERTURE=VID(0), KIND=6 GENERIC_MEMORY, PA>>12 in [56:8]),
+ * so the engine reads the real FBMEM page the client uploaded instead of
+ * an emulated BAR1 shadow that no upload ever populated.
+ */
+static u64 gsp_phys_rmcall_gr_pte_leaf_vid(u64 fb_off)
+{
+ return 1ULL | (((fb_off >> 12) & 0x1ffffffULL) << 8) |
+ ((u64)6u << 56);
+}
+
+/*
+ * r1497: is the engine-visible shadow page at `hpa` entirely empty?
+ * Only then may the real-FB VID leaf replace it; pages the module seeded
+ * (fixed CB), the engine wrote, or the client wrote through the emulated
+ * aperture keep their shadow identity.
+ */
+static bool gsp_phys_gr_shadow_page_empty(u64 hpa)
+{
+ const u8 *p;
+ u32 i;
+
+ if (!hpa || !pfn_valid((unsigned long)(hpa >> PAGE_SHIFT)))
+ return false;
+ p = (const u8 *)__va(hpa);
+ for (i = 0; i < GSP_PHYS_PAGE_SIZE; i++)
+ if (p[i])
+ return false;
+ return true;
+}
+
+/*
+ * r1497: resolve a fetch VA to its real frame-buffer offset when the page
+ * is FB-aperture AND the emulated shadow copy of it is all-zero. This is
+ * the active indexed draw's vertex/index class (r1496): the client's page
+ * tables map the page to vidmem, the upload never reached an emulated
+ * path, and the engine fetched zeros. Returns true with *fb_off set.
+ */
+static bool gsp_phys_gr_fetch_vid_off_locked(u64 va, u64 *fb_off)
+{
+ struct gsp_sema_gr_reloc r;
+
+ if (gsp_sema_gr_reloc_resolve(va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1),
+ &r))
+ return false;
+ if (r.ap != 2 /* SVC_APERTURE_FB */ || !r.hpa)
+ return false;
+ if (!gsp_phys_gr_shadow_page_empty(r.hpa))
+ return false;
+ *fb_off = r.pa;
+ return true;
+}
+
+/*
* GP10X leaf PTE for a frame-buffer (VIDEO_MEMORY aperture 0) address:
* VALID=1 (bit 0), APERTURE=0 (bits 2:1), VOL=1 (bit 3), frame-buffer byte
* offset >> 12 in the VIDEO address field (bits 32:8; dev_mmu.h:592-595,
@@ -17759,6 +17839,9 @@ static u64 gsp_phys_gr_timed_out;
static u64 gsp_phys_gr_zerowin_skips;
static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */
static u64 gsp_phys_gr_pool_exhausted;
+/* r1497: launch fetch pages mapped through a real-FB VID leaf (empty shadow). */
+static u64 gsp_phys_gr_vid_map_pages;
+static u32 gsp_phys_gr_vid_log;
/* m156 (E5): zero-write retries where the warm sem push proved the engine
* wedged (the persistent-lottery class); further retries are skipped. */
static u64 gsp_phys_gr_sem_wedged;
@@ -22888,6 +22971,58 @@ static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va,
return 0;
}
+/*
+ * r1497: install a VID-aperture PTE at the guest VA so the engine reads
+ * the real FBMEM page (fb_off) instead of an empty shadow page. Same
+ * never-rewrite discipline; the cache key is tagged so a VID entry can
+ * never be mistaken for a host-PA alias.
+ */
+static int gsp_phys_rmcall_gr_map_va_vid_locked(void __iomem *bar0,
+ u64 guest_va, u64 fb_off,
+ u32 epoch)
+{
+ u64 page = guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+ u32 pde_index = (u32)(page >> 21);
+ u32 pte_idx = (u32)((page >> 12) & 0x1ffu);
+ u32 line = pte_idx / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES;
+ u64 key = (fb_off & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) |
+ (1ULL << 63);
+ struct gsp_phys_boot_chunk *pt;
+ u8 *pt_va;
+ u32 slot, i;
+ int ret;
+
+ ret = gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot);
+ if (ret)
+ return ret;
+ for (i = 0; i < boot_stage.rmcall_gr_win[slot].pte_count; i++) {
+ if (boot_stage.rmcall_gr_win[slot].pte[i].idx == pte_idx) {
+ if (boot_stage.rmcall_gr_win[slot].pte[i].hpa == key)
+ return 0;
+ return -EALREADY;
+ }
+ if (boot_stage.rmcall_gr_win[slot].pte[i].epoch != epoch &&
+ boot_stage.rmcall_gr_win[slot].pte[i].idx /
+ GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES == line)
+ return -EBUSY;
+ }
+ if (boot_stage.rmcall_gr_win[slot].pte_count >=
+ GSP_PHYS_RMCALL_GR_PT_ENTRIES)
+ return -ENOSPC;
+ pt = &boot_stage.rmcall_gr_pt[boot_stage.rmcall_gr_win[slot].pt_slot];
+ pt_va = gsp_phys_chunk_page_va(pt, 0);
+ if (!pt_va)
+ return -EIO;
+ put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf_vid(fb_off),
+ pt_va + (size_t)pte_idx * 8);
+ dma_wmb();
+ boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx;
+ boot_stage.rmcall_gr_win[slot].pte[i].epoch = epoch;
+ boot_stage.rmcall_gr_win[slot].pte[i].hpa = key;
+ boot_stage.rmcall_gr_win[slot].pte_count++;
+ return 0;
+}
+
static void __iomem *gsp_phys_gr_scratch_bar0;
/*
@@ -25645,6 +25780,14 @@ static u64 gsp_phys_rmcall_gr_stream_page_hpa_locked(
{
u32 i;
+ /*
+ * r1497: a vertex/index fetch page can resolve to an FB page whose
+ * emulated shadow is empty while the same VA has a sysmem mapping
+ * with the real bytes (r1496: the active indexed draw's 0x44fea00
+ * stream read all-zero). Populate the empty shadow page before the
+ * stream line is built. Non-empty pages and SYS answers are no-ops.
+ */
+ gsp_sema_gr_fetch_backing(page);
for (i = 0; i < req->n_pages; i++)
if ((req->pages[i].guest_va &
~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page &&
@@ -25713,8 +25856,11 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked(
u64 base_page = base_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
u64 end_page = (end_va - 1u) & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
u64 hpa[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES];
+ u64 pte[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES];
+ u64 fb_off[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES];
+ bool real[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES];
u64 va0, dummy = 0;
- u32 npages, i, line, page, line_in_page, dummy_n = 0;
+ u32 npages, i, line, page, line_in_page, dummy_n = 0, vid_n = 0;
u8 *pt;
int ret;
@@ -25724,8 +25870,24 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked(
if (npages > GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
return 0;
for (i = 0; i < npages; i++) {
- hpa[i] = gsp_phys_rmcall_gr_stream_page_hpa_locked(req,
- base_page + ((u64)i << 12));
+ u64 pa = base_page + ((u64)i << 12);
+
+ real[i] = false;
+ fb_off[i] = 0;
+ /*
+ * r1497: the active draw's vertex/index page resolves to a
+ * vidmem page whose emulated shadow is empty. Point the
+ * fetch line at the real FBMEM page (the guest's own page
+ * table names the offset) instead of delivering zeros.
+ */
+ if (gsp_phys_gr_fetch_vid_off_locked(pa, &fb_off[i])) {
+ pte[i] = gsp_phys_rmcall_gr_pte_leaf_vid(fb_off[i]);
+ hpa[i] = 0;
+ real[i] = true;
+ vid_n++;
+ continue;
+ }
+ hpa[i] = gsp_phys_rmcall_gr_stream_page_hpa_locked(req, pa);
if (!hpa[i]) {
if (!dummy)
dummy = gsp_phys_gr_surf_dummy_dma_locked();
@@ -25734,6 +25896,7 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked(
hpa[i] = dummy;
dummy_n++;
}
+ pte[i] = gsp_phys_rmcall_gr_pte_leaf(hpa[i]);
}
if (!sh->pages) {
ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
@@ -25756,8 +25919,7 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked(
if (!pt)
return 0;
for (i = 0; i < npages; i++)
- put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa[i]),
- pt + (size_t)(
+ put_unaligned_le64(pte[i], pt + (size_t)(
GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
line_in_page + i) * 8);
dma_wmb();
@@ -25765,14 +25927,16 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked(
va0 = gsp_phys_rmcall_guest_pt_win_va(page) +
((u64)line_in_page << 16);
for (i = 0; i < npages; i++)
- gsp_phys_rmcall_gr_alias_cache_set_locked(hpa[i],
- va0 + ((u64)i << 12));
+ if (!real[i])
+ gsp_phys_rmcall_gr_alias_cache_set_locked(hpa[i],
+ va0 + ((u64)i << 12));
{
static u32 stream_line_log;
u32 w[8] = { 0 }, k;
u64 hpa0 = hpa[0];
- if (pfn_valid((unsigned long)(hpa0 >> PAGE_SHIFT))) {
+ if (!real[0] &&
+ pfn_valid((unsigned long)(hpa0 >> PAGE_SHIFT))) {
const u8 *p = (const u8 *)__va(
hpa0 + (base_va & (GSP_PHYS_PAGE_SIZE - 1)));
@@ -25781,10 +25945,11 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked(
}
if (stream_line_log < 32u) {
stream_line_log++;
- pr_info("vgsp-gr: MME-VSTREAM-LINE h=0x%x base=0x%llx end=0x%llx pages=%u dummy=%u line=0x%llx hpa0=0x%llx first8=%08x %08x %08x %08x %08x %08x %08x %08x\n",
+ pr_info("vgsp-gr: MME-VSTREAM-LINE h=0x%x base=0x%llx end=0x%llx pages=%u dummy=%u vid=%u fb0=0x%llx line=0x%llx hpa0=0x%llx first8=%08x %08x %08x %08x %08x %08x %08x %08x\n",
req ? req->h_object : 0u, base_va, end_va,
- npages, dummy_n, va0, hpa0, w[0], w[1], w[2],
- w[3], w[4], w[5], w[6], w[7]);
+ npages, dummy_n, vid_n, fb_off[0], va0, hpa0,
+ w[0], w[1], w[2], w[3], w[4], w[5], w[6],
+ w[7]);
}
}
return va0;
@@ -28175,6 +28340,41 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
pr_info("vgsp-gr: page[%u] va=0x%llx hpa=0x%llx qmd_word=%u\n",
i, pg->guest_va, pg->hpa, pg->qmd_word);
/*
+ * r1497/r1498: an FB *fetch* page (a QMD pointer pair, not
+ * the data-fill surface) whose shadow is empty is the active
+ * indexed draw's vertex/index class. Map the real FBMEM page
+ * through a VID leaf instead of the empty shadow. The
+ * render-target/surface pages (QMD word NONE) must keep
+ * their shadow identity: the engine's stores have to land in
+ * the emulated page so the copy-out can ship them back to
+ * the guest (r1498: mapping the surface to real FB made the
+ * frame invisible to the guest).
+ */
+ if (pg->hpa && pg->qmd_word != 48u &&
+ pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE) {
+ u64 fb_off = 0;
+
+ if (gsp_phys_gr_fetch_vid_off_locked(pg_va,
+ &fb_off)) {
+ ret = gsp_phys_rmcall_gr_map_va_vid_locked(
+ bar0, pg_va, fb_off, epoch);
+ if (!ret) {
+ mapped = true;
+ gsp_phys_gr_vid_map_pages++;
+ if (gsp_phys_gr_vid_log < 32u) {
+ gsp_phys_gr_vid_log++;
+ pr_info("vgsp-gr: FETCH-VID-MAP va=0x%llx fb=0x%llx qmd=%u h=0x%x push=0x%llx\n",
+ pg_va, fb_off,
+ pg->qmd_word,
+ req->h_object,
+ req->push_va);
+ }
+ } else if (ret == -ENOSPC) {
+ gsp_phys_gr_pool_exhausted++;
+ }
+ }
+ }
+ /*
* SESSION 149 R15 (M2-S1): with the window-0 shadow path
* enabled, force the program page (QMD word 48) OFF the
* post-commit Tier-1 install. r801b proved that install is
@@ -28184,7 +28384,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
* the win0 shadow copy + w48/49 rewrite line, which is the
* proven one. Every other page keeps its Tier-1 route.
*/
- if (pg->hpa && !(pg->qmd_word == 48u &&
+ if (!mapped && pg->hpa && !(pg->qmd_word == 48u &&
gsp_phys_rmcall_gr_shadow_win0_value())) {
ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va,
pg->hpa, epoch);
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index dd02aa0..8668b6f 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1399,6 +1399,8 @@ struct sema_svc {
u64 fwd_copy_skipped;
u64 fwd_copy_soft;
u64 fwd_copy_fail;
+ /* r1497: empty FB fetch pages populated from their sysmem backing. */
+ u64 fwd_fetch_backing;
u64 fwd_rel_deferred;
u64 fwd_rel_applied;
u64 fwd_rel_fallback;
@@ -15453,13 +15455,33 @@ static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object)
if (sema_gl_cb_src_va) {
src_va = sema_gl_cb_src_va;
val = (u32)(src_va >> 8);
- } else {
- if (!svc_scratch18_seen)
- return;
+ } else if (svc_scratch18_seen) {
+ /*
+ * r1497: per-boot discovery. The client's own
+ * SET_MME_SHADOW_SCRATCH(0x18) write (method 0x3460) is the
+ * authority for the fixed-CB base; svc_scratch18_watch()
+ * captures it from whichever decoded push carries it. Use
+ * it instead of a hardcoded VA so a boot whose mapping moved
+ * still reads the real matrix.
+ */
val = READ_ONCE(svc_scratch18_val);
if (!val)
return;
src_va = (u64)val << 8;
+ } else {
+ /*
+ * The 0x3460 write did not reach a decoded push on this
+ * boot (r1496: 0 captures). Fall back to the fixed-CB VA
+ * observed on the boots where the capture did fire
+ * (0x0cd70000) and let the matrix gate decide whether the
+ * page is the real CB. This keeps a real matrix instead of
+ * the synthetic identity whenever the page is present under
+ * the historical VA; a failed gate still falls through to
+ * cached-real / synthetic-id below.
+ */
+ src_va = 0x0cd70000ULL;
+ val = (u32)(src_va >> 8);
+ src = "captured-miss";
}
/*
* r1491: the fixed constant buffer is ENGINE-written (the driver
@@ -31678,6 +31700,9 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req)
*/
static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
u64 *pa_out, u32 *ap_out);
+/* r1497: deliver the sysmem backing of an empty FB fetch page. */
+bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out);
+bool gsp_sema_gr_fetch_backing(u64 va);
static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
@@ -31845,6 +31870,29 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
src = "walk";
note_done:
/*
+ * r1497: an FB launch page whose emulated BAR1 shadow is empty is a
+ * page the engine will fetch as zeros (r1496 wall). If the same VA
+ * also has a sysmem mapping with real bytes, deliver them into the
+ * shadow page now, before the launch maps/copies it. The fb_pa/ap
+ * stay unchanged, so every downstream identity (QMD relocation,
+ * copy-out, ledger) is untouched.
+ */
+ if (svc_fwd_req.gr_pages[i].ap == SVC_APERTURE_FB &&
+ qmd_word != SVC_GR_QMD_WORD_NONE) {
+ u64 gpa = 0;
+
+ if (gsp_sema_gr_fetch_sys_into(page,
+ svc_fwd_req.gr_pages[i].pa,
+ &gpa)) {
+ svc.fwd_fetch_backing++;
+ if (svc.fwd_fetch_backing <= 32)
+ pr_info("sema-gr: FETCH-BACKING note va=0x%llx fb_pa=0x%llx gpa=0x%llx qmd=%u n=%llu\n",
+ page, svc_fwd_req.gr_pages[i].pa,
+ gpa, qmd_word,
+ svc.fwd_fetch_backing);
+ }
+ }
+ /*
* r372 diagnostic: the exact (pa, ap) and branch the launch page
* resolved through - the value the engine's hpa and the copy-out use.
* A D2H readback that lands elsewhere (the sum=0 class) names its
@@ -32065,6 +32113,193 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
}
/*
+ * r1497: fetch-page sysmem backing.
+ *
+ * THE WALL (r1496): the active indexed draw's vertex/index buffer resolves
+ * through the client's own VAS to a vidmem page (FB aperture, low offset)
+ * but the emulated BAR1 shadow at that offset is entirely zero - the upload
+ * never passed through an emulated path (no inline write, no CE copy the
+ * host decoded), so only the real FBMEM has the bytes. The client's object
+ * is nevertheless also mapped in the channel's page tables with a SYS
+ * answer (the CPU/staging copy): walking every registered VAS for the same
+ * VA finds a guest-RAM GPA whose page holds the real vertex/index data
+ * (r1497 live proof: VA 0x44ff000/0x44fea00 -> GPA 0x124c89000/0x12c77e00,
+ * first8 of the latter 00000000 3f800000 32f0b842 c0166666 ...).
+ *
+ * When the FB shadow page is all-zero, copy that SYS page into the shadow
+ * page the engine already reads. The launch map, the MME stream line and
+ * the copy-out keep their existing identities; only the empty page's
+ * content changes. A shadow page with any byte set is never touched, so
+ * every working stream and the CB seed keep their current behaviour.
+ *
+ * The helper runs in process context (service thread or the GR work item);
+ * the walks may sleep in vfio_dma_rw exactly like svc_walk_vaspaces.
+ */
+static bool svc_page_nonzero(const u8 *p, size_t n)
+{
+ size_t i;
+
+ for (i = 0; i + 8u <= n; i += 8u) {
+ u64 v;
+
+ memcpy(&v, p + i, 8);
+ if (v)
+ return true;
+ }
+ for (; i < n; i++)
+ if (p[i])
+ return true;
+ return false;
+}
+
+static int svc_gr_sys_backing_walk(struct nv_vgpu_device *vgpu, u64 va,
+ u64 *gpa_out)
+{
+ struct svc_vaspace local[SVC_MAX_VASPACES];
+ unsigned long flags;
+ unsigned long stamps[SVC_MAX_VASPACES];
+ u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ u8 tmp[32];
+ int i, n = 0;
+
+ /*
+ * The channel currently being serviced is the most specific answer;
+ * it is the VAS that produced the FB mapping in the first place, so
+ * ask it first for the alternate (SYS) encoding of the same VA.
+ */
+ {
+ u64 pa = 0;
+ int ap = SVC_APERTURE_SYS;
+
+ if (!svc_chanvas_lookup(vgpu, page, &pa, &ap) &&
+ ap == SVC_APERTURE_SYS &&
+ !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, tmp,
+ sizeof(tmp)) &&
+ svc_page_nonzero(tmp, sizeof(tmp))) {
+ *gpa_out = pa;
+ return 0;
+ }
+ }
+
+ spin_lock_irqsave(&svc.vas_lock, flags);
+ for (i = 0; i < SVC_MAX_VASPACES; i++) {
+ if (!svc.vas[i].valid)
+ continue;
+ local[n] = svc.vas[i];
+ stamps[n] = local[n].stamp;
+ n++;
+ }
+ spin_unlock_irqrestore(&svc.vas_lock, flags);
+
+ /* Newest registration first, same authority order as svc_walk_vaspaces. */
+ for (i = 1; i < n; i++) {
+ struct svc_vaspace k = local[i];
+ unsigned long ks = stamps[i];
+ int j = i - 1;
+
+ while (j >= 0 && time_before(stamps[j], ks)) {
+ local[j + 1] = local[j];
+ stamps[j + 1] = stamps[j];
+ j--;
+ }
+ local[j + 1] = k;
+ stamps[j + 1] = ks;
+ }
+
+ for (i = 0; i < n; i++) {
+ int alt = local[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS
+ : SVC_APERTURE_FB;
+ int taps[2];
+ int t;
+
+ taps[0] = local[i].ap;
+ taps[1] = alt;
+ for (t = 0; t < 2; t++) {
+ u64 pa = 0;
+ int ap = SVC_APERTURE_SYS;
+
+ if (svc_gmmu_walk(vgpu, local[i].pdb,
+ local[i].entries, page, taps[t],
+ &pa, &ap, false))
+ continue;
+ if (ap != SVC_APERTURE_SYS)
+ continue;
+ if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, tmp,
+ sizeof(tmp)))
+ continue;
+ if (!svc_page_nonzero(tmp, sizeof(tmp)))
+ continue;
+ *gpa_out = pa;
+ return 0;
+ }
+ }
+ return -ENOENT;
+}
+
+bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out)
+{
+ struct nv_vgpu_device *vgpu = svc_fwd_req.vgpu;
+ static u32 back_log;
+ u64 gpa = 0;
+ u8 *base, *dst;
+ u32 wfb[8] = { 0 }, wsys[8] = { 0 };
+ u32 k;
+
+ if (!vgpu || !gpa_out)
+ return false;
+ base = vgpu->bars[1].vaddr;
+ if (!base || fb_pa + SEMA_PAGE_SIZE > svc_fb_shadow_size(vgpu))
+ return false;
+ dst = base + fb_pa;
+ if (svc_page_nonzero(dst, SEMA_PAGE_SIZE))
+ return false;
+ if (svc_gr_sys_backing_walk(vgpu, va, &gpa))
+ return false;
+ /*
+ * Take the first 8 words of the SYS candidate before the copy so the
+ * receipt shows both sides of the delivery even when the copy itself
+ * is what the engine later reads.
+ */
+ if (!svc_read_ap(vgpu, gpa, SVC_APERTURE_SYS, wsys, sizeof(wsys)))
+ ;
+ for (k = 0; k < 8u; k++)
+ memcpy(&wfb[k], dst + (size_t)k * 4u, 4);
+ if (svc_read_ap(vgpu, gpa, SVC_APERTURE_SYS, dst, SEMA_PAGE_SIZE))
+ return false;
+ *gpa_out = gpa;
+ if (back_log < 64u) {
+ back_log++;
+ pr_info("vgsp-sema: FETCH-SYSBACK va=0x%llx fb_pa=0x%llx gpa=0x%llx pages=1 first8_fb=%08x %08x %08x %08x %08x %08x %08x %08x first8_sys=%08x %08x %08x %08x %08x %08x %08x %08x\n",
+ va, fb_pa, gpa,
+ wfb[0], wfb[1], wfb[2], wfb[3], wfb[4], wfb[5],
+ wfb[6], wfb[7], wsys[0], wsys[1], wsys[2], wsys[3],
+ wsys[4], wsys[5], wsys[6], wsys[7]);
+ }
+ return true;
+}
+
+/*
+ * Resolver wrapper for callers that only have the VA (the booter's MME
+ * stream-line builder). Resolve the VA the same way the relocation chain
+ * does; only an FB answer whose shadow page is empty is a candidate.
+ */
+bool gsp_sema_gr_fetch_backing(u64 va)
+{
+ struct nv_vgpu_device *vgpu = svc_fwd_req.vgpu;
+ u64 pa = 0, gpa = 0;
+ u32 ap = 0;
+
+ if (!vgpu)
+ return false;
+ if (svc_gr_src_probe(vgpu, va & ~(u64)(SEMA_PAGE_SIZE - 1), &pa, &ap))
+ return false;
+ if (ap != SVC_APERTURE_FB)
+ return false;
+ return gsp_sema_gr_fetch_sys_into(va, pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+ &gpa);
+}
+
+/*
* relfix PART 3: append one attempt to the resolver's diagnostic chain
* ("name:rc,"). Bounded; the tail is simply dropped once full.
*/
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index f63b9fa..ff3d088 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -474,6 +474,16 @@ struct gsp_sema_gr_reloc {
};
int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out);
+/*
+ * r1497: when a fetch VA resolves to an FB page whose emulated shadow is
+ * empty, copy the same VA's sysmem backing (found by walking every
+ * registered VAS) into that page. gsp_sema_gr_fetch_sys_into() takes the
+ * already-resolved FB PA; gsp_sema_gr_fetch_backing() resolves the VA the
+ * way the relocation chain does and is the booter's entry point. Both
+ * return true only when bytes were delivered.
+ */
+bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out);
+bool gsp_sema_gr_fetch_backing(u64 va);
/*
* SESSION 153-CONTINUED-59: record one VA (page-aligned internally) in the
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-05 23:12:08 +0000