diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 224 |
1 files changed, 212 insertions, 12 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index e7879cc..b20e43e 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -5216,6 +5216,9 @@ static bool gsp_phys_rmcall_gr_mme_iram_probe; * launch thread's time and stops early once the poison is seen). */ static u32 gsp_phys_mme_fast_ms = 400u; +/* r1497: defined with the emission knob below; used by the post-launch + * DS ESR clear. */ +static bool gsp_phys_rmcall_gr_ds_mask_clear; module_param_named(sema_mme_fast_ms, gsp_phys_mme_fast_ms, uint, 0644); MODULE_PARM_DESC(sema_mme_fast_ms, "Spin-sample the MME fault registers right after each GR launch doorbell for this many ms, logging every transition before the GSP-RM clears them (0 = off; default 400)"); @@ -5618,6 +5621,28 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n", tag, h_object, n, poisoned ? " (poison)" : ""); /* + * r1497: clear the DS HWW ESR latch (0x405840, W1C) after the + * launch as well as before it. The pre-clear makes each window's + * validation set readable; a draw that latches a6040e24 otherwise + * leaves the pending report to poison the next window's entry state + * (r1496: ds a6040e24 read at the following begin while poison was + * otherwise low). The clear is W1C and host-only, like the mask + * write above, so the A/B knob covers both sites. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_ds_mask_clear) && bar0) { + static u32 ds_post_log; + u32 esr = ioread32(bar0 + 0x00405840u); + + if (esr) { + iowrite32(esr, bar0 + 0x00405840u); + if (ds_post_log < 24u) { + ds_post_log++; + pr_info("vgsp-gr: DS-ESR post-clear=%08x h=0x%x tag=%s\n", + esr, h_object, tag); + } + } + } + /* * SESSION 153-CONTINUED-107: post-spin IRAM sample - the state left * when the poison (if any) has just cleared, before the poll/retire * work touches anything. @@ -10729,6 +10754,61 @@ static u64 gsp_phys_rmcall_gr_pte_leaf(u64 hpa) } /* + * r1497: a VID-aperture leaf pointing at the page's real frame-buffer + * offset. The guest's own page tables encode exactly this + * (VALID=1, APERTURE=VID(0), KIND=6 GENERIC_MEMORY, PA>>12 in [56:8]), + * so the engine reads the real FBMEM page the client uploaded instead of + * an emulated BAR1 shadow that no upload ever populated. + */ +static u64 gsp_phys_rmcall_gr_pte_leaf_vid(u64 fb_off) +{ + return 1ULL | (((fb_off >> 12) & 0x1ffffffULL) << 8) | + ((u64)6u << 56); +} + +/* + * r1497: is the engine-visible shadow page at `hpa` entirely empty? + * Only then may the real-FB VID leaf replace it; pages the module seeded + * (fixed CB), the engine wrote, or the client wrote through the emulated + * aperture keep their shadow identity. + */ +static bool gsp_phys_gr_shadow_page_empty(u64 hpa) +{ + const u8 *p; + u32 i; + + if (!hpa || !pfn_valid((unsigned long)(hpa >> PAGE_SHIFT))) + return false; + p = (const u8 *)__va(hpa); + for (i = 0; i < GSP_PHYS_PAGE_SIZE; i++) + if (p[i]) + return false; + return true; +} + +/* + * r1497: resolve a fetch VA to its real frame-buffer offset when the page + * is FB-aperture AND the emulated shadow copy of it is all-zero. This is + * the active indexed draw's vertex/index class (r1496): the client's page + * tables map the page to vidmem, the upload never reached an emulated + * path, and the engine fetched zeros. Returns true with *fb_off set. + */ +static bool gsp_phys_gr_fetch_vid_off_locked(u64 va, u64 *fb_off) +{ + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1), + &r)) + return false; + if (r.ap != 2 /* SVC_APERTURE_FB */ || !r.hpa) + return false; + if (!gsp_phys_gr_shadow_page_empty(r.hpa)) + return false; + *fb_off = r.pa; + return true; +} + +/* * GP10X leaf PTE for a frame-buffer (VIDEO_MEMORY aperture 0) address: * VALID=1 (bit 0), APERTURE=0 (bits 2:1), VOL=1 (bit 3), frame-buffer byte * offset >> 12 in the VIDEO address field (bits 32:8; dev_mmu.h:592-595, @@ -17759,6 +17839,9 @@ static u64 gsp_phys_gr_timed_out; static u64 gsp_phys_gr_zerowin_skips; static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; +/* r1497: launch fetch pages mapped through a real-FB VID leaf (empty shadow). */ +static u64 gsp_phys_gr_vid_map_pages; +static u32 gsp_phys_gr_vid_log; /* m156 (E5): zero-write retries where the warm sem push proved the engine * wedged (the persistent-lottery class); further retries are skipped. */ static u64 gsp_phys_gr_sem_wedged; @@ -22888,6 +22971,58 @@ static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va, return 0; } +/* + * r1497: install a VID-aperture PTE at the guest VA so the engine reads + * the real FBMEM page (fb_off) instead of an empty shadow page. Same + * never-rewrite discipline; the cache key is tagged so a VID entry can + * never be mistaken for a host-PA alias. + */ +static int gsp_phys_rmcall_gr_map_va_vid_locked(void __iomem *bar0, + u64 guest_va, u64 fb_off, + u32 epoch) +{ + u64 page = guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 pde_index = (u32)(page >> 21); + u32 pte_idx = (u32)((page >> 12) & 0x1ffu); + u32 line = pte_idx / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; + u64 key = (fb_off & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) | + (1ULL << 63); + struct gsp_phys_boot_chunk *pt; + u8 *pt_va; + u32 slot, i; + int ret; + + ret = gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); + if (ret) + return ret; + for (i = 0; i < boot_stage.rmcall_gr_win[slot].pte_count; i++) { + if (boot_stage.rmcall_gr_win[slot].pte[i].idx == pte_idx) { + if (boot_stage.rmcall_gr_win[slot].pte[i].hpa == key) + return 0; + return -EALREADY; + } + if (boot_stage.rmcall_gr_win[slot].pte[i].epoch != epoch && + boot_stage.rmcall_gr_win[slot].pte[i].idx / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES == line) + return -EBUSY; + } + if (boot_stage.rmcall_gr_win[slot].pte_count >= + GSP_PHYS_RMCALL_GR_PT_ENTRIES) + return -ENOSPC; + pt = &boot_stage.rmcall_gr_pt[boot_stage.rmcall_gr_win[slot].pt_slot]; + pt_va = gsp_phys_chunk_page_va(pt, 0); + if (!pt_va) + return -EIO; + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf_vid(fb_off), + pt_va + (size_t)pte_idx * 8); + dma_wmb(); + boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx; + boot_stage.rmcall_gr_win[slot].pte[i].epoch = epoch; + boot_stage.rmcall_gr_win[slot].pte[i].hpa = key; + boot_stage.rmcall_gr_win[slot].pte_count++; + return 0; +} + static void __iomem *gsp_phys_gr_scratch_bar0; /* @@ -25645,6 +25780,14 @@ static u64 gsp_phys_rmcall_gr_stream_page_hpa_locked( { u32 i; + /* + * r1497: a vertex/index fetch page can resolve to an FB page whose + * emulated shadow is empty while the same VA has a sysmem mapping + * with the real bytes (r1496: the active indexed draw's 0x44fea00 + * stream read all-zero). Populate the empty shadow page before the + * stream line is built. Non-empty pages and SYS answers are no-ops. + */ + gsp_sema_gr_fetch_backing(page); for (i = 0; i < req->n_pages; i++) if ((req->pages[i].guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page && @@ -25713,8 +25856,11 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked( u64 base_page = base_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); u64 end_page = (end_va - 1u) & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); u64 hpa[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES]; + u64 pte[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES]; + u64 fb_off[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES]; + bool real[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES]; u64 va0, dummy = 0; - u32 npages, i, line, page, line_in_page, dummy_n = 0; + u32 npages, i, line, page, line_in_page, dummy_n = 0, vid_n = 0; u8 *pt; int ret; @@ -25724,8 +25870,24 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked( if (npages > GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) return 0; for (i = 0; i < npages; i++) { - hpa[i] = gsp_phys_rmcall_gr_stream_page_hpa_locked(req, - base_page + ((u64)i << 12)); + u64 pa = base_page + ((u64)i << 12); + + real[i] = false; + fb_off[i] = 0; + /* + * r1497: the active draw's vertex/index page resolves to a + * vidmem page whose emulated shadow is empty. Point the + * fetch line at the real FBMEM page (the guest's own page + * table names the offset) instead of delivering zeros. + */ + if (gsp_phys_gr_fetch_vid_off_locked(pa, &fb_off[i])) { + pte[i] = gsp_phys_rmcall_gr_pte_leaf_vid(fb_off[i]); + hpa[i] = 0; + real[i] = true; + vid_n++; + continue; + } + hpa[i] = gsp_phys_rmcall_gr_stream_page_hpa_locked(req, pa); if (!hpa[i]) { if (!dummy) dummy = gsp_phys_gr_surf_dummy_dma_locked(); @@ -25734,6 +25896,7 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked( hpa[i] = dummy; dummy_n++; } + pte[i] = gsp_phys_rmcall_gr_pte_leaf(hpa[i]); } if (!sh->pages) { ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, @@ -25756,8 +25919,7 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked( if (!pt) return 0; for (i = 0; i < npages; i++) - put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa[i]), - pt + (size_t)( + put_unaligned_le64(pte[i], pt + (size_t)( GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + i) * 8); dma_wmb(); @@ -25765,14 +25927,16 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked( va0 = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); for (i = 0; i < npages; i++) - gsp_phys_rmcall_gr_alias_cache_set_locked(hpa[i], - va0 + ((u64)i << 12)); + if (!real[i]) + gsp_phys_rmcall_gr_alias_cache_set_locked(hpa[i], + va0 + ((u64)i << 12)); { static u32 stream_line_log; u32 w[8] = { 0 }, k; u64 hpa0 = hpa[0]; - if (pfn_valid((unsigned long)(hpa0 >> PAGE_SHIFT))) { + if (!real[0] && + pfn_valid((unsigned long)(hpa0 >> PAGE_SHIFT))) { const u8 *p = (const u8 *)__va( hpa0 + (base_va & (GSP_PHYS_PAGE_SIZE - 1))); @@ -25781,10 +25945,11 @@ static u64 gsp_phys_rmcall_gr_stream_line_locked( } if (stream_line_log < 32u) { stream_line_log++; - pr_info("vgsp-gr: MME-VSTREAM-LINE h=0x%x base=0x%llx end=0x%llx pages=%u dummy=%u line=0x%llx hpa0=0x%llx first8=%08x %08x %08x %08x %08x %08x %08x %08x\n", + pr_info("vgsp-gr: MME-VSTREAM-LINE h=0x%x base=0x%llx end=0x%llx pages=%u dummy=%u vid=%u fb0=0x%llx line=0x%llx hpa0=0x%llx first8=%08x %08x %08x %08x %08x %08x %08x %08x\n", req ? req->h_object : 0u, base_va, end_va, - npages, dummy_n, va0, hpa0, w[0], w[1], w[2], - w[3], w[4], w[5], w[6], w[7]); + npages, dummy_n, vid_n, fb_off[0], va0, hpa0, + w[0], w[1], w[2], w[3], w[4], w[5], w[6], + w[7]); } } return va0; @@ -28175,6 +28340,41 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, pr_info("vgsp-gr: page[%u] va=0x%llx hpa=0x%llx qmd_word=%u\n", i, pg->guest_va, pg->hpa, pg->qmd_word); /* + * r1497/r1498: an FB *fetch* page (a QMD pointer pair, not + * the data-fill surface) whose shadow is empty is the active + * indexed draw's vertex/index class. Map the real FBMEM page + * through a VID leaf instead of the empty shadow. The + * render-target/surface pages (QMD word NONE) must keep + * their shadow identity: the engine's stores have to land in + * the emulated page so the copy-out can ship them back to + * the guest (r1498: mapping the surface to real FB made the + * frame invisible to the guest). + */ + if (pg->hpa && pg->qmd_word != 48u && + pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE) { + u64 fb_off = 0; + + if (gsp_phys_gr_fetch_vid_off_locked(pg_va, + &fb_off)) { + ret = gsp_phys_rmcall_gr_map_va_vid_locked( + bar0, pg_va, fb_off, epoch); + if (!ret) { + mapped = true; + gsp_phys_gr_vid_map_pages++; + if (gsp_phys_gr_vid_log < 32u) { + gsp_phys_gr_vid_log++; + pr_info("vgsp-gr: FETCH-VID-MAP va=0x%llx fb=0x%llx qmd=%u h=0x%x push=0x%llx\n", + pg_va, fb_off, + pg->qmd_word, + req->h_object, + req->push_va); + } + } else if (ret == -ENOSPC) { + gsp_phys_gr_pool_exhausted++; + } + } + } + /* * SESSION 149 R15 (M2-S1): with the window-0 shadow path * enabled, force the program page (QMD word 48) OFF the * post-commit Tier-1 install. r801b proved that install is @@ -28184,7 +28384,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * the win0 shadow copy + w48/49 rewrite line, which is the * proven one. Every other page keeps its Tier-1 route. */ - if (pg->hpa && !(pg->qmd_word == 48u && + if (!mapped && pg->hpa && !(pg->qmd_word == 48u && gsp_phys_rmcall_gr_shadow_win0_value())) { ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, pg->hpa, epoch); |