diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 171 |
1 files changed, 161 insertions, 10 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index a915108..d2a1ffc 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -856,6 +856,13 @@ struct svc_pin { struct svc_vaspace { bool valid; + /* + * CONT.400: proven against the client's own content-validated ring (or + * published by RM). The bounded age-out registers UNVERIFIED; keeping + * the distinction lets svc_chan_vas_candidate_promote upgrade a parked + * directory to verified instead of giving up because it is "known". + */ + bool verified; u32 h_vaspace; u64 pdb; u32 entries; /* root entry count */ @@ -2069,6 +2076,31 @@ static bool svc_vaspace_known(u32 h_vaspace) } /* + * CONT.400: is this VA space backed by a directory proven against the + * client's own content-validated ring (or published by RM)? Only then may + * the parked-candidate promote skip its walk. A directory that exists but is + * only UNVERIFIED (bounded age-out) must still be upgraded by content. + */ +static bool svc_vaspace_verified(u32 h_vaspace) +{ + unsigned long flags; + bool found = false; + int i; + + if (!h_vaspace) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].verified && + svc.vas[i].h_vaspace == h_vaspace) { + found = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + return found; +} + +/* * Register a guest channel. Called from the GSP-RM emulation when the guest * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its * entry count and the guest-physical USERD/instance blocks), and dropped when @@ -2695,11 +2727,21 @@ static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) d.pdb = svc_vas_cand[i].pdb; d.num_entries = svc_vas_cand[i].entries; d.b_vidmem = svc_vas_cand[i].vidmem; + /* + * CONT.400: register the parked directory as UNVERIFIED so the + * client can make progress, but KEEP the candidate parked. The + * age-out previously cleared it, which meant that once the + * 1 ms timer fired the content-verified upgrade path + * (svc_chan_vas_candidate_promote) had nothing left to walk - + * exactly why the beef GR ring (VAS 0xbeef0300) stayed + * ring_pa=0 for whole boots. Bumping the stamp only re-arms the + * bounded wait; the candidate stays available for promotion. + */ pr_warn("sema-svc: VAS 0x%x page directory registered UNVERIFIED after %u ms parked (pdb=0x%llx entries=%u; the content-verified ring walk never anchored this boot)\n", d.h_vaspace, svc_vas_cand_age_ms, d.pdb, d.num_entries); gsp_sema_service_vaspace_add(&d); - svc_vas_cand[i].valid = false; + svc_vas_cand[i].stamp = jiffies; } } @@ -2734,6 +2776,7 @@ static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, d.pdb = svc_vas_cand[i].pdb; d.num_entries = svc_vas_cand[i].entries; d.b_vidmem = svc_vas_cand[i].vidmem; + d.verified = true; /* CONT.400: verified ring walk */ gsp_sema_service_vaspace_add(&d); svc_vas_cand[i].valid = false; pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", @@ -2788,7 +2831,20 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) vs->h_vaspace, vs->pdb, desc->h_vaspace); } if (vs) { + /* + * CONT.400: never downgrade a verified directory with an + * unverified one. The bounded age-out re-registers the last + * parked candidate every pass; if that candidate is a stale + * PDB and we let it overwrite the verified one, the ring + * translation regresses to zero after it was working. + */ + if (vs->valid && vs->verified && !desc->verified) { + vs->stamp = jiffies; + spin_unlock_irqrestore(&svc.vas_lock, flags); + return; + } vs->valid = true; + vs->verified = desc->verified; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; @@ -29217,6 +29273,7 @@ static bool svc_xvas_try_root(struct nv_vgpu_device *vgpu, u64 pdb, d.pdb = pdb; d.num_entries = entries; d.b_vidmem = vidmem; + d.verified = true; /* CONT.400: ring VA->PA verified */ gsp_sema_service_vaspace_add(&d); pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (%s root; ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", pdb, entries, vidmem ? "vidmem" : "sysmem", ring_va, ring_pa); @@ -29318,6 +29375,8 @@ static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, * USERD sweep at +15 ms, age-out at +1.36 s, X's DMA test window 10 s). * Anchor on the channel's own gpfifo_va and its entry 0 content instead. */ +static unsigned long svc_promote_diag_at; + static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -29332,9 +29391,26 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, */ if (!h) h = svc_channel_owner_vas(vgpu, ch->h_object); - if (!h || svc_vaspace_known(h) || !ch->gpfifo_va || - ch->gpfifo_entries < 2) + /* + * CONT.400: gate on VERIFIED, not merely known. The bounded age-out + * registers the parked directory UNVERIFIED (so the client can move) + * but the translation is frequently wrong; we must keep trying to + * upgrade it by content until a walk proves it. Only a verified + * directory lets us skip the walk. + */ + if (!h || svc_vaspace_verified(h) || !ch->gpfifo_va || + ch->gpfifo_entries < 2) { + /* CONT.400 diag: name the exact early-return reason. */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u && + time_after(jiffies, svc_promote_diag_at)) { + svc_promote_diag_at = jiffies + msecs_to_jiffies(2000); + pr_info("sema-svc: VAS-PROMOTE h=0x%x skip (h=0x%x verified=%d gpfifo=0x%llx entries=%u)\n", + ch->h_object, h, + h ? svc_vaspace_verified(h) : -1, + ch->gpfifo_va, ch->gpfifo_entries); + } return; + } for (i = 0; i < SVC_VAS_CAND_MAX; i++) { u64 wpa = 0; int wap = SVC_APERTURE_SYS; @@ -29342,16 +29418,25 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, u32 e0 = 0, e1 = 0, len; u64 eva; struct sema_vaspace_desc d; + int wrc; if (!svc_vas_cand[i].valid || svc_vas_cand[i].h_vaspace != h) continue; - if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, - svc_vas_cand[i].entries, - ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), - svc_vas_cand[i].vidmem ? SVC_APERTURE_FB - : SVC_APERTURE_SYS, - &wpa, &wap, false)) + wrc = svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, + ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), + svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + &wpa, &wap, false); + if (wrc) { + /* CONT.400 diag: why the beef ring walk misses. */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) + pr_info("sema-svc: VAS-PROMOTE h=0x%x cand pdb=0x%llx entries=%u walk rc=%d (gpfifo 0x%llx)\n", + ch->h_object, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, wrc, + ch->gpfifo_va); continue; + } if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) continue; memcpy(&e0, e, 4); @@ -29359,19 +29444,85 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if (!eva || len < 4 || len > 0x10000 || - !svc_gr_entry_win_ok(ch->gpfifo_va, eva)) + !svc_gr_entry_win_ok(ch->gpfifo_va, eva)) { + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) + pr_info("sema-svc: VAS-PROMOTE h=0x%x cand pdb=0x%llx -> wpa=0x%llx entry0=%08x %08x eva=0x%llx len=%u REJECT\n", + ch->h_object, svc_vas_cand[i].pdb, + wpa, e0, e1, eva, len); continue; + } memset(&d, 0, sizeof(d)); d.h_vaspace = h; d.pdb = svc_vas_cand[i].pdb; d.num_entries = svc_vas_cand[i].entries; d.b_vidmem = svc_vas_cand[i].vidmem; + d.verified = true; /* CONT.400: content-proven ring walk */ gsp_sema_service_vaspace_add(&d); svc_vas_cand[i].valid = false; pr_info("sema-svc: VAS 0x%x registered at channel 0x%x turn (parked candidate content-validated: gpfifo 0x%llx entry0 va=0x%llx len=%u)\n", h, ch->h_object, ch->gpfifo_va, eva, len); return; } + + /* + * CONT.400: the parked candidate is not always the directory that maps + * the ring (the GL client's GR ring 0x4020000 lives in the RM low + * aperture; its client VAS 0xbeef0300 maps the pushbuffers but not + * necessarily the ring). Probe EVERY registered directory for the ring + * VA and adopt the first that yields a plausible entry-0 push. This is + * the missing link that kept the beef GR channel ring_pa=0 forever, so + * the whole GL draw stream was never decoded (black screen + 0.2 FPS). + */ + { + unsigned long flags; + int k; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (k = 0; k < SVC_MAX_VASPACES; k++) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0, elen; + u64 eva; + struct svc_vaspace snap; + + if (!svc.vas[k].valid) + continue; + snap = svc.vas[k]; + spin_unlock_irqrestore(&svc.vas_lock, flags); + if (!svc_gmmu_walk(vgpu, snap.pdb, snap.entries, + ch->gpfifo_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + snap.ap, &wpa, &wap, false) && + !svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + elen = ((e1 >> 10) & 0x1fffff) << 2; + if (eva && elen >= 4 && elen <= 0x10000 && + svc_gr_entry_win_ok(ch->gpfifo_va, eva)) { + pr_info("sema-svc: RING-VAS h=0x%x gpfifo=0x%llx mapped by VAS 0x%x (pdb=0x%llx ap=%d) -> pa=0x%llx entry0 va=0x%llx len=%u\n", + ch->h_object, ch->gpfifo_va, + snap.h_vaspace, snap.pdb, + snap.ap, wpa, eva, elen); + /* Pin the ring through this directory. */ + svc_pin_store(ch->gpfifo_va, wpa, wap); + svc_lat_store(ch->gpfifo_va, wpa, wap); + svc_cur_vas = snap.h_vaspace; + return; + } + } + spin_lock_irqsave(&svc.vas_lock, flags); + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u && + time_after(jiffies, svc_promote_diag_at)) { + svc_promote_diag_at = jiffies + msecs_to_jiffies(2000); + pr_info("sema-svc: RING-VAS h=0x%x gpfifo=0x%llx NOT mapped by any registered VAS\n", + ch->h_object, ch->gpfifo_va); + } + } } static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, |