summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--kernel/gsp_sema_service.c106
1 files changed, 69 insertions, 37 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 40b8063..948285b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -531,6 +531,10 @@ struct svc_channel {
u64 counter_pa; /* guest PA of the u32 GPU completion counter
* (registered via the sema_counter debug param);
* 0 = not registered */
+ u32 counter_ap; /* aperture of counter_pa: SVC_APERTURE_SYS
+ * (guest RAM) or SVC_APERTURE_FB (emulated
+ * vidmem - the X DDX/kick channels' progress
+ * semaphores live there); 0 = SYS */
u64 release_pa; /* guest PA the push's REPORT_SEM release
* writes (learned from the release VA); the
* publish targets counter_pa instead, so both
@@ -2804,6 +2808,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
if (ch) {
int k;
u64 keep_counter_pa = 0;
+ u32 keep_counter_ap = 0;
u64 keep_release_pa = 0;
u64 keep_release_va = 0;
u8 keep_gr_index = 0, keep_ce_index = 0;
@@ -2830,6 +2835,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
*/
if (same_gen) {
keep_counter_pa = ch->counter_pa;
+ keep_counter_ap = ch->counter_ap;
keep_release_pa = ch->release_pa;
keep_release_va = ch->release_va;
keep_gr_index = ch->gr_index;
@@ -2981,6 +2987,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
: SVC_APERTURE_SYS;
if (same) {
ch->counter_pa = keep_counter_pa;
+ ch->counter_ap = keep_counter_ap;
ch->release_pa = keep_release_pa;
ch->release_va = keep_release_va;
ch->gr_index = keep_gr_index;
@@ -16410,11 +16417,12 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
u32 cur = 0;
u32 nv;
u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa;
+ u32 ap = ch->release_pa ? SVC_APERTURE_SYS :
+ (ch->counter_ap ? ch->counter_ap : SVC_APERTURE_SYS);
if (!payload || !pa)
return;
- if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur,
- sizeof(cur)))
+ if (svc_read_ap(vgpu, pa, ap, &cur, sizeof(cur)))
return;
if (inc) {
if (payload > 0x100000) {
@@ -16432,8 +16440,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
return;
}
nv = inc ? cur + payload : payload;
- if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv,
- sizeof(nv)))
+ if (svc_write_ap(vgpu, pa, ap, &nv, sizeof(nv)))
return;
ch->pending_rel_pay = 0;
ch->pending_rel_inc = 0;
@@ -16483,6 +16490,9 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
static struct {
u64 va_page;
u64 pa_page;
+ u32 ap; /* aperture of pa_page: the fb/kick channels'
+ * counter pages resolve to vidmem (SVC_APERTURE_FB),
+ * not guest RAM */
/*
* SESSION 153-CONTINUED-38: the VAS the page was derived for. The
* counter VA window (0x20400fXXX) is a fixed per-client layout, so
@@ -16495,13 +16505,15 @@ static struct {
static u32 svc_cnt_der_n;
static u32 svc_cnt_der_log;
-static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
+static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out,
+ u32 *ap_out)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
u64 off = va & (SEMA_PAGE_SIZE - 1);
u64 pa = 0;
u32 probe = 0;
u32 i;
+ u32 ap = SVC_APERTURE_SYS;
bool fresh = true;
for (i = 0; i < svc_cnt_der_n; i++) {
@@ -16509,6 +16521,8 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
(!svc_cur_vas || !svc_cnt_der[i].h_vas ||
svc_cnt_der[i].h_vas == svc_cur_vas)) {
pa = svc_cnt_der[i].pa_page | off;
+ ap = svc_cnt_der[i].ap ? svc_cnt_der[i].ap :
+ SVC_APERTURE_SYS;
fresh = false;
break;
}
@@ -16541,35 +16555,37 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
*/
if (!ok && svc_cur_vas &&
!svc_chanvas_lookup(vgpu, page, &p, &a) &&
- a == SVC_APERTURE_SYS &&
+ (a == SVC_APERTURE_SYS || a == SVC_APERTURE_FB) &&
!svc_read_ap(vgpu,
(p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off,
- SVC_APERTURE_SYS, &probe, sizeof(probe))) {
+ a, &probe, sizeof(probe))) {
ok = true;
src = "chanvas";
}
if (!ok && !svc_walk_vaspaces(vgpu, page, &p, &a) &&
- a == SVC_APERTURE_SYS &&
+ (a == SVC_APERTURE_SYS || a == SVC_APERTURE_FB) &&
!svc_read_ap(vgpu,
(p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off,
- SVC_APERTURE_SYS, &probe, sizeof(probe))) {
+ a, &probe, sizeof(probe))) {
ok = true;
src = "vaspaces";
}
if (!ok && !svc_walk_discovered(vgpu, page, &p, &a) &&
- a == SVC_APERTURE_SYS &&
+ (a == SVC_APERTURE_SYS || a == SVC_APERTURE_FB) &&
!svc_read_ap(vgpu,
(p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off,
- SVC_APERTURE_SYS, &probe, sizeof(probe))) {
+ a, &probe, sizeof(probe))) {
ok = true;
src = "discovered";
}
- if (!ok && !svc_targeted_resolve(vgpu, page, &p) &&
- !svc_read_ap(vgpu,
- (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off,
- SVC_APERTURE_SYS, &probe, sizeof(probe))) {
- ok = true;
- src = "targeted";
+ if (!ok && !svc_targeted_resolve(vgpu, page, &p)) {
+ a = SVC_APERTURE_SYS; /* rm_mem content scan */
+ if (!svc_read_ap(vgpu,
+ (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off,
+ a, &probe, sizeof(probe))) {
+ ok = true;
+ src = "targeted";
+ }
}
if (!ok) {
if (svc_cnt_der_log < 16) {
@@ -16580,24 +16596,27 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
return -ENOENT;
}
pa = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off;
+ ap = a;
if (svc_cnt_der_log < 16) {
svc_cnt_der_log++;
- pr_info("sema-chan: counter page 0x%llx resolved via %s (pa=0x%llx)\n",
- page, src, pa);
+ pr_info("sema-chan: counter page 0x%llx resolved via %s (pa=0x%llx ap=%d)\n",
+ page, src, pa, ap);
}
}
/* Read the slot before trusting the mapping: an unreadable page is
* not registered, so the caller keeps the "slot pending" fallback. */
- if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe)))
+ if (svc_read_ap(vgpu, pa, ap, &probe, sizeof(probe)))
return -EIO;
if (fresh && svc_cnt_der_n < SVC_CNT_DER_MAX) {
svc_cnt_der[svc_cnt_der_n].va_page = page;
svc_cnt_der[svc_cnt_der_n].pa_page =
pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+ svc_cnt_der[svc_cnt_der_n].ap = ap;
svc_cnt_der[svc_cnt_der_n].h_vas = svc_cur_vas;
svc_cnt_der_n++;
}
*pa_out = pa;
+ *ap_out = ap;
return 0;
}
@@ -16614,7 +16633,7 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
* is the second client's own page for the same VA.
*/
static int svc_counter_derive_not(struct nv_vgpu_device *vgpu, u64 va,
- u64 avoid_pa, u64 *pa_out)
+ u64 avoid_pa, u64 *pa_out, u32 *ap_out)
{
unsigned long flags;
struct svc_vaspace local[SVC_MAX_VASPACES];
@@ -16636,16 +16655,16 @@ static int svc_counter_derive_not(struct nv_vgpu_device *vgpu, u64 va,
if (svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries,
page, local[i].ap, &p, &a, false))
continue;
- if (a != SVC_APERTURE_SYS)
+ if (a != SVC_APERTURE_SYS && a != SVC_APERTURE_FB)
continue;
p = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off;
if ((p & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
(avoid_pa & ~(u64)(SEMA_PAGE_SIZE - 1)))
continue;
- if (svc_read_ap(vgpu, p, SVC_APERTURE_SYS, &probe,
- sizeof(probe)))
+ if (svc_read_ap(vgpu, p, a, &probe, sizeof(probe)))
continue;
*pa_out = p;
+ *ap_out = a;
return 0;
}
return -ENOENT;
@@ -16708,12 +16727,14 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
spin_lock_irqsave(&svc.chan_lock, flags);
ch->counter_pa = ipa & ~0x3ULL;
+ ch->counter_ap = iap;
for (j = 0; j < SVC_MAX_CHANNELS; j++) {
if (svc.chan[j].valid &&
svc.chan[j].vgpu == vgpu &&
svc.chan[j].h_object == ch->h_object) {
svc.chan[j].counter_pa =
ch->counter_pa;
+ svc.chan[j].counter_ap = iap;
break;
}
}
@@ -16737,7 +16758,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
*/
if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) {
if (!gsp_sema_counter_derive ||
- svc_counter_derive(vgpu, va, &pa))
+ svc_counter_derive(vgpu, va, &pa, &ap))
return;
derived = true;
}
@@ -16781,8 +16802,10 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
* registration must stay pending.
*/
u64 pa2 = 0;
+ u32 ap2 = SVC_APERTURE_SYS;
- if (!svc_counter_derive_not(vgpu, va, pa, &pa2)) {
+ if (!svc_counter_derive_not(vgpu, va, pa, &pa2,
+ &ap2)) {
bool coll2 = false;
spin_lock_irqsave(&svc.chan_lock, flags);
@@ -16803,6 +16826,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
ch->h_object, ch->gr_index,
pa2 & ~0x3ULL, pa);
pa = pa2 & ~0x3ULL;
+ ap = ap2;
collision = false;
}
}
@@ -16825,11 +16849,13 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
*/
spin_lock_irqsave(&svc.chan_lock, flags);
ch->counter_pa = pa;
+ ch->counter_ap = ap;
for (j = 0; j < SVC_MAX_CHANNELS; j++) {
if (svc.chan[j].valid &&
svc.chan[j].vgpu == vgpu &&
svc.chan[j].h_object == ch->h_object) {
svc.chan[j].counter_pa = pa;
+ svc.chan[j].counter_ap = ap;
break;
}
}
@@ -16842,11 +16868,13 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
}
}
if (derived)
- pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
- ch->h_object, ch->gr_index, ch->counter_pa, va);
+ pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx ap=%u (slot va=0x%llx)\n",
+ ch->h_object, ch->gr_index, ch->counter_pa,
+ ch->counter_ap, va);
else
- pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
- ch->h_object, ch->gr_index, ch->counter_pa, va);
+ pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx ap=%u (slot va=0x%llx)\n",
+ ch->h_object, ch->gr_index, ch->counter_pa,
+ ch->counter_ap, va);
svc_counter_flush_pending(vgpu, ch);
}
@@ -16882,7 +16910,7 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
*/
if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) {
if (!gsp_sema_counter_derive ||
- svc_counter_derive(vgpu, sem_va, &pa))
+ svc_counter_derive(vgpu, sem_va, &pa, &ap))
return;
derived = true;
}
@@ -34067,6 +34095,8 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
* fallback for a channel whose pushes are unreadable.
*/
u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa;
+ u32 ap = ch->release_pa ? SVC_APERTURE_SYS :
+ (ch->counter_ap ? ch->counter_ap : SVC_APERTURE_SYS);
if (gsp_sema_trace_h && (u32)ch->h_object == (u32)gsp_sema_trace_h &&
time_after(jiffies, svc_perf_last)) {
@@ -34158,7 +34188,7 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
* advanced the slot past the put position the client last submitted;
* the client polls this u32, so never write it backwards.
*/
- if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur)))
+ if (svc_read_ap(vgpu, pa, ap, &cur, sizeof(cur)))
return;
if (val <= cur) {
if (cur > ch->counter_last)
@@ -34171,11 +34201,11 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
* make a later absolute release a silent skip. Witness every advance.
*/
if (gsp_sema_dbg)
- pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n",
- ch->h_object, pa, val, cur,
+ pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx ap=%u val=%u cur=%u last=%u\n",
+ ch->h_object, pa, ap, val, cur,
ch->counter_last);
{
- int wrc = svc_write_ap(vgpu, pa, SVC_APERTURE_SYS,
+ int wrc = svc_write_ap(vgpu, pa, ap,
&val, sizeof(val));
if (wrc) {
if (gsp_sema_dbg && svc.pend_dbg_log < 1024) {
@@ -40574,7 +40604,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
ok = true;
else if (gsp_sema_counter_derive &&
!svc_counter_derive(vgpu, svc_gr_cand.rel_va,
- &pa))
+ &pa, &ap))
ok = true;
} else if (!svc_resolve_va(vgpu, svc_gr_cand.rel_va, &pa,
&ap)) {
@@ -47897,8 +47927,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
* runs, 2026-10-06). Only fill a missing slot; a
* live authoritative value wins.
*/
- if (!svc.chan[k].counter_pa)
+ if (!svc.chan[k].counter_pa) {
svc.chan[k].counter_pa = ch->counter_pa;
+ svc.chan[k].counter_ap = ch->counter_ap;
+ }
if (!svc.chan[k].release_pa)
svc.chan[k].release_pa = ch->release_pa;
break;
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-13 23:27:31 +0000