diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 76 |
1 files changed, 76 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 904ac65..e297887 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1915,6 +1915,8 @@ struct svc_ipin_alt { }; static struct svc_ipin_alt svc_ipin_alt[SVC_IPIN_ALT_SLOTS]; +static u64 svc_pushalt_hits; /* push reads served by a displaced pair */ +static u32 svc_pushalt_log; static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap) { @@ -6700,6 +6702,7 @@ static void svc_session_flush(void) * entry_log was already exhausted by the first client). */ svc.entry_log = 0; + svc_pushalt_log = 0; memset(svc_hot, 0, sizeof(svc_hot)); memset(svc_retry_q, 0, sizeof(svc_retry_q)); memset(svc_pool_pa, 0, sizeof(svc_pool_pa)); @@ -11942,6 +11945,34 @@ static bool svc_fwd_relay_hold_active(struct nv_vgpu_device *vgpu, svc_fwd_hold.ch_handle == ch->h_object; } +/* + * A push VA whose primary translation did not parse may have been displaced + * by a conflicting injection - the same race the pool-release fan-out + * handles (two host tools inject the same VA page with different PAs and the + * later store wins the table). Try the recorded displaced pair once; content + * plausibility is the arbiter like every other push read. + */ +static bool svc_push_read_displaced(struct nv_vgpu_device *vgpu, u32 h, + u64 va, u8 *buf, u32 len) +{ + u64 pa = 0; + int ap = 0; + + if (svc_ipin_alt_lookup(va, &pa, &ap)) + return false; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), buf, len)) + return false; + if (!svc_method_stream_plausible(buf, len)) + return false; + svc_pushalt_hits++; + if (gsp_sema_dbg && svc_pushalt_log < 32) { + svc_pushalt_log++; + pr_info("sema-pushalt: h=0x%x va=0x%llx pa=0x%llx\n", + h, va, pa); + } + return true; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -12396,7 +12427,52 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } else { unreadable = true; } + /* + * Still implausible (or + * unreadable): the primary + * translation may be the + * loser of a conflicting + * injection. Try the + * displaced pair before + * the rescue/hold paths. + */ + if (unreadable || + !svc_method_stream_plausible(pb, len)) { + u8 *db = kmalloc(len, + GFP_KERNEL); + + if (db && + svc_push_read_displaced(vgpu, + ch->h_object, va, + db, len)) { + memcpy(pb, db, len); + injected = false; + unreadable = false; + } + kfree(db); + } + } + } + + /* + * The entry's page was unreadable through + * every primary source; try the displaced + * injected pair once more before the + * content rescue (it may be the page the + * client actually wrote). + */ + if (unreadable) { + u8 *db = kmalloc(len, GFP_KERNEL); + + if (db && + svc_push_read_displaced(vgpu, + ch->h_object, va, + db, len)) { + memcpy(pb, db, len); + injected = false; + unreadable = false; } + kfree(db); } /* |