diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 205 |
1 files changed, 200 insertions, 5 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index c9abc71..797c4b5 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2076,6 +2076,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) { struct svc_channel *ch = NULL; unsigned long flags; + u64 prev_notify2d_va = 0; + u32 prev_notify2d_gen = 0; int i; if (!desc || !desc->h_object || !desc->gpfifo_va || @@ -2127,6 +2129,28 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } } spin_lock_irqsave(&svc.chan_lock, flags); + /* + * r1652: the GL client's kick channel (0xbfef0002) is re-registered + * while the client keeps the class2d notification target it armed on + * the previous instance: its short kick push carries NOTIFY 0x10c + * without SET_NOTIFY_A/B. The old slot still holds the target after + * gsp_sema_service_channel_del() (valid=false only), but this add's + * memset() would erase it, leaving the short NOTIFY with no address: + * svc_notify2d_write() then drops the completion, no channel event is + * posted, and the client spins in its kick/fence wait. Carry the + * target across a same-session, same-handle re-registration only; a + * different session (or a fresh SET_NOTIFY_A/B) still wins. + */ + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].vgpu != desc->vgpu || + svc.chan[i].h_object != desc->h_object) + continue; + if (svc.chan[i].notify2d_va) { + prev_notify2d_va = svc.chan[i].notify2d_va; + prev_notify2d_gen = svc.chan[i].session_gen; + break; + } + } for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].vgpu == desc->vgpu && svc.chan[i].h_object == desc->h_object) { @@ -2201,6 +2225,19 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->counter_gen = keep_counter_gen; } /* + * r1652: restore the class2d notification target saved before the + * memset. A short kick (NOTIFY only) then still has the address the + * guest programmed on the previous incarnation of this handle, and + * svc_notify2d_write() can author the completion again. The session + * generation check keeps a dead session's VA out of a new client; + * any SET_NOTIFY_A/B in the new incarnation overwrites it. + */ + if (prev_notify2d_va && prev_notify2d_gen == svc_sess_gen) { + ch->notify2d_va = prev_notify2d_va; + pr_info_ratelimited("sema-svc: channel 0x%x kept class2d notify va=0x%llx across re-registration\n", + ch->h_object, ch->notify2d_va); + } + /* * Rank among the live CE channels OF THIS GUEST. The RM hands * out object handles in creation order and the guest's UVM * allocates one push buffer, ring and tracking semaphore per @@ -10144,6 +10181,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, bool blob_trusted = false; const char *blob_src = "none"; const char *op_src = "stream"; + u64 kick_va; /* * SESSION 153 CONT.6 (task 1): record every section arrival (any @@ -10345,11 +10383,23 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * GPU's notification write is the only work * it asks for. */ + /* + * r1652: the short kick form carries only the class2d + * NOTIFY (0x10c), relying on the notification address + * already programmed in the channel context. Use the + * per-channel target when this section did not re-arm + * it; the carried value comes from the previous + * incarnation of the same handle (see + * gsp_sema_service_channel_add). Without the carried + * target the executor never runs and the GL client's + * kick wait spins. + */ + kick_va = svc_gl.kick_notify_va; + if (!kick_va && svc_kick_family(ch)) + kick_va = ch->notify2d_va; if (gsp_sema_kick_notify && - svc_gl.kick_notify_seen && - svc_gl.kick_notify_va && - svc_kick_notify_exec(vgpu, ch, - svc_gl.kick_notify_va, + svc_gl.kick_notify_seen && kick_va && + svc_kick_notify_exec(vgpu, ch, kick_va, svc_gl.kick_notify_val, push_va)) { /* authored once; a replay pass must not @@ -18508,8 +18558,20 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, u64 pa = 0; int ap = SVC_APERTURE_SYS; - if (!ch->notify2d_va) + if (!ch->notify2d_va) { + /* + * r1652: the short kick form (NOTIFY without SET_NOTIFY) + * depends on the target carried from the channel's previous + * registration. Name the miss so a future loss is visible + * instead of a silent spin. + */ + if (gsp_sema_dbg && svc_notify2d_log < 64) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x NOTIFY with no class2d target (short kick before SET_NOTIFY; client spin)\n", + ch->h_object); + } return false; + } if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { if (gsp_sema_dbg && (svc_notify2d_log < 64 || ch->h_object == 0xbfef0002)) { @@ -24521,6 +24583,110 @@ static bool svc_fb_userd_pristine_scan(struct nv_vgpu_device *vgpu, */ static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva); +/* + * r1653: capture the class2d kick channel's notification target from the + * FIRST GPFIFO entry. The GL client arms the class2d subchannel with + * SET_NOTIFY_A/B + NOTIFY in entry 0 and then submits short kick pushes + * (NOTIFY only) that rely on the target staying programmed in the channel + * context. The pump's walk seed consumes only the newest entries ending at + * the live PUT, so on a fresh channel entry 0 can be skipped forever and + * every short kick has no address to author - the client then spins in its + * kick wait (r1652: h=0xbfef0002, "NOTIFY with no class2d target"). Decode + * the skipped entries here, once, before the seed. + */ +static bool svc_kick_scan_push(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 len) +{ + u8 buf[512]; + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 n = min_t(u32, len, sizeof(buf)); + u32 off = 0; + + if (svc_lookup(vgpu, va, &pa, &ap) && + svc_resolve_va(vgpu, va, &pa, &ap)) + return false; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), buf, n)) + return false; + while (off + 4 <= n) { + u32 hdr, op, count, subch, addr, k; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 7; + addr = (hdr & 0x1fff) << 2; + if (!hdr) { + off += 4; + continue; + } + if (op == 1 || op == 3 || op == 5) { + if (!count || off + 4 + count * 4 > n) + break; + if (subch == 3) { + for (k = 0; k < count; k++) { + u32 m = op == 3 ? addr : addr + 4 * k; + u32 v; + + memcpy(&v, buf + off + 4 + 4 * k, 4); + if (m == 0x104) + ch->notify2d_va = + (u64)(v & 0x1ffffff) << 32; + else if (m == 0x108) { + ch->notify2d_va |= v; + if (ch->notify2d_va) + return true; + } + } + } + off += 4 + count * 4; + continue; + } + off += 4; + } + return ch->notify2d_va != 0; +} + +static void svc_kick_capture_arm(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, + int ring_ap, u32 uput) +{ + u32 e, nscan; + + if (!ch->engine_type || ch->h_object < 0xbfef0000u || + ch->h_object > 0xbfef00ffu || ch->notify2d_va || !ring_pa) + return; + /* + * r1653: the arm is normally entry 0, so a missing/zero live PUT + * must not block the decode. Bound the scan by the ring size and + * the live PUT when known. + */ + nscan = ch->gpfifo_entries ? ch->gpfifo_entries : 2; + if (uput && uput < nscan) + nscan = uput; + if (nscan > 8) + nscan = 8; + for (e = 0; e < nscan && !ch->notify2d_va; e++) { + u8 ent[8]; + u32 e0, e1; + u64 pva; + u32 plen; + + if (svc_read_ap(vgpu, ring_pa + (u64)e * 8, ring_ap, ent, + sizeof(ent))) + continue; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + pva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + plen = ((e1 >> 10) & 0x1fffff) << 2; + if (pva && plen >= 4 && plen <= 0x10000) + svc_kick_scan_push(vgpu, ch, pva, plen); + } + if (ch->notify2d_va) + pr_info("sema-chan: kick arm captured h=0x%x notify2d=0x%llx from ring entry (first-discovery decode)\n", + ch->h_object, ch->notify2d_va); +} + static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get, int rrc) { @@ -24827,6 +24993,10 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, ch->userd_pa = upa; ch->userd_as = SVC_APERTURE_SYS; ch->userd_ok = true; + /* r1653: decode the skipped first entries for the + * class2d kick arm before the walk seed below. */ + svc_kick_capture_arm(vgpu, ch, ring_pa, ring_ap, + uput); /* * SESSION 154 (M-GL1.2): seed the walk at the live * block's PUT. The client's put is free-running @@ -40451,6 +40621,31 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * r1653: the class2d kick channel's notification target is + * armed in the first GPFIFO entry, which the walk seed can + * skip. When the live PUT is known and the target is still + * missing, decode the head entries once here (the pump has + * the channel's GPFIFO VA and PUT). Without it every short + * kick push has no address and the GL client spins. + */ + if (!ch->engine_type && !ch->notify2d_va && put && + ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) { + if (ring_pa) { + svc_kick_capture_arm(vgpu, ch, ring_pa, + ring_ap, put); + } else { + u64 rpa = 0; + int rap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, ch->gpfifo_va, &rpa, + &rap)) + svc_kick_capture_arm(vgpu, ch, rpa, + rap, put); + } + } + + /* * r1650: keep the UVM rank-slot release moving for a short * window. Two effects need it: the pool page can move / a * per-client VAS directory can be discovered seconds later |