diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 201 |
1 files changed, 200 insertions, 1 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 82dceaf..f8604b6 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -760,6 +760,29 @@ struct svc_channel { u32 gr_ram_userd_reject_put; /* its PUT at rejection time */ u8 gr_ram_userd_rejects; /* rejections seen (bound) */ /* + * CONT.462 (X-DMA latch lifetime): delivery + backoff state. + * gr_ram_userd_latch_sub snapshots the consumed-entry count + * (ch->submissions) at latch adoption so the pump can tell a latch + * that HAS delivered decoded entries from a frozen lookalike (the + * false fb 0x8f0000 block never consumes: its put maps onto the + * current epoch's already-done slots and submissions stays put). + * A DDX-family latch (h 0x01xxxxxx / 0xbfef00xx) that has served is + * DEFERRED from the 15 s no-advance rejection: the X DDX's + * push-on-demand generations arrive 11-24 s apart (measured c130/ + * c131), so the fixed bound used to kill the block that had just + * passed the DMA test and the next generation then found a frozen FB + * lookalike. gr_ram_userd_defers bounds the deferral (~2 min); on + * every defer/reject decision the DDX family is re-probed (the RAM + * scan may adopt a newer live block in place). + * gr_ram_userd_reject_at stamps the last rejection so the FB shape + * scan backs off after one, leaving the RAM scan the window to find + * the real client block. + */ + u64 gr_ram_userd_latch_sub; /* ch->submissions at latch */ + u8 gr_ram_userd_defers; /* deferrals applied (bound 32) */ + unsigned long gr_ram_userd_reject_at; /* jiffies of last reject */ + unsigned long gr_ram_userd_reprobe_at; /* last live-latch re-probe */ + /* * SESSION 154 (M-GL1.2): content anchor for a client push whose * translation went stale (r913: a validated pin kept answering for * a reused page and the wrong-VAS walk answer was implausible while @@ -5218,6 +5241,7 @@ static void svc_pin_drop(u64 va) */ static inline bool svc_pool_sem_va(u64 va); static bool svc_kick_family(const struct svc_channel *ch); +static bool svc_userd_ddx_family(const struct svc_channel *ch); static void svc_pool_pa_forget(u64 va); /* @@ -10927,6 +10951,7 @@ static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len, * (svc_lookup_trusted -> svc_write_ap), then readback verification. */ static bool svc_kick_family(const struct svc_channel *ch); +static bool svc_userd_ddx_family(const struct svc_channel *ch); static u64 svc_push_pa_ceiling(void); static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, u32 val, @@ -27361,7 +27386,14 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, ch->gr_ram_userd_watch_n || ((ch->h_object >= 0xbfef0000u && ch->h_object <= 0xbfef00ffu) && - ch->gr_ram_userd_kick_sweeps < 64))) { + ch->gr_ram_userd_kick_sweeps < 64) || + /* + * CONT.462: the DDX family re-probe is ALWAYS allowed (clause + * iii): its generations churn blocks every 11-24 s, so a + * retired slice budget must not end the search for the live + * one. The scan stays sliced/bounded per call. + */ + (svc_userd_ddx_family(ch) && ch->gr_ram_userd_rejects < 255))) { u64 upa = 0; u32 uput = 0, uget = 0; @@ -27398,6 +27430,8 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, ch->gr_ram_put_seeded = uput; ch->gr_ram_userd_latch_put = uput; ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_latch_sub = ch->submissions; + ch->gr_ram_userd_defers = 0; ch->gr_ram_userd_confirmed = false; *put = uput; *get = uget; @@ -27455,6 +27489,16 @@ ce_fb_scan: (!ch->engine_type || (ch->h_object & 0xfffff000u) == 0xbeef0000u) && ch->fb_userd_scan_n < 512 && + /* + * CONT.462: after a rejected latch, give the DDX family's RAM + * scan a backoff window before the FB shape scan may re-latch + * (the frozen fb 0x8f0000 lookalike used to win every ~18 s + * churn cycle and keep the channel dark). + */ + !(svc_userd_ddx_family(ch) && ch->gr_ram_userd_reject_at && + time_before(jiffies, + (unsigned long)ch->gr_ram_userd_reject_at + + msecs_to_jiffies(6000))) && (!ch->fb_userd_scan_at || time_after_eq(jiffies, (unsigned long)ch->fb_userd_scan_at + @@ -27765,6 +27809,8 @@ ce_fb_scan: */ ch->gr_ram_userd_latch_put = best_put; ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_latch_sub = ch->submissions; + ch->gr_ram_userd_defers = 0; ch->gr_ram_userd_confirmed = false; pr_info("sema-chan: %s USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", ch->engine_type ? "CE" : "GR", @@ -30555,6 +30601,22 @@ static bool svc_kick_family(const struct svc_channel *ch) } /* + * CONT.462 (X-DMA latch lifetime): the DDX/kick handle families whose + * USERD latch is probed authoritatively by content (the X server's + * 0x01000013 kick channel and the GL kick family share the pattern: the + * control block is client-allocated and moves between push-on-demand + * generations). For these families (and only these) the pump may + * re-probe on a latch decision and the pre-reject FB shape-scan backoff + * applies - the SYS/CE/UVM families keep their existing semantics. + */ +static bool svc_userd_ddx_family(const struct svc_channel *ch) +{ + return ch && !ch->engine_type && + (((ch->h_object & 0xff000000u) == 0x01000000u) || + svc_kick_family(ch)); +} + +/* * Re-validate the kick channel's ring latch each pass (bounded to once a * second per channel). The RAM content scan latches a ring page once; if * the driver later zeroes or reuses that page (the r966-r971 class the @@ -43890,6 +43952,66 @@ static bool svc_userd_put_wrap(const struct svc_channel *ch) ch->userd_as == SVC_APERTURE_FB && !svc_kick_family(ch); } +/* + * CONT.462 (X-DMA latch lifetime): live-latch re-probe for the DDX family. + * + * The X DDX allocates a NEW control block per push-on-demand generation and + * its DMA-test window is only 10 s (c140: the RAM scan latched a lookalike at + * 0x101996f00 put=1 get=1, the real block never landed, the channel was freed + * 10 s later with submissions=0). Waiting for the 15 s latch decision is too + * late, and while a latch is live the pump never re-enters the scan. Run the + * content scan from the live path on a slow cadence: if it validates a + * DIFFERENT block (near-empty page, ring-entry validation, PUT-step + * liveness), switch the latch to it in place - the old block stays serving + * until that moment. Confirmed latches (the client advanced their PUT) are + * left alone; the SYS/CE/UVM families never enter. + */ +static bool svc_userd_ddx_reprobe(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, + int ring_ap, u32 *put, u32 *get) +{ + u64 npa = 0; + u32 nput = 0, nget = 0; + + if (!svc_userd_ddx_family(ch) || !ch->userd_ok || + ch->gr_ram_userd_confirmed || !ch->gr_ram_userd_latch_at) + return false; + if (!svc_gr_ram_userd_scan(vgpu, ch, &npa, &nput, &nget)) + return false; + if (npa == ch->userd_pa) + return false; + { + u64 opa = ch->userd_pa; + u32 oput = ch->gr_ram_userd_latch_put; + + ch->userd_pa = npa; + ch->userd_as = SVC_APERTURE_SYS; + ch->userd_ok = true; + svc_kick_capture_arm(vgpu, ch, ring_pa, ring_ap, nput); + if (ch->gpfifo_entries > 1 && nput) { + u32 span = nput < ch->gpfifo_entries ? + nput : ch->gpfifo_entries - 1; + + ch->last_get = (nput % ch->gpfifo_entries + + ch->gpfifo_entries - span) % + ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->gr_ram_put_seeded = nput; + ch->gr_ram_userd_latch_put = nput; + ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_latch_sub = ch->submissions; + ch->gr_ram_userd_defers = 0; + ch->gr_ram_userd_confirmed = false; + *put = nput; + *get = nget; + pr_info("sema-chan: GR USERD h=0x%x latch SWITCHED to re-probed block pa=0x%llx (put=%u get=%u; old pa=0x%llx latch=%u)\n", + ch->h_object, npa, nput, nget, opa, oput); + } + return true; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { if (svc_pdb_disc_pending) { @@ -44717,6 +44839,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool kfam = svc_kick_family(ch); /* + * CONT.462: keep a slow live-latch re-probe running for the + * DDX family (see svc_userd_ddx_reprobe). The client's 10 s + * DMA-test window does not wait for the 15 s latch decision, + * so a lookalike latch must be able to yield to the real + * block the moment the content scan validates one. + */ + if (svc_userd_ddx_family(ch) && !ch->gr_ram_userd_confirmed && + time_after_eq(jiffies, ch->gr_ram_userd_reprobe_at)) { + ch->gr_ram_userd_reprobe_at = + jiffies + msecs_to_jiffies(2000); + (void)svc_userd_ddx_reprobe(vgpu, ch, ring_pa, + ring_ap, &put, &get); + } + + /* * SESSION 154 (M-GL1.2): speculative-latch confirmation for * the RAM USERD. The scan latches a gated candidate right * away (fast walk start); the pump is the judge: a +1/+2 @@ -44785,6 +44922,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) put > ch->gr_ram_userd_latch_put; if (stepped) { ch->gr_ram_userd_confirmed = true; + ch->gr_ram_userd_defers = 0; if (wrap && put < ch->gr_ram_userd_latch_put) pr_info_ratelimited("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch wrap accepted (put %u latch %u ent %u step %u)\n", @@ -44832,9 +44970,59 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) time_after(jiffies, ch->gr_ram_userd_latch_at + msecs_to_jiffies(15000))) { + /* + * CONT.462 (X-DMA latch lifetime, clause + * iii): before this latch is dropped or + * deferred, ALWAYS re-probe the DDX family. + * The client's push-on-demand generations + * allocate a new control block (c131: + * consecutive RAM-scan adopts landed on three + * different pages), so the useful answer to a + * stuck put is often "the block moved", not + * "the latch is stale". If the content scan + * has just validated a live block, switch to + * it in place; its own gates (near-empty page, + * ring-entry validation, PUT-step liveness) + * are the ownership proof. + */ + bool served = ch->submissions != + ch->gr_ram_userd_latch_sub; + + if (svc_userd_ddx_reprobe(vgpu, ch, ring_pa, + ring_ap, &put, &get)) { + /* + * Switched to a fresh validated block + * (the helper logged and re-seeded the + * latch; put/get now carry it). + */ + } else if (!moved && served && + svc_userd_ddx_family(ch) && + ch->gr_ram_userd_defers < 32) { + /* + * Clause (i): this latch HAS delivered + * decoded entries (submissions advanced + * since adoption) and the client has + * not contradicted it (no decrease). + * The DDX's generations are 11-24 s + * apart; keep the proven block serving + * instead of dropping it at the fixed + * 15 s no-advance bound. Bounded: 32 + * deferrals (~8 min) then the old reject + * path resumes for a dead latch. + */ + ch->gr_ram_userd_defers++; + ch->gr_ram_userd_latch_at = jiffies; + pr_info_ratelimited("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch DEFERRED (served; put=%u latch=%u; staying latched %u/32)\n", + ch->h_object, ch->userd_pa, put, + ch->gr_ram_userd_latch_put, + ch->gr_ram_userd_defers); + } else { unsigned long lflags; int li; + if (svc_userd_ddx_family(ch)) + ch->gr_ram_userd_reject_at = jiffies; + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch REJECTED (put=%u latch=%u%s)\n", ch->h_object, ch->userd_pa, put, ch->gr_ram_userd_latch_put, @@ -44886,6 +45074,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_ram_userd_reject_pa; svc.chan[li].gr_ram_userd_reject_put = ch->gr_ram_userd_reject_put; + svc.chan[li].gr_ram_userd_reject_at = + ch->gr_ram_userd_reject_at; svc.chan[li].gr_ram_userd_rejects = ch->gr_ram_userd_rejects; break; @@ -44893,6 +45083,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) spin_unlock_irqrestore(&svc.chan_lock, lflags); continue; } + } } if (put >= ch->gpfifo_entries) { u32 put_raw = put; @@ -47750,6 +47941,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_ram_userd_latch_put; svc.chan[k].gr_ram_userd_latch_at = ch->gr_ram_userd_latch_at; + svc.chan[k].gr_ram_userd_latch_sub = + ch->gr_ram_userd_latch_sub; + svc.chan[k].gr_ram_userd_defers = + ch->gr_ram_userd_defers; + svc.chan[k].gr_ram_userd_reject_at = + ch->gr_ram_userd_reject_at; + svc.chan[k].gr_ram_userd_reprobe_at = + ch->gr_ram_userd_reprobe_at; svc.chan[k].gr_ram_userd_confirmed = ch->gr_ram_userd_confirmed; /* CONT.407: keep-across-put-reset state. */ |