diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 262 |
1 files changed, 248 insertions, 14 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 45bfdf5..15f728c 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -206,7 +206,14 @@ struct sema_map_entry { #define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ #define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ /* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ -#define SVC_GR_USERD_WATCH 8 +#define SVC_GR_USERD_WATCH 64 +/* + * SESSION 153-CONTINUED-20: a kick-family (h 0xbfef00xx) watched candidate + * whose PUT does not move for this long is a frozen pool lookalike (the + * r972/r973 descriptor-pool class: put=7 forever) - dropped so the ~50 + * frozen entries can never crowd out the real advancing control block. + */ +#define SVC_KICK_WATCH_STALE_MS 20000 /* * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM @@ -612,7 +619,23 @@ struct svc_channel { u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + /* + * SESSION 153-CONTINUED-20: jiffies when the entry was added or last + * stepped; the stale-entry timeout drops a candidate whose PUT has + * not moved for SVC_KICK_WATCH_STALE_MS (the frozen descriptor-pool + * class the kick-family watch exists to defeat). + */ + unsigned long gr_ram_userd_watch_at[SVC_GR_USERD_WATCH]; u8 gr_ram_userd_watch_n; + u32 gr_ram_userd_watch_log; /* bounded add-log counter */ + /* + * SESSION 153-CONTINUED-20: bounded budget re-arms for the + * kick-family watch sweep. When every watch entry has gone stale + * (frozen pool) and the slice budget is spent, one re-arm keeps the + * sweep alive so the real block can still be found; capped like the + * reject counter so a dead channel stops sweeping in bounded time. + */ + u8 gr_ram_userd_kick_sweeps; u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ /* * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The @@ -7879,6 +7902,32 @@ static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, if (!svc_lookup_trusted(vgpu, va, &pa, &ap)) { int r1, r2; + /* + * SESSION 153-CONTINUED-20: prefer the client-visible sysmem + * mapping. The target is the completion word the client's + * flush loop polls through its own memfd; r974/r975 measured + * the serviced-channel VAS walk answering this VA with the FB + * shadow (ap=2, pa=0x20e0b0) - a write no client can observe + * (r973's SYS answer, pa=0x12ae340b0, was the readable one, + * and the ch=NULL dev probe on r975 re-resolved the same VA + * to SYS pa=0x104b640b0 = the kick ring page + 0x1000 + 0xb0, + * exactly the r973 ring-adjacent layout). When the biased + * chain lands in FB, re-resolve with the serviced-channel VAS + * bias cleared and take the sysmem answer when it exists. + */ + if (svc_ap_of(ap) == SVC_APERTURE_FB) { + u64 pa2 = 0; + int ap2 = SVC_APERTURE_FB; + u32 save = svc_cur_vas; + + svc_cur_vas = 0; + if (!svc_lookup_trusted(vgpu, va, &pa2, &ap2) && + svc_ap_of(ap2) == SVC_APERTURE_SYS) { + pa = pa2; + ap = ap2; + } + svc_cur_vas = save; + } r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw)); if (!r1 && !r2 && @@ -17531,10 +17580,18 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget * gate stays open while candidates are under watch, so the * liveness confirmation cannot be starved by slice accounting. + * SESSION 153-CONTINUED-20: for the kick family it also stays open + * while the bounded sweep re-arm budget lasts - the watch set can + * go empty for stretches (all frozen pool entries timed out), and + * the in-scan re-arm needs a next call to re-run the sweep that + * finds the advancing block. */ if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok && (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || - ch->gr_ram_userd_watch_n)) { + ch->gr_ram_userd_watch_n || + ((ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) && + ch->gr_ram_userd_kick_sweeps < 64))) { u64 upa = 0; u32 uput = 0, uget = 0; @@ -19616,10 +19673,47 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } last = ch->gr_ram_userd_watch_put[wi]; if (hp == last) { + /* + * SESSION 153-CONTINUED-20: stale-entry timeout. A + * kick-family watched block whose PUT has not moved + * for the bound is a frozen lookalike (the r972/r973 + * descriptor-pool class, put=7 forever) - drop it so + * the watch set keeps room for the advancing control + * block. Non-kick entries keep the old semantics + * (they are only added by the speculative latch's + * own re-arm today). + */ + if (kick && time_after(jiffies, + ch->gr_ram_userd_watch_at[wi] + + msecs_to_jiffies(SVC_KICK_WATCH_STALE_MS))) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } wi++; continue; } - if (hp != last + 1) { + if (kick && hp < last) { + /* A reset is churn: drop (same class as the jump). */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + if (!kick && hp != last + 1) { /* * SESSION 154: a jump or a reset is the churn * signature (the r916-r919 lookalike pools move by @@ -19634,9 +19728,37 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; ch->gr_ram_userd_watch_hits[wi] = ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; continue; } ch->gr_ram_userd_watch_put[wi] = hp; + ch->gr_ram_userd_watch_at[wi] = jiffies; + /* + * SESSION 153-CONTINUED-20: THE kick-family adoption rule - + * any positive PUT step across passes. The real control block + * is authored by the client's frame loop (measured 140/s), so + * between two sightings its PUT steps by several positions; + * the frozen descriptor-pool entries never step and time out + * above. Do NOT require the +1 step here: for this family a + * free-running put is the LIVE signature, a stall is the + * lookalike. + */ + if (kick) { + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (hg > hp) + continue; /* impossible; keep watching */ + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family ADOPT on PUT step (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, + live_put, live_pa, live_put, live_get); + break; + } /* * SESSION 154: ONE observed +1 step adopts. The live block * advances ~2-8 s per step (measured) while the client's @@ -19693,9 +19815,15 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * latches read 12/8 and 16/16 and every previously * measured lookalike counted 16 or more. Also accept * a free-running put (the old put <= entries bound - * would reject the live block). + * would reject the live block). SESSION 153- + * CONTINUED-20: for the kick family the cap is + * LIFTED to the watch loop's sanity bound (0x100000) + * - its control block is authored by the client's + * frame loop and its PUT free-runs with the frame + * rate; the step-based watch adoption (not the cap) + * is the discriminator there. */ - if (!hp || hp > 8 || hg > hp) + if (!hp || hp > (kick ? 0x100000 : 8) || hg > hp) continue; { u32 z = 0, zi; @@ -19800,6 +19928,64 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, hp == ch->gr_ram_userd_reject_put) continue; /* + * SESSION 153-CONTINUED-20 (watch-adopt-on-PUT- + * step): a kick-family candidate is NOT adopted on + * sight. This family's RAM is littered with frozen + * descriptor-pool lookalikes whose +0x88/+0x8c + * words happen to form the put/get shape (r972/973: + * put=7 frozen, 65 pump rejections in one boot, + * the latch alternating between two pool entries). + * Add the candidate to the watch set (deduped, LRU + * replace when full) and KEEP scanning the slice; + * the watch loop at the top adopts the block only + * when its PUT takes a step across passes, and a + * stale-entry timeout (SVC_KICK_WATCH_STALE_MS) + * evicts frozen entries so the pool can never crowd + * out the real control block. The r971-r973 + * immediate latch is retired for this family. + */ + if (kick) { + u32 wn = ch->gr_ram_userd_watch_n; + bool known = false; + u32 wj; + + for (wj = 0; wj < wn; wj++) + if (ch->gr_ram_userd_watch[wj] == + gpa + o) { + known = true; + break; + } + if (!known) { + if (wn >= SVC_GR_USERD_WATCH) { + u32 old = 0, si; + + for (si = 1; si < wn; si++) + if (time_before( + ch->gr_ram_userd_watch_at[si], + ch->gr_ram_userd_watch_at[old])) + old = si; + wn = old; + } + ch->gr_ram_userd_watch[wn] = gpa + o; + ch->gr_ram_userd_watch_put[wn] = hp; + ch->gr_ram_userd_watch_hits[wn] = 0; + ch->gr_ram_userd_watch_at[wn] = jiffies; + if (wn == ch->gr_ram_userd_watch_n) + ch->gr_ram_userd_watch_n++; + if (ch->gr_ram_userd_watch_log < 48) { + ch->gr_ram_userd_watch_log++; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family watch[%u] pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; adopt on PUT step)\n", + ch->h_object, + ch->gr_index, wn, + gpa + o, hp, hg, + ring_pa, + (long long)(gpa + o - ring_pa) / + (1024 * 1024)); + } + } + continue; /* keep scanning; adopt is deferred */ + } + /* * SESSION 154 (M-GL1.2): a candidate that passed the * shape gate, the entry validation, the page-density * gate and the put cap is latched SPECULATIVELY. The @@ -19817,12 +20003,6 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, live_put = hp; live_get = hg; ch->gr_ram_userd_watch_n = 0; - if (kick) - pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family accept pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; locality gate waived)\n", - ch->h_object, ch->gr_index, live_pa, - live_put, live_get, ring_pa, - (long long)(live_pa - ring_pa) / - (1024 * 1024)); pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put step across passes) -> pa=0x%llx put=%u get=%u\n", ch->h_object, ch->gr_index, live_pa, live_put, live_get); @@ -19833,6 +20013,23 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; ch->gr_ram_userd_passes++; + /* + * SESSION 153-CONTINUED-20: kick-family sweep re-arm. When every + * watched candidate has gone stale (frozen pool) and the slice + * budget is spent, one bounded re-arm keeps the sweep alive so the + * real advancing block can still be found; capped like the reject + * counter so a truly dead channel stops sweeping in bounded time. + */ + if (kick && !live_pa && !ch->gr_ram_userd_watch_n && + ch->gr_ram_userd_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_userd_kick_sweeps < 64) { + ch->gr_ram_userd_kick_sweeps++; + ch->gr_ram_userd_passes = 0; + ch->gr_ram_userd_logged = false; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family sweep re-arm %u/64 (watch empty, budget spent; content scan re-runs)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_userd_kick_sweeps); + } if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && !ch->gr_ram_userd_watch_n) { ch->gr_ram_userd_logged = true; @@ -19865,6 +20062,10 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_rejects; svc.chan[i].gr_ram_userd_watch_n = ch->gr_ram_userd_watch_n; + svc.chan[i].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[i].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { svc.chan[i].gr_ram_userd_watch[wj] = ch->gr_ram_userd_watch[wj]; @@ -19872,6 +20073,8 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_watch_put[wj]; svc.chan[i].gr_ram_userd_watch_hits[wj] = ch->gr_ram_userd_watch_hits[wj]; + svc.chan[i].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; } break; } @@ -30772,6 +30975,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); + /* SESSION 153-CONTINUED-20: the family test shared by the + * kick-confirm relaxation and the churn monitor below. */ + bool kfam = svc_kick_family(ch); + /* * SESSION 154 (M-GL1.2): speculative-latch confirmation for * the RAM USERD. The scan latches a gated candidate right @@ -30782,13 +30989,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * candidate. The live block advances 1->2->3 in 2-10 s * (measured); the client's own release waits are ~11 s, so * the no-advance bound must sit above those. + * + * SESSION 153-CONTINUED-20: for the kick family ANY positive + * PUT step confirms (its put is authored by the client's + * frame loop), only a decrease or the no-advance bound + * rejects - and a CONFIRMED kick latch is not exempt from the + * churn rule: the r974 false confirm (pa=0x11c62e000, put + * walked 1->2->3, then reset to 0/1 and toggled for minutes) + * shows a slowly resetting pool block can pass any + * single-step test. Track its PUT while confirmed and + * unlatch on the first decrease - a client-authored + * free-running put never decreases (the rewind path owns the + * session-reset case). */ + if (kfam && ch->gr_ram_userd_confirmed && + put > ch->gr_ram_userd_latch_put) + ch->gr_ram_userd_latch_put = put; if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok && - !ch->gr_ram_userd_confirmed) { + (!ch->gr_ram_userd_confirmed || + (kfam && put < ch->gr_ram_userd_latch_put))) { + bool stepped; + if (!ch->gr_ram_userd_latch_at) ch->gr_ram_userd_latch_at = jiffies; - if (put == ch->gr_ram_userd_latch_put + 1 || - put == ch->gr_ram_userd_latch_put + 2) { + stepped = kfam ? + put > ch->gr_ram_userd_latch_put : + (put == ch->gr_ram_userd_latch_put + 1 || + put == ch->gr_ram_userd_latch_put + 2); + if (stepped) { ch->gr_ram_userd_confirmed = true; pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", ch->h_object, ch->userd_pa, @@ -33513,6 +33741,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_userd_watch_n = ch->gr_ram_userd_watch_n; + svc.chan[k].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[k].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { svc.chan[k].gr_ram_userd_watch[wj] = @@ -33521,6 +33753,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_ram_userd_watch_put[wj]; svc.chan[k].gr_ram_userd_watch_hits[wj] = ch->gr_ram_userd_watch_hits[wj]; + svc.chan[k].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; } } svc.chan[k].gr_ram_put_seeded = |