diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 270 |
1 files changed, 257 insertions, 13 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 0fc7e69..7b629ed 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2684,6 +2684,38 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) !svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; + /* + * r1774 (V9): the count above is creation order among + * LIVE handles, so a channel whose handle sorts after a + * long-lived one collides with it (r1773 c31: the + * direct-GL GR channel 0xbeef0100 and the live kick + * channel 0xbfef0002 both got rank 2). The rank keys + * the one-shot ring-done bits and the autopin/scan + * gates, so the collision silently disabled the + * content-scan backup for the GL ring. If the counted + * rank is already held by a live GR channel, take the + * smallest unused rank instead (identical result in + * the normal creation-order case). + */ + { + u32 used = 0; + u8 r2; + + for (k = 0; k < SVC_MAX_CHANNELS; k++) + if (svc.chan[k].valid && + svc.chan[k].vgpu == desc->vgpu && + !svc.chan[k].engine_type && + svc.chan[k].gr_index < 32) + used |= 1u << + svc.chan[k].gr_index; + if (used & (1u << rank)) { + for (r2 = 0; r2 < 32; r2++) + if (!(used & (1u << r2))) + break; + if (r2 < 32) + rank = r2; + } + } ch->gr_index = rank; } ch->valid = true; @@ -3033,6 +3065,53 @@ void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, return; } } + /* + * r1774 (V10): the table is 16 slots and the direct-GL client rebuilds + * its page directory every few seconds (r1774 c1: 19 distinct pdbs for + * VAS 0xbeef0300 alone, 26 distinct parks overall). Without eviction + * the first 16 parks filled the table and every FRESH pdb - the one + * that maps the current ring - was silently dropped; once the + * registered directory went stale the promote walk had nothing left + * to try ("cand ... walk rc=-2", "NOT mapped by any registered VAS") + * and the client stalled. Evict the OLDEST candidate of the same VAS + * (its directory is the one being rebuilt); if the VAS has no slot + * (all 16 belong to other VASes), evict the globally oldest one. + */ + { + int victim = -1; + unsigned long oldest = 0; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (!svc_vas_cand[i].valid || + svc_vas_cand[i].h_vaspace != h_vaspace) + continue; + if (!oldest || + time_before(svc_vas_cand[i].stamp, oldest)) { + oldest = svc_vas_cand[i].stamp; + victim = i; + } + } + if (victim < 0) + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (!svc_vas_cand[i].valid) + continue; + if (!oldest || + time_before(svc_vas_cand[i].stamp, + oldest)) { + oldest = svc_vas_cand[i].stamp; + victim = i; + } + } + if (victim >= 0) { + svc_vas_cand[victim].valid = true; + svc_vas_cand[victim].h_vaspace = h_vaspace; + svc_vas_cand[victim].pdb = pdb; + svc_vas_cand[victim].entries = entries; + svc_vas_cand[victim].vidmem = vidmem; + svc_vas_cand[victim].aged_out = false; + svc_vas_cand[victim].stamp = jiffies; + } + } } /* @@ -9684,6 +9763,23 @@ MODULE_PARM_DESC(sema_kick_notify, static u32 svc_kick_log; /* bounded kick-notify witness */ static u64 svc_kick_count; /* executor invocations */ /* + * r1772 (IDLE-DMA-RCA F2, from the r1771 c18 stall): an ADOPTED + * kick-family USERD latch is not yet CONFIRMED (both adoption paths + * seed confirmed=false), so the CONT.407 put=0 keep below did not + * cover it: c18 showed "kick-family ADOPT on PUT step (put 2 -> 3)" + * at [42774.3] then "speculative latch REJECTED (put=0 latch=3)" 7 s + * later, and the kick lane authored nothing for the next 78 s. The + * ADOPT itself required a ring-entry-validated, positive PUT step + * (the same evidence the confirmed path relies on), so an unconfirmed + * latch is kept across a transient put=0 too; the keeps<64 bound + * retains the churn protection. sema_kick_latch_keep=0 restores the + * pre-r1772 condition for A/B. + */ +static bool gsp_sema_kick_latch_keep = true; +module_param_named(sema_kick_latch_keep, gsp_sema_kick_latch_keep, bool, 0644); +MODULE_PARM_DESC(sema_kick_latch_keep, + "Keep an adopted-but-unconfirmed kick-family USERD latch across a transient put=0 (CONT.407 extension; default 1)"); +/* * r982: the notify page a pending-marker hit proved, per knowledge the * exec picked up. Kept in the service's own scope (not the channel * snapshot): the kick channel is the only consumer and the entry is @@ -30407,11 +30503,96 @@ static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, * Anchor on the channel's own gpfifo_va and its entry 0 content instead. */ static unsigned long svc_promote_diag_at; +/* + * r1772 (IDLE-DMA-RCA F3): budget for the "RING-VAS ... mapped by VAS" + * success print in the probe loop below. During the r1771 c18 stall it + * repeated unbudgeted at ~12 Hz (1,034 lines) on the same service thread + * that must author the kick/idle lane. + */ +static unsigned long svc_ringvas_diag_at; +/* + * r1773 (IDLE-DMA-RCA F3 extension): the parked-candidate walk probe + * (`VAS-PROMOTE ... cand ... walk rc=`) was UNBUDGETED for the beef family. + * During the GZDoom V_Init stall the direct-GL client channel's ring VA + * (low arena, e.g. gpfifo 0x42dc000) failed the candidate walk against four + * parked directories every ~40 ms - 100+ pr_info/s plus the walks on the + * single service thread that must also author the kick/idle lane. Budget + * the print to 2 s and back the whole probe off to 250 ms after a failed + * sweep (reset on any promotion). sema_vas_promote_backoff=0 restores the + * pre-fix behaviour for A/B. + */ +static bool gsp_sema_vas_promote_backoff = true; +module_param_named(sema_vas_promote_backoff, gsp_sema_vas_promote_backoff, + bool, 0644); +MODULE_PARM_DESC(sema_vas_promote_backoff, + "Back off the beef-family VAS ring-promote walk after a failed sweep (default 1)"); +static unsigned long svc_vaspromote_diag_at; +static unsigned long svc_vaspromote_backoff_at; +/* + * r1774 (V9): the failed-sweep backoff above was a SINGLE shared stamp, so + * whichever beef-family channel swept first re-armed it every pass and + * starved its siblings (r1773 c31: 0xbeef0101 swept 92 times, 0xbeef0100 + * only 8 - the GR channel whose ring the GLX lane needs). Remember the + * channel that failed and gate only that one; a different beef channel + * always gets its own sweep. + */ +static u32 svc_vaspromote_backoff_h; +/* + * r1774 (V9): when 0, restore the pre-V9 rule where a beef-family channel + * whose VAS is registered VERIFIED skips the ring walk (A/B knob). When 1 + * (default) the direct-GL family always re-walks the ring VA: the client + * rebuilds its page directory every few seconds and the mapping for the + * low-arena ring appears IN the directory only after the client populates + * it (r1773 inc A: the same pdb answered zeros at +3.3 s and the live + * ring pair at +6.6 s), so a verified-but-stale directory must not end the + * probe for this family. + */ +static bool gsp_sema_vas_client_ring = true; +module_param_named(sema_vas_client_ring, gsp_sema_vas_client_ring, bool, 0644); +MODULE_PARM_DESC(sema_vas_client_ring, + "Always re-probe the direct-GL client's low-arena ring VA (default 1; 0 = legacy verified-skip)"); +/* + * r1774 (V9): per-channel rate limit for the direct-GL always-probe so the + * re-walk (up to 16 parked candidates + every registered VAS) runs at most + * ~10/s per channel instead of every service pass; 100 ms still catches the + * client's late ring mapping well inside the DDX's 5 s idle window, and a + * successful probe re-pins the ring so the pump's svc_lookup is served from + * the pin until the next probe. + */ +#define SVC_VASPROMOTE_RL 8 +static struct { + u32 h; + unsigned long at; +} svc_vaspromote_rl[SVC_VASPROMOTE_RL]; + +static bool svc_vaspromote_rl_allow(u32 h) +{ + unsigned long now = jiffies; + int i, free_i = -1; + + for (i = 0; i < SVC_VASPROMOTE_RL; i++) { + if (svc_vaspromote_rl[i].h == h) { + if (time_before(now, svc_vaspromote_rl[i].at)) + return false; + svc_vaspromote_rl[i].at = now + + msecs_to_jiffies(100); + return true; + } + if (!svc_vaspromote_rl[i].h && free_i < 0) + free_i = i; + } + if (free_i >= 0) { + svc_vaspromote_rl[free_i].h = h; + svc_vaspromote_rl[free_i].at = now + msecs_to_jiffies(100); + } + return true; +} static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u32 h = ch->h_vaspace; + bool beef = ((ch->h_object & 0xfffff000u) == 0xbeef0000u); int i; /* @@ -30431,7 +30612,7 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, */ if (!h || !ch->gpfifo_va || ch->gpfifo_entries < 2) { /* CONT.400 diag: name the exact early-return reason. */ - if ((ch->h_object & 0xfffff000u) == 0xbeef0000u && + if (beef && time_after(jiffies, svc_promote_diag_at)) { svc_promote_diag_at = jiffies + msecs_to_jiffies(2000); pr_info("sema-svc: VAS-PROMOTE h=0x%x skip (h=0x%x gpfifo=0x%llx entries=%u)\n", @@ -30445,9 +30626,15 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, * CONT.401: see svc_vaspace_stale_candidate(). Only a verified * directory with no contradicting parked candidate lets us * skip the content walk; a rebuilt one is demoted first. + * + * r1774 (V9): the direct-GL family is exempted (see the + * sema_vas_client_ring comment) - its directory churns and + * the ring mapping lands late, so "verified" is not a + * sufficient proof that the ring VA is resolvable NOW. */ - if (!svc_vaspace_stale_candidate(h)) { - if ((ch->h_object & 0xfffff000u) == 0xbeef0000u && + if (!svc_vaspace_stale_candidate(h) && + !(beef && gsp_sema_vas_client_ring)) { + if (beef && time_after(jiffies, svc_promote_diag_at)) { svc_promote_diag_at = jiffies + msecs_to_jiffies(2000); @@ -30457,10 +30644,29 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, } return; } - pr_info("sema-svc: VAS 0x%x verified directory is stale (rebuilt pdb parked); demoting for re-validation\n", - h); - svc_vaspace_demote(h); + if (svc_vaspace_stale_candidate(h)) { + pr_info("sema-svc: VAS 0x%x verified directory is stale (rebuilt pdb parked); demoting for re-validation\n", + h); + svc_vaspace_demote(h); + } } + /* + * r1773: bound the failed-sweep storm for the beef family (see the + * backoff_at comment above). A promotion (below) clears the window. + * r1774: per-channel (see svc_vaspromote_backoff_h). + */ + if (gsp_sema_vas_promote_backoff && beef && + ch->h_object == svc_vaspromote_backoff_h && + time_before(jiffies, svc_vaspromote_backoff_at)) + return; + /* + * r1774 (V9): rate-limit the direct-GL always-probe (see the + * svc_vaspromote_rl comment). Non-beef channels keep the legacy + * per-pass probe cadence. + */ + if (beef && gsp_sema_vas_client_ring && + !svc_vaspromote_rl_allow(ch->h_object)) + return; for (i = 0; i < SVC_VAS_CAND_MAX; i++) { u64 wpa = 0; int wap = SVC_APERTURE_SYS; @@ -30479,12 +30685,18 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, : SVC_APERTURE_SYS, &wpa, &wap, false); if (wrc) { - /* CONT.400 diag: why the beef ring walk misses. */ - if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) + /* CONT.400 diag: why the beef ring walk misses. + * r1773: budgeted (2 s) - unbudgeted this printed + * 100+/s during the GZDoom V_Init stall. */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u && + time_after(jiffies, svc_vaspromote_diag_at)) { + svc_vaspromote_diag_at = jiffies + + msecs_to_jiffies(2000); pr_info("sema-svc: VAS-PROMOTE h=0x%x cand pdb=0x%llx entries=%u walk rc=%d (gpfifo 0x%llx)\n", ch->h_object, svc_vas_cand[i].pdb, svc_vas_cand[i].entries, wrc, ch->gpfifo_va); + } continue; } if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) @@ -30509,10 +30721,23 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, d.verified = true; /* CONT.400: content-proven ring walk */ gsp_sema_service_vaspace_add(&d); svc_vas_cand[i].valid = false; + svc_vaspromote_backoff_at = 0; /* r1773: promotion clears the backoff */ + svc_vaspromote_backoff_h = 0; /* r1774 (V9) */ pr_info("sema-svc: VAS 0x%x registered at channel 0x%x turn (parked candidate content-validated: gpfifo 0x%llx entry0 va=0x%llx len=%u)\n", h, ch->h_object, ch->gpfifo_va, eva, len); return; } + /* + * r1773: no parked candidate mapped the ring this sweep; back the + * beef-family probe off for 250 ms so a persistently-unresolvable + * client ring cannot re-walk + re-log every service pass (the V_Init + * stall). r1774 (V9): only THIS channel backs off - the shared stamp + * let one failing sibling starve the GR channel's probe. + */ + if (gsp_sema_vas_promote_backoff && beef) { + svc_vaspromote_backoff_h = ch->h_object; + svc_vaspromote_backoff_at = jiffies + msecs_to_jiffies(250); + } /* * CONT.400: the parked candidate is not always the directory that maps @@ -30552,14 +30777,24 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, elen = ((e1 >> 10) & 0x1fffff) << 2; if (eva && elen >= 4 && elen <= 0x10000 && svc_gr_entry_win_ok(ch->gpfifo_va, eva)) { - pr_info("sema-svc: RING-VAS h=0x%x gpfifo=0x%llx mapped by VAS 0x%x (pdb=0x%llx ap=%d) -> pa=0x%llx entry0 va=0x%llx len=%u\n", - ch->h_object, ch->gpfifo_va, - snap.h_vaspace, snap.pdb, - snap.ap, wpa, eva, elen); + if (time_after(jiffies, + svc_ringvas_diag_at)) { + svc_ringvas_diag_at = jiffies + + msecs_to_jiffies(2000); + pr_info("sema-svc: RING-VAS h=0x%x gpfifo=0x%llx mapped by VAS 0x%x (pdb=0x%llx ap=%d) -> pa=0x%llx entry0 va=0x%llx len=%u\n", + ch->h_object, + ch->gpfifo_va, + snap.h_vaspace, + snap.pdb, + snap.ap, wpa, eva, + elen); + } /* Pin the ring through this directory. */ svc_pin_store(ch->gpfifo_va, wpa, wap); svc_lat_store(ch->gpfifo_va, wpa, wap); svc_cur_vas = snap.h_vaspace; + svc_vaspromote_backoff_at = 0; /* r1773 */ + svc_vaspromote_backoff_h = 0; /* r1774 (V9) */ return; } } @@ -42807,7 +43042,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", ch->h_object, ch->userd_pa, ch->gr_ram_userd_latch_put, put); - } else if (kfam && ch->gr_ram_userd_confirmed && + } else if (kfam && (gsp_sema_kick_latch_keep || + ch->gr_ram_userd_confirmed) && put == 0 && ch->gr_ram_userd_latch_put != 0 && ch->gr_ram_userd_keeps < 64) { @@ -42821,6 +43057,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * here 7-8 s before the frame loop's next * kick. Bounded keeps keep a truly churning * zeroing block on the old reject path. + * + * r1772 (IDLE-DMA-RCA F2): also keep an + * ADOPTED-but-unconfirmed latch (r1771 c18: + * ADOPT put 2->3 at [42774.3], REJECTED put=0 + * latch=3 7 s later, kick lane dark 78 s + * after; the ADOPT itself was ring-entry- + * validated). sema_kick_latch_keep=0 restores + * the pre-fix condition for A/B. */ ch->gr_ram_userd_keeps++; if (ch->gr_ram_userd_keep_log < 8) { |