diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 312 |
1 files changed, 215 insertions, 97 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 077eb2b..13caf9a 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -7401,25 +7401,29 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, * gpfifo_va), entry0 at page offset 0 pointing at that push VA with * the same length; the USERD follows at +0x2000. * - * One 64 KB-chunked pass over the registered guest RAM records up to - * SVC_GR_CONTENT_CANDS release-group (push) and ring-entry candidates per - * rank. When a rank has a push candidate but no guest-RAM ring candidate, - * the pass also scans the emulated VRAM mapping (vgpu->bars[1].vaddr) for - * that rank's entry0: FORENSICS-grv1 proved the GR ring lives in the - * vidmem aperture on this guest (the 4 GiB guest-RAM scan found all eight - * pushes but no page-aligned ring entry0; svc_read() cannot address FB), - * and m104's green ring was FB `0x140000 + R*0x3000`. A (ring, push) pair - * is accepted only when the whole gate passes: entry0 bytes exact (push VA - * + length), the push page parses start to end as a method stream, the - * release group sits exactly at len-20 with this rank's slot and payload 1, - * and the USERD reads back a plausible get/put pair (or a readable idle - * 0/0) through the ring's own aperture. Only then are the ring VA and the - * push VA page pinned (the ring in its aperture as found, the push in SYS); - * a wrong or stale candidate pins nothing. Bounded: one pass per 2 s while - * a registered rank 0..7 has no translation, SVC_GR_CONTENT_PASS_MAX passes - * per epoch, and the epoch re-arms when a channel-registration batch - * arrives (the guest recreates channels on every context retry), so the - * discovery can never permanently retire while the guest is still trying. + * Each pass scans the emulated VRAM mapping (vgpu->bars[1].vaddr) FIRST for + * ring-entry candidates - direct host reads, milliseconds - then pairs them + * with RAM push candidates from this or a previous pass; only if a needed + * push is still missing does the pass walk guest RAM (64 KB chunks, + * descending so the high-RAM work allocations are reached immediately, with + * an early exit once every ring-carrying rank has a push). FORENSICS-grv1 + * proved the GR ring lives in the vidmem aperture on this guest (the 4 GiB + * guest-RAM scan found all eight pushes but no page-aligned ring entry0; + * svc_read() cannot address FB), and m104's green ring was FB + * `0x140000 + R*0x3000`. A (ring, push) pair is accepted only when the whole + * gate passes: entry0 bytes exact (push VA + length), the push page parses + * start to end as a method stream, the release group sits exactly at len-20 + * with this rank's slot and payload 1, and the USERD reads back a plausible + * get/put pair (or a readable idle 0/0) through the ring's own aperture. + * Only then are the ring VA and the push VA page pinned (the ring in its + * aperture as found, the push in SYS); a wrong or stale candidate pins + * nothing. Bounded: one pass per 2 s while a registered rank 0..7 has no + * translation, SVC_GR_CONTENT_PASS_MAX passes per epoch with a + * SVC_GR_CONTENT_COOLDOWN rest after a full epoch, and an epoch starts + * immediately on a channel-registration batch. Taking the cap therefore + * never retires discovery while the guest is still trying (FORENSICS-grv3 + * sections 4-5A); it stops - with one log line - only when every registered + * GR channel has a ring. */ #define SVC_GR_CONTENT_RANKS 8 #define SVC_GR_PUSH_VA_BASE 0x200400000ULL @@ -7438,6 +7442,13 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, * is simply not content-discoverable and waits for a structural source. */ #define SVC_GR_CONTENT_FB_LIMIT 0x20000000ULL +/* + * Rest after a full discovery epoch (FORENSICS-grv3 sections 4-5A): the pass + * cap must bound the scanning rate, not retire discovery. When the epoch is + * full the pass count resets and discovery rests this long before starting a + * fresh epoch, for as long as a registered GR channel still lacks a ring. + */ +#define SVC_GR_CONTENT_COOLDOWN (30 * HZ) struct svc_gr_content_cand { u64 pa_page; @@ -7452,7 +7463,9 @@ static struct svc_gr_content_cand static u8 svc_gr_content_nring[SVC_GR_CONTENT_RANKS]; static u8 svc_gr_content_npush[SVC_GR_CONTENT_RANKS]; static unsigned long svc_gr_content_last; +static unsigned long svc_gr_content_cooldown; static u32 svc_gr_content_passes; +static bool svc_gr_content_active; /* a pass ran while rings were missing */ /* * A ring page's bytes after entry0 are either zero (a fresh ring with @@ -7571,6 +7584,69 @@ static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank, return true; } +/* + * True when every rank that has a ring candidate also has a push candidate + * (all inputs of the pairing loop are present) and at least one such rank + * exists. The ring table is rebuilt every pass while the push table is kept + * across passes (FORENSICS-grv3 section 5A), so after the first pass this is + * the fast-path condition that lets a newly appeared FB ring pair without + * re-paying the 6 GiB RAM walk. + */ +static bool svc_gr_content_pushes_ready(const u64 *gva) +{ + bool any = false; + u32 r; + + for (r = 0; r < SVC_GR_CONTENT_RANKS; r++) { + if (!gva[r] || !svc_gr_content_nring[r]) + continue; + any = true; + if (!svc_gr_content_npush[r]) + return false; + } + return any; +} + +/* + * Pair every recorded ring candidate with a recorded push candidate by + * length and run the full content gate (svc_gr_content_pin). Returns the + * number of pins installed by this call. Candidates that fail the gate pin + * nothing, so a wrong or stale pair costs a rejected validation, no write. + */ +static u32 svc_gr_content_pair(struct nv_vgpu_device *vgpu, const u64 *gva, + const u32 *ent) +{ + u32 pinned = 0; + u32 i, a, b; + + for (i = 0; i < SVC_GR_CONTENT_RANKS; i++) { + u32 nr = svc_gr_content_nring[i]; + u32 np = svc_gr_content_npush[i]; + + if (!gva[i] || !nr || !np) + continue; + for (a = 0; a < nr; a++) { + for (b = 0; b < np; b++) { + if (svc_gr_content_ring[i][a].len != + svc_gr_content_push[i][b].len) + continue; + if (!svc_gr_content_pin( + vgpu, i, gva[i], ent[i], + svc_gr_content_ring[i][a].pa_page, + svc_gr_content_ring[i][a].ap, + svc_gr_content_push[i][b].pa_page, + svc_gr_content_ring[i][a].len)) + continue; + pinned++; + break; + } + if (b < np) + break; + } + } + return pinned; +} + static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) { u64 gva[SVC_GR_CONTENT_RANKS] = { 0 }; @@ -7593,6 +7669,7 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) if (READ_ONCE(svc_gr_content_rearm)) { WRITE_ONCE(svc_gr_content_rearm, false); svc_gr_content_passes = 0; + svc_gr_content_cooldown = 0; } spin_lock_irqsave(&svc.chan_lock, flags); @@ -7616,30 +7693,136 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) need = true; } spin_unlock_irqrestore(&svc.chan_lock, flags); - if (!need) + if (!need) { + /* + * Every registered GR channel has a ring: stop scanning and + * say so once (FORENSICS-grv3 section 5A "stop, log once"), + * so a healthy boot stays quiet after the first scan burst. + * The flag only suppresses spam, never a future need: new + * channels set it active again on the next pass. + */ + if (svc_gr_content_active) { + svc_gr_content_active = false; + pr_info("sema-chan: GR content discovery idle: every registered GR channel has a ring\n"); + } return; + } + /* + * Bounded cadence (FORENSICS-grv3 sections 4-5A): a full epoch is + * SVC_GR_CONTENT_PASS_MAX passes at one pass per 2 s. Taking the cap + * used to retire discovery for the rest of the boot - grv3 capped at + * ~4189 and never scanned again while the guest lived ~8 more + * minutes, so any late ring submission was invisible. Reset the pass + * count and rest SVC_GR_CONTENT_COOLDOWN, then start a fresh epoch + * while rings are still missing; registration re-arms immediately + * (above). + */ if (svc_gr_content_passes >= SVC_GR_CONTENT_PASS_MAX) { - pr_info_ratelimited("sema-chan: GR content discovery capped (%u passes, no validated ring; re-arms on the next channel registration)\n", - svc_gr_content_passes); + svc_gr_content_passes = 0; + svc_gr_content_cooldown = jiffies + SVC_GR_CONTENT_COOLDOWN; + pr_info_ratelimited("sema-chan: GR content discovery epoch full (%u passes); cooling down %lus, then re-arming while rings are missing\n", + (unsigned)SVC_GR_CONTENT_PASS_MAX, + (unsigned long)(SVC_GR_CONTENT_COOLDOWN / HZ)); return; } + if (time_before(jiffies, svc_gr_content_cooldown)) + return; if (time_before(jiffies, svc_gr_content_last + 2 * HZ)) return; svc_gr_content_last = jiffies; svc_gr_content_passes++; + svc_gr_content_active = true; + + /* + * Ring candidates are rebuilt every pass (ring content can move); + * push candidates are kept across passes (FORENSICS-grv3 section + * 5A): once a pass has found a rank's staged ctx-init push, a later + * pass can pair a newly appeared FB ring immediately instead of + * re-paying the 6 GiB RAM walk first. The pairing gate re-validates + * every candidate, so a stale entry can only cost a rejected pair. + */ memset(svc_gr_content_ring, 0, sizeof(svc_gr_content_ring)); - memset(svc_gr_content_push, 0, sizeof(svc_gr_content_push)); memset(svc_gr_content_nring, 0, sizeof(svc_gr_content_nring)); - memset(svc_gr_content_npush, 0, sizeof(svc_gr_content_npush)); + /* + * FB aperture scan goes first (FORENSICS-grv3 section 5A): the GR + * ring lives in the emulated vidmem on this guest and this scan is a + * direct host read (bounded to SVC_GR_CONTENT_FB_LIMIT at a page + * stride, milliseconds), so a ring that appears - even late - is + * found at the very start of the pass instead of after the 6 GiB RAM + * walk (grv3's first pin was ~3.5 s late because FB ran last). The + * filter matches the RAM scan, and the candidate is still only a + * candidate - the full gate in svc_gr_content_pin decides. No push + * candidate is required to RECORD a ring; pairing needs one and the + * persistent push table (or the RAM walk below) supplies it. The ring + * chunk starts at a page boundary (m104: FB 0x140000 + R*0x3000) and + * entry0 must sit at the pinned page's offset 0 for the page-granular + * pin to be correct. + */ + if (vgpu->bars[1].vaddr && vgpu->bars[1].size) { + const u8 *fb = (const u8 *)vgpu->bars[1].vaddr; + u64 lim = min_t(u64, vgpu->bars[1].size, + SVC_GR_CONTENT_FB_LIMIT); + u64 off; + + for (off = 0; off + 0x40 <= lim; off += SEMA_PAGE_SIZE) { + u32 w0, e1, r, va_hi, len; + + memcpy(&w0, fb + off, 4); + if ((w0 & 0x3fffff) != 0 || w0 < 0x00400000 || + w0 > 0x02000000) + continue; + r = (w0 - 0x00400000) / 0x400000; + if (r >= SVC_GR_CONTENT_RANKS || !gva[r]) + continue; + if (svc_gr_content_nring[r]) + continue; /* first hit for this rank */ + memcpy(&e1, fb + off + 4, 4); + va_hi = e1 & 0xff; + len = ((e1 >> 10) & 0x1fffff) << 2; + if (va_hi != 2 || (len & 3) || len < 4 || + len > SEMA_PAGE_SIZE) + continue; + if (!svc_gr_ring_tail_ok(fb + off + 8, 0x38)) + continue; + svc_gr_content_record(svc_gr_content_ring[r], + &svc_gr_content_nring[r], + off, len, SVC_APERTURE_FB); + } + } + + /* + * Fast path: every ring candidate's rank already has a push + * candidate (this pass's FB scan plus the persistent push table), so + * pair now and skip the RAM walk entirely. This is what makes a late + * ring submission visible within one 2 s pass instead of after a + * full re-scan. + */ + if (svc_gr_content_pushes_ready(gva)) { + svc_gr_content_pair(vgpu, gva, ent); + return; + } + + /* + * Push (release-group) scan over guest RAM, descending (FORENSICS-grv3 + * section 5A; the same reasoning as the CE content rescue in + * svc_rescue_push_page): the guest's work allocations live in the high + * RAM block (grv3 pushes at 0x1028f...0x12df8..., 4.0-4.8 GiB), so a + * descending walk reaches them within the first chunk or two of the + * valid high range while an ascending walk from 0 needs ~4 GiB. The + * walk stops as soon as every ring-carrying rank has a push candidate, + * so the pass ends in about a second instead of a full 6 GiB walk. + * RAM ring candidates (the session-2 path) are still recorded here for + * boots whose ring is not in FB. + */ buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + for (gpa = SEMA_PA_MAX - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) - continue; + goto next_chunk; for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) { u32 w0; @@ -7695,79 +7878,14 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) SVC_APERTURE_SYS); } } +next_chunk: + if (svc_gr_content_pushes_ready(gva)) + break; + if (!gpa) + break; } kfree(buf); - - /* - * FB aperture fallback (FORENSICS-grv1): on this guest the GR ring is - * in the emulated vidmem, which svc_read() cannot address (it resolves - * the guest DMA space only), while the pushes stay in RAM. For every - * rank that has a RAM push candidate but no guest-RAM ring candidate, - * scan the emulated VRAM mapping for its page-aligned entry0. The same - * filters as the RAM scan apply and the candidate is still only a - * candidate - the full gate in svc_gr_content_pin decides. Bounded to - * SVC_GR_CONTENT_FB_LIMIT at a page stride: the ring chunk starts at a - * page boundary (m104: FB 0x140000 + R*0x3000) and entry0 must sit at - * the pinned page's offset 0 for the page-granular pin to be correct. - */ - if (vgpu->bars[1].vaddr && vgpu->bars[1].size) { - const u8 *fb = (const u8 *)vgpu->bars[1].vaddr; - u64 lim = min_t(u64, vgpu->bars[1].size, - SVC_GR_CONTENT_FB_LIMIT); - u64 off; - - for (off = 0; off + 0x40 <= lim; off += SEMA_PAGE_SIZE) { - u32 w0, e1, r, va_hi, len; - - memcpy(&w0, fb + off, 4); - if ((w0 & 0x3fffff) != 0 || w0 < 0x00400000 || - w0 > 0x02000000) - continue; - r = (w0 - 0x00400000) / 0x400000; - if (r >= SVC_GR_CONTENT_RANKS || !gva[r]) - continue; - if (svc_gr_content_nring[r] || - !svc_gr_content_npush[r]) - continue; /* RAM ring candidate exists */ - memcpy(&e1, fb + off + 4, 4); - va_hi = e1 & 0xff; - len = ((e1 >> 10) & 0x1fffff) << 2; - if (va_hi != 2 || (len & 3) || len < 4 || - len > SEMA_PAGE_SIZE) - continue; - if (!svc_gr_ring_tail_ok(fb + off + 8, 0x38)) - continue; - svc_gr_content_record(svc_gr_content_ring[r], - &svc_gr_content_nring[r], - off, len, SVC_APERTURE_FB); - } - } - - for (i = 0; i < SVC_GR_CONTENT_RANKS; i++) { - u32 nr = svc_gr_content_nring[i]; - u32 np = svc_gr_content_npush[i]; - u32 a, b; - - if (!gva[i] || !nr || !np) - continue; - for (a = 0; a < nr; a++) { - for (b = 0; b < np; b++) { - if (svc_gr_content_ring[i][a].len != - svc_gr_content_push[i][b].len) - continue; - if (!svc_gr_content_pin( - vgpu, i, gva[i], ent[i], - svc_gr_content_ring[i][a].pa_page, - svc_gr_content_ring[i][a].ap, - svc_gr_content_push[i][b].pa_page, - svc_gr_content_ring[i][a].len)) - continue; - break; - } - if (b < np) - break; - } - } + svc_gr_content_pair(vgpu, gva, ent); } /* |