diff options
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 48 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 286 |
2 files changed, 308 insertions, 26 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 2f63bcd..99f394c 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -3137,6 +3137,32 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4); memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8); memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8); + /* + * SESSION 153-CONTINUED-38: the USERD control page sits at + * userdMem.address + userdOffset[subDeviceId] (NV_CHANNEL_ALLOC_PARAMS + * userdOffset[NV_MAX_SUBDEVICES], @64; subDeviceId @136, the + * one-hot bit index is normally 0 but the X client-share channel + * carries 1). The GL client's kernel channels declare userdMem fb + * 0x18fe00 with userdOffset[0]=0x7fe00, so the service was + * probing/monitoring 0x18fe00 while the real control page + * (GPGet/GPPut) lives at 0x20fc00 - both the registered candidate + * and every FB observation were at the wrong offset, which is why + * the direct client's channels never resolved a USERD (CONT-37 + * alloc/free retry loop). Index by the subdevice. + */ + { + u32 sd = 0; + u64 uo = 0; + + if (params_size >= 140) + memcpy(&sd, params + 136, 4); + if (sd >= 8) + sd = 0; + if (params_size >= 64 + (sd + 1) * 8) { + memcpy(&uo, params + 64 + (u64)sd * 8, 8); + desc.userd_pa += uo; + } + } memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM + GSPRM_CHAN_MEM_APERTURE, 4); memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8); @@ -3161,6 +3187,28 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, desc.notifier_pa, desc.notifier_as); } } + /* + * SESSION 153-CONTINUED-38: name the per-subdevice userdOffset (the + * kernel channels' control page may sit inside the declared userdMem + * at a nonzero offset; the service currently assumes address+0). + * Bounded. + */ + { + static u32 uo_log; + + if (uo_log < 8 && params_size >= 80) { + u64 uo0 = 0; + u32 sd = 0; + + memcpy(&uo0, params + 64, 8); + if (params_size >= 140) + memcpy(&sd, params + 136, 4); + uo_log++; + pr_info("vgsp-rm: chan userdOff[0]=0x%llx subDevId=0x%x hObject=0x%x (userd memdesc=0x%llx/%u)\n", + uo0, sd, h_object, desc.userd_pa, + desc.userd_as); + } + } gsp_sema_service_channel_add(&desc); } diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index bc04e25..34dda47 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -12092,6 +12092,14 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, static struct { u64 va_page; u64 pa_page; + /* + * SESSION 153-CONTINUED-38: the VAS the page was derived for. The + * counter VA window (0x20400fXXX) is a fixed per-client layout, so + * two clients (the RC scrubber and the DDX) register the SAME VA and + * a VA-keyed cache/derive hands the second one the first one's page; + * that is the slot-ownership collision CONT-37 measured. 0 = any. + */ + u32 h_vas; } svc_cnt_der[SVC_CNT_DER_MAX]; static u32 svc_cnt_der_n; static u32 svc_cnt_der_log; @@ -12106,7 +12114,9 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) bool fresh = true; for (i = 0; i < svc_cnt_der_n; i++) { - if (svc_cnt_der[i].va_page == page) { + if (svc_cnt_der[i].va_page == page && + (!svc_cur_vas || !svc_cnt_der[i].h_vas || + svc_cnt_der[i].h_vas == svc_cur_vas)) { pa = svc_cnt_der[i].pa_page | off; fresh = false; break; @@ -12127,7 +12137,27 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) * it (r194: the counter page 0x20400f000 never resolved; its * walk hit probed unreadable and targeted was never reached). */ - if (!svc_walk_vaspaces(vgpu, page, &p, &a) && + /* + * SESSION 153-CONTINUED-38: the channel's OWN page tables + * answer first. The global walk sources can only name one + * page for the shared counter VA (0x20400fXXX is a fixed + * per-client layout), so a second client's slot derivation + * collided with the first client's page and its registration + * was rejected forever (CONT-37: "slot owned by another + * channel"). The per-channel walk (svc_cur_vas, set by the + * pump for the channel being serviced) names the page the + * client itself polls. + */ + if (!ok && svc_cur_vas && + !svc_chanvas_lookup(vgpu, page, &p, &a) && + a == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "chanvas"; + } + if (!ok && !svc_walk_vaspaces(vgpu, page, &p, &a) && a == SVC_APERTURE_SYS && !svc_read_ap(vgpu, (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, @@ -12173,6 +12203,7 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) svc_cnt_der[svc_cnt_der_n].va_page = page; svc_cnt_der[svc_cnt_der_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_cnt_der[svc_cnt_der_n].h_vas = svc_cur_vas; svc_cnt_der_n++; } *pa_out = pa; @@ -12180,6 +12211,56 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) } /* + * SESSION 153-CONTINUED-38: counter-slot disambiguation for a channel with + * no usable hVASpace (kernel channels register vas=0x0, so svc_cur_vas + * cannot answer). Two clients' channels can share a rank - the rank formula + * then names the SAME slot VA in each client's own address space - and the + * global walk sources answer with only one client's page, so the second + * client's autoreg collided with the first client's registered slot and was + * rejected forever (CONT-37: the GL client's beef0100 rank=2 vs the X + * client's bfef0002, both 0x2132acfd0). Walk EVERY registered VAS and + * return the first page that is readable AND different from avoid_pa: that + * is the second client's own page for the same VA. + */ +static int svc_counter_derive_not(struct nv_vgpu_device *vgpu, u64 va, + u64 avoid_pa, u64 *pa_out) +{ + unsigned long flags; + struct svc_vaspace local[SVC_MAX_VASPACES]; + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + int i, n = 0; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid) + local[n++] = svc.vas[i]; + spin_unlock_irqrestore(&svc.vas_lock, flags); + + for (i = 0; i < n; i++) { + u64 p = 0; + int a = SVC_APERTURE_SYS; + u32 probe; + + if (svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, + page, local[i].ap, &p, &a, false)) + continue; + if (a != SVC_APERTURE_SYS) + continue; + p = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + if ((p & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (avoid_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + if (svc_read_ap(vgpu, p, SVC_APERTURE_SYS, &probe, + sizeof(probe))) + continue; + *pa_out = p; + return 0; + } + return -ENOENT; +} + +/* * Auto-register a GR channel's completion-counter slot from the observed * layout, without waiting for a readable push: the client zips its GR * channels in ascending handle order onto DESCENDING 16-byte slots, so the @@ -12294,6 +12375,47 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, break; } } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (collision) { + /* + * SESSION 153-CONTINUED-38: the slot VA is shared by + * rank across clients (each client's rank-r channel + * polls 0x20400fec0+(TOTAL-1-r)*0x10 in its OWN VAS), + * so a collision with another client's registered slot + * means the global derive answered with the OTHER + * client's page. Re-resolve through every registered + * VAS and keep the first readable page that differs - + * that page is this client's own slot. Only when no + * such page exists is the collision real and the + * registration must stay pending. + */ + u64 pa2 = 0; + + if (!svc_counter_derive_not(vgpu, va, pa, &pa2)) { + bool coll2 = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (j = 0; j < SVC_MAX_CHANNELS; j++) { + if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && + svc.chan[j].h_object != + ch->h_object && + svc.chan[j].counter_pa == + (pa2 & ~0x3ULL)) { + coll2 = true; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!coll2) { + pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (own-VAS page; contender owner page 0x%llx kept)\n", + ch->h_object, ch->gr_index, + pa2 & ~0x3ULL, pa); + pa = pa2 & ~0x3ULL; + collision = false; + } + } + } if (!collision) { /* * Persist the registration into the authoritative @@ -12310,6 +12432,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, * hung. The authoritative store makes the * registration durable and the log one-shot. */ + spin_lock_irqsave(&svc.chan_lock, flags); ch->counter_pa = pa; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && @@ -12319,8 +12442,8 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, break; } } + spin_unlock_irqrestore(&svc.chan_lock, flags); } - spin_unlock_irqrestore(&svc.chan_lock, flags); if (collision) { pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n", ch->h_object, ch->gr_index, pa); @@ -18342,8 +18465,8 @@ static int svc_read_va_push(struct nv_vgpu_device *vgpu, * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ -static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, - u32 entries, u32 *put, u32 *get) +static int svc_userd_probe_ex(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get, bool loose_get) { u32 p = 0, g = 0; @@ -18355,13 +18478,30 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, return -ENOENT; *put = p; *get = g; - if (p > entries || g > p) + /* + * SESSION 153-CONTINUED-38: the kernel channels' control page can + * carry an uninitialised GP_GET (measured 0x201400 at fb 0x18fe00+0x88 + * while GPPut was a live 3) - GP_GET is GPU-owned ("read only" in the + * Kepler control page) and the driver does not zero it, exactly the + * CE-layout lesson (ISSUE-0002 r47: "the GET word is GPU-owned and its + * initial content is not coherent"). The pump tracks its own position + * in last_get and rewrites GP_GET as it consumes, so `loose_get` + * callers (large GR kernel rings only) accept the block on the PUT + * alone. + */ + if (p > entries || (!loose_get && g > p)) return -EINVAL; if (!p && !g) return -EAGAIN; /* plausible but idle: callers decide */ return 0; } +static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get) +{ + return svc_userd_probe_ex(vgpu, pa, as, entries, put, get, false); +} + /* * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the direct GL client's * channel allocation declares its USERD in the FB aperture at a 0x18fc00/ @@ -18535,6 +18675,12 @@ static bool svc_fb_userd_pristine_scan(struct nv_vgpu_device *vgpu, * aperture stays the channel's own; a wrong second aperture only costs a * failed probe (the values must still read as a coherent put/get pair). */ +/* + * SESSION 153-CONTINUED-38: the shared entry-window gate used by every GR + * ring/entry validator (defined next to svc_gr_ring_candidate below). + */ +static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva); + static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get, int rrc) { @@ -18728,10 +18874,12 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } for (i = 0; i < nc; i++) { + bool loose = !ch->engine_type && ch->gpfifo_entries > 1024; + if (!cand[i]) continue; - if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, - put, get)) { + if (!svc_userd_probe_ex(vgpu, cand[i], as[i], + ch->gpfifo_entries, put, get, loose)) { /* ISSUE-0002 r46: name a fallback latch (CE wrong block). */ if (gsp_sema_dbg && ch->engine_type && ch->ce_index < 32 && @@ -18766,12 +18914,42 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * the in-scan re-arm needs a next call to re-run the sweep that * finds the advancing block. */ - if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok && + /* + * SESSION 153-CONTINUED-38: kernel-VA rings (the direct GL client's + * 0x4000000-class channels) resolve through the channel's own VAS + * walk, never the guest-RAM content scan, so gr_ram_ring stays false + * and the RAM USERD discovery below would never run for exactly the + * channels that need it (CONT-37: beef0100-0102 ok=0, sub=0, then + * the alloc/free retry loop). Anchor the walk-resolved ring as a pin + * - only after entry0 decodes plausibly, so a stale walk answer is + * not cemented - and let the scan run on either anchor. The scan's + * entry validation still proves ownership against the ring. + */ + if (!ch->engine_type && ring_pa && !ch->gr_ram_ring && !ch->userd_ok) { + u8 re[SVC_GPFIFO_ENTRY_SIZE]; + u32 r0, r1; + u64 rva; + u32 rlen; + + if (!svc_read_ap(vgpu, ring_pa, ring_ap, re, sizeof(re))) { + memcpy(&r0, re, 4); + memcpy(&r1, re + 4, 4); + rva = ((u64)(r1 & 0xff) << 32) | + (u64)(r0 & 0xfffffffc); + rlen = ((r1 >> 10) & 0x1fffff) << 2; + if (!(rva & 0xfffff) && rlen >= 4 && rlen <= 0x10000 && + svc_gr_entry_win_ok(ch->gpfifo_va, rva)) { + svc_pin_store(ch->gpfifo_va, ring_pa, ring_ap); + svc_lat_store(ch->gpfifo_va, ring_pa, ring_ap); + } + } + } + if (!ch->engine_type && (ch->gr_ram_ring || ring_pa) && !ch->userd_ok && (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || ch->gr_ram_userd_watch_n || ((ch->h_object >= 0xbfef0000u && - ch->h_object <= 0xbfef00ffu) && - ch->gr_ram_userd_kick_sweeps < 64))) { + ch->h_object <= 0xbfef00ffu) && + ch->gr_ram_userd_kick_sweeps < 64))) { u64 upa = 0; u32 uput = 0, uget = 0; @@ -18871,7 +19049,11 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); - if (!hp || hp > ch->gpfifo_entries || hg > hp) + /* CONT-38: large GR kernel rings may carry an + * uninitialised GPU-owned GP_GET; the PUT alone + * gates there (see svc_userd_probe_ex). */ + if (!hp || hp > ch->gpfifo_entries || + (hg > hp && ch->gpfifo_entries <= 1024)) continue; gate_n++; /* Validate: every claimed entry decodes to a @@ -18903,13 +19085,12 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * Every entry the put claims must name a * VA in the ring's allocation window - * pushbuffers below the ring, the - * progress tracker above it. + * progress tracker above it; the + * kernel-VA family is accepted in its + * own low window (CONT-38). */ - if (!eva || - eva + SVC_GR_RAM_WIN / 2 < - ch->gpfifo_va || - eva >= ch->gpfifo_va + - SVC_GR_RAM_WIN / 2 || + if (!eva || !svc_gr_entry_win_ok( + ch->gpfifo_va, eva) || elen < 4 || elen > 0x100000) { ok = false; rck = (int)k; @@ -19562,6 +19743,30 @@ static bool svc_ring_walk_live(struct nv_vgpu_device *vgpu, u64 gpfifo_va) } /* + * SESSION 153-CONTINUED-38: kernel-VA ring entry window. The direct GL + * client's kernel channels (h 0xbeef01xx) place their GPFIFO ring and + * pushbuffers in the driver's LOW aperture (0x4000000-class, + * SVC_LOW_REL_VA_LO) instead of the userspace GPU-VA window (>= 4 GiB, + * SEMA_VA_LO) every GR gate below assumed, so their entries could never + * validate and the channels never resolved a USERD (the CONT-37 alloc/free + * retry loop). An entry VA in the low family is this channel's push only + * when it sits in the same contiguous allocation as the ring VA; accept it + * within a 256 MB window around the ring, never outside the low family. + */ +static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva) +{ + if (eva + SVC_GR_RAM_WIN / 2 >= ring_va && + eva < ring_va + SVC_GR_RAM_WIN / 2) + return true; + if (eva >= SVC_LOW_REL_VA_LO && + eva < SVC_LOW_REL_VA_LO + 0x10000000ULL && + eva + 0x10000000ULL > ring_va && + eva < ring_va + 0x10000000ULL) + return true; + return false; +} + +/* * Validate a candidate translation for a GR ring: the first GPFIFO entry * must decode to a 1 MB aligned push in the client VA window with a sane * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the @@ -19584,8 +19789,11 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, memcpy(&e1, e + 4, 4); *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); *len = ((e1 >> 10) & 0x1fffff) << 2; - if ((*va & 0xfffff) || *va < SEMA_VA_LO || *va >= SEMA_VA_HI || - *len < 4 || *len > 0x10000) + if ((*va & 0xfffff) || *len < 4 || *len > 0x10000) + return false; + /* SESSION 153-CONTINUED-38: low-aperture kernel-VA ring entries. */ + if ((*va < SEMA_VA_LO || *va >= SEMA_VA_HI) && + !svc_gr_entry_win_ok(ch->gpfifo_va, *va)) return false; /* * SESSION 132: a candidate whose entry0 push provably reads all-zero @@ -21067,7 +21275,18 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * rate; the step-based watch adoption (not the cap) * is the discriminator there. */ - if (!hp || hp > (kick ? 0x100000 : 8) || hg > hp) + /* + * SESSION 153-CONTINUED-38: the cap is also lifted for + * LARGE rings (the kernel channels' 32768-entry + * GPFIFOs): their put is a real free-running count and + * can sit far above 8 while the client waits (measured + * put=16 on beef0100). The remaining gates (near-empty + * page, entry validation against the ring, and the + * pump's confirm/reject) carry the ownership proof. + */ + if (!hp || + hp > (kick || entries > 1024 ? 0x100000 : 8) || + (hg > hp && entries <= 1024)) continue; { u32 z = 0, zi; @@ -21127,10 +21346,7 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, (u64)(a0 & 0xfffffffc); elen = ((a1 >> 10) & 0x1fffff) << 2; if (!eva || - eva + SVC_GR_RAM_WIN / 2 < - ch->gpfifo_va || - eva >= ch->gpfifo_va + - SVC_GR_RAM_WIN / 2 || + !svc_gr_entry_win_ok(ch->gpfifo_va, eva) || elen < 4 || elen > 0x100000) { ok = false; break; @@ -32125,6 +32341,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->engine_type) { u32 pput = 0; + /* + * SESSION 153-CONTINUED-38: set the channel's VAS + * BEFORE the autoreg derive, not after the ring + * lookup. The counter slot must be disambiguated + * through THIS channel's page tables (two clients + * register the same counter VA; CONT-37 collision), + * and the ring lookup + USERD discovery below must + * also resolve through the channel's own VAS first. + */ + svc_cur_vas = ch->h_vaspace; svc_counter_autoreg(vgpu, ch, n_gr); svc_counter_flush_pending(vgpu, ch); /* @@ -32450,7 +32676,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (kfam && ch->gr_ram_userd_confirmed && put > ch->gr_ram_userd_latch_put) ch->gr_ram_userd_latch_put = put; - if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok && + /* + * SESSION 153-CONTINUED-38: the latch confirmation must also + * judge a block adopted through the RAM scan for a + * walk-resolved ring (gr_ram_ring false; gr_ram_userd_latch_at + * stamps the latch), or a wrong low-aperture latch would stick + * forever. + */ + if (!ch->engine_type && ch->userd_ok && + (ch->gr_ram_ring || ch->gr_ram_userd_latch_at) && (!ch->gr_ram_userd_confirmed || (kfam && put < ch->gr_ram_userd_latch_put))) { bool stepped; |