diff options
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 44 | ||||
| -rw-r--r-- | kernel/gsp_rm_tu104.h | 13 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 759 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 8 |
4 files changed, 802 insertions, 22 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 63a4338..eddfab5 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -2456,6 +2456,7 @@ static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx, #define GSPRM_COPY_PDES_MAX_LEVELS 6 static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, + u32 h_object, const u8 *params, u32 params_size) { struct gsp_rm_emu *rm = ctx->rm_emu; @@ -2506,6 +2507,46 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, "vgsp-rm: pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u", i, phys, size, aperture, page_shift); } + + /* + * r496j: this control IS the kernel RM publishing the page-table + * chain of its internal VAS (the level[0..] instances with their + * phys addresses and page shifts). The chain can name the X/GL + * channel's pushbuffer pages and progress semaphore, for which no + * other translation source exists (the fn-54 SET_PAGE_DIRECTORY + * path is external-VAS/UVM only). It is NOT registered directly: + * a blast registration of every client's directory let a stale + * kernel directory answer another client's VA in the walker's + * first-success order (the r459 stale-directory class; observed + * as a CUDA client hanging after DEVICE with r496g). Instead the + * root is offered as a CANDIDATE; the progress service registers + * it only after it has been VERIFIED against a content-validated + * pair of that very client (the ring found by the RAM content + * scan). + */ + if (num_levels >= 1 && rm->pt.levels[0].phys && + rm->pt.levels[0].size >= 16 && + rm->pt.levels[0].size <= 4096 * 8) { + /* + * The level phys values are guest PAs on this platform + * (observed 4+ GiB, above any FB shadow); aperture 3 in the + * message does not match the walker's FB/SYS encoding, so + * classify by address: values inside the emulated VRAM are + * FB, everything else sysmem. The walker also retries the + * other aperture, so a misclassification only costs a walk. + */ + bool vidmem = rm->pt.levels[0].phys < + (u64)ctx->vgpu->bars[1].size; + + gsp_sema_service_vaspace_candidate_add( + h_object, rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), vidmem); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: VAS 0x%x page directory offered as candidate pdb=0x%llx entries=%u (%s; registered only after a content-verified ring walk)", + h_object, (u64)rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), + vidmem ? "vidmem" : "sysmem"); + } } static bool gsp_rm_alloc_interesting(u32 class_id); @@ -3219,7 +3260,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */ - gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE, + gsp_rm_handle_copy_server_pdes(ctx, req.hObject, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize); out_status = 0; out_params_size = 0; diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h index b5ee4d7..3382939 100644 --- a/kernel/gsp_rm_tu104.h +++ b/kernel/gsp_rm_tu104.h @@ -754,7 +754,18 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = { TU104_CE_CAPS0_SYSMEM | \ TU104_CE_CAPS0_P2P | \ TU104_CE_CAPS0_BL_GT_64K) -#define TU104_CE_CAPS1 0x00 +/* + * Caps byte 1: the 2D/3D copy path requires a CE that supports the + * blocklinear access modes (hardware bug 200288104): the X driver's + * nvCEChooseEngine skips every CE without CE_SUPPORTS_NONPIPELINED_BL + * (byte1 0x01; and PIPELINED_BL 0x02 for the preferred pass) and then + * fails with "Failed to find a suitable copy object" - the observed wall + * after the DMA pushbuffer test started passing. Turing's CEs are fully + * capable (pipelined + non-pipelined BL + pitchlinear): byte1 = 0x03. + * (Reference: ctrl2080ce.h NV2080_CTRL_CE_CAPS_CE_SUPPORTS_*_BL 1:0x01/ + * 1:0x02.) + */ +#define TU104_CE_CAPS1 0x03 /* ENGINE_TYPE_COPY0 = 0x9, COPY(i) = COPY0 + i (cl2080_notification.h) */ #define TU104_ENGINE_TYPE_COPY0 0x00000009 diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b328eea..bc21e95 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -160,6 +160,15 @@ struct sema_map_entry { #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c +/* + * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct + * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan + * slice per service pass so the service thread stays responsive. + */ +#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ +#define SVC_GR_RAM_PASS_MAX 16 /* slices before giving up */ +#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ + struct svc_channel { bool valid; u32 session_gen; /* m176: session generation this entry was @@ -385,6 +394,16 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ + /* + * r509: one-shot verdict of the absolute bound. The expiry call + * pushes the deadline (so the hold window restarts) and latches this + * flag; the r351 retry-forward block consumes it within the same + * pass. Without the latch the two call sites in one pass could not + * both observe the expiry (the first push of the deadline made the + * second call see a fresh window - r213-r215's arming bug in + * reverse). + */ + bool gr_hold_bound_hit; unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data- * stalled hold for gr_hold_dead_va. * Fixed once on the first hold of that @@ -420,6 +439,37 @@ struct svc_channel { u64 gr_nocap_va; /* push VA the window was opened for */ unsigned long gr_nocap_since; /* absolute first-seen stamp */ /* + * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in + * guest RAM. The CPU RM maps sysmem objects locally (no + * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so + * svc_lookup_trusted cannot name the ring page; the driver's kickoff + * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan + * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the + * service thread stays responsive. + */ + u64 gr_ram_cursor; /* resume point of the content scan */ + u8 gr_ram_passes; /* slices walked so far */ + bool gr_ram_logged; /* one discovery line per channel */ + /* + * r496e: true when the ring was found by the RAM content scan (a + * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's + * pushes are 2D/3D method streams the emulation does not execute; + * the correct model is consume-and-apply-releases: no QMD capture, + * no launch forward, no no-capture hold. Also allows an all-zero + * NOP push (the driver's pushbuffer sanity test writes exactly one + * zero dword) to be consumed as the no-op it is. + */ + bool gr_ram_ring; + /* + * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the + * registered/ring-derived candidates all fail; the scan validates the + * hit against the ring entries the put claims, so a foreign channel's + * USERD cannot be adopted. Bounded retries: the driver kicks off a + * few passes after channel registration, so a scan before the first + * put must not retire the discovery. + */ + u8 userd_scans; + /* * r471: the completion-counter publish must never advance the * guest's word past a push whose content was still unwritten when * the entry was consumed (the streams_events staging-reuse @@ -1600,6 +1650,90 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ +/* + * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A + * kernel-RM internal VAS publishes its chain there, but registering every + * client's directory directly lets a stale kernel directory answer another + * client's VA in the walker's first-success order (the r459 stale-directory + * class; observed as a CUDA client hanging after DEVICE). Candidates are + * parked here and registered only after a walk of the client's own + * content-validated ring VA through the candidate reproduces that ring's PA + * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify). + */ +#define SVC_VAS_CAND_MAX 16 +static struct { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; + bool vidmem; +} svc_vas_cand[SVC_VAS_CAND_MAX]; + +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace); + +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem) +{ + int i; + + if (!pdb || entries < 2 || entries > 4096) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb == pdb) + return; /* already parked */ + if (!svc_vas_cand[i].valid) { + svc_vas_cand[i].valid = true; + svc_vas_cand[i].h_vaspace = h_vaspace; + svc_vas_cand[i].pdb = pdb; + svc_vas_cand[i].entries = entries; + svc_vas_cand[i].vidmem = vidmem; + return; + } + } +} + +/* + * Verify each parked candidate against one content-validated translation + * (the ring VA -> PA the RAM scan just proved): walk the ring page through + * the candidate and register only the directory that reproduces it. Runs on + * the service thread from the ring-scan latch; the walk itself is cheap (a + * few table reads). + */ +static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, + u64 pa) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = 0; + int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS; + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, va & ~0xfffULL, + ap, &wpa, &wap, false) || + (wpa & ~0xfffULL) != (pa & ~0xfffULL)) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", + d.h_vaspace, d.pdb, d.num_entries, ap); + break; + } +} + void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; @@ -2124,8 +2258,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) } if (n < SVC_PD_MIN_HITS) { n = 0; - if (svc.map_n < SVC_PD_SAMPLES) - return; /* no evidence yet: retry when pairs appear */ + /* + * r510: a LARGE map is a mixed set of VA spaces cached over + * the whole guest boot (the comment above already names the + * consequence: no single directory reproduces a mixed set), so + * scanning with its stride subsample burns the attempts budget + * on evidence that cannot succeed - and by the time the + * push-buffer pairs (the strongest samples) exist, the budget + * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram + * cands=39621 best=0/8)' during the first client, eight RAM + * walks consumed, every later client's high argument pointers + * unresolvable - the terminal class). Only a SMALL map, which + * is one session's own pairs, still falls back; otherwise wait + * for the push-buffer pairs and keep the attempts. + */ + if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) || + svc.map_n < SVC_PD_SAMPLES) + return; /* no coherent evidence yet: retry later */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; @@ -7403,6 +7552,79 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) return free_slot; } +/* + * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op + * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode + * pass and sets svc.op_tr_pending each time, so its push holds for the full + * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op + * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx + * after the first client, dst never translated) ground its whole stream + * at ~10 s per push ("hang after DEVICE" for matmul/multikernel). + * + * The window must still cover the legitimate pin-late retry (observed + * pin latency 0.5-2 s: the H2D copy's operands land a moment after the + * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes), + * so the latch is TIME-based: the FIRST failure of a VA arms a + * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as + * before; after a full bound with no success the op still attempts the + * write but no longer forces the hold - the entry consumes promptly and + * its withheld release applies, exactly the outcome the expired bound + * already sanctioned (op stays skipped, release delivered), minus the + * 10 s tax per subsequent pass. A success clears the record (a later + * failure gets a fresh window). Service-thread only, like every other op + * path. + */ +#define SVC_OPFAIL_MAX 32 +/* + * r507a: the op-failure window rides the opdef hold bound (10000 ms, + * SVC_OPDEF_HOLD_MS defined further down). Keep the literal here so the + * latch (used by svc_run_ops) precedes that define. + */ +#define SVC_OPFAIL_WINDOW_MS 10000u +static struct { + u64 va; + unsigned long since; +} svc_opfail[SVC_OPFAIL_MAX]; +static u32 svc_opfail_log; + +static void svc_opfail_clear(u64 va) +{ + u32 i; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) + if (svc_opfail[i].va == va) + svc_opfail[i].va = 0; +} + +static bool svc_opfail_latched(u64 va) +{ + u32 i; + u32 free_i = SVC_OPFAIL_MAX; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) { + if (svc_opfail[i].va == va) { + if (time_after_eq(jiffies, + svc_opfail[i].since + + msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) { + if (svc_opfail_log < 32) { + svc_opfail_log++; + pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n", + va, SVC_OPFAIL_WINDOW_MS); + } + return true; + } + return false; + } + if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX) + free_i = i; + } + if (free_i < SVC_OPFAIL_MAX) { + svc_opfail[free_i].va = va; + svc_opfail[free_i].since = jiffies; + } + return false; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, u32 entry_idx, u64 pass_hash) @@ -7660,9 +7882,11 @@ svc_wait_done: */ if (rc == SVC_COPY_FORWARDED) continue; - if (rc == 0) + if (rc == 0) { svc.memsets_done++; - else if (rc == -ENOENT) + /* r507a: a success re-arms the failure window. */ + svc_opfail_clear(m->dst); + } else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; @@ -7693,7 +7917,8 @@ svc_wait_done: if (rc != 0 && rc != SVC_COPY_FORWARDED && (m->bytes >= (256u << 10) || (m->type == 1 && m->dst >= SVC_VAS_FLOOR) || - (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR))) + (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) && + !svc_opfail_latched(m->dst)) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -8653,7 +8878,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, inc && inc_apply); return false; } - if (pending || inc_apply) { + /* + * r506w: generic absolute releases are forward-only by VALUE, whichever + * pass discovers the gap. The recorded pending bit came from the + * decode's read of the release word, and that read can be blinded by a + * stale/crossed landing (r506 witness: a withheld tracker release went + * to the flush with pending=0 while the fresh read still showed 0, and + * the release was then dropped by the counter fallback below - the + * client's tracker never advanced and its stream ground at the 10 s + * bound per push). Increments keep the one-shot claim gate (`inc_apply`). + */ + if (pending || inc_apply || (!inc && payload > cur)) { svc.pushes_pending++; if (inc) cur += payload; @@ -8675,10 +8910,20 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, return true; } } - if (ch->engine_type == 0) - svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, - inc && inc_apply); + /* + * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its + * `release_pa ?: counter_pa` fallback writes (and forward-only-skips + * against) the channel's REGISTERED slot, which belongs to a different + * word - the r506 witness: releases to the tracker VA 0x20480fef0 were + * skipped against the polluted PUT counter (cur=96) and lost. For a + * non-counter VA the generic write above is the delivery path. + */ + if (svc_counter_va(sem_va)) { + if (ch->engine_type == 0) + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + } /* * Stall-class fix (R2): when the decode said this absolute release is * genuinely pending and every write attempt above failed without @@ -12999,6 +13244,119 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } } + + /* + * r496b: no candidate matched. For a GR channel whose ring is known + * (translated and pinned - the sysmem content scan or the VRAM + * conventions), the USERD may still be elsewhere: the X/GL driver + * allocates its USERD in VRAM and the CPU RM maps it through a BAR1 + * window, so the driver's GP_PUT write lands at a BAR1 offset that is + * neither the registered memdesc base nor ring-derived. Scan the + * emulated VRAM for the USERD shape and validate each hit against the + * ring: the put it claims must correspond to entries whose pushbuffer + * VAs all sit in the ring's allocation window. A foreign channel's + * USERD cannot pass that test. One scan per channel (the result + * persists through the pump's write-back), logged on latch. + */ + if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { + u64 off, lim; + u8 *base = vgpu->bars[1].vaddr; + u64 best = 0, best_dist = ~0ULL; + u32 best_put = 0, best_get = 0; + bool best_pending = false; + + ch->userd_scans++; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + if (base && lim) { + for (off = 0; off + 0xa0 <= lim; off += 0x200) { + u32 hp, hg; + bool ok = true; + u32 k; + u64 dist; + bool pending; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + /* Validate: every claimed entry decodes to a + * pushbuffer VA in the ring's window. */ + for (k = 0; k < hp && k < 64; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)k * 8, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + /* + * Every entry the put claims must name a + * VA in the ring's allocation window - + * pushbuffers below the ring, the + * progress tracker above it. + */ + if (!eva || + eva + SVC_GR_RAM_WIN / 2 < + ch->gpfifo_va || + eva >= ch->gpfifo_va + + SVC_GR_RAM_WIN / 2 || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + /* + * Several blocks can validate: a previous + * session leaves its old USERD holding the same + * put/entry values while the new channel's block + * sits elsewhere (observed r504: the re- + * registered channel re-scanned, adopted the + * stale block, and its DMA test timed out). + * Prefer a block with work PENDING (get < put); + * among equals prefer the one nearest the + * registered memdesc offset (the driver's own + * hint). The pump's GP_GET write-back goes to + * the adopted block, so a stale adoption strands + * the client's poll. + */ + dist = off > ch->userd_pa ? off - ch->userd_pa + : ch->userd_pa - off; + pending = hg < hp; + if (best && + (best_pending > pending || + (best_pending == pending && dist >= best_dist))) + continue; + best = off; + best_dist = dist; + best_put = hp; + best_get = hg; + best_pending = pending; + } + if (best) { + ch->userd_pa = best; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + *put = best_put; + *get = best_get; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", + ch->h_object, ch->gr_index, best, + best_put, best_get, + best_pending ? "" : ", idle"); + return 0; + } + } + } return -ENOENT; } @@ -14034,6 +14392,139 @@ unsigned gsp_sema_push_hold_ms = 2500; static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out) +{ + static u8 *buf; + u64 start = ch->gr_ram_cursor; + u64 end = start + SVC_GR_RAM_SLICE; + u64 gpa; + int i; + unsigned long flags; + + if (end > SEMA_PA_MAX) + end = SEMA_PA_MAX; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) { + u32 e0, e1, f0, f1; + u64 va0, va1; + u32 len0, len1; + + memcpy(&e0, buf + o, 4); + memcpy(&e1, buf + o + 4, 4); + va0 = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len0 = ((e1 >> 10) & 0x1fffff) << 2; + /* entry0: main pushbuffer, just below the ring */ + if (va0 >= ch->gpfifo_va || + va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va) + continue; + if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) + continue; + if (len0 < 4 || len0 > 0x100000) + continue; + memcpy(&f0, buf + o + 8, 4); + memcpy(&f1, buf + o + 12, 4); + va1 = ((u64)(f1 & 0xff) << 32) | + (u64)(f0 & 0xfffffffc); + len1 = ((f1 >> 10) & 0x1fffff) << 2; + /* entry1: progress tracker, above entry0 */ + if (va1 <= va0 || + va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (len1 < 4 || len1 > 0x100000) + continue; + *pa_out = gpa + o; + return true; + } + } + ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + ch->gr_ram_passes++; + if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { + ch->gr_ram_logged = true; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_passes, + ch->gpfifo_va); + } + /* + * Persist the cursor immediately: the pump's stalled-userd path + * `continue`s past the end-of-pass write-back, so without this the + * scan would restart from slice 0 every pass and never reach the + * ring's page. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +/* + * r496f: discover a kernel-RM client VAS's page directory in the emulated + * VRAM by verification. The CPU RM builds internal-VAS page tables in + * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally + * owned / UVM-only RPC), so the pushbuffer pages and the progress + * semaphore of an X/GL channel cannot be translated from any existing + * source. Try each 4 KB page as the walk root and accept the first whose + * walk reproduces the ring VA -> ring PA pair we already validated by + * content; registering it with the VAS table makes every other VA of the + * same client resolvable through the existing walker. Bounded to the + * vidmem shadow; verified, so a wrong candidate cannot be registered. + */ +static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, + u64 ring_pa) +{ + static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 }; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + unsigned int ei; + + if (!vgpu->bars[1].vaddr || !lim) + return; + for (off = 0; off + 0x1000 <= lim; off += 0x1000) { + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) { + u64 pa = 0; + int ap = 0; + + if (!svc_gmmu_walk(vgpu, off, ent_tab[ei], + ring_va & ~0xfffULL, + SVC_APERTURE_FB, &pa, &ap, false) && + (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) { + struct sema_vaspace_desc d; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | + (u32)(ring_va >> 20); + d.pdb = off; + d.num_entries = ent_tab[ei]; + d.b_vidmem = true; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + off, ent_tab[ei], ring_va, ring_pa); + return; + } + } + } +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -14310,6 +14801,124 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } + + /* + * r495 diagnostic (X/GL channel): the X driver's GR channel + * registers its USERD at a memdesc base that may be an FB + * offset or a BAR1 window. The driver's kickoff writes GP_PUT + * through its CPU mapping of that object; if the write landed + * in the emulated FB shadow, scanning for the USERD shape + * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90, + * both nonzero somewhere in the 0x200 block) names the real + * offset. One-shot per channel, diagnostic only. + */ + if (gsp_sema_dbg && vgpu->bars[1].vaddr && + svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) { + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + int hits = 0; + + for (off = 0; off + 0xa0 <= lim && hits < 8; + off += 0x200) { + u32 hp, hg; + + memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4); + memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n", + off, hp, hg, + *(u32 *)(vgpu->bars[1].vaddr + off + 0x80), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x84), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x88), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x94)); + hits++; + } + pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n", + ch->h_object, hits, lim); + } + } + + /* + * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor + * field comment). The driver's kickoff writes GPFIFO entries whose + * pushbuffer VAs sit just BELOW the ring VA in the same allocation + * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 + * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to + * ring_va + 0x100000 (the progress tracker). A page whose first two + * 8-byte words both name such VAs with sane lengths is the ring; pin + * it (validated) and its allocation window (guesses) so the ring + * entry and pushbuffer reads resolve. Sliced so the service thread + * stays responsive; gives up per channel after + * SVC_GR_RAM_PASS_MAX slices. + */ + if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u64 hit_pa = 0; + + if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) { + /* + * Pin ONLY the content-validated ring page (validated, + * not a window guess). The pushbuffer pages and the + * tracker push of this client resolve through the VAS + * page directory registered from COPY_SERVER_PDES + * (r496g) - the surface is NONCONTIGUOUS, so a linear + * window guess is wrong for them and, being a + * pin_lookup_any hit, would shadow the correct walk in + * svc_lookup_trusted (observed r502: the tracker push + * read zeros through the wrong window page and the + * driver's DMA poll never completed). + */ + svc_pin_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + ch->gr_ram_logged = true; + ch->gr_ram_ring = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + /* + * r496j: with the ring content-validated, verify the + * parked COPY_SERVER_PDES candidates against it and + * register the directory that reproduces the ring's + * PA; its walk then resolves the client's other VAs + * (pushbuffer pages, the progress semaphore the + * driver polls). r496f's blind vidmem sweep stays as + * a fallback for directories the control never + * published. + */ + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + hit_pa); + svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* + * Persist the latch immediately: a later stalled + * path in this pass may `continue` past the + * end-of-pass write-back, and the next pass's + * snapshot must carry gr_ram_ring (the zero-NOP + * exception and the no-capture-hold skip key on + * it). + */ + { + unsigned long flags2; + int i2; + + spin_lock_irqsave(&svc.chan_lock, flags2); + for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++) + if (svc.chan[i2].valid && + svc.chan[i2].vgpu == vgpu && + svc.chan[i2].h_object == + ch->h_object) { + svc.chan[i2].gr_ram_ring = true; + svc.chan[i2].gr_ram_logged = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags2); + } + pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + hit_pa); + return; + } } } @@ -14317,8 +14926,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; - int ap = SVC_APERTURE_SYS; - u8 e[SVC_GPFIFO_ENTRY_SIZE]; + int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { @@ -18578,6 +19186,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) * forever (the same identity bug class as the opdef=955/13/0 * streams2 wedge). The first hold arms; others ride it; a fresh * bound arms once the previous one has passed. + * + * r509: the bound must actually REPORT expiry. The historical arming + * rule re-armed the deadline on the very call that detected expiry + * and then compared against the NEW deadline - so the function could + * never return true, and a data-stalled launch whose arguments never + * resolved held its entry forever (r508 witness: matmul plus ten + * sibling channels in "GR hold ... data args unresolved" with zero + * GR-STALL-hold-bound lines for 10+ minutes; the designed + * retry-forward + consume never ran). The expiry is now one-shot: + * push the deadline, latch gr_hold_bound_hit for the r351 block of + * the same pass, return true. A fresh arm clears the latch (and the + * once-per-launch retry log). */ if (!ch->gr_hold_dead_va || time_after_eq(jiffies, ch->gr_hold_deadline)) { @@ -18585,6 +19205,8 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) ch->gr_hold_dead_since = jiffies; ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); ch->gr_hold_exp_logged = false; + ch->gr_hold_bound_hit = false; + return false; } /* * r420: pause the per-launch consumption bound while the backend @@ -18607,7 +19229,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) launch_va); return false; } - return time_after_eq(jiffies, ch->gr_hold_deadline); + if (time_after_eq(jiffies, ch->gr_hold_deadline)) { + /* + * r509: one-shot expiry. Push the deadline so the hold window + * restarts for the caller's retry, latch the verdict for the + * second call site of this pass (the r351 retry-forward + * block), and report expired. + */ + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_bound_hit = true; + return true; + } + return false; } static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, @@ -20242,8 +20875,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + /* + * r507o stream-order: walk the CE channels BEFORE the GR channels. + * The client's stream order places its H2D copies before the kernel + * that consumes them, but the channels register in handle order (the + * RM's GR channel early, the copy-ce channels later), and a pass that + * decoded the launch before the CE channel's entries ran let the + * forward workqueue execute the launch's copy-in against pre-copy + * data (r480-sep: copy-in 115 ms before its stream's H2D soft + * copies; r481 proved no host semaphore links the two, so the + * emulation must enforce stream order itself). The CE copies execute + * on this same service thread, so consuming the CE entries first + * fixes the inversion directly - no cross-channel hold, no latency + * tax. Stable partition; relative order within each class preserved. + */ for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].engine_type) + local[n++] = svc.chan[i]; + } + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + !svc.chan[i].engine_type) local[n++] = svc.chan[i]; } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -20776,10 +21429,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool same = ident && + /* + * r505b: the per-slot op bookkeeping arrays are + * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL + * channel's ring is 8192 slots. Bound every slot + * index here or a large ring writes out of + * bounds. + */ + bool same = ident && idx < SVC_GPFIFO_MAX_BITS && ch->slot_ops_ident[idx] == ident; - entry_ident = ident; + entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan @@ -20892,7 +21552,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) injected = (svc_ipin_lookup(va, &ipa, &iap) == 0); - if (!svc_method_stream_plausible(pb, len)) { + /* + * r496e: an all-zero push on a + * content-scanned-ring channel is the + * driver's NOP sanity push (one zero + * dword - NVA06F_DMA_NOP), not a bad + * translation: accept it and let the + * decode find no ops. + */ + bool zero_nop = ch->gr_ram_ring && + len >= 4 && + !memchr_inv(pb, 0, len); + + if (!svc_method_stream_plausible(pb, len) && + !zero_nop) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, @@ -21220,7 +21893,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - if (svc.op_tr_pending) + if (svc.op_tr_pending) { /* * Deadline hit: the * operand never @@ -21232,6 +21905,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_flush(vgpu, ch, (u32)idx); + /* + * r505b: mark the slot's + * ops complete. The op was + * permanently skipped, but + * the decode above stored + * complete=0 (op_tr_pending + * was set), so every pin- + * batch rewind replay of + * this slot re-decoded it, + * re-set op_tr_pending and + * re-held the entry in a + * livelock (multikernel on + * r505: opdef held=18359, + * expired=61, the client + * waiting forever while the + * pump looped on slot 1). + * Nothing is owed after the + * flush, so a replay must + * skip the ops. + */ + if ((u32)idx < + SVC_GPFIFO_MAX_BITS) { + ch->slot_ops_complete[idx] = + 1; + ch->slot_ops_done[idx] = + entry_done; + } + } rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -21467,10 +22168,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && - svc_gr_hold_bound_expired(ch, - svc_gr_launch_va(ch)) && + ch->gr_hold_bound_hit && !ch->gr_hold_exp_logged) { ch->gr_hold_exp_logged = true; + /* r509: the latch is this pass's only + * report of the expiry; clear it so the + * NEXT expiry cycle (gr_hold_exp_logged + * now set) falls through to the + * historical consume. */ + ch->gr_hold_bound_hit = false; svc.fwd_gr_hold_exp++; /* * r351: the bound firing used to @@ -21711,6 +22417,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies; } if (gsp_sema_gr_nocap_hold_ms && + !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( @@ -22454,6 +23161,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; + /* r496b: persist the RAM-scan cursor and the + * one-shot USERD-scan latch so they survive + * the per-pass snapshot copy. */ + svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[k].gr_ram_passes = ch->gr_ram_passes; + svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -22531,6 +23246,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_deadline; svc.chan[k].gr_hold_exp_logged = ch->gr_hold_exp_logged; + /* r509: the one-shot expiry latch is snapshot + * state too. */ + svc.chan[k].gr_hold_bound_hit = + ch->gr_hold_bound_hit; /* r361: the opdef absolute-bound record is * snapshot state too (the GRFIX lesson) - * without the write-back the deadline would diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 305a027..ee5c48a 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -81,6 +81,14 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); +/* + * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a + * content-validated walk proves it belongs to a live client; only then is + * it registered (a blind registration let stale kernel directories answer + * other clients' VAs in the walker's first-success order). + */ +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem); /* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state * (direct-map base, content-scanned tables) on the svc thread. Client * session flushes keep that state - see svc_session_flush. */ |