diff options
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 44 | ||||
| -rw-r--r-- | kernel/gsp_rm_tu104.h | 13 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 396 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 8 |
4 files changed, 432 insertions, 29 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 63a4338..eddfab5 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -2456,6 +2456,7 @@ static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx, #define GSPRM_COPY_PDES_MAX_LEVELS 6 static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, + u32 h_object, const u8 *params, u32 params_size) { struct gsp_rm_emu *rm = ctx->rm_emu; @@ -2506,6 +2507,46 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, "vgsp-rm: pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u", i, phys, size, aperture, page_shift); } + + /* + * r496j: this control IS the kernel RM publishing the page-table + * chain of its internal VAS (the level[0..] instances with their + * phys addresses and page shifts). The chain can name the X/GL + * channel's pushbuffer pages and progress semaphore, for which no + * other translation source exists (the fn-54 SET_PAGE_DIRECTORY + * path is external-VAS/UVM only). It is NOT registered directly: + * a blast registration of every client's directory let a stale + * kernel directory answer another client's VA in the walker's + * first-success order (the r459 stale-directory class; observed + * as a CUDA client hanging after DEVICE with r496g). Instead the + * root is offered as a CANDIDATE; the progress service registers + * it only after it has been VERIFIED against a content-validated + * pair of that very client (the ring found by the RAM content + * scan). + */ + if (num_levels >= 1 && rm->pt.levels[0].phys && + rm->pt.levels[0].size >= 16 && + rm->pt.levels[0].size <= 4096 * 8) { + /* + * The level phys values are guest PAs on this platform + * (observed 4+ GiB, above any FB shadow); aperture 3 in the + * message does not match the walker's FB/SYS encoding, so + * classify by address: values inside the emulated VRAM are + * FB, everything else sysmem. The walker also retries the + * other aperture, so a misclassification only costs a walk. + */ + bool vidmem = rm->pt.levels[0].phys < + (u64)ctx->vgpu->bars[1].size; + + gsp_sema_service_vaspace_candidate_add( + h_object, rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), vidmem); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: VAS 0x%x page directory offered as candidate pdb=0x%llx entries=%u (%s; registered only after a content-verified ring walk)", + h_object, (u64)rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), + vidmem ? "vidmem" : "sysmem"); + } } static bool gsp_rm_alloc_interesting(u32 class_id); @@ -3219,7 +3260,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */ - gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE, + gsp_rm_handle_copy_server_pdes(ctx, req.hObject, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize); out_status = 0; out_params_size = 0; diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h index b5ee4d7..3382939 100644 --- a/kernel/gsp_rm_tu104.h +++ b/kernel/gsp_rm_tu104.h @@ -754,7 +754,18 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = { TU104_CE_CAPS0_SYSMEM | \ TU104_CE_CAPS0_P2P | \ TU104_CE_CAPS0_BL_GT_64K) -#define TU104_CE_CAPS1 0x00 +/* + * Caps byte 1: the 2D/3D copy path requires a CE that supports the + * blocklinear access modes (hardware bug 200288104): the X driver's + * nvCEChooseEngine skips every CE without CE_SUPPORTS_NONPIPELINED_BL + * (byte1 0x01; and PIPELINED_BL 0x02 for the preferred pass) and then + * fails with "Failed to find a suitable copy object" - the observed wall + * after the DMA pushbuffer test started passing. Turing's CEs are fully + * capable (pipelined + non-pipelined BL + pitchlinear): byte1 = 0x03. + * (Reference: ctrl2080ce.h NV2080_CTRL_CE_CAPS_CE_SUPPORTS_*_BL 1:0x01/ + * 1:0x02.) + */ +#define TU104_CE_CAPS1 0x03 /* ENGINE_TYPE_COPY0 = 0x9, COPY(i) = COPY0 + i (cl2080_notification.h) */ #define TU104_ENGINE_TYPE_COPY0 0x00000009 diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 75c7645..83bae47 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -441,6 +441,16 @@ struct svc_channel { u8 gr_ram_passes; /* slices walked so far */ bool gr_ram_logged; /* one discovery line per channel */ /* + * r496e: true when the ring was found by the RAM content scan (a + * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's + * pushes are 2D/3D method streams the emulation does not execute; + * the correct model is consume-and-apply-releases: no QMD capture, + * no launch forward, no no-capture hold. Also allows an all-zero + * NOP push (the driver's pushbuffer sanity test writes exactly one + * zero dword) to be consumed as the no-op it is. + */ + bool gr_ram_ring; + /* * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the * registered/ring-derived candidates all fail; the scan validates the * hit against the ring entries the put claims, so a foreign channel's @@ -1630,6 +1640,90 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ +/* + * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A + * kernel-RM internal VAS publishes its chain there, but registering every + * client's directory directly lets a stale kernel directory answer another + * client's VA in the walker's first-success order (the r459 stale-directory + * class; observed as a CUDA client hanging after DEVICE). Candidates are + * parked here and registered only after a walk of the client's own + * content-validated ring VA through the candidate reproduces that ring's PA + * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify). + */ +#define SVC_VAS_CAND_MAX 16 +static struct { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; + bool vidmem; +} svc_vas_cand[SVC_VAS_CAND_MAX]; + +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace); + +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem) +{ + int i; + + if (!pdb || entries < 2 || entries > 4096) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb == pdb) + return; /* already parked */ + if (!svc_vas_cand[i].valid) { + svc_vas_cand[i].valid = true; + svc_vas_cand[i].h_vaspace = h_vaspace; + svc_vas_cand[i].pdb = pdb; + svc_vas_cand[i].entries = entries; + svc_vas_cand[i].vidmem = vidmem; + return; + } + } +} + +/* + * Verify each parked candidate against one content-validated translation + * (the ring VA -> PA the RAM scan just proved): walk the ring page through + * the candidate and register only the directory that reproduces it. Runs on + * the service thread from the ring-scan latch; the walk itself is cheap (a + * few table reads). + */ +static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, + u64 pa) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = 0; + int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS; + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, va & ~0xfffULL, + ap, &wpa, &wap, false) || + (wpa & ~0xfffULL) != (pa & ~0xfffULL)) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", + d.h_vaspace, d.pdb, d.num_entries, ap); + break; + } +} + void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; @@ -7433,6 +7527,54 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) return free_slot; } +/* + * r506w: bounded per-VA failure latch for virtual ops. A virtual op that + * keeps failing -ENOENT (an unresolvable VA) re-runs on every decode pass + * and sets svc.op_tr_pending each time, so its push holds for the full + * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op + * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx + * after the first client, dst never translated) ground its whole stream + * at ~10 s per push ("hang after DEVICE" for matmul/multikernel). + * After SVC_OPFAIL_LATCH failures of the same VA the op still attempts + * the write but no longer forces the hold: the entry consumes promptly and + * its withheld release is applied, exactly the outcome the expired bound + * already sanctioned (op stays skipped, release delivered), minus the + * 10 s tax per pass. Service-thread only, like every other op path. + */ +#define SVC_OPFAIL_LATCH 8 +#define SVC_OPFAIL_MAX 16 +static struct { + u64 va; + u32 n; +} svc_opfail[SVC_OPFAIL_MAX]; +static u32 svc_opfail_log; + +static bool svc_opfail_latched(u64 va) +{ + u32 i; + u32 free_i = SVC_OPFAIL_MAX; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) { + if (svc_opfail[i].va == va) { + svc_opfail[i].n++; + if (svc_opfail[i].n == SVC_OPFAIL_LATCH && + svc_opfail_log < 32) { + svc_opfail_log++; + pr_info("sema-svc: OPFAIL-LATCH va=0x%llx n=%u (op no longer holds its entry)\n", + va, svc_opfail[i].n); + } + return svc_opfail[i].n >= SVC_OPFAIL_LATCH; + } + if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX) + free_i = i; + } + if (free_i < SVC_OPFAIL_MAX) { + svc_opfail[free_i].va = va; + svc_opfail[free_i].n = 1; + } + return false; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, u32 entry_idx, u64 pass_hash) @@ -7723,7 +7865,8 @@ svc_wait_done: if (rc != 0 && rc != SVC_COPY_FORWARDED && (m->bytes >= (256u << 10) || (m->type == 1 && m->dst >= SVC_VAS_FLOOR) || - (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR))) + (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) && + !svc_opfail_latched(m->dst)) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -8683,7 +8826,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, inc && inc_apply); return false; } - if (pending || inc_apply) { + /* + * r506w: generic absolute releases are forward-only by VALUE, whichever + * pass discovers the gap. The recorded pending bit came from the + * decode's read of the release word, and that read can be blinded by a + * stale/crossed landing (r506 witness: a withheld tracker release went + * to the flush with pending=0 while the fresh read still showed 0, and + * the release was then dropped by the counter fallback below - the + * client's tracker never advanced and its stream ground at the 10 s + * bound per push). Increments keep the one-shot claim gate (`inc_apply`). + */ + if (pending || inc_apply || (!inc && payload > cur)) { svc.pushes_pending++; if (inc) cur += payload; @@ -8705,10 +8858,20 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, return true; } } - if (ch->engine_type == 0) - svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, - inc && inc_apply); + /* + * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its + * `release_pa ?: counter_pa` fallback writes (and forward-only-skips + * against) the channel's REGISTERED slot, which belongs to a different + * word - the r506 witness: releases to the tracker VA 0x20480fef0 were + * skipped against the polluted PUT counter (cur=96) and lost. For a + * non-counter VA the generic write above is the delivery path. + */ + if (svc_counter_va(sem_va)) { + if (ch->engine_type == 0) + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + } /* * Stall-class fix (R2): when the decode said this absolute release is * genuinely pending and every write attempt above failed without @@ -13046,6 +13209,9 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { u64 off, lim; u8 *base = vgpu->bars[1].vaddr; + u64 best = 0, best_dist = ~0ULL; + u32 best_put = 0, best_get = 0; + bool best_pending = false; ch->userd_scans++; lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); @@ -13054,6 +13220,8 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 hp, hg; bool ok = true; u32 k; + u64 dist; + bool pending; memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); @@ -13078,10 +13246,17 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, eva = ((u64)(a1 & 0xff) << 32) | (u64)(a0 & 0xfffffffc); elen = ((a1 >> 10) & 0x1fffff) << 2; + /* + * Every entry the put claims must name a + * VA in the ring's allocation window - + * pushbuffers below the ring, the + * progress tracker above it. + */ if (!eva || - eva >= ch->gpfifo_va || eva + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va || + eva >= ch->gpfifo_va + + SVC_GR_RAM_WIN / 2 || elen < 4 || elen > 0x100000) { ok = false; break; @@ -13089,13 +13264,43 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } if (!ok) continue; - ch->userd_pa = off; + /* + * Several blocks can validate: a previous + * session leaves its old USERD holding the same + * put/entry values while the new channel's block + * sits elsewhere (observed r504: the re- + * registered channel re-scanned, adopted the + * stale block, and its DMA test timed out). + * Prefer a block with work PENDING (get < put); + * among equals prefer the one nearest the + * registered memdesc offset (the driver's own + * hint). The pump's GP_GET write-back goes to + * the adopted block, so a stale adoption strands + * the client's poll. + */ + dist = off > ch->userd_pa ? off - ch->userd_pa + : ch->userd_pa - off; + pending = hg < hp; + if (best && + (best_pending > pending || + (best_pending == pending && dist >= best_dist))) + continue; + best = off; + best_dist = dist; + best_put = hp; + best_get = hg; + best_pending = pending; + } + if (best) { + ch->userd_pa = best; ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; - *put = hp; - *get = hg; - pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u, ring-validated)\n", - ch->h_object, ch->gr_index, off, hp, hg); + *put = best_put; + *get = best_get; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", + ch->h_object, ch->gr_index, best, + best_put, best_get, + best_pending ? "" : ", idle"); return 0; } } @@ -14220,6 +14425,54 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, return false; } +/* + * r496f: discover a kernel-RM client VAS's page directory in the emulated + * VRAM by verification. The CPU RM builds internal-VAS page tables in + * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally + * owned / UVM-only RPC), so the pushbuffer pages and the progress + * semaphore of an X/GL channel cannot be translated from any existing + * source. Try each 4 KB page as the walk root and accept the first whose + * walk reproduces the ring VA -> ring PA pair we already validated by + * content; registering it with the VAS table makes every other VA of the + * same client resolvable through the existing walker. Bounded to the + * vidmem shadow; verified, so a wrong candidate cannot be registered. + */ +static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, + u64 ring_pa) +{ + static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 }; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + unsigned int ei; + + if (!vgpu->bars[1].vaddr || !lim) + return; + for (off = 0; off + 0x1000 <= lim; off += 0x1000) { + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) { + u64 pa = 0; + int ap = 0; + + if (!svc_gmmu_walk(vgpu, off, ent_tab[ei], + ring_va & ~0xfffULL, + SVC_APERTURE_FB, &pa, &ap, false) && + (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) { + struct sema_vaspace_desc d; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | + (u32)(ring_va >> 20); + d.pdb = off; + d.num_entries = ent_tab[ei]; + d.b_vidmem = true; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + off, ent_tab[ei], ring_va, ring_pa); + return; + } + } + } +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -14552,24 +14805,63 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, u64 hit_pa = 0; if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) { - u64 win_va = ch->gpfifo_va - SVC_GR_RAM_WIN / 2; - u64 win_pa = hit_pa - SVC_GR_RAM_WIN / 2; - u64 off; - + /* + * Pin ONLY the content-validated ring page (validated, + * not a window guess). The pushbuffer pages and the + * tracker push of this client resolve through the VAS + * page directory registered from COPY_SERVER_PDES + * (r496g) - the surface is NONCONTIGUOUS, so a linear + * window guess is wrong for them and, being a + * pin_lookup_any hit, would shadow the correct walk in + * svc_lookup_trusted (observed r502: the tracker push + * read zeros through the wrong window page and the + * driver's DMA poll never completed). + */ svc_pin_store(ch->gpfifo_va, hit_pa, SVC_APERTURE_SYS); svc_lat_store(ch->gpfifo_va, hit_pa, SVC_APERTURE_SYS); - for (off = 0; off < SVC_GR_RAM_WIN; - off += SEMA_PAGE_SIZE) { - svc_pin_store_win(win_va + off, win_pa + off, - SVC_APERTURE_SYS); - svc_lat_store(win_va + off, win_pa + off, - SVC_APERTURE_SYS); - } ch->gr_ram_logged = true; + ch->gr_ram_ring = true; set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; + /* + * r496j: with the ring content-validated, verify the + * parked COPY_SERVER_PDES candidates against it and + * register the directory that reproduces the ring's + * PA; its walk then resolves the client's other VAs + * (pushbuffer pages, the progress semaphore the + * driver polls). r496f's blind vidmem sweep stays as + * a fallback for directories the control never + * published. + */ + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + hit_pa); + svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* + * Persist the latch immediately: a later stalled + * path in this pass may `continue` past the + * end-of-pass write-back, and the next pass's + * snapshot must carry gr_ram_ring (the zero-NOP + * exception and the no-capture-hold skip key on + * it). + */ + { + unsigned long flags2; + int i2; + + spin_lock_irqsave(&svc.chan_lock, flags2); + for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++) + if (svc.chan[i2].valid && + svc.chan[i2].vgpu == vgpu && + svc.chan[i2].h_object == + ch->h_object) { + svc.chan[i2].gr_ram_ring = true; + svc.chan[i2].gr_ram_logged = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags2); + } pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", ch->h_object, ch->gr_index, ch->gpfifo_va, hit_pa); @@ -21040,10 +21332,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool same = ident && + /* + * r505b: the per-slot op bookkeeping arrays are + * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL + * channel's ring is 8192 slots. Bound every slot + * index here or a large ring writes out of + * bounds. + */ + bool same = ident && idx < SVC_GPFIFO_MAX_BITS && ch->slot_ops_ident[idx] == ident; - entry_ident = ident; + entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan @@ -21156,7 +21455,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) injected = (svc_ipin_lookup(va, &ipa, &iap) == 0); - if (!svc_method_stream_plausible(pb, len)) { + /* + * r496e: an all-zero push on a + * content-scanned-ring channel is the + * driver's NOP sanity push (one zero + * dword - NVA06F_DMA_NOP), not a bad + * translation: accept it and let the + * decode find no ops. + */ + bool zero_nop = ch->gr_ram_ring && + len >= 4 && + !memchr_inv(pb, 0, len); + + if (!svc_method_stream_plausible(pb, len) && + !zero_nop) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, @@ -21484,7 +21796,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - if (svc.op_tr_pending) + if (svc.op_tr_pending) { /* * Deadline hit: the * operand never @@ -21496,6 +21808,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_flush(vgpu, ch, (u32)idx); + /* + * r505b: mark the slot's + * ops complete. The op was + * permanently skipped, but + * the decode above stored + * complete=0 (op_tr_pending + * was set), so every pin- + * batch rewind replay of + * this slot re-decoded it, + * re-set op_tr_pending and + * re-held the entry in a + * livelock (multikernel on + * r505: opdef held=18359, + * expired=61, the client + * waiting forever while the + * pump looped on slot 1). + * Nothing is owed after the + * flush, so a replay must + * skip the ops. + */ + if ((u32)idx < + SVC_GPFIFO_MAX_BITS) { + ch->slot_ops_complete[idx] = + 1; + ch->slot_ops_done[idx] = + entry_done; + } + } rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -21975,6 +22315,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies; } if (gsp_sema_gr_nocap_hold_ms && + !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( @@ -22724,6 +23065,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; svc.chan[k].gr_ram_passes = ch->gr_ram_passes; svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + svc.chan[k].gr_ram_ring = ch->gr_ram_ring; svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 305a027..ee5c48a 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -81,6 +81,14 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); +/* + * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a + * content-validated walk proves it belongs to a live client; only then is + * it registered (a blind registration let stale kernel directories answer + * other clients' VAs in the walker's first-success order). + */ +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem); /* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state * (direct-map base, content-scanned tables) on the svc thread. Client * session flushes keep that state - see svc_session_flush. */ |