summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c79
-rw-r--r--kernel/gsp_rm_emulation.c146
-rw-r--r--kernel/gsp_sema_service.c4007
-rw-r--r--kernel/gsp_sema_service.h27
-rw-r--r--kernel/main.c147
5 files changed, 4274 insertions, 132 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 6fae2f2..534a443 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -1668,6 +1668,20 @@ enum gsp_phys_rmcall_buf {
#define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u
#define GSP_PHYS_RMCALL_POLL_MS 5u
/*
+ * SESSION 148 perf (termbin review, verified in-tree): every RPC reply wait
+ * polled with msleep(5). With several quanta per round trip and ~8
+ * serialized PDE_UPDATE forwards per forwarded GR launch, the 5 ms quantum
+ * dominated per-launch latency (~95% of the ~189 ms host service time per
+ * launch). Wait with sub-millisecond quanta instead: the reply is observed
+ * within one quantum, cutting each round trip to ~1 ms. POLL_MS remains for
+ * the poll accounting prints.
+ */
+#define GSP_PHYS_RMCALL_POLL_US 150u
+static inline void gsp_phys_rmcall_wait_quantum(void)
+{
+ usleep_range(GSP_PHYS_RMCALL_POLL_US, GSP_PHYS_RMCALL_POLL_US + 200u);
+}
+/*
* GR launch release poll window. A CE copy completes in single-digit ms, but
* a GR launch can force the engine's first context switch for the channel;
* the FECS ctxsw path is measured in fractions of a second, so the window is
@@ -2634,6 +2648,18 @@ EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_flags);
static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c,
u32 page)
{
+ /*
+ * SESSION 141: an unallocated chunk (no per-page array AND no base
+ * va) must answer NULL. It previously returned
+ * (u8 *)NULL + page*PAGE_SIZE - a small non-NULL garbage pointer -
+ * so the caller's NULL guard passed and the empty-set post-mortem
+ * print dereferenced it: Oops #2 RIP
+ * gsp_phys_backend_guest_gr_submit+0x4394, CR2=0x1e93c with slot 30
+ * (0x1e000 = 30*0x1000 + 0x900 off). min_data=0's no-external-data
+ * forward hit this on its first pass.
+ */
+ if (!c->page_va && !c->va)
+ return NULL;
if (c->page_va)
return c->page_va[page];
return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE;
@@ -15359,7 +15385,7 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
gsp_phys_status_drain(NULL);
return -ETIMEDOUT;
}
- msleep(GSP_PHYS_RMCALL_POLL_MS);
+ gsp_phys_rmcall_wait_quantum();
}
gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
@@ -15459,7 +15485,7 @@ static void gsp_phys_rmcall_ctx_image_probe(void __iomem *bar0)
offs[k], poll);
break;
}
- msleep(GSP_PHYS_RMCALL_POLL_MS);
+ gsp_phys_rmcall_wait_quantum();
}
if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD)
break;
@@ -15560,7 +15586,7 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va,
*sem_out = poll;
return -ETIMEDOUT;
}
- msleep(GSP_PHYS_RMCALL_POLL_MS);
+ gsp_phys_rmcall_wait_quantum();
}
}
@@ -15743,7 +15769,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va,
gsp_phys_work_timed_out);
return -ETIMEDOUT;
}
- msleep(GSP_PHYS_RMCALL_POLL_MS);
+ gsp_phys_rmcall_wait_quantum();
}
/*
@@ -16262,7 +16288,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
ret = -ETIMEDOUT;
goto out;
}
- msleep(GSP_PHYS_RMCALL_POLL_MS);
+ gsp_phys_rmcall_wait_quantum();
}
/*
@@ -17678,8 +17704,40 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va)
* launches resolve 968-969 records (the client's whole arena); a launch
* with only a handful after the fill's drops has an unresolved argument
* set and must not be handed to the engine (see the submit guard).
+ *
+ * SESSION 127: the Vulkan inline-QMD launch (the C5C0 translator) is a
+ * compute dispatch whose operand set is one descriptor page and one
+ * parameter bank - a handful of records is healthy for it. Tunable module
+ * parameter (default 16 preserves the CUDA-era threshold); the translator
+ * cycles pass gsp_phys_gr_min_data=2.
*/
#define GSP_PHYS_GR_MIN_DATA_RECS 16u
+static unsigned int gsp_phys_gr_min_data_param = GSP_PHYS_GR_MIN_DATA_RECS;
+module_param_named(gsp_phys_gr_min_data, gsp_phys_gr_min_data_param, uint, 0644);
+MODULE_PARM_DESC(gsp_phys_gr_min_data,
+ "GR submit minimum data records (degenerate-set refusal threshold, 0..4096; 0 = explicit bypass for the no-external-data launch class; default 16)");
+
+/*
+ * r801 (SESSION 143): 0 is an explicit BYPASS, not a clamp to 1. The
+ * Vulkan launch's single QMD pointer pair names its own QMD window and its
+ * params are scalars, so the launch has no external data by construction;
+ * the bare n_data>=min gate wrongly classifies it degenerate. The bypass is
+ * safe now that gsp_phys_chunk_page_va answers NULL for an unallocated
+ * chunk (r800): before that guard the empty-set post-mortem print
+ * dereferenced a garbage pointer and oopsed at
+ * gsp_phys_backend_guest_gr_submit+0x4394. The proper predicate (all
+ * candidates launch-internal-filtered, none unresolved => not degenerate)
+ * replaces the bare threshold next; until then 0 stays an explicit,
+ * opt-in debug/test value and the default remains 16.
+ */
+static unsigned int gsp_phys_gr_min_data_value(void)
+{
+ unsigned int v = READ_ONCE(gsp_phys_gr_min_data_param);
+
+ if (v > 4096)
+ v = 4096;
+ return v;
+}
#define GSP_PHYS_GR_GAP_MAX 4096u /* total late pages per launch */
/*
* relfix PART 3 (r217 span containment): the late path authors ONLY the
@@ -19289,10 +19347,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
* into the same software-completion fallback without touching the
* engine, and cannot wedge it for the next client.
*/
- if (req->n_data < GSP_PHYS_GR_MIN_DATA_RECS) {
+ if (req->n_data < gsp_phys_gr_min_data_value()) {
gsp_phys_gr_refused++;
pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n",
- req->n_data, GSP_PHYS_GR_MIN_DATA_RECS,
+ req->n_data,
+ gsp_phys_gr_min_data_value(),
req->push_va);
return -ENODATA;
}
@@ -21190,7 +21249,7 @@ gr_page_unmapped:
ret = -ETIMEDOUT;
goto out;
}
- msleep(GSP_PHYS_RMCALL_POLL_MS);
+ gsp_phys_rmcall_wait_quantum();
}
gsp_phys_gr_ok++;
@@ -21595,7 +21654,7 @@ gr_page_unmapped:
}
if (time_after_eq(jiffies, dl))
break;
- msleep(GSP_PHYS_RMCALL_POLL_MS);
+ gsp_phys_rmcall_wait_quantum();
}
pr_info("vgsp-gr: outcome retry %u: fresh ack %s (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x after %u ms)\n",
nw_attempts,
@@ -22648,7 +22707,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
gsp_phys_status_drain(NULL);
return -ETIMEDOUT;
}
- msleep(GSP_PHYS_RMCALL_POLL_MS);
+ gsp_phys_rmcall_wait_quantum();
}
if (memcmp(gsp_phys_chunk_page_va(src, 0),
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index b8a3e37..7b7e230 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -2665,6 +2665,20 @@ static void gsp_rm_handle_kgr_get_context_buffers_info(
* L2/LTC topology from here. Echo the requested indices and fill in the
* emulated TU104 values.
*/
+/*
+ * m113: FB_GET_INFO_V2 COMPRESSION_SIZE (index 0x01, bytes). The header:
+ * "A return value of 0 indicates the GPU does not support compression" -
+ * but the guest driver still attempts a COMPR-enabled, 64 KB-aligned
+ * comptag allocation (vendor VidHeap alloc, attr COMPR=2) and fails it
+ * NV_ERR_NO_MEMORY (0x51) when the comptag pool has no space, which
+ * libcuda maps to VK_ERROR_OUT_OF_DEVICE_MEMORY (the vkAllocateMemory -2
+ * gate). Report a comptag pool sized like the real Turing parts (~fb/16).
+ * 0 = the historic "no compression" reply.
+ */
+static uint gsp_rm_fb_compr_mb = 128;
+module_param_named(wall2_fb_compr_mb, gsp_rm_fb_compr_mb, uint, 0644);
+MODULE_PARM_DESC(wall2_fb_compr_mb, "FB_GET_INFO_V2 COMPRESSION_SIZE in MiB (default 128, 0 = none)");
+
static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index)
{
u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu);
@@ -2673,7 +2687,8 @@ static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index)
case TU104_FB_INFO_IDX_TILE_REGION_COUNT:
return 0;
case TU104_FB_INFO_IDX_COMPRESSION_SIZE:
- return 0;
+ /* m113: non-zero comptag pool (bytes) - see gsp_rm_fb_compr_mb. */
+ return gsp_rm_fb_compr_mb << 20;
case TU104_FB_INFO_IDX_PARTITION_COUNT:
return 1;
case TU104_FB_INFO_IDX_BAR1_SIZE:
@@ -3037,8 +3052,21 @@ static bool gsp_rm_is_channel_class(u32 class_id)
case 0x906f: /* MAXWELL_CHANNEL_GPFIFO_A */
case 0xa06f: /* KEPLER_CHANNEL_GPFIFO_A */
case 0xb06f: /* PASCAL_CHANNEL_GPFIFO_A */
- case 0xc06f: /* VOLTA_CHANNEL_GPFIFO_A */
+ case 0xc06f: /* VOLTA_CHANNEL_GPFIFO_A (PRE-TURING NAME) */
+ case 0xc36f: /* VOLTA_CHANNEL_GPFIFO_A */
+ /*
+ * r828: the RC watchdog's channel allocates 0xc36f on this
+ * device (the driver's gpfifoMapping loop picks the first
+ * class the gpu reports supported) - without it in this set
+ * the channel never reaches the progress service, its class2d
+ * notifier cycle (SET_NOTIFY_A/B + NOTIFY) never executes and
+ * the guest's krcWatchdog declares the GPU locked after its
+ * 7 s timeout as soon as robust channels are in play.
+ */
case 0xc46f: /* TURING_CHANNEL_GPFIFO_A */
+ case 0xc56f: /* AMPERE_CHANNEL_GPFIFO_A */
+ case 0xc86f: /* HOPPER_CHANNEL_GPFIFO_A */
+ case 0xc96f: /* BLACKWELL_CHANNEL_GPFIFO_A */
return true;
default:
return false;
@@ -3060,8 +3088,18 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
{
struct sema_channel_desc desc;
- if (params_size < GSPRM_CHAN_NOTIFIER_MEM + 8)
+ if (params_size < GSPRM_CHAN_NOTIFIER_MEM + 8) {
+ /*
+ * r828 diag: a channel-class alloc whose params are shorter
+ * than the notifier field is not registered, silently. The
+ * RC watchdog channel (hObject 0x31415900) never appeared in
+ * the sema table; name every such skip with its size so the
+ * shape question is answered by a log, not inference.
+ */
+ pr_info("vgsp-rm: channel alloc hObject=0x%x params_size=%u too short (< %u); NOT registered\n",
+ h_object, params_size, GSPRM_CHAN_NOTIFIER_MEM + 8);
return;
+ }
memset(&desc, 0, sizeof(desc));
desc.vgpu = ctx->vgpu;
@@ -3082,6 +3120,20 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
*/
if (desc.engine_type >= 1 && desc.engine_type <= 8)
desc.engine_type = 0;
+ /*
+ * r835: the channel's own address space (NV_CHANNEL_ALLOC_PARAMS:
+ * hContextShare @24, hVASpace @28 - confirmed by this struct's own
+ * known offsets: engineType @128 = 32 + 8 handles + 8 subdevice
+ * handles + 8 u64 userd offsets). Two clients map different objects
+ * at overlapping VAs (everyone's RM window sits at 0x1200xxxxx), so
+ * the global VA-keyed translation maps conflate them: the X server's
+ * ring VA resolved through the GL client's page tables (walk-probe:
+ * vas h=0xcaf00005 answered 0x120090000 with a zero page) and its
+ * push stream decoded as empty, wedging X exactly like the RC
+ * watchdog's ring. The service resolves a channel's VAs through this
+ * VAS first.
+ */
+ memcpy(&desc.h_vaspace, params + 28, 4);
memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4);
memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8);
memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8);
@@ -3505,6 +3557,41 @@ MODULE_PARM_DESC(wall2_a097_rec6, "0x2080a097 reply word at params+0x1c (0xfffff
module_param_named(wall2_a097_rec7, gsp_rm_a097_rec7, uint, 0644);
MODULE_PARM_DESC(wall2_a097_rec7, "0x2080a097 reply word at params+0x20 (0xffffffff = echo)");
+/*
+ * m112b: the NV04_VID_HEAP_CONTROL enumeration pair 0x90960106/0x90960107
+ * (legacy NVOS32 VidHeap API, class 0x9096). libnvidia-eglcore (file offset
+ * 0x586d80) walks three heaps: per heap it queries 0x90960106 (12 bytes:
+ * {u32 firstIdx, u32 lastIdx}; the client then enumerates the heap's memory
+ * blocks by index via 0x90960107 (56 bytes) while firstIdx+1 <= idx <=
+ * lastIdx). The 56-byte block entry is read at +0x00/+0x08 (two u64s), +0x28
+ * (a u32 type compared against a table at eglcore 0x101b1c0) and +0x30 (a
+ * flag BYTE: when 0 the heap/block is skipped). The generic zero reply makes
+ * every heap look empty, the client concludes out-of-device-memory and
+ * vkAllocateMemory fails -2 WITHOUT any alloc reaching the emulation
+ * (dmesg-corroborated: only 0x9096* controls + a class 0xc5c0 alloc).
+ * Runtime-tunable so one boot can sweep reply variants (same pattern as the
+ * wall2_a097_* words).
+ */
+static uint gsp_rm_9096_lo; /* 0x90960106 firstIdx */
+static uint gsp_rm_9096_hi = 1; /* 0x90960106 lastIdx */
+module_param_named(wall2_9096_lo, gsp_rm_9096_lo, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_lo, "0x90960106 reply firstIdx (default 0)");
+module_param_named(wall2_9096_hi, gsp_rm_9096_hi, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_hi, "0x90960106 reply lastIdx (default 1)");
+static uint gsp_rm_9096_ent_addr; /* 0x90960107 entry +0x00 (u64) */
+module_param_named(wall2_9096_ent_addr, gsp_rm_9096_ent_addr, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_ent_addr, "0x90960107 entry word at +0x00 (default 0)");
+static uint gsp_rm_9096_ent_size = 0x80000000u; /* entry +0x08 (u64) */
+module_param_named(wall2_9096_ent_size, gsp_rm_9096_ent_size, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_ent_size, "0x90960107 entry word at +0x08 (default 0x80000000)");
+static uint gsp_rm_9096_ent_type = 3; /* entry +0x28 (u32) */
+module_param_named(wall2_9096_ent_type, gsp_rm_9096_ent_type, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_ent_type, "0x90960107 entry word at +0x28 (default 3)");
+static uint gsp_rm_9096_ent_flag = 1; /* entry +0x30 (u8) */
+
+module_param_named(wall2_9096_ent_flag, gsp_rm_9096_ent_flag, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_ent_flag, "0x90960107 entry flag byte at +0x30 (default 1)");
+
/*
* The live session's oracle client/object for the 0x20810107 experiment,
@@ -4217,6 +4304,18 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
"vgsp-rm: promote[%u] bufferId=%u bInitialize=%u bNonmapped=%u physAttr=0x%x pa=0x%llx va=0x%llx size=0x%llx",
i, bid, e[30], e[31], attr, pa,
eva, esz);
+ /*
+ * r820: register the driver's own statement
+ * of this kernel buffer's backing (r817
+ * Xorg GL init: the page-table walk answered
+ * a stale zero page for the pushbuffer VA
+ * while this entry named the real one; a
+ * userspace pin scanner can never cover
+ * kernel-owned buffers). See
+ * gsp_sema_service_promote_add().
+ */
+ gsp_sema_service_promote_add(eva, pa, esz,
+ attr, bid);
}
}
out_status = 0;
@@ -4348,6 +4447,47 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
break;
}
+ case 0x90960106: { /* NV04_VID_HEAP_CONTROL block-index range (m112b) */
+ u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE;
+ const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE;
+
+ if (req.paramsSize >= 12) {
+ memcpy(rp, pp, req.paramsSize);
+ *(u32 *)(rp + 0) = gsp_rm_9096_lo;
+ *(u32 *)(rp + 4) = gsp_rm_9096_hi;
+ if (gsp_sema_dbg)
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: ctrl 0x90960106 -> first=%u last=%u",
+ gsp_rm_9096_lo, gsp_rm_9096_hi);
+ }
+ out_status = 0;
+ out_params_size = req.paramsSize;
+ break;
+ }
+
+ case 0x90960107: { /* NV04_VID_HEAP_CONTROL block entry by index (m112b) */
+ u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE;
+ const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE;
+
+ if (req.paramsSize >= 56) {
+ memcpy(rp, pp, req.paramsSize);
+ *(u64 *)(rp + 0x00) = (u64)gsp_rm_9096_ent_addr;
+ *(u64 *)(rp + 0x08) = (u64)gsp_rm_9096_ent_size;
+ *(u32 *)(rp + 0x28) = gsp_rm_9096_ent_type;
+ *(u8 *)(rp + 0x30) = (u8)gsp_rm_9096_ent_flag;
+ if (gsp_sema_dbg)
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: ctrl 0x90960107 -> addr=0x%x size=0x%x type=%u flag=%u",
+ gsp_rm_9096_ent_addr,
+ gsp_rm_9096_ent_size,
+ gsp_rm_9096_ent_type,
+ gsp_rm_9096_ent_flag);
+ }
+ out_status = 0;
+ out_params_size = req.paramsSize;
+ break;
+ }
+
default:
/* Unknown control command - return success with zeroed params.
* Do NOT echo the request: pure queries pass uninitialized
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 993e507..15cc93f 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -205,6 +205,18 @@ struct svc_channel {
u32 h_object;
u32 engine_type;
u32 cid; /* RM channel id (doorbell token vector) */
+ u32 h_vaspace; /* r835: the client VAS this channel was
+ * allocated in (NV_CHANNEL_ALLOC_PARAMS.
+ * hVASpace). Channel-scoped VA resolution
+ * consults this VAS's page tables FIRST:
+ * every client's RM window sits at the same
+ * VAs (0x1200xxxxx) and the global VA-keyed
+ * maps conflate them - the X server's ring
+ * VA resolved through the GL client's page
+ * tables (walk-probe vas h=0xcaf00005
+ * answered 0x120090000) and its push stream
+ * decoded as zero pages, wedging X the same
+ * way the RC watchdog's ring was starved. */
u8 ce_index; /* rank among the live CE channels (creation order) */
u8 gr_index; /* rank among the live GR channels (creation order) */
u64 gpfifo_va;
@@ -506,6 +518,20 @@ struct svc_channel {
*/
u8 userd_scans;
/*
+ * r828: the RC watchdog's notifier VA, captured from the class2d
+ * SET_NOTIFY_A/B methods on subchannel 3 and zeroed at the NOTIFY
+ * (see svc_notify2d_write). Survives the pass snapshot via the
+ * write-back list.
+ */
+ u64 notify2d_va;
+ /* r829: bounded count of zero-entry (bad-translation) holds; see the
+ * zero-entry hold in svc_pump_channels. Persists via the write-back. */
+ u32 zero_held;
+ /* r831: the registered userd value is a real guest PA (the driver's
+ * own declaration; e.g. the RC watchdog channel). The ring auto-pin's
+ * ring+0x2000 convention must not overwrite it. */
+ bool userd_reg;
+ /*
* r471: the completion-counter publish must never advance the
* guest's word past a push whose content was still unwritten when
* the entry was consumed (the streams_events staging-reuse
@@ -652,6 +678,21 @@ static u32 svc_ring_n;
*/
static u8 svc_sess_flush;
static u8 svc_guest_reset_req; /* m177: guest RM reload; wiped on svc thread */
+static u8 svc_sess_flush_forced; /* execute the next flush past the live-channel skip
+ * (ISSUE-0002 client-session boundary aid; see
+ * gsp_sema_service_sess_flush_force) */
+/*
+ * SESSION 148 perf tracing: sema_trace_h selects one channel handle; the
+ * pump then logs a compact state line for it once a second, and
+ * svc_counter_publish logs why it skipped (last_get gate / arming gate) or
+ * the value it wrote. This is the instrument for locating the 22.4 s/frame
+ * glxgears pacer without guessing at windows.
+ */
+unsigned long long gsp_sema_trace_h;
+module_param_named(sema_trace_h, gsp_sema_trace_h, ullong, 0644);
+MODULE_PARM_DESC(sema_trace_h,
+ "Trace one channel handle (0x...; 0=off): 1/s state + publish gates");
+static unsigned long svc_perf_last;
static u8 svc_sess_active; /* a channel of a session has registered (until flush) */
static u32 svc_sess_gen; /* m176: bumped by every executed session flush;
* stamps channels/scrubbers so per-session learned
@@ -786,6 +827,12 @@ static u32 svc_pb_idle; /* consecutive passes with no new base */
static u32 svc_pb_last_n; /* bases before the active pass */
static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */
static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */
+/* SESSION 133: ranks that already consumed their one shadow ring scan. */
+static unsigned long svc_gr_ring_scan_done;
+static unsigned long svc_gr_ring_scan_last; /* rate limit for the scan */
+/* SESSION 138: bounded witness for the GR request builder's page list and
+ * the fill's candidate set (one launch's wiring printed per budget slice). */
+static u32 svc_gr_pages_log;
static bool svc_pb_first_push(const u8 *buf)
{
@@ -958,6 +1005,8 @@ struct sema_svc {
u64 entry_norel_log;
u64 ops_denied; /* work refused: implausible stream */
u64 false_trans; /* resolvable VA mapped to non-push */
+ u64 rescue_map_ok; /* SESSION 148: content-verified
+ * map-rescue successes (GR + CE) */
u64 sem_write_reresolve; /* writes re-resolved after read-back */
u64 pool_alt_applied; /* pool releases fanned to a
* displaced pin (ISSUE-0002) */
@@ -1136,6 +1185,17 @@ struct sema_svc {
u64 fwd_gr_ok;
u64 fwd_gr_fallback;
u64 fwd_gr_refused;
+ u32 fwd_gr_nocand_log; /* SESSION 127: first-N
+ * request-without-candidate
+ * witnesses */
+ u64 fwd_gr_nordy; /* SESSION 127: backend-not-
+ * ready request refusals */
+ u64 fwd_gr_holdbusy;/* SESSION 127: hold-active
+ * request busy verdicts */
+ u64 fwd_gr_inflight;/* SESSION 127: inflight-bit
+ * request busy verdicts */
+ u32 fwd_gr_enoe_log;/* SESSION 127: first-N fill
+ * ENOENT page witnesses */
u64 fwd_gr_defer; /* m150: held for data-arg retry */
u64 fwd_gr_dstpend; /* r541: held for an unexecuted
* device write on a data page */
@@ -1541,10 +1601,21 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
ch->h_object = desc->h_object;
ch->engine_type = desc->engine_type;
ch->cid = desc->cid;
+ ch->h_vaspace = desc->h_vaspace;
ch->gpfifo_va = desc->gpfifo_va;
ch->gpfifo_entries = desc->gpfifo_entries;
ch->userd_pa = desc->userd_pa;
ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS;
+ /*
+ * r831: a registered userd value that looks like a real guest
+ * PA (the RC watchdog's channel carries its own userd block)
+ * is the driver's own declaration and outranks the ring+0x2000
+ * convention guess the ring auto-pin applies (that guess is
+ * right for the CUDA GR layout, where the registered value is
+ * pool-relative - observed 0x10000). Tiny registered values
+ * stay guesses.
+ */
+ ch->userd_reg = desc->userd_pa >= 0x10000000ULL;
ch->instance_pa = desc->instance_pa;
ch->notifier_pa = desc->notifier_pa;
ch->notifier_as = desc->notifier_as ? desc->notifier_as
@@ -1586,12 +1657,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
* channel; the stashed geometry survives, so the scrub path
* does not depend on the channel-table identity at all.
*/
- if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va &&
+ /*
+ * m112b: the CeUtils scrubber is the ONLY exactly-4096-entry
+ * channel (NUM_COPY_BLOCKS). The old `>= 4096` test also
+ * matched every client channel (this driver version: 32768
+ * entries, not the 1024 the comment below assumed), so the
+ * single scrub slot bound to whichever client channel
+ * registered last (observed r701: the slot held h=0xbeee0102
+ * userd=0x8f400/entries=32768 and the pass authored put=0 for
+ * the wrong channel while the real scrubber's payload stayed
+ * zero - the vkAllocateMemory -2 gate).
+ */
+ if (desc->gpfifo_entries == 4096 && desc->gpfifo_va &&
desc->userd_pa) {
struct svc_scrub *s = svc_scrub_get(desc->vgpu, true);
if (!s)
goto scrub_done; /* table full: max guests */
+ pr_info("sema-svc: scrub stash bind vgpu=%px slot=%d h=0x%x entries=%u\n",
+ (void *)desc->vgpu, (int)(s - svc_scrubs),
+ desc->h_object, desc->gpfifo_entries);
/* Re-seed the counting on a fresh scrubber object or a
* new client session (m176: the counter/submitted state
* is per-session; a recycled scrubber handle after a
@@ -1616,6 +1701,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
desc->h_object, desc->gpfifo_va,
desc->gpfifo_entries, desc->userd_pa,
s->userd_as);
+ } else if (desc->gpfifo_entries >= 4096 &&
+ desc->gpfifo_entries != 32768) {
+ /* m112b: a non-32768 channel with >=4096 entries that
+ * fails the stash condition (missing VA or USERD PA)
+ * is a scrubber candidate the service cannot author
+ * for - log it so a silent no-authoring boot is
+ * diagnosable. */
+ pr_info_ratelimited("sema-svc: scrubber-stash MISS h=0x%x entries=%u va=0x%llx userd_pa=0x%llx\n",
+ desc->h_object, desc->gpfifo_entries,
+ desc->gpfifo_va, desc->userd_pa);
}
scrub_done:
pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
@@ -1662,6 +1757,8 @@ static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */
static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */
static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */
static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */
+static u32 svc_gr_push_dump; /* SESSION 127: bounded capture tail dump
+ * (sema_dbg only, 4 per boot) */
static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */
static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */
static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */
@@ -1814,6 +1911,64 @@ void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
* the lesser evil; it is logged loudly for the record.
*/
#define SVC_VAS_CAND_AGE_MS 15000u
+/*
+ * r705h: the candidate age-out deadline is runtime-tunable. The parked
+ * COPY_SERVER_PDES candidate for the GL/Vulkan client ages out (UNVERIFIED)
+ * at exactly the moment the client's channels first register, so the
+ * content-verified walk can never anchor it; raising the deadline keeps the
+ * candidate parked until a host tool supplies a verified ring pair via
+ * sema_pdb_discover. 0 disables the age-out entirely.
+ */
+static u32 svc_vas_cand_age_ms = SVC_VAS_CAND_AGE_MS;
+module_param_named(sema_vas_cand_ms, svc_vas_cand_age_ms, uint, 0644);
+
+/*
+ * SESSION 127 C5C0 translator gate. When 1, the sweep additionally
+ * translates continuation entries into launch work (tail-aware method
+ * parse, inline-QMD capture, LOAD_INLINE_DATA materialization, C5B5
+ * copy-out capture, candidate-owned release withholding).
+ *
+ * SESSION 130 re-test of the r727 "translator regressions" (translator on,
+ * utility sweep OFF, r746): victory_saxpy and victory_math now PASS with
+ * checksums identical to the gate-off baseline (saxpy
+ * csum=981440890.500000), so the saxpy/math half of the r727 regression
+ * is FIXED (the SESSION 129/130 sweep changes); victory_mail PASSES. The
+ * remaining xlate-on regression is victory_multikernel: its launch is
+ * captured and submitted to the physical channel but the forward times
+ * out repeatedly (`GR LAUNCH FAILED: timeout after 2000 ms
+ * (qmd_rel=0xc0ffee01 want=0x600df00d ...) h=0x5c000019
+ * push=0x20043c394`, `hold active`), so the client hangs - the default
+ * stays 0 until that forward path is fixed. The UTILITY SWEEP
+ * (`sema_gr_util_scan`) is the OTHER regression source: at 24 it broke
+ * saxpy (`bad=1047527`) and hung math on r745, so it stays opt-in
+ * (default 0) until its channel reads prefer the direct walk over pins.
+ */
+static bool gsp_sema_gr_xlate;
+module_param_named(sema_gr_xlate, gsp_sema_gr_xlate, bool, 0644);
+MODULE_PARM_DESC(sema_gr_xlate,
+ "C5C0 translator: sweep continuation entries into launch work (default 0 = releases-only sweep)");
+/*
+ * SESSION 128 utility-channel inline-download materialization
+ * (sema_gr_util_scan, default 0 = off): the driver's non-GR upload
+ * channels (the MEM2MEM/A140 + 2D utilities, registered but with an
+ * unresolved USERD so the pump's put-bounded entry loop never decodes
+ * them) carry the engine-side code/data downloads: the
+ * GRCE/INLINE_TO_MEMORY operand group on the mem2mem subchannel (0x180..
+ * 0x18c), LAUNCH_DMA (0x1b0) and a LOAD_INLINE_DATA (0x1b4) run whose
+ * payload the engine fetches from the next GPFIFO entry (the
+ * source-view call). Without this pass those destinations stay zero in
+ * the vidmem shadow - observed: the Vulkan client's code-heap/SASS
+ * upload (12 KB) and command-stream staging (256 B) never land, so the
+ * forwarded launch has no program content. Bounded: scan up to N
+ * entries per channel per pass, materialize once-only keyed by
+ * (push_va, content hash, dst, bytes) via the same
+ * svc_grce_inline_write the GR sweep uses. Requires sema_gr_xlate.
+ */
+static u32 gsp_sema_gr_util_scan;
+module_param_named(sema_gr_util_scan, gsp_sema_gr_util_scan, uint, 0644);
+MODULE_PARM_DESC(sema_gr_util_scan,
+ "Utility-channel i2m sweep: entries scanned per pass (default 0 = off; requires sema_gr_xlate)");
+MODULE_PARM_DESC(sema_vas_cand_ms, "Parked-VAS-candidate age-out (ms, default 15000; 0 = never)");
static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu)
{
@@ -1824,8 +1979,9 @@ static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu)
if (!svc_vas_cand[i].valid)
continue;
- if (time_before(jiffies, svc_vas_cand[i].stamp +
- msecs_to_jiffies(SVC_VAS_CAND_AGE_MS)))
+ if (!svc_vas_cand_age_ms ||
+ time_before(jiffies, svc_vas_cand[i].stamp +
+ msecs_to_jiffies(svc_vas_cand_age_ms)))
continue;
memset(&d, 0, sizeof(d));
d.h_vaspace = svc_vas_cand[i].h_vaspace;
@@ -1833,7 +1989,7 @@ static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu)
d.num_entries = svc_vas_cand[i].entries;
d.b_vidmem = svc_vas_cand[i].vidmem;
pr_warn("sema-svc: VAS 0x%x page directory registered UNVERIFIED after %u ms parked (pdb=0x%llx entries=%u; the content-verified ring walk never anchored this boot)\n",
- d.h_vaspace, SVC_VAS_CAND_AGE_MS, d.pdb,
+ d.h_vaspace, svc_vas_cand_age_ms, d.pdb,
d.num_entries);
gsp_sema_service_vaspace_add(&d);
svc_vas_cand[i].valid = false;
@@ -2651,6 +2807,139 @@ static void svc_lat_store(u64 va, u64 addr, int ap)
}
/*
+ * r820: driver-declared kernel-buffer translations (PROMOTE_CTX).
+ *
+ * The guest RM announces the physical backing of a channel's context
+ * buffers through NV2080_CTRL_CMD_GPU_PROMOTE_CTX: every entry carries
+ * (gpuVirtAddr, gpuPhysAddr, size, physAttr, bufferId). These are
+ * KERNEL-owned buffers - the nvidia-push pushbuffer, the progress-tracker
+ * segment, the FECS/patch/access-map blocks - so the userspace pin
+ * scanner can never cover them (/proc/<pid>/pagemap has no kernel VAs)
+ * and the page-table walk can answer with a stale/wrong page. Observed
+ * r817 (Xorg GL init): the walk resolved the 64 KB pushbuffer VA
+ * 0x120010000 to pa=0x103fed000, a page reading all zeros, while the
+ * driver's own promote entry said pa=0x11e120000. The NOP sanity push of
+ * __nvPushTestPushBuffer then never decoded, GP_GET never advanced and
+ * the X server died with "Failed to initialize DMA".
+ *
+ * The registry is a small table keyed by the entry VA, newest wins: a
+ * client that re-promotes the same buffer - observed between two Xorg
+ * starts, the same channel object announced a NEW pa for the same VA -
+ * replaces the old record, so a stale backing is never preferred over
+ * the live statement. VA-only bookings (bInitialize=0 re-promotes of
+ * already-initialized buffers, zero PA/size) and bNonmapped entries
+ * (va=0) carry no backing and must not erase the last statement.
+ *
+ * Lookups run before the learned cache in svc_lookup_trusted: the wrong
+ * r817 walk answer had been cached in the learned table (svc_lat_store
+ * after every walk success), so a promote hit must both pre-empt and
+ * overwrite that entry; the hit re-stores the corrected page into the
+ * learned table so the direct svc_lat_lookup callers heal too.
+ */
+#define SVC_PROMO_MAX 256
+struct svc_promo_rec {
+ u64 va;
+ u64 pa;
+ u64 size;
+ unsigned long seq;
+ u32 phys_attr;
+ u16 buffer_id;
+ bool valid;
+};
+static struct svc_promo_rec svc_promo[SVC_PROMO_MAX];
+static unsigned long svc_promo_seq;
+static DEFINE_SPINLOCK(svc_promo_lock);
+static u32 svc_promo_hit_log; /* bounded "answered from PROMOTE_CTX" witness */
+
+void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr,
+ u32 buffer_id)
+{
+ struct svc_promo_rec *rec = NULL, *free_rec = NULL, *oldest = NULL;
+ unsigned long flags;
+ int i;
+
+ if (!va || !pa || !size)
+ return;
+
+ spin_lock_irqsave(&svc_promo_lock, flags);
+ for (i = 0; i < SVC_PROMO_MAX; i++) {
+ if (svc_promo[i].valid && svc_promo[i].va == va) {
+ rec = &svc_promo[i];
+ break;
+ }
+ if (!svc_promo[i].valid && !free_rec)
+ free_rec = &svc_promo[i];
+ if (svc_promo[i].valid &&
+ (!oldest || svc_promo[i].seq < oldest->seq))
+ oldest = &svc_promo[i];
+ }
+ if (!rec)
+ rec = free_rec ? free_rec : oldest;
+ if (!rec) {
+ spin_unlock_irqrestore(&svc_promo_lock, flags);
+ return;
+ }
+ rec->va = va;
+ rec->pa = pa;
+ rec->size = size;
+ rec->phys_attr = phys_attr;
+ rec->buffer_id = (u16)buffer_id;
+ rec->seq = ++svc_promo_seq;
+ rec->valid = true;
+ spin_unlock_irqrestore(&svc_promo_lock, flags);
+}
+
+static int svc_promo_lookup(u64 va, u64 *addr, int *ap)
+{
+ unsigned long flags;
+ u64 best_seq = 0;
+ int rc = -ENOENT;
+ int i;
+
+ spin_lock_irqsave(&svc_promo_lock, flags);
+ for (i = 0; i < SVC_PROMO_MAX; i++) {
+ const struct svc_promo_rec *rec = &svc_promo[i];
+
+ /*
+ * r823: answer ONLY the base page. The promoted (va, size)
+ * pair is a VA-space booking, not a contiguous mapping: the
+ * ctx MAIN buffer's span [0x120020000, 0x120153000) contains
+ * OTHER buffers' VAs (the channel GPFIFO at 0x120090000), and
+ * the range arithmetic answered such an interior VA with
+ * MAIN.pa + offset - a zero page (observed r823: the GPFIFO
+ * VA resolved to 0x11ce70000, the ring-entry validation read
+ * va=0x0 len=0 and the latch never fired). Only the page at
+ * the declared VA has a proven claim to the declared PA; the
+ * pushbuffer's NOP sits exactly there (nvPushHeader writes at
+ * the segment base).
+ */
+ if (!rec->valid || va < rec->va ||
+ va - rec->va >= SEMA_PAGE_SIZE)
+ continue;
+ if (best_seq && rec->seq <= best_seq)
+ continue;
+ best_seq = rec->seq;
+ *addr = rec->pa + (va - rec->va);
+ /*
+ * physAttr 0x4 was observed only on vidmem entries (pa in
+ * the emulated FB aperture, e.g. 0x1f4000000), 0x5/0x6 on
+ * system memory. Unknown values fall to SYS, whose read
+ * guard (pa < SEMA_PA_MAX) still rejects nonsense.
+ */
+ *ap = (rec->phys_attr == 0x4) ? SVC_APERTURE_FB :
+ SVC_APERTURE_SYS;
+ rc = 0;
+ }
+ spin_unlock_irqrestore(&svc_promo_lock, flags);
+ if (!rc && svc_promo_hit_log < 64) {
+ svc_promo_hit_log++;
+ pr_info("sema-promo: va=0x%llx -> pa=0x%llx ap=%d (driver PROMOTE_CTX statement)\n",
+ va, *addr, *ap);
+ }
+ return rc;
+}
+
+/*
* VA shadow: page-granular backing for device VAs whose VA->PA translation
* cannot be derived from the guest's page tables.
*
@@ -2984,6 +3273,52 @@ struct svc_ipin_alt {
static struct svc_ipin_alt svc_ipin_alt[SVC_IPIN_ALT_SLOTS];
static u64 svc_pushalt_hits; /* push reads served by a displaced pair */
static u32 svc_pushalt_log;
+/* r707x: per-client push-read attempt witness (pump read site). */
+static u32 svc_rdw_log;
+/* r820: bounded per-session USERD-scan decision witness (svc_chan_userd).
+ * GR channels only and capped: a hot CE channel re-running this every
+ * ~16 ms burned the whole budget before the channel under diagnosis ever
+ * appeared. Each scan reports its decision, a scan that found nothing
+ * reportable logs a sweep summary. */
+static u32 svc_userdp_log;
+/* r829: bounded witness for the zero-entry holds (bad ring translations). */
+static u32 svc_zero_entry_log;
+static u8 svc_read_last_src;
+/* r715: PA the winning source of the last push read resolved to. */
+static u64 svc_read_last_pa;
+/*
+ * r820: set when the last svc_read_va_push_ex chunk was answered by a
+ * driver-declared PROMOTE_CTX kernel buffer. The push decode uses it to
+ * accept the NOP sanity push: nvidia-push's __nvPushTestPushBuffer writes
+ * NVA16F_DMA_NOP (0x00000000) into a promoted kernel pushbuffer, so the
+ * whole small push is zeros - unparseable as a method stream, but exactly
+ * right from a translation the driver itself declared.
+ */
+static bool svc_read_last_promote;
+/* r715: per-client push-decode branch witness (pump read site). */
+static u32 svc_pushdec_log;
+/* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */
+static u32 svc_pushfail_log;
+/*
+ * SESSION 132: the util-sweep witnesses are per-CLIENT budgets (reset in
+ * svc_session_flush), not per-boot: on a multi-client boot the driver-init
+ * reads of an earlier client exhausted the boot budget and hid the later
+ * clients' rows (the same fix the r707x/r715 pump witnesses got).
+ */
+static u32 svc_util_dec_log;
+static u32 svc_gr_util_log;
+static struct { u64 va; u64 h; } svc_util_dec_keys[32];
+/*
+ * SESSION 143 (r803): per-channel sweep VISIBILITY. The r802 evidence could
+ * not say whether the Vulkan client's copy-class channels (0xbeef0101/2,
+ * eng 11/12) were swept at all: the per-pass log capped at 24 rows and the
+ * early return (no gpfifo/entries) prints nothing. These one-shot (per
+ * client, reset in svc_session_flush) keys name the first sweep of EVERY
+ * channel - begin or skip + reason - so "channel absent from the log" can
+ * never again be confused with "channel not in the table".
+ */
+static u32 svc_util_seen_keys[32];
+static u32 svc_util_seen_n;
static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap)
{
@@ -3265,11 +3600,56 @@ static bool svc_resolve_in_progress(u64 va)
* VA above the window base, including sysmem-backed VAs, so callers that must
* not act on a guess (push execution) use this variant.
*/
+/*
+ * r835: the VAS handle of the channel currently being serviced (0 = none).
+ * Channel-scoped VA resolution consults this VAS's page tables FIRST: the
+ * global pin/lat/map layers are VA-keyed and every client's RM window sits
+ * at the same VAs (0x1200xxxxx), so a lookup outside the owning client's
+ * page tables can answer ANOTHER client's page - the X server's ring
+ * resolved to the GL client's (zero) page and its pushes decoded empty,
+ * wedging X's readback wait exactly like the RC watchdog's ring starvation.
+ */
+static u32 svc_cur_vas;
+
+static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
+ int *ap)
+{
+ int i;
+
+ if (!svc_cur_vas)
+ return -ENOENT;
+ for (i = 0; i < SVC_MAX_VASPACES; i++) {
+ u64 pa = 0;
+ int a = SVC_APERTURE_SYS;
+
+ if (!svc.vas[i].valid ||
+ svc.vas[i].h_vaspace != svc_cur_vas)
+ continue;
+ if (svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries,
+ va & ~(u64)(SEMA_PAGE_SIZE - 1),
+ svc.vas[i].ap, &pa, &a, false))
+ continue;
+ *addr = pa + (va & (SEMA_PAGE_SIZE - 1));
+ *ap = a;
+ return 0;
+ }
+ return -ENOENT;
+}
+
static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
int *ap)
{
u64 pa = 0;
+ /*
+ * r835: the decoded channel's own page tables answer first. When a
+ * channel is being serviced (svc_cur_vas set) its VAS is the only
+ * authoritative source for its VAs; the VA-keyed layers below may
+ * hold another client's pages for the same numeric VA. A failed
+ * walk falls through to the old chain unchanged.
+ */
+ if (!svc_chanvas_lookup(vgpu, va, addr, ap))
+ return 0;
if (!svc_pin_lookup_any(va, addr, ap)) {
return 0;
}
@@ -3313,6 +3693,23 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
svc_lat_store(va, pa, SVC_APERTURE_SYS);
return 0;
}
+ /*
+ * r826: driver-declared PROMOTE_CTX buffers are the LAST resort, not
+ * the first. The promoted (va, pa) pair names a BOOKING, and even its
+ * base page can belong to a different buffer than the one mapped at
+ * that VA (r823: the ctx-MAIN booking's span swallowed the GPFIFO VA;
+ * r826: the PATCH booking's base VA 0x120001000 hosts nvidia-push's
+ * separately mapped 4 KB progress semaphore - writing the release to
+ * the booking's pa left the driver's own polled page at zero and the
+ * X server failed at its 10 s wait while the release value sat in the
+ * wrong page). The page-table walks describe the real mappings and
+ * must answer first; the promote record is only a fallback for VAs
+ * the walks cannot resolve at all.
+ */
+ if (!svc_promo_lookup(va, addr, ap)) {
+ svc_lat_store(va, *addr, *ap);
+ return 0;
+ }
return -ENOENT;
}
@@ -6330,10 +6727,11 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
svc_gr_cand.param_size, svc_gr_cand.rel_va,
svc_gr_cand.rel_payload, svc_gr_cand.rel_flags,
svc_gr_cand.n_pointer, svc_gr_cand.n_inline);
- if (push && push_len >= 16) {
+ if (gsp_sema_dbg && svc_gr_push_dump < 4 && push && push_len >= 16) {
u32 n = push_len / 4;
u32 i, j;
+ svc_gr_push_dump++;
if (n > 256)
n = 256;
for (i = 0; i < n; i += 16) {
@@ -6401,6 +6799,31 @@ static bool svc_fwd_defer_arm(void)
gsp_phys_work_defer_release_enabled();
}
+/*
+ * r824: the release deferral must be owned by a candidate of THIS push.
+ * svc_fwd_defer_arm() answers globally (any unoffered candidate of the
+ * boot's ring); an unrelated lingering candidate then parked the release
+ * of a push that has no forwarding at all. Observed: the X/GL DMA test's
+ * progress-tracker release (the kernel's progress semaphore) was recorded
+ * into svc_fwd_rel and dropped at the next pass's reset, every pass, so
+ * the semaphore never advanced and Xorg failed at its 10 s wait - while
+ * every other stage of the chain (latch, NOP acceptance, decode) had
+ * just started working. A release whose push owns no candidate cannot
+ * wait for a forwarded copy that will never exist; it applies inline.
+ */
+static bool svc_fwd_cand_owns_push(u64 push_va)
+{
+ u32 ri;
+
+ for (ri = 0; ri < SVC_GR_CAND_RING; ri++) {
+ if (svc_gr_cand_ring[ri].valid &&
+ !svc_gr_cand_ring[ri].offered &&
+ svc_gr_cand_ring[ri].push_va == push_va)
+ return true;
+ }
+ return false;
+}
+
/*
* P6b wave-1.5 deferred guest releases: every release group the current
@@ -6954,6 +7377,20 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
u64 *done_out, u32 rows, u32 dpitch, u32 spitch,
u32 owner_ch, u32 owner_idx, u64 owner_hash)
{
+ /*
+ * fail4: name every copy the executor is asked to run (pre-
+ * resolution), so a copy that fails or is held before any witness
+ * still has an entry record. SESSION 132: one record per COPY OP,
+ * not per row - a 480-row frame copy logged 480 rows and exhausted
+ * the per-session budget (128) inside the first op, hiding every
+ * later copy (including any SASS upload).
+ */
+ if (gsp_sema_dbg && svc.copyent_log < 128) {
+ svc.copyent_log++;
+ pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu rows=%u dpitch=0x%x spitch=0x%x dphys=%u dap=%u sphys=%u sap=%u\n",
+ svc.cur_ch, dst, src, total, rows, dpitch, spitch,
+ dst_phys, dst_aperture, src_phys, src_aperture);
+ }
if (rows > 1 && dpitch) {
u64 done = 0;
int rc = 0;
@@ -7003,17 +7440,6 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot
*done_out = 0; /* 0 until the first byte lands */
if (!total || total > (256ULL << 20))
return -EINVAL;
- /*
- * fail4: name every copy the executor is asked to run (pre-
- * resolution), so a copy that fails or is held before any witness
- * still has an entry record.
- */
- if (gsp_sema_dbg && svc.copyent_log < 128) {
- svc.copyent_log++;
- pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n",
- svc.cur_ch, dst, src, total, dst_phys, dst_aperture,
- src_phys, src_aperture);
- }
snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch,
owner_idx, owner_hash);
@@ -8848,18 +9274,20 @@ static bool svc_stream_parse(const u8 *buf, u32 len)
memcpy(&hdr, buf + off, 4);
if (hdr == 0) {
/*
- * UVM's large page-table pushes begin with a
- * zero-filled header region (observed 0x1004
- * bytes) before the first method, so zeros before
- * the first method are leading padding; zeros
- * after one are trailing padding and end the
- * stream.
+ * NVC06F_DMA_NOP is 0x00000000: a zero word is
+ * NOP padding. UVM's large page-table pushes begin
+ * with a zero-filled header region (observed 0x1004
+ * bytes) and the GR submit pushes carry zero gaps
+ * between method groups (observed r715: the 1432-
+ * and 17316-byte pushes pad between the context
+ * setup and the dispatch groups), so zeros are
+ * skipped anywhere in the stream, not only before
+ * the first method or at the tail. The exact-end
+ * requirement and the valid-header walk below are
+ * the false-positive arbiters.
*/
- if (!methods) {
- off += 4;
- continue;
- }
- break; /* trailing zero padding */
+ off += 4;
+ continue;
}
op = (hdr >> 29) & 0x7;
count = (hdr >> 16) & 0xfff;
@@ -8879,16 +9307,27 @@ static bool svc_stream_parse(const u8 *buf, u32 len)
return false;
off += 4 + count * 4;
break;
+ case 0: /* GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK */
+ case 2: /* GRP2_USE_TERT */
case 4: /* IMMD_DATA_METHOD: header only */
- off += 4;
- break;
case 7: /* END_PB_SEGMENT: header only */
+ /*
+ * r715: the class header defines op 0 as
+ * GRP0_USE_TERT (0) and op 2 as GRP2_USE_TERT -
+ * 4-byte header-only formats. The GR submit
+ * pushes emit 0x0001fff0 (SET_SUBDEVICE_MASK,
+ * value 0xfff) around their semaphore groups; the
+ * old parser rejected the first one on every big
+ * push, so the whole push read implausible and
+ * the entries were skipped without decode
+ * (session 124's consume-without-decode).
+ */
off += 4;
break;
default:
return false;
}
- if (++methods > 1024)
+ if (++methods > 8192)
return false;
}
while (off < len && buf[off] == 0)
@@ -8947,6 +9386,127 @@ static unsigned long svc_put_log_at[32];
*/
static u32 svc_userd_dbg;
static u32 svc_userd_dbg_ch[32];
+/* SESSION 134: per-CE-channel budget of the first-submission ring captures
+ * (see svc_ce_firstsub_capture) and the start of an unlatched CE channel's
+ * completion hold (see svc_ce_hold_ms). Session state; reset by
+ * svc_session_flush. */
+static u32 svc_ce_firstsub_n[32];
+static unsigned long svc_ce_hold_since[32];
+/*
+ * SESSION 134c: the FB offset each CE rank's first-submission capture latched
+ * (0 = none). The shadow scan is rank-blind on its own - it takes the first
+ * ring-shaped page in the shadow - and on r779 that latched rank 0's block
+ * (fb 0x80000, userd 0x90000) for rank 1 as well (both channels reported
+ * scan_hits=1 latch=1 off=0x80000). The rank-1 channel's first release was
+ * then decoded from rank 0's ring/page and stale-dropped (cur=1 already from
+ * rank 0's applied release), so the client's rank-1 tracker wait never
+ * completed and the client spun forever (victory_mail stall, three boots).
+ * The capture now prefers the rank's convention offset and refuses any
+ * offset another CE rank already latched. Session state; reset by
+ * svc_session_flush.
+ */
+static u64 svc_ce_firstsub_off[32];
+/*
+ * SESSION 144: FB ring-block ownership ACROSS engine classes. The GR
+ * autopin fallback scan (SESSION 133c) and the CE first-submission
+ * capture (SESSION 134c) each solved the same problem only within their
+ * own class; a live-but-foreign ring block passes every shape gate, so a
+ * channel whose own ring is not written yet latches another channel's
+ * block permanently and never reads its own entries. Observed on
+ * r805-r808: every CUDA GR rank's userd-shape scan latch resolved the CE
+ * rank-0 block (fb 0x80000, userd 0x90000; e.g.
+ * "auto-pinned GR ring h=0x5c000019 rank=0 ... -> fb 0x80000 (userd-shape
+ * scan ... entry0 va=0x120000000 len=40 put=35 get=35)" while the CE
+ * handle decoded the identical entry[35] raw words from pa=0x10350641c),
+ * so no GR launch was ever captured and victory_mail hung after DEVICE.
+ * A channel that cannot see its own block must stay unlatched (and retry
+ * next pass) rather than steal a foreign one. Keyed by the FB ring
+ * offset; claims are session state, reset by svc_session_flush.
+ */
+struct svc_fb_claim {
+ u32 h; /* owning channel handle */
+ u64 ring_off; /* FB ring offset */
+ u64 userd_off; /* FB USERD offset (0 = unknown) */
+};
+#define SVC_CE_RING_FB_BASE 0x80000ULL
+#define SVC_CE_RING_FB_STRIDE 0x20000ULL
+static struct svc_fb_claim svc_fb_claim_tab[40];
+static u32 svc_fb_claim_n;
+
+static void svc_fb_claim_add(u32 h, u64 ring_off, u64 userd_off)
+{
+ u32 i;
+
+ for (i = 0; i < svc_fb_claim_n; i++) {
+ if (svc_fb_claim_tab[i].h == h) {
+ svc_fb_claim_tab[i].ring_off = ring_off;
+ svc_fb_claim_tab[i].userd_off = userd_off;
+ return;
+ }
+ }
+ if (svc_fb_claim_n < ARRAY_SIZE(svc_fb_claim_tab)) {
+ svc_fb_claim_tab[svc_fb_claim_n].h = h;
+ svc_fb_claim_tab[svc_fb_claim_n].ring_off = ring_off;
+ svc_fb_claim_tab[svc_fb_claim_n].userd_off = userd_off;
+ svc_fb_claim_n++;
+ }
+}
+
+/*
+ * true = the candidate FB block (ring at ring_off, USERD at userd_off if
+ * known) is already claimed by a DIFFERENT channel, or is a live CE
+ * rank's convention block even before that rank latched, i.e. it must not
+ * be latched or stolen. Same-channel re-latches pass.
+ */
+static bool svc_fb_claim_foreign(struct nv_vgpu_device *vgpu, u32 self_h,
+ u64 ring_off, u64 userd_off)
+{
+ u32 i;
+ unsigned long flags;
+ int k;
+
+ for (i = 0; i < svc_fb_claim_n; i++) {
+ if (svc_fb_claim_tab[i].h == self_h)
+ continue;
+ if (svc_fb_claim_tab[i].ring_off == ring_off)
+ return true;
+ if (userd_off && svc_fb_claim_tab[i].userd_off == userd_off)
+ return true;
+ }
+ /*
+ * SESSION 144b: reserve every live CE rank's convention block even
+ * before that rank has latched. r809: CE rank 1's block at fb
+ * 0xa0000 was still unclaimed when GR rank 0's userd-shape scan ran
+ * (rank 1 latched through the plain autopin path, not the
+ * first-submission capture) and the GR channel stole it again.
+ */
+ spin_lock_irqsave(&svc.chan_lock, flags);
+ for (k = 0; k < SVC_MAX_CHANNELS; k++) {
+ u64 conv;
+
+ if (!svc.chan[k].valid || svc.chan[k].vgpu != vgpu ||
+ !svc.chan[k].engine_type ||
+ svc.chan[k].h_object == self_h ||
+ svc.chan[k].ce_index >= 32)
+ continue;
+ conv = SVC_CE_RING_FB_BASE +
+ SVC_CE_RING_FB_STRIDE * svc.chan[k].ce_index;
+ if (ring_off == conv ||
+ (userd_off && userd_off == conv + 0x10000)) {
+ spin_unlock_irqrestore(&svc.chan_lock, flags);
+ return true;
+ }
+ }
+ spin_unlock_irqrestore(&svc.chan_lock, flags);
+ return false;
+}
+
+static void svc_fb_claim_reset(void)
+{
+ svc_fb_claim_n = 0;
+ memset(svc_fb_claim_tab, 0, sizeof(svc_fb_claim_tab));
+}
+
/*
* ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD
* probe fails at discovery. Session/reset-scoped, scheduled on the first
@@ -10014,8 +10574,19 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
}
if (st->qmd_n > svc_gr_qmd_seen)
svc_gr_qmd_seen = st->qmd_n;
- if (st->qmd_have == 3)
- st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8;
+ /*
+ * SESSION 127 C5C0 translator: the Vulkan driver writes
+ * SET_INLINE_QMD_ADDRESS_B only (the upper half stays absent for a
+ * <4G QMD; observed beec0100 idx23: `0x31c = 0x4b85f` then a 64-word
+ * 0x320 run). Accept the B-only form with an implicit zero upper
+ * half - the old A+B requirement left qmd_dev_va at 0 and the
+ * candidate logged with qmd=0x0 (the field is identity/logging
+ * only, but the 0x0 read confused every prior session's forensics).
+ */
+ if (st->qmd_have & 2)
+ st->qmd_dev_va =
+ (((u64)((st->qmd_have & 1) ? st->qmd_hi : 0) << 32) |
+ st->qmd_lo) << 8;
if (st->qmd_n == SVC_QMD_WORDS) {
/*
* Static: a 16-entry record array is ~1.2 KB and this is
@@ -10037,15 +10608,85 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
svc_grce_dst_ring_next : SVC_GRCE_DST_MAX;
if (have > GSP_BRIDGE_INLINE_MAX)
have = GSP_BRIDGE_INLINE_MAX;
- for (i = 0; i < have; i++) {
- u32 idx = (svc_grce_dst_ring_next - 1 - i) %
- SVC_GRCE_DST_MAX;
- const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx];
-
- recs[n].dst_va = rec->dst_va;
- recs[n].len = rec->len;
- memcpy(recs[n].data, rec->data, rec->len);
- n++;
+ /*
+ * SESSION 127 C5C0 translator: the rec ring is session-wide
+ * and mixes every client's LOAD_INLINE_DATA destinations. A
+ * stale rec whose page this QMD does not name made the fill
+ * refuse the whole launch (GR fill ENOENT
+ * va=0x80412000 qmd_word=NONE covered=1 - the Vulkan
+ * client's cancel page inherited from the driver-init
+ * phase). Keep only recs whose page is one of the pages the
+ * captured QMD names (the QMD buffer, the parameter bank,
+ * the program, the pointer pairs) - exactly the pages the
+ * backend's inline-served fill can use; keep everything when
+ * the QMD names none (fail-open).
+ */
+ {
+ u64 keep[12];
+ u32 nkeep = 0, k;
+
+ if (st->qmd_dev_va)
+ keep[nkeep++] = st->qmd_dev_va &
+ ~(u64)(SEMA_PAGE_SIZE - 1);
+ keep[nkeep++] =
+ ((u64)st->qmd[32] |
+ ((u64)(st->qmd[33] & 0x1ffff) << 32)) &
+ ~(u64)(SEMA_PAGE_SIZE - 1);
+ keep[nkeep++] =
+ ((u64)st->qmd[48] |
+ ((u64)(st->qmd[49] & 0x1ffff) << 32)) &
+ ~(u64)(SEMA_PAGE_SIZE - 1);
+ for (k = 0; k < SVC_GR_POINTER_MAX - 1 && nkeep < 12; k++) {
+ u32 w = 34 + 2 * k;
+ u64 pv = (u64)st->qmd[w] |
+ ((u64)(st->qmd[w + 1] & 0x1ffff)
+ << 32);
+
+ if (pv)
+ keep[nkeep++] = pv &
+ ~(u64)(SEMA_PAGE_SIZE - 1);
+ }
+ for (i = 0; i < have; i++) {
+ u32 idx = (svc_grce_dst_ring_next - 1 - i) %
+ SVC_GRCE_DST_MAX;
+ const struct svc_grce_rec *rec =
+ &svc_grce_dst_ring[idx];
+ u64 rpage = rec->dst_va &
+ ~(u64)(SEMA_PAGE_SIZE - 1);
+ bool keepit = !nkeep;
+ u32 j;
+
+ for (j = 0; j < nkeep; j++)
+ if (keep[j] == rpage) {
+ keepit = true;
+ break;
+ }
+ /*
+ * SESSION 127 (gated): a stale session-wide
+ * rec whose page resolves nowhere made the
+ * fill refuse the whole launch; keep it when
+ * it resolves (the CUDA inline-served
+ * pages), drop only unresolvable unnamed
+ * ones. With the translator off, the
+ * historical keep-all behaviour stands.
+ */
+ if (!keepit && gsp_sema_gr_xlate) {
+ u64 rpa = 0;
+ u32 rap = SVC_APERTURE_SYS;
+
+ if (svc_resolve_va(vgpu, rpage, &rpa,
+ &rap) &&
+ svc_walk_vaspaces(vgpu, rpage,
+ &rpa, &rap))
+ continue;
+ } else if (!keepit) {
+ continue;
+ }
+ recs[n].dst_va = rec->dst_va;
+ recs[n].len = rec->len;
+ memcpy(recs[n].data, rec->data, rec->len);
+ n++;
+ }
}
/*
* C10 (GAP-2 A5): duplicate destinations in the newest-first
@@ -10202,6 +10843,60 @@ static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va,
h_object, push_va, off, len);
}
+/*
+ * r828: the guest kernel's RC watchdog (krcWatchdog_IMPL, 7 s notify
+ * timeout) posts a notifier cycle on its own channel (object 0x31415900)
+ * through the class2d subchannel (NVA06F_SUBCHANNEL_2D = 3): SET_NOTIFY_A/B
+ * name the notifier VA, NOTIFY (WRITE_ONLY) is the GPU's "write a zeroed
+ * NvNotification" - the driver set the status word to 0xFFFF and declares
+ * "GPU is probably locked" when it never comes back as 0. Observed: robust
+ * channels enabled by the GL client (GZDOOM), then NVRM krcWatchdog every
+ * 8 s and Xorg dead on its screen readback. The emulation must author the
+ * notification write, exactly like svc_notify_complete does for the
+ * work-submit token.
+ */
+#define SVC_2D_SUBCH 3 /* NVA06F_SUBCHANNEL_2D */
+#define SVC_2D_SET_NOTIFY_A 0x0104 /* payload: addr bits [56:32] */
+#define SVC_2D_SET_NOTIFY_B 0x0108 /* payload: addr bits [31:0] */
+#define SVC_2D_NOTIFY 0x010c /* payload: 0 = WRITE_ONLY */
+
+static u64 svc_notify2d_done;
+static u32 svc_notify2d_log;
+
+static void svc_notify2d_write(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch)
+{
+ /* 16 bytes = the NvNotification (status word at +14). */
+ u8 zero[16] = { 0 };
+ u64 pa = 0;
+ int ap = SVC_APERTURE_SYS;
+
+ if (!ch->notify2d_va)
+ return;
+ if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) {
+ if (gsp_sema_dbg && svc_notify2d_log < 16) {
+ svc_notify2d_log++;
+ pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n",
+ ch->h_object, ch->notify2d_va);
+ }
+ return;
+ }
+ if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero)))
+ return;
+ svc_notify2d_done++;
+ /*
+ * r834: pin the page that accepted the ack; the same stale-map
+ * defense as the push-side pin in the decode.
+ */
+ svc_pin_store(ch->notify2d_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+ pa & ~(u64)(SEMA_PAGE_SIZE - 1), svc_ap_of(ap));
+ if (gsp_sema_dbg && svc_notify2d_log < 16) {
+ svc_notify2d_log++;
+ pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (RC watchdog acked)\n",
+ ch->h_object, ch->notify2d_va, pa);
+ }
+}
+
static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
u32 len, u64 page_pa, bool allow_ops,
struct svc_channel *ch, u32 entry_idx,
@@ -10219,6 +10914,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
* and the matmul r322 starved/hung note).
*/
svc.cur_ch = ch->h_object;
+ /* r835: decode-scoped VAS for channel-first VA resolution. */
+ svc_cur_vas = ch->h_vaspace;
/*
* r34x: the acquire-wait capture is PER RING ENTRY. It used to be
* reset only at SET_OBJECT, so a decode pass over a later entry (or
@@ -10752,6 +11449,48 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
continue;
}
+ if (subch == SVC_2D_SUBCH && count >= 1 && addr >= 0x0100 &&
+ addr <= 0x010c) {
+ /*
+ * r828: the RC watchdog's notifier cycle (class2d
+ * methods on subchannel 3). See svc_notify2d_write:
+ * without the zeroed notification the guest's
+ * krcWatchdog declares the GPU locked after 7 s and
+ * the X/GL stack wedges on screen readback.
+ */
+ switch (addr) {
+ case SVC_2D_SET_NOTIFY_A:
+ ch->notify2d_va = (u64)(v0 & 0x1ffffff) << 32;
+ /*
+ * r834: this method stream reached us through
+ * a REAL read (svc_read_last_pa); pin its page
+ * so the watchdog's push translation cannot go
+ * stale later - the content-scanned map answers
+ * a zero page once the injected pin ages out,
+ * staging exactly the starvation the ack must
+ * not depend on.
+ */
+ if (svc_read_last_pa)
+ svc_pin_store(push_va &
+ ~(u64)(SEMA_PAGE_SIZE - 1),
+ svc_read_last_pa &
+ ~(u64)(SEMA_PAGE_SIZE - 1),
+ SVC_APERTURE_SYS);
+ break;
+ case SVC_2D_SET_NOTIFY_B:
+ ch->notify2d_va |= (u64)v0;
+ break;
+ case SVC_2D_NOTIFY:
+ if (count == 1 && v0 == 0) /* WRITE_ONLY */
+ svc_notify2d_write(vgpu, ch);
+ break;
+ default:
+ break;
+ }
+ off += 4 + count * 4;
+ continue;
+ }
+
if (subch == 0 && addr == 0) {
/* SET_OBJECT on the channel subchannel: a push starts */
n_ops = 0;
@@ -10844,7 +11583,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
or.va = va;
or.payload = payload;
or.entry_idx = entry_idx;
+ /*
+ * r824: the deferral needs a
+ * candidate that OWNS this push
+ * (see svc_fwd_cand_owns_push) -
+ * a release from a push with no
+ * forwarding has nothing to wait
+ * for and applies inline.
+ */
if (svc_fwd_defer_arm() &&
+ svc_fwd_cand_owns_push(push_va) &&
!svc.op_tr_pending) {
svc_opdef_drop(vgpu, ch,
entry_idx);
@@ -12655,6 +13403,17 @@ static void svc_gr_defer_reset(void);
static void svc_session_flush(void)
{
+ /* r707x: per-client witness budgets. */
+ svc_pushfail_log = 0;
+ svc_rdw_log = 0;
+ svc_pushdec_log = 0;
+ /* SESSION 132: util-sweep witnesses + their dedup keys are per-client. */
+ svc_util_dec_log = 0;
+ svc_gr_util_log = 0;
+ memset(svc_util_dec_keys, 0, sizeof(svc_util_dec_keys));
+ /* SESSION 143: the per-channel sweep-visibility keys are per-client. */
+ svc_util_seen_n = 0;
+ memset(svc_util_seen_keys, 0, sizeof(svc_util_seen_keys));
unsigned long flags;
u32 i;
@@ -12673,6 +13432,7 @@ static void svc_session_flush(void)
*/
{
bool live = false;
+ bool forced = READ_ONCE(svc_sess_flush_forced);
spin_lock_irqsave(&svc.chan_lock, flags);
for (i = 0; i < SVC_MAX_CHANNELS; i++)
@@ -12681,10 +13441,15 @@ static void svc_session_flush(void)
break;
}
spin_unlock_irqrestore(&svc.chan_lock, flags);
- if (live) {
+ if (live && !forced) {
pr_info("sema-svc: session flush skipped: live channel(s) present (deferred flush raced a new session registration)\n");
return;
}
+ if (forced) {
+ svc_sess_flush_forced = 0;
+ pr_info("sema-svc: session flush (forced) executing with live=%d channel(s)\n",
+ live);
+ }
}
svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */
@@ -12760,6 +13525,7 @@ static void svc_session_flush(void)
svc_pb_last = 0;
svc_pb_last_n = 0;
svc_gr_ring_done = 0;
+ svc_gr_ring_scan_done = 0;
svc_ring_n = 0;
/*
* The learned tables are session state too: the primary VA->PA pin
@@ -12785,6 +13551,16 @@ static void svc_session_flush(void)
* name the previous client's pages (ISSUE-0002 fan-out).
*/
memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt));
+ /*
+ * r820: PROMOTE_CTX records name one client's kernel buffers, and a
+ * new client's channels re-announce their own (the same VAs can back
+ * with different PAs across sessions): session state. A flush skips
+ * entirely while any channel is live, so a running client's records
+ * cannot be wiped under it.
+ */
+ memset(svc_promo, 0, sizeof(svc_promo));
+ svc_promo_seq = 0;
+ svc_promo_hit_log = 0;
memset(svc_cnt_der, 0, sizeof(svc_cnt_der));
svc_cnt_der_n = 0;
/* r551: delivered-release marks are per-session (same handles reused). */
@@ -12869,6 +13645,10 @@ static void svc_session_flush(void)
memset(svc_put_log_at, 0, sizeof(svc_put_log_at));
svc_userd_dbg = 0;
memset(svc_userd_dbg_ch, 0, sizeof(svc_userd_dbg_ch));
+ memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n));
+ memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since));
+ memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off));
+ svc_fb_claim_reset();
svc_userd_sweep_log = 0;
memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at));
memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n));
@@ -13835,7 +14615,14 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len)
if (off + 4 > len)
return true; /* all zeros: not refutable here */
op = (hdr >> 29) & 0x7;
- if (op == 4 || op == 7) /* header-only methods */
+ /*
+ * r715: op 0 (GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK) and op 2
+ * (GRP2_USE_TERT) are 4-byte header-only formats per the class
+ * header; the GR submit pushes start with 0x0001fff0
+ * (SET_SUBDEVICE_MASK), so a correct injected pin for such a page
+ * was rejected here and the read fell through to the walk.
+ */
+ if (op == 0 || op == 2 || op == 4 || op == 7) /* header-only */
return true;
if (op != 1 && op != 3 && op != 5)
return false;
@@ -13884,12 +14671,30 @@ static u32 svc_pushpg_split_log;
* validated candidate). Reads stay defensively copied through
* svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds.
*/
+/* r707d: bounded push-read failure witness (see below). */
+static u32 svc_pushfail_log;
+/*
+ * r707e: last content-backed ring pair resolved by any channel. The VAS
+ * candidate verify is retried with it on EVERY pump pass: the parked
+ * COPY_SERVER_PDES candidate for the client's user VAS parks at
+ * device-create and must register before the submit's first push read
+ * (a failed GR entry read is never retried with a fresh VAS), but the ring
+ * resolution and the park race within one pass - the every-pass retry
+ * closes that race in a single boot.
+ */
+static u64 svc_lastgood_ring_va;
+static u64 svc_lastgood_ring_pa;
+
+
static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
struct svc_channel *ch, u64 va, void *buf,
size_t len, bool skip_ipin)
{
u8 *p = buf;
+ /* r820: per-read; the decode reads it right after the call. */
+ svc_read_last_promote = false;
+
while (len) {
u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
size_t chunk = min_t(size_t, len,
@@ -13899,6 +14704,7 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
bool have_win, done = false;
bool first = (p == buf);
int last_rc = -ENOENT;
+ int trusted_rc = 0;
u32 src;
have_win = !svc_pb_win_lookup(ch, page_va, &win_pa);
@@ -13952,7 +14758,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
default:
/* learned cache / discovered map / page
* directories / targeted rm_mem resolve */
- if (svc_lookup_trusted(vgpu, va, &pa, &ap))
+ trusted_rc = svc_lookup_trusted(vgpu, va,
+ &pa, &ap);
+ if (trusted_rc)
continue;
break;
}
@@ -14013,10 +14821,47 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
*/
if (src == 3 && first && !memchr_inv(p, 0, chunk))
continue;
+ svc_read_last_src = (u8)src;
+ svc_read_last_pa = pa;
+ /*
+ * r820: name whether the trusted answer came from a
+ * driver-declared PROMOTE_CTX buffer; the decode uses
+ * it for the zero-NOP acceptance (see the flag).
+ */
+ if (src == 4) {
+ u64 promo_pa = 0;
+ int promo_ap = SVC_APERTURE_SYS;
+
+ svc_read_last_promote =
+ !svc_promo_lookup(page_va, &promo_pa,
+ &promo_ap);
+ }
done = true;
}
- if (!done)
+ if (!done) {
+ svc_read_last_src = 0xff;
+ if (svc_pushfail_log < 96) {
+ u64 ipa = 0, apa = 0;
+ int iap = 0, aap = 0;
+ bool has_ipin =
+ (svc_ipin_lookup(va, &ipa, &iap) == 0);
+ bool has_alt =
+ (svc_ipin_alt_lookup(va, &apa, &aap) == 0);
+
+ svc_pushfail_log++;
+ pr_info("sema-pushfail: va=0x%llx chunk=0x%llx page=0x%llx last_rc=%d trusted_rc=%d ipin=%d pa=0x%llx alt=%d pa=0x%llx win=%d pa=0x%llx\n",
+ (unsigned long long)va,
+ (unsigned long long)chunk,
+ (unsigned long long)page_va,
+ last_rc, trusted_rc, has_ipin,
+ (unsigned long long)ipa,
+ has_alt,
+ (unsigned long long)apa,
+ have_win,
+ (unsigned long long)win_pa);
+ }
return last_rc;
+ }
p += chunk;
va += chunk;
len -= chunk;
@@ -14081,6 +14926,22 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
: SVC_APERTURE_FB;
/*
+ * r820 diag: the Xorg DMA-test window is 10 s and the latch hinges on
+ * this function's bounded scan re-runs (userd_scans < 8); a pass
+ * cadence stretched by a rebuild can spend every scan before the VAS
+ * candidate registers (r820: the ring resolved at +1 s but no latch
+ * line followed). One bounded row per call names the state the scan
+ * decides on.
+ */
+ if (gsp_sema_dbg && !ch->engine_type && svc_userdp_log < 512) {
+ svc_userdp_log++;
+ pr_info("sema-chan: userdp h=0x%x eng=%u scans=%u rrc=%d ring_pa=0x%llx ring_ap=%d userd_pa=0x%llx as=%d ok=%u\n",
+ ch->h_object, ch->engine_type, ch->userd_scans,
+ rrc, ring_pa, ring_ap, ch->userd_pa, ch->userd_as,
+ ch->userd_ok ? 1 : 0);
+ }
+
+ /*
* ISSUE-0002 (r40): prefer the layout-derived CE USERD and accept it
* while it is still idle. In a second session the registered userdMem
* offset (pool-relative, see below) aliases a DIFFERENT rank's real
@@ -14276,12 +15137,27 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
* USERD cannot pass that test. One scan per channel (the result
* persists through the pump's write-back), logged on latch.
*/
- if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) {
+ /*
+ * r820: the latch scan is capped per channel so a channel that
+ * never anchors cannot sweep forever. But the resolution the scan
+ * needs can arrive LATE: the Xorg GL channel's ring resolves only
+ * when the client's VAS page directory registers (COPY_SERVER_PDES
+ * age-out), and r820 showed every one of the 8 pre-resolution scans
+ * spent before that moment (the pass cadence stretched by a
+ * rebuild), so the 10 s DMA-test window closed with the ring
+ * resolved and no retry left. Once ring_pa is non-zero the scan is
+ * cheap (the sweep validates against a resolved ring), so it may
+ * keep retrying up to a wider cap; the client's own timeout still
+ * bounds the channel's life.
+ */
+ if (!ch->engine_type && ring_pa &&
+ ch->userd_scans < (ring_pa ? 64u : 8u)) {
u64 off, lim;
u8 *base = vgpu->bars[1].vaddr;
u64 best = 0, best_dist = ~0ULL;
u32 best_put = 0, best_get = 0;
bool best_pending = false;
+ u32 gate_n = 0, vfail_n = 0;
ch->userd_scans++;
lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
@@ -14292,23 +15168,33 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
u32 k;
u64 dist;
bool pending;
+ int rck = -1;
+ u64 rkva = 0;
+ u32 rklen = 0;
memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4);
memcpy(&hg, base + off + SVC_USERD_GP_GET, 4);
if (!hp || hp > ch->gpfifo_entries || hg > hp)
continue;
+ gate_n++;
/* Validate: every claimed entry decodes to a
* pushbuffer VA in the ring's window. */
for (k = 0; k < hp && k < 64; k++) {
u8 e[8];
u64 eva;
u32 elen, a0, a1;
+ int rdrc;
- if (svc_read_va(vgpu,
+ rdrc = svc_read_va(vgpu,
ch->gpfifo_va +
(u64)k * 8,
- e, sizeof(e))) {
+ e, sizeof(e));
+ if (rdrc) {
ok = false;
+ rck = (int)k;
+ /* -1: read failure rc; else the
+ * window check below. */
+ rkva = (u64)rdrc;
break;
}
memcpy(&a0, e, 4);
@@ -14329,11 +15215,29 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
SVC_GR_RAM_WIN / 2 ||
elen < 4 || elen > 0x100000) {
ok = false;
+ rck = (int)k;
+ rkva = eva;
+ rklen = elen;
break;
}
}
- if (!ok)
+ if (!ok) {
+ /*
+ * r820 diag: name why a scanned
+ * block was rejected (read rc or
+ * the entry VA/len window).
+ */
+ vfail_n++;
+ if (gsp_sema_dbg && !ch->engine_type &&
+ svc_userdp_log < 512) {
+ svc_userdp_log++;
+ pr_info("sema-chan: userdp scan off=0x%llx put=%u rejected k=%d (read_rc=%lld va=0x%llx len=%u)\n",
+ off, hp, rck,
+ (long long)rkva,
+ rkva, rklen);
+ }
continue;
+ }
/*
* Several blocks can validate: a previous
* session leaves its old USERD holding the same
@@ -14373,6 +15277,17 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
best_pending ? "" : ", idle");
return 0;
}
+ /*
+ * r820 diag: the sweep found nothing adoptable. gates
+ * counts put/get-coherent blocks seen, vfails the ones
+ * rejected by the ring-entry validation.
+ */
+ if (gsp_sema_dbg && svc_userdp_log < 512) {
+ svc_userdp_log++;
+ pr_info("sema-chan: userdp sweep h=0x%x scans=%u gates=%u vfails=%u best=0x%llx\n",
+ ch->h_object, ch->userd_scans,
+ gate_n, vfail_n, best);
+ }
}
}
return -ENOENT;
@@ -14401,6 +15316,293 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
* pump's completion clear before the normal path resumes. */
#define SVC_NOTIFIER_ERR_HOLD (2 * HZ)
+/*
+ * SESSION 134: CE first-submission capture.
+ *
+ * The CE (copy-engine) rings are placed by the guest's allocator at a
+ * location that varies per boot and is NOT reliably described by the
+ * channel's registered gpfifo VA through the discovered page tables (r763
+ * matched the fixed FB convention and the sweep decoded the inline-SASS
+ * entry; r765-r772 read zeros everywhere, the CE channels never latched,
+ * the driver's inline LAUNCH_DMA code downloads never executed and the
+ * frames stayed black). The logged instance where ring content and USERD
+ * were both live is the channel's first notifier clear - the guest's
+ * doorbell for its first submission, processed by the pump in the same
+ * pass that writes the ring. At that instant this hook:
+ * 1. reads entry0 through the channel's own translation (svc_lookup),
+ * 2. reads entry0 through the trusted walk (the VAS may have just
+ * landed since the registration read),
+ * 3. scans the shadow (0x100 step, first 64 MB, the GR-scan bound) for
+ * a ring-shaped page whose entry0 names a 1MB-aligned push VA in the
+ * CE window [0x120000000,0x121000000), and for every hit runs the
+ * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate
+ * whose USERD reads coherent (or idle 0/0) is PINNED and LATCHED
+ * exactly like svc_ring_autopin's CE path.
+ * Bounded: at most 4 captures per CE channel, 16 witness rows per boot;
+ * resets with the session (svc_session_flush).
+ */
+static u64 svc_ce_firstsub_log;
+
+/*
+ * SESSION 134: bounded completion hold for unlatched CE channels.
+ *
+ * The clear write in svc_notify_complete is what makes the guest treat the
+ * submission as consumed; the driver then recycles the ring and its content
+ * is gone before the client's VAS lands (sema_vas_cand_ms), leaving the CE
+ * ring latch with nothing readable (r769-r774: rrc=-2 at the first
+ * submission, the walk answering a zero page later). Holding the notifier
+ * busy for a bounded window keeps the driver's submission in flight - and
+ * with it the ring content that the regular per-pass latch path
+ * (svc_chan_userd through svc_lookup) needs - until the VAS registers and
+ * the ring reads coherently. 0 = off (the pre-SESSION-134 behaviour).
+ */
+static u64 svc_ce_hold_ms;
+module_param_named(sema_ce_hold_ms, svc_ce_hold_ms, ullong, 0644);
+MODULE_PARM_DESC(sema_ce_hold_ms, "Hold an unlatched CE channel's completion open for up to N ms so the ring stays live for the latch (0 = off)");
+
+/*
+ * SESSION 134c rank-aware first-submission capture (see the capture below).
+ * The CE block convention: ring at 0x80000 + 0x20000 * rank, USERD at
+ * ring + 0x10000 (observed for every CE channel across boots; the same
+ * convention svc_chan_userd applies in its regular per-pass latch).
+ */
+#define SVC_CE_RING_FB_BASE 0x80000ULL
+#define SVC_CE_RING_FB_STRIDE 0x20000ULL
+
+/*
+ * One candidate offset for the CE first-submission capture: validate the
+ * ring shape (`scan_hits` counts shape hits) and, on a shape hit, run the
+ * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate whose
+ * USERD reads coherent (or idle 0/0) is PINNED and LATCHED exactly like
+ * svc_ring_autopin's CE path. Returns true when latched.
+ */
+static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch, u8 *sbase,
+ u64 lim, u64 off, u32 *scan_hits,
+ u64 *latch_off, u64 *latch_upa,
+ u32 *latch_put, u32 *latch_get,
+ bool own_block)
+{
+ static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 };
+ u32 h0, h1, hlen;
+ u64 hva;
+ unsigned int ui;
+
+ memcpy(&h0, sbase + off, 4);
+ memcpy(&h1, sbase + off + 4, 4);
+ hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc);
+ hlen = ((h1 >> 10) & 0x1fffff) << 2;
+ if ((hva & 0xfffff) || hva < 0x120000000ULL ||
+ hva >= 0x121000000ULL || hlen < 4 || hlen > 0x10000)
+ return false;
+ (*scan_hits)++;
+ for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) {
+ u64 upa = off + udeltas[ui];
+ u32 p3 = 0, g3 = 0;
+ int prc;
+
+ if (upa + 4 > lim)
+ continue;
+ prc = svc_userd_probe(vgpu, upa,
+ SVC_APERTURE_FB,
+ ch->gpfifo_entries,
+ &p3, &g3);
+ /*
+ * SESSION 144d: the rank's OWN convention block (own_block,
+ * the +0x10000 candidate at the rank's convention offset) is
+ * latched even when its stale content probes incoherent
+ * (g > p) or idle - the guest rewrites put/get on its next
+ * submission and the pump's per-channel last_get processes
+ * entry0 exactly once. r813: client #2's convention block at
+ * fb 0x90000 read put=1/get=146 (g > p -> -EINVAL) at the
+ * capture instant, the coherence gate rejected it, and the
+ * never-used 0x82000 zero block was latched instead, so the
+ * client's first push never drained and it hung. The gates
+ * below stay for every heuristic candidate (other deltas and
+ * the fallback scan), preserving the SESSION 135 protection.
+ */
+ if (!own_block || ui != 0) {
+ if (prc && prc != -EAGAIN)
+ continue;
+ }
+ /*
+ * SESSION 135: at the FIRST-SUBMISSION instant the channel's
+ * submission is by definition unconsumed - the notifier clear
+ * this capture runs before is what would let the driver treat
+ * it as consumed. A candidate whose USERD reads a consumed
+ * state (get >= put && put > 0) cannot be this channel's live
+ * just-submitted work; it is a PREVIOUS session's leftover
+ * block surviving in VRAM (r785: the capture latched the dead
+ * CUDA client's caf00012 block at fb 0x80000, put=1 get=1,
+ * for the Vulkan channel beef0101 - the decoded entry was the
+ * old session's, the client stalled on its expectations).
+ * Idle 0/0 stays acceptable (svc_userd_probe's own rule).
+ */
+ /*
+ * SESSION 144c: this consumed-state rejection is for FOREIGN
+ * blocks found by the heuristic deltas. The rank's OWN
+ * convention block (+0x10000) legitimately carries the
+ * previous session's consumed state until the guest
+ * re-initialises the reused allocation - rejecting it made
+ * the r812 second client fall through to a never-used
+ * 0x82000 block and stall forever (cefirst row: off=0x80000
+ * upa=0x82000 put=0 get=0 while the client waited for the
+ * first push's completion). The pump's per-channel last_get
+ * still processes entry0 exactly once, so latching the
+ * convention block with stale put/get is correct.
+ */
+ if (!own_block && p3 && p3 == g3)
+ continue;
+ /* latch exactly like the CE autopin */
+ svc_pin_store(ch->gpfifo_va, off, SVC_APERTURE_FB);
+ svc_lat_store(ch->gpfifo_va, off, SVC_APERTURE_FB);
+ ch->userd_pa = upa;
+ ch->userd_as = SVC_APERTURE_FB;
+ ch->userd_ok = true;
+ set_bit(ch->ce_index, &svc_ring_done);
+ svc_ring_n++;
+ *latch_off = off;
+ *latch_upa = upa;
+ *latch_put = p3;
+ *latch_get = g3;
+ return true;
+ }
+ return false;
+}
+
+static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch)
+{
+ u8 *sbase = vgpu->bars[1].vaddr;
+ u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL);
+ u64 ring_pa = 0, wpa = 0;
+ u32 e0 = 0, e1 = 0, walk_e0 = 0, walk_e1 = 0;
+ u32 scan_hits = 0, latched = 0;
+ u64 latch_off = 0, latch_upa = 0;
+ u32 latch_put = 0, latch_get = 0;
+ int ring_ap = 0, wap = SVC_APERTURE_SYS;
+ int rrc = -1, wrc = -1;
+
+ if (ch->engine_type == 0 || ch->ce_index >= 32)
+ return;
+ if (svc_ce_firstsub_n[ch->ce_index] >= 4)
+ return;
+ svc_ce_firstsub_n[ch->ce_index]++;
+
+ if (sbase && lim >= 0x1000) {
+ /* 1. the channel's own translation at this instant. */
+ rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap);
+ if (!rrc && ring_pa) {
+ u8 e[8];
+
+ if (!svc_read_ap(vgpu, ring_pa, ring_ap, e,
+ sizeof(e))) {
+ memcpy(&e0, e, 4);
+ memcpy(&e1, e + 4, 4);
+ }
+ }
+ /* 2. the trusted walk at this instant. */
+ if (!svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap)) {
+ u8 e[8];
+
+ wrc = 0;
+ if (!svc_read_ap(vgpu, wpa, svc_ap_of(wap), e,
+ sizeof(e))) {
+ memcpy(&walk_e0, e, 4);
+ memcpy(&walk_e1, e + 4, 4);
+ }
+ }
+ /* 3. shadow scan for a CE-window ring page. */
+ {
+ /*
+ * SESSION 134c rank-aware order: try the rank's
+ * convention offset first, then the bounded scan,
+ * and never accept an offset another CE rank already
+ * latched. The rank-blind first-hit latched rank 0's
+ * block for rank 1 on r779 (both rows read
+ * off=0x80000 upa=0x90000), so the rank-1 channel's
+ * first release decoded from rank 0's ring/page and
+ * was stale-dropped (cur=1 already) - the client's
+ * rank-1 tracker wait never completed and it spun
+ * forever. A rank whose block is not in the shadow
+ * at this instant now simply stays unlatched for the
+ * regular per-pass CE latch instead of stealing a
+ * foreign block.
+ */
+ u64 exp = SVC_CE_RING_FB_BASE +
+ SVC_CE_RING_FB_STRIDE * ch->ce_index;
+ u64 off;
+
+ latched = svc_ce_firstsub_try_off(vgpu, ch, sbase,
+ lim, exp, &scan_hits,
+ &latch_off, &latch_upa,
+ &latch_put, &latch_get,
+ true);
+ for (off = 0; !latched && off + 0x90 <= lim;
+ off += 0x100) {
+ unsigned int ri;
+ bool foreign = false;
+
+ if (off == exp)
+ continue;
+ for (ri = 0; ri < 32; ri++) {
+ if (ri == ch->ce_index ||
+ !svc_ce_firstsub_off[ri])
+ continue;
+ if (svc_ce_firstsub_off[ri] == off) {
+ foreign = true;
+ break;
+ }
+ }
+ if (foreign)
+ continue;
+ /*
+ * SESSION 144: symmetric protection - a CE
+ * fallback scan must not steal a block another
+ * channel (CE rank or GR) already claims.
+ */
+ if (svc_fb_claim_foreign(vgpu, ch->h_object, off,
+ off + 0x10000))
+ continue;
+ latched = svc_ce_firstsub_try_off(vgpu, ch,
+ sbase, lim,
+ off, &scan_hits,
+ &latch_off,
+ &latch_upa,
+ &latch_put,
+ &latch_get,
+ false);
+ }
+ if (latched) {
+ svc_ce_firstsub_off[ch->ce_index] = latch_off;
+ /*
+ * SESSION 144: publish the claim so a GR rank
+ * whose own ring is not written yet cannot
+ * latch this CE block (r805-r808: every CUDA
+ * GR rank stole the CE rank-0 block and the
+ * launch was never captured).
+ */
+ svc_fb_claim_add(ch->h_object, latch_off,
+ latch_upa);
+ }
+ }
+ }
+
+ if (gsp_sema_dbg && svc_ce_firstsub_log < 16) {
+ svc_ce_firstsub_log++;
+ pr_info("sema-cefirst: h=0x%x rank=%u n=%u rrc=%d rpa=0x%llx rap=%d e0=0x%x e1=0x%x wrc=%d wpa=0x%llx wap=%d we0=0x%x we1=0x%x scan_hits=%u latch=%u off=0x%llx upa=0x%llx put=%u get=%u\n",
+ ch->h_object, ch->ce_index,
+ svc_ce_firstsub_n[ch->ce_index], rrc, ring_pa,
+ ring_ap, e0, e1, wrc, wpa, wap, walk_e0, walk_e1,
+ scan_hits, latched, latch_off, latch_upa,
+ latch_put, latch_get);
+ }
+ if (latched)
+ pr_info("sema-cefirst: LATCHED h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd 0x%llx put=%u get=%u; first-submission capture)\n",
+ ch->h_object, ch->ce_index, ch->gpfifo_va,
+ latch_off, latch_upa, latch_put, latch_get);
+}
+
static void svc_notify_complete(struct nv_vgpu_device *vgpu,
struct svc_channel *ch)
{
@@ -14453,6 +15655,41 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
return;
/*
+ * SESSION 134: a visible doorbell means a submission just landed -
+ * for a CE-class channel this is the one instant its ring content
+ * and USERD are both live. Capture/snapshot it BEFORE the clear
+ * write below (the clear is what makes the guest treat the
+ * submission as complete and recycle the ring), see
+ * svc_ce_firstsub_capture.
+ */
+ if (ch->engine_type)
+ svc_ce_firstsub_capture(vgpu, ch);
+
+ /*
+ * SESSION 134: keep an unlatched CE submission open so the ring
+ * content survives until the latch can read it (see
+ * svc_ce_hold_ms). The regular per-pass latch path retries while
+ * the hold runs; the hold expires un-latched into the old
+ * behaviour, so a channel the emulation can never resolve cannot
+ * strand the guest.
+ */
+ if (ch->engine_type && !ch->userd_ok && svc_ce_hold_ms &&
+ ch->ce_index < 32) {
+ if (!svc_ce_hold_since[ch->ce_index])
+ svc_ce_hold_since[ch->ce_index] = jiffies;
+ if (time_before(jiffies,
+ svc_ce_hold_since[ch->ce_index] +
+ msecs_to_jiffies(svc_ce_hold_ms)))
+ return;
+ if (svc_ce_firstsub_log < 16) {
+ svc_ce_firstsub_log++;
+ pr_info("sema-cefirst: h=0x%x rank=%u completion hold expired (%llu ms) unlatched; clearing\n",
+ ch->h_object, ch->ce_index, svc_ce_hold_ms);
+ }
+ svc_ce_hold_since[ch->ce_index] = jiffies;
+ }
+
+ /*
* Clear both slots: they are adjacent, and whichever one the client is
* polling must read as completed for a healthy channel.
*/
@@ -14495,8 +15732,6 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
* against the ring's first GPFIFO entry, so a layout change costs a failed
* pin, not a wrong one.
*/
-#define SVC_CE_RING_FB_BASE 0x80000ULL
-#define SVC_CE_RING_FB_STRIDE 0x20000ULL
static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */
static bool svc_fbs_scan_done; /* one-shot FB entry scan diagnostic */
@@ -14532,6 +15767,62 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
bool gsp_sema_gr_rings = true;
/*
+ * SESSION 132 dead-entry guard for ring candidates.
+ *
+ * The r759 false positive: a vidmem candidate (0xa0000) whose first GPFIFO
+ * entry (push 0x120100000/40) passed the plausibility gate - the USERD
+ * probe hit the channel's REAL USERD at 0xb0000, which the VRAM shape scan
+ * had just found 0x10000 after the candidate by coincidence - was pinned
+ * as the GR ring while the client's own page tables resolve the ring VA to
+ * a guest-RAM page (0x11623d000) whose entry0 is live (push 0x4000000/
+ * 0x8bc, decoded by the util sweep). The pump then read the pinned page,
+ * could not fetch the entry's push (all-zero through every source), and
+ * skipped the entry after 39 failed rescues - the live ring stayed
+ * undrained and every upload behind it invisible.
+ *
+ * A candidate whose first entry cannot deliver a method stream is not this
+ * channel's ring. The push head is read through the client's own page
+ * tables (guest truth); a push that cannot be resolved at all does NOT
+ * mark the entry dead (the VAS may not be registered yet on an early
+ * pass, and vidmem-backed pushes are not in the client's tables at all).
+ */
+static bool svc_ring_entry_dead(struct nv_vgpu_device *vgpu, u32 e0, u32 e1)
+{
+ u64 va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+ u32 len = ((e1 >> 10) & 0x1fffff) << 2;
+ u8 head[16];
+
+ if (!va || len < 8 || len > 0x10000)
+ return false; /* unusable entry; the gate rejects it */
+ if (svc_read_va_trusted(vgpu, va, head, sizeof(head)))
+ return false; /* cannot tell: do not reject */
+ return !memchr_inv(head, 0, sizeof(head));
+}
+
+/*
+ * SESSION 132: does the client's own page table (VAS walk only - never the
+ * pin/lat/map sources, so a bad pin cannot vote for itself) resolve the
+ * ring VA to a page whose first entry is LIVE? Used to reject a dead
+ * candidate only when a live alternative provably exists.
+ */
+static bool svc_ring_walk_live(struct nv_vgpu_device *vgpu, u64 gpfifo_va)
+{
+ u64 wpa = 0;
+ int wap = SVC_APERTURE_SYS;
+ u8 we[8];
+ u32 w0, w1;
+
+ if (svc_walk_vaspaces(vgpu, gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+ &wpa, &wap))
+ return false; /* walk cannot resolve: no verdict */
+ if (svc_read_ap(vgpu, wpa, wap, we, sizeof(we)))
+ return false;
+ memcpy(&w0, we, 4);
+ memcpy(&w1, we + 4, 4);
+ return !svc_ring_entry_dead(vgpu, w0, w1);
+}
+
+/*
* Validate a candidate translation for a GR ring: the first GPFIFO entry
* must decode to a 1 MB aligned push in the client VA window with a sane
* length, and the USERD at +SVC_GR_USERD_OFF must be readable through the
@@ -14558,6 +15849,14 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu,
*len < 4 || *len > 0x10000)
return false;
/*
+ * SESSION 132: a candidate whose entry0 push provably reads all-zero
+ * through the client's own page tables cannot deliver a method
+ * stream (the pump's rescue path eventually skips exactly these) -
+ * reject it so the autopin falls through to a live candidate.
+ */
+ if (svc_ring_entry_dead(vgpu, e0, e1))
+ return false;
+ /*
* Gate: the USERD must be readable. rc==0 is an active get/put
* pair; rc==-EAGAIN is a readable idle (0/0) block, accepted so a
* freshly created channel is not rejected. -EINVAL (implausible
@@ -14567,6 +15866,20 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu,
*get = 0;
prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap,
ch->gpfifo_entries, put, get);
+ /*
+ * SESSION 132: the driver places the GPFIFO and the USERD
+ * independently (r759: ring in guest RAM 0x11623d000, USERD in
+ * vidmem 0xb0000), so a walk-resolved ring cannot pass a
+ * same-aperture USERD probe. Accept the channel's own USERD when
+ * one is already known - the put/get liveness gate is unchanged.
+ */
+ if (prc != 0 && prc != -EAGAIN && ch->userd_ok && ch->userd_pa &&
+ ch->userd_pa != cand_pa + SVC_GR_USERD_OFF) {
+ *put = 0;
+ *get = 0;
+ prc = svc_userd_probe(vgpu, ch->userd_pa, ch->userd_as,
+ ch->gpfifo_entries, put, get);
+ }
return prc == 0 || prc == -EAGAIN;
}
@@ -15597,6 +16910,58 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
}
/*
+ * r705h: host-triggered pdb discovery. The automatic latch
+ * (svc_vas_candidates_verify + svc_xvas_pdb_discover) only runs from the RAM
+ * ring scan, which sweeps [0,6GB) before the client's pushes exist and then
+ * gives up for the boot; and per-client VAS state is flushed at session end.
+ * This entry point lets a host tool that CONTENT-LOCATED a verified
+ * {ring VA -> ring PA} pair (the ring page's GPFIFO entries are its
+ * fingerprint) hand the pair to the service while the client is alive, so:
+ * 1. every parked COPY_SERVER_PDES candidate is verified against the pair
+ * (fast: a walk each), and
+ * 2. if none verifies, the vidmem walk-root sweep anchors on the pair.
+ * Registered VASes make the client's pushbuffer VAs walkable and the pump
+ * can then decode the submit's pushes (the fence release among them).
+ */
+static u64 svc_pdb_disc_va;
+static u64 svc_pdb_disc_pa;
+static bool svc_pdb_disc_pending;
+
+/*
+ * r707r: bounded channel rewind (debug). After the VAS pdb anchor lands,
+ * entries skipped before it never re-decode on their own (their unread
+ * window expires into a skip), so the release they carry would be lost for
+ * that put epoch. A host tool asks for a rewind (sema_chan_rewind: a hex
+ * h_object, or "all"); the pump re-processes the channel from slot 0 with
+ * the consumed-slot bitmap and the per-entry rescue state cleared.
+ */
+static u32 svc_rewind_h;
+static bool svc_rewind_all;
+static bool svc_rewind_pending;
+
+int gsp_sema_service_chan_rewind(u32 h_object)
+{
+ if (!svc.task)
+ return -ENODEV;
+ svc_rewind_h = h_object;
+ svc_rewind_all = (h_object == 0xffffffffu);
+ svc_rewind_pending = true;
+ wake_up_process(svc.task);
+ return 0;
+}
+
+int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa)
+{
+ if (!svc.task)
+ return -ENODEV;
+ svc_pdb_disc_va = ring_va & ~0xfffULL;
+ svc_pdb_disc_pa = ring_pa & ~0xfffULL;
+ svc_pdb_disc_pending = true;
+ wake_up_process(svc.task);
+ return 0;
+}
+
+/*
* r496f: discover a kernel-RM client VAS's page directory in the emulated
* VRAM by verification. The CPU RM builds internal-VAS page tables in
* vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally
@@ -15677,6 +17042,42 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
return; /* wait until the CE channels are known */
/*
+ * SESSION 148 perf (r858/r859 evidence): a GR channel's ring page
+ * sits at (notifier_pa page + 0x400). Verified for the RC watchdog
+ * across r848/r853/r858; extended to all GR channels after r859
+ * showed the X channel (0x1000013) NEVER resolving its ring (no
+ * auto-pin all boot, entries decoded from garbage: va=0x41c000/
+ * 0x2017a20/0x10104 with lens 20464..49152, unreadable, skipped;
+ * ~11 s per frame). The candidate probe (entry0 in the channel's
+ * own alloc + USERD shape) content-gates every derivation, so a
+ * wrong guess costs a failed probe - exactly the current behaviour.
+ */
+ if (!ch->engine_type && ch->notifier_pa) {
+ u64 rpa = (ch->notifier_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) +
+ 0x400;
+
+ if (svc_gr_ring_candidate(vgpu, ch, rpa, SVC_APERTURE_SYS,
+ &va, &len, &put, &get)) {
+ svc_pin_store(ch->gpfifo_va, rpa, SVC_APERTURE_SYS);
+ svc_lat_store(ch->gpfifo_va, rpa, SVC_APERTURE_SYS);
+ if (ch->userd_reg)
+ ch->userd_ok = true;
+ else {
+ ch->userd_pa = rpa + SVC_GR_USERD_OFF;
+ ch->userd_as = SVC_APERTURE_SYS;
+ ch->userd_ok = true;
+ }
+ svc_gr_ring_base = rpa;
+ set_bit(ch->gr_index, &svc_gr_ring_done);
+ svc_ring_n++;
+ pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=1 (notifier-derived; entry0 va=0x%llx len=%u put=%u get=%u)\n",
+ ch->h_object, ch->gr_index, ch->gpfifo_va,
+ rpa, va, len, put, get);
+ return;
+ }
+ }
+
+ /*
* Guest truth first: the channel registration carries the USERD
* memdesc base for THIS session's allocation, and the ring sits one
* USERD block before it (observed for every GR channel across
@@ -15731,11 +17132,23 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
ch->gpfifo_entries, &put,
&get))
continue;
+ /*
+ * SESSION 132: reject a dead candidate when
+ * the client's own page tables provably
+ * resolve the ring VA to a live page (the
+ * r759 vidmem false positive).
+ */
+ if (svc_ring_entry_dead(vgpu, e0, e1) &&
+ svc_ring_walk_live(vgpu, ch->gpfifo_va))
+ continue;
svc_pin_store(ch->gpfifo_va, cand, cap);
svc_lat_store(ch->gpfifo_va, cand, cap);
ch->userd_as = cap;
ch->userd_ok = true;
svc_gr_ring_base = cand;
+ if (cap == SVC_APERTURE_FB)
+ svc_fb_claim_add(ch->h_object, cand,
+ ch->userd_pa);
set_bit(ch->gr_index, &svc_gr_ring_done);
svc_ring_n++;
pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n",
@@ -15766,13 +17179,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
len >= 4 && len <= 0x10000 &&
!svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF,
SVC_APERTURE_FB, ch->gpfifo_entries,
- &put, &get)) {
+ &put, &get) &&
+ !(svc_ring_entry_dead(vgpu, e0, e1) &&
+ svc_ring_walk_live(vgpu, ch->gpfifo_va))) {
svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
ch->userd_pa = pa + SVC_GR_USERD_OFF;
ch->userd_as = SVC_APERTURE_FB;
ch->userd_ok = true;
svc_gr_ring_base = base;
+ svc_fb_claim_add(ch->h_object, pa,
+ pa + SVC_GR_USERD_OFF);
set_bit(ch->gr_index, &svc_gr_ring_done);
svc_ring_n++;
pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n",
@@ -15804,7 +17221,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
(u64)(e0 & 0xfffffffc);
len = ((e1 >> 10) & 0x1fffff) << 2;
if (!(va & 0xfffff) && va >= SEMA_VA_LO &&
- va < SEMA_VA_HI && len >= 4 && len <= 0x10000) {
+ va < SEMA_VA_HI && len >= 4 && len <= 0x10000 &&
+ !(svc_ring_entry_dead(vgpu, e0, e1) &&
+ svc_ring_walk_live(vgpu, ch->gpfifo_va))) {
/*
* Best effort only: a wrapped GP_PUT
* legitimately exceeds the entry count, so a
@@ -15819,10 +17238,20 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
&get);
svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap);
svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap);
- ch->userd_pa = pin_pa + SVC_GR_USERD_OFF;
- ch->userd_as = svc_ap_of(pin_ap);
- ch->userd_ok = true;
+ /* r831: keep a registered real-PA userd. */
+ if (ch->userd_reg) {
+ ch->userd_ok = true;
+ } else {
+ ch->userd_pa =
+ pin_pa + SVC_GR_USERD_OFF;
+ ch->userd_as = svc_ap_of(pin_ap);
+ ch->userd_ok = true;
+ }
svc_gr_ring_base = base;
+ if (svc_ap_of(pin_ap) == SVC_APERTURE_FB)
+ svc_fb_claim_add(ch->h_object, pin_pa,
+ pin_pa +
+ SVC_GR_USERD_OFF);
set_bit(ch->gr_index, &svc_gr_ring_done);
svc_ring_n++;
pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (entry0 va=0x%llx len=%u put=%u get=%u)\n",
@@ -15854,10 +17283,23 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
&get)) {
svc_pin_store(ch->gpfifo_va, wpa, wap);
svc_lat_store(ch->gpfifo_va, wpa, wap);
- ch->userd_pa = wpa + SVC_GR_USERD_OFF;
- ch->userd_as = wap;
- ch->userd_ok = true;
+ /*
+ * r831: the ring+0x2000 userd convention is a guess
+ * for the CUDA GR layout; a channel whose REGISTERED
+ * userd is a real PA (the RC watchdog) keeps its own
+ * (the candidate gate above probes it as fallback).
+ */
+ if (ch->userd_reg) {
+ ch->userd_ok = true;
+ } else {
+ ch->userd_pa = wpa + SVC_GR_USERD_OFF;
+ ch->userd_as = wap;
+ ch->userd_ok = true;
+ }
svc_gr_ring_base = wpa;
+ if (wap == SVC_APERTURE_FB)
+ svc_fb_claim_add(ch->h_object, wpa,
+ wpa + SVC_GR_USERD_OFF);
set_bit(ch->gr_index, &svc_gr_ring_done);
svc_ring_n++;
pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n",
@@ -15867,6 +17309,185 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
}
}
+ /*
+ * SESSION 133: bounded shadow ring scan fallback.
+ *
+ * The CE-class rings live in VIDMEM and the FB convention only
+ * matches when the guest's VRAM allocator happens to place them at
+ * the conventional offsets (r763: matched, fb 0x80000/0xa0000, the
+ * channel decoded; r765: read zeros, the ring VA resolved to a
+ * stale RAM page whose entry0 was dead, the CE channel never
+ * latched and its inline SASS copies never decoded - black
+ * frames). When every deterministic candidate has failed, scan the
+ * shadow for a ring-shaped page: entry0 must name a 1 MB aligned
+ * push VA in the client window with a sane length, must not be
+ * provably dead (svc_ring_entry_dead - the r759 false-positive
+ * guard), and the USERD at the observed deltas (+0x10000 for the
+ * CE family, +0x2000 for GR) must read a coherent put/get. One
+ * bounded scan per rank per boot; a false candidate still costs
+ * only a failed probe.
+ */
+ if (!test_bit(ch->gr_index, &svc_gr_ring_scan_done)) {
+ u8 *sbase = vgpu->bars[1].vaddr;
+ u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL);
+ u64 off;
+
+ /*
+ * SESSION 133b: the rings are NOT page-aligned (r767: the
+ * USERD shape scan found the channel blocks at 0xfee00/
+ * 0xff000/0xff200, so the rings sit at unaligned offsets like
+ * 0xefe00); step by 0x100 and RETRY (rate-limited) - a first
+ * pass can run before the client has written the ring or the
+ * USERD, and a consumed once-bit would never rescan.
+ */
+ if (time_before(jiffies, (unsigned long)svc_gr_ring_scan_last +
+ msecs_to_jiffies(1500)))
+ return;
+ svc_gr_ring_scan_last = jiffies;
+ if (sbase && lim > 0x1000) {
+ static const u64 udeltas[] = { 0x10000, SVC_GR_USERD_OFF,
+ 0x1000 };
+
+ /*
+ * SESSION 133c pass 1: find the USERD by its OWN
+ * shape (put@+0x8c, get@+0x88 - the find_userd
+ * layout) and derive the ring from it. The USERD
+ * words appear at channel creation and stay; the
+ * ring entry may be written later, so this pass is
+ * the stabler trigger. Each hit is validated by the
+ * derived ring's entry0 (shape + not-dead), so a
+ * coincidental put/get pair costs a failed check.
+ */
+ for (off = 0; off + 0x90 <= lim; off += 0x100) {
+ u32 up, ug;
+ unsigned int ui;
+
+ memcpy(&up, sbase + off + 0x8c, 4);
+ if (up < 1 || up > 65535)
+ continue;
+ memcpy(&ug, sbase + off + 0x88, 4);
+ if (ug > up)
+ continue;
+ for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) {
+ u64 roff;
+ u32 h0, h1, p3 = 0, g3 = 0;
+ u64 hva;
+ u32 hlen;
+
+ if (off < udeltas[ui])
+ continue;
+ roff = off - udeltas[ui];
+ memcpy(&h0, sbase + roff, 4);
+ memcpy(&h1, sbase + roff + 4, 4);
+ hva = ((u64)(h1 & 0xff) << 32) |
+ (u64)(h0 & 0xfffffffc);
+ hlen = ((h1 >> 10) & 0x1fffff) << 2;
+ if (!hva || (hva & 0xfffff) ||
+ hva < SEMA_VA_LO ||
+ hva >= SEMA_VA_HI || hlen < 4 ||
+ hlen > 0x10000)
+ continue;
+ if (svc_ring_entry_dead(vgpu, h0, h1))
+ continue;
+ /*
+ * SESSION 144: never latch a foreign
+ * block. Without this every CUDA GR
+ * rank stole the CE rank-0 block while
+ * its own ring was unwritten
+ * (r805-r808); staying unlatched here
+ * lets the FB-convention path latch the
+ * real ring once the client writes it.
+ */
+ if (svc_fb_claim_foreign(vgpu,
+ ch->h_object,
+ roff, off))
+ continue;
+ if (svc_userd_probe(vgpu, off,
+ SVC_APERTURE_FB,
+ ch->gpfifo_entries,
+ &p3, &g3))
+ continue;
+ svc_pin_store(ch->gpfifo_va, roff,
+ SVC_APERTURE_FB);
+ svc_lat_store(ch->gpfifo_va, roff,
+ SVC_APERTURE_FB);
+ ch->userd_pa = off;
+ ch->userd_as = SVC_APERTURE_FB;
+ ch->userd_ok = true;
+ svc_gr_ring_base = roff;
+ svc_fb_claim_add(ch->h_object, roff, off);
+ set_bit(ch->gr_index,
+ &svc_gr_ring_done);
+ set_bit(ch->gr_index,
+ &svc_gr_ring_scan_done);
+ svc_ring_n++;
+ pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd-shape scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n",
+ ch->h_object, ch->gr_index,
+ ch->gpfifo_va, roff,
+ udeltas[ui], hva, hlen, p3,
+ g3);
+ return;
+ }
+ }
+ /* pass 2: ring-entry-shaped pages (below) */
+ for (off = 0; off + 8 <= lim; off += 0x100) {
+ u32 h0, h1;
+ u64 hva;
+ u32 hlen;
+ unsigned int ui;
+
+ memcpy(&h0, sbase + off, 4);
+ memcpy(&h1, sbase + off + 4, 4);
+ hva = ((u64)(h1 & 0xff) << 32) |
+ (u64)(h0 & 0xfffffffc);
+ hlen = ((h1 >> 10) & 0x1fffff) << 2;
+ if (!hva || (hva & 0xfffff) ||
+ hva < SEMA_VA_LO || hva >= SEMA_VA_HI ||
+ hlen < 4 || hlen > 0x10000)
+ continue;
+ if (svc_ring_entry_dead(vgpu, h0, h1))
+ continue;
+ for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) {
+ u64 upa = off + udeltas[ui];
+ u32 p3 = 0, g3 = 0;
+
+ if (upa + 4 > lim)
+ continue;
+ if (svc_fb_claim_foreign(vgpu,
+ ch->h_object,
+ off, upa))
+ continue;
+ if (svc_userd_probe(vgpu, upa,
+ SVC_APERTURE_FB,
+ ch->gpfifo_entries,
+ &p3, &g3))
+ continue;
+ svc_pin_store(ch->gpfifo_va, off,
+ SVC_APERTURE_FB);
+ svc_lat_store(ch->gpfifo_va, off,
+ SVC_APERTURE_FB);
+ ch->userd_pa = upa;
+ ch->userd_as = SVC_APERTURE_FB;
+ ch->userd_ok = true;
+ svc_gr_ring_base = off;
+ svc_fb_claim_add(ch->h_object, off,
+ upa);
+ set_bit(ch->gr_index,
+ &svc_gr_ring_done);
+ set_bit(ch->gr_index,
+ &svc_gr_ring_scan_done);
+ svc_ring_n++;
+ pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (shadow scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n",
+ ch->h_object, ch->gr_index,
+ ch->gpfifo_va, off,
+ udeltas[ui], hva, hlen, p3,
+ g3);
+ return;
+ }
+ }
+ }
+ }
+
if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) {
u8 d1[8] = { 0 }, d2[8] = { 0 };
int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS;
@@ -15970,8 +17591,24 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
* entry and pushbuffer reads resolve. Sliced so the service thread
* stays responsive; gives up per channel after
* SVC_GR_RAM_PASS_MAX slices.
- */
- if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX &&
+ *
+ * SESSION 147 (r840 evidence): the scan must NOT run for the kernel
+ * RC-watchdog channel (h=0x31415900). Its ring is a plain kernel
+ * allocation whose VA is always resolvable through the driver-
+ * published VAS walk - the walk is the correct, deterministic
+ * source. On r840 the walk lost a registration race, the scan ran
+ * first and latched a FOREIGN page (pa=0x101cc3000, garbage
+ * entries: raw=0xffff88f10403e000), and the bad latch poisoned
+ * every later entry/push read: the driver's DMA test never saw its
+ * push complete and Xorg died with "Failed to initialize DMA"
+ * (the same boot passed on r835/836/838 where the trusted walk
+ * won the race). Gating the scan keeps the per-pass walk retry
+ * alive for the watchdog; the walk landing late is indistinguishable
+ * from the r838 flow, and never landing is no worse than a poisoned
+ * latch.
+ */
+ if (ch->h_object != 0x31415900u &&
+ ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX &&
!test_bit(ch->gr_index, &svc_gr_ring_done)) {
u64 hit_pa = 0;
@@ -16075,6 +17712,13 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+ /*
+ * SESSION 144b: record the block ownership here too - CE ranks that
+ * latch through this plain autopin path (not the first-submission
+ * capture) were invisible to the GR fallback scan, which then stole
+ * their block (r809: CE rank 1 at fb 0xa0000).
+ */
+ svc_fb_claim_add(ch->h_object, pa, pa + 0x10000);
set_bit(ch->ce_index, &svc_ring_done);
svc_ring_n++;
pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n",
@@ -16599,6 +18243,65 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
u32 ncand = 0, i;
/*
+ * SESSION 148 perf (measured on r843): content-verified full-chain
+ * rescue, usable by GR channels too. The CE-only gate below refused
+ * every GR push outright, yet the same svc_lookup the diagnostic
+ * probe uses resolves the stalled entries correctly (X's frame
+ * pushes at 0x120000514 -> vidmem shadow pa=0x200514 with plausible
+ * stream content) while the pump's own read went through a stale
+ * shadowing translation, failed plausibility, and skipped the entry
+ * - one full retry window per push, the measured 22.4 s/frame
+ * glxgears cadence. Resolve via svc_lookup, verify the bytes parse
+ * as a method stream, then pin the verified page so the pump's
+ * re-read lands on it. Cheap: one chain lookup + one page read.
+ */
+ if (entry_va && entry_len && entry_len <= SEMA_PAGE_SIZE) {
+ u64 mpa = 0;
+ int map = SVC_APERTURE_SYS;
+ u8 *vb = kmalloc(entry_len, GFP_KERNEL);
+ int lrc = -ENOMEM, rrc = -ENOMEM;
+ bool plaus = false;
+
+ if (vb) {
+ lrc = svc_lookup(vgpu, entry_va, &mpa, &map);
+ if (!lrc)
+ rrc = svc_read_ap(vgpu, mpa, map, vb,
+ entry_len);
+ if (!rrc)
+ plaus = svc_method_stream_plausible(vb,
+ entry_len);
+ if (gsp_sema_trace_h &&
+ (u32)ch->h_object == (u32)gsp_sema_trace_h &&
+ time_after(jiffies, svc_perf_last)) {
+ svc_perf_last = jiffies + HZ;
+ pr_info("sema-perf: maprescue h=0x%x va=0x%llx len=%u lookup=%d pa=0x%llx ap=%d read=%d plaus=%d\n",
+ ch->h_object, entry_va, entry_len,
+ lrc, mpa, map, rrc, plaus);
+ }
+ if (!lrc && !rrc && plaus) {
+ svc_pin_store(entry_va &
+ ~(u64)(SEMA_PAGE_SIZE - 1),
+ mpa &
+ ~(u64)(SEMA_PAGE_SIZE - 1),
+ map);
+ svc_lat_store(entry_va &
+ ~(u64)(SEMA_PAGE_SIZE - 1),
+ mpa &
+ ~(u64)(SEMA_PAGE_SIZE - 1),
+ map);
+ svc.rescue_map_ok++;
+ if (svc.rescue_map_ok <= 8)
+ pr_info("sema-svc: rescue via map h=0x%x va=0x%llx -> pa=0x%llx ap=%d (verified stream)\n",
+ ch->h_object, entry_va, mpa,
+ map);
+ kfree(vb);
+ return 0;
+ }
+ kfree(vb);
+ }
+ }
+
+ /*
* The matcher needs only the channel rank (slot) and the release-push
* shape; it does NOT need the pool semaphore translation that
* expected_sem_payload is learned from. Requiring the learned payload
@@ -16902,6 +18605,16 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
*/
u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa;
+ if (gsp_sema_trace_h && (u32)ch->h_object == (u32)gsp_sema_trace_h &&
+ time_after(jiffies, svc_perf_last)) {
+ svc_perf_last = jiffies + HZ;
+ pr_info("sema-perf: pub h=0x%x put=%u last=%u armed=%d pa=0x%llx cnt=%u gate=%s\n",
+ ch->h_object, put, (u32)ch->last_get, ch->gr_pub_armed,
+ pa, ch->counter_last,
+ !pa ? "no-pa" :
+ (ch->last_get != put) ? "outstanding" :
+ (!ch->gr_pub_armed ? "unarmed" : "publish"));
+ }
if (!pa)
return;
/*
@@ -18534,6 +20247,156 @@ static void svc_fwd_hold_direct_note(void)
}
/*
+ * SESSION 127 C5C0 translator: pending GRCE copy-out list.
+ *
+ * The Vulkan compute client submits its frame work in the pre-built
+ * continuation beyond GP_PUT: beec0100 idx23 carries the inline-QMD launch
+ * (0x31c = 0x4b85f, 64-word 0x320 run) and idx24 carries the TURING_DMA_COPY_A
+ * (0xC5B5, subch 4) image->buffer copies that un-tile the storage image into
+ * the client's sysmem readback buffer. Both pushes are only visible to the
+ * sema_gr_ahead sweep (see svc_gr_ahead_push), not to the put-bounded main
+ * walk.
+ *
+ * Copy ordering: the copy must read the image AFTER the forwarded launch
+ * completed (the engine's output is copied back into the VA shadow on the
+ * completion edge). The sweep only records the copy; it is executed inside
+ * svc_fwd_hold_gr_release, i.e. after the backend published done and after
+ * the copy-out, immediately before the launch's guest release word is
+ * authored - so the client's fence never observes a pre-render image.
+ * A bounded fallback run (sweep, when no forward is owed) keeps the list
+ * from stalling when the launch was refused.
+ */
+#define SVC_GR_COPY_PEND 8
+struct svc_gr_copy_pend {
+ bool valid;
+ u32 h_object;
+ u64 push_va;
+ u64 hash;
+ u64 src;
+ u64 dst;
+ u32 bytes; /* per-row byte count */
+ u32 rows;
+ u32 dpitch;
+ u32 spitch;
+ u32 dma; /* LAUNCH_DMA word (remap/transfer bits, logging) */
+ u32 remap; /* SET_REMAP_COMPONENTS (logging) */
+};
+static struct svc_gr_copy_pend svc_gr_copy_pends[SVC_GR_COPY_PEND];
+static u32 svc_gr_copy_pend_next;
+static u64 svc_gr_copy_run_total;
+static u64 svc_gr_copy_run_log;
+
+static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash,
+ u64 src, u64 dst, u32 bytes, u32 rows,
+ u32 dpitch, u32 spitch, u32 dma, u32 remap)
+{
+ u32 i;
+
+ if (!bytes || !rows || !src || !dst)
+ return;
+ for (i = 0; i < SVC_GR_COPY_PEND; i++) {
+ struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i];
+
+ if (p->valid && p->push_va == push_va && p->src == src &&
+ p->dst == dst && p->bytes == bytes && p->rows == rows)
+ return; /* already recorded (idempotent re-sweep) */
+ }
+ for (i = 0; i < SVC_GR_COPY_PEND; i++) {
+ struct svc_gr_copy_pend *p =
+ &svc_gr_copy_pends[svc_gr_copy_pend_next++ %
+ SVC_GR_COPY_PEND];
+
+ if (p->valid)
+ continue; /* keep live ones; take a free slot */
+ p->valid = true;
+ p->h_object = h_object;
+ p->push_va = push_va;
+ p->hash = hash;
+ p->src = src;
+ p->dst = dst;
+ p->bytes = bytes;
+ p->rows = rows;
+ p->dpitch = dpitch ? dpitch : bytes;
+ p->spitch = spitch ? spitch : bytes;
+ p->dma = dma;
+ p->remap = remap;
+ pr_info("sema-gr: copy pend h=0x%x push=0x%llx src=0x%llx dst=0x%llx row=0x%x rows=%u dpitch=0x%x spitch=0x%x dma=0x%x remap=0x%x\n",
+ h_object, push_va, src, dst, bytes, rows, p->dpitch,
+ p->spitch, dma, remap);
+ return;
+ }
+ /* All slots live: overwrite the oldest position. */
+ {
+ struct svc_gr_copy_pend *p =
+ &svc_gr_copy_pends[svc_gr_copy_pend_next++ %
+ SVC_GR_COPY_PEND];
+
+ pr_warn_ratelimited("sema-gr: copy pend overflow (dropping push=0x%llx)\n",
+ p->push_va);
+ memset(p, 0, sizeof(*p));
+ }
+}
+
+/*
+ * Execute and retire every pending GRCE copy. Called from the completion
+ * edge (svc_fwd_hold_gr_release) and, as a bounded fallback, from the sweep
+ * when no forward is owed. The executor is the same svc_copy() the CE/GCE
+ * op path uses (handles vidmem shadow sources and sysmem destinations).
+ */
+static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu)
+{
+ u32 i;
+
+ for (i = 0; i < SVC_GR_COPY_PEND; i++) {
+ struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i];
+ u64 done = 0;
+ int rc;
+
+ if (!p->valid)
+ continue;
+ rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0,
+ SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done,
+ p->rows, p->dpitch, p->spitch, p->h_object, 0,
+ p->hash);
+ svc_gr_copy_run_total++;
+ if (rc == 0 || rc == SVC_COPY_FORWARDED) {
+ p->valid = false;
+ if (svc_gr_copy_run_log < 64) {
+ svc_gr_copy_run_log++;
+ pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n",
+ p->h_object, p->push_va, p->dst,
+ (u64)p->bytes * p->rows, p->rows, rc);
+ }
+ } else if (svc_gr_copy_run_log < 64) {
+ svc_gr_copy_run_log++;
+ pr_info("sema-gr: copy retry h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rc=%d done=0x%llx\n",
+ p->h_object, p->push_va, p->dst,
+ (u64)p->bytes * p->rows, rc, done);
+ }
+ }
+}
+
+/*
+ * Is this release VA owned by a live, not-yet-authored launch candidate
+ * captured from THIS push? (Only the launch's own push's release is
+ * withheld; a continuation fence entry elsewhere in the ring keeps the
+ * historical immediate-forward behaviour.)
+ */
+static bool svc_gr_ahead_owned_rel_va(u64 va, u64 push_va)
+{
+ u32 ri;
+
+ for (ri = 0; ri < SVC_GR_CAND_RING; ri++) {
+ if (svc_gr_cand_ring[ri].valid &&
+ !svc_gr_cand_ring[ri].rel_done &&
+ svc_gr_cand_ring[ri].rel_va == va &&
+ svc_gr_cand_ring[ri].push_va == push_va)
+ return true;
+ }
+ return false;
+}
+
+/*
* Author a GR launch completion release (the QMD-level semaphore the
* physical engine writes when the launch completes) in software,
* forward-only. Used when a hold resolves without a successful hardware
@@ -18581,6 +20444,14 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu,
{
if (!svc_fwd_req.gr)
return;
+ /*
+ * SESSION 127 C5C0 translator: the launch is complete and its output
+ * pages were copied back into the VA shadow on this edge - run any
+ * pending GRCE copy-outs (image -> linear readback buffer) BEFORE
+ * authoring the client's release word, so the fence can never clear
+ * on a pre-render image.
+ */
+ svc_gr_copy_pend_run(vgpu);
svc_gr_sw_release(vgpu, ch ? ch->h_object : 0,
svc_fwd_req.gr_rel_va,
svc_fwd_req.gr_rel_payload);
@@ -18718,6 +20589,13 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu)
svc_fwd_req.gr_push_va, waited);
return;
}
+ /*
+ * SESSION 127 C5C0 translator: this edge authors the guest release
+ * without a hold - run any pending GRCE copy-outs first, exactly like
+ * the hold's success edge, so the fence cannot clear on a pre-render
+ * image.
+ */
+ svc_gr_copy_pend_run(vgpu);
svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object,
svc_fwd_req.gr_rel_va,
svc_fwd_req.gr_rel_payload);
@@ -20062,8 +21940,34 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
goto note_done;
}
}
- if (svc_resolve_va(vgpu, page, &pa, &ap))
- return;
+ if (svc_resolve_va(vgpu, page, &pa, &ap)) {
+ /*
+ * SESSION 127 C5C0 translator (gated): a client-VAS page. The
+ * Vulkan
+ * QMD's program, parameter and descriptor pages live in the
+ * client's own page tables (vidmem aperture) and the trusted
+ * sources - pins, learned cache, discovered map - do not
+ * carry them, which made the fill refuse the launch with
+ * -ENOENT. Resolve through the registered-VAS walk instead:
+ * the exact path walk_probe proves (r707w).
+ */
+ u64 wpa = 0;
+ u32 wap = SVC_APERTURE_SYS;
+
+ if (!gsp_sema_gr_xlate ||
+ svc_walk_vaspaces(vgpu, page, &wpa, &wap))
+ return;
+ pa = wpa;
+ ap = svc_ap_of(wap);
+ svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+ svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap);
+ svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+ (u32)svc_ap_of(ap));
+ svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+ (int)svc_ap_of(ap));
+ src = "vas";
+ goto note_done;
+ }
svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap);
/*
@@ -20551,6 +22455,14 @@ static void svc_gr_scan_cand(u64 *wins, u32 *n_cand, const u8 *data, u32 bytes)
if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3))
continue;
+ /* SESSION 139: float-shaped words are scalars, not device VAs
+ * (see the scan-site note; r789's 480.0f = 0x43f00000). */
+ {
+ u32 fexp = (u32)(v >> 23) & 0xffu;
+
+ if (fexp >= 0x6cu && fexp <= 0x8fu)
+ continue;
+ }
wv = v & ~(u64)(SEMA_PAGE_SIZE - 1);
for (w = 0; w < *n_cand; w++)
if (wins[w] == wv)
@@ -21056,18 +22968,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i];
u64 hpa = 0;
const u8 *page;
+ u8 pbuf[SEMA_PAGE_SIZE];
u32 k;
if (pp->qmd_word != 32u || !pp->pa)
continue;
- kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
- SVC_APERTURE_SYS;
- if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0,
- SEMA_PAGE_SIZE, &hpa, &off) || off)
- continue;
- page = (const u8 *)__va(hpa);
- if (!page)
- continue;
+ /*
+ * SESSION 140: read the page through the pin-aware reader
+ * first. The r791 evidence: the QMD page (va 0x4b85000)
+ * resolves to a vidmem shadow that reads empty, while the
+ * content the capture decoded comes from the push-buffer
+ * discovery's pinned staging copy - so a raw host mapping of
+ * pp->pa sees zeros and the pointer values never become
+ * candidates. svc_read_va consults ipin/pin/lat before the
+ * walk and returns the staged bytes.
+ */
+ if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) {
+ page = pbuf;
+ } else {
+ kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
+ SVC_APERTURE_SYS;
+ if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0,
+ SEMA_PAGE_SIZE, &hpa, &off) || off)
+ continue;
+ page = (const u8 *)__va(hpa);
+ if (!page)
+ continue;
+ }
for (k = 0x150u / 8u; k < 256; k++) {
u64 v = get_unaligned_le64(page + k * 8);
u64 wv;
@@ -21075,6 +23002,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
if (v < 0x10000000ULL || v >= (1ULL << 47) ||
(v & 3))
continue;
+ /*
+ * SESSION 139: reject float-shaped words. The launch
+ * param page carries f32 scalars (r789: 0.2f, 640.0f,
+ * 480.0f at +0x630) whose bit patterns pass the
+ * VA-shape filter (0x43f00000 = 480.0f) and were the
+ * fill's only candidate - an unresolvable "arg base".
+ * A plausible f32 magnitude has an exponent byte in
+ * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47)
+ * as f32 land in subnormal/tiny exponents, so this
+ * does not reject pointers.
+ */
+ {
+ u32 fexp = (u32)(v >> 23) & 0xffu;
+
+ if (fexp >= 0x6cu && fexp <= 0x8fu)
+ continue;
+ }
wv = v & ~(u64)(SEMA_PAGE_SIZE - 1);
for (w = 0; w < n_cand; w++)
if (wins[w] == wv)
@@ -21109,18 +23053,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i];
u64 hpa = 0;
const u8 *page;
+ u8 pbuf[SEMA_PAGE_SIZE];
u32 k;
if (pp->qmd_word != 32u || !pp->pa)
continue;
- kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
- SVC_APERTURE_SYS;
- if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0,
- SEMA_PAGE_SIZE, &hpa, &off) || off)
- continue;
- page = (const u8 *)__va(hpa);
- if (!page)
- continue;
+ /*
+ * SESSION 140: read the page through the pin-aware reader
+ * first. The r791 evidence: the QMD page (va 0x4b85000)
+ * resolves to a vidmem shadow that reads empty, while the
+ * content the capture decoded comes from the push-buffer
+ * discovery's pinned staging copy - so a raw host mapping of
+ * pp->pa sees zeros and the pointer values never become
+ * candidates. svc_read_va consults ipin/pin/lat before the
+ * walk and returns the staged bytes.
+ */
+ if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) {
+ page = pbuf;
+ } else {
+ kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
+ SVC_APERTURE_SYS;
+ if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0,
+ SEMA_PAGE_SIZE, &hpa, &off) || off)
+ continue;
+ page = (const u8 *)__va(hpa);
+ if (!page)
+ continue;
+ }
for (k = 0; k < SEMA_PAGE_SIZE / 8; k++) {
u64 v, wv;
@@ -21130,6 +23089,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
if (v < 0x10000000ULL || v >= (1ULL << 47) ||
(v & 3))
continue;
+ /*
+ * SESSION 139: reject float-shaped words. The launch
+ * param page carries f32 scalars (r789: 0.2f, 640.0f,
+ * 480.0f at +0x630) whose bit patterns pass the
+ * VA-shape filter (0x43f00000 = 480.0f) and were the
+ * fill's only candidate - an unresolvable "arg base".
+ * A plausible f32 magnitude has an exponent byte in
+ * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47)
+ * as f32 land in subnormal/tiny exponents, so this
+ * does not reject pointers.
+ */
+ {
+ u32 fexp = (u32)(v >> 23) & 0xffu;
+
+ if (fexp >= 0x6cu && fexp <= 0x8fu)
+ continue;
+ }
wv = v & ~(u64)(SEMA_PAGE_SIZE - 1);
for (w = 0; w < n_cand; w++)
if (wins[w] == wv)
@@ -21142,19 +23118,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i];
u64 hpa = 0;
const u8 *page;
+ u8 pbuf[SEMA_PAGE_SIZE];
u32 k;
- if (pp->qmd_word < 32 || pp->qmd_word > 47 ||
+ /* SESSION 139: include word 48 (see the r788 finding: the
+ * word-34 pointer dedupes into the program's 4K page whose
+ * entry carries qmd_word=48; the old 32..47-even filter left
+ * that page unscanned). */
+ if (pp->qmd_word < 32 || pp->qmd_word > 48 ||
(pp->qmd_word & 1) || !pp->pa)
continue;
- kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
- SVC_APERTURE_SYS;
- if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0,
- SEMA_PAGE_SIZE, &hpa, &off) || off)
- continue;
- page = (const u8 *)__va(hpa);
- if (!page)
- continue;
+ /*
+ * SESSION 140: read the page through the pin-aware reader
+ * first. The r791 evidence: the QMD page (va 0x4b85000)
+ * resolves to a vidmem shadow that reads empty, while the
+ * content the capture decoded comes from the push-buffer
+ * discovery's pinned staging copy - so a raw host mapping of
+ * pp->pa sees zeros and the pointer values never become
+ * candidates. svc_read_va consults ipin/pin/lat before the
+ * walk and returns the staged bytes.
+ */
+ if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) {
+ page = pbuf;
+ } else {
+ kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
+ SVC_APERTURE_SYS;
+ if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0,
+ SEMA_PAGE_SIZE, &hpa, &off) || off)
+ continue;
+ page = (const u8 *)__va(hpa);
+ if (!page)
+ continue;
+ }
/*
* relfix PART 3: whole page, matching the rewrite's scan
* range (it walks 0x0..0xfff qword-pairs); the old 0..0x7ff
@@ -21167,6 +23162,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
if (v < 0x10000000ULL || v >= (1ULL << 47) ||
(v & 3))
continue;
+ /*
+ * SESSION 139: reject float-shaped words. The launch
+ * param page carries f32 scalars (r789: 0.2f, 640.0f,
+ * 480.0f at +0x630) whose bit patterns pass the
+ * VA-shape filter (0x43f00000 = 480.0f) and were the
+ * fill's only candidate - an unresolvable "arg base".
+ * A plausible f32 magnitude has an exponent byte in
+ * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47)
+ * as f32 land in subnormal/tiny exponents, so this
+ * does not reject pointers.
+ */
+ {
+ u32 fexp = (u32)(v >> 23) & 0xffu;
+
+ if (fexp >= 0x6cu && fexp <= 0x8fu)
+ continue;
+ }
wv = v & ~(u64)(SEMA_PAGE_SIZE - 1);
for (w = 0; w < n_cand; w++)
if (wins[w] == wv)
@@ -21179,6 +23191,58 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
svc_gr_scan_inline(wins, &n_cand, pp->va);
}
/*
+ * SESSION 140 (the n_data wiring): seed the candidate set with the
+ * launch's OWN QMD pointer pairs (words 34..47, already decoded into
+ * gr_qmd from the push's inline-QMD run). These ARE the launch's
+ * data-window pointers - the r792 evidence: the capture holds
+ * ptr[0]=0xdff0febc00 but the fill's page scans read only empty
+ * shadow pages (the QMD page's vidmem shadow is zero; its content
+ * never arrives - the invisible-vidmem-write class), so the pointer
+ * values could never become candidates and n_data stayed 0 (refused
+ * degenerate). The values need no page scan - use them directly.
+ */
+ {
+ u32 w2;
+
+ for (w2 = 0; w2 + 1 < 16; w2 += 2) {
+ u64 pv = (u32)svc_fwd_req.gr_qmd[34 + w2] |
+ ((u64)(svc_fwd_req.gr_qmd[35 + w2] &
+ 0x1ffffu) << 32);
+ u64 pw;
+ u32 fexp;
+
+ if (pv < 0x10000000ULL || pv >= (1ULL << 47) ||
+ (pv & 3))
+ continue;
+ fexp = (u32)(pv >> 23) & 0xffu;
+ if (fexp >= 0x6cu && fexp <= 0x8fu)
+ continue;
+ pw = pv & ~(u64)(SEMA_PAGE_SIZE - 1);
+ for (w = 0; w < n_cand; w++)
+ if (wins[w] == pw)
+ break;
+ if (w == n_cand && n_cand < ARRAY_SIZE(wins)) {
+ wins[n_cand++] = pw;
+ is_arg[n_cand - 1] = true;
+ if (svc_gr_pages_log < 64) {
+ svc_gr_pages_log++;
+ pr_info("sema-gr: qmd-ptr seed va=0x%llx (word %u) h=0x%x\n",
+ pw, 34 + w2,
+ svc_fwd_req.gr_h_object);
+ }
+ }
+ }
+ }
+ if (gsp_sema_dbg && svc_gr_pages_log < 48) {
+ svc_gr_pages_log++;
+ pr_info("sema-gr: fill cand n=%u pages=%u push=0x%llx\n",
+ n_cand, svc_fwd_req.gr_n_pages,
+ svc_fwd_req.gr_push_va);
+ for (w = 0; w < n_cand && w < 12; w++)
+ pr_info("sema-gr: win[%u] va=0x%llx arg=%u\n",
+ w, wins[w], is_arg[w]);
+ }
+ /*
* r635: the launch-level CONTENT anchor for the host-side fallback,
* computed BEFORE the span loop so candidate order cannot decide.
* An argument candidate whose span already holds the client's own
@@ -21817,7 +23881,12 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
u64 qpa = 0;
u32 qap = SVC_APERTURE_SYS;
- if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) {
+ if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap) ||
+ (gsp_sema_gr_xlate &&
+ !svc_walk_vaspaces(vgpu,
+ svc_gr_cand.qmd_dev_va &
+ ~(u64)(SEMA_PAGE_SIZE - 1),
+ &qpa, &qap))) {
svc_fwd_req.gr_qmd_pa =
qpa & ~(u64)(SEMA_PAGE_SIZE - 1);
svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap);
@@ -21877,12 +23946,42 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32);
if (svc_gr_cand.prog_va)
svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48);
+ /*
+ * SESSION 140: the QMD page itself carries the launch's pointer
+ * VALUES (QMD words 32..49 live inside it - the param, the
+ * pointer-region pairs and the program). Without this note the fill
+ * never scans it, so the pointer values (r790: word 34 =
+ * 0xdff0febc00) can never become candidates and n_data stays 0
+ * (refused: degenerate data set). Noted under word 34 so the
+ * 32..48-even bank-page scan covers it.
+ */
+ if (svc_gr_cand.qmd_dev_va)
+ svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34);
for (i = 0; i < svc_gr_cand.n_pointer; i++)
svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i],
svc_gr_cand.pointer_word[i]);
for (i = 0; i < svc_gr_cand.n_inline; i++)
svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va,
SVC_GR_QMD_WORD_NONE);
+ if (gsp_sema_dbg && svc_gr_pages_log < 24) {
+ svc_gr_pages_log++;
+ pr_info("sema-gr: cand h=0x%x param=0x%llx prog=0x%llx n_ptr=%u inline=%u n_pages=%u push=0x%llx\n",
+ svc_gr_cand.h_object, svc_gr_cand.param_va,
+ svc_gr_cand.prog_va, svc_gr_cand.n_pointer,
+ svc_gr_cand.n_inline, svc_fwd_req.gr_n_pages,
+ svc_gr_cand.push_va);
+ for (i = 0; i < svc_gr_cand.n_pointer && i < 8; i++)
+ pr_info("sema-gr: ptr[%u] va=0x%llx word=%u\n", i,
+ svc_gr_cand.pointer_va[i],
+ svc_gr_cand.pointer_word[i]);
+ for (i = 0; i < svc_fwd_req.gr_n_pages && i < 12; i++) {
+ const struct svc_gr_page *pg =
+ &svc_fwd_req.gr_pages[i];
+
+ pr_info("sema-gr: page[%u] va=0x%llx pa=0x%llx ap=%u qmd_word=%u\n",
+ i, pg->va, pg->pa, pg->ap, pg->qmd_word);
+ }
+ }
if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages)
return -E2BIG;
for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
@@ -21890,10 +23989,19 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
if (pg->pa)
continue;
- if (pg->qmd_word == SVC_GR_QMD_WORD_NONE)
- return -ENOENT;
- if (!svc_gr_page_inline_covers(pg->va))
+ if (pg->qmd_word == SVC_GR_QMD_WORD_NONE ||
+ !svc_gr_page_inline_covers(pg->va)) {
+ if (svc.fwd_gr_enoe_log < 8) {
+ svc.fwd_gr_enoe_log++;
+ pr_info("vgsp-work: GR fill ENOENT va=0x%llx qmd_word=%u inline=%u covered=%d n_pages=%u push=0x%llx\n",
+ pg->va, pg->qmd_word,
+ svc_gr_cand.n_inline,
+ svc_gr_page_inline_covers(pg->va),
+ svc_fwd_req.gr_n_pages,
+ svc_gr_cand.push_va);
+ }
return -ENOENT;
+ }
}
/*
* GRFIX FIX1c: the fill hosts the self-enforcing decision point and
@@ -22480,11 +24588,25 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
}
}
if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid ||
- svc_gr_cand.h_object != ch->h_object)
+ svc_gr_cand.h_object != ch->h_object) {
+ if (svc.fwd_gr_nocand_log < 8) {
+ svc.fwd_gr_nocand_log++;
+ pr_info("vgsp-work: GR request without candidate h=0x%x (forward=%d cand_valid=%d cand_h=0x%x offered=%d)\n",
+ ch->h_object, gsp_phys_gr_forward_enabled(),
+ svc_gr_cand.valid, svc_gr_cand.h_object,
+ svc_gr_cand.offered);
+ }
return false;
+ }
svc_gr_data_stalled = false; /* m150: only the fill sets this */
svc.fwd_gr_req++;
if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) {
+ svc.fwd_gr_nordy++;
+ if ((svc.fwd_gr_nordy & 0x3F) == 1)
+ pr_info("vgsp-work: GR request refused: backend not ready h=0x%x ready=%d gmem=%d n=%llu\n",
+ ch->h_object, gsp_phys_backend_gr_ready(),
+ gsp_phys_guest_mem_enabled(),
+ svc.fwd_gr_nordy);
svc.fwd_gr_refused++;
svc.fwd_gr_fallback++;
svc_gr_bridge_fallback(vgpu);
@@ -22495,6 +24617,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
* request's done/hw_rc are still being read. The capture falls back.
*/
if (svc_fwd_hold.active) {
+ svc.fwd_gr_holdbusy++;
+ if ((svc.fwd_gr_holdbusy & 0x3F) == 1)
+ pr_info("vgsp-work: GR request busy: hold active h=0x%x (hold ch=0x%x n=%llu)\n",
+ ch->h_object, svc_fwd_hold.ch_handle,
+ svc.fwd_gr_holdbusy);
svc.fwd_gr_refused++;
svc.fwd_gr_fallback++;
/*
@@ -22515,6 +24642,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
if (svc_fwd_paused ||
test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) {
mutex_unlock(&svc_fwd_lock);
+ svc.fwd_gr_inflight++;
+ if ((svc.fwd_gr_inflight & 0x3F) == 1)
+ pr_info("vgsp-work: GR request busy: inflight h=0x%x (state=0x%lx n=%llu)\n",
+ ch->h_object, svc_fwd_state,
+ svc.fwd_gr_inflight);
svc.fwd_gr_refused++;
svc.fwd_gr_fallback++;
if (!gsp_cuda_bridge_enabled()) {
@@ -22541,7 +24673,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
mutex_unlock(&svc_fwd_lock);
svc.fwd_gr_refused++;
svc.fwd_gr_fallback++;
- if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1)
+ if (svc.fwd_gr_refused < 8 ||
+ (svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1)
pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n",
rc, ch->h_object, svc_gr_cand.push_va);
svc_gr_bridge_fallback(vgpu);
@@ -23508,8 +25641,1148 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu,
put, get, ch->submissions, waited, reason);
}
+/*
+ * ---------------------------------------------------------------------
+ * SESSION 126 - FENCE FOLD: channel-anchored sweep of a GR-class
+ * channel's PRE-BUILT ring continuation.
+ *
+ * The Vulkan client's fence waits never clear unaided because the
+ * driver's kick boundary (GP_PUT) stops at the first batch while the ring
+ * already carries the continuation entries it pre-built for the next
+ * submits - and the fence's own INC-5 RELEASE group (observed
+ * [2005c017][0x45ba090][0][payload 1][0][0x02100001]) lives in one of
+ * those entries, so the pump's GP_PUT-bounded walk never decodes it
+ * (r715/r716: put=11, entries 12..47 carried content, no dmesg row ever
+ * mentioned 0x45ba). The host-side fenceword_assist wrote the wait word
+ * directly; this fold makes the service author it from the driver's own
+ * push bytes.
+ *
+ * Doctrine: unlike the disabled page-content sweep (svc_apply_page_data,
+ * off since the false-positive corruption incidents), this sweep is
+ * CHANNEL-ANCHORED: it only walks entries reachable through the ring's
+ * validated GPFIFO translation, only accepts entries whose bytes pass the
+ * strict method-stream gate, and only applies the semaphore release
+ * groups it finds - forward-only, through the standard pending ledger
+ * (svc_pending_add). No consumption bookkeeping is touched (last_get,
+ * put_prev, slot_done_epoch and the replay state stay as the pump left
+ * them), so a later real kick of the same entries decodes them normally;
+ * repeated sweeps are idempotent (the pending ledger dedups (va,payload)
+ * and the apply path is forward-only).
+ *
+ * Gated by the same live-epoch arm as the counter publish
+ * (ch->gr_pub_armed) so a residual ring from a previous session cannot
+ * author releases before the session switch is proven.
+ * ---------------------------------------------------------------------
+ */
+unsigned gsp_sema_gr_ahead_max = 64;
+module_param_named(sema_gr_ahead, gsp_sema_gr_ahead_max, uint, 0644);
+MODULE_PARM_DESC(sema_gr_ahead,
+ "GR-class channels: max GPFIFO entries beyond GP_PUT whose channel-anchored pushes are swept for semaphore releases (0=off)");
+
+
+static u32 svc_gr_ahead_log;
+static u64 svc_gr_ahead_entries;
+static u64 svc_gr_ahead_rels;
+static u64 svc_gr_ahead_skips;
+/* SESSION 127: releases withheld because a live candidate owns the word. */
+static u64 svc_gr_ahead_owned;
+/* SESSION 127: bounded witness for the sweep-caller's launch request. */
+static u32 svc_gr_ahead_req_log;
+/* SESSION 128: inline (LOAD_INLINE_DATA) writes authored by the sweep. */
+static u64 svc_gr_ahead_inline_writes;
+
+/*
+ * SESSION 129 util-channel witness: the utility channel's subchannel-2
+ * LOAD_INLINE_DATA arm never fired (writes=0 on r737) while the same arm
+ * fired for in-push payloads on the GR channel - i.e. the failure sits in
+ * the split (tail-served) shape. This struct carries the per-entry story
+ * out of the translator for one bounded, deduplicated witness line:
+ * which read source answered (attempt A = the channel path, 1 = the direct
+ * registered-VAS walk), whether the strict parse accepted the entry, and
+ * what the LID arm did (saw/fired/refused a still-zero payload).
+ */
+struct svc_util_wit {
+ u32 p_src; /* attempt that won: 0 = channel, 1 = VAS walk, 0xff none */
+ u32 pa_ok; /* attempt-A parse (method count) */
+ u32 pb_ok; /* attempt-B parse (method count) */
+ u32 t_src; /* tail read source: 0 = channel, 1 = walk, 0xff none */
+ u32 t_len; /* tail length served */
+ u32 t0, t1; /* SESSION 130: first two words of the served tail */
+ bool saw_lid; /* a LOAD_INLINE_DATA arm was reached */
+ bool fired; /* ... and authored its write */
+ bool zero; /* ... but the tail payload was still all-zero (pending) */
+ u32 dst;
+ u32 line_len;
+ u32 line_lc;
+ u32 count;
+};
+static struct svc_util_wit svc_util_wit;
+static u64 svc_util_pending_zero;
+
+/*
+ * SESSION 127 C5C0 translator, bounded tail materialization buffer: a
+ * LOAD_INLINE_DATA run whose payload overruns the entry end is served from
+ * the next ring entry (the driver splits large inline runs into a methods
+ * entry plus a dedicated data slot entry). 4 KB covers the observed
+ * param-bank write (1568 B) in full and the first page of the big uploads.
+ */
+#define SVC_AHEAD_TAIL_INLINE_MAX 32768
+static u8 svc_ahead_inline_buf[SVC_AHEAD_TAIL_INLINE_MAX];
+static u64 svc_ahead_inline_trunc;
+
+/*
+ * SESSION 128: one-copy materialization bound for the utility/CE sweep.
+ * The engine-side copy-in the sweep replays (a TURING_DMA_COPY_A pitch
+ * copy from a client-owned staging buffer into the vidmem shadow) moves
+ * the driver's code/constant uploads; the observed SASS/code-heap class is
+ * tens of KB, so 4 MB is a generous cap that still refuses a bogus
+ * geometry before it can walk the shadow.
+ */
+#define SVC_AHEAD_COPY_IN_MAX (4ULL << 20)
+
+/*
+ * SESSION 128: materialize one TURING_DMA_COPY_A pitch copy-in that an
+ * unprocessed copy-class channel carried - src bytes live in the client's
+ * staging buffer (guest RAM, resolved through the trusted walk like any
+ * operand page), dst is the vidmem page the driver staged the upload for
+ * (code heap / constant store). This is the emulated engine write for the
+ * copy the real CE would perform; it is once-only keyed by (push_va,
+ * content hash, dst, bytes) so re-scans of a stable ring are free.
+ */
+static u64 svc_ahead_copy_in_skip;
+static void svc_ahead_copy_in(struct nv_vgpu_device *vgpu, u32 ch_h,
+ u64 push_va, u64 hash, u64 src_va, u64 dst_va,
+ u32 row, u32 rows, u32 dma)
+{
+ static u8 cb[65536];
+ u64 bytes = (u64)row * (rows ? rows : 1);
+ u64 done = 0;
+ u64 spa = 0, dpa = 0;
+ int sap = SVC_APERTURE_SYS, dap = SVC_APERTURE_SYS;
+ int rc = 0;
+ bool src_ok, dst_ok;
+
+ if (!bytes || src_va < SVC_LOW_REL_VA_LO || src_va >= SEMA_VA_HI ||
+ dst_va < SVC_LOW_REL_VA_LO || dst_va >= SEMA_VA_HI)
+ return;
+ /*
+ * Only the upload direction is materialized: the source must be a
+ * resolved client page in sysmem/guest RAM (the staging buffer the
+ * client CPU-wrote), and the destination must be vidmem (resolved to
+ * the FB aperture) or a not-yet-resolved VA whose bytes the VA
+ * shadow will hold. A resolved SYS destination is a copy-OUT (the
+ * engine reading vidmem into a client buffer): replaying it here
+ * would overwrite the client's buffer with shadow garbage before the
+ * producing launch ran, so it is skipped - the GR channels' copy-out
+ * machinery owns that direction.
+ */
+ src_ok = !svc_lookup(vgpu, src_va, &spa, &sap) && sap != SVC_APERTURE_FB;
+ dst_ok = svc_lookup(vgpu, dst_va, &dpa, &dap) ||
+ dap == SVC_APERTURE_FB;
+ if (!src_ok || !dst_ok) {
+ svc_ahead_copy_in_skip++;
+ if (svc_ahead_copy_in_skip <= 8)
+ pr_info("sema-utilsweep: copy-in skipped (src_ok=%d dst_ok=%d) push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx\n",
+ src_ok, dst_ok, push_va, src_va, dst_va,
+ bytes);
+ return;
+ }
+ if (bytes > SVC_AHEAD_COPY_IN_MAX) {
+ svc_ahead_inline_trunc++;
+ if (svc_ahead_inline_trunc <= 8)
+ pr_warn("sema-utilsweep: copy-in too large push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx cap=0x%llx\n",
+ push_va, src_va, dst_va, bytes,
+ SVC_AHEAD_COPY_IN_MAX);
+ return;
+ }
+ if (svc_inline_once_seen(push_va, hash, dst_va, (u32)bytes))
+ return;
+ svc_inline_once_store(push_va, hash, dst_va, (u32)bytes);
+ while (done < bytes) {
+ u32 chunk = bytes - done > sizeof(cb) ?
+ (u32)sizeof(cb) : (u32)(bytes - done);
+
+ rc = svc_read_va(vgpu, src_va + done, cb, chunk);
+ if (rc) {
+ rc = -EIO;
+ break;
+ }
+ rc = svc_grce_inline_write(vgpu, dst_va + done, cb, chunk,
+ ch_h);
+ if (rc)
+ break;
+ done += chunk;
+ }
+ svc_gr_ahead_inline_writes++;
+ pr_info("sema-utilsweep: copy-in src=0x%llx -> dst=0x%llx bytes=0x%llx dma=0x%x done=0x%llx rc=%d h=0x%x\n",
+ src_va, dst_va, bytes, dma, done, rc, ch_h);
+}
+
+/* One 4-byte word of a method stream that may continue into the tail. */
+static u32 svc_ahead_w32(const u8 *buf, u32 len, const u8 *tail, u32 taillen,
+ u32 off)
+{
+ u32 v = 0;
+
+ if (off + 4 <= len)
+ memcpy(&v, buf + off, 4);
+ else if (tail && off >= len && (u64)(off - len) + 4 <= taillen)
+ memcpy(&v, tail + (off - len), 4);
+ return v;
+}
+
+/*
+ * Strict method-stream parse with a one-entry tail (see the sweep comment).
+ * A run whose payload overruns this entry's end may be completed from the
+ * next ring entry; anything else invalid means the page is not a method
+ * stream (the same false-positive arbiter as svc_stream_full_ok). Returns
+ * the method count, or 0; *used_tail reports that the tail served a
+ * trailing data run (nothing after it is parsed).
+ */
+static u32 svc_ahead_parse(const u8 *buf, u32 len, const u8 *tail, u32 taillen,
+ bool *used_tail)
+{
+ u32 off = 0, methods = 0;
+
+ *used_tail = false;
+ for (;;) {
+ u32 hdr, op, count;
+ u64 end;
+
+ if (off + 4 > len)
+ break;
+ memcpy(&hdr, buf + off, 4);
+ if (hdr == 0) {
+ off += 4;
+ continue;
+ }
+ op = (hdr >> 29) & 0x7;
+ count = (hdr >> 16) & 0xfff;
+ switch (op) {
+ case 1: /* INC_METHOD */
+ case 3: /* NON_INC_METHOD */
+ case 5: /* ONE_INC */
+ if (!count || count > 0xfff)
+ return 0;
+ end = (u64)off + 4 + (u64)count * 4;
+ if (end > len) {
+ if (!tail || (u64)(end - len) > taillen)
+ return 0;
+ *used_tail = true;
+ off = len;
+ goto done;
+ }
+ off = (u32)end;
+ break;
+ case 0: /* GRP0_USE_TERT */
+ case 2: /* GRP2_USE_TERT */
+ case 4: /* IMMD_DATA_METHOD */
+ case 7: /* END_PB_SEGMENT */
+ off += 4;
+ break;
+ default:
+ return 0;
+ }
+ if (++methods > 8192)
+ return 0;
+ }
+done:
+ while (off < len && buf[off] == 0)
+ off++;
+ return (methods > 0 && (off == len || *used_tail)) ? methods : 0;
+}
+
+/*
+ * SESSION 127 C5C0 translator sweep pass over one coherent push. Besides the
+ * original release extraction (svc_gr_ahead_push, below), it recognizes, for
+ * pre-built continuation entries:
+ *
+ * - the C5C0 inline-QMD window (SET_INLINE_QMD_ADDRESS_A/B + the 64-word
+ * LOAD_INLINE_QMD_DATA run) and feeds it to the standard svc_qmd_accum
+ * capture, so the physical GR forward (the CUDA QMD path) fires for a
+ * launch that never reaches the GP_PUT-bounded main walk - observed
+ * beec0100 idx23: `0x31c = 0x4b85f` + `0x320 x 64`;
+ * - GRCE / KEPLER_INLINE_TO_MEMORY_B LOAD_INLINE_DATA runs (op 3 at
+ * 0x1b4, offsets from the 0x180/0x184/0x188/0x18c group), materialized
+ * into the VA shadow exactly like the main decode does - observed
+ * beef0100 idx00 (256 B -> 0x423f000, payload in idx01) and idx08
+ * (12288 B -> 0x4448000, payload in idx09), and beec0100 idx23
+ * (1568 B -> 0x4b86020, payload in-push);
+ * - TURING_DMA_COPY_A (0xC5B5, subch >= 2) copy operands
+ * (0x400/0x404 in, 0x408/0x40c out, 0x410/0x414 pitch, 0x418/0x41c
+ * line length/count, 0x300 LAUNCH_DMA, 0x700..0x708 remap) recorded as
+ * pending copy-outs for the completion edge (svc_gr_copy_pend_run) -
+ * observed beec0100 idx24: image 0xdfd8d60000 -> readback 0x050b0000,
+ * 640x480 rows.
+ *
+ * Returns the number of release groups queued (as before). The buffer must
+ * pass the strict tail-aware parse first; a page that is not a method
+ * stream is not processed at all.
+ */
+static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+ const u8 *buf, u32 len, const u8 *tail,
+ u32 taillen, u64 push_va, bool inline_only)
+{
+ static struct svc_push_state stq; /* service thread only */
+ static u32 dw[66]; /* QMD window words */
+ u32 off = 0, nrel = 0, methods, i;
+ u64 off_in = 0, off_out = 0;
+ u32 have_in = 0, have_out = 0;
+ u32 pitch_in = 0, pitch_out = 0;
+ u32 line_len = 0, line_count = 0;
+ u32 last_dma = 0, last_remap = 0;
+ u32 cp_subch = ~0u;
+ bool used_tail = false;
+ bool forward = gsp_phys_gr_forward_enabled();
+ bool xlate = gsp_sema_gr_xlate;
+
+ if (!xlate) {
+ /* Historical sweep: releases from exactly-covered entries. */
+ tail = NULL;
+ taillen = 0;
+ }
+ methods = svc_ahead_parse(buf, len, tail, taillen, &used_tail);
+ if (!methods)
+ return 0;
+
+ memset(&stq, 0, sizeof(stq));
+ stq.push_va = push_va;
+
+ while (off + 4 <= len) {
+ u32 hdr, op, addr, count, subch;
+ u32 w0, w1, w2, w3, w4;
+ u64 end;
+
+ memcpy(&hdr, buf + off, 4);
+ if (!hdr) {
+ off += 4;
+ continue;
+ }
+ op = (hdr >> 29) & 0x7;
+ addr = (hdr & 0x1fff) << 2;
+ count = (hdr >> 16) & 0xfff;
+ subch = (hdr >> 13) & 0x7;
+ if (op == 0 || op == 2 || op == 4 || op == 7) {
+ off += 4;
+ continue;
+ }
+ if (op != 1 && op != 3 && op != 5)
+ break;
+ if (!count || count > 0xfff)
+ break;
+ end = (u64)off + 4 + (u64)count * 4;
+ /*
+ * Tail-served trailing run: process from the head word, then
+ * stop (the tail is a data slot, nothing parses after it).
+ */
+ w0 = svc_ahead_w32(buf, len, tail, taillen, off + 4);
+ w1 = svc_ahead_w32(buf, len, tail, taillen, off + 8);
+ w2 = svc_ahead_w32(buf, len, tail, taillen, off + 12);
+ w3 = svc_ahead_w32(buf, len, tail, taillen, off + 16);
+ w4 = svc_ahead_w32(buf, len, tail, taillen, off + 20);
+
+ /*
+ * C5C0 inline-QMD window (mirrors the main decode's intercept):
+ * only accumulate once the run declares itself a QMD window.
+ */
+ if (xlate && forward && !inline_only && subch >= 1 &&
+ addr >= SVC_QMD_ADDR_A &&
+ addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 &&
+ (stq.qmd_have ||
+ (addr <= SVC_QMD_ADDR_B &&
+ (u64)addr + (u64)count * 4 > SVC_QMD_ADDR_A))) {
+ u32 n = count > 66 ? 66 : count;
+
+ for (i = 0; i < n; i++)
+ dw[i] = svc_ahead_w32(buf, len, tail, taillen,
+ off + 4 + 4 * i);
+ svc_qmd_accum(vgpu, ch, &stq, addr, n,
+ (const u8 *)dw, buf, len, true, false);
+ if (end > len)
+ break;
+ off += 4 + count * 4;
+ continue;
+ }
+
+ /*
+ * GRCE / INLINE_TO_MEMORY_B LOAD_INLINE_DATA: materialize the
+ * DMA write (the engine would do this; the emulation authors
+ * the bytes into the VA shadow so the forwarded launch's
+ * operand pages - parameter banks - are real).
+ */
+ if (xlate && op == 3 && subch >= 1 &&
+ addr == GRCE_M_LOAD_INLINE_DATA &&
+ have_out && line_len) {
+ u64 nbytes = (u64)line_len *
+ (line_count ? line_count : 1);
+ u32 cap = count * 4;
+ u32 bytes, k, words;
+ u64 h = svc_push_hash(buf, len);
+ bool tail_served = end > len;
+ bool refuse_zero = false;
+
+ if (nbytes > cap)
+ nbytes = cap;
+ bytes = (u32)nbytes;
+ if (bytes > SVC_AHEAD_TAIL_INLINE_MAX) {
+ bytes = SVC_AHEAD_TAIL_INLINE_MAX;
+ svc_ahead_inline_trunc++;
+ if (svc_ahead_inline_trunc <= 8)
+ pr_warn("sema-ahead: inline write truncated push=0x%llx dst=0x%llx want=%llu cap=%u\n",
+ push_va, off_out, nbytes,
+ SVC_AHEAD_TAIL_INLINE_MAX);
+ }
+ words = (bytes + 3) / 4;
+ for (k = 0; k < words; k++)
+ memcpy(svc_ahead_inline_buf + 4 * k,
+ &(u32){ svc_ahead_w32(buf, len, tail,
+ taillen,
+ off + 4 + 4 * k) },
+ 4);
+ /*
+ * SESSION 129: a tail-served payload whose head is still
+ * all zeros means the tail source answered with an empty
+ * page (a stale pin shadow, or the driver has not staged
+ * the bytes yet) - authoring zeros over the destination
+ * would corrupt the code heap. Leave the write pending
+ * (no once-store, the sweep retries every pass) until a
+ * nonzero payload arrives. In-push payloads are never
+ * gated: the bytes are part of the flushed submit.
+ */
+ if (inline_only && tail_served && bytes &&
+ !memchr_inv(svc_ahead_inline_buf, 0,
+ min_t(u32, bytes, 16)))
+ refuse_zero = true;
+ if (inline_only) {
+ svc_util_wit.saw_lid = true;
+ svc_util_wit.dst = (u32)off_out;
+ svc_util_wit.line_len = line_len;
+ svc_util_wit.line_lc = line_count;
+ svc_util_wit.count = count;
+ if (refuse_zero)
+ svc_util_wit.zero = true;
+ }
+ if (refuse_zero) {
+ /*
+ * SESSION 130: a zero-headed tail-served
+ * payload is applied under a VARIANT once-key
+ * (h | 1) instead of being refused forever:
+ * the destination is currently empty, so
+ * authoring zeros is idempotent and correct
+ * for the driver's internal zero buffer (the
+ * observed 256 B utility i2m = exactly
+ * GF100_ZERO_BUFFER_SIZE), while a later
+ * non-zero payload for the same entry still
+ * matches its own (h) key and lands - the
+ * refusal can no longer stall the arm
+ * (r737/r739: `writes=0` with the payload
+ * page stably zero).
+ */
+ svc_util_pending_zero++;
+ if (bytes && !svc_inline_once_seen(push_va,
+ h | 1,
+ off_out,
+ bytes)) {
+ svc_inline_once_store(push_va, h | 1,
+ off_out, bytes);
+ svc_gr_ahead_inline_writes++;
+ if (inline_only)
+ svc_util_wit.fired = true;
+ (void)svc_grce_inline_write(vgpu,
+ off_out,
+ svc_ahead_inline_buf,
+ bytes,
+ ch->h_object);
+ }
+ } else if (bytes && !svc_inline_once_seen(push_va, h,
+ off_out,
+ bytes)) {
+ svc_inline_once_store(push_va, h, off_out,
+ bytes);
+ svc_gr_ahead_inline_writes++;
+ if (inline_only)
+ svc_util_wit.fired = true;
+ (void)svc_grce_inline_write(vgpu, off_out,
+ svc_ahead_inline_buf,
+ bytes,
+ ch->h_object);
+ }
+ /*
+ * Record the destination and the fresh payload bytes
+ * for the launch fill's inline-served pages, exactly
+ * like the main decode's LOAD_INLINE_DATA arm: the
+ * QMD capture later in this entry collects the ring
+ * and the fill can serve the parameter bank from it
+ * even while its shadow page is still empty.
+ */
+ {
+ struct svc_grce_rec *rec =
+ &svc_grce_dst_ring[svc_grce_dst_ring_next %
+ SVC_GRCE_DST_MAX];
+
+ /*
+ * A refused (still-zero) tail payload is not
+ * recorded either: the next pass re-reads the
+ * tail and the real bytes land then.
+ */
+ if (!refuse_zero) {
+ rec->dst_va = off_out;
+ rec->len = min_t(u32, bytes,
+ GSP_BRIDGE_INLINE_BYTES);
+ memcpy(rec->data, svc_ahead_inline_buf,
+ rec->len);
+ svc_grce_dst_ring_next++;
+ }
+ }
+ if (end > len)
+ break;
+ off += 4 + count * 4;
+ continue;
+ }
+
+ /*
+ * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c), for
+ * subchannel 1 AND 2. The mem2mem subchannel (2, the
+ * A140/KEPLER_INLINE_TO_MEMORY_B upload lane the utility
+ * channels drive) carries the same operand methods as the
+ * compute subchannel (1). This group MUST be parsed BEFORE
+ * the TURING-copy branch below: that branch's `subch >= 2`
+ * guard also matches subchannel 2 and would swallow the
+ * 0x180..0x18c methods unread, leaving have_out/line_len
+ * unset - the exact reason the utility channel's subch-2
+ * LOAD_INLINE_DATA arm never fired (r737/r739 witness:
+ * `lidn=1 subch=2 ... saw=0 dst=0 ll=0`).
+ */
+ if (xlate && (subch == 1 || subch == 2) &&
+ addr >= 0x180 && addr <= 0x18c && count) {
+ u32 k, n = count > 4 ? 4 : count;
+ u32 vv[4] = { w0, w1, w2, w3 };
+
+ for (k = 0; k < n; k++) {
+ switch (addr + 4 * k) {
+ case 0x180:
+ line_len = vv[k];
+ break;
+ case 0x184:
+ line_count = vv[k];
+ break;
+ case 0x188:
+ off_out = (off_out & 0xffffffffULL) |
+ ((u64)(vv[k] & 0x1ffff)
+ << 32);
+ have_out = 1;
+ break;
+ case 0x18c:
+ off_out = (off_out &
+ ~0xffffffffULL) | vv[k];
+ have_out = 1;
+ break;
+ default:
+ break;
+ }
+ }
+ } else if (xlate && subch >= 2) {
+ /* TURING_DMA_COPY_A operand runs (subch >= 2). */
+ u32 k, n = count > 4 ? 4 : count;
+ u32 vv[4] = { w0, w1, w2, w3 };
+
+ if (addr >= 0x400 && addr <= 0x41c && n) {
+ for (k = 0; k < n; k++) {
+ switch (addr + 4 * k) {
+ case 0x400:
+ off_in = (off_in &
+ 0xffffffffULL) |
+ ((u64)(vv[k] & 0x1ffff)
+ << 32);
+ have_in = 1;
+ break;
+ case 0x404:
+ off_in = (off_in &
+ ~0xffffffffULL) |
+ vv[k];
+ have_in = 1;
+ break;
+ case 0x408:
+ off_out = (off_out &
+ 0xffffffffULL) |
+ ((u64)(vv[k] & 0x1ffff)
+ << 32);
+ have_out = 1;
+ break;
+ case 0x40c:
+ off_out = (off_out &
+ ~0xffffffffULL) |
+ vv[k];
+ have_out = 1;
+ break;
+ case 0x410:
+ pitch_in = vv[k];
+ break;
+ case 0x414:
+ pitch_out = vv[k];
+ break;
+ case 0x418:
+ line_len = vv[k];
+ break;
+ case 0x41c:
+ line_count = vv[k];
+ break;
+ default:
+ break;
+ }
+ }
+ } else if (addr >= 0x700 && addr <= 0x708 && n) {
+ for (k = 0; k < n; k++) {
+ if (addr + 4 * k == 0x700)
+ last_remap = (last_remap &
+ ~0xffffu) |
+ (vv[k] & 0xffffu);
+ else if (addr + 4 * k == 0x708)
+ last_remap = (last_remap &
+ 0xffffu) |
+ ((vv[k] & 0xffffu)
+ << 16);
+ }
+ } else if (addr == 0x300 && count) {
+ u32 dma = w0;
+
+ /*
+ * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a
+ * release-only / semaphore op - no data
+ * movement, nothing to record).
+ */
+ last_dma = dma;
+ if ((dma & 0x3) && have_in && have_out &&
+ line_len && line_count &&
+ off_in && off_out) {
+ if (inline_only)
+ svc_ahead_copy_in(vgpu,
+ ch->h_object, push_va,
+ svc_push_hash(buf,
+ len),
+ off_in, off_out,
+ line_len * 4,
+ line_count, dma);
+ else if (forward) {
+ svc_gr_copy_pend_add(ch->h_object,
+ push_va, svc_push_hash(buf,
+ len),
+ off_in, off_out,
+ line_len * 4,
+ line_count,
+ pitch_out ? pitch_out :
+ line_len * 4,
+ pitch_in, dma,
+ last_remap);
+ cp_subch = subch;
+ }
+ }
+ }
+ }
+
+ if (!inline_only && addr == HOST_M_SEM_ADDR_LO && count >= 5) {
+ u64 va;
+
+ va = ((u64)(w1 & 0xff) << 32) |
+ (u64)(w0 & 0xfffffffc);
+ op = w4 & 0x7;
+ if ((op == HOST_SEM_OP_RELEASE ||
+ op == HOST_SEM_OP_REDUCTION) && w2 &&
+ va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_HI) {
+ /*
+ * A release targeting a live launch candidate's
+ * own QMD release word is owned by the
+ * completion edge (post-copy-out); sweeping it
+ * here would clear the client's fence before
+ * the engine output landed.
+ */
+ if (xlate && forward &&
+ svc_gr_ahead_owned_rel_va(va, push_va)) {
+ svc_gr_ahead_owned++;
+ } else {
+ svc_pending_add(va, w2,
+ op == HOST_SEM_OP_REDUCTION ?
+ 1 : 0);
+ nrel++;
+ }
+ }
+ }
+ if (end > len)
+ break;
+ off += 4 + count * 4;
+ }
+ (void)cp_subch;
+ return nrel;
+}
+
+static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch, u32 put)
+{
+ static u8 abuf[2048];
+ static u8 nbuf[2048];
+ u32 i, maxent, nrel = 0, swept = 0, nskip = 0;
+
+ if (!put || !ch->gpfifo_entries)
+ return;
+ maxent = min_t(u32, gsp_sema_gr_ahead_max, 512);
+ for (i = 0; i < maxent; i++) {
+ u32 idx = (put + i) % ch->gpfifo_entries;
+ u8 ent[SVC_GPFIFO_ENTRY_SIZE];
+ u32 e0, e1, len, rlen, tail_len = 0;
+ u64 va;
+
+ if (svc_read_va(vgpu,
+ ch->gpfifo_va +
+ (u64)idx * SVC_GPFIFO_ENTRY_SIZE,
+ ent, sizeof(ent)))
+ break;
+ memcpy(&e0, ent, 4);
+ memcpy(&e1, ent + 4, 4);
+ va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+ len = ((e1 >> 10) & 0x1fffff) << 2;
+ if (!va && !len)
+ break; /* unwritten slot: end of the continuation */
+ if (!va || !len || len > SVC_PUSH_MAX)
+ break;
+ if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI)
+ break;
+ rlen = min_t(u32, len, sizeof(abuf));
+ if (svc_read_va_push(vgpu, ch, va, abuf, rlen)) {
+ /*
+ * Unreadable here (pre-anchor translation) is a
+ * reason to stop this pass: the continuation is not
+ * proven coherent, so nothing is authored.
+ */
+ nskip++;
+ break;
+ }
+ /*
+ * SESSION 127 C5C0 translator: read the next ring entry as
+ * the tail of a stream whose final inline run overruns this
+ * entry (the driver splits large LOAD_INLINE_DATA payloads
+ * into a dedicated data slot entry; observed beef0100
+ * idx00->idx01 and idx08->idx09).
+ */
+ {
+ u32 nidx = (put + i + 1) % ch->gpfifo_entries;
+ u8 nent[SVC_GPFIFO_ENTRY_SIZE];
+ u32 ne0, ne1, nlen;
+ u64 nva;
+
+ if (!svc_read_va(vgpu,
+ ch->gpfifo_va +
+ (u64)nidx * SVC_GPFIFO_ENTRY_SIZE,
+ nent, sizeof(nent))) {
+ memcpy(&ne0, nent, 4);
+ memcpy(&ne1, nent + 4, 4);
+ nva = ((u64)(ne1 & 0xff) << 32) |
+ (u64)(ne0 & 0xfffffffc);
+ nlen = ((ne1 >> 10) & 0x1fffff) << 2;
+ if (nva && nlen < SVC_PUSH_MAX) {
+ tail_len = min_t(u32, nlen,
+ sizeof(nbuf));
+ if (svc_read_va_push(vgpu, ch, nva,
+ nbuf, tail_len))
+ tail_len = 0;
+ }
+ }
+ }
+ swept++;
+ nrel += svc_gr_ahead_push(vgpu, ch, abuf, rlen, nbuf,
+ tail_len, va, false);
+ }
+ svc_gr_ahead_entries += swept;
+ svc_gr_ahead_rels += nrel;
+ svc_gr_ahead_skips += nskip;
+ if (swept && (nrel || svc_gr_ahead_log < 64)) {
+ svc_gr_ahead_log++;
+ pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u owned=%llu skipped=%u (total rels=%llu)\n",
+ ch->h_object, put, swept, nrel, svc_gr_ahead_owned,
+ nskip, svc_gr_ahead_rels);
+ }
+}
+
+/*
+ * SESSION 129: read a VA straight through the registered-VAS walks only
+ * (newest registration first), one page at a time - the exact path
+ * walk_probe and sema_probe prove. The util sweep's channel read can be
+ * shadowed by a stale injected pin / window guess for the utility VAs
+ * (observed r737: the subchannel-2 LID arm never fired although the push
+ * page content is real and the VAS walk resolves both the methods entry
+ * and its payload slot); this helper is the arbitration fallback that
+ * bypasses those sources entirely.
+ */
+static int svc_util_walk_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+ size_t len)
+{
+ u8 *p = buf;
+
+ while (len) {
+ u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ size_t off = va & (SEMA_PAGE_SIZE - 1);
+ size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - off);
+ u64 pa = 0;
+ int ap = SVC_APERTURE_SYS;
+
+ if (svc_walk_vaspaces(vgpu, page, &pa, &ap))
+ return -ENOENT;
+ if (svc_read_ap(vgpu, pa + off, svc_ap_of(ap), p, chunk))
+ return -EIO;
+ p += chunk;
+ va += chunk;
+ len -= chunk;
+ }
+ return 0;
+}
+
+/*
+ * SESSION 129: does this entry window carry a LOAD_INLINE_DATA header? Used
+ * by the bounded util-sweep witness to select only the entries that can
+ * exercise the arm (one per distinct content, so a stable ring logs once).
+ */
+static u32 svc_util_lid_scan(const u8 *buf, u32 len, u32 *first_subch)
+{
+ u32 k, n = 0;
+
+ *first_subch = 0xff;
+ for (k = 0; k + 4 <= len; k += 4) {
+ u32 hdr;
+
+ memcpy(&hdr, buf + k, 4);
+ if (((hdr >> 29) & 0x7) == 3 &&
+ ((hdr & 0x1fff) << 2) == GRCE_M_LOAD_INLINE_DATA) {
+ if (*first_subch == 0xff)
+ *first_subch = (hdr >> 13) & 0x7;
+ n++;
+ }
+ }
+ return n;
+}
+
+/*
+ * SESSION 128: utility-channel inline-download materialization.
+ *
+ * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D
+ * composite: observed h=0xbeef0100 with gpfifo=0x4020000) are registered
+ * with an unresolved USERD (userd_ok=0), so the put-bounded entry loop in
+ * svc_pump_channels never decodes their pushes and the LOAD_INLINE_DATA
+ * downloads they carry (the client's code-heap/SASS upload, command-stream
+ * staging) never land anywhere the emulation can see: the destinations
+ * stay zero in the vidmem shadow.
+ *
+ * This pass scans a bounded window of the channel's GPFIFO directly (the
+ * gpfifo VA resolves through the trusted walk even when the USERD does
+ * not) and feeds each entry + its successor (the source-view call carrying
+ * the payload) through the proven sweep decoder in inline-only mode: the
+ * GRCE/INLINE_TO_MEMORY operand group (0x180..0x18c on subch 1/2),
+ * LAUNCH_DMA and the LOAD_INLINE_DATA run are materialized by
+ * svc_grce_inline_write; releases, QMD windows and copy-pend records are
+ * skipped so nothing about the client's fence/completion state changes.
+ * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring
+ * costs only the reads. Requires sema_gr_xlate; bounded by
+ * sema_gr_util_scan entries per channel per pass (0 = off).
+ *
+ * SESSION 129 read arbitration: attempt A is the channel path (unchanged);
+ * when A fails to read, or reads content the strict parse rejects, attempt
+ * B re-reads the push and the tail through svc_util_walk_read and the
+ * parse arbitrates - a source that yields a method stream wins. The r737
+ * evidence (arm never fired while the content is real) pointed at exactly
+ * this: a shadowed channel read consumed the entry without a parseable
+ * buffer while the walk resolves the true pages.
+ */
+static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch)
+{
+/*
+ * The observed utility-channel pushes are the ~17 KB setup block and the
+ * two split uploads (256 B command stream + 12 KB code/heap payload); a
+ * 64 KB window covers every one of them with room to spare while keeping
+ * the two static buffers modest.
+ */
+#define SVC_UTIL_BUF (64 * 1024)
+ static u8 ubuf[SVC_UTIL_BUF];
+ static u8 nbuf[SVC_UTIL_BUF];
+ static u8 bbuf[SVC_UTIL_BUF];
+ static u8 tbuf[SVC_UTIL_BUF];
+ static u64 svc_gr_util_entries;
+ static u64 svc_gr_util_writes;
+ static u64 svc_util_tail_alt; /* SESSION 130: tails re-read through
+ * the direct walk because the
+ * channel read was zero-headed */
+ static u64 svc_util_tail_try; /* ... and the walk attempts (alt is
+ * the subset that yielded non-zero) */
+ u32 i, maxent = min_t(u32, gsp_sema_gr_util_scan, 64);
+ u32 swept = 0, nmade = 0;
+ bool first_sight = false;
+
+ if (!gsp_sema_gr_xlate || !gsp_sema_gr_util_scan)
+ return;
+ if (!ch->gpfifo_va || !ch->gpfifo_entries) {
+ /* SESSION 143: name every channel the sweep cannot cover. */
+ u32 k;
+
+ for (k = 0; k < svc_util_seen_n; k++)
+ if (svc_util_seen_keys[k] == ch->h_object)
+ break;
+ if (k == svc_util_seen_n &&
+ svc_util_seen_n < ARRAY_SIZE(svc_util_seen_keys)) {
+ svc_util_seen_keys[svc_util_seen_n++] = ch->h_object;
+ pr_info("sema-utilsweep: skip h=0x%x eng=%u reason=no-gpfifo gpfifo=0x%llx entries=%u userd=%d\n",
+ ch->h_object, ch->engine_type, ch->gpfifo_va,
+ ch->gpfifo_entries, ch->userd_ok ? 1 : 0);
+ }
+ return;
+ }
+ {
+ u32 k;
+
+ for (k = 0; k < svc_util_seen_n; k++)
+ if (svc_util_seen_keys[k] == ch->h_object)
+ break;
+ if (k == svc_util_seen_n &&
+ svc_util_seen_n < ARRAY_SIZE(svc_util_seen_keys)) {
+ svc_util_seen_keys[svc_util_seen_n++] = ch->h_object;
+ first_sight = true;
+ pr_info("sema-utilsweep: begin h=0x%x eng=%u gpfifo=0x%llx entries=%u userd=%d cid=%u\n",
+ ch->h_object, ch->engine_type, ch->gpfifo_va,
+ ch->gpfifo_entries, ch->userd_ok ? 1 : 0,
+ ch->cid);
+ }
+ }
+ for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) {
+ u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE];
+ u32 e0, e1, len, rlen;
+ u64 va, before;
+
+ if (svc_read_va(vgpu,
+ ch->gpfifo_va +
+ (u64)i * SVC_GPFIFO_ENTRY_SIZE,
+ ent, sizeof(ent)))
+ break;
+ memcpy(&e0, ent, 4);
+ memcpy(&e1, ent + 4, 4);
+ va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+ len = ((e1 >> 10) & 0x1fffff) << 2;
+ if (!va && !len)
+ continue; /* unwritten slot */
+ if (!va || !len || len > SVC_PUSH_MAX)
+ continue;
+ if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI)
+ continue;
+ /*
+ * SESSION 132 stale-translation arbitration: when the
+ * channel-path entry is dead (its push provably reads
+ * all-zero through the client's page tables) while the
+ * direct VAS walk answers a live entry on the same ring
+ * slot, the channel path is reading a stale page (the r759
+ * pin false positive) - the walk's entry is the channel's
+ * ring. A dead entry with no live walk alternative is left
+ * to the existing push arbitration below.
+ */
+ if (svc_ring_entry_dead(vgpu, e0, e1)) {
+ u8 went[SVC_GPFIFO_ENTRY_SIZE];
+
+ if (!svc_util_walk_read(vgpu,
+ ch->gpfifo_va +
+ (u64)i * SVC_GPFIFO_ENTRY_SIZE,
+ went, sizeof(went))) {
+ u32 we0, we1;
+ u64 wva;
+ u32 wlen;
+
+ memcpy(&we0, went, 4);
+ memcpy(&we1, went + 4, 4);
+ wva = ((u64)(we1 & 0xff) << 32) |
+ (u64)(we0 & 0xfffffffc);
+ wlen = ((we1 >> 10) & 0x1fffff) << 2;
+ if (!svc_ring_entry_dead(vgpu, we0, we1) &&
+ wva && wlen && wlen <= SVC_PUSH_MAX &&
+ wva >= SVC_LOW_REL_VA_LO &&
+ wva < SEMA_VA_HI) {
+ e0 = we0;
+ e1 = we1;
+ va = wva;
+ len = wlen;
+ }
+ }
+ }
+ rlen = min_t(u32, len, sizeof(ubuf));
+ /*
+ * SESSION 129 read arbitration + bounded witness (see the
+ * sweep's header comment). Attempt A: the channel path,
+ * unchanged. Attempt B: the direct registered-VAS walk, used
+ * when A cannot read or cannot parse; the strict parse then
+ * arbitrates which content is the method stream.
+ */
+ {
+ u32 push_rc, parse_a = 0, parse_b = 0;
+ u32 tl_a = 0, tl_b = 0;
+ u32 lidn, lidsc = 0xff, slot;
+ u64 nva = 0;
+ u32 nlen = 0;
+ u64 hkey;
+ u8 *pbuf = ubuf;
+ bool ut = false, seen = false;
+
+ push_rc = svc_read_va_push(vgpu, ch, va, ubuf,
+ rlen);
+ /*
+ * The payload of a trailing LOAD_INLINE_DATA run rides
+ * the next GPFIFO entry (the driver's source-view
+ * GPFIFO call, observed beef0100 idx00->idx01 and
+ * idx08->idx09): read it as the tail exactly like
+ * svc_gr_ahead_sweep does.
+ */
+ if (svc_read_va(vgpu,
+ ch->gpfifo_va +
+ (u64)((i + 1) % ch->gpfifo_entries) *
+ SVC_GPFIFO_ENTRY_SIZE,
+ nent, sizeof(nent)) == 0) {
+ u32 ne0, ne1;
+
+ memcpy(&ne0, nent, 4);
+ memcpy(&ne1, nent + 4, 4);
+ nva = ((u64)(ne1 & 0xff) << 32) |
+ (u64)(ne0 & 0xfffffffc);
+ nlen = ((ne1 >> 10) & 0x1fffff) << 2;
+ }
+ if (!push_rc && nva && nlen && nlen < SVC_PUSH_MAX) {
+ tl_a = min_t(u32, nlen, sizeof(nbuf));
+ if (svc_read_va_push(vgpu, ch, nva, nbuf,
+ tl_a))
+ tl_a = 0;
+ }
+ if (!push_rc)
+ parse_a = svc_ahead_parse(ubuf, rlen, nbuf,
+ tl_a, &ut);
+ if (push_rc || !parse_a) {
+ bool bpush = false;
+
+ if (nva && nlen && nlen < SVC_PUSH_MAX) {
+ u32 want = min_t(u32, nlen,
+ sizeof(nbuf));
+
+ if (!svc_util_walk_read(vgpu, nva,
+ nbuf,
+ want))
+ tl_b = want;
+ }
+ if (!svc_util_walk_read(vgpu, va, bbuf,
+ rlen)) {
+ bpush = true;
+ parse_b = svc_ahead_parse(bbuf, rlen,
+ nbuf, tl_b,
+ &ut);
+ /*
+ * A's buffer is authoritative only
+ * when A actually read it; otherwise
+ * (or when only B parses) B's walk
+ * content is the stream.
+ */
+ if (push_rc ||
+ (parse_b && !parse_a)) {
+ pbuf = bbuf;
+ tl_a = tl_b;
+ }
+ }
+ if (push_rc && !bpush)
+ continue; /* unreadable both ways */
+ }
+ /*
+ * SESSION 130 tail-content arbitration: a tail-served
+ * payload whose channel head is still all-zero is not
+ * the staged bytes (the page the channel translation
+ * points at is empty, or the read is shadowed by a
+ * stale pin). Try the direct registered-VAS walk -
+ * the same path attempt B proves - before handing
+ * the tail to the LID arm, whose zero-headed refusal
+ * would otherwise stall the upload every pass.
+ */
+ if (tl_a && nva &&
+ !memchr_inv(nbuf, 0, min_t(u32, tl_a, 16))) {
+ u32 want = tl_a;
+
+ svc_util_tail_try++;
+ if (!svc_util_walk_read(vgpu, nva, tbuf,
+ want) &&
+ memchr_inv(tbuf, 0,
+ min_t(u32, want, 16))) {
+ memcpy(nbuf, tbuf, want);
+ svc_util_tail_alt++;
+ }
+ }
+ memset(&svc_util_wit, 0, sizeof(svc_util_wit));
+ if (tl_a >= 4)
+ memcpy(&svc_util_wit.t0, nbuf, 4);
+ if (tl_a >= 8)
+ memcpy(&svc_util_wit.t1, nbuf + 4, 4);
+ before = svc_gr_ahead_inline_writes;
+ (void)svc_gr_ahead_push(vgpu, ch, pbuf, rlen, nbuf,
+ tl_a, va, true);
+ if (svc_gr_ahead_inline_writes != before)
+ nmade++;
+ swept++;
+ /*
+ * Bounded witness, one line per distinct LID-carrying
+ * (or unparseable) entry content: which attempt was
+ * used, the parses, the tail resolution and what the
+ * LID arm did.
+ */
+ lidn = svc_util_lid_scan(pbuf, rlen, &lidsc);
+ if (!lidn && parse_a)
+ continue;
+ hkey = svc_push_hash(pbuf, rlen);
+ for (slot = 0; slot < 32; slot++)
+ if (svc_util_dec_keys[slot].h == hkey &&
+ svc_util_dec_keys[slot].va == va) {
+ seen = true;
+ break;
+ }
+ if (seen || svc_util_dec_log >= 96)
+ continue;
+ slot = svc_util_dec_log & 31;
+ svc_util_dec_keys[slot].va = va;
+ svc_util_dec_keys[slot].h = hkey;
+ svc_util_dec_log++;
+ pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x t8=%08x %08x alt=%llu/%llu lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n",
+ ch->h_object, i, va, rlen,
+ pbuf == bbuf ? 1u : 0u, parse_a, parse_b,
+ nva, tl_a, svc_util_wit.t0, svc_util_wit.t1,
+ svc_util_tail_alt, svc_util_tail_try,
+ lidn, lidsc,
+ svc_util_wit.saw_lid ? 1 : 0,
+ svc_util_wit.fired ? 1 : 0,
+ svc_util_wit.zero ? 1 : 0,
+ svc_util_wit.dst, svc_util_wit.line_len,
+ svc_util_wit.line_lc,
+ rlen >= 4 ? *(const u32 *)pbuf : 0,
+ rlen >= 8 ? *(const u32 *)(pbuf + 4) : 0);
+ }
+ }
+ svc_gr_util_entries += swept;
+ svc_gr_util_writes += nmade;
+ if (first_sight && !swept)
+ pr_info("sema-utilsweep: read=none h=0x%x gpfifo=0x%llx entries=%u\n",
+ ch->h_object, ch->gpfifo_va, ch->gpfifo_entries);
+ if (swept && (nmade || svc_gr_util_log < 128)) {
+ svc_gr_util_log++;
+ pr_info("sema-utilsweep: h=0x%x swept=%u writes=%u (total %llu/%llu)\n",
+ ch->h_object, swept, nmade, svc_gr_util_entries,
+ svc_gr_util_writes);
+ }
+}
+
static void svc_pump_channels(struct nv_vgpu_device *vgpu)
{
+ if (svc_pdb_disc_pending) {
+ svc_pdb_disc_pending = false;
+ pr_info("sema-pdb-disc: requested va=0x%llx pa=0x%llx\n",
+ svc_pdb_disc_va, svc_pdb_disc_pa);
+ svc_vas_candidates_verify(vgpu, svc_pdb_disc_va,
+ svc_pdb_disc_pa);
+ svc_xvas_pdb_discover(vgpu, svc_pdb_disc_va,
+ svc_pdb_disc_pa);
+ }
/* Only ever called from the service thread. */
static struct svc_channel local[SVC_MAX_CHANNELS];
/*
@@ -23523,6 +26796,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
bool stalled = false;
spin_lock_irqsave(&svc.chan_lock, flags);
+ if (svc_rewind_pending) {
+ svc_rewind_pending = false;
+ for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+ struct svc_channel *c = &svc.chan[i];
+
+ if (!c->valid || c->vgpu != vgpu)
+ continue;
+ if (!svc_rewind_all && c->h_object != svc_rewind_h)
+ continue;
+ pr_info("sema-chan: rewind h=0x%x last_get=%u put=%u (slot bitmap + rescue state cleared)\n",
+ c->h_object,
+ (unsigned int)c->last_get,
+ (unsigned int)c->put_prev);
+ bitmap_zero(c->slot_done_epoch, SVC_GPFIFO_MAX_BITS);
+ c->last_get = 0;
+ c->rescue_fail = 0;
+ c->unread_va = 0;
+ c->unread_until = 0;
+ c->defer_since = 0;
+ }
+ }
+ /*
+ * r707e: every-pass candidate verify with the last-good ring pair.
+ * Cheap (<=16 parked candidates, one walk each) and idempotent; the
+ * first pass after the user-VAS candidate parks registers it, so the
+ * client's very first push reads resolve.
+ */
+ if (svc_lastgood_ring_pa)
+ svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va,
+ svc_lastgood_ring_pa);
/*
* r507o stream-order: walk the CE channels BEFORE the GR channels.
* The client's stream order places its H2D copies before the kernel
@@ -23709,8 +27012,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc_fwd_hold.ch_handle == ch->h_object) &&
!svc_read_ap(vgpu,
ch->userd_pa + SVC_USERD_GP_PUT,
- ch->userd_as, &pput, sizeof(pput)))
+ ch->userd_as, &pput, sizeof(pput))) {
+ if (gsp_sema_trace_h &&
+ (u32)ch->h_object ==
+ (u32)gsp_sema_trace_h &&
+ time_after(jiffies, svc_perf_last)) {
+ svc_perf_last = jiffies + HZ;
+ pr_info("sema-perf: pass h=0x%x pput=%u last=%u armed=%d cnt=%u sub=%llu held=%llu starved=%llu\n",
+ ch->h_object, pput,
+ (u32)ch->last_get,
+ ch->gr_pub_armed,
+ ch->counter_last,
+ ch->submissions,
+ svc.gpfifo_held,
+ svc.gpfifo_starved);
+ }
svc_counter_publish(vgpu, ch, pput);
+ }
}
/*
@@ -23740,6 +27058,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
if (!rrc)
svc_pb_pair(vgpu, ch);
+ if (!rrc && ring_pa) {
+ svc_lastgood_ring_va = ch->gpfifo_va;
+ svc_lastgood_ring_pa = ring_pa;
+ }
+ /*
+ * r707f (FOLD): the moment ANY channel's ring resolves through
+ * the channel object, verify the parked COPY_SERVER_PDES VAS
+ * candidates against that content-backed pair. The candidates
+ * are parked at device-create time, before the client's compute
+ * channels first submit, so registering here lands the VAS page
+ * directory BEFORE the submit's entries get their first push
+ * read - the window the host-triggered path could not win (the
+ * entry's first read and the ring's first resolution happen in
+ * the same pass; a pair that arrives later cannot help, because
+ * a failed GR entry read is never retried with the fresh VAS).
+ */
+ if (!rrc)
+ svc_vas_candidates_verify(vgpu, ch->gpfifo_va,
+ ring_pa);
/*
* ISSUE-0002 (r47 diag): run the deferred CE USERD sweep for a
@@ -23760,6 +27097,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
jiffies + 27 * HZ;
}
+ /*
+ * SESSION 128: utility/upload channels carry the engine's
+ * inline code/data downloads (LOAD_INLINE_DATA on the
+ * mem2mem subchannel) and the copy-class channels (eng=11/12)
+ * carry the driver's pitch copy-ins (staging -> code heap /
+ * constant store). With an unresolved USERD the put-bounded
+ * entry loop below never decodes them (the !userd_ok branch
+ * continues before it); with a resolved USERD the main
+ * decode's LOAD_INLINE_DATA/operand arms still only accept
+ * subchannel 1, so the subchannel-2 downloads are missed
+ * either way. Scan every channel's GPFIFO directly here and
+ * materialize the downloads (see svc_gr_util_sweep); the
+ * inline-only mode plus the once-only key make the overlap
+ * with the main loop and the GR sweep a no-op.
+ */
+ if (gsp_sema_gr_util_scan && gsp_sema_gr_xlate)
+ svc_gr_util_sweep(vgpu, ch);
+
if (!ch->userd_ok) {
if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
rrc ? -1 : ring_ap, &put, &get,
@@ -23784,6 +27139,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
}
/*
+ * r832: the RC watchdog's notifier ack must not depend on the
+ * ring reads. Its VA is known from the channel's first decode
+ * (ch->notify2d_va, captured from SET_NOTIFY_A/B) and the ack
+ * is the same zeroed NvNotification the decode path writes:
+ * the guest re-arms status=0xFFFF every cycle and checks it
+ * back to 0 within 7 s. Under heavy GL traffic the watchdog
+ * ring's translation can go stale for stretches (its entries
+ * then read as zeros and are consumed as empty), which starved
+ * exactly this write and wedged the guest with "GPU is
+ * probably locked" while every other channel kept running.
+ * Writing the ack on every pass that sees work pending is
+ * idempotent.
+ */
+ if (!ch->engine_type && ch->notify2d_va && put != get)
+ svc_notify2d_write(vgpu, ch);
+
+ /*
* ISSUE-0002 diag: put snapshots for CE channels, bounded per
* session. The throttle is indexed by rank (the channel struct
* iterated here is a local copy, so a per-channel stamp never
@@ -23977,6 +27349,50 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
ch->put_prev = put;
/*
+ * SESSION 126 fence fold: sweep the ring continuation beyond
+ * GP_PUT for semaphore releases (see svc_gr_ahead_sweep).
+ * GR-class channels only, live-armed epochs only, and only
+ * with the ring translation proven through the channel
+ * object (rrc == 0).
+ */
+ if (!ch->engine_type && ch->gr_pub_armed &&
+ gsp_sema_gr_ahead_max && !rrc && ring_pa) {
+ svc_gr_ahead_sweep(vgpu, ch, put);
+ /*
+ * SESSION 127 C5C0 translator: offer a launch the
+ * sweep captured from the continuation (the Vulkan
+ * client's inline-QMD push lives beyond GP_PUT, so
+ * the main walk's per-entry request never sees it).
+ * svc_gr_request is a no-op without a matching
+ * unoffered candidate and keeps it on a busy
+ * verdict, so a bounded per-pass retry here is the
+ * sweep-captured launch's entire dispatch path.
+ */
+ if (gsp_sema_gr_xlate &&
+ gsp_phys_gr_forward_enabled()) {
+ if (svc_gr_cand.valid &&
+ svc_gr_cand.h_object == ch->h_object &&
+ svc_gr_ahead_req_log < 8) {
+ svc_gr_ahead_req_log++;
+ pr_info("sema-ahead: request h=0x%x push=0x%llx offered=%d\n",
+ ch->h_object,
+ svc_gr_cand.push_va,
+ svc_gr_cand.offered);
+ }
+ (void)svc_gr_request(vgpu, ch);
+ }
+ /*
+ * Fallback copy-out retirement: when no forward is
+ * owed (the launch was already consumed or the
+ * backend is off), run any pending GRCE copies so
+ * they cannot sit forever (the completion edge is
+ * the primary path).
+ */
+ if (gsp_sema_gr_xlate && !svc_gr_forward_pending())
+ svc_gr_copy_pend_run(vgpu);
+ }
+
+ /*
* STALLFREE r225: stale-epoch / stale-replay wedge watchdog.
* Runs before the entry loop so a wedged session epoch is
* recovered (and logged) instead of spinning: see
@@ -24018,6 +27434,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
get_before = ch->last_get;
applied_before = svc.releases_applied;
+ /* r835: this channel's VAS answers its VAs first (see
+ * svc_chanvas_lookup). Refreshed per channel turn; a channel
+ * without a known VAS clears it back to the old chain. */
+ svc_cur_vas = ch->h_vaspace;
/*
* C4 (GAP-1 H1): the GR walk witness - does put advance, and
@@ -24182,6 +27602,40 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
}
/*
+ * r829: a zero-length / zero-VA entry is a bad read
+ * (a stale or wrong translation), never work. The old
+ * path consumed it as an empty push and advanced
+ * GPGet; observed r830 on the RC watchdog channel:
+ * two bogus zero entries consumed through a
+ * mis-resolved ring page pushed the userd's GPGet past
+ * the driver's own GPPut, every later userd probe then
+ * read get>put and the channel's REAL userd was
+ * rejected - the channel stopped being serviced and
+ * the guest declared the GPU locked. Hold the entry
+ * instead (bounded; the per-channel counter persists
+ * through the pass snapshot); a corrected translation
+ * (the ring auto-pin lands within seconds) re-reads
+ * it. Past the cap the old consume-as-empty behavior
+ * resumes so an unresolvable ring cannot livelock the
+ * channel.
+ */
+ if (va == 0 || len == 0) {
+ if (ch->zero_held < 512 && !svc_hold_bypass()) {
+ ch->zero_held++;
+ svc.gpfifo_held++;
+ if (gsp_sema_dbg &&
+ svc_zero_entry_log < 16) {
+ svc_zero_entry_log++;
+ pr_info("sema-chan: zero-entry hold h=0x%x idx=%llu raw=%08x%08x (bad ring translation; not consumed)\n",
+ ch->h_object, idx,
+ e1, e0);
+ }
+ stalled = true;
+ continue;
+ }
+ }
+
+ /*
* Cheap wait for an entry already inside its
* expensive-wait window: read + rescue + hold were
* already paid for this VA, and a pin-batch rewind
@@ -24229,9 +27683,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
if (pb) {
int rescue_rc;
- bool unreadable =
- svc_read_va_push(vgpu, ch, va,
- pb, len) != 0;
+ int rd_rc;
+ bool unreadable;
+ bool zero_nop = false;
+ bool displaced = false;
+ bool have_ipin = false;
+
+ svc_read_last_src = 0xff;
+ svc_read_last_pa = 0;
+ rd_rc = svc_read_va_push(vgpu, ch, va,
+ pb, len);
+ unreadable = rd_rc != 0;
+ if (gsp_sema_dbg && svc_rdw_log < 6000 &&
+ !ch->engine_type) {
+ svc_rdw_log++;
+ pr_info("sema-rdw: h=0x%x va=0x%llx len=%u rc=%d src=%u\n",
+ ch->h_object, va, len,
+ rd_rc,
+ (unsigned)svc_read_last_src);
+ }
bool injected = false;
bool allow_ops = (ch->engine_type != 0);
@@ -24256,8 +27726,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
u64 ipa;
int iap;
- injected = (svc_ipin_lookup(va, &ipa,
- &iap) == 0);
+ have_ipin =
+ (svc_ipin_lookup(va, &ipa,
+ &iap) == 0);
+ /*
+ * r715: the entry's treatment
+ * follows the source that
+ * ACTUALLY answered the read,
+ * not a fresh page-granularity
+ * ipin lookup. The read falls
+ * through to the trusted walk
+ * (src 4) whenever an ipin's
+ * page fails its first-chunk
+ * guard, and taking the
+ * injected branch then skipped
+ * the only source that could
+ * answer: the entry was
+ * consumed without ever
+ * consulting the registered
+ * VAS walk (session 124).
+ */
+ injected = have_ipin &&
+ (svc_read_last_src == 0);
/*
* r496e: an all-zero push on a
* content-scanned-ring channel is the
@@ -24265,8 +27755,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
* dword - NVA06F_DMA_NOP), not a bad
* translation: accept it and let the
* decode find no ops.
+ * r820/r826: the same push arrives on
+ * KERNEL channels (nvidia-push's
+ * __nvPushTestPushBuffer writes the
+ * NOP into the kernel pushbuffer); its
+ * promote entry is a booking, so the
+ * trusted walk answers - a SMALL
+ * all-zero GR push resolved by the
+ * trusted chain (src=4) is the NOP.
+ * Real launches are large (a QMD push
+ * is hundreds of bytes), so the size
+ * bound keeps an unread legitimate
+ * push from being consumed as empty.
*/
- bool zero_nop = ch->gr_ram_ring &&
+ zero_nop = (ch->gr_ram_ring ||
+ (!ch->engine_type &&
+ svc_read_last_src == 4 &&
+ len <= 16)) &&
len >= 4 &&
!memchr_inv(pb, 0, len);
@@ -24299,7 +27804,82 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
kfree(fb);
}
} else {
- unreadable = true;
+ /*
+ * r707v: a primary-source
+ * hit (injected pin, window
+ * guess, lat cache) whose
+ * content is implausible
+ * must not shadow the
+ * registered VAS walk - a
+ * stale pin otherwise makes
+ * every push read "resolve"
+ * to the wrong page, fail
+ * plausibility and turn the
+ * entry unreadable forever.
+ * Retry once with the pins
+ * bypassed so the trusted
+ * walk (svc_walk_vaspaces)
+ * answers; accept only a
+ * plausible stream.
+ */
+ u8 *vb = kmalloc(len,
+ GFP_KERNEL);
+
+ if (vb) {
+ u64 wpa = 0;
+ int wap = SVC_APERTURE_SYS;
+ bool ok = false;
+
+ /*
+ * r707w: consult ONLY the
+ * registered VAS walk (the
+ * exact path walk_probe
+ * proves) - not
+ * svc_lookup_trusted, whose
+ * pin/lat hits would answer
+ * with the same wrong page
+ * again and keep the entry
+ * unreadable.
+ */
+ int wr = svc_walk_vaspaces(vgpu,
+ va & ~0xfffULL,
+ &wpa, &wap);
+ int rr = wr ? wr : svc_read_ap(vgpu,
+ wpa + (va & 0xfff),
+ wap == SVC_APERTURE_FB ?
+ SVC_APERTURE_FB :
+ SVC_APERTURE_SYS,
+ vb, len);
+ bool pl = !rr ? svc_method_stream_plausible(vb, len)
+ : false;
+
+ if (gsp_sema_dbg &&
+ svc_pushpg_fb_log < 64) {
+ svc_pushpg_fb_log++;
+ pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk rc=%d pa=0x%llx plaus=%d\n",
+ ch->h_object, va,
+ wr ? wr : rr,
+ wpa + (va & 0xfff),
+ pl ? 1 : 0);
+ }
+ if (!wr && !rr && pl) {
+ memcpy(pb, vb, len);
+ svc.pushpg_fallback++;
+ ok = true;
+ if (gsp_sema_dbg &&
+ svc_pushpg_fb_log < 64) {
+ svc_pushpg_fb_log++;
+ pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk ok pa=0x%llx\n",
+ ch->h_object, va,
+ wpa + (va & 0xfff));
+ }
+ }
+ kfree(vb);
+ if (!ok)
+ unreadable = true;
+ } else {
+ unreadable = true;
+ }
}
/*
* Still implausible (or
@@ -24315,18 +27895,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
u8 *db = kmalloc(len,
GFP_KERNEL);
- if (db &&
- svc_push_read_displaced(vgpu,
- ch->h_object, va,
- db, len)) {
- memcpy(pb, db, len);
- injected = false;
- unreadable = false;
- }
- kfree(db);
- }
- }
- }
+ if (db &&
+ svc_push_read_displaced(vgpu,
+ ch->h_object, va,
+ db, len)) {
+ memcpy(pb, db, len);
+ injected = false;
+ unreadable = false;
+ displaced = true;
+ }
+ kfree(db);
+ }
+ }
+ }
/*
* The entry's page was unreadable through
@@ -24345,11 +27926,40 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
memcpy(pb, db, len);
injected = false;
unreadable = false;
+ displaced = true;
}
kfree(db);
}
/*
+ * r715 pushdec witness: name the branch
+ * the entry takes straight after the
+ * read, before the rescue/hold paths.
+ * Session 124 had an entry consumed
+ * between the read (rc=0 src=4) and the
+ * decode with no row naming why; this
+ * line makes the branch and the content
+ * (first 8 bytes) visible on every read.
+ * Budget: 512 rows per client (reset in
+ * svc_session_flush).
+ */
+ if (gsp_sema_dbg && svc_pushdec_log < 2048) {
+ svc_pushdec_log++;
+ pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n",
+ ch->h_object, va, len,
+ (unsigned)svc_read_last_src,
+ svc_read_last_pa,
+ svc_method_stream_plausible(pb, len) ? 1 : 0,
+ injected ? 1 : 0,
+ have_ipin ? 1 : 0,
+ zero_nop ? 1 : 0,
+ displaced ? 1 : 0,
+ unreadable ? 1 : 0,
+ len >= 4 ? *(const u32 *)pb : 0,
+ len >= 8 ? *(const u32 *)(pb + 4) : 0);
+ }
+
+ /*
* Content rescue before any fake-release
* fallback (CE channels): the push bytes
* are in guest RAM even when the VA->PA
@@ -25195,7 +28805,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
if (!ch->engine_type && va && len &&
gsp_phys_gr_forward_enabled() &&
(svc_gr_qmd_seen > 0 ||
- all_zero) &&
+ (all_zero && !zero_nop)) &&
(!entry_replay ||
svc_gr_cand_push_owed(va)) &&
!(svc_gr_cand.valid &&
@@ -25503,6 +29113,31 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
idx, va, len);
svc_log_stalled_va(va);
/*
+ * GR stale-skip clock: the
+ * rescue-failure skip gate below
+ * requires defer_since, yet every
+ * GR arming site is gated on a
+ * nonzero expected_sem_payload. A
+ * zero-expectation GR entry (its
+ * release was already delivered)
+ * therefore never satisfied the
+ * gate and was re-read on every
+ * pass while the channel ring
+ * stalled behind it (live
+ * evidence: one 96-byte entry
+ * re-read 362 times over ~18 s;
+ * ~8 s of ring stall per frame
+ * with frames at 0.045 FPS). Arm
+ * the clock at first sighting; the
+ * skip window itself stays
+ * governed by gsp_sema_gr_grace_ms.
+ */
+ if (!ch->engine_type &&
+ !ch->expected_sem_payload &&
+ !ch->defer_since)
+ ch->defer_since = jiffies -
+ msecs_to_jiffies(gsp_sema_gr_grace_ms + 1);
+ /*
* Open the cheap-wait window
* for this entry (see the
* loop-top guard) so rewind
@@ -26072,6 +29707,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc.chan[k].gr_ram_logged = ch->gr_ram_logged;
svc.chan[k].gr_ram_ring = ch->gr_ram_ring;
svc.chan[k].userd_scans = ch->userd_scans;
+ svc.chan[k].notify2d_va = ch->notify2d_va;
+ svc.chan[k].zero_held = ch->zero_held;
+ svc.chan[k].userd_reg = ch->userd_reg;
svc.chan[k].logged = ch->logged;
svc.chan[k].logged_expected = ch->logged_expected;
svc.chan[k].notified = ch->notified;
@@ -26485,12 +30123,13 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu)
svc_read_ap(vgpu, ch->notifier_pa + 16 + 8,
ch->notifier_as, &info1, 4);
}
- pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n",
+ pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx armed=%d notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n",
ch->h_object, ch->engine_type, ch->gpfifo_va,
ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
ch->userd_ok, ch->last_get, ch->submissions,
ch->expected_sem_payload, ch->last_rel_va,
ch->counter_pa,
+ ch->gr_pub_armed,
ch->notifier_pa,
ch->notifier_as, info0, st0, info1, st1,
ch->notified);
@@ -26792,6 +30431,92 @@ int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap)
}
/*
+ * X/GL frame-pacer mirror (SESSION 148 perf, proven on r861): the X
+ * server's present path waits for a GPU-progress word - a lone u32
+ * (format 0x14 | base nibble at bits 16..19 | consumed-dword index in
+ * bits 0..17) in its own guest-RAM page - to reach its per-frame
+ * submitted-byte counter ((word & 0x3ffff) << 2 == counter). The
+ * emulation does not execute the 2D method streams, so nothing ever
+ * advances the word: every frame paid the driver's full ~10 s wait
+ * timeout (xorg.log "(EE) NVIDIA(GPU-0): WAIT (2, 8, 0x8000, 0x8d4,
+ * 0x<growing>)" every 10 s -> glxgears 0.09 FPS). A gdb poke of the
+ * word to (0x140000 | counter >> 2) made the wait complete in 3 ms
+ * ("WAIT (0, ... <equal>, <equal>)" appears ms after the timeout print)
+ * and silenced all further timeouts for the session.
+ *
+ * This kthread mirrors the driver's counter word (guest PA cnt_pa, a
+ * u32 in the X process's channel object that advances by the frame's
+ * submitted bytes) into the progress word (guest PA mem_pa) every few
+ * ms, preserving the word's high bits. Both PAs are discovered per boot
+ * (content/handle search in guest RAM) and injected with the
+ * sema_xw_mirror parameter; "0,0" stops the thread. Pure tuning aid for
+ * the 1.0.0 demo path - no product logic depends on it.
+ */
+static u64 svc_xw_cnt_pa;
+static u64 svc_xw_mem_pa;
+static struct task_struct *svc_xw_task;
+
+static int svc_xw_mirror_loop(void *unused)
+{
+ while (!kthread_should_stop()) {
+ struct nv_vgpu_device *d, *vgpu = NULL;
+ u32 c = 0, cur = 0;
+
+ mutex_lock(&nv_vgpu_driver.lock);
+ list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
+ if (atomic_read(&d->vfio_dev_open_count) > 0) {
+ vgpu = d;
+ break;
+ }
+ }
+ mutex_unlock(&nv_vgpu_driver.lock);
+ if (vgpu &&
+ !svc_read_ap(vgpu, svc_xw_cnt_pa, SVC_APERTURE_SYS,
+ &c, sizeof(c)) &&
+ !svc_read_ap(vgpu, svc_xw_mem_pa, SVC_APERTURE_SYS,
+ &cur, sizeof(cur))) {
+ u32 m = (cur & 0xfffc0000u) |
+ ((c >> 2) & 0x3ffffu);
+
+ if (m != cur)
+ svc_write_ap(vgpu, svc_xw_mem_pa,
+ SVC_APERTURE_SYS, &m, sizeof(m));
+ }
+ msleep(3);
+ }
+ return 0;
+}
+
+int gsp_sema_service_xw_mirror(u64 cnt_pa, u64 mem_pa)
+{
+ if (!cnt_pa || !mem_pa) {
+ if (svc_xw_task) {
+ kthread_stop(svc_xw_task);
+ svc_xw_task = NULL;
+ pr_info("sema-xw: mirror stopped\n");
+ }
+ return 0;
+ }
+ if ((cnt_pa | mem_pa) & 0x3)
+ return -EINVAL;
+ svc_xw_cnt_pa = cnt_pa;
+ svc_xw_mem_pa = mem_pa;
+ if (!svc_xw_task) {
+ svc_xw_task = kthread_run(svc_xw_mirror_loop, NULL,
+ "vgsp-xwmirror");
+ if (IS_ERR(svc_xw_task)) {
+ int err = PTR_ERR(svc_xw_task);
+
+ svc_xw_task = NULL;
+ return err;
+ }
+ }
+ pr_info("sema-xw: mirror cnt=0x%llx -> mem=0x%llx running\n",
+ (unsigned long long)cnt_pa, (unsigned long long)mem_pa);
+ return 0;
+}
+
+/*
* Dump a range of the emulated vidmem (BAR1 shadow) into the kernel log.
* Debug aid for the CE data path: verifies whether a decoded fill/copy
* actually landed where the CE operand said it should.
@@ -26992,19 +30717,43 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu)
u32 put = 0;
u64 delta;
- if (!s || !s->userd_ok || !s->pa_ok || !s->entries)
+ if (!s)
+ return;
+ if (!s->userd_ok || !s->pa_ok || !s->entries) {
+ /* m112b: silent-return diagnostic (bounded per boot). */
+ if (s->log_budget < 8) {
+ s->log_budget++;
+ pr_info("sema-svc: scrub pass idle vgpu=%px slot=%d userd_ok=%d pa_ok=%d entries=%u gen=%u\n",
+ (void *)vgpu, (int)(s - svc_scrubs),
+ s->userd_ok, s->pa_ok, s->entries,
+ s->session_gen);
+ }
return;
+ }
if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as,
&put, sizeof(put))) {
/* Retry through the alternate aperture (as registration did). */
if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT,
s->userd_as == SVC_APERTURE_FB ?
SVC_APERTURE_SYS : SVC_APERTURE_FB,
- &put, sizeof(put)))
+ &put, sizeof(put))) {
+ if (s->log_budget < 12) {
+ s->log_budget++;
+ pr_info("sema-svc: scrub pass read-fail userd=0x%llx as=%u\n",
+ s->userd_pa, s->userd_as);
+ }
return;
+ }
}
- if (put >= s->entries)
+ if (put >= s->entries) {
+ /* m112b: junk-translation diagnostic (bounded). */
+ if (s->log_budget < 12) {
+ s->log_budget++;
+ pr_info("sema-svc: scrub pass junk put=%u entries=%u userd=0x%llx\n",
+ put, s->entries, s->userd_pa);
+ }
return; /* junk translation; retry next pass */
+ }
if (!s->seeded) {
s->seeded = true;
@@ -27205,6 +30954,26 @@ static void svc_stall_watch_check(void)
svc_force_consume = 4;
}
+/*
+ * ISSUE-0002 client-session boundary aid: force a session flush that
+ * executes even while channels are live. The channel-close trigger only
+ * fires when the table empties, which never happens on a guest where the
+ * driver-global kernel channels (RC watchdog 0x31415900, the tiny notifier
+ * channel) stay valid; later client sessions then inherit the previous
+ * session's map/pin/pool discovery state and wedge (false translations ->
+ * failed rescues -> driver DMA-init timeout). Raised through the
+ * sema_force_flush module parameter so the boundary can be tested before
+ * wiring automatic detection.
+ */
+void gsp_sema_service_sess_flush_force(void)
+{
+ WRITE_ONCE(svc_sess_flush_forced, 1);
+ WRITE_ONCE(svc_sess_flush, 1);
+ if (READ_ONCE(svc.task))
+ wake_up_process(svc.task);
+ pr_info("sema-svc: forced session flush requested\n");
+}
+
static int svc_thread(void *arg)
{
struct nv_vgpu_device *vgpus[SVC_MAX_GUESTS];
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index fbc7e4d..e96ce1c 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -31,6 +31,7 @@ struct sema_channel_desc {
u32 h_object;
u32 engine_type;
u32 cid; /* RM channel id (the doorbell token vector) */
+ u32 h_vaspace; /* r835: VAS handle the channel was allocated in */
u64 gpfifo_va;
u32 gpfifo_entries;
u64 userd_pa; /* NvA06FControl: GPGet @0x88, GPPut @0x8c */
@@ -89,6 +90,23 @@ struct sema_vaspace_desc {
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc);
void gsp_sema_service_vaspace_del(u32 h_vaspace);
+
+/*
+ * r820: a driver-declared kernel-buffer translation from an
+ * NV2080_CTRL_CMD_GPU_PROMOTE_CTX entry (gpuVirtAddr/gpuPhysAddr/size/
+ * physAttr of one context buffer). These buffers - the nvidia-push
+ * pushbuffer, the progress-tracker segment, the FECS/patch/access-map
+ * blocks - are kernel-owned, so the userspace pin scanner cannot cover
+ * them and the page-table walk can answer with a stale/wrong page
+ * (r817 Xorg GL init: the walk resolved the 64 KB pushbuffer VA to a
+ * zero page while the promote entry named the real one). The sema
+ * service consults these records before the learned cache and the walks
+ * in svc_lookup_trusted; newest statement per VA wins. VA-only bookings
+ * (zero pa/size) and bNonmapped entries (va=0) are ignored. Records are
+ * session state, cleared by svc_session_flush.
+ */
+void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr,
+ u32 buffer_id);
/*
* r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a
* content-validated walk proves it belongs to a live client; only then is
@@ -102,6 +120,12 @@ void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
* session flushes keep that state - see svc_session_flush. */
void gsp_sema_service_guest_reset(void);
+/*
+ * ISSUE-0002 aid: force a client-session flush that executes even while
+ * channels are live (see addendum 8 and the sema_force_flush parameter).
+ */
+void gsp_sema_service_sess_flush_force(void);
+
int gsp_sema_service_start(void);
void gsp_sema_service_stop(void);
int gsp_sema_service_probe(u64 gpu_va, u32 count);
@@ -109,7 +133,10 @@ int gsp_sema_service_walk_probe(u64 gpu_va);
int gsp_sema_service_dump_channels(void);
int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap);
int gsp_sema_service_dump_vidmem(u64 addr, u64 len);
+int gsp_sema_service_xw_mirror(u64 cnt_pa, u64 mem_pa);
int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits);
+int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa);
+int gsp_sema_service_chan_rewind(u32 h_object);
int gsp_sema_service_counter_add(u32 h_object, u64 pa);
void gsp_sema_service_doorbell(u32 token);
diff --git a/kernel/main.c b/kernel/main.c
index dd11d12..0e9452b 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2554,6 +2554,25 @@ module_param_cb(sema_probe, &nv_vgpu_sema_probe_ops, NULL, 0200);
MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA");
/*
+ * ISSUE-0002 aid: force a client-session flush past the live-channel skip.
+ * Write anything (echo 1 > .../parameters/sema_force_flush).
+ */
+static int nv_vgpu_sema_force_flush_set(const char *val,
+ const struct kernel_param *kp)
+{
+ gsp_sema_service_sess_flush_force();
+ return 0;
+}
+
+static const struct kernel_param_ops nv_vgpu_sema_force_flush_ops = {
+ .set = nv_vgpu_sema_force_flush_set,
+};
+
+module_param_cb(sema_force_flush, &nv_vgpu_sema_force_flush_ops, NULL, 0200);
+MODULE_PARM_DESC(sema_force_flush,
+ "Force a client-session flush even with live channels (ISSUE-0002 aid)");
+
+/*
* Register the guest PMA scrubber channel buffer words (finish payload PA,
* host-semaphore PA) the guest scanner resolved:
* echo 0x11a8ff004,0x11a8ff000 > .../parameters/sema_scrubbuf_add
@@ -2960,6 +2979,61 @@ module_param_cb(sema_map_add, &nv_vgpu_sema_map_add_ops, NULL, 0200);
MODULE_PARM_DESC(sema_map_add, "Inject a GPU VA -> guest PA mapping (hex va,hex pa)");
/*
+ * Debug: retrigger the client-VAS page-directory discovery with a host
+ * content-verified {ring VA -> ring PA} pair (the ring page's GPFIFO
+ * entries are the fingerprint):
+ * echo 0x<va>,0x<pa> > .../parameters/sema_pdb_discover
+ */
+static int nv_vgpu_sema_pdb_discover_set(const char *val,
+ const struct kernel_param *kp)
+{
+ char buf[64];
+ char *comma;
+ u64 va = 0, pa = 0;
+
+ strscpy(buf, val, sizeof(buf));
+ comma = strchr(buf, ',');
+ if (!comma)
+ return -EINVAL;
+ *comma = '\0';
+ if (kstrtoull(buf, 0, &va) || kstrtoull(comma + 1, 0, &pa))
+ return -EINVAL;
+
+ return gsp_sema_service_pdb_discover(va, pa);
+}
+
+static const struct kernel_param_ops nv_vgpu_sema_pdb_discover_ops = {
+ .set = nv_vgpu_sema_pdb_discover_set,
+};
+
+module_param_cb(sema_pdb_discover, &nv_vgpu_sema_pdb_discover_ops, NULL, 0200);
+MODULE_PARM_DESC(sema_pdb_discover, "Retrigger the VAS pdb discovery with a verified ring pair (hex va,hex pa)");
+
+/*
+ * Debug: bounded channel rewind. Clears the consumed-slot bitmap and the
+ * per-entry rescue state for one channel (hex h_object) or every channel
+ * ("all"), so the pump re-processes its ring from slot 0 - used after the
+ * VAS pdb anchor lands so entries skipped before it re-decode (the release
+ * they carry would otherwise be lost for that put epoch).
+ */
+static int nv_vgpu_sema_chan_rewind_set(const char *val,
+ const struct kernel_param *kp)
+{
+ u32 h = 0xffffffffu;
+
+ if (strncmp(val, "all", 3) && kstrtouint(val, 0, &h))
+ return -EINVAL;
+ return gsp_sema_service_chan_rewind(h);
+}
+
+static const struct kernel_param_ops nv_vgpu_sema_chan_rewind_ops = {
+ .set = nv_vgpu_sema_chan_rewind_set,
+};
+
+module_param_cb(sema_chan_rewind, &nv_vgpu_sema_chan_rewind_ops, NULL, 0200);
+MODULE_PARM_DESC(sema_chan_rewind, "Rewind a channel's ring (hex h_object or 'all')");
+
+/*
* Debug: register a channel's GPU completion counter (guest PA of the u32
* tracking counter the GPU would write when a push completes):
* echo <h_object>,<pa> > .../parameters/sema_counter
@@ -3023,6 +3097,38 @@ module_param_cb(sema_vidmem_dump, &nv_vgpu_sema_vidmem_dump_ops, NULL, 0200);
MODULE_PARM_DESC(sema_vidmem_dump, "Dump emulated vidmem (addr,len) to the kernel log");
/*
+ * X/GL frame-pacer mirror (SESSION 148 perf): mirror the X server's
+ * submitted-byte counter (guest PA) into its GPU-progress word (guest PA)
+ * so the driver's per-frame 10 s wait completes in milliseconds.
+ * echo 0x112345678,0x112345000 > .../parameters/sema_xw_mirror
+ * echo 0,0 > .../parameters/sema_xw_mirror (stop)
+ */
+static int nv_vgpu_sema_xw_mirror_set(const char *val,
+ const struct kernel_param *kp)
+{
+ char buf[64];
+ char *comma;
+ u64 cnt = 0, mem = 0;
+
+ strscpy(buf, val, sizeof(buf));
+ comma = strchr(buf, ',');
+ if (!comma)
+ return -EINVAL;
+ *comma = '\0';
+ if (kstrtoull(buf, 0, &cnt) || kstrtoull(comma + 1, 0, &mem))
+ return -EINVAL;
+
+ return gsp_sema_service_xw_mirror(cnt, mem);
+}
+
+static const struct kernel_param_ops nv_vgpu_sema_xw_mirror_ops = {
+ .set = nv_vgpu_sema_xw_mirror_set,
+};
+
+module_param_cb(sema_xw_mirror, &nv_vgpu_sema_xw_mirror_ops, NULL, 0200);
+MODULE_PARM_DESC(sema_xw_mirror, "Mirror X frame counter (cnt_pa,mem_pa; 0,0=stop)");
+
+/*
* Debug: search the whole emulated vidmem for an 8-byte pattern:
* echo 0x0000000400000001,16 > .../parameters/sema_vidmem_find
*/
@@ -3427,6 +3533,47 @@ static int nv_vgpu_dump_vidmem_set(const char *val, const struct kernel_param *k
mutex_unlock(&nv_vgpu_driver.lock);
return 0;
}
+ if (!strncmp(buf, "map", 3)) {
+ /*
+ * Coarse non-zero map: one line per 1 MB block that holds
+ * any non-zero dword (SESSION 131: locate the live client's
+ * uploaded content - the image BO, the code heap, the rings -
+ * inside the 256 MB aperture without 65536 range dumps).
+ */
+ mutex_lock(&nv_vgpu_driver.lock);
+ list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
+ u8 *base = vgpu->bars[1].vaddr;
+ size_t size = vgpu->bars[1].size;
+ size_t blk, nblk = (size + (1 << 20) - 1) >> 20;
+ size_t o;
+
+ if (!base || !size)
+ continue;
+ for (blk = 0; blk < nblk; blk++) {
+ size_t start = blk << 20;
+ size_t end = start + (1 << 20);
+ u32 nz = 0, first = 0;
+
+ if (end > size)
+ end = size;
+ for (o = start; o + 4 <= end; o += 4) {
+ u32 v;
+
+ memcpy(&v, base + o, 4);
+ if (v) {
+ if (!nz)
+ first = (u32)(o & 0xFFFFF);
+ nz++;
+ }
+ }
+ if (nz)
+ pr_info("vgsp: vidmem map blk 0x%zx nz=%u first=0x%x\n",
+ start, nz, first);
+ }
+ }
+ mutex_unlock(&nv_vgpu_driver.lock);
+ return 0;
+ }
comma = strchr(buf, ',');
if (comma)
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-27 20:15:26 +0000