diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 13 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 2130 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 1 | ||||
| -rw-r--r-- | kernel/main.c | 24 |
4 files changed, 2087 insertions, 81 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 534a443..85bdb89 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19665,7 +19665,18 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); pr_info("vgsp-gr: page[%u] va=0x%llx hpa=0x%llx qmd_word=%u\n", i, pg->guest_va, pg->hpa, pg->qmd_word); - if (pg->hpa) { + /* + * SESSION 149 R15 (M2-S1): with the window-0 shadow path + * enabled, force the program page (QMD word 48) OFF the + * post-commit Tier-1 install. r801b proved that install is + * ineffective for a late GR page (the engine kept reading + * the empty guest page: CODEHASH h0=0, FAULT-CAP + * code_src=guest-hpa); skipping it lets the page fall to + * the win0 shadow copy + w48/49 rewrite line, which is the + * proven one. Every other page keeps its Tier-1 route. + */ + if (pg->hpa && !(pg->qmd_word == 48u && + gsp_phys_rmcall_gr_shadow_win0_value())) { ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, pg->hpa, epoch); if (!ret) { diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 831f920..3581ce9 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -187,8 +187,54 @@ struct sema_map_entry { * slice per service pass so the service thread stays responsive. */ #define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ -#define SVC_GR_RAM_PASS_MAX 16 /* slices before giving up */ +/* + * SESSION 149 R15 (M1.1b): 24 slices cover two full sweeps of the 6 GB + * guest before the boot-time budget retires; the put-triggered keep-alive + * in the pump grants up to SVC_GR_RAM_REFRESH_MAX further budgets. + */ +#define SVC_GR_RAM_PASS_MAX 24 /* slices before a budget retires */ +#define SVC_GR_RAM_REFRESH_MAX 3 /* keep-alive budget refreshes */ +#define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ #define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ +/* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ +#define SVC_GR_USERD_WATCH 8 + +/* + * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM + * can extend above SEMA_PA_MAX: the 8 GiB guest's e820 top is + * 0x280000000, and the GL client channel's GPFIFO ring page was measured + * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there) - + * above the old 6 GB cap, so the content scan could never reach it. Only + * the ring scan uses this wider top; every other table/bound in the + * service stays sized on SEMA_PA_MAX. Reads past mapped RAM (the + * 0x80000000-0x100000000 PCI hole, or a smaller guest) fail fast, so a + * too-large top costs a few failed probes per sweep. The module parameter + * is the per-cycle guard: shrink it to the boot's real RAM top. + */ +unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL; +module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644); +#define SVC_GR_SCAN_TOP_MAX 0x400000000ULL /* 16 GiB sanity clamp */ + +/* + * Ring-scan slice budget: two full sweeps at the configured top, with the + * original 24-slice floor and a u8-safe cap (gr_ram_passes). + */ +static u32 svc_gr_ram_pass_budget(void) +{ + u64 top = gsp_sema_gr_scan_top; + u32 slices; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top <= SVC_GR_RAM_SLICE) + return SVC_GR_RAM_PASS_MAX; + slices = (u32)(top / SVC_GR_RAM_SLICE) * 2; + if (slices < SVC_GR_RAM_PASS_MAX) + slices = SVC_GR_RAM_PASS_MAX; + if (slices > 250) + slices = 250; + return slices; +} struct svc_channel { bool valid; @@ -499,6 +545,68 @@ struct svc_channel { u8 gr_ram_passes; /* slices walked so far */ bool gr_ram_logged; /* one discovery line per channel */ /* + * SESSION 149 R15 (M1.1b keep-alive): the boot-time slice budget can + * exhaust long before the client creates its ring objects (r886: 16 + * slices spent, the GL client's ring never scanned, no pairing). A + * GPFIFO put advance proves the client is live, so grant a fresh + * bounded budget - spaced and capped so a never-found ring cannot + * keep the scan hot. + */ + u32 gr_ram_kick; /* put value at the last budget refresh */ + u8 gr_ram_refreshes; /* budget refreshes granted so far */ + unsigned long gr_ram_refresh_at; /* next eligible refresh */ + /* + * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose + * ring was content-validated in RAM (gr_ram_ring). Such a client's + * USERD is a system-memory object like its ring (the kernel-RM/X GL + * path), so the FB-shadow shape scan can never reach it and the pump + * never iterates its entries. Same sliced/cursor discipline as the + * ring scan; the cursor is persisted immediately because the + * stalled-userd pump path skips the end-of-pass write-back. + */ + u64 gr_ram_userd_cursor; /* resume point of the USERD scan */ + u8 gr_ram_userd_passes; /* slices walked so far */ + bool gr_ram_userd_logged; /* one discovery/give-up line */ + /* + * SESSION 154 (M-GL1.2): liveness confirmation for the RAM USERD + * latch. The shape+entry gates alone admitted frozen or slowly + * churning lookalike pages (r913: pa 0x7c000 adopted at put=1 only + * survived by luck; r914: pa 0x9e200 adopted at put=12 get=8 and + * its put%2==0 collapsed the walk - submissions stayed 0 while the + * client retried for minutes). A candidate is WATCHED across passes + * instead: only a block whose PUT takes the client's own single +1 + * step is adopted as the live control block - the measured live + * block advances 1->2->3 while the lookalikes jump by larger + * amounts or churn to zero. + */ + u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ + u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ + u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + u8 gr_ram_userd_watch_n; + u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The + * RAM USERD scan latches a gated candidate immediately; the pump + * confirms it from its own per-pass PUT read (a +1/+2 step), and + * unlatches a churning/frozen block (see svc_pump_channels). + */ + u32 gr_ram_userd_latch_put; /* PUT at the speculative latch */ + unsigned long gr_ram_userd_latch_at; /* jiffies at the latch */ + bool gr_ram_userd_confirmed; /* +1/+2 step observed */ + /* + * SESSION 154 (M-GL1.2): content anchor for a client push whose + * translation went stale (r913: a validated pin kept answering for + * a reused page and the wrong-VAS walk answer was implausible while + * the real push bytes sat in RAM on a page no directory covered). + * Scan guest RAM for the client's release-group signature (sliced, + * cursor-resumed, like the ring scans), validate the page and pin + * it so the pump's retry decodes the real bytes. + */ + u64 gr_anchor_cursor; /* resume point of the anchor scan */ + u64 gr_anchor_va; /* entry VA the sweep is for */ + u8 gr_anchor_sweeps; /* completed sweeps for that VA */ + bool gr_anchor_logged; /* one start line per channel */ + /* * r496e: true when the ring was found by the RAM content scan (a * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's * pushes are 2D/3D method streams the emulation does not execute; @@ -645,6 +753,12 @@ static void svc_mark_hot(u64 pa) * push-buffer base) and the channel (its slot). */ #define SVC_PB_MAX 32 +/* + * R10 (patch A): svc_pb[] slot namespace for GR (eng==0) channels' + * content-anchored ring windows (svc_pb_pair_gr). CE slots are + * 4*ce_index (0..124), so the high bit keeps the namespaces apart. + */ +#define SVC_PB_GR_SLOT_BASE 0x80000000u struct svc_pb_hit { u64 pa_page; u64 va; /* push-buffer base VA, once paired with a channel */ @@ -826,6 +940,8 @@ static u64 svc_pb_last; static u32 svc_pb_idle; /* consecutive passes with no new base */ static u32 svc_pb_last_n; /* bases before the active pass */ static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ +/* R10 (patch A): GR ranks whose ring-anchored window is paired. */ +static unsigned long svc_pb_gr_done; static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ /* SESSION 133: ranks that already consumed their one shadow ring scan. */ static unsigned long svc_gr_ring_scan_done; @@ -1446,6 +1562,17 @@ static void svc_chan_rearm_rank(struct svc_channel *ch) clear_bit(ch->ce_index, &svc_pb_done); } else { clear_bit(ch->gr_index, &svc_gr_ring_done); + /* + * r914: re-arm the ring-anchored window pairing too. The GL + * client re-registers with a NEW ring PA on every attempt, + * and with only the ring bit cleared the stale first pair + * (svc_pb_win_lookup returns the first slot match) kept + * answering the push reads with the previous attempt's + * pages - which the guest had since reused for kernel + * structures (measured r913: the push read resolved to a + * page of kernel pointers and every entry went unreadable). + */ + clear_bit(ch->gr_index, &svc_pb_gr_done); } } @@ -3236,6 +3363,36 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) } /* + * SESSION 154 (M-GL1.2): drop a validated pin whose content no longer + * parses as the page's kind of stream (a stale translation kept answering + * for a page the guest has since reused - r913: va 0x1203c0000 read + * `first8=00000001` through a validated pin while the real push bytes sat + * elsewhere). The next read then consults the window guess, the trusted + * walk and the content sources instead of the dead pin. + */ +static void svc_pin_drop(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_PIN_PROBE; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page) + return; + if (p->va_page == page) + break; + } + if (probe == SVC_PIN_PROBE) + return; + memset(&svc.pin[(h + probe) & mask], 0, sizeof(struct svc_pin)); + if (svc.pin_n) + svc.pin_n--; +} + +/* * Injected (host-tool) translations. Kept separate from the discovered pin * table so a hand-verified VA->PA pair can never be shadowed, evicted or * silently dropped by the self-discovery stream: in the shared table a hot @@ -6609,6 +6766,20 @@ static_assert(sizeof(struct svc_gr_page) == 24, "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes"); static u64 svc_push_hash(const u8 *p, u32 len); +/* SESSION 149 R15: exactly-once guard for the hash-verified replay capture. */ +static bool svc_gr_cand_push_seen(u64 push_va, u64 hash); +/* SESSION 149 R16 (M3): trusted-walk VA read, defined below the decode. */ +static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); +/* + * SESSION 149 R17 (M4): the section's memory-writing upload commands are + * applied through the shared inline-write path (defined below the section + * code) with its (push, hash, dst, bytes) once-only guard. + */ +static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, + const u8 *data, u32 bytes, u32 ch_h); +static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes); +static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes); /* QMD field decode: the exact shape the host CUDA bridge uses. */ static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c) @@ -6748,6 +6919,505 @@ static void svc_gr_cand_capture(struct svc_channel *ch, } /* + * SESSION 149 R14 (sema_gl_xlate, default 0 = off): GL/3D direct method + * stream -> synthesized GR candidate recognizer. + * + * The GL channel's large per-frame pushes bind the TURING_A 3D class + * (0xC597) on the channel subchannel and carry no CUDA QMD, so the + * inline-QMD capture above never fires for them (the victim-section + * classification of the r882 stream, push VA 0x120010004 / 1524 B: + * SET_OBJECT C597, SET_TEX_HEADER_POOL_A -> 0x1214fa220/0x10, 20x + * LINE_LENGTH_IN(0x180)+LAUNCH_DMA(0x1b0) inline uploads to + * 0x1214f8f00+0x100k, SET_PIPELINE_PROGRAM_ADDRESS_A/B(0x2014/0x2018 + * + j*64) -> 0x121201b00, BIND_GROUP_CONSTANT_BUFFER(0)). This section + * collects the OBSERVED VA operands (nothing is invented) and synthesizes + * the 64-word QMD those fields would carry; the candidate then flows + * through the unchanged svc_gr_request / svc_fwd_gr_work / + * gsp_phys_backend_guest_gr_submit machinery. + * + * Strictly additive: gated by sema_gl_xlate and inert for every stream + * without the C597 class bind (0xC5B5 copy-class and 902D 2D binds never + * open a section). Class numbers from the public class headers + * (clc597.h: TURING_A 0xC597). + */ +static bool gsp_sema_gl_xlate; +module_param_named(sema_gl_xlate, gsp_sema_gl_xlate, bool, 0644); +MODULE_PARM_DESC(sema_gl_xlate, + "GL/3D (C597) direct stream -> synthesized GR candidate (default 0 = off)"); + +/* + * SESSION 149 R17 (M4): one recorded section upload command. The 3D-class + * i2m shape is a 0x180 (LINE_LENGTH_IN) run naming the destination followed + * by a 0x1b0 (LAUNCH_DMA + LOAD_INLINE_DATA) run carrying the payload. + */ +#define SVC_GL_UPLOAD_MAX 24 +struct svc_gl_up { + u64 dst_va; /* from the 0x180 run's OFFSET_OUT_UPPER|OFFSET_OUT */ + u32 len; /* payload bytes copied (payload = run data minus cfg) */ + u8 data[SVC_GR_INLINE_BYTES]; +}; + +struct svc_gl_sec { + bool seen; + u32 subch; + u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ + u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ + u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ + u64 rel_va; /* last VALID release decoded on the channel */ + u32 rel_payload; + u64 ptr[SVC_GR_POINTER_MAX]; + u32 n_ptr; + /* + * SESSION 149 R17 (M4): the section's memory-writing upload commands + * collected in order. On real hardware the 3D engine executes these + * i2m copies (constants, tex pools, ...); in this emulation no engine + * ran them, so every destination stayed zero. svc_gl_flush applies + * the recorded bytes through svc_grce_inline_write (FB aperture -> + * vidmem shadow, sysmem -> guest RAM, unresolved -> VA shadow). + */ + u64 up_dst; /* destination of the last 0x180 run */ + u32 up_bytes; /* its length word (stream); witness only */ + u32 n_up; + struct svc_gl_up up[SVC_GL_UPLOAD_MAX]; + /* + * SESSION 149 R15 (M2-S1): bytes of the first LAUNCH_DMA / + * LOAD_INLINE_DATA run observed inside the section (the observed + * upload payload). Attached to the synthesized candidate as an + * inline record at the program VA so the backend's window-0 shadow + * copy has a non-empty source even when the guest program page + * resolves empty (the GL program page is not among the section's + * own upload destinations). + */ + u8 blob[SVC_GR_INLINE_BYTES]; + u32 blob_len; +}; +static struct svc_gl_sec svc_gl; /* service thread only */ +static u32 svc_gl_rep_log; /* bounded replay-already-captured witness */ +static u32 svc_gl_prog_log; /* R16 bounded program-read witness */ +static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ +/* + * SESSION 149 R17 (M4): explicit program bytes for the synthesized GL + * launch (hex string, <= GSP_BRIDGE_INLINE_BYTES). The GL program page + * (e.g. 0x121201b00) is empty in every buffer this emulation can see: no + * memory-writing command in any readable push targets it and the + * context-init upload never reached a decodable ring. When set, these + * bytes are attached as the program record (after a trusted VA read, so a + * real upload still wins). Development use only: a minimal real sm_75 + * SASS program (EXIT;NOP;...) proves the engine executes OUR bytes. + */ +static char *sema_gl_prog_hex; +module_param_named(sema_gl_prog_hex, sema_gl_prog_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_prog_hex, + "explicit program bytes (hex) for the synthesized GL launch (<=64 B)"); +static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_prog_len; +static bool svc_gl_prog_parsed; + +static void svc_gl_prog_apply_param(void) +{ + const char *h = sema_gl_prog_hex; + u32 n, i; + + if (svc_gl_prog_parsed) + return; + svc_gl_prog_parsed = true; + if (!h || !h[0]) + return; + n = (u32)strlen(h); + if (n % 2 || n / 2 > GSP_BRIDGE_INLINE_BYTES) { + pr_warn("sema-gl: prog-hex param ignored (bad length %u)\n", + n); + return; + } + for (i = 0; i < n; i += 2) { + unsigned int b; + + if (sscanf(h + i, "%2x", &b) != 1) { + pr_warn("sema-gl: prog-hex param bad at %u\n", i); + return; + } + svc_gl_prog[i / 2] = (u8)b; + } + svc_gl_prog_len = n / 2; + pr_info("sema-gl: prog-hex param: %u bytes stored\n", + svc_gl_prog_len); +} +/* + * SESSION 149 R17 (M4): standalone GL-class upload tracker. A context-init + * upload push (the one-shot program upload) is NOT required to sit inside a + * C597 section, but it carries the same subchannel-0 0x180/0x1b0 i2m shape. + * The tracker pairs the last 0x180 destination with each following 0x1b0 + * payload and applies it on sight (sema_gl_xlate, eng==0, trusted/allow, + * once-only) so the destination (e.g. the program page) gets its content + * even when the push carries no class bind this pass recognises. + */ +static u64 svc_gl_spot_dst; /* last 0x180 destination (per pass) */ +static u32 svc_gl_spot_bytes; /* its length word (witness) */ +static u32 svc_gl_lid_log; /* bounded standalone-apply witness */ + +static void svc_gl_add_ptr(u64 va) +{ + u32 k; + + if (!va) + return; + for (k = 0; k < svc_gl.n_ptr; k++) + if (svc_gl.ptr[k] == va) + return; + if (svc_gl.n_ptr < SVC_GR_POINTER_MAX - 1) + svc_gl.ptr[svc_gl.n_ptr++] = va; +} + +/* + * Collect one observed VA operand from the open C597 section. Every case + * is transcribed from the decoded r882 stream words; the upper-word masks + * follow the class definitions (ADDRESS_UPPER 7:0 / lower 31:0). + */ +static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) +{ + u32 v[4] = { 0, 0, 0, 0 }; + u32 n = count > 4 ? 4 : count, k; + + for (k = 0; k < n; k++) { + if (4 * (k + 1) > dlen) + break; + memcpy(&v[k], data + 4 * k, 4); + } + /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). + * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. + */ + if (addr >= 0x2014 && addr < 0x4000 && (addr & 0x3f) == 0x14 && + count >= 2 && !svc_gl.prog_va) { + svc_gl.prog_va = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + return; + } + switch (addr) { + case 0x1574: /* SET_TEX_HEADER_POOL_A/B/C: upper, lower, index. + * Observed: [00000001 214fa220 00000010]. + */ + if (count >= 3 && !svc_gl.param_va) { + svc_gl.param_va = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + svc_gl.param_size = v[2]; /* observed 0x10 */ + } + return; + case 0x155c: /* SET_TEX_SAMPLER_POOL_A/B/C: [1 214fa200 0] */ + if (count >= 3) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x0f84: /* SET_VERTEX_STREAM_SUBSTITUTE_A/B: [1 21200000] */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x0180: /* LINE_LENGTH_IN run: VALUE, LINE_COUNT, + * OFFSET_OUT_UPPER, OFFSET_OUT, PITCH_OUT. + * Observed: [0000000c 1 1 214f8f00 0000000c]. + */ + if (count >= 4) + svc_gl_add_ptr(((u64)(v[2] & 0xff) << 32) | + (u64)(v[3] & 0xfffffffc)); + /* R17: this run names the destination the next 0x1b0 + * LAUNCH_DMA/LOAD_INLINE_DATA run writes to. */ + if (count >= 5) { + svc_gl.up_dst = ((u64)(v[2] & 0xff) << 32) | + (u64)(v[3] & 0xfffffffc); + svc_gl.up_bytes = v[0]; + } + return; + case 0x01b0: /* LAUNCH_DMA + LOAD_INLINE_DATA (op3/op5): the run's + * data words ARE the observed upload payload. Keep the + * FIRST run's bytes for the M2-S1 inline attachment. + * Observed: a004006c -> [00000041 00000670 00000690 + * 000006b0] = LAUNCH_DMA cfg + 3 LID words = 12 B. + * SESSION 149 R16 (M3): keep only the LID payload (the + * cfg word is not code) and bound the copy by the run's + * OWN count - dlen reaches the end of the push, so the + * R15 copy over-read the run (blob=64 = 12 real bytes + + * 52 B of the following method words). These bytes are + * only the fallback: svc_gl_flush reads the real + * program bytes from prog_va via the trusted walk. + */ + if (!svc_gl.blob_len && count > 1 && dlen > 4) { + u32 n = (count - 1) * 4; + + if (n > dlen - 4) + n = dlen - 4; + if (n > sizeof(svc_gl.blob)) + n = (u32)sizeof(svc_gl.blob); + if (n) { + memcpy(svc_gl.blob, data + 4, n); + svc_gl.blob_len = n; + } + } + /* + * R17: record the run's payload at the destination the + * preceding 0x180 run named (the payload starts after the + * LAUNCH_DMA config word - same split as the blob above). + */ + if (svc_gl.up_dst && count > 1 && dlen > 4 && + svc_gl.n_up < SVC_GL_UPLOAD_MAX) { + struct svc_gl_up *u = &svc_gl.up[svc_gl.n_up]; + u32 n = (count - 1) * 4; + + if (n > dlen - 4) + n = dlen - 4; + if (n > sizeof(u->data)) + n = (u32)sizeof(u->data); + if (n) { + u->dst_va = svc_gl.up_dst; + u->len = n; + memcpy(u->data, data + 4, n); + svc_gl.n_up++; + } + } + return; + default: + return; + } +} + +/* + * Close the open section: synthesize the QMD from the collected fields and + * capture it as the forwarding candidate. The QMD word layout mirrors + * svc_gr_qmd_decode() (the CUDA bridge contract): 48/49 program, 32/33 + * parameter + size, 23..25 release, 34..47 pointer pairs, qmd[4] b10 + * REL_ENABLE. Every VA is copied from a decoded operand; qmd_dev_va stays 0 + * (synthesized - no QMD buffer exists in guest memory) so the backend uses + * its qmdwin/scratch path. + */ +static void svc_gl_flush(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, + const struct svc_push_state *st, const u8 *buf, + u32 len, u64 push_va, bool trusted, bool replay, + u32 at_off) +{ + struct svc_push_state s2; + struct gsp_bridge_inline pr; + u32 n_recs = 0, k; + /* + * SESSION 149 R16 (M3): the inline attachment's source. Priority: + * the REAL program bytes read from prog_va through the trusted walk + * (the GL program page), else the observed LID payload bounded to its + * own run. r888's blob=64 over-read the 12 B LID run. + */ + u8 blob[SVC_GR_INLINE_BYTES]; + u32 blob_n = 0; + int blob_rc = -1; + bool blob_trusted = false; + const char *blob_src = "none"; + + svc_gl_prog_apply_param(); + if (!svc_gl.seen || !svc_gl.prog_va || !svc_gl.param_va || + !svc_gl.rel_va || !trusted) + goto out; + /* + * SESSION 149 R15 (M1.1a): admit a rewind replay when the same slot + * re-read valid bytes whose content hash matches the recorded pass + * (the pump sets cap_trust only for such a hash-verified replay; see + * the trust admission at the decode call site). The session-wide + * (push_va, hash) capture history is the exactly-once guard: a push + * already captured/forwarded makes every later replay a no-op. + */ + if (replay && svc_gr_cand_push_seen(push_va, svc_push_hash(buf, len))) { + if (svc_gl_rep_log < 16) { + svc_gl_rep_log++; + pr_info("sema-gl: replay already captured h=0x%x push=0x%llx (no re-capture)\n", + ch->h_object, push_va); + } + goto out; + } + s2 = *st; + memset(s2.qmd, 0, sizeof(s2.qmd)); + s2.qmd[48] = (u32)svc_gl.prog_va; + s2.qmd[49] = ((u32)(svc_gl.prog_va >> 32)) & 0x1ffff; + s2.qmd[32] = (u32)svc_gl.param_va; + s2.qmd[33] = (((u32)(svc_gl.param_va >> 32)) & 0x1ffff) | + ((svc_gl.param_size >> 4) << 19); + s2.qmd[23] = (u32)svc_gl.rel_va; + s2.qmd[24] = ((u32)(svc_gl.rel_va >> 32)) & 0xff; + s2.qmd[25] = svc_gl.rel_payload; + s2.qmd[4] |= 1u << 10; /* REL_ENABLE */ + /* + * qmd[24] b31 (REL_ONE_WORD) stays clear on the first launch: the + * backend then redirects release0 to the module semaphore and the + * sema service authors the guest word (set it once rel_hpa resolving + * is wanted). qmd[24] b26 (REDUCTION) likewise stays clear. + */ + /* + * SESSION 149 R16 (M3): the scheduling fields the r888 engine + * refused to schedule without. Word/field offsets from the public + * QMDV02_03 class header (nvidia/inc/class/clc5c0qmd.h; the booter's + * own 48/49 program pair matches its QMDV02_03 offsets, so the + * rest of the layout is the same class): + * w12 CTA_RASTER_WIDTH MW(415:384) + * w13 CTA_RASTER_HEIGHT MW(431:416) + * w14 CTA_RASTER_DEPTH MW(463:448) + * w18 QMD_VERSION MW(579:576), QMD_MAJOR_VERSION MW(583:580), + * CTA_THREAD_DIMENSION0 MW(607:592) -> w18 hi word + * w19 CTA_THREAD_DIMENSION1 MW(623:608), + * CTA_THREAD_DIMENSION2 MW(639:624) + * w20 REGISTER_COUNT_V MW(656:648) <- SKEDCHECK06 field + * w29 BARRIER_COUNT MW(959:955) + * w31 SASS_VERSION MW(1023:1016) + * SKEDCHECK16 fired on CTA_THREAD_DIMENSION0==0 and SKEDCHECK06 on + * REGISTER_COUNT_V==0. The register count is the stream's own value: + * REGISTER_COUNT(1) = 0x1a = 26 (method 0x200c + 1*0x40 = 0x204c, + * words [0x1a, 0]) - the same number the driver's C5C0 path writes + * ("a NULL QMD will result in SKEDCHECK06_REGISTER_COUNT", cui + * turing_launch.c:781). CTA dims: the 0xC597 stream carries no grid / + * CTA operands we can locate (the 3D-class runs are format/viewport/ + * shader config; the C597 SHADER_CONFIG "CTA raster" words are not in + * this push), so a minimal 1x1x1 grid of 1x1x1 CTAs is used and + * documented as the choice: the checks only reject ZERO dims and a + * zero total-thread count, both of which a 1x1x1 launch satisfies. + * SHARED_MEMORY_SIZE (w17) stays 0 = "no shared memory" (valid and + * NOT among the failed checks); the m72 local-memory prelude already + * covers SKEDCHECK05. The driver's cache-control block (w5 top bits) + * is set as every captured guest QMD carries it; the backend turns it + * into its icache-invalidate bit for the shadow line. + */ + s2.qmd[12] = 1; /* CTA raster width (CTAs) */ + s2.qmd[13] = 1; /* CTA raster height (CTAs) */ + s2.qmd[14] = 1; /* CTA raster depth (CTAs) */ + s2.qmd[18] = 0x23u | (1u << 16); /* QMD 2.3, CTA dim0 = 1 */ + s2.qmd[19] = 1u | (1u << 16); /* CTA dim1 = 1, dim2 = 1 */ + s2.qmd[20] = 0x1au << 8; /* REGISTER_COUNT_V = 26 */ + s2.qmd[29] |= 1u << 27; /* BARRIER_COUNT = 1 */ + s2.qmd[31] |= 0x75u << 24; /* SASS_VERSION 7.5 (sm_75) */ + s2.qmd[5] |= 0x3c000000u; /* driver cache-control block */ + for (k = 0; k < svc_gl.n_ptr && k < SVC_GR_POINTER_MAX - 1; k++) { + s2.qmd[34 + 2 * k] = (u32)svc_gl.ptr[k]; + s2.qmd[35 + 2 * k] = ((u32)(svc_gl.ptr[k] >> 32)) & 0x1ffff; + } + s2.qmd_dev_va = 0; + s2.qmd_hdr = 0; + s2.qmd_end = 0; + s2.qmd_have = 0; + /* + * SESSION 149 R15 (M2-S1) + R16 (M3): attach program-page content as + * an inline record at the synthesized program VA. The GL program page + * (0x121201b00) is NOT among the section's own upload destinations, + * so without this the backend's w48 code-source screen sees an empty + * page, withdraws to the guest VA and the launch dies at FAULT_PDE; + * with it the proven win0 shadow copy + w48/49 rewrite line runs. + * + * R16 source order: the REAL program bytes read from prog_va through + * the trusted walk (the read machinery translates GL VAs now; src=4 + * answered this channel's ring/pushes in r888), else the observed + * LID payload bounded to its own run (12/24/28 B - "materialize + * exactly that"). A trusted read that lands on an all-zero page is + * treated as "not real" and falls through to the LID bytes. + */ + /* + * SESSION 149 R17 (M4): apply the section's memory-writing upload + * commands BEFORE reading the program page, so an upload that + * targets the program VA is visible to the read below (the section's + * engine commands never ran: every destination - constants, tex + * pools, program - stayed zero). The shared inline-write path + * resolves each destination (FB aperture -> emulated vidmem shadow, + * sysmem -> guest RAM, unresolved -> VA shadow) and the once-only + * guard is keyed by (push, content hash, dst, bytes) so a + * replay/rescue pass cannot double-apply. + */ + { + u64 ph = svc_push_hash(buf, len); + u32 ap_n = 0, ap_b = 0; + + for (k = 0; k < svc_gl.n_up && k < SVC_GL_UPLOAD_MAX; k++) { + struct svc_gl_up *u = &svc_gl.up[k]; + + if (svc_inline_once_seen(push_va, ph, u->dst_va, + u->len)) + continue; + svc_inline_once_store(push_va, ph, u->dst_va, u->len); + if (!svc_grce_inline_write(vgpu, u->dst_va, u->data, + u->len, ch->h_object)) { + ap_n++; + ap_b += u->len; + } + } + if (gsp_sema_dbg && svc_gl_up_log < 8) { + svc_gl_up_log++; + pr_info("sema-gl: uploads applied=%u/%u bytes=%u h=0x%x push=0x%llx\n", + ap_n, svc_gl.n_up, ap_b, ch->h_object, + push_va); + } + } + memset(blob, 0, sizeof(blob)); + blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, + sizeof(blob)); + if (!blob_rc) { + u32 z = 0; + + for (k = 0; k < 16; k++) + z |= blob[k]; + if (z) + blob_trusted = true; + } + if (blob_trusted) { + blob_n = sizeof(blob); + blob_src = "trusted-prog"; + } else if (svc_gl_prog_len) { + blob_n = svc_gl_prog_len; + memcpy(blob, svc_gl_prog, blob_n); + blob_src = "prog-hex"; + } else if (svc_gl.blob_len) { + blob_n = svc_gl.blob_len; + memcpy(blob, svc_gl.blob, blob_n); + blob_src = "lid-fallback"; + } + if (blob_n) { + memset(&pr, 0, sizeof(pr)); + pr.dst_va = svc_gl.prog_va; + pr.len = blob_n; + memcpy(pr.data, blob, blob_n); + n_recs = 1; + } + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u\n", + ch->h_object, push_va, len, at_off, svc_gl.prog_va, + svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, + svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, + replay ? 1u : 0u); + /* R16 bounded witness: what the synthesized QMD now schedules and + * where the code bytes came from. */ + if (gsp_sema_dbg && svc_gl_prog_log < 8) { + svc_gl_prog_log++; + pr_info("sema-gl: M3 sched w12=%u w13=%u w14=%u w18=0x%08x w19=0x%08x w20=0x%08x w29=0x%08x w31=0x%08x w5=0x%08x\n", + s2.qmd[12], s2.qmd[13], s2.qmd[14], s2.qmd[18], + s2.qmd[19], s2.qmd[20], s2.qmd[29], s2.qmd[31], + s2.qmd[5]); + pr_info("sema-gl: M3 code src=%s read_rc=%d len=%u lid=%u first=%08x %08x %08x %08x\n", + blob_src, + blob_rc, blob_n, svc_gl.blob_len, + get_unaligned_le32(blob), + get_unaligned_le32(blob + 4), + get_unaligned_le32(blob + 8), + get_unaligned_le32(blob + 12)); + } + svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len, + push_va); +out: + /* + * SESSION 149 R15 (M1.1a): preserve the last valid release across + * the reset - it belongs to the channel (decoded from the bracketing + * 24 B release pushes), not to this section; wiping it here and at + * the section open starved the flush guard of rel_va on the trusted + * first pass (the r883/r886 one-shot decode). + */ + { + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + } +} + +/* * Consume the current GR candidate without forwarding it: hand the launch to * the host CUDA bridge when that path is enabled, then clear the candidate. * Exactly-once via `offered`. Called on the service thread (capture/refusal) @@ -10973,6 +11643,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* r391f: the inline-write once-only key needs the push's identity. */ svc_pass_hash = svc_push_hash(buf, len); + /* SESSION 149 R17 (M4): per-pass reset of the standalone 0x180/0x1b0 + * upload pair (the tracker pairs one run with its successor). */ + svc_gl_spot_dst = 0; + svc_gl_spot_bytes = 0; /* * r316: pending releases record the channel they were decoded for; @@ -11437,6 +12111,109 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } + /* + * SESSION 149 R14 (sema_gl_xlate): GL/3D direct-stream section + * recognition. A SET_OBJECT (op 1, addr 0, one class word) + * binds a class to a subchannel; C597 (TURING_A, the observed + * GL class) on the channel subchannel opens the section. Any + * other class bind closes it (the copy-class C5B5 and 2D 902D + * binds surrounding the GL push must not open sections). The + * collected VA operands live in svc_gl until the section closes + * at the end of the pass or at the next class bind. + */ + if (gsp_sema_gl_xlate && op == 1 && addr == 0 && count == 1 && + off + 8 <= len) { + u32 cls; + + memcpy(&cls, buf + off + 4, 4); + if (cls == 0xc597) { + /* + * SESSION 149 R15 (M1.1a): carry the last + * valid release across the section reset. + * It was decoded from the bracketing 24 B + * release pushes BEFORE this class bind; + * the old wipe-and-open here left rel_va + * at 0 at flushtime, so the trusted first + * pass of the one-shot setup push was + * rejected by the flush guard and the + * section was never captured (r883/r886: + * exactly one trusted decode of the 1524 B + * push, zero captures). + */ + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + + if (svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.seen = true; + svc_gl.subch = subch; + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + } else if (svc_gl.seen) { + svc_gl_flush(vgpu, ch, &st, buf, len, + push_va, trusted, replay, off); + } + } else if (gsp_sema_gl_xlate && svc_gl.seen && + subch == svc_gl.subch) { + svc_gl_field(addr, count, buf + off + 4, + len - off - 4); + } + + /* + * SESSION 149 R17 (M4): standalone GL-class upload apply. The + * context-init upload push (the one-shot program upload) need + * not sit inside a recognised C597 section, but it carries + * the same subchannel-0 0x180 (LINE_LENGTH_IN) + 0x1b0 + * (LAUNCH_DMA + LOAD_INLINE_DATA) i2m shape. Pair the last + * 0x180 destination with each following 0x1b0 payload and + * apply it through the shared inline-write path (the engine + * never ran these commands). Once-only by (push, content + * hash, dst, bytes); the witness line is bounded. + */ + if (gsp_sema_gl_xlate && ch->engine_type == 0 && subch == 0) { + if (op == 1 && addr == 0x180 && count >= 4) { + u32 w2, w3; + + memcpy(&w2, buf + off + 12, 4); + memcpy(&w3, buf + off + 16, 4); + svc_gl_spot_dst = + ((u64)(w2 & 0xff) << 32) | + (u64)(w3 & 0xfffffffc); + memcpy(&svc_gl_spot_bytes, buf + off + 4, 4); + } else if (op == 5 && addr == 0x1b0 && count > 1 && + svc_gl_spot_dst && + (trusted || allow_ops)) { + u32 bytes = (count - 1) * 4; + + if (bytes && !svc_inline_once_seen(push_va, + svc_pass_hash, svc_gl_spot_dst, + bytes)) { + svc_inline_once_store(push_va, + svc_pass_hash, + svc_gl_spot_dst, bytes); + (void)svc_grce_inline_write(vgpu, + svc_gl_spot_dst, + buf + off + 8, bytes, + ch->h_object); + if (gsp_sema_dbg && + svc_gl_lid_log < 24) { + svc_gl_lid_log++; + pr_info("sema-gl-lid: h=0x%x push=0x%llx dst=0x%llx bytes=%u len=%u trusted=%d first=%08x %08x %08x\n", + ch->h_object, push_va, + svc_gl_spot_dst, bytes, + svc_gl_spot_bytes, + trusted, + *(const u32 *)(buf + off + 8), + off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0, + off + 20 <= len ? *(const u32 *)(buf + off + 16) : 0); + } + } + } + } + if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data @@ -11586,6 +12363,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + /* SESSION 149 R14: keep the last VALID release + * decoded on this channel for the GL section + * synthesizer (the 24 B release pushes bracket + * the large GL launch push in ring order). + */ + if (gsp_sema_gl_xlate) { + svc_gl.rel_va = va; + svc_gl.rel_payload = payload; + } svc_wait_rel_va = va; svc_wait_rel_payload = payload; svc_wait_rel_off = off; @@ -13301,6 +14087,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off += 4 + count * 4; } + /* + * SESSION 149 R14: close an open GL/3D section when the pass ends - + * this is the normal capture point (one section per large GL push). + */ + if (gsp_sema_gl_xlate && svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, len, push_va, trusted, + replay, off); + if (gsp_sema_dbg && ch->h_object && svc.entry_log < 16384) { svc.entry_log++; @@ -13525,6 +14319,7 @@ static void svc_session_flush(void) svc_dst_guard_reset(); /* r609: per-session device-dst windows */ svc.map_n = 0; svc_pb_n = 0; + svc_pb_gr_done = 0; svc_pd_found = 0; svc_pd_attempts = 0; svc_pd_best = 0; @@ -13803,6 +14598,7 @@ static void svc_guest_reset_wipe(void) svc_pd_r_ev_hash = 0; svc_pd_r_n = 0; svc_pb_n = 0; + svc_pb_gr_done = 0; svc_pool_boot_n = 0; svc_pool_boot_next = 0; svc_pool_pa_n = 0; @@ -14610,9 +15406,23 @@ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, { u32 slot, i; - if (!ch || !ch->engine_type || ch->ce_index >= 32) + if (!ch) return -ENOENT; - slot = 4 * ch->ce_index; + /* + * R10 (patch A): GR (eng==0) channels whose ring was content- + * validated own a ring-anchored window pair (svc_pb_pair_gr). + * The CE path is unchanged; a GR channel without a paired + * window is rejected exactly as before. + */ + if (!ch->engine_type) { + if (ch->gr_index >= 32 || !ch->gr_ram_ring) + return -ENOENT; + slot = SVC_PB_GR_SLOT_BASE | ch->gr_index; + } else { + if (ch->ce_index >= 32) + return -ENOENT; + slot = 4 * ch->ce_index; + } for (i = 0; i < svc_pb_n; i++) { u64 pa; @@ -14752,6 +15562,27 @@ static u64 svc_lastgood_ring_va; static u64 svc_lastgood_ring_pa; +/* + * SESSION 154 (M-GL1.2): the PA ceiling for the push read. The GL client's + * sysmem allocations live ABOVE SEMA_PA_MAX (the ring scan already reads to + * gsp_sema_gr_scan_top for exactly this client - r908: its ring page at gpa + * 0x231e5d000); the content anchor pinned its push page at pa 0x2328e8000 + * (r920, measured) and the old 6 GiB read bound then rejected the freshly + * pinned page (src=2 -> -EIO) while the stale window answered instead + * (src=3 garbage, every push "unreadable"). Use the same widened ceiling + * the scans use. + */ +static u64 svc_push_pa_ceiling(void) +{ + u64 top = READ_ONCE(gsp_sema_gr_scan_top); + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PA_MAX) + top = SEMA_PA_MAX; + return top; +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) @@ -14831,7 +15662,8 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, break; } - if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) { + if (ap == SVC_APERTURE_SYS && + pa >= svc_push_pa_ceiling()) { last_rc = -EIO; continue; } @@ -14966,6 +15798,11 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, return 0; } +/* r912: guest-RAM USERD shape scan (defined next to the ring RAM scan). */ +static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out); + /* * Locate a channel's USERD. The allocation parameters carry an offset that is * relative to a client pool the emulation cannot see, so it is only one @@ -15191,6 +16028,63 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } /* + * r912 (M-GL1.2): a GR channel whose ring was content-validated in + * guest RAM (gr_ram_ring) has its USERD in guest RAM too - the + * FB-shadow sweep below can never reach it. The guest-RAM shape + * scan is sliced and budgeted like the ring scan; it validates + * every entry the candidate's put claims against the ring window + * and admits the block only after its PUT advances across sightings + * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget + * gate stays open while candidates are under watch, so the + * liveness confirmation cannot be starved by slice accounting. + */ + if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok && + (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || + ch->gr_ram_userd_watch_n)) { + u64 upa = 0; + u32 uput = 0, uget = 0; + + if (svc_gr_ram_userd_scan(vgpu, ch, &upa, &uput, &uget)) { + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_SYS; + ch->userd_ok = true; + /* + * SESSION 154 (M-GL1.2): seed the walk at the live + * block's PUT. The client's put is free-running + * (measured 0->3 with entries=2), so the pump's + * `put %= entries` alone can collapse onto + * last_get and consume NOTHING - r914 stayed at + * submissions=0 for minutes with put=12 % 2 = 0 and + * last_get=0. Consume up to entries-1 positions + * ENDING at the PUT: the newest slots the client is + * waiting on; normal slot-space tracking resumes + * from there. + */ + if (ch->gpfifo_entries > 1 && uput) { + u32 span = uput < ch->gpfifo_entries ? + uput : ch->gpfifo_entries - 1; + + ch->last_get = (uput % ch->gpfifo_entries + + ch->gpfifo_entries - span) % + ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->gr_ram_put_seeded = uput; + ch->gr_ram_userd_latch_put = uput; + ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_confirmed = false; + *put = uput; + *get = uget; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by RAM scan -> pa=0x%llx (put=%u get=%u%s; entries ring-validated; speculative, pump confirms; walk seed last_get=%u)\n", + ch->h_object, ch->gr_index, upa, uput, uget, + uget < uput ? "" : ", idle", + (u32)ch->last_get); + return 0; + } + } + + /* * r496b: no candidate matched. For a GR channel whose ring is known * (translated and pinned - the sysmem content scan or the VRAM * conventions), the USERD may still be elsewhere: the X/GL driver @@ -16894,14 +17788,31 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *pa_out) { static u8 *buf; + u64 top = gsp_sema_gr_scan_top; u64 start = ch->gr_ram_cursor; - u64 end = start + SVC_GR_RAM_SLICE; + u64 end; u64 gpa; + /* + * r908: the GPFIFO entries start AT the registered ring VA, so their + * in-page offset is the ring VA's own offset (X server ring + * 0x120090000 -> offset 0; RC watchdog 0x120000400 -> 0x400; the GL + * client 0x1203cfff0 -> 0xff0, measured). The scan used to test only + * page offset 0, so any ring whose VA is not page-aligned could never + * match no matter how good the entry contents were. + */ + u32 eoff = (u32)(ch->gpfifo_va & (SEMA_PAGE_SIZE - 1)); int i; unsigned long flags; - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) @@ -16914,31 +17825,46 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, if (svc_read(vgpu, gpa, buf, chunk)) continue; - for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) { + for (o = 0; o + eoff + 2 * SVC_GPFIFO_ENTRY_SIZE <= chunk; + o += SEMA_PAGE_SIZE) { u32 e0, e1, f0, f1; u64 va0, va1; u32 len0, len1; - memcpy(&e0, buf + o, 4); - memcpy(&e1, buf + o + 4, 4); + memcpy(&e0, buf + o + eoff, 4); + memcpy(&e1, buf + o + eoff + 4, 4); va0 = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len0 = ((e1 >> 10) & 0x1fffff) << 2; - /* entry0: main pushbuffer, just below the ring */ - if (va0 >= ch->gpfifo_va || - va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va) + /* + * entry0: a pushbuffer VA in the ring's allocation + * window - just below the ring for the long-standing + * convention (X ring: 0x120010000 for ring + * 0x120090000), or just above it (r908: the GL + * client's PROMOTE_CTX ordering shows the main + * pushbuffer above the ring on this boot; accept + * both sides, still bounded to +/-1 MB of the ring). + */ + if (va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va || + va0 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) continue; if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) continue; if (len0 < 4 || len0 > 0x100000) continue; - memcpy(&f0, buf + o + 8, 4); - memcpy(&f1, buf + o + 12, 4); + memcpy(&f0, buf + o + eoff + 8, 4); + memcpy(&f1, buf + o + eoff + 12, 4); va1 = ((u64)(f1 & 0xff) << 32) | (u64)(f0 & 0xfffffffc); len1 = ((f1 >> 10) & 0x1fffff) << 2; - /* entry1: progress tracker, above entry0 */ - if (va1 <= va0 || + /* + * entry1: the progress tracker above entry0 - or the + * SAME push submitted again (r908 measured on the GL + * client's ring: {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}). Never below entry0; + * still bounded by the window above the ring. + */ + if (va1 < va0 || va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) continue; if (len1 < 4 || len1 > 0x100000) @@ -16947,7 +17873,7 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, return true; } } - ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + ch->gr_ram_cursor = (end >= top) ? 0 : end; ch->gr_ram_passes++; if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { ch->gr_ram_logged = true; @@ -16968,6 +17894,10 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; svc.chan[i].gr_ram_passes = ch->gr_ram_passes; svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + /* SESSION 149 R15: the M1.1b keep-alive budget state. */ + svc.chan[i].gr_ram_kick = ch->gr_ram_kick; + svc.chan[i].gr_ram_refreshes = ch->gr_ram_refreshes; + svc.chan[i].gr_ram_refresh_at = ch->gr_ram_refresh_at; break; } } @@ -16976,6 +17906,484 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, } /* + * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring + * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan). + * Such a client's USERD is a system-memory object like its ring, so the + * FB-shadow shape scan inside svc_chan_userd can never reach it: the + * registered userd value is pool-relative, every FB candidate reads + * garbage, and the pump never iterates the GPFIFO entries ("unregistered + * (submissions=0)"). Scan guest RAM for the USERD control block shape + * (GET at +0x88, PUT at +0x8c - SVC_USERD_GP_*), with the same sliced, + * cursor-resumed budget as the ring scan (one SVC_GR_RAM_SLICE slice per + * pass, svc_gr_ram_pass_budget() slices before giving up). + * + * A candidate passes the shape and entry gates first (every entry its PUT + * claims must decode to a pushbuffer VA inside the ring's allocation + * window: entry k at ring VA + (k % entries)*8, within +/-SVC_GR_RAM_WIN/2 + * of the ring VA with a sane length), but admission is by LIVENESS, not by + * shape (SESSION 154): the GET/PUT gate alone admitted frozen or churning + * lookalike pages (r914: a low-RAM table whose +0x88/+0x8c words happened + * to read 12/8 and later 8/4 - adopted, the pump then read PUT=12 with + * entries=2, the modulo collapsed to 0 and NOTHING was consumed while the + * client retried for minutes). A passing candidate is WATCHED across + * passes; only a block whose PUT takes the client's own small step (+1 or + * +2) on two separate sightings is adopted as the live control block (the + * measured live block advances 1->2->3; the lookalikes sit still or jump + * by 4). The watch runs every call, so it keeps working past the slice + * budget (the gate in svc_chan_userd keeps calling while a watch is live). + */ +static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 ring_pa = 0; + u64 start = ch->gr_ram_userd_cursor; + u64 end, gpa; + u32 entries = ch->gpfifo_entries; + u64 live_pa = 0; + u32 live_put = 0, live_get = 0, live_prev = 0; + int ring_ap = SVC_APERTURE_SYS; + u32 wi; + int i; + unsigned long flags; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !entries) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + (void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap); + /* + * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates. + * Cheap (one 4-byte PUT read each, every pass) and the primary adoption + * site: a watched block whose PUT takes the client's own single +1 step + * is the live control block (the measured live block advances 1->2->3 + * in 2-8 s steps; the registration window is 25-45 s, so one step must + * suffice). A block whose PUT jumps by a larger amount or decreases is + * a churning lookalike (r914's pa 0x9e200 marched -4; the dense + * lookalike pools churn +14) and its streak resets; one that goes + * unreadable or zero is dropped. + */ + for (wi = 0; wi < ch->gr_ram_userd_watch_n; ) { + u64 wpa = ch->gr_ram_userd_watch[wi]; + u32 hp = 0, hg = 0, last; + + if (svc_read(vgpu, wpa + SVC_USERD_GP_PUT, &hp, + sizeof(hp)) || !hp || hp > 0x100000) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + last = ch->gr_ram_userd_watch_put[wi]; + if (hp == last) { + wi++; + continue; + } + if (hp != last + 1) { + /* + * SESSION 154: a jump or a reset is the churn + * signature (the r916-r919 lookalike pools move by + * +4/-4/+14 or collapse to zero between sightings) - + * drop the entry instead of keeping a dead watch slot + * that would evict newer, better candidates. + */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + ch->gr_ram_userd_watch_put[wi] = hp; + /* + * SESSION 154: ONE observed +1 step adopts. The live block + * advances ~2-8 s per step (measured) while the client's + * channel registration window is ~25-45 s, so waiting for + * a second step can outlive the window; a churning lookalike + * moves in larger jumps (measured +4/-4/+14) and a random + * page cannot produce this exact one-step signature with + * the shape+entry gates already applied. + */ + ch->gr_ram_userd_watch_hits[wi]++; + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (!hg || hg > hp) { + ch->gr_ram_userd_watch_hits[wi] = 0; + wi++; + continue; + } + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, live_put, + live_pa, live_put, live_get); + break; + } + if (!live_pa) + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 0x90 <= chunk; o += 0x100) { + u32 hp, hg, k; + bool ok = true; + + memcpy(&hg, buf + o + SVC_USERD_GP_GET, 4); + memcpy(&hp, buf + o + SVC_USERD_GP_PUT, 4); + /* + * r914: the control block lives in a near-empty page + * (only the put/get pair and a few engine-owned words + * are written - the measured X-channel FB block and + * the GL client's live sysmem block both leave the + * leading 0x80 bytes zero). The shape gate alone is + * far too weak: a low-RAM page at pa 0x7c000 carrying + * coincidental small numbers at +0x88/+0x8c was + * latched on r913 (the pass then read its put + * forever and never advanced). SESSION 154: the put + * cap is 8 - the GL client's live block cycles 0..3 + * with entries=2 (measured), while the r914 false + * latches read 12/8 and 16/16 and every previously + * measured lookalike counted 16 or more. Also accept + * a free-running put (the old put <= entries bound + * would reject the live block). + */ + if (!hp || hp > 8 || hg > hp) + continue; + { + u32 z = 0, zi; + + for (zi = 0; zi < 0x80; zi++) + if (!buf[o + zi]) + z++; + if (z < 0x60) + continue; + } + /* + * Never adopt the ring page itself: the ring VA's + * own page holds GPFIFO entries and ring metadata, + * and a latch there would make the pump's GP_GET + * write-back corrupt it. + */ + if (ring_pa && (gpa + o) == + (ring_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + for (k = 0; k < hp && k < 8; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + /* + * r914: entry slot j of a free-running put is + * j % entries, not j. + */ + u32 slot = k % entries; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)slot * SVC_GPFIFO_ENTRY_SIZE, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + if (!eva || + eva + SVC_GR_RAM_WIN / 2 < + ch->gpfifo_va || + eva >= ch->gpfifo_va + + SVC_GR_RAM_WIN / 2 || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + /* + * SESSION 154: the live control block lives in a + * NEAR-EMPTY page (measured: put/get plus two more + * bytes non-zero - r914's live block; the X-channel FB + * block and the GL sysmem block both leave the leading + * 0x80 bytes zero). The r916-r919 candidate floods all + * came from dense pages (a dozen+ shape hits per page + * in tables and pools), so a page with more than 32 + * non-zero bytes is rejected before it can occupy a + * watch slot. + */ + { + u32 pstart = o & ~0xfffu; + u32 zc = 0, zi2; + + for (zi2 = 0; zi2 < SEMA_PAGE_SIZE; zi2++) + if (!buf[pstart + zi2]) + zc++; + if (zc < SEMA_PAGE_SIZE - 32) + continue; + } + /* + * SESSION 154 (M-GL1.2): a candidate that passed the + * shape gate, the entry validation, the page-density + * gate and the put cap is latched SPECULATIVELY. The + * old code latched on shape alone and a frozen/ + * churning lookalike could hold the walk forever + * (r913: pa 0x7c000; r914: pa 0x9e200 at put=12); + * the strengthened gates reject those classes, and the + * pump now CONFIRMS the latch from its own per-pass + * read: a +1/+2 step confirms the block as the live + * one, a reset/jump or no movement within the bound + * unlatches it so the scan can adopt a better + * candidate (see svc_pump_channels). + */ + live_pa = gpa + o; + live_put = hp; + live_get = hg; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put step across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_pa, + live_put, live_get); + break; + } + if (live_pa) + break; + } + ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; + ch->gr_ram_userd_passes++; + if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && + !ch->gr_ram_userd_watch_n) { + ch->gr_ram_userd_logged = true; + pr_info("sema-chan: GR USERD h=0x%x rank=%u RAM scan found no candidate in %u slice(s) (top 0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_userd_passes, + top); + } + /* + * Persist the cursor immediately: the stalled-userd pump path + * `continue`s past the end-of-pass write-back (the same reason + * svc_gr_ram_ring_scan persists its cursor). + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + u32 wj; + + svc.chan[i].gr_ram_userd_cursor = + ch->gr_ram_userd_cursor; + svc.chan[i].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[i].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + svc.chan[i].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { + svc.chan[i].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[i].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[i].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + } + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!live_pa) + return false; + *pa_out = live_pa; + *put_out = live_put; + *get_out = live_get; + return true; +} + +/* + * SESSION 154 (M-GL1.2): content anchor for a GR client push whose + * translation went stale. r913: the entry's read resolved through a + * VALIDATED pin that kept answering for a page the guest had since reused + * (`first8=00000001` through src=2) while the trusted walk answered from a + * wrong VASPACE (`pa=0x5c0000 aperture=vidmem plaus=0`) and the client's + * real push bytes sat in RAM on a page no registered directory covered. + * The client's submit pushes carry two stable anchors - the first method + * pair (SET_SUBDEVICE_MASK 0x0001fff0, measured on every GR submit push of + * this driver) and the release-group signature pair measured deep in the + * same pushes - so sweep guest RAM for them (one SVC_GR_RAM_SLICE slice per + * call, cursor-resumed, a few sweeps per entry VA), validate the page by + * re-reading it (prefix at the entry's own page offset + full method-stream + * parse + the signature present in the same page - a stale fragment page + * carrying only the signature fails), then pin it with svc_pin_store so the + * pump's retry reads the REAL bytes like any validated page. + */ +static const u8 svc_gr_anchor_pre[8] = { + 0xf0, 0xff, 0x01, 0x00, 0x00, 0x00, 0x01, 0x20, +}; +static const u8 svc_gr_anchor_sig[16] = { + 0x43, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, +}; +#define SVC_GR_ANCHOR_SWEEPS 3 + +static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 len) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 start = ch->gr_anchor_cursor; + u64 end, gpa, page_pa = 0; + u32 poff = (u32)(va & (SEMA_PAGE_SIZE - 1)); + int i; + unsigned long flags; + + if (len > SEMA_PAGE_SIZE) + return false; + if (ch->gr_anchor_va != va) { + ch->gr_anchor_va = va; + ch->gr_anchor_sweeps = 0; + ch->gr_anchor_cursor = 0; + start = 0; + } + if (ch->gr_anchor_sweeps >= SVC_GR_ANCHOR_SWEEPS) + return false; + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !len) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + if (!ch->gr_anchor_logged) { + ch->gr_anchor_logged = true; + pr_info("sema-anchor: h=0x%x va=0x%llx len=%u content anchor scan started\n", + ch->h_object, va, len); + } + for (gpa = start; !page_pa && gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 24 <= chunk; o += 4) { + /* + * (a) the push's own first method pair - it sits at + * the entry VA's page offset, so only a hit whose + * page offset matches the entry's is a candidate. + */ + if (!memcmp(buf + o, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + ((gpa + o) & (SEMA_PAGE_SIZE - 1)) == poff) { + page_pa = gpa + o - poff; + break; + } + /* + * (b) the release-group signature measured deep in + * the client's submit pushes; the page is validated + * below before it is pinned. + */ + if (!memcmp(buf + o, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + page_pa = (gpa + o) & + ~(u64)(SEMA_PAGE_SIZE - 1); + break; + } + } + } + ch->gr_anchor_cursor = (end >= top) ? 0 : end; + if (ch->gr_anchor_cursor == 0) + ch->gr_anchor_sweeps++; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[i].gr_anchor_va = ch->gr_anchor_va; + svc.chan[i].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[i].gr_anchor_logged = ch->gr_anchor_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!page_pa) + return false; + /* + * Validate the candidate BEFORE pinning: the push's first method + * pair at the entry's own page offset, a full method-stream parse, + * and the release-group signature present in the same page (the + * remnant fragment at r914's pa 0x84aef000 carries the signature + * but fails the prefix/parse gate). + */ + { + u8 *pb = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + u32 n = min_t(u32, len, SEMA_PAGE_SIZE - poff); + bool ok = false; + + if (pb) { + const u8 *sig = NULL; + + ok = !svc_read(vgpu, page_pa, pb, SEMA_PAGE_SIZE); + if (ok) { + u32 z; + + for (z = 0; z + 16 <= SEMA_PAGE_SIZE; z++) + if (!memcmp(pb + z, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + sig = pb + z; + break; + } + } + ok = ok && sig && + !memcmp(pb + poff, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + svc_push_prefix_ok(pb + poff, n) && + svc_method_stream_plausible(pb + poff, n); + kfree(pb); + } + if (!ok) { + pr_info("sema-anchor: h=0x%x va=0x%llx candidate pa=0x%llx rejected (content gate)\n", + ch->h_object, va, page_pa); + return false; + } + } + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + svc_lat_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + pr_info("sema-anchor: h=0x%x va=0x%llx -> pa=0x%llx (release-group signature anchored; pinned)\n", + ch->h_object, va, page_pa); + return true; +} + +/* * r705h: host-triggered pdb discovery. The automatic latch * (svc_vas_candidates_verify + svc_xvas_pdb_discover) only runs from the RAM * ring scan, which sweeps [0,6GB) before the client's pushes exist and then @@ -17028,17 +18436,65 @@ int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa) } /* - * r496f: discover a kernel-RM client VAS's page directory in the emulated - * VRAM by verification. The CPU RM builds internal-VAS page tables in - * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally - * owned / UVM-only RPC), so the pushbuffer pages and the progress - * semaphore of an X/GL channel cannot be translated from any existing - * source. Try each 4 KB page as the walk root and accept the first whose - * walk reproduces the ring VA -> ring PA pair we already validated by - * content; registering it with the VAS table makes every other VA of the - * same client resolvable through the existing walker. Bounded to the - * vidmem shadow; verified, so a wrong candidate cannot be registered. + * R10 (patch B): one candidate walk-root probe. Shared by the vidmem + * and guest-sysmem phases of svc_xvas_pdb_discover: a candidate + * registers only when its walk reproduces the content-verified + * ring_va -> ring_pa pair, so a wrong candidate costs a rejected walk, + * never a wrong registration. */ +static bool svc_xvas_try_root(struct nv_vgpu_device *vgpu, u64 pdb, + u32 entries, bool vidmem, u64 ring_va, + u64 ring_pa) +{ + struct sema_vaspace_desc d; + u64 pa = 0; + int ap = 0; + + if (svc_gmmu_walk(vgpu, pdb, entries, ring_va & ~0xfffULL, + vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS, + &pa, &ap, false) || + (pa & ~0xfffULL) != (ring_pa & ~0xfffULL)) + return false; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | (u32)(ring_va >> 20); + d.pdb = pdb; + d.num_entries = entries; + d.b_vidmem = vidmem; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (%s root; ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + pdb, entries, vidmem ? "vidmem" : "sysmem", ring_va, ring_pa); + return true; +} + +/* + * r496f: discover a kernel-RM client VAS's page directory by verification. + * The CPU RM builds internal-VAS page tables and never publishes them + * (SET_PAGE_DIRECTORY is an externally owned / UVM-only RPC), so the + * pushbuffer pages and the progress semaphore of an X/GL channel cannot + * be translated from any existing source. Try each 4 KB page as the + * walk root and accept the first whose walk reproduces the ring VA -> + * ring PA pair we already validated by content; registering it with the + * VAS table makes every other VA of the same client resolvable through + * the existing walker. + * + * R10 (patch B): bounded guest-sysmem phase. A sysmem client (the X/GL + * kernel-RM channel whose ring is guest RAM) can keep its internal-VAS + * page tables in guest RAM too, where the vidmem sweep cannot see them + * (r880: ring VA 0x120090000 -> pa 0x10643d000 content-validated, no + * root ever reproduced it). Candidates come only from guest-sysmem + * ranges this module already knows: + * - the ring's allocation window, +/- SVC_GR_RAM_WIN/2 around the + * caller's content-verified ring PA (SVC_GR_RAM_WIN, the span + * svc_gr_ram_ring_scan validated the client's ring/pushes in), and + * - the merged guest-allocation map svc.map[]/svc.map_n (struct + * sema_map_entry - "the allocations the guest has described to us", + * see svc_apply_mapped), capped to SVC_XVAS_SYS_MAP_MAX entries. + * Both are additionally bounded by SEMA_PA_MAX (guest RAM); every + * candidate passes the identical verification (svc_xvas_try_root), so a + * wrong candidate cannot be registered. + */ +#define SVC_XVAS_SYS_MAP_MAX 16384 static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, u64 ring_pa) { @@ -17047,30 +18503,51 @@ static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, u64 off; unsigned int ei; - if (!vgpu->bars[1].vaddr || !lim) - return; - for (off = 0; off + 0x1000 <= lim; off += 0x1000) { - for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) { - u64 pa = 0; - int ap = 0; - - if (!svc_gmmu_walk(vgpu, off, ent_tab[ei], - ring_va & ~0xfffULL, - SVC_APERTURE_FB, &pa, &ap, false) && - (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) { - struct sema_vaspace_desc d; - - memset(&d, 0, sizeof(d)); - d.h_vaspace = 0x5a5a0000u | - (u32)(ring_va >> 20); - d.pdb = off; - d.num_entries = ent_tab[ei]; - d.b_vidmem = true; - gsp_sema_service_vaspace_add(&d); - pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", - off, ent_tab[ei], ring_va, ring_pa); - return; - } + /* Vidmem shadow sweep (r496f; unchanged behaviour). */ + if (vgpu->bars[1].vaddr && lim) { + for (off = 0; off + 0x1000 <= lim; off += 0x1000) + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], true, + ring_va, ring_pa)) + return; + } + + /* + * R10 (patch B): bounded guest-sysmem phase. The ring allocation + * window first (the strongest guess - the ring scan's own + * convention), then the merged allocation map, skipping pages the + * window phase already covered. + */ + if (ring_pa >= SVC_GR_RAM_WIN / 2 && + ring_pa + SVC_GR_RAM_WIN / 2 <= SEMA_PA_MAX) { + u64 lo = (ring_pa - SVC_GR_RAM_WIN / 2) & ~0xfffULL; + u64 hi = (ring_pa + SVC_GR_RAM_WIN / 2) & ~0xfffULL; + + for (off = lo; off < hi; off += 0x1000) + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], false, + ring_va, ring_pa)) + return; + } + { + u32 nmap = min_t(u32, svc.map_n, SVC_XVAS_SYS_MAP_MAX); + u32 mi; + + for (mi = 0; mi < nmap; mi++) { + off = svc.map[mi].pa_page & ~0xfffULL; + if (!off || off >= SEMA_PA_MAX) + continue; + if (ring_pa >= SVC_GR_RAM_WIN / 2 && + off >= ring_pa - SVC_GR_RAM_WIN / 2 && + off < ring_pa + SVC_GR_RAM_WIN / 2) + continue; /* window phase covered it */ + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], false, + ring_va, ring_pa)) + return; } } } @@ -17651,12 +19128,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, * pushbuffer VAs sit just BELOW the ring VA in the same allocation * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to - * ring_va + 0x100000 (the progress tracker). A page whose first two - * 8-byte words both name such VAs with sane lengths is the ring; pin - * it (validated) and its allocation window (guesses) so the ring - * entry and pushbuffer reads resolve. Sliced so the service thread - * stays responsive; gives up per channel after - * SVC_GR_RAM_PASS_MAX slices. + * ring_va + 0x100000 (the progress tracker). r908 relaxations, + * measured on the GL client ring (va=0x1203cfff0, gpa 0x231e5d000): + * the entry pair is tested at the ring VA's own in-page offset (the + * entries start AT the registered VA), entry0 may sit either side of + * the ring within the window, and entry1 may repeat entry0's VA + * (that client queues {push,64} then {push,32}). A page whose entry + * pair both name such VAs with sane lengths is the ring; pin it + * (validated) and its allocation window (guesses) so the ring entry + * and pushbuffer reads resolve. Sliced so the service thread stays + * responsive; gives up per channel after svc_gr_ram_pass_budget() + * slices (two sweeps of sema_gr_scan_top). * * SESSION 147 (r840 evidence): the scan must NOT run for the kernel * RC-watchdog channel (h=0x31415900). Its ring is a plain kernel @@ -17674,7 +19156,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, * latch. */ if (ch->h_object != 0x31415900u && - ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + ch->gr_ram_passes < svc_gr_ram_pass_budget() && !test_bit(ch->gr_index, &svc_gr_ring_done)) { u64 hit_pa = 0; @@ -17792,6 +19274,90 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, } /* + * R10 (patch A): content-anchored ring window for a GR (eng==0) channel. + * + * A GR channel whose ring was found by the guest-RAM content scan + * (gr_ram_ring; svc_gr_ram_ring_scan) has exactly one content-verified + * anchor: the ring page itself, pinned by the scan. The scan's contract + * bounds the channel's pushbuffer VAs to +/- SVC_GR_RAM_WIN/2 around the + * ring VA (entry0 just below the ring, the progress tracker above), so + * install the window as two one-megabyte svc_pb[] entries with the same + * +1 slope convention CE uses (cross-checked on the RC watchdog r880: + * ring 0x120000400 -> pa 0x10567a400, its push 0x120000000 -> + * pa 0x10567a000). + * + * Strictly additive: only eng==0 channels whose ring was content- + * validated are eligible - the kernel-RM sysmem clients the scan exists + * for. CUDA channels keep their verified VAS/walk sources; the RC + * watchdog is excluded from the scan gate. The entries are consumed + * only by svc_pb_win_lookup, i.e. as an untrusted last-resort source in + * the push read, after every per-page validated source missed, with the + * existing all-zero first-chunk rejection still applied. No pin, lat + * or other lookup consumer sees these guesses. + */ +static void svc_pb_pair_gr(struct svc_channel *ch) +{ + u64 ring_pa = 0, ring_va; + int ring_ap = SVC_APERTURE_SYS; + u32 want, n = 0; + + if (ch->gr_index >= 32 || !ch->gr_ram_ring || !ch->gpfifo_va) + return; + if (test_bit(ch->gr_index, &svc_pb_gr_done)) + return; + if (svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap)) + return; + ring_pa &= ~(u64)(SEMA_PAGE_SIZE - 1); + if (!ring_pa || ring_pa >= SEMA_PA_MAX) + return; + ring_va = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); + want = SVC_PB_GR_SLOT_BASE | ch->gr_index; + + /* + * r914: replace this rank's previous pair instead of appending. + * svc_pb_win_lookup returns the FIRST matching slot entry, so a + * stale earlier pair (previous attempt's ring PA) would keep + * winning over the refreshed one; svc_chan_rearm_rank re-arms this + * pairing per (re-)registration for exactly that reason. + */ + { + u32 i; + + for (i = 0; i < svc_pb_n; i++) { + if (svc_pb[i].slot == want) { + svc_pb[i] = svc_pb[svc_pb_n - 1]; + svc_pb_n--; + i--; + } + } + } + + /* Lower half: [ring - 1 MB, ring) -> [ring_pa - 1 MB, ring_pa). */ + if (svc_pb_n < SVC_PB_MAX && ring_va >= SVC_GR_RAM_WIN / 2 && + ring_pa >= SVC_GR_RAM_WIN / 2) { + svc_pb[svc_pb_n].va = ring_va - SVC_GR_RAM_WIN / 2; + svc_pb[svc_pb_n].pa_page = ring_pa - SVC_GR_RAM_WIN / 2; + svc_pb[svc_pb_n].slot = want; + svc_pb_n++; + n++; + } + /* Upper half: [ring, ring + 1 MB) -> [ring_pa, ring_pa + 1 MB). */ + if (svc_pb_n < SVC_PB_MAX && + ring_pa + SVC_GR_RAM_WIN / 2 < SEMA_PA_MAX) { + svc_pb[svc_pb_n].va = ring_va; + svc_pb[svc_pb_n].pa_page = ring_pa; + svc_pb[svc_pb_n].slot = want; + svc_pb_n++; + n++; + } + if (!n) + return; + set_bit(ch->gr_index, &svc_pb_gr_done); + pr_info("sema-svc: GR ring window paired h=0x%x rank=%u ring va=0x%llx -> pa=0x%llx (%u entries; untrusted last-resort push window)\n", + ch->h_object, ch->gr_index, ring_va, ring_pa, n); +} + +/* * Pair a channel's push buffer with its base page: the ring's first entry * carries the push-buffer VA, the base page was found by content and its * release names the channel's slot (4 bytes per rank). @@ -17801,7 +19367,11 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) u64 hit_pa = 0, va = 0; u32 len = 0, i, hit_i = 0, want, off; - if (!ch->engine_type || ch->ce_index >= 32) + if (!ch->engine_type) { + svc_pb_pair_gr(ch); + return; + } + if (ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_pb_done)) return; @@ -27230,6 +28800,33 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_notify2d_keepalive(vgpu, ch); /* + * SESSION 149 R15 (M1.1b): GR RAM-scan keep-alive. The boot + * budget can retire before the client creates its ring (r886: + * 16 slices spent, no pairing). A GPFIFO put advance proves + * the client is live NOW, so grant a fresh bounded budget - + * capped and spaced so an unfound ring cannot keep the scan + * hot. The scan itself stops on its own once the ring is + * content-validated (gr_ram_ring). + */ + if (!ch->engine_type && ch->h_object != 0x31415900u && + ch->gr_index < 32 && + !test_bit(ch->gr_index, &svc_gr_ring_done) && + ch->gr_ram_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_refreshes < SVC_GR_RAM_REFRESH_MAX && + time_after_eq(jiffies, ch->gr_ram_refresh_at) && + ch->gr_ram_kick != put) { + ch->gr_ram_kick = put; + ch->gr_ram_refreshes++; + ch->gr_ram_refresh_at = jiffies + + msecs_to_jiffies(SVC_GR_RAM_REFRESH_MS); + ch->gr_ram_passes = 0; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan keep-alive %u/%u (put=%u moved; budget reset)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_refreshes, SVC_GR_RAM_REFRESH_MAX, + put); + } + + /* * ISSUE-0002 diag: put snapshots for CE channels, bounded per * session. The throttle is indexed by rank (the channel struct * iterated here is a local copy, so a per-channel stamp never @@ -27284,8 +28881,96 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); - if (put >= ch->gpfifo_entries) + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation for + * the RAM USERD. The scan latches a gated candidate right + * away (fast walk start); the pump is the judge: a +1/+2 + * step of the block's PUT confirms it as the client's live + * control block, while a jump, a reset or no movement within + * the bound rejects it so the scan can adopt a better + * candidate. The live block advances 1->2->3 in 2-10 s + * (measured); the client's own release waits are ~11 s, so + * the no-advance bound must sit above those. + */ + if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok && + !ch->gr_ram_userd_confirmed) { + if (!ch->gr_ram_userd_latch_at) + ch->gr_ram_userd_latch_at = jiffies; + if (put == ch->gr_ram_userd_latch_put + 1 || + put == ch->gr_ram_userd_latch_put + 2) { + ch->gr_ram_userd_confirmed = true; + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", + ch->h_object, ch->userd_pa, + ch->gr_ram_userd_latch_put, put); + } else if (put != ch->gr_ram_userd_latch_put || + time_after(jiffies, + ch->gr_ram_userd_latch_at + + msecs_to_jiffies(15000))) { + unsigned long lflags; + int li; + + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch REJECTED (put=%u latch=%u%s)\n", + ch->h_object, ch->userd_pa, put, + ch->gr_ram_userd_latch_put, + put != ch->gr_ram_userd_latch_put ? + "" : " no advance in 15 s"); + ch->userd_ok = false; + ch->userd_pa = 0; + ch->userd_as = 0; + ch->gr_ram_userd_latch_at = 0; + ch->gr_ram_userd_confirmed = false; + ch->gr_ram_put_seeded = 0; + ch->last_get = 0; + /* + * Persist immediately: the pump `continue`s + * here, skipping the end-of-pass write-back + * for this channel. + */ + spin_lock_irqsave(&svc.chan_lock, lflags); + for (li = 0; li < SVC_MAX_CHANNELS; li++) + if (svc.chan[li].valid && + svc.chan[li].vgpu == vgpu && + svc.chan[li].h_object == + ch->h_object) { + svc.chan[li].userd_ok = false; + svc.chan[li].userd_pa = 0; + svc.chan[li].userd_as = 0; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, lflags); + continue; + } + } + if (put >= ch->gpfifo_entries) { + u32 put_raw = put; + put %= ch->gpfifo_entries; + /* + * SESSION 154 (M-GL1.2): a free-running producer (the + * GL client's control block counts past any small ring + * - put=12 with entries=2, measured) can collapse + * onto the pump's own position by the modulo: an even + * number of positions ahead and last_get == put, so + * the walk consumes NOTHING and the client's release + * is never applied (r914: submissions stayed 0 while + * the client retried for minutes). One-shot per put + * value: step the walk back one slot so it consumes + * the newest entry; normal slot-space tracking + * resumes from there and the next put value walks + * on. + */ + if (ch->gpfifo_entries > 1 && + put == (u32)ch->last_get && + put_raw != ch->gr_ram_put_seeded) { + ch->gr_ram_put_seeded = put_raw; + ch->last_get = (put + ch->gpfifo_entries - 1) % + ch->gpfifo_entries; + pr_info_ratelimited("sema-chan: GR free-running put h=0x%x put=%u get=%u ent=%u -> walk seed last=%u (modulo collapse consumed nothing)\n", + ch->h_object, put_raw, get, + ch->gpfifo_entries, + (u32)ch->last_get); + } + } /* * A wrapped put means every slot may hold a new entry: clear @@ -28006,6 +29691,47 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * SESSION 154 (M-GL1.2): a GR RAM-ring + * client push whose every primary + * source failed (r913: a validated pin + * kept answering for a page the guest + * reused and the wrong-VASPACE walk + * answer was implausible). The + * client's submit pushes carry stable + * content anchors, so sweep RAM for + * the page, validate it, pin it and + * re-read the entry through the fresh + * pin. When the anchor has not found + * the page yet, drop a stale validated + * pin (the read that failed came from + * src=2) so later reads prefer the + * walk and the anchor over it. + */ + if (unreadable && !ch->engine_type && + ch->gr_ram_ring && va && len) { + if (svc_gr_anchor_scan(vgpu, ch, + va, len) && + !svc_read_va_push(vgpu, ch, + va, pb, + len) && + svc_method_stream_plausible(pb, + len)) { + ch->rescue_last = 0; + ch->push_hold_until = 0; + svc.gpfifo_rescued++; + unreadable = false; + } else if (svc_read_last_src == 2) { + svc_pin_drop(va); + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx stale validated pin dropped (content failed every source)\n", + ch->h_object, + va); + } + } + } + + /* * r715 pushdec witness: name the branch * the entry takes straight after the * read, before the rescue/hold paths. @@ -28179,13 +29905,57 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) pr_info("vgsp-gr: GR rehold capture idx=%llu va=0x%llx\n", (unsigned long long)idx, va); } - if (!cap_trust && !entry_replay && + /* + * SESSION 149 R15 + * (M1.1a): a rewind + * replay whose slot + * identity AND content + * hash both match the + * recorded pass is the + * same push re-read + * (entry words alone + * cannot tell it from a + * rewritten push - the + * hash can, and the + * r354 checks above + * cleared entry_replay + * on every mismatch, so + * a surviving replay + * plus the explicit hash + * compare here is the + * verified case). + * Admit it as trusted so + * the GL section / + * synthesis capture can + * still be made when the + * first pass never + * captured (the r883 + * one-shot setup push + * decoded exactly once, + * as a replay, and the + * old suppression threw + * that single chance + * away). Ops stay + * suppressed: + * entry_replay keeps + * them off at the + * decode call, and the + * capture sites hold + * their own exactly-once + * guards. + */ + if (!cap_trust && !ch->engine_type && gsp_phys_gr_forward_enabled() && !unreadable && svc_method_stream_plausible(pb, - len)) - cap_trust = true; + len)) { + if (!entry_replay || + (entry_ident && + ch->slot_ops_hash[idx] == + svc_push_hash(pb, len))) + cap_trust = true; + } svc_entry_outcome("decode", ch, idx, va, len); @@ -29779,6 +31549,51 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; svc.chan[k].gr_ram_passes = ch->gr_ram_passes; svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + /* r912: the guest-RAM USERD scan cursor (also + * persisted immediately inside the scan - the + * stalled path skips this write-back). */ + svc.chan[k].gr_ram_userd_cursor = + ch->gr_ram_userd_cursor; + svc.chan[k].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[k].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + /* SESSION 154 (M-GL1.2): liveness watch + the + * free-running-put seed + the push anchor + * cursor all survive the snapshot copy. */ + { + u32 wj; + + svc.chan[k].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + for (wj = 0; wj < SVC_GR_USERD_WATCH; + wj++) { + svc.chan[k].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[k].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[k].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + } + } + svc.chan[k].gr_ram_put_seeded = + ch->gr_ram_put_seeded; + svc.chan[k].gr_ram_userd_latch_put = + ch->gr_ram_userd_latch_put; + svc.chan[k].gr_ram_userd_latch_at = + ch->gr_ram_userd_latch_at; + svc.chan[k].gr_ram_userd_confirmed = + ch->gr_ram_userd_confirmed; + svc.chan[k].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[k].gr_anchor_va = ch->gr_anchor_va; + svc.chan[k].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[k].gr_anchor_logged = ch->gr_anchor_logged; + /* SESSION 149 R15: M1.1b keep-alive state. */ + svc.chan[k].gr_ram_kick = ch->gr_ram_kick; + svc.chan[k].gr_ram_refreshes = + ch->gr_ram_refreshes; + svc.chan[k].gr_ram_refresh_at = + ch->gr_ram_refresh_at; svc.chan[k].gr_ram_ring = ch->gr_ram_ring; svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].notify2d_va = ch->notify2d_va; @@ -30532,6 +32347,9 @@ static struct task_struct *svc_xw_task; static int svc_xw_mirror_loop(void *unused) { + u32 orig_top = 0; + u32 ticks = 0; + while (!kthread_should_stop()) { struct nv_vgpu_device *d, *vgpu = NULL; u32 c = 0, cur = 0; @@ -30551,28 +32369,180 @@ static int svc_xw_mirror_loop(void *unused) &cur, sizeof(cur))) { /* * SESSION 148 (r867): member = counter (dword index, - * high bits preserved). The r867 "one learned step - * ahead" experiment produced a PERMANENT +0x70 - * mismatch at every check (member re-advances right - * after each counter change, so the driver's == test - * can never land: the xorg.log shows - * "(2, ..., member_ahead, counter)" timeouts and the - * GL client gets nothing). The plain tracking law is + * high bits preserved). The plain tracking law is * the correct one: the driver's re-check cadence - * (~3 ms-scale re-probes observed as 3 s "(0)" success - * prints) gives the mirror room to settle on the new - * counter between the driver's increment and its - * first probe. + * (~3 ms-scale re-probes) gives the mirror room to + * settle on the new counter between the driver's + * increment and its first probe. + * + * SESSION 148 r873 page-reuse guard: the word page + * is only ours while the driver's progress word + * lives there. When the page is freed/reused (X + * server died, batch BO recycled) its content + * changes and the base bits (bits 16..23 = 0x10 or + * 0x14) vanish. r872 evidence: the mirror kept + * writing 0x26d into a page that had become another + * process's PTE table (guest "Bad page map ... + * pmd:102656067" pointing at exactly the mirrored + * page's PA), corrupting unrelated guest memory. + * NEVER write once the base bits have changed; stop + * the thread so nothing more touches the page. */ - u32 m = (cur & 0xfffc0000u) | - ((c >> 2) & 0x3ffffu); + u32 top = cur & 0xfffc0000u; + + if (!orig_top) { + if (!top) { + /* not a progress word: write nothing */ + msleep(3); + continue; + } + orig_top = top; + } + if (top != orig_top) { + pr_warn("sema-xw: mirror STOP: word page 0x%llx repurposed (top 0x%x -> 0x%x)\n", + (unsigned long long)svc_xw_mem_pa, + orig_top, cur); + break; + } + if (!(++ticks & 63)) { + /* counter-side guard: channel signature */ + u64 sig = 0; + + if (svc_read_ap(vgpu, svc_xw_cnt_pa - 0x50, + SVC_APERTURE_SYS, &sig, + sizeof(sig)) || + sig != 0x0100001300000001ULL) { + pr_warn("sema-xw: mirror STOP: counter object 0x%llx repurposed (sig=0x%llx)\n", + (unsigned long long)svc_xw_cnt_pa, + (unsigned long long)sig); + break; + } + } + { + u32 m = top | ((c >> 2) & 0x3ffffu); + u16 st = 0; - if (m != cur) - svc_write_ap(vgpu, svc_xw_mem_pa, - SVC_APERTURE_SYS, &m, sizeof(m)); + if (m != cur) + svc_write_ap(vgpu, svc_xw_mem_pa, + SVC_APERTURE_SYS, &m, + sizeof(m)); + /* + * SESSION 148 r873: submit-kick completion. + * The X submit path arms member status + * 0x8000 at value_word+0xE (gdb-proven layout: + * value @member+0, status @member+0xE) and + * polls until it reads 0; nothing else ever + * clears it, so each kick paid the driver's + * full timeout. Clear it here, right after + * the value write, so the poll passes ms + * after the frame is submitted. + */ + if (!svc_read_ap(vgpu, svc_xw_mem_pa + 0xE, + SVC_APERTURE_SYS, &st, + sizeof(st)) && + st == 0x8000) { + u16 z = 0; + + svc_write_ap(vgpu, svc_xw_mem_pa + 0xE, + SVC_APERTURE_SYS, &z, + sizeof(z)); + if (!(++ticks % 256)) + pr_info("sema-xw: mirror alive (clears=%u cnt=0x%x)\n", + ticks, c); + } + } } msleep(3); } + svc_xw_task = NULL; + return 0; +} + +/* + * X/GL submit-kick completion clearer (SESSION 148 r873). The X server's + * submit path (decoded at nvidia_drv dc170) arms a 16-byte member - + * u32 value at +0x0 (0 while pending) and u16 status 0x8000 at +0xE - and + * then polls dc330 until EVERY member status reads 0; nothing in the + * emulation ever writes that completion, so each kick paid the driver's + * full retry/timeout cycle. The member lives in a page whose page offset + * is 0xB0 (membase is page-aligned), so the status word is at page + * offset 0xBE. The finder in guest RAM reports that PA during an armed + * window; this thread clears status 0x8000 -> 0 (and only that exact + * pattern) so the kick poll passes within a few ms. Pure tuning aid - + * no product logic depends on it. + */ +static u64 svc_xw_comp_pa; /* PA of the u16 status word (page off 0xBE) */ +static struct task_struct *svc_xw_comp_task; + +static int svc_xw_comp_loop(void *unused) +{ + u64 n = 0; + + while (!kthread_should_stop()) { + struct nv_vgpu_device *d, *vgpu = NULL; + u32 val = 1; + u16 st = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu && + !svc_read_ap(vgpu, svc_xw_comp_pa - 0xE, + SVC_APERTURE_SYS, &val, sizeof(val)) && + !svc_read_ap(vgpu, svc_xw_comp_pa, SVC_APERTURE_SYS, + &st, sizeof(st))) { + /* + * Clear ONLY the exact pending pattern (value still + * 0, status 0x8000). Anything else is either a + * completed member (status 0) or a repurposed page - + * leave it alone. + */ + if (st == 0x8000 && val == 0) { + u16 z = 0; + + svc_write_ap(vgpu, svc_xw_comp_pa, + SVC_APERTURE_SYS, &z, sizeof(z)); + if (!(++n % 25)) + pr_info("sema-xw: comp clears=%llu\n", + (unsigned long long)n); + } + } + msleep(2); + } + svc_xw_comp_task = NULL; + return 0; +} + +int gsp_sema_service_xw_comp(u64 comp_pa) +{ + if (!comp_pa) { + if (svc_xw_comp_task) { + kthread_stop(svc_xw_comp_task); + svc_xw_comp_task = NULL; + pr_info("sema-xw: comp stopped\n"); + } + return 0; + } + if (comp_pa & 1) + return -EINVAL; + svc_xw_comp_pa = comp_pa; + if (!svc_xw_comp_task) { + svc_xw_comp_task = kthread_run(svc_xw_comp_loop, NULL, + "vgsp-xwcomp"); + if (IS_ERR(svc_xw_comp_task)) { + int err = PTR_ERR(svc_xw_comp_task); + + svc_xw_comp_task = NULL; + return err; + } + } + pr_info("sema-xw: comp status_pa=0x%llx running\n", + (unsigned long long)comp_pa); return 0; } diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index e96ce1c..69d6f33 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -134,6 +134,7 @@ int gsp_sema_service_dump_channels(void); int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap); int gsp_sema_service_dump_vidmem(u64 addr, u64 len); int gsp_sema_service_xw_mirror(u64 cnt_pa, u64 mem_pa); +int gsp_sema_service_xw_comp(u64 comp_pa); int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits); int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa); int gsp_sema_service_chan_rewind(u32 h_object); diff --git a/kernel/main.c b/kernel/main.c index 0e9452b..509c560 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -3129,6 +3129,30 @@ module_param_cb(sema_xw_mirror, &nv_vgpu_sema_xw_mirror_ops, NULL, 0200); MODULE_PARM_DESC(sema_xw_mirror, "Mirror X frame counter (cnt_pa,mem_pa; 0,0=stop)"); /* + * X/GL submit-kick completion clearer (SESSION 148 r873): PA of the u16 + * status word (member page offset 0xBE) that the X submit path arms to + * 0x8000 and polls until 0. Cleared host-side so the kick settles in ms. + * echo 0x1123456be > .../parameters/sema_xw_comp + * echo 0 > .../parameters/sema_xw_comp (stop) + */ +static int nv_vgpu_sema_xw_comp_set(const char *val, + const struct kernel_param *kp) +{ + u64 pa = 0; + + if (kstrtoull(val, 0, &pa)) + return -EINVAL; + return gsp_sema_service_xw_comp(pa); +} + +static const struct kernel_param_ops nv_vgpu_sema_xw_comp_ops = { + .set = nv_vgpu_sema_xw_comp_set, +}; + +module_param_cb(sema_xw_comp, &nv_vgpu_sema_xw_comp_ops, NULL, 0200); +MODULE_PARM_DESC(sema_xw_comp, "Clear X submit member status at PA (0=stop)"); + +/* * Debug: search the whole emulated vidmem for an 8-byte pattern: * echo 0x0000000400000001,16 > .../parameters/sema_vidmem_find */ |