diff options
Diffstat (limited to 'tools/guest-diag')
106 files changed, 10894 insertions, 178 deletions
diff --git a/tools/guest-diag/egl_frame.c b/tools/guest-diag/egl_frame.c new file mode 100644 index 0000000..ed10e39 --- /dev/null +++ b/tools/guest-diag/egl_frame.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * egl_frame.c - minimal EGL/GLES2 offscreen render test for the vGPU guest. + * + * Tries, in order: the EGL_EXT_platform_device platform with each enumerated + * EGLDeviceEXT, EGL_MESA_platform_surfaceless, then EGL_DEFAULT_DISPLAY. + * For the first display that initializes it creates an offscreen pbuffer + * context, renders colour bands into an FBO, reads them back and writes a + * PPM. One greppable line per stage. + * + * Build (guest): gcc -O2 -o egl_frame egl_frame.c -lEGL -lGLESv2 + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <EGL/egl.h> +#include <EGL/eglext.h> +#include <GLES2/gl2.h> + +#define W 320 +#define H 240 + +static EGLDisplay try_display(const char *name, EGLDisplay dpy, + const char *dev_name) +{ + EGLint major = 0, minor = 0; + + if (dpy == EGL_NO_DISPLAY) { + printf("EGL_PLATFORM %s -> no display\n", name); + return EGL_NO_DISPLAY; + } + if (!eglInitialize(dpy, &major, &minor)) { + printf("EGL_PLATFORM %s -> init fail 0x%x\n", name, + eglGetError()); + return EGL_NO_DISPLAY; + } + printf("EGL_PLATFORM %s ok %d.%d vendor=%s%s%s\n", name, major, minor, + eglQueryString(dpy, EGL_VENDOR), + dev_name ? " device=" : "", dev_name ? dev_name : ""); + return dpy; +} + +static EGLDisplay get_display(void) +{ + PFNEGLGETPLATFORMDISPLAYEXTPROC gp = + (PFNEGLGETPLATFORMDISPLAYEXTPROC)eglGetProcAddress( + "eglGetPlatformDisplayEXT"); + PFNEGLQUERYDEVICESEXTPROC qd = + (PFNEGLQUERYDEVICESEXTPROC)eglGetProcAddress( + "eglQueryDevicesEXT"); + EGLDisplay dpy; + + if (gp && qd) { + EGLDeviceEXT devs[8]; + EGLint n = 0; + int i; + + if (qd(8, devs, &n) && n > 0) { + printf("EGL_DEVICES n=%d\n", n); + for (i = 0; i < n; i++) { + EGLDisplay d = gp(EGL_PLATFORM_DEVICE_EXT, + devs[i], NULL); + + dpy = try_display("device", d, "enum"); + if (dpy != EGL_NO_DISPLAY) + return dpy; + } + } else { + printf("EGL_DEVICES n=0\n"); + } + if (gp) { + dpy = try_display("surfaceless", + gp(EGL_PLATFORM_SURFACELESS_MESA, + NULL, NULL), NULL); + if (dpy != EGL_NO_DISPLAY) + return dpy; + } + } + return try_display("default", eglGetDisplay(EGL_DEFAULT_DISPLAY), + NULL); +} + +static int render(EGLDisplay dpy) +{ + EGLConfig cfg; + EGLint ncfg = 0; + EGLSurface surf; + EGLContext ctx; + GLuint fbo = 0, tex = 0; + unsigned char *px; + FILE *f; + int i; + EGLint attrs[] = { + EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, EGL_ALPHA_SIZE, 8, + EGL_NONE + }; + EGLint pattrs[] = { EGL_WIDTH, W, EGL_HEIGHT, H, EGL_NONE }; + EGLint cattrs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE }; + + if (!eglChooseConfig(dpy, attrs, &cfg, 1, &ncfg) || ncfg < 1) { + printf("EGL_CONFIG_FAIL err=0x%x\n", eglGetError()); + return 1; + } + surf = eglCreatePbufferSurface(dpy, cfg, pattrs); + if (surf == EGL_NO_SURFACE) { + printf("EGL_PBUFFER_FAIL err=0x%x\n", eglGetError()); + return 1; + } + ctx = eglCreateContext(dpy, cfg, EGL_NO_CONTEXT, cattrs); + if (ctx == EGL_NO_CONTEXT) { + printf("EGL_CONTEXT_FAIL err=0x%x\n", eglGetError()); + return 1; + } + if (!eglMakeCurrent(dpy, surf, surf, ctx)) { + printf("EGL_CURRENT_FAIL err=0x%x\n", eglGetError()); + return 1; + } + printf("GL_RENDERER %s\n", glGetString(GL_RENDERER)); + printf("GL_VERSION %s\n", glGetString(GL_VERSION)); + + glGenTextures(1, &tex); + glBindTexture(GL_TEXTURE_2D, tex); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, W, H, 0, GL_RGBA, + GL_UNSIGNED_BYTE, NULL); + glGenFramebuffers(1, &fbo); + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, tex, 0); + if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != + GL_FRAMEBUFFER_COMPLETE) { + printf("GL_FBO_FAIL\n"); + return 1; + } + for (i = 0; i < 16; i++) { + glEnable(GL_SCISSOR_TEST); + glScissor(0, i * H / 16, W, H / 16); + glClearColor((float)i / 16.0f, 1.0f - (float)i / 16.0f, + 0.5f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + } + glFinish(); + printf("GL_ERR 0x%x\n", glGetError()); + px = (unsigned char *)malloc(W * H * 4); + glReadPixels(0, 0, W, H, GL_RGBA, GL_UNSIGNED_BYTE, px); + { + const char *out = getenv("EGL_FRAME_OUT"); + + if (!out) + out = "/tmp/egl_frame.ppm"; + f = fopen(out, "wb"); + if (!f) { + printf("PPM_OPEN_FAIL\n"); + return 1; + } + fprintf(f, "P6\n%d %d\n255\n", W, H); + for (i = 0; i < W * H; i++) + fwrite(px + i * 4, 1, 3, f); + fclose(f); + printf("EGL_FRAME_WRITTEN %s\n", out); + } + printf("EGL_FRAME_OK\n"); + return 0; +} + +int main(void) +{ + EGLDisplay dpy; + const char *ext; + + setvbuf(stdout, NULL, _IONBF, 0); + ext = eglQueryString(EGL_NO_DISPLAY, EGL_EXTENSIONS); + printf("EGL_CLIENT_EXT %s\n", ext ? ext : "(none)"); + dpy = get_display(); + if (dpy == EGL_NO_DISPLAY) { + printf("EGL_ALL_PLATFORMS_FAILED\n"); + return 1; + } + return render(dpy); +} diff --git a/tools/guest-diag/egl_retry.c b/tools/guest-diag/egl_retry.c new file mode 100644 index 0000000..ce45d45 --- /dev/null +++ b/tools/guest-diag/egl_retry.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * egl_retry.c - EGL/GLES2 offscreen probe with an explicit eglMakeCurrent + * retry loop. The r581-era trace showed the glsi engine-init returning + * 0xee00001 ("busy") exactly once and setting the device state bit 0x20; + * eglcore does not retry. This probe retries the make-current (and the whole + * display/context creation) up to N times, printing the precise EGL error of + * every attempt, so we can tell "one-shot busy, retry succeeds" from + * "permanent failure". + * + * Build (guest): gcc -O2 -o egl_retry egl_retry.c -lEGL -lGLESv2 + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <EGL/egl.h> +#include <EGL/eglext.h> +#include <GLES2/gl2.h> + +static EGLDisplay get_device_display(void) +{ + PFNEGLGETPLATFORMDISPLAYEXTPROC gp = + (PFNEGLGETPLATFORMDISPLAYEXTPROC)eglGetProcAddress( + "eglGetPlatformDisplayEXT"); + PFNEGLQUERYDEVICESEXTPROC qd = + (PFNEGLQUERYDEVICESEXTPROC)eglGetProcAddress( + "eglQueryDevicesEXT"); + EGLDeviceEXT devs[8]; + EGLint n = 0; + + if (!gp || !qd || !qd(8, devs, &n) || n <= 0) + return EGL_NO_DISPLAY; + printf("EGL_DEVICES n=%d\n", n); + return gp(EGL_PLATFORM_DEVICE_EXT, devs[0], NULL); +} + +int main(int argc, char **argv) +{ + int tries = argc > 1 ? atoi(argv[1]) : 10; + EGLDisplay dpy; + EGLint major = 0, minor = 0; + EGLConfig cfg; + EGLint ncfg = 0; + EGLSurface surf; + EGLContext ctx = EGL_NO_CONTEXT; + EGLint attrs[] = { + EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, EGL_NONE + }; + EGLint pb[] = { EGL_WIDTH, 64, EGL_HEIGHT, 64, EGL_NONE }; + EGLint cattr[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE }; + int i; + + setvbuf(stdout, NULL, _IONBF, 0); + + dpy = get_device_display(); + if (dpy == EGL_NO_DISPLAY) { + printf("NO_DEVICE_DISPLAY\n"); + return 1; + } + if (!eglInitialize(dpy, &major, &minor)) { + printf("EGL_INIT_FAIL 0x%x\n", eglGetError()); + return 1; + } + printf("EGL_INIT ok %d.%d vendor=%s\n", major, minor, + eglQueryString(dpy, EGL_VENDOR)); + if (!eglBindAPI(EGL_OPENGL_ES_API)) { + printf("BIND_API_FAIL 0x%x\n", eglGetError()); + return 1; + } + if (!eglChooseConfig(dpy, attrs, &cfg, 1, &ncfg) || ncfg < 1) { + printf("CHOOSE_CONFIG_FAIL 0x%x n=%d\n", eglGetError(), ncfg); + return 1; + } + for (i = 0; i < tries; i++) { + surf = eglCreatePbufferSurface(dpy, cfg, pb); + if (surf == EGL_NO_SURFACE) { + printf("TRY %d CREATE_SURFACE_FAIL 0x%x\n", i + 1, + eglGetError()); + usleep(200000); + continue; + } + if (ctx == EGL_NO_CONTEXT) { + ctx = eglCreateContext(dpy, cfg, EGL_NO_CONTEXT, + cattr); + if (ctx == EGL_NO_CONTEXT) { + printf("TRY %d CREATE_CTX_FAIL 0x%x\n", i + 1, + eglGetError()); + eglDestroySurface(dpy, surf); + usleep(200000); + continue; + } + } + if (eglMakeCurrent(dpy, surf, surf, ctx)) { + const char *rend = (const char *)glGetString( + GL_RENDERER); + const char *ver = (const char *)glGetString( + GL_VERSION); + printf("TRY %d CURRENT OK renderer=%s version=%s\n", + i + 1, rend ? rend : "?", ver ? ver : "?"); + /* draw one clear + readback so the pipeline runs */ + glClearColor(0.1f, 0.4f, 0.9f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + { + unsigned char px[4]; + glReadPixels(0, 0, 1, 1, GL_RGBA, + GL_UNSIGNED_BYTE, px); + printf("PIXEL %02x%02x%02x%02x\n", px[0], + px[1], px[2], px[3]); + } + printf("EGL_RETRY_OK on try %d\n", i + 1); + return 0; + } + printf("TRY %d MAKE_CURRENT_FAIL 0x%x\n", i + 1, + eglGetError()); + eglDestroySurface(dpy, surf); + usleep(200000); + } + printf("EGL_RETRY_EXHAUSTED tries=%d\n", tries); + return 1; +}
\ No newline at end of file diff --git a/tools/guest-diag/fbcopy_pub.c b/tools/guest-diag/fbcopy_pub.c new file mode 100644 index 0000000..a064882 --- /dev/null +++ b/tools/guest-diag/fbcopy_pub.c @@ -0,0 +1,718 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * fbcopy_pub.c - capture the vGPU framebuffer WITHOUT X and publish frames + * into the kvmfr ivshmem BAR. + * + * The X server's screen has 1280x720 pixels in the emulated VRAM, which the + * guest maps as the vGPU's BAR1 (resource1). The X SHM readback path can + * stall behind the X server's own push wait; this tool reads the framebuffer + * pixels straight out of the BAR1 mapping instead (host RAM backing, no GPU + * or X round-trip), swizzles BGRA -> RGBA and writes the kvmfr frame + * protocol the host reader already decodes. + * + * Usage (guest, as root): + * ./fbcopy_pub test one solid red frame (pipeline check) + * ./fbcopy_pub find [passes] [ms] motion-scan first 64 MB of BAR1 + * ./fbcopy_pub findp [off] [sz] [p] [ms] + * pread motion-scan of any BAR1 range + * (mmap only reaches the first 64 MB) + * ./fbcopy_pub pub <off> [n] [fps] [w] [h] [pitch] + * copy w*h from BAR1@off (pitch), + * mmap path (off < 64 MB) + * ./fbcopy_pub pubp <off> [n] [fps] [w] [h] [pitch] + * same but pread path (any offset) + * + * Find/build on the host (same glibc as the guest): gcc -O2 -o fbcopy_pub + * fbcopy_pub.c + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <dirent.h> +#include <errno.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <time.h> + +#define FRAME_OFFSET 0x100000UL +#define HDR_MAGIC 0x4b564d46u /* "FMVK" */ +#define DEF_W 1280 +#define DEF_H 720 +#define DEF_PITCH 5120 /* xorg-vgpu.conf: 1280*4 rounded */ + +struct frame_hdr { + uint32_t magic; + uint32_t version; + uint32_t width; + uint32_t height; + uint32_t stride; + uint32_t format; /* 0 = RGBA8 */ + uint32_t seq; + uint32_t reserved; + uint64_t timestamp_ns; +}; + +static uint64_t now_ns(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + ts.tv_nsec; +} + +static const char *find_vgpu_resource1(void) +{ + static char path[320]; + DIR *d = opendir("/sys/bus/pci/devices"); + struct dirent *de; + + if (!d) + return NULL; + while ((de = readdir(d))) { + char p[340], buf[32]; + FILE *f; + int n; + + if (de->d_name[0] == '.') + continue; + snprintf(p, sizeof(p), "/sys/bus/pci/devices/%s/vendor", + de->d_name); + f = fopen(p, "r"); + if (!f) + continue; + n = fread(buf, 1, sizeof(buf) - 1, f); + fclose(f); + if (n <= 0) + continue; + buf[n] = 0; + if (strtoul(buf, NULL, 16) != 0x10de) + continue; + snprintf(path, sizeof(path), + "/sys/bus/pci/devices/%s/resource1", de->d_name); + closedir(d); + return path; + } + closedir(d); + return NULL; +} + +/* + * The vfio mdev region mmap is limited: only file offset 0 with len <= 64 MB + * works (nonzero offsets and 256 MB both EINVAL). Map the first 64 MB once at + * offset 0 and index into the mapping for every access. + */ +#define CHUNK_MAX (64UL << 20) + +static unsigned char *map_vram(int fd, size_t want, size_t *got) +{ + size_t len = want > CHUNK_MAX ? CHUNK_MAX : want; + unsigned char *p; + + p = mmap(NULL, len, PROT_READ, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) + return NULL; + *got = len; + return p; +} + +static double t_ms(const struct timespec *a, const struct timespec *b) +{ + return (double)(b->tv_sec - a->tv_sec) * 1e3 + + (double)(b->tv_nsec - a->tv_nsec) / 1e6; +} + +/* + * Find the largest contiguous run of "dense" pages (>= 64 of 256 sampled + * bytes nonzero) - an image in VRAM looks like one long dense run. Small + * gaps (<= 16 pages) are tolerated. + */ +static void report_cluster(const uint32_t *nz, size_t pages, + unsigned long long base_off, const char *tag) +{ + size_t p, run_start = 0, last_dense = 0, best_start = 0, best_len = 0; + int in_run = 0; + + for (p = 0; p <= pages; p++) { + int dense = p < pages && nz[p] >= 64; + + if (dense) { + if (!in_run) { + run_start = p; + in_run = 1; + } + last_dense = p; + } else if (in_run && p - last_dense > 16) { + size_t len = last_dense - run_start + 1; + + if (len > best_len) { + best_len = len; + best_start = run_start; + } + in_run = 0; + } + } + if (best_len >= 8) + printf("%s_CLUSTER off=0x%llx pages=%zu bytes=%zu\n", tag, + base_off + ((unsigned long long)best_start << 12), + best_len, best_len << 12); +} + +int main(int argc, char **argv) +{ + const char *mode = argc > 1 ? argv[1] : "find"; + const char *bar_path = getenv("FBCOPY_BAR") ? getenv("FBCOPY_BAR") : + "/sys/bus/pci/devices/0000:00:03.0/resource2"; + const char *res1 = getenv("FBCOPY_RES1") ? getenv("FBCOPY_RES1") : + find_vgpu_resource1(); + struct stat st; + size_t map_size, i; + unsigned char *bar; + struct frame_hdr *hdr; + volatile uint32_t *seqp; + int fd, vram_fd; + + setvbuf(stdout, NULL, _IONBF, 0); + if (!res1) { + printf("NO_VGPU\n"); + return 1; + } + printf("VGPU_RES1 %s\n", res1); + if (stat(res1, &st) || !st.st_size) { + printf("STAT_FAIL\n"); + return 1; + } + map_size = (size_t)st.st_size; + printf("VRAM_SIZE 0x%zx (%zu MB)\n", map_size, map_size >> 20); + vram_fd = open(res1, O_RDONLY | O_SYNC); + if (vram_fd < 0) { + perror(res1); + return 1; + } + + /* kvmfr BAR next: every mode needs it except find. */ + fd = open(bar_path, O_RDWR | O_SYNC); + if (fd < 0) { + perror(bar_path); + return 1; + } + bar = (unsigned char *)mmap(NULL, 128UL << 20, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + close(fd); + if (bar == MAP_FAILED) { + perror("mmap kvmfr"); + return 1; + } + hdr = (struct frame_hdr *)(bar + FRAME_OFFSET); + seqp = &hdr->seq; + + if (!strcmp(mode, "zero")) { + /* Reset the kvmfr header seq so a reader seeding from it will + * accept the next pub run's frame 1. */ + hdr->magic = HDR_MAGIC; + hdr->version = 1; + hdr->width = DEF_W; + hdr->height = DEF_H; + hdr->stride = DEF_W * 4; + hdr->format = 0; + hdr->timestamp_ns = now_ns(); + __sync_synchronize(); + *seqp = 0; + printf("SEQ_ZEROED\n"); + return 0; + } + + if (!strcmp(mode, "test")) { + unsigned char *dst = bar + FRAME_OFFSET + + sizeof(struct frame_hdr); + size_t n = (size_t)DEF_W * DEF_H * 4; + + hdr->magic = HDR_MAGIC; + hdr->version = 1; + hdr->width = DEF_W; + hdr->height = DEF_H; + hdr->stride = DEF_W * 4; + hdr->format = 0; + for (i = 0; i < n; i += 4) { + dst[i + 0] = (unsigned char)(argc > 2 ? atoi(argv[2]) + : 255); + dst[i + 1] = (unsigned char)(argc > 3 ? atoi(argv[3]) + : 0); + dst[i + 2] = (unsigned char)(argc > 4 ? atoi(argv[4]) + : 0); + dst[i + 3] = 0xff; + } + hdr->timestamp_ns = now_ns(); + __sync_synchronize(); + *seqp = 1; + printf("TEST_FRAME_PUBLISHED\n"); + return 0; + } + + if (!strcmp(mode, "find")) { + /* + * Motion + content scan of the first 64 MB of BAR1 (the only + * window the vfio mdev mmap accepts). Sample 256 B of every + * 4 KB page over `passes` passes `interval_ms` apart; score + * pages by total differing bytes (motion) and by nonzero + * density (content). Rendering can be slower than one frame + * per second, so the interval defaults to 3 s and is + * configurable: find [passes] [interval_ms] + */ + int passes = argc > 2 ? atoi(argv[2]) : 3; + int interval_ms = argc > 3 ? atoi(argv[3]) : 3000; + size_t scan_size = map_size > CHUNK_MAX ? CHUNK_MAX : map_size; + size_t pages = scan_size >> 12; + unsigned char *s0, *s1, *tmp, *w; + uint32_t *motion, *nz; + size_t p, wlen; + int top, pass; + + if (passes < 2) + passes = 2; + w = map_vram(vram_fd, scan_size, &wlen); + if (!w) { + printf("CHUNK_MAP_FAIL\n"); + return 1; + } + printf("SCAN_WINDOW 0x%zx pages=%zu passes=%d interval=%dms\n", + wlen, pages, passes, interval_ms); + s0 = malloc(pages * 256); + s1 = malloc(pages * 256); + motion = calloc(pages, sizeof(uint32_t)); + nz = calloc(pages, sizeof(uint32_t)); + if (!s0 || !s1 || !motion || !nz) + return 1; + for (p = 0; p < pages; p++) + memcpy(s0 + p * 256, w + (p << 12), 256); + for (pass = 1; pass < passes; pass++) { + struct timespec a, b; + uint64_t moved = 0; + + usleep((useconds_t)interval_ms * 1000); + clock_gettime(CLOCK_MONOTONIC, &a); + for (p = 0; p < pages; p++) + memcpy(s1 + p * 256, w + (p << 12), 256); + clock_gettime(CLOCK_MONOTONIC, &b); + for (p = 0; p < pages; p++) { + unsigned char *x = s0 + p * 256; + unsigned char *y = s1 + p * 256; + uint32_t d = 0; + int j; + + for (j = 0; j < 256; j++) + if (x[j] != y[j]) + d++; + motion[p] += d; + moved += d; + } + printf("PASS %d moved_bytes=%llu dt=%.0fms\n", + pass, (unsigned long long)moved, t_ms(&a, &b)); + tmp = s0; s0 = s1; s1 = tmp; + } + /* nonzero density of the last sample */ + for (p = 0; p < pages; p++) { + unsigned char *x = s0 + p * 256; + uint32_t c = 0; + int j; + + for (j = 0; j < 256; j++) + if (x[j]) + c++; + nz[p] = c; + } + report_cluster(nz, pages, 0, "FIND"); + /* per-16MB summary */ + for (i = 0; i + 4096 <= pages; i += 4096) { + uint64_t s = 0, c = 0; + size_t j; + + for (j = i; j < i + 4096; j++) { + s += motion[j]; + c += nz[j]; + } + if (s || c) + printf("BLOCK off=0x%zx moving_bytes=%llu nonzero_bytes=%llu\n", + i << 12, (unsigned long long)s, + (unsigned long long)c); + } + for (top = 0; top < 16; top++) { + size_t best = 0; + uint32_t bs = 0; + + for (p = 0; p < pages; p++) + if (motion[p] > bs) { + bs = motion[p]; + best = p; + } + if (!bs) + break; + printf("HOT off=0x%zx score=%u\n", best << 12, bs); + motion[best] = 0; + } + for (top = 0; top < 16; top++) { + size_t best = 0; + uint32_t bs = 0; + + for (p = 0; p < pages; p++) + if (nz[p] > bs) { + bs = nz[p]; + best = p; + } + if (!bs) + break; + printf("CONTENT off=0x%zx nonzero=%u\n", + best << 12, bs); + nz[best] = 0; + } + printf("FIND_DONE\n"); + return 0; + } + + if (!strcmp(mode, "probe")) { + /* + * Ground truth on what BAR1 access actually works: try mmap + * windows at several offsets/sizes and pread at several + * offsets, report each result. Read-only. + */ + struct { + unsigned long long off, len; + } mm[] = { + { 0, 64UL << 20 }, + { 0, 256UL << 20 }, + { 0, 4UL << 20 }, + { 4UL << 20, 4UL << 20 }, + { 64UL << 20, 64UL << 20 }, + { 128UL << 20, 4UL << 20 }, + { 192UL << 20, 4UL << 20 }, + { 60UL << 20, 8UL << 20 }, + }; + unsigned long long po[] = { + 0, 0x100000, 4UL << 20, 60UL << 20, + 64UL << 20, 128UL << 20, 252UL << 20, + }; + unsigned char buf[64]; + size_t k; + + for (k = 0; k < sizeof(mm) / sizeof(mm[0]); k++) { + unsigned char *p; + + if (mm[k].off + mm[k].len > map_size) { + printf("MMAP off=0x%llx len=0x%llx SKIP(>bar)\n", + mm[k].off, mm[k].len); + continue; + } + errno = 0; + p = mmap(NULL, (size_t)mm[k].len, PROT_READ, + MAP_SHARED, vram_fd, (off_t)mm[k].off); + if (p == MAP_FAILED) + printf("MMAP off=0x%llx len=0x%llx FAIL errno=%d\n", + mm[k].off, mm[k].len, errno); + else { + printf("MMAP off=0x%llx len=0x%llx OK first=%02x%02x%02x%02x\n", + mm[k].off, mm[k].len, p[0], p[1], p[2], p[3]); + munmap(p, (size_t)mm[k].len); + } + } + for (k = 0; k < sizeof(po) / sizeof(po[0]); k++) { + ssize_t r; + + errno = 0; + r = pread(vram_fd, buf, 16, (off_t)po[k]); + if (r != 16) + printf("PREAD off=0x%llx FAIL r=%zd errno=%d\n", + po[k], r, errno); + else + printf("PREAD off=0x%llx OK %02x%02x%02x%02x\n", + po[k], buf[0], buf[1], buf[2], buf[3]); + } + printf("PROBE_DONE\n"); + return 0; + } + + if (!strcmp(mode, "findp")) { + /* + * Same motion + content scan as `find`, but via pread so the + * whole 256 MB BAR is reachable (mmap stops at 64 MB). + * findp [start_off] [size] [passes] [interval_ms] + */ + unsigned long long start = argc > 2 ? + strtoull(argv[2], NULL, 0) : 0; + unsigned long long size = argc > 3 ? + strtoull(argv[3], NULL, 0) : map_size - start; + int passes = argc > 4 ? atoi(argv[4]) : 3; + int interval_ms = argc > 5 ? atoi(argv[5]) : 3000; + size_t pages = size >> 12; + unsigned char *s0, *s1, *tmp; + uint32_t *motion, *nz; + size_t p; + int top, pass; + + if (passes < 2) + passes = 2; + if (start + size > map_size) { + printf("RANGE_FAIL start=0x%llx size=0x%llx bar=0x%zx\n", + start, size, map_size); + return 1; + } + printf("SCANP start=0x%llx size=0x%llx pages=%zu passes=%d interval=%dms\n", + start, size, pages, passes, interval_ms); + s0 = malloc(pages * 256); + s1 = malloc(pages * 256); + motion = calloc(pages, sizeof(uint32_t)); + nz = calloc(pages, sizeof(uint32_t)); + if (!s0 || !s1 || !motion || !nz) + return 1; + for (pass = 0; pass < passes; pass++) { + struct timespec a, b; + uint64_t moved = 0; + + if (pass) + usleep((useconds_t)interval_ms * 1000); + clock_gettime(CLOCK_MONOTONIC, &a); + for (p = 0; p < pages; p++) { + unsigned char *dst = pass ? s1 : s0; + ssize_t r = pread(vram_fd, dst + p * 256, 256, + (off_t)(start + (p << 12))); + + if (r != 256) { + printf("PREAD_FAIL off=0x%llx r=%zd errno=%d\n", + start + (p << 12), r, errno); + return 1; + } + } + clock_gettime(CLOCK_MONOTONIC, &b); + if (pass) { + for (p = 0; p < pages; p++) { + unsigned char *x = s0 + p * 256; + unsigned char *y = s1 + p * 256; + uint32_t d = 0; + int j; + + for (j = 0; j < 256; j++) + if (x[j] != y[j]) + d++; + motion[p] += d; + moved += d; + } + printf("PASS %d moved_bytes=%llu dt=%.0fms\n", + pass, (unsigned long long)moved, + t_ms(&a, &b)); + tmp = s0; s0 = s1; s1 = tmp; + } else { + printf("PASS 0 sampled dt=%.0fms\n", + t_ms(&a, &b)); + } + } + for (p = 0; p < pages; p++) { + unsigned char *x = s0 + p * 256; + uint32_t c = 0; + int j; + + for (j = 0; j < 256; j++) + if (x[j]) + c++; + nz[p] = c; + } + report_cluster(nz, pages, start, "FINDP"); + for (i = 0; i + 4096 <= pages; i += 4096) { + uint64_t s = 0, c = 0; + size_t j; + + for (j = i; j < i + 4096; j++) { + s += motion[j]; + c += nz[j]; + } + if (s || c) + printf("BLOCK off=0x%llx moving_bytes=%llu nonzero_bytes=%llu\n", + start + ((unsigned long long)i << 12), + (unsigned long long)s, + (unsigned long long)c); + } + for (top = 0; top < 16; top++) { + size_t best = 0; + uint32_t bs = 0; + + for (p = 0; p < pages; p++) + if (motion[p] > bs) { + bs = motion[p]; + best = p; + } + if (!bs) + break; + printf("HOT off=0x%llx score=%u\n", + start + ((unsigned long long)best << 12), bs); + motion[best] = 0; + } + for (top = 0; top < 16; top++) { + size_t best = 0; + uint32_t bs = 0; + + for (p = 0; p < pages; p++) + if (nz[p] > bs) { + bs = nz[p]; + best = p; + } + if (!bs) + break; + printf("CONTENT off=0x%llx nonzero=%u\n", + start + ((unsigned long long)best << 12), bs); + nz[best] = 0; + } + printf("FINDP_DONE\n"); + return 0; + } + + if (!strcmp(mode, "pub")) { + unsigned long long off = argc > 2 ? + strtoull(argv[2], NULL, 0) : 0; + int frames = argc > 3 ? atoi(argv[3]) : 30; + int fps = argc > 4 ? atoi(argv[4]) : 5; + int w = argc > 5 ? atoi(argv[5]) : DEF_W; + int h = argc > 6 ? atoi(argv[6]) : DEF_H; + int pitch = argc > 7 ? atoi(argv[7]) : DEF_PITCH; + unsigned long long span = (unsigned long long)(h - 1) * pitch + + (unsigned long long)w * 4; + unsigned long long base = off & ~0xfffULL; + size_t wlen = (size_t)((off - base) + span); + unsigned char *wmap; + int f, y, x; + + if (off + span > map_size) { + printf("OFF_RANGE_FAIL need=0x%llx bar=0x%zx\n", + off + span, map_size); + return 1; + } + /* 2026-10-28 (session 149): nonzero-offset mmaps of BAR1 work + * on this mdev (earlier builds assumed only offset 0 <= 64 MB + * was mappable). Map directly at the requested offset; fall + * back to the old offset-0 64 MB window if that ever fails. */ + wlen = (wlen + 0xfff) & ~0xfffUL; + wmap = mmap(NULL, wlen, PROT_READ, MAP_SHARED, vram_fd, + (off_t)base); + if (wmap == MAP_FAILED) { + size_t got = 0; + wmap = map_vram(vram_fd, map_size, &got); + if (wmap) { + base = 0; + wlen = got; + } + if (!wmap || off + span > wlen) { + printf("WIN_MAP_FAIL off=0x%llx len=0x%zx errno=%d\n", + base, wlen, errno); + return 1; + } + } + printf("PUB off=0x%llx base=0x%llx len=0x%zx frames=%d fps=%d %dx%d pitch=%d\n", + off, base, wlen, frames, fps, w, h, pitch); + hdr->magic = HDR_MAGIC; + hdr->version = 1; + hdr->width = (uint32_t)w; + hdr->height = (uint32_t)h; + hdr->stride = (uint32_t)w * 4; + hdr->format = 0; + for (f = 0; f < frames; f++) { + uint64_t t0 = now_ns(); + unsigned char *dst = bar + FRAME_OFFSET + + sizeof(struct frame_hdr); + unsigned char *src = wmap + (size_t)(off - base); + + for (y = 0; y < h; y++) { + unsigned char *s = src + (size_t)y * pitch; + unsigned char *d = dst + (size_t)y * w * 4; + + for (x = 0; x < w; x++) { + d[4 * x + 0] = s[4 * x + 2]; + d[4 * x + 1] = s[4 * x + 1]; + d[4 * x + 2] = s[4 * x + 0]; + d[4 * x + 3] = 0xff; + } + } + hdr->timestamp_ns = now_ns(); + __sync_synchronize(); + *seqp = (uint32_t)(f + 1); + __sync_synchronize(); + printf("FRAME seq=%d ms=%.1f\n", f + 1, + (double)(now_ns() - t0) / 1e6); + if (fps > 0) + usleep((useconds_t)(1000000 / fps)); + } + printf("FRAME_DONE frames=%d\n", f); + return 0; + } + + if (!strcmp(mode, "pubp")) { + unsigned long long off = argc > 2 ? + strtoull(argv[2], NULL, 0) : 0; + int frames = argc > 3 ? atoi(argv[3]) : 30; + int fps = argc > 4 ? atoi(argv[4]) : 5; + int w = argc > 5 ? atoi(argv[5]) : DEF_W; + int h = argc > 6 ? atoi(argv[6]) : DEF_H; + int pitch = argc > 7 ? atoi(argv[7]) : DEF_PITCH; + unsigned long long span = (unsigned long long)(h - 1) * pitch + + (unsigned long long)w * 4; + unsigned long long base = off & ~0xfffULL; + size_t len = (size_t)((off - base) + span); + unsigned char *buf; + int f, y, x; + + len = (len + 0xfff) & ~0xfffUL; + if (off + span > map_size) { + printf("OFF_RANGE_FAIL need=0x%llx bar=0x%zx\n", + off + span, map_size); + return 1; + } + buf = malloc(len); + if (!buf) + return 1; + printf("PUBP off=0x%llx frames=%d fps=%d %dx%d pitch=%d len=0x%zx\n", + off, frames, fps, w, h, pitch, len); + hdr->magic = HDR_MAGIC; + hdr->version = 1; + hdr->width = (uint32_t)w; + hdr->height = (uint32_t)h; + hdr->stride = (uint32_t)w * 4; + hdr->format = 0; + for (f = 0; f < frames; f++) { + uint64_t t0 = now_ns(); + unsigned char *dst = bar + FRAME_OFFSET + + sizeof(struct frame_hdr); + unsigned char *src; + ssize_t r = pread(vram_fd, buf, len, (off_t)base); + + if (r != (ssize_t)len) { + printf("PREAD_FAIL off=0x%llx r=%zd errno=%d\n", + base, r, errno); + return 1; + } + src = buf + (size_t)(off - base); + for (y = 0; y < h; y++) { + unsigned char *s = src + (size_t)y * pitch; + unsigned char *d = dst + (size_t)y * w * 4; + + for (x = 0; x < w; x++) { + d[4 * x + 0] = s[4 * x + 2]; + d[4 * x + 1] = s[4 * x + 1]; + d[4 * x + 2] = s[4 * x + 0]; + d[4 * x + 3] = 0xff; + } + } + hdr->timestamp_ns = now_ns(); + __sync_synchronize(); + *seqp = (uint32_t)(f + 1); + __sync_synchronize(); + printf("FRAME seq=%d ms=%.1f\n", f + 1, + (double)(now_ns() - t0) / 1e6); + if (fps > 0) + usleep((useconds_t)(1000000 / fps)); + } + printf("FRAME_DONE frames=%d\n", f); + return 0; + } + + printf("USAGE: fbcopy_pub test|find|findp|pub|pubp " + "<off> [n] [fps] [w] [h] [pitch]\n"); + return 1; +} diff --git a/tools/guest-diag/fenceword_assist.py b/tools/guest-diag/fenceword_assist.py new file mode 100644 index 0000000..9e207e2 --- /dev/null +++ b/tools/guest-diag/fenceword_assist.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# fenceword_assist.py - keep the Vulkan fence wait word of a live vk_frame120 +# client released (session 125). +# +# SUPERSEDED (SESSION 126, commit 95e7f57e): the service now authors these +# releases itself. The `sema_gr_ahead` fence fold sweeps the channel's +# pre-built ring continuation beyond GP_PUT through the ring's validated +# GPFIFO translation, decodes the INC-5 RELEASE groups it finds (addr 0x5c +# count>=5) and queues them in the standard pending ledger; r717 evidence: +# `sema-ahead: h=0xbeec0100 put=11 swept=10 rels=4` followed 30 ms later by +# `sema-svc: deferred release va=0x45ba090 payload=1 (was cur=0)`, with +# VKFRAME seq/DONE and NO assist running (see +# backups/evidence-20261002-victory/r717-fencefold/). Keep this tool only as +# the A/B fallback for boots that predate the fold (or with +# sema_gr_ahead=0). +# +# The client's fence wait polls a word in a 16 KB /dev/nvidiactl BO mapped +# read-write into the client: the init wait word lives at +0x80 (the t122 +# "BW" chaser writes it once) and the per-submit fence word at +0x90 of the +# SAME BO (GPU VA <base>+0x90, delivered by the submit push's INC-5 semaphore +# group). While the emulation does not apply that group, this host assist +# writes 1 to the +0x90 slot whenever it reads 0 - the direct predecessor of +# the service-side fold (see Handoff SESSION 125/126). +# +# Usage (host, while a t122-driven client runs): +# tools/guest-diag/fenceword_assist.py [seconds] [guest-pid] +import os +import subprocess +import sys +import time + +VMEXEC = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "..", "scripts", "vgpu_vm.py")) +PY = r''' +import struct, sys, time +pids = [p for p in sys.argv[1:] if p] +maps_of = {} +def vma(pid, addr): + for line in open("/proc/%d/maps" % pid): + rng, perms = line.split(None, 2)[:2] + a, b = [int(x, 16) for x in rng.split("-")] + if a <= addr < b: + return line.strip() + return None +found = {} +for pid in pids: + try: + maps = [l for l in open("/proc/%d/maps" % pid) if "nvidia" in l and "rw-s" in l] + except OSError: + continue + for l in maps: + rng, perms, off = l.split(None, 3)[:3] + a, b = [int(x, 16) for x in rng.split("-")] + if b - a != 0x4000: + continue + try: + mem = open("/proc/%d/mem" % pid, "r+b", 0) + mem.seek(a + 0x80); init = mem.read(4) + mem.seek(a + 0x90); fw = mem.read(4) + except Exception: + continue + if init == b"\x01\x00\x00\x00" and fw == b"\x00\x00\x00\x00": + found[pid] = (a + 0x90, mem) + print("found pid=%s word=%#x" % (pid, a + 0x90), flush=True) +end = time.time() + float(sys.argv[1]) +while time.time() < end: + for pid, (w, mem) in list(found.items()): + try: + mem.seek(w) + if mem.read(4) == b"\x00\x00\x00\x00": + mem.seek(w); mem.write(struct.pack("<I", 1)); mem.flush() + print("released pid=%s word=%#x" % (pid, w), flush=True) + except Exception: + del found[pid] + time.sleep(0.5) +print("assist done", flush=True) +''' + + +def main(): + dur = sys.argv[1] if len(sys.argv) > 1 else "300" + pid = sys.argv[2] if len(sys.argv) > 2 else "" + b64 = __import__("base64").b64encode(PY.encode()).decode() + subprocess.run([VMEXEC, "exec", "--timeout", str(int(dur) + 60), + "echo %s | base64 -d > /tmp/fwa.py; sudo python3 /tmp/fwa.py %s %s" + % (b64, dur, pid)]) + + +if __name__ == "__main__": + main() diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index a33a183..8768f2f 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -266,11 +266,114 @@ def _vmalloc_pages_for(kc, va): return 0 -def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0): +def _nv_vmap_areas(kc): + """nv_vm_map_pages vmalloc areas: [(lo, hi, pages)] (5 s cache). + + Every CE push-buffer rm_mem is a vmap created by nv_vm_map_pages; the + demand pinner walks these areas to find the page holding a stalled + push VA by content, without needing any kprobe event or object chain. + """ + now = time.time() + if getattr(_nv_vmap_areas, "ts", 0) > now - 5.0: + return _nv_vmap_areas.areas + areas = [] + try: + for line in open("/proc/vmallocinfo"): + if "nv_vm_map_pages" not in line: + continue + p = line.split() + if len(p) < 2: + continue + try: + lo, hi = p[0].split("-") + lo = int(lo, 16) + hi = int(hi, 16) + except ValueError: + continue + pages = (hi - lo) >> 12 + if pages > 0: + areas.append((lo, hi, pages)) + except OSError: + pass + _nv_vmap_areas.areas = areas + _nv_vmap_areas.ts = now + return areas + + +# Learned page-offset per nv_vm_map_pages area for the demand content match +# (2026-10-16b): the first successful resolution records which page index the +# VA maps to, and later pushes in the same allocation retry that offset first +# instead of walking the whole +/-16 neighbourhood. +_area_off = {} + + +def resolve_va_by_content(kc, va, out): + """Demand pin: find the page carrying GPU VA 'va' by content match. + + The host reports the CE push-buffer VAs whose GPFIFO entries it cannot + read (dmesg 'stalled VA #N: 0x...'). The guest walks every + nv_vm_map_pages vmap area and tests the candidate leaf page at the VA's + in-window offset (plus a neighbourhood, for the per-mem header-page + offset ambiguity) against the host's method-stream prefix check. On a + match it emits the exact pair, so the host can translate and consume the + entry - no kprobe event, no object-chain discovery, no resolution + lottery. Only the requested VA is pinned, so a wrong candidate cannot + poison other VAs. + + 2026-10-16b: the offset search window was +/-1 page. Pushes of a client + whose pushbuffer vmap carries a larger per-allocation header (observed + with multi-MB CE staging: pushes at 0x120000f58+ never resolved and the + demand loop re-sent the same VA for minutes while the client crawled) + fall outside it. Widen to +/-16 pages and LEARN the winning offset per + vmap area, because consecutive pushes advance sequentially inside the + same allocation and share the offset. + """ + if not (PB_CE_LO <= va < PB_CE_HI): + return False + off = (va - PB_CE_LO) & 0xFFFFFF + k = off >> 12 + areas = _nv_vmap_areas(kc) + for (lo, _hi, pages) in areas: + learned = _area_off.get(lo) + if learned is not None: + adjs = [learned] + else: + adjs = [0, 1, -1] + adjs += [d for m in (2, 4, 8, 16) for d in (m, -m)] + for adj in adjs: + idx = k + adj + if idx < 0 or idx >= pages: + continue + cpu = lo + idx * 0x1000 + d = kc.read(cpu, 64) + if not d or not any(d[:32]): + continue + if not _push_prefix_ok(d): + continue + pa = kc.va_to_pa(cpu) + if pa is None: + continue + _area_off[lo] = adj + out.append("0x%x,0x%x,1\n" % (va & ~0xFFF, pa)) + dbg("demand-va: 0x%x -> pa=0x%x (area off=%d adj=%+d)" + % (va & ~0xFFF, pa, idx, adj)) + return True + dbg("demand-va: 0x%x not found by content (%d areas)" + % (va & ~0xFFF, len(areas))) + return False + + +def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0, + page_off=0): """uvm_rm_mem kptr -> GPU-VA/GPA runs (direct map or page-table walk). va_lo/va_hi optionally restrict the emitted GPU VA range; the kcore - chain resolver targets the CE push-buffer window only. + chain resolver targets the CE push-buffer window only. page_off shifts + the vmap page paired with GPU data page k: the CE push-buffer vmap is + one page longer than its GPU data window on some allocations (a leading + header page), so GPU data page k lives at vmap page k + page_off. + page_off is derived per allocation (mem_page_off): a fixed offset is + wrong (r116/r117 regressed client #1 with a hard +1). """ if not is_kern_ptr(mem): return 0 @@ -281,7 +384,7 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0): and (not va_hi or v < va_hi)] if not cpu or not gpus: return 0 - size = kc.read64(mem + 0x60) or 0 + size = _mem_size(kc, mem) dm = _direct_map_pa(kc, cpu) emitted = 0 if dm is not None: @@ -290,14 +393,15 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0): out.append("0x%x,0x%x,%d\n" % (gv, dm, npages)) emitted += 1 return emitted + area = _vmalloc_pages_for(kc, cpu) npages = (max(1, min(size >> 12, 4098)) if size - else _vmalloc_pages_for(kc, cpu)) - if not npages: - npages = 1 + else area or default_pages) + if area and page_off + npages > area: + npages = max(1, area - page_off) runs = [] cur = None for k in range(npages): - pa = kc.va_to_pa(cpu + k * 0x1000) + pa = kc.va_to_pa(cpu + (k + page_off) * 0x1000) if pa is None: if cur: runs.append(cur) @@ -318,6 +422,196 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0): return emitted +def _mem_size(kc, mem): + """uvm_rm_mem size (bytes) from the 570.144 layout, with a fallback. + + The struct is type(4)+pad(4)+mask(8)+vas[33] (264 B)+proxy_vas+ + gpu_owner+size, so size lives at +0x128; +0x60 (inside vas[]) reads 0 + for unused processors and is kept only as a fallback. + """ + for off in (0x128, 0x60): + v = kc.read64(mem + off) or 0 + if 0 < v <= (256 << 20): + return v + return 0 + + +def _push_prefix_ok(buf): + """First-4KB method-stream plausibility, mirroring the host's check.""" + off = 0 + while off + 4 <= len(buf): + hdr = struct.unpack_from('<I', buf, off)[0] + if hdr: + break + off += 4 + if off + 4 > len(buf): + return True # all zeros: not refutable + op = (hdr >> 29) & 0x7 + if op in (4, 7): + return True + if op not in (1, 3, 5): + return False + return ((hdr >> 16) & 0xfff) != 0 + + +def _page_content(kc, va): + pa = kc.va_to_pa(va & ~0xFFF) + if pa is None: + return None + return kc.read(kc.page_offset_base + pa, 64) + + +def mem_page_off(kc, mem): + """Per-allocation vmap offset for a uvm_rm_mem's GPU data pages. + + Derived, never fixed: the vmap area's page count (vmallocinfo) minus + the rm_mem's data page count (size at +0x128). A 4097-page area with + 4096 data pages means one leading header page (offset 1); an area that + exactly matches the data means offset 0. When the size is unreadable, + the two candidate offsets are content-probed with the host's method- + stream plausibility test and the decisively-passing one wins; an + ambiguous result keeps 0 (the status-quo pairing). + """ + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + if not cpu or _direct_map_pa(kc, cpu) is not None: + return 0 + area = _vmalloc_pages_for(kc, cpu) + if area <= 1: + return 0 + size = _mem_size(kc, mem) + if size: + data = min(size >> 12, 4098) + if 0 <= area - data <= 2: + return area - data + return 0 + g = [] + for off in (0, 1): + if off >= area: + g.append(0) + continue + buf = _page_content(kc, cpu + off * 0x1000) + g.append(1 if (buf is not None and _push_prefix_ok(buf)) else 0) + if g[0] != g[1]: + return 0 if g[0] else 1 + return 0 + + +def _ce_mem_pages(kc, mem): + """CE mem walk parameters: (gpus, cpu, off, npages) or None. + + Walks the same page set resolve_mem_pins used: the whole vmalloc area + (minus the header offset) when known, else size at +0x128, else the + default. The area governs: the live pushes are written sequentially and + may run past the size field, so a size-derived prefix stopped one page + short of the write frontier (r144: last page 0x120030000 emitted, the + live page 0x120031000 never in any pin set). all CE-window GPU VAs share + the CPU mapping. + """ + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + gpus = [v for v in vas if v and PB_CE_LO <= v < PB_CE_HI] + if not cpu or not gpus: + return None + if _direct_map_pa(kc, cpu) is not None: + return None # never a vmap: not the push-buffer shape + off = mem_page_off(kc, mem) + area = _vmalloc_pages_for(kc, cpu) + size = _mem_size(kc, mem) + size_pages = max(1, min(size >> 12, 4098)) if size else 0 + area_pages = area - off if area and area > off else 0 + npages = min(max(size_pages, area_pages) or 64, 4098) + return (gpus, cpu, off, npages) + + +def _emit_ce_mem_delta(kc, st, out, fresh_map=False): + """Emit the CE mem's live {va,pa} pairs as merged runs. + + Two emission modes share one merge step: + + * full re-assert (KC_FULLREM_S cadence, and every fresh_map pass): every + page that translates is emitted, content-independent. This is the + fix for the host's session flush: at a client switch the host wipes + its whole sema map when the previous client's last channel closes + ("session flush (all channels closed)"), and a pin batch that raced + in for the new client is lost. The old content-gated delta never + re-emitted those unchanged pairs, so the new client's CE pushes were + permanently unreadable ("NO ENTRY" in sema-probe) and every GPFIFO + entry was skipped after the 8 s grace (r144/r145 second-client stall, + both mail and matmul). Re-asserting the full pair set every couple of + seconds heals any number of flush wipes; merged contiguous runs keep + the traffic to a few lines per pass. Zero-content pages are pinned + deliberately: the pin carries the translation, the host reads the + live content, and the client writes it later. + + * delta (between full passes): content-gated, emits pages whose PA is + new or changed, so a remap or a freshly-faulted page is picked up + without waiting for the next full pass. Never drops a previously + emitted pair (the host's last-store-wins pin sees the truth, and a + failed burst never clears the whole map). + """ + gpus, cpu, off, npages = st["gpus"], st["cpu"], st["page_off"], st["npages"] + last = st["last"] + nowm = time.monotonic() + full = nowm >= st.get("full_next", 0.0) + if full: + age = nowm - st.get("resolved_at", nowm) + st["full_next"] = nowm + (KC_FULLREM_S if age < KC_FULLREM_FAST_S + else KC_FULLREM_SLOW_S) + if fresh_map: + for k in range(npages): + kc._pa_cache.pop(cpu + (k + off) * 0x1000, None) + changed = [] + if full: + # Re-walk pages that did not translate on an earlier pass: the + # vmap's leaf sizes are huge (2 MB/1 GB), so most pages translate + # immediately; a straggler (the vmap populated lazily) is retried + # with its negative PA-cache entry popped, bounded per pass so the + # sweep stays cheap. + miss = st.setdefault("miss", set()) + retry = 0 + for k in range(npages): + page = cpu + (k + off) * 0x1000 + pa = kc.va_to_pa(page) + if pa is None: + if k in miss and retry < 512: + kc._pa_cache.pop(page, None) + retry += 1 + else: + miss.add(k) + continue + miss.discard(k) + for gv in gpus: + changed.append((gv + k * 0x1000, pa)) + else: + for k in range(npages): + d = kc.read(cpu + (k + off) * 0x1000, 32) + if not d or not any(d): + continue # zero/unreadable: nothing to pin yet + pa = kc.va_to_pa(cpu + (k + off) * 0x1000) + if pa is None: + continue + for gv in gpus: + if last.get(gv + k * 0x1000) != pa: + changed.append((gv + k * 0x1000, pa)) + changed.sort() + merged = [] + for va, pa in changed: + if merged and va == merged[-1][0] + merged[-1][2] * 0x1000 and \ + pa == merged[-1][1] + merged[-1][2] * 0x1000: + merged[-1][2] += 1 + else: + merged.append([va, pa, 1]) + for va, pa, n in merged: + out.append("0x%x,0x%x,%d\n" % (va, pa, n)) + for va, pa in changed: + last[va] = pa + st["emit_total"] = st.get("emit_total", 0) + len(merged) + if full: + st["full_total"] = st.get("full_total", 0) + 1 + return len(merged) + + def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out): """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT buffer usually sits) from the ring's CPU mapping.""" @@ -440,11 +734,55 @@ def resolve_sem_pins(kc, sem, out): return resolve_mem_pins(kc, mem, out, default_pages=16) +def resolve_pool_va(kc, va, out, sems): + """Pool/waiter VA -> PA via tracker-sem -> page -> mem; NO content match. + + The word at POOL base + 4*idx is a slot in a UVM semaphore-pool page: a + sysmem uvm_rm_mem whose PA is only reachable through its kernel CPU + alias (the client pagemap has no present entry for it). Same kcore walk + the CE pushbuffer uses; the chain is shorter (sem+0 = page, page+0 = + mem). The page sits 16 MB above the CE push buffer base, outside + resolve_sem_pins' default_pages window, which is why the host saw + 'pending release starved' for it on every client's first seconds. + """ + page = va & ~0xFFF + if not (POOL_SEM_LO <= page < POOL_SEM_HI): + return False + for sem in list(sems): + if not is_kern_ptr(sem): + continue + try: + pg = kc.read64(sem) + if not is_kern_ptr(pg): + continue + mem = kc.read64(pg) + if not is_kern_ptr(mem): + continue + got = [] + resolve_mem_pins(kc, mem, got, default_pages=1, + va_lo=page, va_hi=page + 0x1000) + except Exception: + continue + for line in got: + try: + lva = int(line.split(',')[0], 16) + except Exception: + continue + if lva == page: + out.append(line) + dbg('pool-va: 0x%x -> %s (mem=0x%x)' + % (page, line.strip(), mem)) + return True + dbg('pool-va: 0x%x unresolved (%d known sems)' % (page, len(sems))) + return False + + def resolve_pb_pins(kc, pb, out): mem = kc.read64(pb + 8) if not mem: return 0 - return resolve_mem_pins(kc, mem, out, default_pages=64) + return resolve_mem_pins(kc, mem, out, default_pages=64, + page_off=mem_page_off(kc, mem)) # The CE push-buffer window is the one UVM region the client never maps into @@ -458,12 +796,49 @@ def resolve_pb_pins(kc, pb, out): # channel->pool->manager->pushbuffer->rm_mem chain is walkable in kernel # memory, so resolve the CE push buffer directly with kcore: no trace event, # nothing to lose. +# +# The kptr-keyed one-shot latches below are NOT object identity: UVM frees +# and re-creates these rm_mems and channels during a session, and the slab +# reuses the same addresses. A resolution taken before such a move pins +# pages that are stale by the time the host reads them (observed r125: the +# pinned CE pages read all-zero while the channel's real pushes lived at +# new vmap pages of the same rm_mem kptr; every ring entry then stalled in +# the host's 8 s grace since no release could be decoded from zeros). So a +# resolved mem is periodically re-validated against the channel's own ring +# entries and re-resolved when it stops matching (KC_PB_STATE/_kc_pb_fresh), +# and when several mem candidates are reachable the one whose pages actually +# hold the ring-entry pushes is chosen (content anchor, _mem_content_ok). PB_CE_LO = 0x120000000 PB_CE_HI = 0x121000000 +# m165 (D-F1a): UVM semaphore-pool page range. The pool is a 4 KB sysmem +# uvm_rm_mem placed by the UVM arena right after the 16 MB CE push buffer; +# the client's tracker words (0x121000000 + 4*idx) live in its first page. +# The client pagemap has no present entry for it, so only the kcore +# tracker-sem -> page -> mem walk can translate it. +POOL_SEM_LO = 0x121000000 +POOL_SEM_HI = 0x122000000 +KC_POOL_REEMIT_S = 2.0 # re-assert the pool page pair this often too KC_CHAIN_BACK = 0x2000 KC_CHAIN_SPAN = 0xA000 KC_CHAIN_MAX = 400 RESOLVED_KC_PB = set() +KC_PB_STATE = {} # sem kptr -> resolved-mem state (fingerprint + pins) +KC_REFRESH_S = 2.0 # re-validate a resolved CE mem at most this often +KC_REEMIT_S = 2.0 # sweep the CE mem's live pages this often +KC_REMAP_S = 30.0 # full re-translation sweep (catches remaps) +# Full re-assert cadence: every pass at least this far apart re-emits the +# whole translatable pair set regardless of the delta, healing host-side +# session-flush wipes (see _emit_ce_mem_delta). Must stay well under the +# host's gsp_sema_ce_grace_ms (8 s) so a wiped-and-needed page is restored +# inside the hold window. The fast cadence applies to a mem's first +# KC_FULLREM_FAST_S seconds (the flush race lives at the client switch); +# after that the re-assert backs off to KC_FULLREM_SLOW_S - a fully +# fragmented pair set (descending PAs) emits one page per line, and 4 k +# lines every 2 s saturated a guest under multi-client load (r148: load 19, +# three scanners from an install race, clients wedged). +KC_FULLREM_S = float(os.environ.get("STREAM_FULLREM_S", "1.5")) +KC_FULLREM_FAST_S = float(os.environ.get("STREAM_FULLREM_FAST_S", "45")) +KC_FULLREM_SLOW_S = float(os.environ.get("STREAM_FULLREM_SLOW_S", "10.0")) def _chain_mems_at(kc, w): @@ -484,13 +859,96 @@ def _chain_mems_at(kc, w): return mems +def _mem_gpu_cpu(kc, mem): + """The CE-window GPU VA and CPU alias of an rm_mem (None/None when absent).""" + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + gpu = next((v for v in vas + if v and PB_CE_LO <= v < PB_CE_HI), None) + return gpu, cpu + + +def _mem_content_score(kc, mem, gpu): + """Rough 'is this the live CE push buffer' score for a mem candidate. + + The channel's CE rings live in (emulated) video memory, so the guest + cannot read the ring entries to anchor a push VA; but the CE push + buffer's guest-RAM pages are CPU-written method streams during channel + setup and client work. Count nonzero u32 pairs across the first 128 + pages of the candidate's window: a live mem scores high, a freed page + range that a reused kptr still describes reads zeros and scores 0. + """ + cpu = None + for i in range(16): + v = kc.read64(mem + 0x10 + i * 8) + if is_kern_ptr(v): + cpu = v + break + if cpu is None: + return -1 + score = 0 + for k in range(128): + buf = kc.read(cpu + k * 0x1000, 0x1000) + if not buf: + continue + for off in range(0, 0x1000, 16): + if buf[off:off + 8] != b"\x00" * 8: + score += 1 + return score + + +def _kc_pb_fresh(kc, sem): + """The pinned CE mem still matches the channel's live backing. + + Cheap check: the mem's own {cpu,gpu} fields and the guest PA of its + first window page must still match the recorded resolution. A vmap area + never moves while the mem object lives, so either check failing means + the object was freed and (possibly) re-created at the same kptr - the + caller then re-resolves and re-emits the current pages. + """ + st = KC_PB_STATE.get(sem) + if not st: + return False + mem = st.get("mem") + if not mem or not is_kern_ptr(mem): + return False + if (kc.read64(mem + 0x30) or 0) != st.get("cpu") or \ + (kc.read64(mem + 0x38) or 0) != st.get("gpu"): + return False + if not st.get("page_va"): + return True + if not st.get("page_pa"): + # Unknown PA at store time: the manual page-table walk could not + # translate the vmap page (it may have been read before the kernel + # page tables were walkable). Treat the record as fresh instead of + # failing every check - a permanently-failing freshness test made + # the scanner drop the latch and re-resolve in a tight loop, and + # each failed re-resolution left the host with no pins at all + # (observed r131: "kc-chain stale: re-resolving" every second for + # five sems while smiley sat in cudaMalloc). + return True + pa = kc.va_to_pa(st["page_va"]) + if pa is None or pa != st.get("page_pa"): + return False + return True + + def resolve_pb_chain(kc, sem, out): """Tracking-sem kptr -> CE push-buffer rm_mem pins (kcore chain walk). Returns True when a CE push-buffer mem was found for this sem, so the caller marks it resolved even when the mem was already pinned; new pin lines are appended to out. Bounded: one window read plus a few reads per - candidate, and each rm_mem is resolved at most once per session. + candidate, and each rm_mem is resolved at most once per session (until a + freshness failure drops it again, see _kc_pb_fresh). + + When several mem candidates are reachable, the content score decides: a + stale candidate (freed memory read through a reused kptr) reads zeros + and is skipped in favour of the live push-buffer pages, so it cannot + shadow them on the host's last-store-wins pin table. An all-zero + candidate set emits nothing: pinning zeros is what made the host read + zero pushes and stall every CE entry; the drift retries (retry_at) and + the periodic refresh pick the pages up as soon as content appears. """ if not is_kern_ptr(sem): return False @@ -499,26 +957,92 @@ def resolve_pb_chain(kc, sem, out): return False words = struct.unpack_from("<%dQ" % (len(data) // 8), data, 0) cands = list(dict.fromkeys(w for w in words if is_kern_ptr(w))) - found = False + # Channel-anchored expansion (r134 live dump): the tracking sem is a + # member of a uvm_channel_t table; the channel records sit in the sem's + # own window and carry their CE ring GPU VA ([0x121010000, 0x121400000)) + # plus back-to-back 32-bit object handles. The pool/manager/pushbuffer + # pointers hang off those records' targets, which the bare sem-window + # scan never reaches - the second-client stall is exactly this: the only + # live uvmt event is the spinning sem, and no mem candidate is found + # around it. For each ring-VA word, scan the surrounding record for + # kptrs, take those targets, and try the chain shape on both. Additive + # and bounded; content scoring still decides, so a wrong candidate + # cannot shadow the live pages. + extra = [] + for i, w in enumerate(words): + if not w or not (0x121010000 <= w < 0x121400000): + continue + lo = max(0, i - 0x60) + hi = min(len(words), i + 0x60) + for c in words[lo:hi]: + if not is_kern_ptr(c): + continue + extra.append(c) + tgt = kc.read(c, 0x100) + if tgt: + tw = struct.unpack_from("<%dQ" % (len(tgt) // 8), + tgt, 0) + for w2 in tw: + if is_kern_ptr(w2): + extra.append(w2) + if len(extra) >= 512: + break + if extra: + head = [] + for w in extra: + if w not in head and w not in cands: + head.append(w) + cands = head + cands + mems = [] for w in cands[:KC_CHAIN_MAX]: for mem in _chain_mems_at(kc, w): - vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] - if not any(v and PB_CE_LO <= v < PB_CE_HI for v in vas): - continue - found = True - if mem in RESOLVED_KC_PB: - continue - RESOLVED_KC_PB.add(mem) - n = resolve_mem_pins(kc, mem, out, default_pages=64, - va_lo=PB_CE_LO, va_hi=PB_CE_HI) - try: - with open('/tmp/guest_pin_stream.log', 'a') as f: - f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x ' - 'pin-runs=%d\n' - % (time.strftime('%T'), mem, sem, n)) - except OSError: - pass - return found + if mem not in mems: + mems.append(mem) + found = False + scored = [] + for mem in mems: + gpu, cpu = _mem_gpu_cpu(kc, mem) + if not gpu or not cpu: + continue + found = True + score = _mem_content_score(kc, mem, gpu) + scored.append((score, mem, gpu, cpu)) + if not scored: + return found + best = max(scored, key=lambda c: c[0]) + if best[0] <= 0: + dbg("kc-chain all candidates zero (sem=0x%x, n=%d): deferring" + % (sem, len(scored))) + return found + if len(scored) > 1: + dbg("kc-chain scores sem=0x%x: %s" % + (sem, ' '.join('%#x=%d' % (m, s) for s, m, _g, _c in scored))) + _score, mem, gpu, cpu = best + if mem in RESOLVED_KC_PB: + return True + RESOLVED_KC_PB.add(mem) + params = _ce_mem_pages(kc, mem) + if not params: + return found + gpus, cpu, off, npages = params + st = {"mem": mem, "gpu": gpu, "cpu": cpu, "gpus": gpus, + "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, + "page_off": off, "npages": npages, "last": {}, + "resolved_at": time.monotonic()} + # Fresh translation for the first sweep: the Kcore PA cache may hold + # pairs from before this mem's vmap area was (re)populated. + kc._pa_cache.clear() + n = _emit_ce_mem_delta(kc, st, out) + KC_PB_STATE[sem] = st + try: + with open('/tmp/guest_pin_stream.log', 'a') as f: + f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x ' + 'pin-runs=%d off=%d npages=%d size=%d score=%d\n' + % (time.strftime('%T'), mem, sem, n, off, npages, + _mem_size(kc, mem), _score)) + except OSError: + pass + return True # UvmGpuChannelInfo field offsets (nv_uvm_types.h, x86_64): @@ -613,15 +1137,72 @@ def resolve_channel_pins(kc, ch, out): -def clear_trace(): - """Drop stale ring events so the fallback tail reader cannot re-learn a - previous client's freed sems after a client switch. The new client's - events are delivered fresh through the consuming trace_pipe reader.""" +def _read_trace_snapshot(max_bytes=1 << 20, budget=0.5): + """Best-effort non-blocking read of the trace ring file. + + Spawning `tail` here left uninterruptible D-state processes when the + seq_file read blocked under load (observed r127: dozens of stuck tails, + load 26); a direct os.read with O_NONBLOCK returns whatever is ready and + never accumulates processes. + """ try: - with open(TRACE, "w") as f: - f.write("") + fd = os.open(TRACE, os.O_RDONLY | os.O_NONBLOCK) except OSError: - pass + return b"" + chunks = [] + got = 0 + t0 = time.time() + try: + while got < (8 << 20) and time.time() - t0 < budget: + try: + d = os.read(fd, 262144) + except BlockingIOError: + break + except OSError: + break + if not d: + break + chunks.append(d) + got += len(d) + finally: + os.close(fd) + return b"".join(chunks)[-max_bytes:] + + +def seed_events_from_trace(seen_sems, seen_pbs, seen_chs): + """One-shot tail scan of the trace ring after a pid switch / start. + + The uvmt/uvbp/uvch kprobe events are one-shot per object, and the + trace_pipe reader only sees events that arrive after it attaches: a + client whose channels are registered in the instant around the scanner's + pid switch would otherwise never appear in the sets (observed r126: + vecadd_loop ran for minutes with sem=0 while its CE entries paid the + 8 s grace). The ring keeps recent events, so scan its tail once and add + every distinct kptr. Stale previous-client pointers that are also + re-learned are harmless: their chains no longer resolve to content + (the mem candidates are scored and deferred). + """ + try: + out = _read_trace_snapshot() + except Exception: + return 0 + n = 0 + for m in re.finditer(rb"sem=0x([0-9a-fA-F]+)", out): + v = int(m.group(1), 16) + if v not in seen_sems: + seen_sems.add(v) + n += 1 + for m in re.finditer(rb"pb=0x([0-9a-fA-F]+)", out): + v = int(m.group(1), 16) + if v not in seen_pbs: + seen_pbs.add(v) + n += 1 + for m in re.finditer(rb"ch=0x([0-9a-fA-F]+)", out): + v = int(m.group(1), 16) + if v not in seen_chs: + seen_chs.add(v) + n += 1 + return n class TraceReader: @@ -724,6 +1305,75 @@ def new_sems(seen): return found +# --------------------------------------------------------------- scrubber +# PMA scrubber channel discovery (third-client scrub freeze). The guest's +# PMA scrubber is an internal CeUtils CE channel (4096 GPFIFO entries, +# pbGpuVA 0x120000000, ring at pbGpuVA+0x64000, finish payload at +# pbGpuVA+0x6c004). It lives in a kernel vmap area, not in any user pagemap, +# and its GPU VAs collide numerically with the client CE push-buffer window +# in the host's global translation map - so the host cannot resolve it and +# must be told the finish/host-semaphore word PAs directly. Resolve them +# here (vmallocinfo + manual kernel page-table walk) and emit one +# #scrub <fin_pa>,<hsem_pa> +# line; pin_gr_stream.py registers them through sema_scrubbuf_add. +SCRUB_AREA_SIZES = (0x6e000, 0x6d000, 0x6c000, 0x6b000, 0x6f000, + 0x70000, 0x71000, 0x72000) +_scrub_sent = {} + + +def _scrub_ring_ok(kc, ring_pa): + """A 906F GPFIFO ring page: all-zero (fresh) or entries whose GET names a + push inside the channel's own 0x120000000 window (encoded as + push_va - 0x100000000, 4-byte aligned).""" + buf = kc.read(kc.page_offset_base + ring_pa, 0x1000) + if not buf: + return False + for off in range(0, 0x100, 8): + e0, _e1 = struct.unpack_from('<II', buf, off) + if e0 == 0: + continue + if not (0x20000000 <= e0 < 0x20080000) or (e0 & 3): + return False + return True + + +def scrub_scan(kc, lines): + """Emit the scrubber buffer words' PAs once (re-emitted if they move).""" + try: + f = open('/proc/vmallocinfo') + except OSError: + return + with f: + for line in f: + if 'nvidia' not in line or 'ioremap' in line or 'vmap' not in line: + continue + parts = line.split() + try: + a, b = parts[0].split('-') + lo, hi = int(a, 16), int(b, 16) + except (ValueError, IndexError): + continue + if (hi - lo) not in SCRUB_AREA_SIZES: + continue + fin_pa = kc.va_to_pa(lo + 0x6c000) + ring_pa = kc.va_to_pa(lo + 0x64000) + if fin_pa is None or ring_pa is None: + continue + if not _scrub_ring_ok(kc, ring_pa): + dbg('scrub candidate 0x%x rejected (ring shape)' % lo) + continue + fin = fin_pa + 4 + hsem = fin_pa + key = (fin, hsem) + if _scrub_sent.get('key') == key: + return + _scrub_sent['key'] = key + lines.append('#scrub 0x%x,0x%x\n' % (fin, hsem)) + dbg('scrub buffer emitted lo=0x%x fin=0x%x hsem=0x%x' + % (lo, fin, hsem)) + return + + def nvidia_fds(pid): """CUDA device fds this pid holds: 'uvm', 'gpu', or None. @@ -748,16 +1398,49 @@ def nvidia_fds(pid): return kind +def pid_starttime(pid): + """Starttime (clock ticks, field 22) of /proc/<pid>, or None. + + Part of the client identity (pid, starttime): pids are recycled, so a + per-client reset keyed on the pid alone can mis-fire on a recycled pid. + Parsed after the last ')' so comm names with spaces/parens are safe.""" + try: + with open('/proc/%d/stat' % pid) as f: + st = f.read() + rp = st.rfind(')') + if rp < 0: + return None + toks = st[rp + 2:].split() + if len(toks) < 20: # starttime = field 22 overall = index 19 + return None + return int(toks[19]) + except (OSError, ValueError): + return None + + +def pid_line(tag, pid, start): + """'#<tag> <pid> [starttime]' - the optional starttime keeps the line + backward compatible with older hosts (they parse the pid only).""" + if start is None: + return "#%s %d\n" % (tag, pid) + return "#%s %d %d\n" % (tag, pid, start) + + def cdt_pid(): prefixes = tuple(p for p in os.environ.get("CLIENT_PREFIXES", "cdt,va,victory").split(",") if p) # Processes that hold a /dev/nvidia* fd but are not CUDA clients; the # scanner must never commit to them (a transient nvidia-smi at boot was # picked once and its exited pagemap read back EOF forever). - non_clients = ("nvidia-smi", "nvidia-persistenced", "Xorg", "X") - best = None - uvm = None - gpu = None + # + # SESSION 146: Xorg/X used to be excluded here too, but the GL/GZDOOM + # path needs the X server's pages pinned - Xorg screen init fails with + # "Failed to initialize DMA / Failed to allocate push buffer" when the + # nvidia-push buffers cannot be resolved (observed r816). Xorg is a + # long-lived process (unlike the nvidia-smi transient the exclusion + # guarded against), so it is a valid, stable pin target. + non_clients = ("nvidia-smi", "nvidia-persistenced") + cands = [] for d in os.listdir('/proc'): if not d.isdigit(): continue @@ -772,28 +1455,24 @@ def cdt_pid(): rp = st.rfind(')') if rp >= 0 and st[rp + 2:rp + 3] == 'Z': continue - if any(name.startswith(p) for p in prefixes): - best = int(d) - continue if any(name.startswith(p) for p in non_clients): continue # Arbitrary CUDA binaries (any comm) are clients too: match by # the CUDA device they hold open so streaming never depends on - # the binary's name. Prefer a UVM holder (compute client) over a - # plain /dev/nvidia0 holder (nvidia-smi, persistenced); the last - # match wins, which is the newest client when several overlap. + # the binary's name. The newest candidate (highest pid) is the + # client being streamed: preferring a name-prefix match first + # pinned the scanner to a previous client that was still tearing + # down while the new client had already started pushing (its + # first pushes were unreadable, the expectation drifted, and the + # stall followed). kind = nvidia_fds(d) - if kind == 'uvm': - uvm = int(d) - elif kind == 'gpu': - gpu = int(d) + if kind is not None or any(name.startswith(p) for p in prefixes): + cands.append(int(d)) except OSError: pass - if best is not None: - return best - if uvm is not None: - return uvm - return gpu + if not cands: + return None + return max(cands) def scan_window(fd, lo, hi, span, out): @@ -872,6 +1551,18 @@ def scan_vmas(fd, pid, out): parts = line.split() if len(parts) < 5: continue + # Skip mappings with no read permission: a CUDA device buffer + # is a PROT_NONE reservation (e.g. `---p 00000000 00:00 0`), + # potentially hundreds of MB. Its pages are never present in + # the pagemap, so scanning it produces no pairs - but it + # consumed the VMA_MAX_PAGES budget and the scan never reached + # the client's present staging buffers that sit ABOVE the + # device reservation (observed: a client whose device range + # started below its staging buffer got no pins at all and its + # H2D copies failed with rc=-2 forever). + perms = parts[1] if len(parts) >= 2 else '' + if 'r' not in perms: + continue # Anonymous mappings (no path field) are the client's UVM sysmem # buffers when the driver maps them without a /dev/zero backing: # the second client's cudaMalloc'd data buffers show up exactly @@ -960,6 +1651,9 @@ def main(): resolved_chs = set() resolved_kc_pb = set() retry_at = {} + kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} tracer = TraceReader() kc = Kcore() kc_ok = kc.load() @@ -970,13 +1664,34 @@ def main(): except OSError: pass last_pid = None + last_start = None last_log = 0.0 + scrub_next = 0.0 + demand_next = 0.0 + # m165 (D-F1a): pool page VAs learned from demand, re-asserted every + # KC_POOL_REEMIT_S. Boot-stable (UVM frees pool pages only at GPU + # deinit), so the set deliberately persists across client switches. + pool_pages = {} + pool_reemit_next = 0.0 + _want_state = {"mtime": None, "done": set()} loop = 0 while True: pid = cdt_pid() loop += 1 if pid is None: + if last_pid is not None and sock is not None: + # In-band client-death signal: the host tools purge the dead + # client's per-client state on this line, so the NEXT client + # works without a VM cycle (the kill cascade). Best-effort - + # if the socket is gone the host still has its own probe. + try: + sock.sendall(pid_line("nopid", last_pid, + last_start).encode()) + except OSError: + sock.close() + sock = None last_pid = None + last_start = None seen = {} seen_sems = set() seen_pbs = set() @@ -985,6 +1700,7 @@ def main(): continue if pid != last_pid: last_pid = pid + last_start = pid_starttime(pid) seen = {} seen_sems = set() seen_pbs = set() @@ -995,17 +1711,27 @@ def main(): resolved_chs = set() resolved_kc_pb = set() retry_at = {} + kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} # Client switch: the old client's channel/put pointers are freed # memory. Clearing the #put map stops stale reads from being # emitted as if they were the new client's counters. CHAN_PUTS.clear() PUT_LAST.clear() - clear_trace() tracer.reset() RESOLVED_KC_PB.clear() + KC_PB_STATE.clear() + # One-shot: recover the new client's one-shot channel events + # from the trace ring (they can fall in the switch instant and + # be invisible to the pipe reader; the old clear_trace() made + # that permanent - part of the r126 stall class). + _seeded = seed_events_from_trace(seen_sems, seen_pbs, seen_chs) + if _seeded: + dbg("switch seed: +%d event kptr(s)" % _seeded) if sock is not None: try: - sock.sendall(("#pid %d\n" % pid).encode()) + sock.sendall(pid_line("pid", pid, last_start).encode()) except OSError: sock.close() sock = None @@ -1039,6 +1765,123 @@ def main(): new = [(va, pa) for va, pa in cur.items() if seen.get(va) != pa] pending_cur = cur lines = [] + # Demand pins (host -> this scanner): the host appends the CE + # push-buffer VAs of GPFIFO entries it cannot read to + # /tmp/guest_want_vas.txt (written via the guest agent by + # pin_gr_stream); resolve each by content against the + # nv_vm_map_pages areas and emit the pair. This is the + # event-independent hole filler for the stall class where the + # one-shot #sem kprobe event was lost and the object-chain + # resolver never learns the client. + if kc_ok and time.time() >= demand_next: + demand_next = time.time() + 1.0 + try: + st_d = os.stat("/tmp/guest_want_vas.txt") + if st_d.st_mtime != _want_state.get("mtime"): + with open("/tmp/guest_want_vas.txt") as fw: + # r582: tolerate malformed historical lines. The + # r581 boot's scanner died here: a transport-escaped + # append merged VAs into one token containing + # literal backslash-n ('0x20400f000\\n0x121000000\\n + # ...') and int(t, 16) raised ValueError out of + # main(), stopping ALL new pins (the observed + # pool-pin latency). Extract every 0x token instead; + # a bad token is skipped, never fatal. + wvas = [] + for t in re.findall(r"0x[0-9a-fA-F]+", + fw.read()): + try: + wvas.append(int(t, 16)) + except ValueError: + pass + _want_state["mtime"] = st_d.st_mtime + for wv in wvas[-64:]: + if wv in _want_state["done"]: + continue + got = [] + if POOL_SEM_LO <= (wv & ~0xFFF) < POOL_SEM_HI: + # m165 (D-F1a): pool/waiter VAs have no method + # stream content and the CE offset math does not + # apply; route them to the structural tracker-sem + # walk instead. m168: the tracker sems are + # long-lived objects (allocated once) whose page + # pointer follows the pool across clients, so try + # every sem we have ever resolved a chain for - + # seen_sems alone can miss the live generation. + _psems = set(seen_sems) | set(KC_PB_STATE.keys()) \ + | set(resolved_kc_pb) + try: + ok = resolve_pool_va(kc, wv, got, + _psems) + except Exception: + ok = False + got = [] + dbg('demand-pool-va: 0x%x %s' + % (wv, 'ok' if ok else 'pending')) + if ok and got: + pool_pages.setdefault(wv & ~0xFFF, True) + else: + try: + ok = resolve_va_by_content(kc, wv, got) + except Exception: + ok = False + got = [] + # r586: a demanded VA outside the CE + # pushbuffer window (the GRFILL staging pages, + # high client VAs) is a plain client mapping + # page: resolve it through the client's own + # pagemap. Without this the demand for those + # VAs was a silent no-op (resolve_va_by_content + # rejects anything outside PB_CE_LO..PB_CE_HI) + # and the fills kept dropping them. + if not ok and not got and pid: + try: + pfd = os.open('/proc/%d/pagemap' % pid, + os.O_RDONLY) + try: + sub = {} + vpg = wv & ~0xFFF + scan_range(pfd, vpg, vpg + 0x1000, + sub) + finally: + os.close(pfd) + if sub: + for v, p in sub.items(): + got.append('0x%x,0x%x,1\n' + % (v, p)) + ok = True + except OSError: + pass + if ok and got: + lines.extend(got) + _want_state["done"].add(wv) + except OSError: + pass + # m165 (D-F1a): re-assert the pool page pair every KC_POOL_REEMIT_S. + # The pool page is boot-stable (UVM frees it only at GPU deinit) but + # the host pin table is wiped at every session flush, so a one-shot + # pin leaves later clients starving until the slow structural walks + # answer; the periodic re-assert heals the wipe within seconds (the + # same argument as the CE mem sweep above). + if kc_ok and pool_pages and time.time() >= pool_reemit_next: + pool_reemit_next = time.time() + KC_POOL_REEMIT_S + for pv in list(pool_pages): + got = [] + try: + ok = resolve_pool_va(kc, pv, got, seen_sems) + except Exception: + ok = False + got = [] + if ok and got: + lines.extend(got) + # PMA scrubber buffer discovery (throttled): resolve the finish/host- + # semaphore word PAs and stream them for host registration. + if kc_ok and time.time() >= scrub_next: + scrub_next = time.time() + 5.0 + try: + scrub_scan(kc, lines) + except Exception as e: + dbg('scrub scan failed: %s' % e) for ctl in tracer.drain(): parts = ctl.split() if ctl.startswith("#sem") and len(parts) >= 2: @@ -1064,6 +1907,103 @@ def main(): added += len(got) else: retry_at[sem] = now_r + 1.0 + # CE push-buffer pin freshness: a resolved mem is re-probed + # against the channel's live ring content. UVM frees and + # re-creates these rm_mems/channels mid-session and the slab + # reuses the same kptrs, so a one-shot resolution can pin + # pages that are stale (observed reading all zeros) while + # the live pushes sit at new pages; dropping the latches + # makes the resolution below re-emit the current truth. + if sem in resolved_kc_pb and \ + now_r >= kc_refresh_at.get(sem, 0): + kc_refresh_at[sem] = now_r + KC_REFRESH_S + try: + _fresh = _kc_pb_fresh(kc, sem) + except Exception: + _fresh = True + if not _fresh: + # Re-resolve, but never lose the latch on a failed + # attempt: a dropped latch means the host keeps no + # pins for this channel at all, which is worse than + # a possibly-stale pin (the host content-checks the + # first page and now rejects all-FF continuation + # chunks, so stale pages are bounded). On failure + # keep the old state and defer the next attempt. + _st = KC_PB_STATE.get(sem) + if _st and _st.get("mem"): + RESOLVED_KC_PB.discard(_st["mem"]) + got = [] + try: + _found2 = resolve_pb_chain(kc, sem, got) + except Exception: + _found2 = False + got = [] + if _found2 and got: + _new = KC_PB_STATE.get(sem) + _newmem = _new.get("mem") if _new else None + dbg("kc-chain stale: re-resolved sem=0x%x " + "new-mem=%s runs=%d" + % (sem, "yes" if _newmem != + (_st.get("mem") if _st else None) + else "no", len(got))) + lines.extend(got) + added += len(got) + resolved_kc_pb.add(sem) + else: + if _st and _st.get("mem"): + RESOLVED_KC_PB.add(_st["mem"]) + if sem not in KC_PB_STATE: + KC_PB_STATE[sem] = _st + kc_refresh_at[sem] = now_r + 5.0 + dbg("kc-chain stale: keep old latch sem=0x%x" + % sem) + else: + # CE mem delta sweep (second-client pins): the one-shot + # emission pinned the mem once; pages the client writes + # later (or whose vmap pairing moved) were never + # re-emitted, so the host kept zeros for the other CE + # ranks' pushes. Sweep every KC_REEMIT_S, emitting only + # new/changed nonzero pairs; burst the Kcore PA cache + # every KC_REMAP_S so a remap is not shadowed. + _st = KC_PB_STATE.get(sem) + if _st is not None and \ + now_r >= kc_reemit_at.get(sem, 0): + _fresh_map = now_r >= kc_remap_at.get(sem, 0) + if _fresh_map: + kc_remap_at[sem] = now_r + KC_REMAP_S + kc_reemit_at[sem] = now_r + KC_REEMIT_S + got = [] + try: + _n2 = _emit_ce_mem_delta(kc, _st, got, + fresh_map=_fresh_map) + except Exception: + _n2 = 0 + got = [] + if got: + lines.extend(got) + added += len(got) + dbg("ce-delta sem=0x%x runs=%d mem=0x%x" + % (sem, _n2, _st.get("mem", 0))) + # CE push-buffer chain (second-client stall fix): resolve + # pool -> manager -> pushbuffer -> rm_mem directly, so the + # push-buffer pages never depend on the racy one-shot uvbp + # event. Independent of the ring/pool gates below (the + # channel gate's continue must not starve this path). + if sem not in resolved_kc_pb and \ + now_r >= retry_at.get(("pk", sem), 0): + got = [] + try: + _found = resolve_pb_chain(kc, sem, got) + except Exception: + _found = False + got = [] + if _found: + resolved_kc_pb.add(sem) + if got: + lines.extend(got) + added += len(got) + else: + retry_at[("pk", sem)] = now_r + 2.0 # ISSUE-0002: the pool pin (mem) and the channel ring are # resolved independently. The sem is marked resolved as soon # as the pool page pins, but the channel_info anchor may only @@ -1083,26 +2023,6 @@ def main(): added += len(got) else: retry_at[("cy", sem)] = now_r + 2.0 - # CE push-buffer chain (second-client stall fix): resolve - # pool -> manager -> pushbuffer -> rm_mem directly, so the - # push-buffer pages never depend on the racy one-shot uvbp - # event. Independent of the ring/pool gates above. - if sem in resolved_kc_pb or \ - now_r < retry_at.get(("pk", sem), 0): - continue - got = [] - try: - _found = resolve_pb_chain(kc, sem, got) - except Exception: - _found = False - got = [] - if _found: - resolved_kc_pb.add(sem) - if got: - lines.extend(got) - added += len(got) - else: - retry_at[("pk", sem)] = now_r + 2.0 for pb in list(seen_pbs): if pb in resolved_pbs or now_r < retry_at.get(pb, 0): continue @@ -1131,6 +2051,26 @@ def main(): added += len(got) else: retry_at[ch] = now_r + 1.0 + # A uvch channel kptr is another early seed for the CE + # push-buffer chain (its window contains the pool pointer): + # resolve it here so the pins never depend on the uvmt sem + # events alone. + if ch in resolved_kc_pb or \ + now_r < retry_at.get(("pk", ch), 0): + continue + got = [] + try: + _found = resolve_pb_chain(kc, ch, got) + except Exception: + _found = False + got = [] + if _found: + resolved_kc_pb.add(ch) + if got: + lines.extend(got) + added += len(got) + else: + retry_at[("pk", ch)] = now_r + 2.0 if added: try: with open('/tmp/guest_pin_stream.log', 'a') as f: @@ -1185,7 +2125,8 @@ def main(): sock.connect((CID, PORT)) sock.settimeout(None) if last_pid is not None: - sock.sendall(("#pid %d\n" % last_pid).encode()) + sock.sendall(pid_line("pid", last_pid, + last_start).encode()) except OSError: if sock: sock.close() diff --git a/tools/guest-diag/launch_qemu_rootport.sh b/tools/guest-diag/launch_qemu_rootport.sh index 444bdfc..d38321b 100755 --- a/tools/guest-diag/launch_qemu_rootport.sh +++ b/tools/guest-diag/launch_qemu_rootport.sh @@ -1,4 +1,4 @@ #!/bin/bash # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2026 Open-IOV contributors -exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -object memory-backend-file,id=pc.ram,size=4096M,mem-path=/dev/shm/vgpu-guest-ram,share=yes,prealloc=on -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 4096 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/<MDEV-UUID>,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait +exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -object memory-backend-file,id=pc.ram,size=8192M,mem-path=/dev/shm/vgpu-guest-ram,share=yes,prealloc=on -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 8192 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/<MDEV-UUID>,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait diff --git a/tools/guest-diag/nvkmstrace.c b/tools/guest-diag/nvkmstrace.c new file mode 100644 index 0000000..ca72e46 --- /dev/null +++ b/tools/guest-diag/nvkmstrace.c @@ -0,0 +1,62 @@ +// nvkmstrace.c - LD_PRELOAD shim: log NVKMS_IOCTL_IOWR commands +#define _GNU_SOURCE +#include <stdio.h> +#include <stdarg.h> +#include <stdint.h> +#include <string.h> +#include <dlfcn.h> +#include <sys/ioctl.h> + +struct nvkms_ioctl_params { + uint32_t cmd; + uint32_t size; + uint64_t address; +}; + +#define NVKMS_REQ 0xC0106D00UL + +static int (*real_ioctl)(int, unsigned long, ...); + +int ioctl(int fd, unsigned long req, ...) +{ + va_list ap; + void *arg; + + va_start(ap, req); + arg = va_arg(ap, void *); + va_end(ap); + if (!real_ioctl) + real_ioctl = dlsym(RTLD_NEXT, "ioctl"); + if (req == NVKMS_REQ && arg) { + struct nvkms_ioctl_params *p = arg; + int rc; + + rc = real_ioctl(fd, req, arg); + fprintf(stderr, "NVKMS_IOCTL cmd=%u size=%u addr=%p rc=%d\n", + p->cmd, p->size, (void *)(uintptr_t)p->address, rc); + if (p->address && p->size) { + unsigned char *q = (unsigned char *)(uintptr_t)p->address; + unsigned n = p->size < 64 ? p->size : 64, i; + + fprintf(stderr, " data:"); + for (i = 0; i < n; i++) + fprintf(stderr, " %02x", q[i]); + fprintf(stderr, "\n"); + if (p->size >= 700) { + /* reply.status candidates: request = 32 + 4 + + * 4*1 (NvBool=NvU8) + 16*(32+4) = 616 bytes */ + unsigned o; + + for (o = 600; o <= 700; o += 4) { + unsigned v; + memcpy(&v, q + o, 4); + if (v <= 16) + fprintf(stderr, " small@%u=%u\n", + o, v); + } + } + } + return rc; + } + return real_ioctl(fd, req, arg); +} diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py index b864ab5..ff7e299 100644 --- a/tools/guest-diag/pin_gr_stream.py +++ b/tools/guest-diag/pin_gr_stream.py @@ -16,6 +16,7 @@ import re import socket import subprocess import sys +import threading import time HERE = os.path.dirname(os.path.abspath(__file__)) @@ -31,12 +32,44 @@ LO = os.environ.get("PINWIN_LO", "0x200000000") HI = os.environ.get("PINWIN_HI", "0x206000000") PAMAX = os.environ.get("PINWIN_PA_MAX", "0x180000000") SEMSTREAM = os.environ.get("SEM_STREAM_FILE", "/tmp/opencode/sem_stream.txt") +# Liveness file checked by pin_supervisor.sh (mtime must stay < 75 s; the +# supervisor grants a 120 s startup grace on top). +HEARTBEAT_FILE = os.environ.get("PIN_GR_HEARTBEAT", "/tmp/opencode/pin_gr_heartbeat") +# Inter-tool re-assert request (written by semfix_uvmt.py's stall watchdog): +# when its mtime moves, re-assert every pair this tool has ever pinned. +REASSERT_REQ = os.environ.get("PIN_REASSERT_FILE", + "/tmp/opencode/pin_reassert_request") +# Stall watchdog: a known client with no pin/data activity for this long gets +# an automatic recovery (full re-assert + demand of the fixed counter/pool +# VAs). Pre-capture (no pin yet for this client) uses the tighter limit. +STALL_PRE_S = float(os.environ.get("PIN_STALL_PRE", "60")) +STALL_POST_S = float(os.environ.get("PIN_STALL_POST", "120")) +STALL_RETRY_S = float(os.environ.get("PIN_STALL_RETRY", "30")) +STALL_ESCALATE_S = float(os.environ.get("PIN_STALL_ESCALATE", "180")) def log(msg): print("[%s] %s" % (time.strftime("%H:%M:%S"), msg), flush=True) +def _heartbeat(): + try: + with open(HEARTBEAT_FILE, "w") as f: + f.write("%.3f\n" % time.time()) + except OSError: + pass + + +def _hb_loop(): + """Refresh the heartbeat on a fixed cadence, independent of data-path + activity: recv can be legally idle for minutes between client phases and + the supervisor used to read that silence as a hung tool (r198 restarts at + 34 s staleness while injection bursts were running).""" + while True: + _heartbeat() + time.sleep(5) + + def _mirror(line): """Append guest control lines (#pid/#sem) to the shared host file so the single-QMP watcher can consume them without guest-agent round trips.""" @@ -56,18 +89,68 @@ def vexec(cmd, timeout=40): return "" +def _scanner_alive(): + """True when the in-guest scanner process still exists. + + install_and_start() pkill's and restarts the scanner, which resets every + in-guest latch; probing first avoids killing a healthy scanner that is + merely silent because no client exists yet (r198 pre-client churn).""" + out = vexec("sudo sh -c 'kill -0 $(cat /tmp/guest_pin_stream.pid) " + "2>/dev/null && echo ALIVE'") + return "ALIVE" in (out or "") + + +def _demand_write_cmd(vas): + """Build a transport-proof append command for the guest demand file. + + r582: the old `printf '%s' '<lines with real LF>'` form was mangled by + the guest-agent transport under load - the newlines arrived as LITERAL + backslash-n text and merged VAs onto one line, and the guest scanner's + parser then crashed on `int('0x20400f000\\\\n0x121000000...', 16)` + (ValueError in main; r581 boot scanner log). The scanner's death stopped + ALL new pins while the host only re-asserted known pairs - the observed + pool-pin-latency (0x121000000 never arrived while other injections + flowed). One `echo <va> >> file` per VA carries no newlines and no + backslashes at all, so no transport layer can escape them.""" + return "; ".join("echo 0x%x >> /tmp/guest_want_vas.txt" % va + for va in vas) + + +def _demand_send(vas): + """Append VA hints to the guest scanner's demand file (best-effort). + + Same channel pin_gr_stream._demand_loop uses for host-dmesg stalls; the + stall watchdog uses it to ask the scanner to re-resolve the fixed + pre-capture VAs (counter page / UVM pool) by content/pagemap.""" + cmd = _demand_write_cmd(vas) + if not cmd: + return + try: + vexec(cmd, 30) + except Exception: + pass + + def _start_cmd(): with open(GUEST_SCRIPT) as f: b64 = subprocess.run(["base64", "-w0"], stdin=f, - capture_output=True, text=True).stdout + capture_output=True, text=True, + timeout=10).stdout prefixes = os.environ.get("CLIENT_PREFIXES", "cdt,va,victory") vma_every = os.environ.get("STREAM_VMA_EVERY", "1") - return ("echo '%s' | base64 -d > /tmp/guest_pin_stream.py; " - "if [ -f /tmp/guest_pin_stream.pid ]; then " - "p=$(cat /tmp/guest_pin_stream.pid 2>/dev/null); " - "[ -n \"$p\" ] && grep -q guest_pin_stream " - "/proc/$p/cmdline 2>/dev/null && sudo kill $p 2>/dev/null; " - "fi; " + # Kill EVERY existing scanner, not just the pid-file one: a retry after + # a partially-failed first attempt (pid file not written yet) used to + # leave the first scanner alive, and multiple scanners each re-assert + # the whole CE pair set - r148 ran three and drove the guest to load 19 + # (clients wedged in userspace spins). The pattern is anchored + # (^python3) and assembled at runtime via printf: an unanchored pattern + # or a literal one would also match this command's own shell (its + # cmdline contains the script path), and pkill would kill the shell + # before it could start the new scanner - every start attempt then + # failed silently (r149). + return ("P=$(printf '^python3 /tmp/guest_pin_%%s.py' stream); " + "sudo pkill -f \"$P\" 2>/dev/null; sleep 1; " + "echo '%s' | base64 -d > /tmp/guest_pin_stream.py; " "sudo rm -f /tmp/guest_pin_stream.pid; " "sudo setsid nohup env CLIENT_PREFIXES='%s' " "STREAM_VMA_EVERY='%s' " @@ -82,6 +165,7 @@ def install_and_start(): # or a stale qga client); retry a few times instead of giving up, because # a boot without the scanner streams no pins and the client then hangs. for attempt in range(5): + _heartbeat() out = vexec(_start_cmd(), 60) if any(ch.isdigit() for ch in out): return True @@ -93,78 +177,297 @@ def install_and_start(): def inject(va, pa, ln): cmd = "printf '0x%x,0x%x,0x%x,1' > %s/sema_map_add" % (va, pa, ln, PARAM) for _ in range(60): - r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], - capture_output=True, text=True) + try: + r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], + capture_output=True, text=True, timeout=10) + except subprocess.TimeoutExpired: + time.sleep(0.05) + continue if r.returncode == 0: return True time.sleep(0.05) return False +def scrub_register(fin_pa, hsem_pa): + """Register the PMA scrubber buffer words (guest-resolved PAs) with the + service so it can author the scrub progress (third-client scrub freeze).""" + cmd = ("printf '0x%x,0x%x' > %s/sema_scrubbuf_add" + % (fin_pa, hsem_pa, PARAM)) + for _ in range(60): + try: + r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], + capture_output=True, text=True, timeout=10) + except subprocess.TimeoutExpired: + time.sleep(0.05) + continue + if r.returncode == 0: + log("scrub registered fin=0x%x hsem=0x%x" % (fin_pa, hsem_pa)) + return True + time.sleep(0.05) + log("scrub register FAILED fin=0x%x hsem=0x%x" % (fin_pa, hsem_pa)) + return False + + +def _demand_loop(): + """Push the host's unreadable CE push VAs to the guest scanner. + + The service logs each distinct stalled VA once per boot ('stalled VA + #N: 0x...'). Each sighting is tracked and re-sent every 20 s for five + minutes: the guest resolves the VA by content against the + nv_vm_map_pages areas (event-independent, no object chain), so a page + that was not yet written when first asked is picked up on a retry. + This closes the stall class where the one-shot #sem kprobe event was + lost and the chain resolver never learns the client. + """ + track = {} # va -> [first_seen, last_sent] + while True: + time.sleep(3) + try: + r = subprocess.run(["sudo", "-n", "dmesg"], + capture_output=True, text=True, timeout=20) + except Exception: + continue + now = time.time() + for m in re.finditer(r"(?:stalled VA #\d+[:=]\s*|pending release starved va=)(0x[0-9a-f]+)", + (r.stdout or "")[-200000:]): + va = int(m.group(1), 16) + track.setdefault(va, [now, 0.0]) + for va in list(track): + first, last = track[va] + if now - first > 300.0: + del track[va] + todo = [va for va, (f, l) in track.items() if now - l > 20.0] + if not todo: + continue + cmd = _demand_write_cmd(todo) + try: + vexec(cmd, 30) + except Exception: + continue + for va in todo: + track[va][1] = now + log("demand: sent %d stalled VA(s) to guest" % len(todo)) + + +_DEMAND_STARTED = False +_HB_STARTED = False +_REASSERT_REQ_SEEN = 0.0 + + def main(): + global _DEMAND_STARTED, _HB_STARTED + _heartbeat() + if not _HB_STARTED: + _HB_STARTED = True + try: + threading.Thread(target=_hb_loop, daemon=True).start() + except Exception as e: + log("WARN: heartbeat thread failed to start: %s" % e) if not install_and_start(): log("WARN: could not start guest scanner (will retry via vsock wait)") + if not _DEMAND_STARTED: + _DEMAND_STARTED = True + try: + threading.Thread(target=_demand_loop, daemon=True).start() + except Exception as e: + log("WARN: demand thread failed to start: %s" % e) srv = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) srv.bind((socket.VMADDR_CID_ANY, PORT)) + srv.settimeout(10) srv.listen(4) log("listening on vsock port %d (window %s..%s)" % (PORT, LO, HI)) seen = {} pin_total = 0 - buf = b"" + last_conn = time.time() + last_reinstall = 0.0 + # Stall-watchdog state for the connected client. client_pid is the + # scanner-reported pid (None until a client appears); last_activity marks + # the last pin/data/control event; pins_since_client counts pages pinned + # for the current client (0 = pre-capture). + client_pid = None + client_start = None + last_activity = time.time() + pins_since_client = 0 + stall_since = None + recover_next = 0.0 + escalated = False + no_conn_warn_next = 0.0 + + def _activity(): + nonlocal last_activity, stall_since, escalated + last_activity = time.time() + stall_since = None + escalated = False + while True: try: conn, _ = srv.accept() + except socket.timeout: + now = time.time() + # No guest scanner connection for 30 s: it died or never started + # (a boot without it streams no pins and every client then hangs). + # Probe the scanner first instead of reinstalling blindly: + # install_and_start() pkill's a live scanner and resets all + # in-guest latches, and pre-client silence is normal (r198 churn). + if now - last_conn > 30.0 and now - last_reinstall >= 60.0: + last_reinstall = now + if _scanner_alive(): + log("no guest connection for %.0fs but scanner alive; " + "not reinstalling" % (now - last_conn)) + if now - last_conn > 180.0 and now >= no_conn_warn_next: + no_conn_warn_next = now + 60.0 + log("STALL WATCHDOG: no guest scanner connection for " + "%.0fs (scanner process alive) - pins/demand may " + "be stranded" % (now - last_conn)) + continue + log("reinstall attempt: no guest connection for %.0fs" + % (now - last_conn)) + try: + install_and_start() + except Exception as e: + log("reinstall failed: %s" % e) + continue except OSError as e: log("accept failed: %s" % e) time.sleep(1) continue + last_conn = time.time() + _heartbeat() conn.settimeout(5) log("guest connected") buf = b"" runs = [] - while True: - try: - chunk = conn.recv(1 << 20) - except socket.timeout: + try: + while True: + try: + chunk = conn.recv(1 << 20) + except socket.timeout: + if runs: + novel = _novel_pages(runs, seen) + pages = _flush(runs, seen) + pin_total += pages + pins_since_client += pages + if novel: + _activity() + runs = [] + _maybe_reassert_request(seen) + now = time.time() + # Stall watchdog: a known client with no pin/data + # activity gets a bounded, idempotent recovery (re-assert + # everything + demand the fixed counter/pool VAs), then a + # loud escalation if the stall persists. This is the + # "first cudaMalloc / first GR capture must not stall + # silently" guard. recover_next rate-limits attempts (the + # loop wakes every 5 s on the recv timeout). + if client_pid is not None: + limit = STALL_PRE_S if pins_since_client == 0 \ + else STALL_POST_S + idle = now - last_activity + if idle >= limit and now >= recover_next: + recover_next = now + STALL_RETRY_S + if stall_since is None: + stall_since = now + log("STALL WATCHDOG: client pid %d - no pin " + "activity for %.0fs (pre-capture=%s) - " + "re-asserting all known pairs + demanding " + "counter/pool VAs" + % (client_pid, idle, pins_since_client == 0)) + if seen: + _flush([], seen, full=True) + _demand_send([0x20400f000, 0x121000000]) + if not escalated and stall_since is not None \ + and now - stall_since >= STALL_ESCALATE_S: + escalated = True + log("STALL WATCHDOG: recovery FAILED for pid %d " + "after %.0fs - stall persists, needs manual " + "attention" % (client_pid, now - stall_since)) + continue + except OSError: + break + if not chunk: + break + _heartbeat() + buf += chunk + while b"\n" in buf: + line, buf = buf.split(b"\n", 1) + text = line.decode(errors="replace").strip() + if not text: + continue + if text.startswith("#pid"): + m = re.match(r"#pid\s+(\d+)(?:\s+(\d+))?", text) + seen = {} + client_pid = int(m.group(1)) if m else None + client_start = (int(m.group(2)) if m and m.group(2) + else None) + pins_since_client = 0 + _activity() + log("guest reports cdt pid %s%s" + % (client_pid if client_pid is not None else "?", + (" start=%s" % client_start + if client_start is not None else ""))) + _mirror(text) + continue + if text.startswith("#nopid"): + # In-band client death: the guest scanner tells us the + # client is gone (with its identity). Drop the per-pair + # dedupe map so the next client's pairs are injected + # unconditionally; semfix_uvmt consumes the mirrored + # line and purges its per-client state too. + client_pid = None + client_start = None + seen = {} + pins_since_client = 0 + _activity() + log("guest reports client gone (%s)" % text) + _mirror(text) + continue + if text.startswith("#sem"): + _mirror(text) + _activity() + continue + if text.startswith("#pb"): + _mirror(text) + _activity() + continue + if text.startswith("#put"): + log(text) + _mirror(text) + _activity() + continue + if text.startswith("#scrub"): + m = re.search(r"0x([0-9a-fA-F]+),0x([0-9a-fA-F]+)", + text) + _mirror(text) + if m: + scrub_register(int(m.group(1), 16), + int(m.group(2), 16)) + continue + m = re.match(r"^(0x[0-9a-f]+),(0x[0-9a-f]+)(?:,(\d+))?$", + text) + if m: + runs.append((int(m.group(1), 16), + int(m.group(2), 16), + int(m.group(3)) if m.group(3) else 1)) if runs: - _flush(runs, seen) + novel = _novel_pages(runs, seen) + pages = _flush(runs, seen) + pin_total += pages + pins_since_client += pages + if novel: + _activity() runs = [] - continue - except OSError: - break - if not chunk: - break - buf += chunk - while b"\n" in buf: - line, buf = buf.split(b"\n", 1) - text = line.decode(errors="replace").strip() - if not text: - continue - if text.startswith("#pid"): - m = re.search(r"(\d+)", text) - seen = {} - log("guest reports cdt pid %s" % (m.group(1) if m else "?")) - _mirror(text) - continue - if text.startswith("#sem"): - _mirror(text) - continue - if text.startswith("#pb"): - _mirror(text) - continue - if text.startswith("#put"): - log(text) - _mirror(text) - continue - m = re.match(r"^(0x[0-9a-f]+),(0x[0-9a-f]+)(?:,(\d+))?$", text) - if m: - runs.append((int(m.group(1), 16), int(m.group(2), 16), - int(m.group(3)) if m.group(3) else 1)) - if runs: - pin_total += _flush(runs, seen) - runs = [] + # Poll the semfix re-assert request on every recv cycle, not + # only on recv-timeouts: sustained scanner traffic starves the + # timeout branch, so the request fired post-mortem in r203 + # (119 s late, `seen` already cleared by #nopid). + _maybe_reassert_request(seen) + except Exception as e: + # A transient error on one connection must not kill the process: + # log it, drop the connection and go back to accept(). + log("connection handler error: %s (continuing)" % e) log("guest disconnected (pins this session: %d)" % pin_total) try: conn.close() @@ -172,19 +475,82 @@ def main(): pass -def _flush(runs, seen): +def _maybe_reassert_request(seen): + """Honour a full-re-assert request written by semfix_uvmt's watchdog.""" + global _REASSERT_REQ_SEEN + try: + m = os.stat(REASSERT_REQ).st_mtime + except OSError: + return + if m > _REASSERT_REQ_SEEN: + _REASSERT_REQ_SEEN = m + if seen: + log("semfix requested a full pin re-assert - re-asserting %d " + "pair(s)" % len(seen)) + _flush([], seen, full=True) + + +def _coalesce_seen(seen): + """Coalesce the seen {va: pa} per-page map into (va, pa, npages) runs.""" + items = sorted(seen.items()) + runs = [] + i = 0 + while i < len(items): + va, pa = items[i] + n = 1 + while (i + n < len(items) + and items[i + n][0] == va + n * 0x1000 + and items[i + n][1] == pa + n * 0x1000): + n += 1 + runs.append((va, pa, n)) + i += n + return runs + + +def _novel_pages(runs, seen): + """Count pages in runs whose translation is new or changed. + + Progress accounting for the stall watchdog: the guest re-streams the CE + window and UVM pool every couple of seconds even when nothing moves, so a + pure repeat of an already-known pair must not count as pin activity + (r203: those repeats masked the watchdog through every hang).""" + return sum(1 for va, pa, n in runs for i in range(n) + if seen.get(va + i * 0x1000) != pa + i * 0x1000) + + +def _flush(runs, seen, full=False): """Inject run lines (one batched sudo pass), return number of pages pinned. The old implementation spawned one sudo process per run; with ~400 runs that cost ~4 s, which landed the GR pins after the client's context-init deadline. The batch writes all runs in a single shell, retrying only the ones the kernel queue rejected (ENOSPC) individually. + + full=True re-asserts every pair ever pinned (coalesced), not just this + batch's runs: the kernel session flush wipes the pin table when a client's + channels close, and the stall watchdog / semfix re-assert request use this + to heal it without touching the guest. Writes are idempotent. """ - todo = [] + todo = {} for va, pa, n in runs: - if seen.get(va) == pa and n == 1: + # The CE window (0x120000000..0x121000000) and the UVM semaphore + # pool page range (0x121000000..0x122000000) are re-asserted by the + # guest every couple of seconds on purpose: the host's session flush + # wipes the whole pin table when the previous client's last channel + # closes, and a pin batch that raced in for the new client must be + # allowed to land again even when the pair is unchanged. Skipping + # repeats there stranded the second client's CE pushes (the r144/ + # r145 stall: sema-probe "NO ENTRY" for the live push pages) and the + # pool/tracker page (pending release starved, every client's first + # seconds). + ce_win = 0x120000000 <= va < 0x122000000 + if not full and not ce_win and seen.get(va) == pa and n == 1: continue - todo.append((va, pa, n)) + todo[(va, pa, n)] = True + if full: + for va, pa, n in _coalesce_seen(seen): + todo[(va, pa, n)] = True + todo = list(todo) if not todo: return 0 @@ -202,6 +568,7 @@ def _flush(runs, seen): CHUNK = 200 for start in range(0, len(script), CHUNK): batch = script[start:start + CHUNK] + _heartbeat() try: r = subprocess.run(["sudo", "-n", "sh", "-c", "\n".join(batch)], capture_output=True, text=True, timeout=60) @@ -235,4 +602,12 @@ def _flush(runs, seen): if __name__ == "__main__": - main() + # main() loops forever; only an unexpected exception exits it. Restart + # instead of dying silently - a dead listener means no pins and every + # later CUDA client hangs at first cudaMalloc. + while True: + try: + main() + except Exception as e: + log("FATAL: main() exited: %r - restarting in 2s" % (e,)) + time.sleep(2) diff --git a/tools/guest-diag/pin_supervisor.sh b/tools/guest-diag/pin_supervisor.sh new file mode 100755 index 0000000..b87d5c1 --- /dev/null +++ b/tools/guest-diag/pin_supervisor.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Open-IOV contributors +# pin_supervisor.sh - supervise the host-side pin tools (pin_gr_stream.py and +# semfix_uvmt.py) so a silent death cannot strand every later CUDA client at +# first cudaMalloc. +# +# Both tools write a heartbeat file; this script checks, every 5 s, that each +# process is alive (exact script path) and that its heartbeat mtime is < 75 s. +# A startup grace (120 s, skipped early once the tool log shows it has done +# first useful work) keeps a slow cold start from being restarted. On failure +# it kills the tool (exact path match) and restarts it, appending RESTART +# lines to the supervisor log. After 10 restarts in one hour for a tool it +# logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever started +# (QEMU's QMP socket accepts a single client). +# +# Liveness policy (do not tighten without re-checking the soak evidence): +# * the tools refresh their heartbeats from dedicated loops/threads even +# while idle (pin_gr_stream: 5 s thread; semfix: per loop + around every +# blocking guest-agent/dmesg call), so 75 s never trips on a healthy +# client between phases; +# * a DEAD process is restarted on the next 5 s tick regardless of grace; +# * dead-pid/state recovery (the kill cascade) is handled INSIDE the tools +# (in-band #nopid + idle liveness probe + stall watchdog), NOT by this +# supervisor - a log-based restart here could fire mid-run of a healthy +# client (the m178 lesson). +# +# Usage: pin_supervisor.sh LOGDIR TAG +# Logs: LOGDIR/12-pin-supervisor-TAG.log (supervisor) +# LOGDIR/12-pin_gr_stream-TAG.log, LOGDIR/12-semfix-TAG.log (tools) +set -uo pipefail + +LOGDIR="${1:?usage: pin_supervisor.sh LOGDIR TAG}" +TAG="${2:?usage: pin_supervisor.sh LOGDIR TAG}" + +HERE=$(cd "$(dirname "$0")" && pwd) +PIN="$HERE/pin_gr_stream.py" +SEMFIX="$HERE/semfix_uvmt.py" +PIN_HB="${PIN_GR_HEARTBEAT:-/tmp/opencode/pin_gr_heartbeat}" +SEMFIX_HB="${SEMFIX_HEARTBEAT_FILE:-/tmp/opencode/semfix_heartbeat}" +SUPLOG="$LOGDIR/12-pin-supervisor-$TAG.log" +PIN_LOG="$LOGDIR/12-pin_gr_stream-$TAG.log" +SEMFIX_LOG="$LOGDIR/12-semfix-$TAG.log" +WINDOW=3600 +CAP=10 +STALE=75 +GRACE=120 +mkdir -p "$LOGDIR" /tmp/opencode + +log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$SUPLOG"; } + +tool_alive() { pgrep -f "python3 $1" >/dev/null 2>&1; } + +hb_age() { # age in seconds of the heartbeat file, 9999 if missing + local f="$1" m + m=$(stat -c %Y "$f" 2>/dev/null) || { echo 9999; return; } + echo $(( $(date +%s) - m )) +} + +kill_tool() { + pkill -f "python3 $1" 2>/dev/null + sleep 1 + if pgrep -f "python3 $1" >/dev/null 2>&1; then + pkill -9 -f "python3 $1" 2>/dev/null + sleep 1 + fi + return 0 +} + +start_pin() { + if tool_alive "$PIN"; then return 0; fi + touch "$PIN_HB" 2>/dev/null || true + ( setsid nohup env SEMFIX_COUNTER_RETRY=10 \ + python3 "$PIN" >> "$PIN_LOG" 2>&1 & ) + pin_started=$(date +%s) + log "START pin_gr_stream" +} + +start_semfix() { + # pgrep exact-count check before start: if an instance already exists + # (ours or a leftover), never start a second one. + if tool_alive "$SEMFIX"; then + log "semfix already running - not starting another" + return 0 + fi + touch "$SEMFIX_HB" 2>/dev/null || true + ( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 \ + python3 "$SEMFIX" 0.5 >> "$SEMFIX_LOG" 2>&1 & ) + semfix_started=$(date +%s) + log "START semfix_uvmt" +} + +prune() { # $1 now, $2 list -> list with entries older than WINDOW dropped + local now="$1" t out="" + for t in $2; do + [ $(( now - t )) -lt "$WINDOW" ] && out="$out $t" + done + printf '%s' "$out" +} + +count_list() { # $1 list -> element count + local n=0 t + for t in $1; do n=$(( n + 1 )); done + echo "$n" +} + +restarts_pin="" +restarts_semfix="" +pin_started=$(date +%s) +semfix_started=$(date +%s) + +restart() { # $1 label, $2 script path, $3 start function + local now list n + now=$(date +%s) + if [ "$1" = pin_gr_stream ]; then + restarts_pin=$(prune "$now" "$restarts_pin") + n=$(count_list "$restarts_pin") + else + restarts_semfix=$(prune "$now" "$restarts_semfix") + n=$(count_list "$restarts_semfix") + fi + if [ "$n" -ge "$CAP" ]; then + log "GIVEUP $1: $n restarts within ${WINDOW}s" + exit 1 + fi + kill_tool "$2" + if [ "$1" = pin_gr_stream ]; then + restarts_pin="$restarts_pin $now" + else + restarts_semfix="$restarts_semfix $now" + fi + log "RESTART $(( n + 1 )) $1 $(date +%Y-%m-%dT%H:%M:%S)" + "$3" +} + +check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function, + # $5 tool start TS, $6 tool log file, $7 first-work pattern + local age runtime + age=$(hb_age "$3") + if tool_alive "$2" && [ "$age" -le "$STALE" ]; then + return 0 + fi + if tool_alive "$2"; then + # Startup grace: a cold tool can spend its first ~2 min installing the + # guest scanner or connecting QMP with a lagging heartbeat; only + # enforce staleness once the grace has passed or the log shows the + # tool has already done successful work. + runtime=$(( $(date +%s) - $5 )) + if [ "$runtime" -lt "$GRACE" ] && \ + ! grep -q -- "$7" "$6" 2>/dev/null; then + return 0 + fi + log "$1 heartbeat stale (${age}s) - restarting" + else + log "$1 not running (heartbeat age ${age}s) - restarting" + fi + restart "$1" "$2" "$4" +} + +log "pin_supervisor up (LOGDIR=$LOGDIR TAG=$TAG)" +start_pin +start_semfix +while true; do + sleep 5 + check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin "$pin_started" \ + "$PIN_LOG" "listening on vsock" + check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix \ + "$semfix_started" "$SEMFIX_LOG" "semfix_uvmt: watching" +done diff --git a/tools/guest-diag/pool_page.py b/tools/guest-diag/pool_page.py new file mode 100755 index 0000000..a9f7b15 --- /dev/null +++ b/tools/guest-diag/pool_page.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# pool_page.py <pa-hex> - dump the UVM semaphore-pool page's u32 slots via +# /proc/kcore (guest root). The pool VA 0x121000000 maps to the PA the +# scanner+host pinned; nonzero slots are the applied release values. Compare +# against the tracker's expected payload (tracker_walk.py) and the driver's +# uvsv wait value to find write-target divergence / starvation. +import sys +import struct + +pa = int(sys.argv[1], 16) if len(sys.argv) > 1 else 0x107fd8000 +sys.argv = ["pool_page"] +sys.path.insert(0, "/tmp") +import guest_pin_stream as g # noqa: E402 + +kc = g.Kcore() +if not kc.load(): + print("KCORE LOAD FAILED") + sys.exit(1) + +nz = [] +for off in range(0, 0x1000, 4): + d = kc.read(kc.page_offset_base + pa + off, 4) + v = struct.unpack("<I", d)[0] if d else 0 + if v: + nz.append((off, v)) +print("pa=0x%x nonzero slots=%d" % (pa, len(nz))) +for off, v in nz[:64]: + print("off=0x%x val=%u" % (off, v)) diff --git a/tools/guest-diag/qga_fetch.py b/tools/guest-diag/qga_fetch.py new file mode 100755 index 0000000..5a940fc --- /dev/null +++ b/tools/guest-diag/qga_fetch.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# Fetch a file from the guest via the QEMU guest agent protocol +# (/tmp/qemu-guest-agent.sock, the virtio-serial QGA chardev - the same +# socket vgpu_vm.py exec uses). Usage: qga_fetch.py <guest_path> <host_path> [chunk] +import base64 +import json +import socket +import sys + +GUEST = sys.argv[1] +HOST = sys.argv[2] +CHUNK = int(sys.argv[3]) if len(sys.argv) > 3 else 1 << 20 +SOCK = "/tmp/qemu-guest-agent.sock" + + +def recv_line(s): + buf = b"" + while b"\n" not in buf: + chunk = s.recv(65536) + if not chunk: + break + buf += chunk + return buf.split(b"\n", 1)[0] + + +def cmd(s, obj): + s.sendall((json.dumps(obj) + "\r\n").encode()) + line = recv_line(s) + return json.loads(line) + + +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +s.settimeout(30.0) +s.connect(SOCK) +r = cmd(s, {"execute": "guest-file-open", + "arguments": {"path": GUEST, "mode": "r"}}) +h = r["return"] +print(f"handle={h}") +total = 0 +with open(HOST, "wb") as out: + while True: + r = cmd(s, {"execute": "guest-file-read", + "arguments": {"handle": h, "count": CHUNK}}) + d = r["return"] + buf = base64.b64decode(d.get("buf-b64", "")) + if buf: + out.write(buf) + total += len(buf) + print(f"\r{total}", end="", flush=True) + if d.get("eof"): + break +cmd(s, {"execute": "guest-file-close", "arguments": {"handle": h}}) +print(f"\nFETCHED {total} bytes -> {HOST}") diff --git a/tools/guest-diag/qga_snip.py b/tools/guest-diag/qga_snip.py new file mode 100644 index 0000000..cd3b992 --- /dev/null +++ b/tools/guest-diag/qga_snip.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# Fetch a byte range from a guest file via the QEMU guest agent. +# Usage: qga_snip.py <guest_path> <offset> <length> <host_out> +import base64 +import json +import socket +import sys + +GUEST, OFF, LEN, HOST = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), sys.argv[4] +import os +SOCK = os.environ.get("QGA_SOCK", "/tmp/qemu-guest-agent.sock") + + +def recv_line(s): + buf = b"" + while b"\n" not in buf: + c = s.recv(65536) + if not c: + break + buf += c + return buf.split(b"\n", 1)[0] + + +def cmd(s, obj): + s.sendall((json.dumps(obj) + "\r\n").encode()) + return json.loads(recv_line(s)) + + +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +s.settimeout(30.0) +s.connect(SOCK) +h = cmd(s, {"execute": "guest-file-open", + "arguments": {"path": GUEST, "mode": "r"}})["return"] +cmd(s, {"execute": "guest-file-seek", + "arguments": {"handle": h, "offset": OFF, "whence": 0}}) +r = cmd(s, {"execute": "guest-file-read", + "arguments": {"handle": h, "count": LEN}}) +data = base64.b64decode(r["return"].get("buf-b64", "")) +open(HOST, "wb").write(data) +cmd(s, {"execute": "guest-file-close", "arguments": {"handle": h}}) +print(f"GOT {len(data)} bytes @ {OFF} -> {HOST}") diff --git a/tools/guest-diag/read_proc_va.py b/tools/guest-diag/read_proc_va.py new file mode 100644 index 0000000..c035840 --- /dev/null +++ b/tools/guest-diag/read_proc_va.py @@ -0,0 +1,16 @@ +import os, sys +pat = sys.argv[1] +vas = [int(a, 16) for a in sys.argv[2:]] +pids = os.popen("pgrep -x " + pat).read().split() +if not pids: + print("NO-PID"); sys.exit(0) +pid = int(pids[0]) +print("pid", pid) +fd = os.open("/proc/%d/mem" % pid, os.O_RDONLY) +for a in vas: + try: + os.lseek(fd, a, 0) + d = os.read(fd, 4) + print(hex(a), d.hex()) + except OSError as e: + print(hex(a), "ERR", e) diff --git a/tools/guest-diag/redboot-triage.sh b/tools/guest-diag/redboot-triage.sh new file mode 100755 index 0000000..041e9a6 --- /dev/null +++ b/tools/guest-diag/redboot-triage.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# redboot-triage.sh <live-log> [base-va] - pull the r613b/r613d wedge-class +# evidence from a boot's dmesg capture in one pass. +# usage: redboot-triage.sh /tmp/opencode/rNNN-live.log [0x7fc716c00000] +set -u +LOG="${1:?usage: redboot-triage.sh <log> [base-va]}" +VA="${2:-}" +echo "=== $LOG ===" +echo "--- sema-shadow-dst (r614 witness; the red-boot entry point)" +grep -h "sema-shadow-dst" "$LOG" | head -12 +echo "--- GRFILL-DROP + held-for-retry + degenerate refusal" +grep -hE "GRFILL-DROP|launch held for a bounded retry|GR submit refused|GR forward failed|GR-STALL-fwd-fail|GR-COMPLETE-SW" "$LOG" | tail -24 +echo "--- arg-miss walk lists (r614: full, untruncated)" +grep -h "arg-miss" "$LOG" | tail -6 +echo "--- cand[0] arg base probes (first resolution of each client)" +grep -h "cand\[0\] arg base" "$LOG" | tail -12 +echo "--- EPOST drop signature (status-ring starvation rounds)" +grep -h "no postable event" "$LOG" | tail -6 +if [ -n "$VA" ]; then + echo "--- all lines for base $VA" + grep -h "$VA" "$LOG" | grep -vE "ldatrace|dmatrace" | head -30 +fi +echo "=== done ==="
\ No newline at end of file diff --git a/tools/guest-diag/sdl_cap.c b/tools/guest-diag/sdl_cap.c new file mode 100644 index 0000000..23c83ab --- /dev/null +++ b/tools/guest-diag/sdl_cap.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: BSD-3-Clause +// sdl_cap.c - LD_PRELOAD capture shim: records every frame GZDoom presents +// through SDL into an append-only file, so the host can harvest the guest's +// real rendered frames without any X/driver readback path. +// +// Hooks: +// SDL_RenderPresent(SDL_Renderer*) -> SDL_RenderReadPixels(ARGB8888) +// SDL_UpdateWindowSurface(SDL_Window*) -> SDL_GetWindowSurface pixels +// Both forward to the real SDL after capturing. +// +// File layout (little endian): +// [0] 8B magic "GZCAPV01" (+ 4088 B reserved zero) +// [4096] frame records: +// 4B "GZF2" | u32 w | u32 h | u32 stride | u32 bpp +// u64 ts_ns (CLOCK_MONOTONIC) | u64 seq | stride*h bytes (top-down) +// A trailing record with seq = 0xFFFFFFFFFFFFFFFF marks clean stop. +// +// Env: GZCAP (path, default /dev/shm/gzframes.bin), GZCAP_MAXF (0=unlimited) +#define _GNU_SOURCE +#include <dlfcn.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#define ARGB8888 0x16362004u + +typedef struct SDL_Renderer SDL_Renderer; +typedef struct SDL_Window SDL_Window; + +typedef struct SDL_Surface { + uint32_t flags; + void *format; + int w, h; + int pitch; + void *pixels; + void *userdata; + int locked; + void *list_blitmap; + struct { int x, y, w, h; } clip_rect; + void *map; + int refcount; +} SDL_Surface; + +static int (*real_RenderReadPixels)(SDL_Renderer *, const void *, uint32_t, + void *, int); +static int (*real_RenderPresent)(SDL_Renderer *); +static SDL_Surface *(*real_GetWindowSurface)(SDL_Window *); +static int (*real_UpdateWindowSurface)(SDL_Window *); +static int (*real_GetRendererOutputSize)(SDL_Renderer *, int *, int *); +static void (*real_GL_SwapWindow)(SDL_Window *); +static int (*real_GL_GetDrawableSize)(SDL_Window *, int *, int *); +static void (*real_glReadPixels)(int, int, int, int, unsigned, unsigned, void *); + +static FILE *cap_fp; +static unsigned long long cap_count, cap_max = 0; +static int cap_stop; + +__attribute__((constructor)) static void cap_init(void) +{ + const char *p = getenv("GZCAP"); + const char *m = getenv("GZCAP_MAXF"); + char hdr[4096]; + + if (m) + cap_max = strtoull(m, NULL, 0); + if (!p) + p = "/dev/shm/gzframes.bin"; + real_RenderReadPixels = dlsym(RTLD_NEXT, "SDL_RenderReadPixels"); + real_RenderPresent = dlsym(RTLD_NEXT, "SDL_RenderPresent"); + real_GetWindowSurface = dlsym(RTLD_NEXT, "SDL_GetWindowSurface"); + real_UpdateWindowSurface = dlsym(RTLD_NEXT, "SDL_UpdateWindowSurface"); + real_GetRendererOutputSize = + dlsym(RTLD_NEXT, "SDL_GetRendererOutputSize"); + real_GL_SwapWindow = dlsym(RTLD_NEXT, "SDL_GL_SwapWindow"); + real_GL_GetDrawableSize = dlsym(RTLD_NEXT, "SDL_GL_GetDrawableSize"); + real_glReadPixels = dlsym(RTLD_NEXT, "glReadPixels"); + if (!real_glReadPixels) + real_glReadPixels = dlsym(RTLD_DEFAULT, "glReadPixels"); + if (!real_glReadPixels) { + void *h = dlopen("libGL.so.1", RTLD_NOW | RTLD_GLOBAL); + + if (!h) + h = dlopen("libGLESv2.so.2", RTLD_NOW | RTLD_GLOBAL); + if (h) + real_glReadPixels = dlsym(h, "glReadPixels"); + } + fprintf(stderr, "[gzcap] init path=%s present=%p updwin=%p swap=%p read=%p gros=%p\n", + p, (void *)real_RenderPresent, (void *)real_UpdateWindowSurface, + (void *)real_GL_SwapWindow, (void *)real_RenderReadPixels, + (void *)real_GetRendererOutputSize, (void *)real_glReadPixels); + cap_fp = fopen(p, "wb"); + if (!cap_fp) + return; + memset(hdr, 0, sizeof(hdr)); + memcpy(hdr, "GZCAPV01", 8); + fwrite(hdr, 1, sizeof(hdr), cap_fp); + fflush(cap_fp); +} + +static void cap_record(const void *px, int w, int h, int stride, int bpp) +{ + struct timespec ts; + uint64_t tns, seq; + uint32_t rh[4]; + char m[4] = { 'G', 'Z', 'F', '2' }; + + if (!cap_fp || cap_stop) + return; + if (cap_max && cap_count >= cap_max) { + uint64_t endseq = 0xFFFFFFFFFFFFFFFFull; + + fwrite(m, 1, 4, cap_fp); + fwrite(&endseq, 8, 1, cap_fp); + fflush(cap_fp); + cap_stop = 1; + return; + } + clock_gettime(CLOCK_MONOTONIC, &ts); + tns = (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; + seq = cap_count; + rh[0] = (uint32_t)w; + rh[1] = (uint32_t)h; + rh[2] = (uint32_t)stride; + rh[3] = (uint32_t)bpp; + fwrite(m, 1, 4, cap_fp); + fwrite(rh, 4, 4, cap_fp); + fwrite(&tns, 8, 1, cap_fp); + fwrite(&seq, 8, 1, cap_fp); + fwrite(px, 1, (size_t)stride * h, cap_fp); + fflush(cap_fp); + cap_count++; + if ((cap_count & 31) == 0) + fprintf(stderr, "[gzcap] frames=%llu\n", cap_count); +} + +int SDL_RenderPresent(SDL_Renderer *r) +{ + static int hit; + + if (!hit++) + fprintf(stderr, "[gzcap] hook RenderPresent\n"); + if (cap_fp && !cap_stop && real_RenderReadPixels && + real_GetRendererOutputSize) { + int w = 0, h = 0; + + if (real_GetRendererOutputSize(r, &w, &h) == 0 && w > 0 && h > 0 && + w <= 4096 && h <= 4096) { + static unsigned char *buf; + static size_t bufsz; + size_t need = (size_t)w * 4 * h; + + if (bufsz < need) { + free(buf); + buf = malloc(need); + bufsz = buf ? need : 0; + } + if (buf && real_RenderReadPixels(r, NULL, ARGB8888, buf, + w * 4) == 0) + cap_record(buf, w, h, w * 4, 32); + } + } + if (real_RenderPresent) + return real_RenderPresent(r); + return 0; +} + +int SDL_UpdateWindowSurface(SDL_Window *win) +{ + static int hit; + + if (!hit++) + fprintf(stderr, "[gzcap] hook UpdateWindowSurface\n"); + if (cap_fp && !cap_stop && real_GetWindowSurface) { + SDL_Surface *s = real_GetWindowSurface(win); + + if (s && s->pixels && s->w > 0 && s->h > 0 && s->pitch > 0 && + s->w <= 4096 && s->h <= 4096) + cap_record(s->pixels, s->w, s->h, s->pitch, 32); + } + if (real_UpdateWindowSurface) + return real_UpdateWindowSurface(win); + return 0; +} + +void SDL_GL_SwapWindow(SDL_Window *win) +{ + static int hit; + static unsigned char *gbuf; + static size_t gbufsz; + + if (!hit++) + fprintf(stderr, "[gzcap] hook GLSwapWindow\n"); + if (cap_fp && !cap_stop && real_glReadPixels && real_GL_GetDrawableSize) { + int w = 0, h = 0; + + if (real_GL_GetDrawableSize(win, &w, &h) == 0 && w > 0 && h > 0 && + w <= 4096 && h <= 4096) { + size_t need = (size_t)w * 4 * h; + + if (gbufsz < need) { + free(gbuf); + gbuf = malloc(need); + gbufsz = gbuf ? need : 0; + } + if (gbuf) { + real_glReadPixels(0, 0, w, h, 0x1908, 0x1401, gbuf); + /* GL rows are bottom-up: flip into a scratch record */ + { + unsigned char *tmp = malloc(need); + int y; + + if (tmp) { + for (y = 0; y < h; y++) + memcpy(tmp + (size_t)y * w * 4, + gbuf + (size_t)(h - 1 - y) * w * 4, + (size_t)w * 4); + cap_record(tmp, w, h, w * 4, 32); + free(tmp); + } + } + } + } + } + if (real_GL_SwapWindow) + real_GL_SwapWindow(win); +} diff --git a/tools/guest-diag/semfix_uvmt.py b/tools/guest-diag/semfix_uvmt.py index cf54445..11c191c 100755 --- a/tools/guest-diag/semfix_uvmt.py +++ b/tools/guest-diag/semfix_uvmt.py @@ -53,14 +53,47 @@ SKIP_PB = os.environ.get("SEMFIX_SKIP_PB", "0") == "1" # Seconds between counter-page pin attempts (the guest-agent exec blocks for # its full timeout when the client saturates the guest). COUNTER_RETRY_S = float(os.environ.get("SEMFIX_COUNTER_RETRY", "10")) +# Liveness file checked by pin_supervisor.sh (mtime must stay < 75 s; the +# supervisor grants a 120 s startup grace on top). +SEMFIX_HB_FILE = os.environ.get("SEMFIX_HEARTBEAT_FILE", + "/tmp/opencode/semfix_heartbeat") +# Dead-pid/stall watchdog tuning (seconds). The tracked client is probed in +# the guest only when it has been idle this long, on a cadence, so live +# clients are never probed and a killed client is detected within ~1 min. +LIVENESS_IDLE_S = float(os.environ.get("SEMFIX_LIVENESS_IDLE", "15")) +LIVENESS_EVERY_S = float(os.environ.get("SEMFIX_LIVENESS_EVERY", "25")) +# Pre-capture stall: client alive, counter page not pinned, no progress for +# this long -> recovery attempt; escalate loudly if it persists. +STALL_PRECAP_S = float(os.environ.get("SEMFIX_STALL_PRECAP", "60")) +STALL_RETRY_S = float(os.environ.get("SEMFIX_STALL_RETRY", "30")) +STALL_ESCALATE_S = float(os.environ.get("SEMFIX_STALL_ESCALATE", "180")) +# Inter-tool re-assert request: written here, consumed by pin_gr_stream.py +# (full ping re-assert of every pair it has ever pinned). +REASSERT_REQ = os.environ.get("PIN_REASSERT_FILE", + "/tmp/opencode/pin_reassert_request") + + +def _hb(): + try: + with open(SEMFIX_HB_FILE, "w") as f: + f.write("%.3f\n" % time.time()) + except OSError: + pass class Stream: """Tail the guest scanner's control lines mirrored by pin_gr_stream.py. - The guest emits "#pid <n>" and "#sem 0x<k>" over vsock; the listener - appends them here. This replaces per-loop guest-agent trace reads, which - were too slow to catch short-lived tracking semaphores. + The guest emits "#pid <n> [starttime]" and "#sem 0x<k>" over vsock; the + listener appends them here. This replaces per-loop guest-agent trace + reads, which were too slow to catch short-lived tracking semaphores. + + Replay safety (the m178 regression): the file persists across tool + restarts, so a fresh process re-reads the whole history. Lines written + before this process started (byte offset < init_size) are absorbed as + HISTORY - they seed the dedupe/identity state and the sem/pb lists but + can never trigger a per-client reset. Only lines appended afterwards + are live. This makes a supervisor restart mid-run harmless. """ def __init__(self, path): @@ -70,6 +103,15 @@ class Stream: self.pids = [] self.pbs = [] self.new_pbs = [] + self.new_nopids = [] + self.history_pids = [] # (pid, start) identities from history + self.pid_keys = set() # (pid, start) dedupe, per generation + try: + # Everything already in the file when this process starts is + # history (a restart replays it); never reset on it. + self._init_size = os.path.getsize(path) + except OSError: + self._init_size = 0 def reset(self): """Forget everything learned about the previous client. A new client @@ -80,6 +122,16 @@ class Stream: self.pbs = [] self.pids = [] self.new_pbs = [] + self.new_nopids = [] + self.pid_keys = set() + + @staticmethod + def _ident(line): + """Parse '#pid 1234' or '#pid 1234 5678900' -> (pid, start|None).""" + nums = re.findall(r"\d+", line) + if not nums: + return None, None + return int(nums[0]), (int(nums[1]) if len(nums) > 1 else None) def poll(self): try: @@ -90,12 +142,19 @@ class Stream: self.off = 0 f.seek(self.off) data = f.read() + base = self.off self.off = f.tell() except OSError: - return [], [] + return [], [], [] new_sems, new_pids = [], [] self.new_pbs = [] + self.new_nopids = [] + pos = base for line in data.splitlines(): + # A line is live when it ends past the startup boundary; history + # lines are absorbed (identity/dedupe) but never emitted as live. + live = pos + len(line) + 1 > self._init_size + pos += len(line) + 1 if line.startswith("#sem"): m = re.search(r"0x([0-9a-fA-F]+)", line) if m: @@ -111,13 +170,116 @@ class Stream: self.pbs.append(v) self.new_pbs.append(v) elif line.startswith("#pid"): - m = re.search(r"(\d+)", line) - if m: - v = int(m.group(1)) - if v not in self.pids: - self.pids.append(v) - new_pids.append(v) - return new_sems, new_pids + pid, start = self._ident(line) + if pid is not None and (pid, start) not in self.pid_keys: + self.pid_keys.add((pid, start)) + self.pids.append(pid) + if live: + new_pids.append((pid, start)) + else: + self.history_pids.append((pid, start)) + elif line.startswith("#nopid"): + # In-band client-death signal (new guest scanner; older ones + # simply never send it and the host probe covers the case). + pid, start = self._ident(line) + if live and pid is not None: + self.new_nopids.append((pid, start)) + return new_sems, new_pids, self.new_nopids + + +class ClientTracker: + """Monotonic, replay-proof client-identity tracking. + + A client's identity is (pid, starttime). Rules, in priority order: + * an identity adopted once is never reset a second time (duplicate + lines, trace/stream replays, supervisor restarts); + * a backwards pid, or the same pid with an older starttime, is a + replay - never reset; + * a pid recycled with a NEWER starttime is a real new client - reset + immediately (the previous holder is definitionally gone); + * a newer unseen identity only resets the per-client state once the + CURRENT client is CONFIRMED gone - a reset fired while the previous + client was still active is the m178 third-client hang regression. + + Pure logic: the caller feeds guest liveness results in and performs the + actual state purge when an action says so. + """ + + def __init__(self): + self.seen = set() + self.current = None + self.pending = None + self.dead = set() + + @staticmethod + def _start_match(a, b): + # Unknown starttimes match conservatively (legacy scanner lines). + return a[1] == b[1] or a[1] is None or b[1] is None + + def observe_history(self, pid, start): + """Absorb an identity found in pre-restart history: the process has + no per-client state yet, so history only seeds identity tracking and + the freshest identity becomes current - no reset is ever requested + for history lines.""" + key = (pid, start) + if key in self.seen: + return + self.seen.add(key) + self.current = key + + def note_dead(self, pid, start): + """A #nopid or a successful dead probe: identity is gone.""" + key = (pid, start) + self.dead.add(key) + if self.current and self.current[0] == pid \ + and self._start_match(self.current, key): + self.current = None + if self.pending and self.pending[0] == pid \ + and self._start_match(self.pending, key): + self.pending = None + + def is_current(self, pid, start): + """True when (pid, start) is the currently tracked identity (used to + decide whether a death signal may purge - a stale #nopid for a + recycled pid must never purge the new holder).""" + return self.current is not None and self.current[0] == pid \ + and self._start_match(self.current, (pid, start)) + + def decide(self, pid, start): + """Classify a live #pid line: adopt | skip | skip_backwards | + maybe_reset | reset.""" + key = (pid, start) + if key in self.seen: + return "skip" + self.seen.add(key) + cur = self.current + if cur is None: + self.current = key + return "adopt" + if pid == cur[0] and self._start_match(cur, key): + return "skip" + if pid == cur[0] and start is not None and cur[1] is not None \ + and start > cur[1]: + # Recycled pid, newer process: safe to reset now. + self.current = key + return "reset" + if pid < cur[0] or (pid == cur[0] and start is not None + and cur[1] is not None and start < cur[1]): + return "skip_backwards" + self.pending = key + return "maybe_reset" + + def resolve(self, old_dead): + """Caller learned whether the CURRENT client is dead. Returns + 'reset' (purge + adopt the pending client) or 'defer'.""" + if self.pending is None: + return None + if old_dead: + key = self.pending + self.pending = None + self.current = key + return "reset" + return "defer" def log(msg): @@ -196,7 +358,9 @@ def dmesg_releases(seen): Only lines after the last "sema-svc: thread started" marker count: dmesg keeps the previous module load's lines and the old client's releases. + Returns the number of newly printed lines (used as watchdog progress). """ + n = 0 try: r = subprocess.run( ["sudo", "-n", "sh", "-c", @@ -205,13 +369,15 @@ def dmesg_releases(seen): "| grep 'sema-push' | tail -300"], capture_output=True, text=True, timeout=10) except Exception: - return + return 0 for line in r.stdout.splitlines(): line = line.strip() if not line or line in seen: continue seen.add(line) + n += 1 log("push: %s" % line.replace("vgsp: ", "")) + return n def cdt_pid(): @@ -224,6 +390,34 @@ def cdt_pid(): return int(pids[-1]) if pids else None +def _pid_alive_cmd(pid): + """Guest shell that reports ALIVE / DEAD for a pid. Zombie-aware: a + killed (SIGKILLed) client lingers as a defunct /proc entry, and a plain + `test -d /proc/<pid>` would latch it as alive forever (the m178 + dead-pid latch).""" + return ("f=/proc/%d/stat; if [ -r \"$f\" ]; then " + "s=$(sed -e 's/^[^)]*) //' -e 's/ .*//' \"$f\"); " + "if [ \"$s\" = Z ] || [ -z \"$s\" ]; then echo DEAD; " + "else echo ALIVE; fi; else echo DEAD; fi" % pid) + + +def pid_alive(pid): + """Guest liveness of a client pid: True alive, False dead/zombie/gone, + None unknown (agent error). Callers must treat None as alive (never + purge a client on an inconclusive probe).""" + try: + r = subprocess.run([VMEXEC, "exec", _pid_alive_cmd(pid)], + capture_output=True, text=True, timeout=20) + except Exception: + return None + out = r.stdout or "" + if "ALIVE" in out: + return True + if "DEAD" in out: + return False + return None + + def pin_counter_page(pid, pinned_pids): """Pin the client's completion-counter page from its own pagemap. @@ -268,10 +462,87 @@ def main(): dseen = set() counter_pids = set() counter_next_try = {} + # Current client pid/starttime, mirrored from the tracker (starttime + # disambiguates recycled pids). last_pid = None + last_start = None + # Stale-sem tracking: consecutive unreadable reads, first-unreadable time, + # and the per-sem read skip deadline once a sem has failed 3 times. + unread_count = {} + first_unreadable_t = {} + skip_until = {} + # Failed pool-pin tracking: retry with a 15 s backoff after 2 failures. + sem_pin_fail = {} + sem_pin_skip_until = {} + qmp_next_try = 0.0 stream = Stream(SEMSTREAM) + tracker = ClientTracker() + # Dead-pid/backwards-transition proofing and the stall watchdog. + history_absorbed = False + adopted_from_history = False + probe_next = 0.0 # next allowed idle-liveness probe + recover_next = 0.0 # next watchdog recovery attempt + stall_since = None # first observation of the pre-capture stall + escalated = False + progress_t = time.time() # last host-side progress for the client loop = 0 + def _clear_client_state(): + """Purge every per-client latch. ONLY valid when the previous client + is CONFIRMED dead (or at fresh process start): the m178 third-client + hang came from wiping the state of an ACTIVE client on a replayed + (#pid) identity.""" + stream.reset() + known_sems.clear() + seen_sem_pin.clear() + last_state.clear() + last_seen_t.clear() + pinned_ranges.clear() + seen_ch.clear() + seen_pb.clear() + counter_pids.clear() + counter_next_try.clear() + sem_pin_fail.clear() + sem_pin_skip_until.clear() + unread_count.clear() + first_unreadable_t.clear() + skip_until.clear() + + def _adopt(pid, start, why): + nonlocal last_pid, last_start, progress_t, adopted_from_history + nonlocal stall_since, escalated + last_pid, last_start = pid, start + progress_t = time.time() + adopted_from_history = False + stall_since = None + escalated = False + log("semfix: tracking client pid %d%s (%s)" + % (pid, " start=%s" % start if start is not None else "", why)) + + def _watchdog_reset(): + nonlocal stall_since, escalated + stall_since = None + escalated = False + + def _watchdog_recover(): + """Idempotent pre-capture stall recovery: re-arm every sem latch so + the next loop re-resolves and re-pins, force an immediate counter + retry, and request a full pin re-assert from pin_gr_stream.""" + for sem in set(known_sems) | set(stream.sems): + skip_until.pop(sem, None) + sem_pin_skip_until.pop(sem, None) + sem_pin_fail.pop(sem, None) + unread_count.pop(sem, None) + first_unreadable_t.pop(sem, None) + seen_sem_pin.discard(sem) + if last_pid is not None: + counter_next_try[last_pid] = 0.0 + try: + with open(REASSERT_REQ, "w") as f: + f.write("%.3f\n" % time.time()) + except OSError: + pass + def pin_range_once(gpu_vas, cpu, gpa, size=0): key = tuple(gpu_vas), cpu, size st = pinned_ranges.get(key) @@ -293,40 +564,104 @@ def main(): while True: loop += 1 + _hb() sems = set() pbs = set() chans = set() - new_sems, new_pids = stream.poll() + new_sems, new_pids, new_nopids = stream.poll() for s in new_sems: log("stream: new uvmt sem 0x%x" % s) - for p in new_pids: - log("stream: client pid %d" % p) - if last_pid is not None and p != last_pid: - stream.reset() - known_sems.clear() - seen_sem_pin.clear() - last_state.clear() - last_seen_t.clear() - pinned_ranges.clear() - seen_ch.clear() - seen_pb.clear() - counter_pids.clear() - log("semfix: new client pid %d - reset per-client pin state" % p) - last_pid = p + if new_sems and last_pid is not None: + progress_t = time.time() + + # In-band client-death signals (new guest scanner): purge the dead + # client's per-client state immediately so the NEXT client works + # without a VM cycle (the kill cascade). + for p, st in new_nopids: + log("stream: client gone (%d%s)" + % (p, " start=%s" % st if st is not None else "")) + was_current = tracker.is_current(p, st) + tracker.note_dead(p, st) + if was_current and last_pid == p: + _clear_client_state() + last_pid = last_start = None + _watchdog_reset() + log("semfix: purged per-client state of exited pid %d" % p) + # The old client is gone: adopt any client whose reset was deferred + # while it was still (assumed) alive. + if last_pid is None and tracker.pending is not None \ + and tracker.current is None: + if tracker.resolve(old_dead=True) == "reset": + p, st = tracker.current + _clear_client_state() + _adopt(p, st, "deferred reset") + # Pre-restart history (a supervisor restart re-reads the file): absorb + # identities WITHOUT any reset - the fresh process already has empty + # state; the freshest identity is the client that was running. + if stream.history_pids and not history_absorbed: + for p, st in stream.history_pids: + tracker.observe_history(p, st) + history_absorbed = True + if tracker.current is not None and last_pid is None: + p, st = tracker.current + _adopt(p, st, "resumed from pre-restart history") + adopted_from_history = True + + for p, st in new_pids: + cur_before = tracker.current + action = tracker.decide(p, st) + tag = " start=%s" % st if st is not None else "" + if action == "skip": + log("stream: client pid %d%s seen before - no reset" % (p, tag)) + elif action == "skip_backwards": + log("semfix: client pid %d%s backwards/replay - no reset" + % (p, tag)) + elif action == "adopt": + log("stream: client pid %d%s" % (p, tag)) + _adopt(p, st, "new client") + elif action == "reset": + # Recycled pid, newer starttime: the previous holder of this + # pid is definitionally gone, safe to reset now. + log("semfix: pid %d recycled (start=%s) - reset per-client " + "pin state" % (p, st)) + _clear_client_state() + _adopt(p, st, "pid recycled") + elif action == "maybe_reset": + old = cur_before + alive = pid_alive(old[0]) if old else False + if alive is False: + log("semfix: new client pid %d%s (pid %d gone) - reset " + "per-client pin state" + % (p, tag, old[0] if old else -1)) + _clear_client_state() + tracker.resolve(old_dead=True) + _adopt(p, st, "client switch") + else: + # NEVER clear the active client's state on the account of + # a newer pid: the m178 hang. Deferred until the old pid + # is confirmed gone (periodic idle probe below). + log("semfix: client pid %d%s appeared while pid %s is " + "alive%s - reset deferred" + % (p, tag, old[0] if old else "?", + "" if alive else " (probe unknown)")) for pb in getattr(stream, "new_pbs", []): log("stream: new pushbuffer 0x%x" % pb) pbs.add(pb) + if getattr(stream, "new_pbs", []) and last_pid is not None: + progress_t = time.time() sems.update(stream.sems) # The guest-agent trace read is slow under client load; use it only # as a fallback for channel/pushbuffer discovery and to recover sems # the scanner may have missed. if loop % TRACE_EVERY == 0: + _hb() try: t_sems, t_pbs, t_chans = guest_trace() except Exception: t_sems, t_pbs, t_chans = set(), set(), set() + _hb() sems.update(t_sems) pbs.update(t_pbs) chans.update(t_chans) @@ -336,16 +671,54 @@ def main(): for sem in sorted(sems - known_sems): known_sems.add(sem) - for sem in sorted(known_sems): + for hb_i, sem in enumerate(sorted(known_sems)): + if hb_i % 16 == 0: + # A full pass can run 60-90 s (thousands of QMP reads); keep + # the heartbeat fresh inside it (r203: 79 s gap -> supervisor + # restarted the tool mid-pass). + _hb() + now = time.time() + if now < skip_until.get(sem, 0): + continue try: st = uvmt_state(q, sem) except Exception: st = None + if st is None: + # Stale-sem hygiene: the trace ring replays dead clients' + # kernel pointers forever; without eviction every loop burns + # QMP reads on them and the log floods with "unreadable". + cnt = unread_count.get(sem, 0) + 1 + unread_count[sem] = cnt + first_unreadable_t.setdefault(sem, now) + if cnt >= 3: + skip_until[sem] = now + 30.0 + if cnt == 3: + log("semfix: sem=0x%x unreadable x3 - retrying every " + "30s" % sem) + age = now - first_unreadable_t[sem] + if age >= 60.0 and sem not in stream.sems: + known_sems.discard(sem) + unread_count.pop(sem, None) + first_unreadable_t.pop(sem, None) + skip_until.pop(sem, None) + last_state.pop(sem, None) + last_seen_t.pop(sem, None) + seen_sem_pin.discard(sem) + sem_pin_fail.pop(sem, None) + sem_pin_skip_until.pop(sem, None) + log("semfix: evicting stale sem=0x%x (unreadable %.0fs, " + "absent from stream)" % (sem, age)) + continue + else: + unread_count.pop(sem, None) + first_unreadable_t.pop(sem, None) + skip_until.pop(sem, None) key = state_key(st) - now = time.time() if key != last_state.get(sem): last_state[sem] = key last_seen_t[sem] = now + progress_t = time.time() log("uvmt %s" % state_str(sem, st)) elif now - last_seen_t.get(sem, 0) >= HEARTBEAT: last_seen_t[sem] = now @@ -354,18 +727,34 @@ def main(): # Pin the pool page once per (sem, word) resolution. The payload # word is what the releasing push writes; pinning the page makes # the host able to apply that release. - if st and sem not in seen_sem_pin: + if (st and sem not in seen_sem_pin + and now >= sem_pin_skip_until.get(sem, 0)): try: pairs = resolve_sem_pool(q, sem) except Exception as e: pairs = [] log("semfix: sem=0x%x resolve failed: %s" % (sem, e)) if pairs: - seen_sem_pin.add(sem) + ok_any = False for gv, gpa in pairs: ok = pin(gv, gpa) + ok_any = ok_any or ok log("semfix: sem=0x%x pool va=0x%x -> gpa=0x%x pin=%s" % (sem, gv, gpa, "ok" if ok else "FAILED")) + if not ok_any: + # Mark pinned only after a successful pin(): a failed + # pin used to be recorded as done and never retried. + fails = sem_pin_fail.get(sem, 0) + 1 + sem_pin_fail[sem] = fails + if fails >= 2: + sem_pin_skip_until[sem] = now + 15.0 + log("semfix: sem=0x%x pool pin failed x%d - retry " + "in 15s" % (sem, fails)) + continue + seen_sem_pin.add(sem) + sem_pin_fail.pop(sem, None) + sem_pin_skip_until.pop(sem, None) + progress_t = time.time() if SKIP_PB: continue d = {} @@ -385,6 +774,7 @@ def main(): d["exception"] = str(e)[:40] if res: pin_range_once(*res) + progress_t = time.time() log("semfix: sem=0x%x pushbuf -> %s" % (sem, diag_str(d))) else: @@ -393,7 +783,22 @@ def main(): else: log("semfix: sem=0x%x pool unresolved (retry)" % sem) + if getattr(q, "_fails", 0) >= 5: + # QMP went away (QEMU restart, monitor died): all reads have been + # empty for 5 consecutive replies. Recreate the client loudly + # instead of reading empty forever ("uvmt(hb) ... unreadable"). + now = time.time() + if now >= qmp_next_try: + log("semfix: QMP unreadable (%d consecutive empty replies) - " + "recreating connection" % q._fails) + try: + q = Qmp(QMP_SOCK) + except Exception as e: + log("semfix: QMP recreate failed: %s" % e) + qmp_next_try = now + 5.0 + for ch in sorted(chans - seen_ch): + _hb() # a resolve can block for tens of seconds d = {} try: res = resolve_from_channel(q, ch, d) @@ -403,10 +808,12 @@ def main(): if res: seen_ch.add(ch) pin_range_once(*res) + progress_t = time.time() log("semfix: ch=0x%x -> %s" % (ch, diag_str(d))) else: log("semfix: ch=0x%x pushbuf miss [%s]" % (ch, diag_str(d))) for pb in sorted(pbs - seen_pb): + _hb() # a resolve can block for tens of seconds d = {} try: res = resolve_from_pushbuffer(q, pb, d) @@ -416,12 +823,15 @@ def main(): if res: seen_pb.add(pb) pin_range_once(*res) + progress_t = time.time() log("semfix: pb=0x%x -> %s" % (pb, diag_str(d))) else: log("semfix: pb=0x%x pushbuf miss [%s]" % (pb, diag_str(d))) if DLOG: - dmesg_releases(dseen) + if dmesg_releases(dseen): + progress_t = time.time() + _hb() # Retry incomplete push-buffer pins: gva2gpa of the kernel vmalloc # alias fails while the sampled vCPU runs in user mode (PTI), so a @@ -430,24 +840,89 @@ def main(): if loop % 8 == 0: for st in list(pinned_ranges.values()): if not st["done"] and st["tries"] > 0: + _hb() # pin_range_once can walk thousands of QMP reads pin_range_once(*st["args"]) + # Dead-pid detection and the pre-capture stall watchdog. The guest is + # probed ONLY when the tracked client has been quiet for a while (a + # live, pushing client is never probed), so the agent is not burdened + # during healthy runs. A confirmed-dead client is purged immediately: + # the NEXT client then starts with clean state - no VM cycle needed + # (the kill cascade). + now = time.time() + if adopted_from_history and last_pid is not None: + # The client resumed from pre-restart history may have died while + # this process was down; do not latch a dead pid into the counter + # retries. + alive = pid_alive(last_pid) + adopted_from_history = False + if alive is False: + log("semfix: resumed pid %d already gone - dropping" % last_pid) + tracker.note_dead(last_pid, last_start) + last_pid = last_start = None + _watchdog_reset() + if last_pid is not None and now >= probe_next \ + and now - progress_t >= LIVENESS_IDLE_S: + probe_next = now + LIVENESS_EVERY_S + alive = pid_alive(last_pid) + if alive is False: + log("semfix: client pid %d gone (probe) - purging per-client " + "state" % last_pid) + tracker.note_dead(last_pid, last_start) + _clear_client_state() + last_pid = last_start = None + _watchdog_reset() + if tracker.pending is not None and tracker.current is None \ + and tracker.resolve(old_dead=True) == "reset": + p, st = tracker.current + log("semfix: adopting deferred client pid %d" % p) + _adopt(p, st, "deferred reset") + elif alive is True and last_pid not in counter_pids: + # Pre-capture stall: the client is alive but has no pinned + # counter page and nothing progressed. Recover (idempotent), + # then escalate loudly if the stall persists - the goal is + # that a first cudaMalloc / first GR capture never stalls + # silently. + idle = now - progress_t + if idle >= STALL_PRECAP_S: + if stall_since is None: + stall_since = now + if now >= recover_next: + recover_next = now + STALL_RETRY_S + log("STALL WATCHDOG: client pid %d alive, no " + "pre-capture progress for %.0fs - recovery " + "attempt (re-arm sems, re-pin counter, request " + "pin re-assert)" % (last_pid, idle)) + _watchdog_recover() + if not escalated and now - stall_since >= STALL_ESCALATE_S: + escalated = True + log("STALL WATCHDOG: recovery FAILED for pid %d " + "after %.0fs (alive, counter page not pinned) - " + "pre-capture hang persists, needs manual " + "attention" % (last_pid, now - stall_since)) + # Pin the completion-counter page of a newly started client, before its # first GR pushes decode their releases. The pid comes from the guest # scanner; the guest-agent fallback is only a safety net. The exec can # block for its whole timeout under client load, so retry on a slow # cadence: hammering the agent here starves it and wedges the run. if last_pid is None and loop % TRACE_EVERY == 0: - last_pid = cdt_pid() - if last_pid: - log("counter: client pid %d (agent)" % last_pid) + p = cdt_pid() + _hb() + if p: + if tracker.current is None and (p, None) not in tracker.seen: + tracker.observe_history(p, None) + last_pid, last_start = p, None + log("counter: client pid %d (agent)" % p) if last_pid and last_pid not in counter_pids: now = time.time() if now >= counter_next_try.get(last_pid, 0): try: - pin_counter_page(last_pid, counter_pids) + if pin_counter_page(last_pid, counter_pids): + progress_t = time.time() except Exception as e: log("counter: pin failed: %s" % e) + _hb() counter_next_try[last_pid] = now + COUNTER_RETRY_S time.sleep(interval) diff --git a/tools/guest-diag/semfix_watch.py b/tools/guest-diag/semfix_watch.py index 5124805..4b60fd6 100644 --- a/tools/guest-diag/semfix_watch.py +++ b/tools/guest-diag/semfix_watch.py @@ -58,12 +58,47 @@ CHAN_MAX_CANDS = int(os.environ.get("CHAN_MAX_CANDS", "256")) class Qmp: def __init__(self, path): + self.path = path + self._cache = {} + self._fails = 0 + self._last_reconnect = 0.0 + self._reconnecting = False self.s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.s.connect(path) self.s.settimeout(15) self._rd() self.cmd("qmp_capabilities") - self._cache = {} + + def reconnect(self): + """Re-establish the QMP connection after the socket died. + + QEMU's QMP socket accepts a single client and closes when the guest + reboots; cmd() used to swallow every exception and return {}, so all + reads stayed empty forever ("uvmt(hb) sem=... unreadable") and + pinning stopped. Rate-limited to one attempt per second so a failing + monitor does not turn every read into a connect storm.""" + if self._reconnecting or time.time() - self._last_reconnect < 1.0: + return + self._reconnecting = True + self._last_reconnect = time.time() + try: + try: + self.s.close() + except Exception: + pass + self.s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.s.connect(self.path) + self.s.settimeout(15) + self._rd() + self.cmd("qmp_capabilities") + self._cache = {} + self._ncpu = 0 + self._cpu = 0 + self._fails = 0 + except Exception: + pass + finally: + self._reconnecting = False def _rd(self): """Read one QMP reply (JSON object ends with a newline). Poll with a @@ -91,8 +126,17 @@ class Qmp: def cmd(self, name, **args): try: self.s.sendall(json.dumps({"execute": name, "arguments": args}).encode() + b"\n") - return json.loads(self._rd() or "{}") + reply = self._rd() + if not reply: + raise OSError("empty QMP reply") + self._fails = 0 + return json.loads(reply) except Exception: + # Broken socket (QEMU restart, monitor hiccup): try exactly one + # reconnect, rate-limited inside reconnect(). + self._fails = getattr(self, "_fails", 0) + 1 + if not getattr(self, "_reconnecting", False): + self.reconnect() return {} def hmp(self, line): @@ -143,8 +187,14 @@ class Qmp: break m = re.search(r":(.*)$", out, re.S) if not m: + if not out.strip(): + # Empty monitor output: the socket is broken, not the address. + self.reconnect() return [] vals = [int(x, 16) for x in re.findall(r"0x([0-9a-fA-F]+)", m.group(1))][:n] + if not vals: + self.reconnect() + return [] if cache: self._cache[key] = vals return vals @@ -189,18 +239,46 @@ def guest_trace(): Read *before* clearing: the create-time probes fire once per client and would be wiped by a clear that races them. + + 2026-10-16 robustness fix (the cascade root): this used to run a bare + `tail` over the guest agent. A tracefs read that blocks under a busy + guest left *uninterruptible* tail processes behind (observed dozens in + D state), the guest-agent exec then blocked for its full 90 s timeout + every iteration, and the pool/tracker pins this tool supplies stopped + flowing - after ~5 clients every new client froze at DEVICE waiting on + its UVM tracker with `pending release starved ... pool/waiter page`. + Now the read is bounded remotely (`timeout -s KILL 12`) so no tail can + outlive its call, and SSH (which stays healthy when the guest agent + wedges) is preferred with the agent as fallback. """ - inner = ("tail -c 131072 /sys/kernel/tracing/trace " + inner = ("timeout -s KILL 12 tail -c 131072 /sys/kernel/tracing/trace " "| grep -E \"uvmt|uvch|uvbp|uvpbf\" " "| tail -400") if not os.environ.get("KEEP_TRACE"): inner += "; echo > /sys/kernel/tracing/trace" - cmd = "sudo sh -c '%s'" % inner + outer = "sudo sh -c '%s'" % inner + out = "" + ssh = os.environ.get("SEMFIX_SSH_PORT", "2222") + ssh_cmd = [ + "sshpass", "-p", os.environ.get("VGPU_GUEST_PASS", "ubuntu"), + "ssh", "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "PubkeyAuthentication=no", "-o", "ConnectTimeout=6", + "-p", ssh, "ubuntu@127.0.0.1", outer, + ] try: - out = subprocess.run([VMEXEC, "exec", cmd], capture_output=True, - text=True, timeout=90).stdout + r = subprocess.run(ssh_cmd, capture_output=True, text=True, + timeout=25) + out = r.stdout or "" except Exception: - return set(), set(), set() + out = "" + if not out: + try: + out = subprocess.run([VMEXEC, "exec", outer], + capture_output=True, text=True, + timeout=25).stdout or "" + except Exception: + return set(), set(), set() sems = {int(m, 16) for m in re.findall(r"sem=0x([0-9a-fA-F]+)", out)} pbs = {int(m, 16) for m in re.findall(r"pb=0x([0-9a-fA-F]+)", out)} chans = {int(m, 16) for m in re.findall(r"ch=0x([0-9a-fA-F]+)", out)} @@ -369,8 +447,11 @@ def resolve_pushbuffer_wide(q, sem, d): def pin(va, pa): cmd = "printf '0x%x,0x%x' > %s/sema_map_add" % (va, pa, PARAM) for _ in range(40): - r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], - capture_output=True, text=True) + try: + r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], + capture_output=True, text=True, timeout=5) + except subprocess.TimeoutExpired: + continue if r.returncode == 0 and "timed out" not in (r.stderr or ""): return True time.sleep(0.05) # async queue full: drain and retry @@ -380,8 +461,11 @@ def pin(va, pa): def pin_bulk(va, pa, nbytes): cmd = "printf '0x%x,0x%x,0x%x' > %s/sema_map_add" % (va, pa, nbytes, PARAM) for _ in range(40): - r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], - capture_output=True, text=True) + try: + r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], + capture_output=True, text=True, timeout=5) + except subprocess.TimeoutExpired: + continue if r.returncode == 0 and "timed out" not in (r.stderr or ""): return True time.sleep(0.05) diff --git a/tools/guest-diag/stuck_client_capture.sh b/tools/guest-diag/stuck_client_capture.sh new file mode 100755 index 0000000..6acd8b5 --- /dev/null +++ b/tools/guest-diag/stuck_client_capture.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# stuck_client_capture.sh - one-shot capture for a stuck CUDA client in the +# vGPU guest (run on the HOST; talks to the guest over ssh_guest.sh). +# +# The in-guest pin scanner consumes trace_pipe continuously, so kprobe reads +# return entries=0 while it runs. This tool: +# 1. pauses the scanner (kill -STOP), +# 2. reads the retained kprobe trace tail for the client's UVM waits +# (uvtw/uvsv/uvmt) - the tracker semaphore kptr and the value waited, +# 3. walks the tracker object (tracker_walk.py) for its expected payload, +# 4. dumps the semaphore-pool page (pool_page.py) for applied values, +# 5. resumes the scanner. +# Usage: stuck_client_capture.sh <client-comm> [pool-pa-hex] +# e.g. stuck_client_capture.sh victory_matmul 0x107fd8000 +set -u +REPO="$(cd "$(dirname "$0")/../.." && pwd)" +SSH="$REPO/scripts/ssh_guest.sh" +CLIENT="${1:?usage: stuck_client_capture.sh <client-comm> [pool-pa]}" +POOL="${2:-0x107fd8000}" + +echo "=== pushing walk helpers to the guest ===" +"$SSH" "cat > /tmp/tracker_walk.py" < "$REPO/tools/guest-diag/tracker_walk.py" +"$SSH" "cat > /tmp/pool_page.py" < "$REPO/tools/guest-diag/pool_page.py" + +echo "=== pausing scanner ===" +timeout 30 "$SSH" "echo ubuntu | sudo -S sh -c 'pkill -STOP -f guest_pin_stream.py; echo STOPPED'" 2>/dev/null | tail -1 + +echo "=== kprobe trace tail (client $CLIENT) ===" +timeout 60 "$SSH" "echo ubuntu | sudo -S sh -c 'tail -c 300000 /sys/kernel/tracing/trace | grep -a \"\$CLIENT\" | grep -aE \"uvtw|uvsv|uvmt\" | tail -8'" 2>/dev/null | tail -10 + +echo "=== tracker walk (kptr from the uvsv line above) ===" +echo " (run: sudo python3 /tmp/tracker_walk.py 0x<sem-kptr>)" + +echo "=== pool page $POOL ===" +timeout 90 "$SSH" "echo ubuntu | sudo -S python3 /tmp/poolpage.py $POOL" 2>/dev/null | tail -12 + +echo "=== resuming scanner ===" +timeout 30 "$SSH" "echo ubuntu | sudo -S sh -c 'pkill -CONT -f guest_pin_stream.py; echo RESUMED'" 2>/dev/null | tail -1 +echo "=== done ===" diff --git a/tools/guest-diag/tracker_walk.py b/tools/guest-diag/tracker_walk.py new file mode 100755 index 0000000..3846fa1 --- /dev/null +++ b/tools/guest-diag/tracker_walk.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# tracker_walk.py - read a UVM tracking semaphore's expected payload and walk +# its semaphore object via /proc/kcore (run in the guest as root). +# +# Use with a live stuck client: the udvt/uvtw kprobes or the scanner's #sem +# lines give the tracker kptr (e.g. 0xffff899a885b5480); this prints +# tracker+0x0 = semaphore kptr +# tracker+0x28 = expected payload (the value the client waits to observe) +# semaphore+0.. = its pointers (rm_mem/page) and candidate value words +# Pair it with the host's pool-wr/deferred-release logs and a pool page dump +# (pool_page.py) to compare what the driver expects vs what the host wrote. +import sys +import struct + +sys.argv = ["tracker_walk"] +sys.path.insert(0, "/tmp") +import guest_pin_stream as g # noqa: E402 + +kc = g.Kcore() +if not kc.load(): + print("KCORE LOAD FAILED") + sys.exit(1) + + +def r64(a): + d = kc.read(a, 8) + return struct.unpack("<Q", d)[0] if d else None + + +def r32(a): + d = kc.read(a, 4) + return struct.unpack("<I", d)[0] if d else None + + +for arg in sys.argv[1:] or ["0xffff899a885b5480"]: + SEM = int(arg, 16) + print("tracker 0x%x:" % SEM) + for off in range(0, 0x40, 8): + print(" +0x%x = 0x%x" % (off, r64(SEM + off) or 0)) + sp = r64(SEM) + print("semaphore=0x%x" % (sp or 0)) + if sp and g.is_kern_ptr(sp): + for off in range(0, 0x80, 8): + v = r64(sp + off) + print(" sem+0x%x = 0x%x" % (off, v or 0)) + for off in range(0, 0x80, 8): + v = r64(sp + off) + if v and g.is_kern_ptr(v): + w = r32(v) + print(" *sem+0x%x -> dword=0x%x (%u)" % (off, w or 0, w or 0)) diff --git a/tools/guest-diag/wall2/README-session74.md b/tools/guest-diag/wall2/README-session74.md new file mode 100644 index 0000000..58587eb --- /dev/null +++ b/tools/guest-diag/wall2/README-session74.md @@ -0,0 +1,66 @@ +# WALL #2 / device-init completion — session 74 findings (r675) + +## The client's "11 s stalls" fully root-caused (this is NOT the channel-init release) +strace (ioctl-only) shows THREE exact 11.00 s gaps with NO ioctl in flight; a +full strace shows the client doing a pure `sched_yield` spin (~57K/s) + +`clock_gettime` (perf) inside **libnvidia-eglcore.so.570.144** — i.e. a +userspace timeout loop. gdb backtraces place the spin inside the Vulkan ICD's +EGL/GL init (`vkCreateInstance` -> libGLX_nvidia vk_icdNegotiate... -> +libEGL -> eglcore wait loop). + +## The wait target: completion slots in /dev/nvidiactl mappings +- The wait loop (eglcore file offsets ~0xb284b0 delay, ~0xb33d00 wait loop, + ~0xb1c750 slot check) polls a slot table on the client's heap object: + table at object+0x1428, count at object+0x22efc. +- Slot check: `esi` is offset by 0x43; for esi-0x43 > 4 it reads a **u16 at + slot+0xe** and treats `0xffff` as "empty/not complete". +- gdb sampling during the stall: 13164 checks with `u16@slot+0xe = 0xffff`, + 1 with `0` — the client spins until the flag changes. +- The slot pointer = a **/dev/nvidiactl rw-s mapping** (page-sized; base+0x20 + = slot). These are the client's mappings of RM memory (the channel + notifier pages: service logs `notif=0x11627f060` per channel). + +## Filling the slots (guest-side) WORKS mechanically but is not the -4 cause +- `slotfill.sh` / `slotfill2.sh`: while vk_frame runs, write 0 to + `base+0x2e` of every small /dev/nvidiactl rw-s mapping when it reads 0xffff. +- Result: the 11 s gaps vanish (5 fills within ~2 s; the client proceeds + immediately) — **but vkCreateDevice still fails -4**. So the 11 s waits are + not the failure; they are an init-side slowdown the client tolerates. + +## The next wall: the polled RM control 0x2080a097 +- With the slots filled, the client settles into **NVOS54 (ioctl nr=0x2a) + polling every ~1.25 s** (strace), and the host dmesg shows + `GSP_RM_CONTROL cmd=0x2080a097 hClient=0xc1d000bb hObject=0xbeef0004 + paramsSize=1168 flags=0x0` -> `unknown cmd=0x2080a097 -> generic SUCCESS` + (all-zero reply), 36 polls in 45 s. +- The NVOS54 struct (gdb dump): {hClient=0xc1d000c0, hObject=0xbeef0004, + cmd=0x2080a097, flags=0, params=ptr, paramsSize=0x490, status=ptr, ...}. +- The params buffer = 1168 bytes of 32-byte records with a `0x25` field at + +0x10 (dump in /tmp/opencode evidence; gdbparam.sh captures it live). +- The client polls until the control's status/reply signals completion; our + generic all-zero reply never does. **This is the wall that must be solved to + get vkCreateDevice past -4** (either a real reply for 0x2080a097 or making + the polled status field change). + +## Other session-74 results +- The GR ring-entry writer was NOT found by disassembly-guessing: nvidia.ko + `channelFillGpFifo` and nvidia-modeset `nvWriteGpEntry` never fire for this + channel's kick (gdb hbreaks); nvidia/modeset kprobes are `notrace`; QEMU gdb + hbreak works. +- `sema_push_raw` dumps (module param) captured the service's reads of the + push: entry0 decodes only 2 methods then zeros (`consumed=0x288/1432`), + entry2 starts `SET_OBJECT 0xc597` then zeros; **no release in the push** + (`have_rel=0` for all three entries) — the channel-init release theory for + these waits is dead. +- `find_ring_va.py` / `vmap_watch2.py`: the ring entries are NOT in any + vmallocinfo area during the run; the nv_vm_map_pages areas hold the CE + pushes; the GR push content is not found by content searches. + +## Files +- gdbpoll6.gdb, gdbu16.gdb, gdbfill.gdb, gdbwatch.gdb, gdbparam.sh, + gdbgap3.sh, gdbspin.sh — the gdb probes (slot check, u16 flag, fill, + NVOS54 param dump, backtraces). +- slotfill.sh / slotfill2.sh — the guest-side slot-fill watcher. +- stracevk.sh, fastrun.sh, fastrun2.sh — the strace/perf recipes. +- perfstall5.sh — perf sampling with map-relative offsets. +- find_ring_va.py, vmap_watch2.py — the vmap/kcore watchers. diff --git a/tools/guest-diag/wall2/README-session76.md b/tools/guest-diag/wall2/README-session76.md new file mode 100644 index 0000000..828b3b9 --- /dev/null +++ b/tools/guest-diag/wall2/README-session76.md @@ -0,0 +1,134 @@ +# session 76 notes (r680) — the 0x2080a097 engine-monitor re-examined + +## What the firmware says (ground truth for the a097 family) + +The shipped GSP-RM 570.144 image embeds the host-side control +descriptor table (file offset 0x41df8..0x42450, entry = 0x20 bytes +{u32 cmd, u32 params_size, u32 0x4049580, u32 0, u32 handler_ptr, +u32 0, u32 out_size, u32 0}). Entries seen: + + 0x2080a07f size 0x408 handler vaddr 0x551b2c0 + 0x2080a079 size 0x14804 + 0x2080a097 size 0x490 handler vaddr 0x55166f4 + 0x2080a0d1 size 0x7e8 + ... + +The real 0x2080a097 handler (0x55166f4) writes up to FOUR {u32,u32} +mode pairs into params+4..+0x23 (params[0] low byte = the request's +count flag): for i=1..4 it calls an engine-status helper +(0x5516334 / 0x55d7f04 / 0x55d7c1c / 0x55d7d60) with mode=i and emits +{pair} = {engine value, 1000 * u16(engine+0x1bc0)}; the loop stops at +the first helper error. So the real reply overwrites the client's +request record {1,0,0,0}. + +## What the client does with the reply + +libnvidia-eglcore's local handler (file offset 0xecd2e0, the userspace +implementation of the internal cmd 0x2080a040) copies back: + + result+0 = reply+0 (count) + result+0x10 = reply+0x204 (flag) + result+0x20 = reply+0x488 + result+0x24 = reply+0x48c + count*16 bytes from reply+4 -> *(result+8) + +For the engine tick (file offset 0xd5e3e0, called from the wait loop +0xd6ef40 per iteration) the result struct IS the caller's request at +rsp+0x10, and *(result+8) points at a 16-byte "inner" buffer +(rsp+0). The tick then reads inner+4 and stores it to session+0xe104 +(the per-engine "progress" cache; poll interval ~1.25 s from the same +function). A second consumer of the tick's return value compares it +against per-boot config at lib+0x2019320: + + 0xfc8 = 1000, 0xffc = 95 (progress threshold), + 0xfec = 10, 0xfed = 20 (bit-history cutoffs), + 0x1000 = 500 + +and ORs the comparison bit into a round-robin bitfield (+0xb4) of the +state machine object. + +## Live evidence on r680 (fresh boot, full-param recipe) + +* The 0x2080a097 polls (NVOS54 0x2a, 1.25 s cadence) come from a + WORKER thread (gdb bt: start_thread -> eglcore a11b2a -> wait loop + 0xd6ef40 -> tick 0xd5e3e0). The engine object E (0x604101365e90) + sits at E+0x150 = 0x00200000 (bit 21 only), E+0x1b038 = 1, + E+0x1b039 = 1, *(E+0x1b030)+8 != 0 — all static for 5+ minutes. +* MAIN thread (vkCreateDevice) is in a poll(fd=/dev/nvidia0, events + POLLIN|POLLPRI, timeout=10s) loop inside eglcore (bt: poll <- + eglcore+0xa12de4 <- 0xd69159 <- 0xd6a02b <- 0xdcebc6 <- ... <- + libvulkan vkCreateDevice). No -4 is ever printed in this state: the + client waits indefinitely (killed by its own timeout in runs). +* GR channel kicks continue concurrently (sema-entry lines for + h=0xbeec0100 va=0x4ca0000 len=1432 etc.) and end as + "skipping stale entry ... after N failed rescues" (gr_wedge>0). +* Reply sweep on r680 (module params) — flag=0/2/1/1..0xffffffff, + v488, v48c AND the new record matrix count/rec0..rec7 = + 0/{1,0,0,0}/100/1000/100000/1000000/0xdeadbeef — produced ZERO + observable change: same a097 cadence, no writers to + E+0x150/E+0x1b038/E+0x1b039 ever caught, no client progress. The + hw watchpoints on those words fired zero times over ~5 min. + +## Tools + +* t75g.py — in-guest gdb python: dispatcher-type log, a097 issuer + dump, tick (d5e449) state + cfg dump, E/S capture, hw watchpoints on + E+0x150 / E+0x1b038 / E+0x1b039 / S+0x120, POST a040 consumption + dump. +* diag76.py — in-guest gdb python: thread-apply bt + map-relative + address resolution (used to find the main-thread poll wait). +* slots_probe.sh — read the u16 completion-slot words of every small + /dev/nvidiactl rw-s mapping (SESSION 74's slotfill ABI). +* run75g.sh — trace + record-word sweep driver. + +## Verdict + +The a097 monitor is a background worker, not an obviously blocking +gate: every knob the client reads from it can be varied with no +observable state change, while the main thread blocks elsewhere +(/dev/nvidia0 poll loop) and the GR channel-init pushes still cannot +be read by the sema service. Next lever is the main thread's wait +(what would make the /dev/nvidia0 poll return readable) or the GR +channel-init completion — not the a097 reply words. + +## ADDENDUM (r681) — THE MAIN-THREAD vkCreateDevice WAIT DECODED + +Live gdb on the main thread (r681, fresh boot) resolved the loop at +eglcore 0xdceb80 (frame chain: poll <- a12de4 <- d69159 <- d6a02b <- +dcebc6 <- d3838a <- ... <- libvulkan vkCreateDevice): + +* rbx (waitable, vt 0x...3f58f8): vt+0x18 = d69ee0 = "should wait" + (returns `(*0x48(*(rbx+0x70)) != 1)`); vt+0x28 = d69f50 = "wait" + (builds a pollfd array from the object's fd holder and poll()s up to + 16 fds with the 10 s timeout). +* rbp (loop object, vt 0x...3f71f8): vt+0x38 = dcee20 = the give-up + predicate. dcee20 returns nonzero (-> return -4 = the client's + VKFAIL -4 origin) when either object at rbp+0x110 / rbp+0x120 shows + an error (b29d60(...) != 0 or *(obj+0x70cc) != 0); otherwise 0 and + the loop repeats. It also throttles itself at 1e6 (units) via + rbp+0x1b8. +* The readiness value: sub = *(rbx+0x70) (0x...f7f0c90); sub+0x58 -> + obj2; obj2's vt+0x10 = 0xce71f0 = `read u32 at (*(obj2+0x58)) + + *(u32*)(obj2+0x60)` — i.e. a u32 word at holder_base + 0x80 (the + live object had obj2+0x60 = 0x80, obj2+0x64 = 0x1e). Observed: 0x0 + every poll (39/39) -> should-wait = 1 forever. +* The word lives in a **/dev/nvidiactl rw-s mapping** + (0x7623d0480000-0x7623d0484000) — a VM_IO driver page, not plain + guest RAM: /proc/PID/mem reads of it return zeros, and writing the + guest-RAM file at the pagemap-derived PAs (0x1036dd080 and both + hole-shifted candidates) did NOT change the client's behaviour. + The writer is therefore the guest kernel driver/RM when the awaited + operation completes (the same class as the SESSION-74 completion + slots) — not a host RAM poke. +* All 20 sampled polls of the wait helper returned 0 (timeout): the + fd (/dev/nvidia0) never becomes readable (nv_poll reports + POLLPRI|POLLIN only when an nv_event is pending on the file). + +So vkCreateDevice waits for (a) the readiness word == 1, and the wait +ends only when the driver/RM signals that word (or an nv_event arrives +on /dev/nvidia0). The give-up path (-4) fires only if the RM error +flags appear. Next: identify the operation whose completion writes the +word (the object chain sub -> obj2 -> holder is a "signal/semaphore" +object; dump the holder object and its creating control), then make +the emulation drive the guest kernel to complete it (the engine/ +channel-init completion the sema service already tries to deliver). diff --git a/tools/guest-diag/wall2/README-session77.md b/tools/guest-diag/wall2/README-session77.md new file mode 100644 index 0000000..d830bef --- /dev/null +++ b/tools/guest-diag/wall2/README-session77.md @@ -0,0 +1,1195 @@ +# session 77 (r683/r684) — vkCreateDevice PASSES: the 0x7E event validation and the readiness-word completion + +## The two-part unblock + +1. **r682 fix validated** (`e3c1cdc5`, loaded on r683/r684): the + nvkms/Vulkan path allocates its completion event as + NV01_EVENT_KERNEL_CALLBACK_EX (class 0x7E); the emulation now + registers it alongside 0x79. Live proof (r683, sema_dbg=1): + + GSP_RM_ALLOC class=0x7e paramsSize=24 hClient=0xc1e00011 + hParent=0xcaf00001 hObject=0xcaf00003 + event registered hEvent=0xcaf00003 hSrc=0x0 + notifyIndex=1006632972 (off 8, slot 13) + params=1100e0c1000000007e0000000c00003c0000000000000000 + EVPOST all ev[13] cl=0xc1e00011 ev=0xcaf00003 nidx=0x3c00000c rc=0 + + (notifyIndex 0x3c00000c = subdevice 0x3c, index 0xc = + NV2080_NOTIFIERS_GRAPHICS). The guest consumes the records + (`guest_read == our_write` in the rmstate dump), but the guest's + event dispatch does NOT write the event's notification word — the + residual delivery gap. + +2. **The completion word found and driven (the actual unblock)**. The + main-thread vkCreateDevice wait (eglcore 0xdceb80, session 76) reads + a u32 readiness word: `should-wait = (word != 1)`; the word lives at + `+0x80` of a **16 KB rw-s /dev/nvidiactl mapping** (the event + notification buffer; holder chain sub -> obj2 -> holder, resolved + live by t79.py). The host can write it directly in the guest RAM + file: + + PA = pagemap PFN << 12 (+0x80); file offset = PA - 0x80000000 + (above-4G RAM; below-4G would be PA itself per the cycle's + guest-RAM registration: gpa 0x0/0x100000000 -> off 0x0/0x80000000) + + Writing 1 flipped the wait and **vkCreateDevice returned success** + (`VKDEVICE ok family=0`) — reproduced three times: + * r683: manual write after a t79 probe; + * r684: nudger (all 16 KB mapping words) + slotfill restart; + * r684d: probe -> single-word write (the reliable recipe). + +## The reliable recipe (per run) + +1. r682+ module; `sema_dbg=1 gsp_rm_ev_log=300` optional. +2. Launch `vk_frame` + `slotfill2.sh` (EGL completion-slot fills; the + slots re-arm, so re-run slotfill if the client is stuck in the + sched_yield spin before the word-wait). +3. Wait for the main thread in `do_poll` (the word-wait). +4. `t79.py` (gdb) -> `WAIT word=0x...`; guest pagemap -> PFN -> PA; + host write `01 00 00 00` at `PA-0x80000000`. +5. `VKDEVICE ok` follows within ~10 s. The client then proceeds to + image/memory/pipeline setup; its next wait is NOT the same word + (the word stays 1) — next session: probe the follow-on state + (attach fails while the client spins in the post-device phase; + use the /proc-only checks + a longer observation window). + +## Cautions learned + +* Writing ALL 16 KB mapping words (the multi-page nudger) can trip the + -4 give-up path (an RM error flag word gets written): restrict nudges + to the single probed word. +* Attaching gdb to the client while it is in the tight post-device spin + can hang the attach (leave a `timeout`-wrapped gdb, then kill that + gdb by PID and `kill -CONT` the client — the pending SIGSTOP from a + dead tracer otherwise wedges it). +* zsh truncates 64-bit hex literals in `$(( ))`: compute PFN/PA in + python, not zsh. + +## Tools + +* `t79.py` — resolves the wait chain (sub -> obj2 -> holder -> word VA) + and prints the client's nvidiactl rw-s mapping sizes. +* `guest_words.sh` — prints VA/PA of the +0x80 word for every 16 KB + nvidiactl mapping (pagemap; run in the guest as root). +* `nudge.sh` — host-side loop: poll guest_words.sh and write 1 to any + zero word (multi-page variant; prefer the single-word recipe). + +## ADDENDUM (r685, session 78) — POST-DEVICE WAIT IDENTIFIED: the NvOgl (GL ucode) engine init; push windows now READABLE (resolver6) + +* The post-device spin is the **GL module bring-up**: the main thread's + A8-layer retry loop (eglcore 0xa85d24) works on a work item whose + embedded name is ASCII **"NvOgl_m0"** (+0xf04: "NvOgl_m40"); the D6 + submit layer (0xd624e0) ORs **0x200000** into its pending word (the + same engine-pending bit SESSION 75/76 watched on E+0x150), then the + A8 loop yields via the B284B0 spin helper (sched_yield) waiting for + the GL ucode init to complete on the GR engine. +* **The push windows ARE now readable** (first time): the resolver6 + recipe (window-record pair {push_va,0x20000}+{ring_va,0x40000} -> + same-page 0x20000 memdesc PA -> `sema_map_add`) injected + `0x4480000 -> 0x120cfb000` (content `2001c00d 70000000 2001c00d + 68000000` = the 2-method semaphore-like push) and `0x4ca0000 -> + 0x12e51e000` (the 1432-byte GR init stream: SET_OBJECT 0xc5c0/0xc5b5, + `0x200121ed`, `0x200120a9`, bulk methods, tail `2001c00d + 70/68`). The service now decodes the streams (`sema-push ... tail=0dc0 + 012000000070...`) where SESSION 73-74 saw "unread". + CAUTION: the resolver injects every candidate; a zero-page candidate + injected last replaces the good mapping for that VA (re-inject the + good page after the scan). +* **Remaining gate**: the service still reports `have_rel=0` for every + GR push (the CE-shaped release matcher) and — decisively — the + GR-forward machinery is NOT engaging: `drain held=4 starved=149 + forwarded=0 ... gr_nocap=4`. The GL ucode init work is never + forwarded/executed, so the completion (and the 0x200000 bit clear) + never arrives. Next: the GR forward/capture path for these channels + (operand capture + release decode for the GR shape), then the client + proceeds to VKFRAME. + +## ADDENDUM (r686, session 79) — the low-VA release fix and the corrected injection recipe + +* The GR/GL init releases decode only with the low-aperture acceptance + (`SEMA_VA_LO` bound relaxed to >= 0x4000000 in both release forms, + commit cb8b75c5): live `rel=0x4b19b10 pay=1` (REPORT_SEM group + `200426c0 00000000 04b19b10 00000001 10000000`), `rel=0x4b17bb0 + pay=3` and `rel=0x4b17e30 pay=1` (0x5c-form `2005c017 04923060 ... + 02000001`); `have_rel=1`, `applied` rising, un-translated releases + queue in svc_pending (417 observed). +* Corrected pair-specific push injections for r686: + `0x4480000 -> 0x125692000` (record page 0x1163c9000, live desc), + `0x4ca0000 -> 0x131649000` (page 0x122f20000), plus + `0x4b16000 -> 0x12a76a000` (the {0x4b16000,0x1000} window). + CAUTION: resolver7 injects each page's descs for BOTH pairs (union + loop) - fix it to keep per-pair pages or inject manually from the + per-pair scan. The window-record layout is {tag, VA, SIZE, 0, + "tail", ...} stride 0x40 (search the VA u64 with size at +8 and + "tail" at +0x10). +* Remaining: the semaphore VAs (0x4b17bb0/0x4b17e30/0x4b19b10) have no + window records in the known record pages; the service cannot resolve + them (targeted rm_mem miss) so the releases stay pending and the + NvOgl wait spins. Find their PAs (client mapping pages / RM + VA-tracker / live poll-watch), inject, and the pending releases + should apply. + +## ADDENDUM (r686, session 80) — ALL low-aperture window records found (the "liat" tag fix); the semaphore windows identified and their releases applying + +* **The record scan fix**: the RM window record is 0x40 bytes + {u64 tag; u64 VA; u64 SIZE; u64 0; u32 0x7461696c; u32 0}. The tag is + stored as the u32 0x7461696c - on disk the BYTES are "liat", NOT + "tail" (the hex display `7461696c` reads as "tail" only as a number). + Searching for b'tail' finds nothing; `wall2_recscan.py` searches for + the byte pattern 6c 69 61 74 and decodes {tag,VA,SIZE} at + -0x20/-0x18/-0x10. It found **all 75 records** in one pass. +* **The GR init semaphore windows** (r686): + `va=0x4b17000 size=0x2000 -> pa=0x12a76f000` (verified by content: + +0xbb0=3, +0xe30=1 - exactly the rel=0x4b17bb0 pay=3 / + rel=0x4b17e30 pay=1 payloads) and + `va=0x4b19000 size=0x1000 -> pa=0x122f9b000` (after injection the + release applied there: +0xb10 became 2). Injections: + `echo 0x4b17000,0x12a76f000,0x2000,1 > sema_map_add` and + `echo 0x4b19000,0x122f9b000,0x1000,1 > sema_map_add`. +* **State after the injections**: the low-VA releases decode + (`have_rel=1`) and apply to the semaphore pages; `pending` hovers at + ~418 (the client re-kicks continuously); the NvOgl wait still spins + (the A8 loop compares per-entry counter tables r14+0x5e8 vs the + wanted set - the completion is not just the semaphore value). Next: + the A8 counter tables' source (what fills r14+0x5e8 / the "wanted" + table - likely the GR channel's own progress counters) or the GR + forward/ucode execution (`forwarded=0 gr_nocap=7` still). + +## ADDENDUM (r686, session 81) — the counter-autoreg gap analysed: the counter page 0x20400f000 is unresolvable; the manual `sema_counter` registration is the bypass + +* `svc_counter_autoreg` (per-pass for eng==0 channels, n_gr=20) needs a + translation for the rank slot VA `0x20400fec0 + (19-r)*0x10` + (r = gr_index = rank by ascending handle among the guest's GR + channels): either an ipin pin (`svc_ipin_lookup`, 9275 entries live + from the guest pin stream) or `svc_counter_derive` (the + vaspaces/discovered/targeted walk chain, `sema_counter_derive=1` + default). Both miss for the counter page -> no "counter auto" log, + `pub_calls=0` -> the A8 loop's per-entry counter tables never fill and + the NvOgl wait spins. +* The client does NOT map the counter pool: `/proc/<pid>/pagemap` for + 0x20400f000/0x20400ffd0/0x20400ffc0/0x20400ffb0 is all-zero (not + present) and no VMA covers it - it is a UVM kernel-side object, so the + pin stream cannot resolve it through the client's page tables. The + stream's log shows `sem=0` (zero semaphore resolutions) even though + the want-list (`/tmp/guest_want_vas.txt`, host-written) contains 44 + `20400f` entries; the stream's "channel-anchored expansion" for the + UVM tracking sems (guest_pin_stream.py ~line 950-1010) is the + machinery that must find them (it is written for the CUDA second-client + stall - "the only live uvmt event is the spinning sem"). +* Bypass available now: the `sema_counter` parameter registers a channel + slot manually - `echo <h_object>,<pa> > .../parameters/sema_counter`. + Once the counter page's PA is known (via the guest kcore/UVM walk), + this wires the publish path without a kernel rebuild. +* The rank slots for the GL client's channels (handles 0xbeec0100, + 0xbeec0104, 0xbeee0100 -> ranks 0,1,2): 0x20400ffd0 / 0x20400ffc0 / + 0x20400ffb0. + +## ADDENDUM (r686, session 82) — the counter-page PA hunt: pagemap route confirmed dead, vidmem-window hypothesis tested and not supported + +* The client's own pagemap does NOT contain the counter pool: the + semfix_uvmt tool's resolver (`guest_pmap_win.py <pid> 0x20400f000 + 0x204010000 64`) returns NOTHING, as does the UVM pool range + (`0x121000000 0x121100000`). The tool's `pin_counter_page` has + therefore never succeeded for these clients (the STALL WATCHDOG's + "re-pin counter" attempts across sessions produced no "counter: + pinned" line). +* VIDMEM-window hypothesis tested: the counter VA 0x20400f000 is in the + emulated vidmem window (>= SVC_VIDMEM_VA_BASE 0x200080000), so its + window offset is 0x3F87000 (0x20400f000 - 0x200080000). The shadow + there is reachable and all-zero (`sema_vidmem_dump 0x3f87000,64`); + poking the rank slots (+0xfd0/+0xfc0/+0xfb0 = ranks 0/1/2) with 6 and + then 0x40 via `poke_vidmem` changed the shadow (verified by dump) but + did NOT move the client's wait. The vidmem route is not (or not only) + the completion path; the A8 loop's tables have another source. +* gdb-attach hazard (again): attaches to the client in its post-device + spin hang with no output; two gdb chains had to be killed by PID + (never a pattern) and the client SIGCONT'd. Probe only while the + client is in do_poll. + +## ADDENDUM (r687, session 83) — THE A8 COUNTER SOURCE DECODED: a client readback window the host can write (first live read-back confirmed) + +* The A8 loop calls `a82f30(obj)` per entry; a82f30 calls the object's + method pointer at ***(obj+0x19b8)** = eglcore+0xb250f0, which returns + `(hi<<32)|count`. The count comes from **b1bf00**: + - fast path (obj+0x224 != 0): `*(u32*)(obj+0x1078) - 1`; + - slow path (observed: obj+0x224 == 0): **b27c90(obj, 0x800000bb/0xbc)** + which reads `*(u32*)(*(*(obj+0x1480)+0x58) + idx*16)` - a **client + READBACK WINDOW** (idx = 0xbb..0xbb+n, n = *(u32*)(obj+0x22efc)); + the loop takes the MIN of those reads. +* Live (r687, object handle 0xbeec0100): the window = VA + 0x7827e6c12000 (a /dev/nvidiactl mapping), all zeros; obj+0x224=0 + (slow path active), obj+0x1078=7 (fast-path field unused). +* **The host CAN write it**: resolving the window's PA via the client's + pagemap and writing 0x100 into the window (offsets 0xbb0..0x1ff0, via + the RAM file) made the client read it - **obj+0x1070 = 0x100** (the + method stores the returned count there) and the A8 "actual" table + entry became `0xffffffff00000100`. FIRST LIVE PROOF that the client's + completion counter can be driven from the host. +* The wait still spins: the A8 loop's first scan (compare + a85e1d: actual vs want, want observed 0) passes with our value, but + the loop continues into a second scan (compare a85ec1, driven by + b29dc0) and the `*(rsp+0x38)` branch (a85e60). NEXT: decode those + (the second scan's actual source and the rsp+0x38 flag), and whether + the other objects (entries 0-2 of the A8 table) need the same window + writes. +* The other objects' windows were not observed (only 0xbeec0100 called + a82f30 in the sampled window); the nvidia0-side window (obj+0x1478) + is NOT present in the pagemap (VM_IO; host cannot write it via the + RAM file). + +## ADDENDUM (r687, session 84) — the a86038 want-scan mapped: ALL WANTS PASS with the driven window value; the residual gate is the A8 loop's internal callback/retry machinery + +* Live dump of the a86038 scan (BP at 0xa8608e, per-index): + i=0 want=0x0 actual=0x0 pend=no + i=1 want=0x0 actual=0x0 pend=no + i=2 want=0x0 actual=0x0 pend=no + i=3 want=0x6 actual=0xffffffff00000100 pend=no + (want from a82fd0(job_obj, i) = the job object's +0x98 array maxed + against global+0x600; actual from r14+0x5e8 = our driven counter). + NO pending entries -> the scan completes. +* Writing 0x100 to the ENTIRE readback window (offsets 0x0..0x1ff0 of VA + 0x7827e6c12000) did not clear the wait either. +* The loop still cycles: the bt stays at the a85f08-a85f1a yield call + (b284b0(*(rsp+0x10), 0xb) - a bounded pause/yield spin of ~0x14 + units, hence the slow cadence), and the exit-attempt path a85f30 + decrements *(rbp) (w0=1) while firing the global callbacks + (0x206f790/0x206f7a0 vtables: *(%rax), *0x10, *0x18) with args from + rbp+0x30/0x38/0x40. The remaining gate is this callback/retry + machinery (or another wait object), not the readback values. +* NEXT: resolve the global callback targets (0x206f790/0x206f7a0 + vtables at runtime) and trace the a85f30 exit path (what the + callbacks do and what makes *(rbp) reach 0); or find how the real GPU + writes the readback window (the op-4 methods 0x800000bb/0xbc the + service skips) and emulate that write so the client's own machinery + completes. + +## ADDENDUM (r687, session 85) — **THE NvOgl WAIT CLEARED** (the session-count gate) and the client advanced to vkAllocateMemory (new wall: -2 OUT_OF_DEVICE_MEMORY) + +* The A8 loop's exit requires the session count `*(u32*)(r14+0x5bc)` + (= lib_base + 0x2070b7c, the global at +0x20705c0+0x5bc) to reach 0; + the loop head increments `*(rbp)` and the entry scans pass with the + driven window value, but with count=4 the exit checks + (a85b48/a85e78: `if (r12d == 0)`) are never taken - the loop waits for + the engine sessions to complete (count -> 0). +* **DIRECT TEST (succeeded)**: `set {unsigned int}0x<base+0x2070b7c> = 0` + via gdb -> the loop exited, the NvOgl init completed, and the client + ADVANCED past the post-device wait for the first time: + + VKDEVICE ok family=0 + VKFAIL vkAllocateMemory(dev, &ma, NULL, &img_mem) -> -2 (line 204) + +* **NEW WALL**: `vkAllocateMemory` for the image (DEVICE_LOCAL) returns + -2 (VK_ERROR_OUT_OF_DEVICE_MEMORY) at vk_frame.c:204 - the guest + driver's heap/type selection fails before any alloc RPC (no + GSP_RM_ALLOC in dmesg). Next: the memory-heap reporting + (FB_GET_INFO_V2 0x20801303 is handled; check the heap-size/type + queries the driver uses for the DEVICE_LOCAL type, and the emulated + VRAM size the driver sees). +* Recipe for the next run: slotfill + word nudge -> VKDEVICE ok; then + write 0 to lib_base+0x2070b7c (gdb; the harness) once the NvOgl wait + starts (the A8 loop's count gate); then the memory-alloc wall. + +## ADDENDUM (r689, session 86) — **THE DEVICE_LOCAL -2 WALL CRACKED**: the 512 MiB emulated VRAM was the gate; the "nvidia-nvidia-vgpu-turing-2g" mdev type (2 GiB) lets the allocation proceed + +* Evidence chain: the ICD hooks (t91.py) captured the client's allocation + `ALLOC sType=5 size=0x4000 typeIndex=1` (a valid 16 KB DEVICE_LOCAL + request); the standalone memory-properties dump (vk_memprops.py) + showed device-local heap[0] = 512 MiB with a separate 2936 MiB + non-local heap. With 512 MiB the driver failed the allocation + (VK_ERROR_OUT_OF_DEVICE_MEMORY, no RM RPC). +* **FIX TESTED**: the cycle with `VGPU_MDEV_TYPE=nvidia-nvidia-vgpu-turing-2g` + (start_vm_keep.sh honours the env var) -> `heap[0] size=0x80000000 + (2048 MiB)` (vk_memprops.py) -> the SAME client with the SAME recipe + (slotfill + word nudge + count auto-clear) reaches `VKDEVICE ok` and + the allocation NO LONGER fails -2: the client proceeds into + `os_schedule` (a deeper wait). +* REMAINING in the new state: the client's later pushes (va=0x44852d4 / + 0x4ca52d4, then +0x586c) extend beyond the first window page and read + `pa=0x0`/zeros (the sema injections covered page 1 only; the window's + page list is scattered - the memdesc carries only the FIRST page). + NEXT: extend the window injections to the full scattered page list + (from the memdesc's page array) or accept 1-page pushes; then the + allocation completes and the client proceeds to the buffer/pipeline. +* Tools: vk_memprops.py (standalone memory-properties dump; safe + alongside other clients), vk_alloc_test.py (standalone allocation + reproducer - NOTE: it hits vkCreateDevice -4 without the harness + (word nudge + count clear), so it cannot replace the vk_frame run), + t91.py (auto-clear + ICD memprops/alloc hooks). + +## ADDENDUM (r689, session 87) — the window page-list hunt: the pages are scattered; content matches found but the VA<->page relation is unresolved + +* The 2 GiB cycle's window records (recscan89): `0x4480000` descs = + {0x107c47000 (live init stream), 0x103c06000, 0x12ecc2000}; + `0x4ca0000` descs = {0x11dbfa000, 0x12f398000 (live init stream)}. + The window's later offsets (va+0x52d4, +0x586c, +0x9000) at + first_PA+offset are ALL ZEROS -> the windows are NOT contiguous + (scattered pages). +* The memdesc record (page 0x106128000) is {size=0x20000, flags + 0x03008000, PA=0x107c47000, kernel pointers 0xffff8d2e5e21ec50 / + 0x4fe7ec50 / 0x461287e8 ... , "head"/"tail"} — the page list hangs off + the kernel pointers. A guest kcore walk of those pointers returned + "unreadable" (stale addresses by the time of the walk; the walk must + happen on FRESH pointers in the same pass as the dump). +* Content search for the distinctive init-stream prefix (`00 80 01 20 + b5 c5 00 00 04 00 04 20`) found the window pages: + 0x107c47000+0x0 (the injected init page), 0x103c06000+0x858, + 0x11dbfa000+0x858, 0x126d69000+0x59c — the later pages hold the same + engine-init streams at arbitrary offsets; the VA<->page relation is + NOT linear (the +0x59c push appears in pages whose +0x0 is zero), so + the mapping cannot be reconstructed by offset alone. +* NEXT: (a) dump the memdesc + kcore-walk its page-list pointers in ONE + guest pass (fresh addresses); or (b) inject the content-matched pages + at trial VAs and watch the service's reads; or (c) the service-side + decode of the window page lists. The client (2632) blocks in + os_schedule at the allocation — the unread later pushes are the + suspected gate. + +## Session 88 addendum - the push-page fix, the gdb-kill wedge, the ring-mapping gap + +**1. PATH-5 RULE (the push pages).** A GR window's pushes do NOT live at +the window base page: the r689-family windows carry their method streams in +WINDOW PAGE 5 (va offsets 0x52d4/0x559c/0x586c/0x596c all live in +0x5000-0x5fff). Standing injections at the base page resolve pa=0x0 for the +pushes (the service's `sema-pushall ... pa=0x0` lines) and the channel +holds/skips entries as "GR no-capture". The fix: page PA = +match_filoff - (va & 0xfff) + 0x80000000 for a content match of the push's +first8 bytes constrained to the same in-page offset; then +`echo 0x<va&~0xfff>,0x<pa>,0x1000,1 > .../sema_map_add`. Proven on r689: +channel beec0104's pushes at va=0x4d00000 (page -> PA 0x12f398000) went +from first8=00000000 to first8=20012000 0000c5c0, and `fast sweep` +applied 3281 -> 3809 (+30/sweep sustained). + +**2. host_resolve.py** automates 1: it polls dmesg for sema-pushall lines +with pa=0x0 and non-zero first8, searches /dev/shm/vgpu-guest-ram +(offset-constrained), and injects the page. The service's 6 s no-capture +hold is the race window - run it BEFORE the client and keep it polling. + +**3. NEVER SIGKILL AN ATTACHED GDB.** On r689, killing the stale t91.py +gdb (kill -9 on PID 5334) killed its tracees (client threads +2883/2884/2885/3048); one held the guest RM API rwsem write lock; the lock +leaked and every later RM call blocked (`[vkps] Update blocked for more +than 1228 seconds` in rmapiLockAcquire -> RmGssLegacyRpcCmd -> +_nv04ControlWithSecInfo). The t91 harness design is correct: `timeout 500` +SIGTERMs the gdb and it detaches cleanly. If a gdb must go, use SIGTERM +and wait; SIGKILL only when the guest is about to be cycled anyway. A +leaked kernel rwsem cannot be repaired live. + +**4. The r690 gap: the GR channel ring mapping.** Fresh cycle r690 (mdev +f5a796b0): every RM RPC of the client init is answered (GR ctx commit, +channel allocs class 0x9067 hObject 0xcef00000/0xcef00080, EVPOSTs), but +the service reports `gpfifo done=0 nomap=938 pushes=0` - it cannot read +any GPFIFO ring, so no entry decodes, no pushall line appears, and the +guest's completion poll loops on RM control 0x20800a70 until the ~10 s RM +timeout -> vkCreateDevice returns -4 (VK_ERROR_DEVICE_LOST). r689's +service had done=19108 (rings mapped) - so the ring mapping is the +missing ingredient for a fresh client in r690; candidate paths: the FIFO +alloc params (12 bytes, GPFIFO VA inside) are not logged; the scanner saw +the CE window (ipins 4592) but no GR ring this boot. Next: log the 0x9067 +alloc params (or scan the guest for the ring VA) and inject the ring page. + +**5. gr_walk.py / gr_walk2.py** are the guest-side kcore walkers (fresh-pass +memdesc discovery; 12 records found, kptrs readable in-pass) kept as the +alternate grounding path. + +## Session 89 addendum - THE PT WALK WORKS (live); the completion-word gap + +**1. GMMU PT walk resolved LIVE (the mission's PT-binding step: DONE).** +The service's own walker math (svc_gmmu_walk: root_bits=ilog2(entries), +shift=49-root_bits; L1 shift38, L2 shift29, PD0 idx=(va>>21)&0xff with 16-byte +dual big/small entries, PT idx=(va>>12)&0x1ff; PDE bit0=0, PTE bit0=1; +pte_addr=(e>>8)<<12) was replicated host-side over /dev/shm/vgpu-guest-ram +(walk.py / live_walk.py). The LIVE published pdb (dmesg: "page directory +... pdb=0x... entries=4" - a NEW pdb per client, e.g. 0x105f0a000 / +0x12a359000 / 0x107012000) resolves every client VA while the client runs: +ring 0x44a0000 -> PA, windows 0x4480000/0x4485000/0x4ca0000..., releases +0x4213000/0x45a0bb0, and the injection via sema_map_add (ap=1 sysmem) makes +the service read them. VALIDATED BY CONTENT: the ring page held exactly the +GPFIFO entries the service logged (entry[1] raw=8000040004480598 va=0x4480598 +len=4), and the push page at va=0x4480598 held the method stream +(...0x20020047...). Post-mortem reads fail: the guest frees/zeros the +structures and pdb pages when the client dies - ALL resolution must run +LIVE during the client's ~11 s life (live_walk.py polls dmesg every 0.4 s, +parses pdb + VAs, walks, injects; static family list pre-seeds the +deterministic VAs). + +**2. Progress with the walk+inject live: gpfifo done=0 -> 6-9.** +With the live injections the service decoded and CONSUMED the whole GR ring +(entries raw logged, releases decoded: rel=0x4213000 pay=1, rel=0x45a0bb0 +pay=3, have_rel=1) and submissions reached 6. The client STILL ends +vkCreateDevice -4 (~11 s, RM 10 s poll of control 0x20800a70) because the +releases are decoded but NOT APPLIED and the client's completion counter +page is unresolvable. + +**3. Why releases do not apply (two gates found).** +(a) svc_fwd_defer_arm(): with gsp_phys_gr_forward=1 a GR ring candidate that +is valid but not "offered" keeps defer armed - every GR release goes to the +forwarded-hardware path (which never runs) instead of the inline software +apply. Cycling with gsp_phys_gr_forward=0 removes the arm, but the release +still does not apply - the decode's apply path is additionally gated by +svc.op_tr_pending (a global "operand targeted scan in progress" state) via +svc_opdef_release: while op_tr_pending=1 every release is recorded into an +opdef slot and withheld, silently when the slot already contains it. The +live symptom: `counter page 0x20400f000 unreadable via +vaspaces/discovered/targeted; slot stays pending` repeated at the client's +poll rate plus repeated `targeted resolve va=0x20400f000: full RAM scan +found nothing`. + +**4. The counter/notifier page record tables (host-side resolve).** +Searching the RAM for the counter-page VA as u64 finds 16-byte {pa,va} +record arrays, e.g. {0x5FEDC000,0x20400D800},{0x5FEDD000,0x20400E000}, +{0x5FEDE000,0x20400E800},{0x5FEDF000,0x20400F000},... (VA step 0x800, +PA step 0x1000) and a second variant ({0x5FEEE000,0x20400E000}, +{0x5FEEF000,0x20400F000}). These are how the counter page (and probably +the notifier, notif=0x1xxxxxxx0 per registration) can be resolved WITHOUT +the VA-space walk: search the VA u64, parse the neighbouring u64 as PA, +inject. Must be done live (records vanish post-mortem). NOTE: the service's +own targeted scan reports "found nothing" for this VA - the record layout +the host parses is not what that scan looks for. + +**5. Next steps (ranked) for session 90.** +1. Extend live_walk.py with a record-resolve fallback: for any VA the + service logs as unreadable/pending/stalled (regex set already widens), + search the RAM for its u64, parse {pa,va} records (both orders), inject + candidates; feed it counter-page (0x20400f000 family), notifier + (notif=... per registration) and release VAs BEFORE the decode (static + list already covers the deterministic ones: add 0x20400f000 with the + record-resolve because the pdb walk cannot reach the UVM/counter space). +2. Or fix the two release gates kernel-side: (a) don't arm the GR deferral + when the forwarding engine has no proven channel (offered flag); (b) + treat op_tr_pending as per-request state (clear when no scan is actually + in progress for the entry) so a stuck global does not withhold every + release. Both are small, testable changes (gsp_sema_service.c). +3. Watch for the first live sign: `rel-apply ... written` / `pub-adv` / + `counter auto/derived` lines on the client's channel; then VKDEVICE + should pass and the flow proceeds to vkAllocateMemory. + +**6. Counter-page record resolve PROVEN LIVE (session 89 end).** +`echo 0x20400f000,0x5FEDF000,0x1000,1 > sema_map_add` (the {pa,va} record +found by the u64 search) made the service learn the slot: +`sema-chan: counter auto h=0xbeef0100 rank=0 pa=0x5fedfff0 (slot +va=0x20400fff0)` and the completion publish went from ZERO to 172 calls +(`counter pub_calls=72 -> 172` over the client's life; previously 0 on every +boot of this session). The client still ends vkCreateDevice -4, so the +remaining unmet wait is the GR ctx-init release word (rel=0x4213000 pay=1, +rel=0x45a0bb0 pay=3 - decoded, have_rel=1, never applied: no rel-apply / +pub-adv line for either). With the counter now advancing, the next +instrumented step is naming the exact GR release-handling branch that +swallows the apply (rebuild with a bounded print at each branch of +svc_rel_apply_906f / svc_opdef_release / svc_rel_apply_launch), then fix +that gate. The gr_forward defer-arm gate is already understood (see 3a) and +gr_forward=0 is the correct test configuration; the remaining suspect is +the service-global svc.op_tr_pending path in svc_opdef_release (it records +the release into an opdef slot and returns true silently when the slot +already contains it). + +**7. Session 90: the CURRENT-boot wait loop site (the old recipe no longer matches).** +The client's long spin on this driver/boot sits INSIDE vkCreateInstance (bt: +libEGL -> eglcore -> 0xB2852D helper), a bounded yield loop at +eglcore+0xB2852D: `mov (%r11),%r12d / test $1,%r12b / jne <exit> / +mov 0x4(%r11),%eax / cmp $1,%eax / jne <retry>` - the loop waits for an +IN-PROCESS word pair at r11 (e.g. 0x7ffc2a1c5080, stack/heap; guest-written +only, the host RAM file cannot feed it because it is not a GPU buffer). +Evidence the old site moved: t91's breakpoint at eglcore+0xa85d24 never +fires this boot (`clears=0`) and a read-watchpoint (t80r.py) on the +0x80 +words of the two 16 KB rw-s nvidiactl mappings does not fire either (the +16K event-buffer wait of sessions 77/78 is not the live target now; the +spin stack is +0xB2852D <- +0xA5325A <- +0xA539AD <- +0xA47491 <- +0x9FF515 +<- +0xA20D04 <- libEGL, NOT the 0xA85F1A/A8 path). A widened multi-word +nudge (nudge2.sh/guest_words2.sh: all rw-s nvidia mappings <=256K, +0x80) +DID fire on 20+ words but the client still returned -4 - the session-77 +caution stands: multi-word writes trip the error path; single-word only, +after identifying the true target. +NEXT: (a) rwatch WRITE on r11+4 to name the writer of the in-process word +(the component that must run; if nothing ever writes it, disassemble the +exit branch 0x...ebc9 vs retry 0x...eb93 to see the pass/fail contract); +(b) the same for the 0xA53xxx caller chain - one of these callers is the +A8/NvOgl GL-ucode wait of sessions 78/79 (its completion comes from the +emulated GR engine path); (c) only then re-test the count-clear + single +word nudge recipes. + +**8. Session 91: THE vkCreateInstance WALL BREACHED (the marker-clear recipe).** +The client's retry loop (eglcore+0xB33D30) waits until NO event-slot entry has +the pending marker 0xffff: per entry ptr = *(u64*)(obj+0x1428+i*8), +cnt = *(u32*)(obj+0x22efc), the checked slot = ptr + n*0x10 (n = the call's +esi), marker = u16@+0xe == 0xffff (or u32@+0 when (esi-0x43)<=4) via the +check at +0xB1C750 (OR of all pending markers). The guest's event dispatch +NEVER clears it, so the client burns its 10 s -4 timeout. PROVEN FIX: +clear {u16@slot+0xe, u16@slot+0xc, u32@slot+0} to 0 at every hit of ++0xB33D30 (t93 single-shot; t94 auto-clearer, 4000 hits) -> the client +passes vkCreateInstance AND device enumeration FOR THE FIRST TIME +(`VKINSTANCE ok / VKDEVS 2 / VKPICK`, previously the 10 s -4 fired during +this phase). t94 cleared 5+ distinct module objects (obj=0x5d67063a84a0, +0x5d670661ad80, 0x5d670673eed0, 0x5d67067ebb50, ...) over the client's life. +The client then sat in vkCreateDevice's poll(/dev/nvidia0, nfds=1, +timeout=10 ms) loop at eglcore+0xA12DDF..+0xA12DE4 WITHOUT any -4 for 5+ +minutes (vs the old ~11 s death) - the longest the client has ever run. + +**9. The remaining gap (single): the event-fd signal.** The device-phase wait +is a poll() on /dev/nvidia0 (fd 27) - nvidia_poll wakes only when the guest +RM signals the client's event state; the guest consumes our POST_EVENT +records but never performs the event signaling (session 78's "residual +delivery gap"). Writing to /proc/PID/fd/<nvidia0> is not a signal. The +session-84 count write (base+0x2070b7c, read 4, wrote 0, t97) did not flip +this wait alone. NEXT: fix the host POST_EVENT -> guest dispatch association +(gsp_rm_emulation.c r579 area: the payload fields hClient/hEvent/notifyIndex/ +data/info/status; compare against the guest's _kgspRpcPostEvent -> +gpuNotifySubDeviceEvent expectations) so the guest signals the event and +nvidia_poll returns POLLIN; then the same t94-pattern (or a service-side +equivalent) durably unblocks both waits per boot. + +**10. Session 92: THE AUTOMATED PIPELINE - fresh boot to VKDEVICE with ZERO manual steps.** +On a fresh r692 boot (cycle params unchanged, gr_forward=0) a single gdb +script (t99.py) carrying two breakpoint handlers drove the client end-to-end: +(a) the instance-phase marker cleaner at eglcore+0xB33D30 (t94 pattern: clear +u16@slot+0xe / u16@slot+0xc / u32@slot to 0 per entry at ptr+n*0x10 for +entries *(obj+0x1428+i*8), cnt *(obj+0x22efc)) - fired CLEAR#1..#5 across +five module objects; (b) the device-phase wait-word chaser at +eglcore+0xDCEBD9 (t79 chain: rbx -> *(rbx+0x70) -> *(sub+0x58)=obj2 -> + off=*(u32)(obj2+0x60), holder=*(obj2+0x58), base=*(holder+0x58), +word=base+off) - found WORD addr=0x734781a86080 val=0x0 and wrote 1. +Result: `VKINSTANCE ok / VKDEVS 2 / VKPICK / VKDEVICE ok family=0` with no +manual intervention (r692 client 1725). The client then sat in the +POST-DEVICE wait (the next phase - allocation/engine-init), which t99 does +not cover. The live_walk PT-walk injector ran alongside and injected the +window/ring VAs (0x4d20000, 0x4923000...; note 0x4b19b10 resolved ap=FB - +the GL init release is vidmem-mapped and skipped by the SYS-only injector). + +**11. HARD LESSON (attach discipline, again).** A gdb attached to a client +that never hits its breakpoints wedges: `continue` never returns, the +timeout's SIGTERM does not tear it down, and the ptrace slot blocks all +later attaches (hit 3x this session: 146084, the t91-on-1725 attach, both +left holding). RULE: only attach with breakpoints that are KNOWN to fire +for the client's current phase, keep windows <= ~60 s, and prefer +self-terminating handlers (hit caps). A wedged attach leaves the client +RUNNING (state R) but unattachable; cycle the VM for a clean environment +when more attaches are needed. + +**12. Continuing the chain (next session).** (1) Identify the post-device +wait of the VKDEVICE-ok client (bt sample once attachable; likely the +allocation/UVM phase or the A8/NvOgl engine-init with its count gate at +lib_base+0x2070b7c=4 -> 0 and/or the 0xdceb80 wait loop - add the t91 +count-clear BP to the t99 bundle if the A8 path is live). (2) Extend t99 +into the "durable harness bundle": marker-clean + word-chase + count-clear ++ (once VKFRAME) the release/alloc phases. (3) The bNotifyList=1 +(gsp_rm_ev_nlist) path stays armed for the event-signal experiment; +post/EVPOST logging re-budgeted (gsp_rm_ev_log=2000 set). + +**13. Session 93: THE ZERO-PTRACE PIPELINE (r693) - VKDEVICE with no gdb-wedge risk.** +Split the harness by risk: (a) ONE gdb (t101.py) for the INSTANCE-phase +marker clears only (base+0xB33D30 handler, clears u16@+0xe/+0xc and u32@+0 +of each entry; self-disables after 24 clears; NO other BPs so it cannot +fire constantly); (b) the WAIT WORDS handled by /tmp/unwait.py - a guest +python poller with ZERO ptrace that scans the newest vk_frame's rw-s +nvidia mappings of exactly 0x4000 bytes and writes 1 to any zero u32 at +mapping+0x80 via /proc/PID/mem (the session-77 geometry, confirmed by t79 +chains: word = the +0x80 of a 16 KB rw-s nvidiactl mapping). Result on +fresh r693 (mdev 261980af): t101 CLEAR#1..#5 then idle; unwait wrote FIVE +wait words (0x7876ead57080, 0x7876f1816080, 0x7876ea47f080, +0x7876ea81a080, 0x7876ead51080); client reached `VKINSTANCE ok / VKDEVS 2 +/ VKPICK / VKDEVICE ok family=0` with NO wedge exposure. live_walk (PT-walk +injector) ran alongside. The client then sat in the post-device wait that +the +0x80 poller does not cover (t102.py = the one-shot safe diagnostic: +attach -> print PC/base/A8-count(base+0x2070b7c)/bt -> detach, NO continue +and NO breakpoints - use it when the slot is free). + +**14. THE RM-LOCK WEDGE ROOT CAUSE + RULE.** A gdb killed (SIGTERM) while +the tracee sits AT a breakpoint leaves the tracee STOPPED; if that stop is +inside an RM api critical section (the rwlock held across an RM call!), the +RM write lock stays held and EVERY other client blocks in +os_acquire_rwlock_write (state D) - observed directly: old client 1725 +stopped by its wedged t99 gdb -> new client 7967 hung `state=D wchan= +os_acquire_rwlock_write` on ioctl 0xc020462b; killing the gdb + `kill -CONT` +resumed 1725 out of the ptrace stop but it resumed INSIDE the same RM wait, +so the lock stayed held until the VM was cycled. RULES: never let a gdb +timeout fire while a tracee is at a frequent-firing BP; prefer BPs that +fire once or self-disable; prefer no-ptrace pollers for anything with a +repeating pattern; the safe diagnostic is attach -> read -> detach without +continue. + +**15. Session 94: THE INSTANCE CLEARS ARE REQUIRED (zero-gdb disproved); t104 is the safe form.** +Pure zero-gdb run (fresh r694, client + unwait.py ONLY, no gdb): unwait +wrote three +0x80 words and the client still ended `VKFAIL vkCreateDevice +... -4` - WITHOUT the instance-phase marker clears the device gate fails +even with the words. With t104.py (single BP at +0xB33D30; clears +u16@+0xe/+0xc/u32@+0; self-exits on cleared>=8 OR chits>=40 OR 25 s since +the last clear -> returns True so the script tail runs and batch gdb exits +cleanly, no outer-timeout dependence) + unwait + live_walk: CLEAR#1..#5, +two words written, client reached `VKINSTANCE ok / VKDEVS 2 / VKPICK / +VKDEVICE ok family=0` (r694 client 3725). The client then sat in the +post-device wait again; the t102 diagnostic could not attach because the +t104 gdb still held the slot (its 25 s-idle exit never fired - no +post-instance B33D30 hits - and its 150 s timeout likely fired at a stop += the known wedge risk). NEXT FORM: make t104 detach ITSELF from inside +the handler once the clears are done (`gdb.execute("detach")` from the +stop handler; the tracee resumes; the script ends and gdb exits) so NO +gdb is ever held while the client runs - then t102 always has the slot +and the wedge class disappears end-to-end. + +**16. The allocation/post-device phase is the remaining gate.** With +VKDEVICE green the client waits in a site that is neither the +0x80 word +pattern nor (likely) the A8 count path; the next session should (a) get +the detach-first t104 live, (b) run t102 immediately after VKDEVICE to +name the wait (read PC/bt/A8 count = base+0x2070b7c), and (c) inspect the +service for this client (pushes decoded / releases applied / counter +pub_calls) to see whether the allocation needs the live injections only +or a service-side gap. + +**17. Session 95: THE GDB THREAD-STOP INTERFERENCE + the SIGKILL nuance.** +The t105 attach (t=1.5 s, before the client's worker threads spawned) left +four worker threads in `ptrace_stop.part.0` (futex) - the client's main +thread span waiting on them, so the client froze at the post-device wait +with the gdb holding the trace. TERM/INT/HUP on the wedged gdb were all +ignored (batch gdb stuck in continue). SIGKILL on the gdb (1584) was tried +with the reasoning that all stopped threads sat at futex (NO RM lock held): +result - the tracer died, the tracees DETACHED+RESUMED (client 1518 +SURVIVED with its workers back in futex_wait_queue; contrast session 88 +where SIGKILLed-tracer tracees were KILLED mid-syscall). So the SIGKILL +outcome depends on the tracee's stop context; the mission's never-SIGKILL +rule stands, but this datapoint matters when judging recoverability. +After the gdb died, 1518 still held the RM write lock from its own +post-device wait (its wait word was no longer polled because unwait.py +tracks the NEWEST vk_frame and a fresh test client 6555 was launched and +blocked D-state on os_acquire_rwlock_write) - the unwait-newest coupling +must be replaced by ALL-clients polling in the next form. + +**18. HARNESS DESIGN RULES (accumulated).** (1) A gdb attached across +thread-spawn windows stops worker threads and freezes the client - attach +only AFTER all threads exist, or ensure the gdb explicitly resumes new +threads, or do the work with NO gdb. (2) Any gdb that must run through +thread-spawn windows MUST self-detach quickly (the instance phase spans +~10-20 s and the clears arrive in ~5 bursts) and NEVER rely on outer +timeouts. (3) unwait.py should poll EVERY live vk_frame (not just the +newest). (4) The durable answer for all of this is the service-side +folding: the instance markers and the +0x80 words are guest-RAM writes a +kernel-side service can perform on the same triggers (the marker semantics +and the word semantics are understood) - that removes gdb and /proc +helpers from the loop entirely. + +**19. Session 96: THE DETACH-FIRST CLEARER PROVEN (t106) + the orphan-tracer plague.** +t106.py (single +0xB33D30 BP; clears markers; detaches FROM THE HANDLER as +soon as cleared>=5 or chits>=14: `gdb.execute("detach")` + return True so +the batch gdb exits) on fresh r696 (mdev 241af1c1): CLEAR#1..#5, +`BC-DETACHED cleaned=5 chits=5`, `T106-DONE chits=5 cleared=5`, +`TracerPid: 0` - THE SLOT IS LEFT FREE. unwait.py wrote five words and the +client (1549) reached `VKINSTANCE ok / VKDEVS 2 / VKPICK / VKDEVICE ok +family=0`. THE CLEAN FORM OF THE CLEARER EXISTS NOW. + +**20. THE ORPHAN-TRACER PLAGUE (root cause of every t102 failure).** Every +INTERRUPTED gdb attempt leaves an attached orphan: (a) piping gdb to +`head` closes the pipe early and the gdb stays attached (hit twice); (b) an +ssh command dying/timing out mid-gdb-add leaves the gdb running (hit +again). Symptoms: later attaches fail with `Could not attach ... already +traced by process N` / `ptrace: Inappropriate ioctl for device` and the +t102 log stays empty. RECOVERY (session-95 evidence): TERM is usually +ignored (wedged in continue); SIGKILL the orphan + `kill -CONT <client>`; +the tracee SURVIVES when it was running/at-futex and detaches cleanly +(TracerPid 0). STRICT DISCIPLINE: never pipe gdb output (redirect to a +file, read it after); never let an ssh command end while a gdb may be +attached (generous timeouts); after every gdb run verify TracerPid 0. + +**21. THE NEXT FORM (t107, one gdb, one window).** Combine the proven +clearer and the diagnostic in ONE batch gdb: attach -> clears -> detach -> +`python time.sleep(...)` idle -> RE-ATTACH (gdb can attach again after +detach) -> read PC / bt / A8 count (t102 content) -> detach -> print - +one ssh session, one window, everything self-contained, no orphan risk +(every path ends in an explicit detach; wrap each phase in try/except with +a final detach). Expected: names the post-device wait (allocation/UVM vs +A8 count base+0x2070b7c) so the allocation can be passed and the first +VKFRAME reached. + +**22. Session 97: THE DEEP SUBMISSION STATE + THE COUNTER-SLOT WEB.** +The r696 client (1549) reached the deepest state yet: the service reported +`gpfifo done=11576 pushes=10932 counter pub_calls=1356` and climbing +(gpfifo done was 0 in the pre-pipeline boots) - the client submits the full +work stream and the service decodes/processes it (pairs=14395 map=14318, +ipins=9240 from live_walk). The client then sits in the KNOWN userspace +post-device spin (main thread `running` in all samples; one tid in +os_acquire_rwlock_read). NEW: a counter-slot collision spam +(`counter auto ... pa=0x5fedfff0 rejected (slot owned by another channel; +not installed)`, 232+ callbacks suppressed) - the 0x20400f000 injection +resolved the WHOLE counter VA family to ONE page, so both live channels +collided; the manual bypass `echo <h>,<pa> > .../parameters/sema_counter` +(h_object,pa syntax, main.c:2965) was applied for both handles +(0xbeee0100,0xbeec0100 -> 0x5fedfff0) and pub_calls jumped 1356 -> 3739. +The client gate (VKDEVICE -> frames) persists: its remaining wait is the +userspace spin word NOT covered by the +0x80/16K poller (candidate: the +tracker/counter the service now advances - watch whether pub traffic +alone resolves the spin, else t107-GDB the spin site). ENVIRONMENT: one +orphan gdb (10092 on 1549) + stale channels (beec0100) from prior +attempts; the orphan-kill discipline applies (SIGKILL by PID + CONT + +verify TracerPid). + +**23. THE IOCTL-ARG SAMPLING AXIS (no ptrace).** /proc/<tid>/syscall shows +the running ioctl's ARGS (fd, cmd, arg) when caught - sampling the main +thread repeatedly during an ioctl-loop phase names the exact stuck RM +control without any gdb. Use it before ptrace-based diagnostics when the +thread does syscalls at a catchable rate. + +**24. Session 98: THE POST-DEVICE SPIN NAMED - THE SEC2 SCRUBBER AUTH-TAG WAIT.** +perf (no ptrace!) on the spun client: top symbol `channelReadChannelMemdesc` +24.33% (then scheduler internals) - the guest's nvidia driver hot-loops +reading the CHANNEL MEMDESC. Source decode (nvidia-570.144-reference, +sec2_utils.c): the only reference caller is +`_sec2utilsUpdateGetPtr` -> `channelReadChannelMemdesc(pChannel, +authTagBufSemaOffset)` inside `_sec2utilsGetNextAuthTagSlot`, whose loop +waits for the scrub auth-tag slots to free; `authTagBufSemaOffset = +finishPayloadOffset + CHANNEL_ENGINE_SEMAPHORE_SIZE` (4). So the spin = +the SEC2 MEMORY-SCRUBBER channel waiting for its auth-tag semaphore (a u32 +inside the channel memdesc buffer) to advance - the GPU never delivers it +in our emulation, so the client waits forever (each poll = one full memdesc +transfer = the 24% cost). THE FIX DIRECTION: the service must complete the +scrubber's semaphore updates - the scrubber channel registers in the +service (`scrubber channel 0x... detected (gpfifo=... userd=...)`) and its +submits carry semaphore releases targeting the auth-tag VA +(pbGpuVA + authTagBufSemaOffset) in the channel buffer; on decode, write +the release value to that VA's page (the channel buffer is a guest-RAM +allocation the service can resolve/pin; the pending queue holding 17 +entries is the suspect pool for the unresolvable ones - inspect +svc_pending[]). Service-side counter to watch after the fix: the scrub +loop exits, the client proceeds to the allocation/pipeline. Tool: perf +record/report on the guest is the no-ptrace spin-pointer (`perf record -p +<pid> -- sleep 3; perf report --stdio --sort=symbol,dso`). + +**25. Session 99: THE SCRUBBER GATE CONNECTED TO THE CE CHANNELS' ok=0.** +chan_dump on r696 (client 1549 spun): six channels - +`h=0xbeee0100 eng=0 gpfifo=0x44a0000 userd=0xfee00 ok=1 get=6 sub=6354 rel=0x4213020 crel=0x5fedfff0 notif=0x11cd51060` +`h=0xbeec0100 eng=0 gpfifo=0x4cc0000 userd=0xfee00 ok=1 rel=0x4923020 crel=0x5fedfff0` +`h=0xbeec0104 eng=0 gpfifo=0x4d20000 userd=0xfee00 ok=1 rel=0x4b17e30 crel=0x5fedffe0` +and the CE-class channels `beee0101 (eng=11, gpfifo=0x4500000 userd=0x8f600 as=2 ok=0)`, +`beee0102 (eng=12, gpfifo=0x4560000 userd=0x8f400 ok=0)`, `beed0101 (eng=12, +gpfifo=0x49c0000 userd=0x8f200 ok=0)` - ALL ok=0: the CE userd discovery +(failed probe: EAGAIN 0/0 or EINVAL) means no CE pump, no CE sema applies, +so the SEC2 scrubber channel (a CE-class channel) never sees its auth-tag +semaphore advance and the client spins in _sec2utilsGetNextAuthTagSlot. +The service's candidates (registered PA, ring+0x10000 in both apertures, +8-slot list) all miss for the CE channels. `find_userd` (FB scan for a +live put@0x8c/get@0x88 pair) found ZERO candidates - the CE userds are not +in FB with live puts (either the client's PUT writes go to a page the +service's candidates miss, or the CE userds genuinely idle). A raw host-RAM +shape scan (put in 1..65535, get<=put) is too noisy (24 hits in the first +28KB of BIOS-era low RAM) - needs tighter validation. NEXT APPROACH: find +the REAL userd page by DIFF-WATCH - sample the candidate pages (and the +CE ring pages) twice around a scrub submit and diff for the page whose ++0x88/+0x8c pair changes; the guest kernel writes GP_PUT there, so the +changing page is the true userd. Then fix the discovery (or add a +sema_userd-style injection param) so the CE pump engages and the scrubber +completes. NOTE: the GR channels' userd=0xfee00 (as=2, FB) and ok=1 with +get=6/sub=6354 - the GR machinery runs fine; only the CE/scrubber path is +dark. + +**26. Session 100: THE CE USERD HUNT EXHAUSTED THE STANDARD LAYOUTS; THE CIRCULAR WAIT.** +Candidates examined for the CE userds this session: (a) the registered FB +offsets (0x8f600/0x8f400/0x8f200 as=2) - ALL ZERO in the FB shadow +(sema_vidmem_dump 0x8f100,0x600 twice); (b) the ring-PA + 0x10000 linear +guess - random/zero/table garbage (the allocations are NON-CONTIGUOUS); +(c) the PDB walks of gpfifo+0x10000 (0x4501000->pa 0x125e91000, +0x4561000->pa 0x132032000, 0x49c1000->pa 0x11bddd000 via pdb 0x1152f4000) +- ALL THREE PAGES ALL ZERO. The CE userds were never written - and the +deeper reading: the SEC2 scrub flow is a CIRCULAR WAIT: the scrubber waits +for its previous scrub pushes to be CONSUMED (the auth-tag sema advances +only when the GPU runs the scrub) -> the consumption needs the CE pump -> +the CE pump needs a live userd -> the userd is written BY SUBMITS -> the +submits are held by the wait. In a working boot the guest's first scrub +push happens BEFORE the wait loop (the loop only waits when the slots are +full) - so in our boots the FIRST scrub push must have gone into the CE +ring, the service failed to consume it (ok=0/userd dark), and the client +now waits for a completion that will never come. + +**27. THE DIRECT BYPASS (next session, ranked).** (1) Write the auth-tag +semaphore in the CHANNEL BUFFER (memdesc) page directly: the scrub loop's +`_sec2utilsUpdateGetPtr` reads `channelReadChannelMemdesc(pChannel, +authTagBufSemaOffset)` = a u32 at memdesc+finishPayloadOffset+4; advancing +that value frees the slots and unblocks the client WITHOUT the CE pump. +The memdesc = the channel buffer allocation (a VA in the client's space); +resolve it via the PDB (the guest's channel struct gives the buffer VA; +the service's channel registration/alloc params carry it) and write the +incremented value from the service (or the host RAM file). (2) OR decode +the scrub pushes the service already sees (check sema-push for the SCRUBBER +channel - its submits carry the auth-tag VA; the release decode should +target it; the service's release applies would write the sema if the VA +resolves). (3) The registered FB userd offsets being zero suggests the +client's kernel maps the userd through a VA whose PA the discovery misses; +the diff-watch (two samples around activity) remains the fallback proof. + +**28. Session 101: THE LOOP SEMANTICS + perf trace (NO-PTRACE SYSCALL TRACING).** +The wait loop decoded exactly (sec2_utils.c:420-441): +`while (nextPut == pSec2Utils->authTagGetIndex) { gpuCheckTimeout(); +authTagGetIndex = _sec2utilsUpdateGetPtr(); osSpinLoop(); }` - it exits +when the memdesc sema (u32 at pChannelBufferMemdesc + +authTagBufSemaOffset; authTagBufSemaOffset = finishPayloadOffset + 4) +advances past nextPut; it also has a GPU timeout that would error out. +perf trace on the spun client (no ptrace!) revealed the live pattern: +tid 1793 = a 10ms-timed futex loop (WAKE val=1 then +WAIT_BITSET|CLOCK_REALTIME 10ms - the paced retry); tid 1788 = repeated +`openat -> read(fd, 299 bytes) -> close -> futex-wait` - a transient-fd +POLLING READER of a small blob (the fd is opened and closed each cycle, so +fd 28 no longer exists; the client's persistent fds: /dev/nvidia-modeset +(10), /dev/nvidia0 (11/12/14/15), GLCache .toc/.bin (16/17), memfd +\"allocation fd\" (18)). The 299-byte read is the likely userland face of +the memdesc read path (the 24% channelReadChannelMemdesc hot loop). +NEXT: (a) catch the transient fd path by sampling `ls -l /proc/<pid>/fd` +every ~0.2 s in a loop around the burst (or perf trace -s during a burst); +(b) if it is a driver read() of the memdesc, resolve its buffer and write +the auth-tag sema value (advance past nextPut) via the RAM file; (c) the +loop timeout (gpuCheckTimeout) may eventually error the client - the write +must land first. + +**29. Session 102: fd-sampler miss, no nvidia debugfs; the memdesc hunt narrows.** +The 0.15 s /proc/1549/fd sampler over 20 s caught NO transient path (the +open/read(299)/close reader is bursty or dormant between bursts; the static +fds are /dev/nvidiactl, /dev/nvidia0, /dev/nvidia-modeset, a socket, the +GLCache .toc/.bin pair, and a memfd "allocation fd"). No nvidia debugfs +tree exists in the guest. Remaining ranked paths to the auth-tag sema +(memdesc + finishPayloadOffset + 4): (1) content-locate the scrubber's +channel buffer by its method stream (the C5B5/C56F shapes: SET_OBJECT +0xc5c0/0xc5b5, headers 0x200121ed/0x200120a9 observed in the session-78 +window dumps) and read/write the sema inside; (2) the scrubber channel's +own sema-push decodes (its pushes carry the auth-tag VA = pbGpuVA + +authTagBufSemaOffset; grep the boot's sema-pushall for the scrubber +channel id and read the rel= VA - then PDB-walk it); (3) sample the fd +burst narrower (0.02 s) right after a client restart when the burst is +hot. The service-side ideal: on detecting the scrubber channel, resolve +its channel buffer and pre-advance the auth-tag sema (a durable fold). + +**30. Session 103: THE SCRUB STREAM FOUND; ONE RELEASE DELIVERED; THE SEMA PAGE STILL AMBIGUOUS.** +Content-located the CE/scrub init method stream (headers 0x200121ed/ +0x200120a9 searched): full stream at `pa=0x114e98000` = SET_OBJECT +0xC5C0/0xC5B5 (0x20012000/0x20012080 headers), engine-init methods +0x200121ed(3)/0x200120a9(1)/0x2001209c(0x0c000001), bulk methods +(0x8000xxxx), then TWO REPORT_SEM releases in one buffer: +`0x200426c0 00000000 04b19b10 ...` (va=0x4b19b10) and +`0x200426c0 00000000 04b17e30 ...` (va=0x4b17e30), tail 0x2001c00d +70/68. Twins at pa=0x1026ad000+0x87c and pa=0x12afba000+0x87c (identical +content at identical offsets - multi-mapped copies). Release resolution: +`0x4b17e30 -> pa=0x114efbe30 (SYS) cur=1` - THAT release WAS delivered (the +service applied it, cur=1). `0x4b19b10` resolves via the walker's big-half +2MB branch to FB 0x660b10 holding garbage (0xD9F26F94) - poke_vidmem +tests ([0x660b10]=1 then =0x40000000) did NOT flip the client, and the +linear-block guesses (buffer VA 0x4b00000 <-> stream pa 0x114e98000, sema +candidate 0x114eb1b10) read all zero. NEXT: the auth-tag sema page must be +picked by VALIDATION (the existing value must be a small plausible +GetPtr): walk 0x4b19000 returning ALL chains (small/64K/2MB both +apertures), pick the page whose u32 at +0x1b10 (or the offset the memdesc +actually uses) is a small number; or take the buffer aperture from the +channel's alloc params (the service's registration) and walk with that +aperture exclusively. The fix remains: advance that u32 (write via the +host RAM file if SYS, poke_vidmem if FB). + +**31. Session 104: THE SEMA VA 0x4b19b10 IS NOT THE LOOP'S PAGE; THE KPROBE IS THE ANSWER.** +The validated multi-chain walk (small-4KB and big chains BOTH) resolves +VA 0x4b19b10 -> FB 0x660b10 (both chains converge; the guest's own PTs say +the object lives in vidmem). poke_vidmem writes at 0x660b10 (0x1, +0x40000000) and 0x660b14 (0x1, 0x40000000) LANDED (verified via +sema_vidmem_dump: [0x660b10]=0x11112222) yet the client does not flip and +perf still shows channelReadChannelMemdesc at 24.52%. CONCLUSION: the +REPORT_SEM VA 0x4b19b10 (and 0x4b17e30) are NOT the auth-tag sema the loop +reads - they are the GL-init release objects; the real auth-tag sema is at +pChannelBufferMemdesc + finishPayloadOffset + 4 (a VA and page we do not +yet know; finishPayloadOffset is NOT 0x19b10). THE DEFINITIVE INSTRUMENT: +a guest kprobe on channelReadChannelMemdesc (the offset argument = %si = +the authTagBufSemaOffset VALUE; and the pChannel pointer lets us read the +channel struct -> pChannelBufferMemdesc): `echo 'p:crcd +channelReadChannelMemdesc offset=%si' >> /sys/kernel/tracing/kprobe_events` ++ enable the event + read tracing pipe WHILE the loop runs. This names the +offset and the memdesc in one shot, then the sema page = walk(memdesc+off) +and the advance (RAM file if SYS; FB + poke if the converged mapping). + +**32. Session 105: KPROBE BLOCKED (notrace); THE BAR1-PATH ANALYSIS; THE CHANNEL-FIELD OFFSETS.** +(1) The kprobe on channelReadChannelMemdesc is IMPOSSIBLE: the guest's +dmesg says `trace_kprobe: Could not probe notrace function +channelReadChannelMemdesc [nvidia]` - the function is annotated notrace +(ftrace/perf-probe/BPF-kprobe all blocked). The tracefs filesystem had to +be mounted first (mount -t tracefs tracefs /sys/kernel/tracing) and it +already carries UVM kprobes from earlier sessions (uvmt/uvch/uvbp/uvtw/ +uvsv...). The static sec2utils functions (the callers) are absent from +kallsyms (inlined) - no probe target on that path. +(2) BAR1-path analysis: the guest driver's memdesc read uses +TRANSFER_FLAGS_USE_BAR1|SHADOW_ALLOC|SHADOW_INIT_MEM - a CPU read of the +GUEST'S BAR1 mapping, which the mdev mmap maps onto the HOST's vidmem +shadow (the same buffer poke_vidmem writes). The pokes at FB 0x660b10/ +0x660b14 LANDED (verified) yet the loop did not advance, so the memdesc +page is NOT the FB 0x660b10 the PT walk converged on (VA 0x4b19b10 belongs +to the GL-init release object, not the channel-buffer memdesc). +(3) The channel registration parse exposes the guest-visible channel +fields and their alloc-param offsets (gsp_rm_emulation.c +gsp_rm_register_channel: GSPRM_CHAN_GPFIFO_OFFSET/ENTRIES, ENGINE_TYPE, +CID, INSTANCE_MEM, USERD_MEM(+APERTURE), NOTIFIER_MEM(+APERTURE)) - the +channel-buffer/memdesc field (pChannelBufferMemdesc) is not among them; +the next session should dump a CE channel's FULL alloc params (the +service's rate-limited alloc dumps or a fresh boot) and look for the +channel-buffer VA/PA field, then sema = buffer + finishPayloadOffset + 4 +(per-channel offset, e.g. the 0x19b10-style for the GL channel). +(4) Fallback if the field is not in the params: per-VA content hunt for +the auth-tag sema VALUE (the retprobe would have given it; without +kprobes: the sema value = a small slot counter - scan the CE channel's +allocation candidates for a plausible counter that CHANGES across two +samples). + +**33. Session 106: THE CHANNEL BUFFER IS NOT A CLIENT PARAM; THE TAIL SWEEP AND HASH SCAN NEGATIVE.** +NV_CHANNEL_ALLOC_PARAMS (alloc_channel.h:300-341) layout mapped to the +service offsets: gpFifoOffset@8/entries@16, engineType@128, cid@132, +instanceMem@144, userdMem@168, ramfcMem@192, mthdbufMem@216, +hPhysChannelGroup/internalFlags, errorNotifierMem@248... - THE CHANNEL +BUFFER (pChannelBufferMemdesc) IS NOT IN THE PARAMS: it is a +driver-internal memdesc created by channelAlloc, so the alloc-params path +cannot name it (the kprobe/debuginfo path was the right one; it is +notrace-blocked). The finish-payload tail of the stream page +(0x114e98000 +0xd4..+0x118) was swept with 0x40000000 (18 u32s; the +pre-sweep values: +0xd8=0x68000000 finish marker, +0xe0=0x901025a6, ++0xe4..+0xec=0x8000xxxx methods) - no flip. A 16-page high-entropy scan of +0x114e98000..0x114ea8000 for the auth-tag SHA-256 array: ZERO high-entropy +32-byte blocks (the hash array is elsewhere / another allocation). NEXT: +(1) the memmgrMemDescBeginTransfer SHADOW insight - the read happens from +a FRESH kmalloc'd shadow copied from the memdesc each call; the shadow's +source = the memdesc's backing page; the memdesc page must be found by +TRANSFER-TIME properties (not the VA walk); (2) or a guest perf hardware +watchpoint on the driver's BAR1 ioremap VA + the (unknown) memdesc offset; +(3) or the service-side broad assist: increment a RANGE of plausible FB +sema cells periodically (blind but bounded), watching for the flip. + +**34. Session 107: FB ASSIST SWEEPS NEGATIVE - THE READ IS NOT FB-SOURCED; THE TWINS ARE THE NEXT TARGET.** +Two bounded FB "sema assist" sweeps (distinct values per cell, +0x40000000|idx): (a) the walked page 0x660b00..0x661000 (320 cells) - no +flip; (b) the CE-channel allocation cluster 0x8f000..0x90000 (1024 cells, +where the registered userds live) - no flip; perf re-confirmed +channelReadChannelMemdesc 23.13% (the loop still runs). CONCLUSION: the +memdesc read does NOT come from the FB shadow the CPU can poke - the +transfer's SHADOW_INIT must read the memdesc through a path that bypasses +these bytes, or the memdesc backing is SYSTEM memory via an unknown VA. +NEXT (one-minute test): the STREAM TWINS - pa 0x1026ad000 and +0x12afba000 hold BYTE-IDENTICAL copies of the CE method stream (verified at ++0x7c0/+0x880: same 0x68000000 finish marker area, same 0x200120a9/9c +methods) - one of them is very likely the memdesc BACKING in sysmem (the +driver copies the same stream per channel/boot). Sweep BOTH twins' +stream-area pages (reuse the assist code; 1 page each) and watch for the +flip; if it flips, bisect the winning page to the sema cell and make it +durable (host RAM file writes = zero-cost, no poke_vidmem needed for +sysmem). Failing that: content-diff the twins across a driver transition +(the memdesc copy updates during submits; the copy that CHANGES is the +backing). + +**35. Session 108: THE CONTENT-DIFF - 9 ACTIVE PAGES + THE CONSTANT-READ DEDUCTION.** +Page-hash diff over three +/-256KB windows (twins 0x1026ad000/0x12afba000, +stream 0x114e98000) across 15 s: NINE pages change: +0x102690000, 0x102698000 (440 word diffs - a buffer written continuously = +the push copy!), 0x1026a7000 (+0xe8: 934 -> 940 - a slow clean counter), +0x1026b3000 (+0x48: 32768 -> 1 transition), 0x1026b5000 (+0x244/+0x24c: +24987 -> 32765, two u16s at ~650/s), 0x1026c2000 (+0xe8: 232 -> 3, a +wrapping counter mod ~233), 0x1026e6000 (+0x18: 2764 -> 1930, +0x88/+0x90 +decreasing - countdowns), 0x114e6b000, 0x114e6e000. THE DEDUCTION: the +stuck loop's read value must be CONSTANT (its page shows no writes; the +GPU never advances the sema), so the auth-tag sema is a static cell - +likely zero - and the fix is to make the read NONZERO (advance it) at the +right page. None of the 9 changing pages is it (their cells move on their +own). NEXT: (1) restrict the search to CONSTANT pages among the CE +channel's candidate backing (the twins region pages that DID NOT change +but sit adjacent to the active set); (2) target the sema cell: the cell +that equals nextPut (small constant, e.g. 0) at the memdesc+off; sweep the +CONSTANT pages' zero u32s in a bounded ring around the twins/active set +with 1 (the sysmem write is free); (3) the durable service fold once +found. + +**36. Session 109: THE BURST vs HEARTBEAT DISCOVERY + THE 200ms READER.** +perf trace of the whole process over 10 s with -e ioctl: ZERO ioctls from +ANY thread; meanwhile the process shows a strict 200 ms pattern on thread +1788: `openat(filename ptr 0x5eaec740) -> read(fd 28, count 299) -> close` +(the transient fd the fd-sampler missed!) and thread 1793's 10 ms +futex re-arms. The perf-reported 24% channelReadChannelMemdesc (a kernel +function only reachable via RM calls) CONTRADICTS the zero-ioctl window: +the hot kernel work is BURSTY (periodic rounds between the 200 ms +heartbeats), not continuous. The 299-byte blob read every 200 ms = the +client's health poll (unnamed path; candidate: /proc/driver/nvidia/... +informational node - find it by reading /proc/PID/io or by narrowing the +sampler to the 200 ms phase). THE MODEL NOW: the client idles (futex +waits) + wakes every 200 ms to poll a node + periodically runs RM bursts +(the burst = a full retry round hitting channelReadChannelMemdesc). NEXT: +(1) name the 299-byte node (it may BE the wait's condition - a status the +service must change); (2) capture the BURST with perf (a long record + +per-thread breakdown) to see which thread runs the memdesc reads (the RM +burst thread) and correlate with the host-side RPC traffic in the same +window (the host dmesg RPC lines during a burst = the exact RM call!). + +**37. Session 110: THE LOOP IS CONTINUOUS INSIDE ONE LONG IOCTL (THE LOCK MODEL CLOSED).** +(1) The 299-byte/200ms reader: the filename pointer 0x5eaec740 is NEVER in +the client's maps (30 Hz sampler over 15 s, zero hits) yet openat returns +28 every cycle - a decode artifact/transience, and at 5 reads/s it is +irrelevant to the 24% blocker. Treat it as a RED HERRING. (2) A 20-second +perf record (80K samples): channelReadChannelMemdesc 24.65% + a scheduler +family (pick_next_task_fair/update_curr/pick_eevdf/__schedule/ +schedule_timeout/os_schedule ~30% total) = the SEC2 wait loop running +CONTINUOUSLY. The earlier zero-ioctl windows are explained: THE MAIN +THREAD SITS INSIDE ONE LONG-LIVED ioctl (entered hours ago; the loop runs +in the kernel inside it) - and that ioctl HOLDS THE RM WRITE LOCK, which +closes the lock model: tid 1791 waits on the RM read lock, new clients +block D-state (os_acquire_rwlock_write), all consistent. (3) PEBS/ +mem-loads events are unavailable in the guest perf (no memory-address +sampling) - the read address cannot be sampled. Instruments for the sema +address are now exhausted short of the notrace blocker or the blind +service-side assist. +NEXT: (1) the blind assist, refined: the loop's READ is CONTINUOUS and +the sema is CONSTANT - sweep ONLY constant small-value cells in the CE +candidate pages (see session 108's active set for the region), one value +at a time, 2 s per batch, watching the flip; (2) or a graft: since the +loop runs inside the client's blocked ioctl, unloading the CLIENT (kill) +releases the RM lock and the fresh client restarts the same phase - use +fresh-client runs for any A/B of service-side fixes; (3) the durable +service-side fold remains the endgame once the cell is known. + +**38. Session 111: THE FB IS FULLY DISPROVED (WHOLE-FB FILL, NO FLIP); THE SYS-FILL WEDGED THE GUEST (CYCLE NEXT).** +(1) scan_vidmem for the stream signatures (0x200121ed, 0x200120a9) and even +0x68: ZERO hits across 256MB - THE FB SHADOW WAS ALL ZEROS (before any +pokes): the GPU-side objects are never written in our emulation (the +driver->GPU DMA never runs), so the driver's memdesc SHADOW_INIT copies +zeros and the sec2 loop reads 0 forever (nextPut==authTagGetIndex==0 -> +infinite wait; the complete model). (2) The refined assist: swept FB +0x80000..0x90000 (16384 cells, distinct values; 60s/32KB via poke_vidmem) +- no flip. (3) THE DEFINITIVE TEST: `fill_vidmem 0x40000000` filled the +WHOLE 256MB FB shadow in-kernel (`vgsp: fill_vidmem value=0x40000000 +size=0x10000000 done`) - STILL NO FLIP: THE LOOP'S READ DOES NOT COME FROM +THE FB SHADOW AT ALL; the channel-buffer memdesc is SYS-BACKED (or another +backing) in our emulation, not the FB. (4) The follow-up SYS fill (128 +candidate pages near the active set overwritten with 0x40000000 via the +RAM file) CLOBBERED LIVE DRIVER DATA and WEDGED THE GUEST (SSH banner +timeout; QEMU alive) - the next session MUST CYCLE (fresh boot); lesson: +never bulk-overwrite pages adjacent to the identified active set (0x102698000 +was a hot staging buffer). +NEXT: (1) fresh cycle; (2) the sema is a SYS page - apply the full-page +marker fill ONLY to genuinely-constant, non-active pages (re-run the +content-diff first, then fill the constant set in small batches with a +client check after each, NOT the active neighborhoods); (3) or the +service-side durable route: have the SERVICE write the marker into the +channel-buffer page once it can resolve it (the memdesc VA = the CE +channel's buffer; the service's channel registration fields + the PT walk +give the page candidates). + +**39. Sessions 112b-114: THE SCRUB GATE FELL, THE COMPR WALL WAS DECODED AND FLIPPED, AND THE FIRST VULKAN SUBMISSION LANDED (VKFENCES IS THE NEW WALL).** +Context first (sessions 112b/112c, commits `97489187`/`cb7d8ce7`/`752e2b30`/ +`d6ccdaf3`/`8700f65a`): the CeUtils scrubber's finish payload is authored +in-service (bound to the EXACT 4096-entry channel), the client then leaves +`scrubCheckAndWaitForSize` permanently and runs +vkCreateImage -> vkGetImageMemoryRequirements -> vkAllocateMemory; the +remaining `-2` was decomposed to the guest-internal VidHeap `#149`-shape +call (attr COMPR_ANY + 2MB page + owner 0x474C0000) returning 0x51 +NV_ERR_NO_MEMORY; session 113 read that as a 64KB comptag/CBM alloc. This +session REFINED it: the COMPR+2MB class is not always 64KB - the r705 +runs show the *image data* alloc itself (size 0x140000 = 1.25 MB for +640x480x4 + pad, attr 0x1002000, flags 0xc10d) in exactly the same shape. +The "separate 64KB CBM" reading was too narrow; every alloc of the class +{attr&0xF000 == 0x2000, 2MB page, owner 0x474C0000} fails 0x51. + +**A. THE ZERO-TOUCH CREATE RECIPE (NEW, SCRIPTED).** Two live assists are +required per client, both proven again this session: (1) the +instance-phase marker clears at eglcore+0xB33D30 (t106/t99 `BC`, 5 +clears); (2) THE DEVICE-PHASE WAIT-WORD CHASER at eglcore+0xDCEBD9 +(t99 `BW`: word = *(holder+0x58)+*(u32)(obj2+0x60) with +obj2 = *(*(rbx+0x70)+0x58); write 1 when 0). Without (2) the client +reaches `VKDEVICE`'s poll(`/dev/nvidia0`, 10 ms) retry loop and sits +there INDEFINITELY (session-76 state; the a097 engine-monitor worker +keeps polling 0x2080a097 every 1.25 s regardless - its reply words are +still a red herring for the main-thread gate). With (1)+(2) the run is +`VKINSTANCE ok / VKDEVS 2 / VKDEVICE ok family=0` then straight to the +image path. `t99.py`-style attach works too: attaching `BW` to an +already-stuck client writes the word and the client immediately +proceeds (proven on a stuck pid 15797: WORDWROTE x2 -> VKDEVICE ok -> +VKFAIL -2 at the alloc). +NEW TOOLS: `t120.py` (BC+BW+single-shot COMPR->non-COMPR flip), +`t121.py` (BC+BW+full VidHeap trace from #120), `t122.py` (BC+BW+flip +for ANY COMPR+2MB internal alloc, size >= 0x10000), `a8clear.py` (the +no-ptrace count-gate clearer for base+0x2070b7c, reconstructed). + +**B. THE FLIP (THE DECISIVE EXPERIMENT).** With `t122.py`: at the +vidheap ioctl whose params match (attr&0xF000)==0x2000, owner +0x474C0000, size>=0x10000, the script clears attr bits 13:12 +(COMPR_ANY -> NONE) in the client's own params before the kernel sees +them. Result (r705c, reproducible twice on the same boot): +`HIT149 size=0x140000 attr=0x1002000 -> 0x1000000` (image data alloc +SUCCEEDS), a second `HIT149 size=0x140000 attr=0x1022004 -> 0x1020004` +(buffer-class COMPR alloc) also succeeds, and the client runs the WHOLE +pipeline for the first time in project history: +vkCreateImageView -> buffer -> shader module -> descriptor set +layout/pool -> pipeline layout -> compute pipeline -> command pool -> +command buffer -> fence -> vkResetFences -> **vkQueueSubmit OK** -> +`VKFAIL vkWaitForFences(dev, 1, &fence, VK_TRUE, 30000000000ull) -> 2 +(line 459)` - THE FIRST VULKAN SUBMISSION, then the FIRST FENCE TIMEOUT +(30 s, the client's own bound; the run exits rc=1 cleanly). + +**C. THE NEW WALL = THE FIRST FENCE RELEASE.** Host-side evidence in the +submit window: the client's GR channel registers +(`sema-chan: h=0xbeec0100 eng=0 gpfifo=0x4cc0000 ent=32768 +userd=0xfee00 put=11 get=2 ring rc=0 pa=0x1217e3000 ap=1`) - the guest +driver published 11 GPFIFO entries, the emulation's reader consumed 2; +the service still reports the channel-init pushes unread +(`sema-entry: unread h=0xbeec0100 idx=2 va=0x4ca059c len=976 want=0 +exp=0 unread=1`) and `gr_wedge` grew 3 -> 9 over the runs. So the fence's +release is never authored after the submit - this is the session-76/93 +"wall #2" class (push-page readability/completion), now the ONLY gate +between the flipped client and the first `VKFRAME seq/DONE` + PPM. + +**D. DURABLE-FIX LEADS (for the COMPR class, no guest patching).** +(1) memmgrAllocHal_GM107 (570.144) downgrades COMPR_ANY -> NONE when +`pMemorySystemConfig->bDisableCompbitBacking` is set - the guest's +MEMORY_SYSTEM_STATIC_CONFIG comes from the GSP (the MEMSYS_GET_STATIC_ +CONFIG control 0x20800a1c / the GSP static info path); answering it (or +the same knob wherever the guest's config actually comes from) is the +host-side durable equivalent of the flip. 0x20800a1c was NEVER observed +in the RM_CONTROL log, so first find where the guest's config is fed +(control cache? GSP init args?) before coding the handler. +(2) Alternatively locate glcore's COMPR decision (attr construction at +eglcore+0x2c83fb/0x2c8465 and +0x50e038/0x50e894/0x50e8bd per session +113) and answer the capability that drives it. (3) The session-85 +device-local -2 story (2g mdev type) remains a prerequisite context. + +NEXT (ranked): (1) author the fence release for the first submit (the +push-page-5 content-match / pinning machinery; or have the service +complete the fence when it consumes the submit's GPFIFO entries); (2) +then the first VKFRAME seq/DONE + PPM; (3) fold the COMPR flip into a +host-side answer (see D); (4) the mission series (Xorg/GZDOOM, KVMFR, +CUDA 15/15 + soak, test series, video, Windows). + +**40. Session 115: THE FENCE WALL DECOMPOSED - THE CLIENT VAS PAGE DIRECTORY IS UNDISCOVERABLE AND THE PUSHES ARE NONCONTIGUOUS; RING PAGES ARE CONTENT-LOCATABLE AND INJECTABLE, BUT THE RING PAIR ALONE DOES NOT UNLOCK THE PUSH DECODE.** +(1) The fence timeout is upstream of everything else: the sema service cannot +decode the submit's GR pushes (`sema-entry: unread ... va=0x4ca0000 +len=1432` -> `skipping stale entry ... after 185-194 failed rescues`; +`work_gr submitted=0` - no GR forward ever starts). (2) The pump's push +reads resolve ONLY through the client VAS page directory (or pins). The +service has NO pdb for this client: `walk_probe` of the ring and push VAs +prints `discovered pdb=0x0 found=0` and no registered VASes; the r496f +vidmem pdb discovery never ran (it is only called from the RAM ring-scan +latch), and the r496j COPY_SERVER_PDES candidate (0x109b9c000) is garbage +(its content is not a page table). (3) THE RAM RING SCAN MISSES BY +CONSTRUCTION for this client: it sweeps [0, 6 GB) in 12 slices and gives +up (`found no candidate in 12 slice(s)`), but the sweep runs BEFORE the +client's pushes exist and never re-runs - while the ring page content is +provably in RAM (content-located below). (4) CONTENT LOCATION WORKS: the +GR ring page (ring VA 0x4cc0000, first entries {va=0x4ca0000 len=1432, +va=0x4ca0598 len=4}) was found in the guest RAM file by its first two GPFIFO +entries (at GPA 0x12C600000 then 0x12898c000 on later runs - per-run +allocations), and `sema_map_add` injection of the TRUE ring pair is +accepted (`INJECT va=0x4d20000 pa=0x13619d000 : ok`). BUT the pushes do not +become readable from the ring pair alone: the pushbuffer is NONCONTIGUOUS +(the window guess `ring_pa - 0x20000` reads zeros for both the GR and the +CE channels), the pushes are not CPU-mapped in the client process (the +ring GPA is absent from /proc/PID/pagemap of every rw-s nvidia mapping), +and a bottom-up search for the level-3 table (a 4 KB page whose index-2 +big/small entry maps into the ring's 2 MB window) found ZERO candidates in +RAM - the page tables are almost certainly in VIDMEM (r496f's note) or use +a shape the walker does not model. (5) The service's OWN successful read of +one CE init push (`sema-push: h=0xbeec0104 va=0x4d00000 ... rel=0x4b19b10 +pay=1 have_rel=1`, `ops_runs=0`) did NOT apply its release either - +whatever resolved that one read is not reproducible for the GR pushes. +NEW TOOLS: `fence_assist.py` (host: watches a `dmesg -w` capture for the +pump's entry decodes, content-locates the ring pages by their first two +entries in the RAM file, injects the pairs via sema_map_add - proven to +find and inject, but the ring pair alone is insufficient) and +`find_push.py` (guest: tests whether the client CPU-maps the ring GPA). +NEXT (ranked, concrete): (1) find the client VAS page directory: extract the +pdb from the GUEST kernel's RM objects (kcore walk of the VAS/GPU object +for this channel - the GPFIFO ring page content anchors the search), then +register it via the service's VAS table (a new sysfs/debugfs entry point +or the existing candidate path) - this is the "PDB injection" the mission +names; (2) or scan VIDMEM for the pdb with the ring pair as the verifier +(extend `svc_xvas_pdb_discover`'s trigger: call it from the ring-injection +path, not only from the RAM scan latch); (3) make the RAM ring scan re-arm +per channel with the pushes present (its 12-slice give-up is the reason +the auto-discovery never latches on this client); (4) then the submit +decodes, the release lands and `vkWaitForFences` returns -> VKFRAME. diff --git a/tools/guest-diag/wall2/README.md b/tools/guest-diag/wall2/README.md new file mode 100644 index 0000000..1ef9558 --- /dev/null +++ b/tools/guest-diag/wall2/README.md @@ -0,0 +1,85 @@ +# WALL #2 (GR channel-init release) — session 73 findings + tools + +## Goal +The vk_frame Vulkan client fails `vkCreateDevice -> VKFAIL -4` because the +kernel RM kicks its GR channel (h=0xbeee0100, ring VA 0x44a0000, push VA +0x4480000) and waits ~11 s per attempt for a completion the sema service +cannot apply: it cannot translate the push VA ("false translation for push +va=0x4480000 ... rescuing"), so no release is decoded and the wait times out. + +## Findings (r674/r675, 8+ client runs) +1. The push is RM-built (nvidia.ko), NOT UVM: a kprobe on + `uvm_hal_turing_host_semaphore_release` never fired during a client run. +2. nvidia.ko / nvidia-modeset.ko functions are `notrace` -> `kprobe_events` + refuses them ("Could not probe notrace function ... [nvidia]"). UVM module + probes DO work. QEMU gdbstub hardware breakpoints (hbreak) work fine. +3. The service's rescue matcher is CE-shaped only (hdr 0x20038090, hi==1, + lo&0xfff==slot, w5/v5 shape) -> it can never match a GR push. Entries 0/1/2 + are consumed as "skipping stale entry ... after ~150 failed rescues"; + gr_wedge=1. +4. Host-side content searches of /dev/shm/vgpu-guest-ram during the stall found + NO push data: neither the CE release group (0x20038090) nor the host-class + release header (0x20050017) appears outside driver code pages. +5. Ring entry encoding (from `channelFillGpFifo` in nvidia.ko and + `nvWriteGpEntry` in nvidia-modeset.ko): [0] = va & ~3, [1] = + ((va>>32)&0xff) | ((len<<8)&0x7ffffc00) | 0x80000000 (flag bit seen on + entry[1]). +6. Neither `channelFillGpFifo` (nvidia.ko) nor `nvWriteGpEntry` + (nvidia-modeset.ko) is called during the client run (gdb hbreaks, 0 hits + for the GR channel) -> the GR channel's ring entries are written by an + unidentified writer (possibly the emulation/GSP path); the push content + was never located in guest RAM. +7. RM window records + memdescs (host-readable, stable layout): + - a page holds the client channel's window records + {VA=0x4480000, SIZE=0x20000} (push) and {VA=0x44a0000, SIZE=0x40000} + (ring), each inside head/tail list nodes; + - memdescs `{SIZE, 0x03008000, FIRST_PA, ptr}` live in the same page: + the ring memdesc's FIRST_PA == the service's validated ring pa + (verified twice: 0x1158ce000, 0x129415000) -> memdesc PA == the PA of the + window's first VA page; + - the push memdesc `{0x20000, 0x03008000, PA}` was injected as + 0x4480000 -> PA (resolver4, r675: PA 0x11595c000; "injected mapping + va=0x4480000 ... (sysmem)" in dmesg) and the service then read the page + WITHOUT the "false translation" line - but decoded no release (the page + content does not look like the method stream; later candidates read as + zero). +8. The kernel-side stall shows no GSP RPCs and no ioctls; the wait function + `channelWaitForFinishPayload` (polls `channelReadChannelMemdesc(channel, + channel->0x50)`, which reads `*(channel+0xe8)+off`) was NEVER called during + a client run (gdb hbreak) -> the RM's wait path for this kick is elsewhere. + +## Tools (this directory) +- wall2_resolver2.py — v2: CE release-group content search + tuple arrays + (pa&~0xfff, strides 0x20/0x40/0x60). No candidates live. +- wall2_resolver3.py — v3: ring memdesc -> same-page push memdesc recipe. +- wall2_resolver4.py — v4: window-record pair + ring memdesc cross-check + (this one injected successfully; rdesc[0] bug fixed in v5). +- wall2_resolver5.py — v5: continuous scan (no ring-line gate), injects every + 0x20000 memdesc PA found in pages holding both window records. +- wall2_livecap{,2,3}.py — live stall-time scanners for the CE group and the + host release header (both negative; useful for re-checks). +- scan_r673*.py — RAM structure analysis (ring-PA refs, record pages, memdesc + scans). +- trace_push674.py — VA-reference tracer with direct-map pointer following. +- gdb675*.gdb — QEMU gdbstub scripts (channelFillGpFifo / nvWriteGpEntry + probes, channel struct dumps). + +## Next steps (ranked) +1. Find the GR ring-entry writer (gdb watchpoint on the ring page's CPU VA; + get the VA from the channel struct; find the channel struct via the ring + memdesc page's head/tail pointers). The writer's caller holds the push's + CPU mapping -> PA -> inject -> service decodes the release. +2. Or: gdb-break at the stall on the RM's wait (sample the sleeping task's + kernel stack; the current candidates never fired) and read the semaphore + value it polls; bump it host-side via the ring-page PA (the finish payload + may live at ring_cpu + channel->0x50). +3. Or: fix the service's rescue matcher to accept GR-shaped releases once the + push content is captured (see item 1). + +## Ops +- pkill -f self-matches: the bracket trick fails when the pattern text appears + elsewhere in the agent's own command line; kill by explicit PID from + `ps -eo pid,comm,args | awk '$2=="python3" && /<script>/ {print $1}'`. +- r674 wedged after a gdb-interrupted run; fresh cycle r675 used (no host + reboot). gdb674d/gdb675d sessions left attached (never kill gdbs). +- Never reload the guest nvidia modules; never kill clients mid-flight. diff --git a/tools/guest-diag/wall2/a8clear.py b/tools/guest-diag/wall2/a8clear.py new file mode 100644 index 0000000..261312a --- /dev/null +++ b/tools/guest-diag/wall2/a8clear.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# a8clear.py - no-ptrace A8/NvOgl count clearer: finds each live vk_frame's +# libnvidia-eglcore base (the off==0 mapping) and writes 0 to the u32 at +# base+0x2070b7c whenever it is nonzero (the count gate t91 cleared via gdb). +import os, subprocess, sys, time +DUR = float(sys.argv[1]) if len(sys.argv) > 1 else 600 +def pids(): + try: + out = subprocess.run(["pgrep", "-x", "vk_frame"], capture_output=True).stdout + return [int(x) for x in out.split()] + except Exception: + return [] +log = set() +end = time.time() + DUR +while time.time() < end: + for pid in pids(): + base = None + try: + for line in open("/proc/%d/maps" % pid): + p = line.split() + if len(p) < 6 or "libnvidia-eglcore" not in p[-1]: + continue + try: + off = int(p[2], 16) + except Exception: + continue + if off == 0: + base = int(p[0].split("-")[0], 16) + break + except Exception: + continue + if not base: + continue + w = base + 0x2070b7c + try: + fd = os.open("/proc/%d/mem" % pid, os.O_RDWR) + os.lseek(fd, w, 0) + v = os.read(fd, 4) + if len(v) == 4: + n = int.from_bytes(v, "little") + if n: + os.lseek(fd, w, 0) + os.write(fd, b"\x00\x00\x00\x00") + k = (pid, n) + if len(log) < 40: + log.add(k) + print("A8CLEAR pid=%d count=%u -> 0" % (pid, n), flush=True) + os.close(fd) + except Exception: + pass + time.sleep(0.5) +print("A8CLEAR-DONE", flush=True) diff --git a/tools/guest-diag/wall2/diag76.py b/tools/guest-diag/wall2/diag76.py new file mode 100644 index 0000000..989abd3 --- /dev/null +++ b/tools/guest-diag/wall2/diag76.py @@ -0,0 +1,47 @@ +import gdb + +inf = gdb.selected_inferior() +pid = inf.pid +maps = [] +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + maps.append((lo, hi, off, parts[5] if len(parts) > 5 else "")) + except Exception: + pass + +def resolve(addr): + for lo, hi, off, name in maps: + if lo <= addr < hi: + return "%s+0x%x" % (name.split('/')[-1], off + (addr - lo)) + return "?" + +gdb.execute("set pagination off") +print("=== THREADS ===") +for t in inf.threads(): + try: + t.switch() + except Exception as e: + print("switch err", e) + continue + pc = int(gdb.parse_and_eval("$pc")) + print("TID %s pc=%s" % (t.ptid, resolve(pc))) + try: + bt = gdb.execute("bt 12", to_string=True) + except Exception as e: + bt = "bt err %s" % e + for ln in bt.splitlines(): + ln = ln.strip() + if ln.startswith("#"): + try: + a = int(ln.split()[1], 16) + print(" ", ln, "=>", resolve(a)) + except Exception: + print(" ", ln) + else: + print(" ", ln) +print("DIAG-DONE") +gdb.execute("detach") +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/fastrun.sh b/tools/guest-diag/wall2/fastrun.sh new file mode 100644 index 0000000..73b9d80 --- /dev/null +++ b/tools/guest-diag/wall2/fastrun.sh @@ -0,0 +1,10 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /tmp/slotfill2.log 2>&1 & ) +sleep 0.5 +sudo timeout 60 strace -f -T -tt -e trace=ioctl -o /tmp/vk_strace3.out /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk77.out 2>&1 +echo "=== client ===" +cat /tmp/vk77.out +echo "=== last ioctls ===" +tail -12 /tmp/vk_strace3.out diff --git a/tools/guest-diag/wall2/fastrun2.sh b/tools/guest-diag/wall2/fastrun2.sh new file mode 100644 index 0000000..3c7b943 --- /dev/null +++ b/tools/guest-diag/wall2/fastrun2.sh @@ -0,0 +1,12 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /dev/null 2>&1 & ) +sleep 0.5 +sudo timeout 45 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk78.out 2>&1 +echo "=== client ===" +cat /tmp/vk78.out +echo "=== repeated controls (top) ===" +sudo dmesg -T | grep -oE "GSP_RM_CONTROL cmd=0x[0-9a-f]+" | sort | uniq -c | sort -rn | head -12 +echo "=== last controls ===" +sudo dmesg -T | grep "GSP_RM_CONTROL cmd" | tail -8 diff --git a/tools/guest-diag/wall2/fasttick.py b/tools/guest-diag/wall2/fasttick.py new file mode 100644 index 0000000..413d3ba --- /dev/null +++ b/tools/guest-diag/wall2/fasttick.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# fasttick.py - watch the service log for the GR0 channel's ring resolution +# (sema-chan: h=0xbeec0100 ... ring rc=0 pa=0x...) and immediately hand the +# verified {ring VA -> ring PA} pair to the service (sema_pdb_discover), so +# the parked VAS pdb candidate can be verified before/at age-out. +import re, subprocess, sys, time + +PARAM = "/sys/module/vgsp/parameters/sema_pdb_discover" +RING_VA = 0x4cc0000 + +def main(): + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 120 + end = time.time() + dur + fired = set() + seen_pdbs = set() + print("# fasttick start (%.0fs)" % dur, flush=True) + while time.time() < end: + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + tail = out.splitlines()[-120:] + for ln in tail: + m = re.search(r"sema-chan: h=0xbeec0100 eng=\d+ cid=\d+ gpfifo=0x([0-9a-f]+) ent=\d+ userd=0x[0-9a-f]+ as=\d+ put=\d+ get=\d+ ring rc=0 pa=0x([0-9a-f]+)", ln) + if m: + ring_va = int(m.group(1), 16) + ring_pa = int(m.group(2), 16) + key = (ring_va, ring_pa) + if key not in fired: + fired.add(key) + cmd = "echo 0x%x,0x%x > %s" % (ring_va, ring_pa, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print("[%s] TRIGGER va=0x%x pa=0x%x rc=%d" % ( + time.strftime("%H:%M:%S"), ring_va, ring_pa, r.returncode), flush=True) + m2 = re.search(r"page directory (?:registered UNVERIFIED after \d+ ms parked \()?pdb=0x([0-9a-f]+) entries=(\d+)", ln) + if not m2: + m2 = re.search(r"VA space 0x[0-9a-f]+ page directory 0x([0-9a-f]+) entries=(\d+)", ln) + if m2: + key = int(m2.group(1), 16) + if key not in seen_pdbs: + seen_pdbs.add(key) + print("[%s] PDB candidate 0x%x entries=%s" % ( + time.strftime("%H:%M:%S"), key, m2.group(2)), flush=True) + time.sleep(0.3) + print("# fasttick end", flush=True) + +main() diff --git a/tools/guest-diag/wall2/fence_assist.py b/tools/guest-diag/wall2/fence_assist.py new file mode 100644 index 0000000..4ea0b0c --- /dev/null +++ b/tools/guest-diag/wall2/fence_assist.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# fence_assist.py - content-locate the client's ring pages in guest RAM from the +# service's own GPFIFO entry decodes (dmesg) and inject the TRUE VA->PA pairs +# via sema_map_add, so the pump's push reads (and its pushbuffer pairing) have +# a correct anchor. +import mmap, re, struct, subprocess, sys, threading, time + +F = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" + +def gpa_of(fo): + return fo if fo < 0x80000000 else fo - 0x80000000 + 0x100000000 + +def inject(va, pa, ln=0x1000, flag=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va & ~0xfff, pa & ~0xfff, ln, flag, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print("INJECT va=0x%x pa=0x%x : %s" % (va & ~0xfff, pa & ~0xfff, + "ok" if r.returncode == 0 else "FAIL"), flush=True) + +def search(mm, pats, results): + CH = 8 << 20 + size = len(mm) + off = 0 + while off < size: + d = mm[off:off+CH] + for (name, pat) in pats.items(): + if name in results: + continue + i = d.find(pat) + if i >= 0: + results[name] = gpa_of(off + i) + off += CH - 32 + +def main(): + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 120 + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + end = time.time() + dur + pats = {} # name -> pattern bytes + ring_va = {} # name -> ring VA + seen = set() # raw entry lines processed + done = set() # injected names + print("# fence_assist start (%.0fs)" % dur, flush=True) + while time.time() < end: + try: + with open("/tmp/opencode/dmsg_w.txt", "r", errors="ignore") as df: + tail = df.read().splitlines()[-6000:] + out = "\n".join(tail) + except OSError: + out = "" + tail = [] + tail = [ln for ln in tail if "sema-chan:" in ln or "sema-push" in ln] + cur = {} + for ln in tail: + m = re.search(r"sema-chan: entry\[(\d+)\] h=0x([0-9a-f]+) eng=\d+ raw=([0-9a-f]+) va=0x([0-9a-f]+)", ln) + if m: + idx, h, raw, va = int(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4), 16) + cur.setdefault(h, {})[idx] = raw + if idx == 0: + ring_va[h] = va # entry[0] va is the push VA (ring_va - 0x20000) + for h, ents in cur.items(): + key = (h, tuple(sorted(ents.items()))) + if key in seen or len(ents) < 2: + continue + seen.add(key) + name = h + pat = b"".join(struct.pack("<Q", ents[i]) for i in sorted(ents)[:2]) + pats[name] = pat + print("# channel h=0x%s entries=%s pattern ready" % (h, sorted(ents)), flush=True) + if pats: + results = {} + threads = [] + # split the map among 4 threads + LO = 0x100000000; HI = 0x180000000 + size = len(mm) + step = size // 4 + class Sl(shim := object): + def __init__(self, a, b): + self.a, self.b = a, b + def worker(a, b): + CH = 8 << 20 + off = a + while off < b: + if off < 0x80000000 and b <= 0x80000000: pass + d = mm[off:off+CH] + for (nm, p) in list(pats.items()): + if nm in results: continue + i = d.find(p) + if i >= 0: + results[nm] = gpa_of(off + i) + off += CH - 32 + for k in range(4): + a = 0x80000000 + k * (0x80000000 // 4) + b = 0x80000000 + (k + 1) * (0x80000000 // 4) + t = threading.Thread(target=worker, args=(a, b)) + t.start(); threads.append(t) + for t in threads: t.join() + for nm, gpa in results.items(): + if nm in done: continue + done.add(nm) + # ring page content starts with entry[0]; the ring VA is the + # channel's gpfifo VA = we reconstruct: entry0 va is the push, + # ring = first "gpfifo=0x..." dmesg line for this h + m = re.search(r"sema-chan: h=0x%s eng=\d+ cid=\d+ gpfifo=0x([0-9a-f]+)" % nm, out) + rva = int(m.group(1), 16) if m else None + if rva: + print("RING FOUND h=0x%s gpfifo_va=0x%x -> gpa=0x%x" % (nm, rva, gpa), flush=True) + inject(rva, gpa, 0x1000) + time.sleep(0.6) + print("# fence_assist done, injected %d" % len(done), flush=True) + +main() diff --git a/tools/guest-diag/wall2/find_push.py b/tools/guest-diag/wall2/find_push.py new file mode 100644 index 0000000..4bf8ccc --- /dev/null +++ b/tools/guest-diag/wall2/find_push.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# find_push.py - locate the client's CPU mapping that owns the GR ring page +# (GPA 0x12C600000 as content-located on the host) and print the CPU VA<-> +# GPU VA correspondence for the low GPU VAs (0x4ca0000 etc). +import os, struct, sys, subprocess, mmap, time + +RING_GPA = 0x12C600000 +PUSH_GVA = 0x4ca0000 +RING_GVA = 0x4cc0000 + +def pids(): + out = subprocess.run(["pgrep", "-x", "vk_frame"], capture_output=True).stdout + return [int(x) for x in out.split()] + +def read_pagemap(pid, va): + try: + with open("/proc/%d/pagemap" % pid, "rb") as f: + f.seek((va // 4096) * 8) + d = f.read(8) + if len(d) < 8: + return None + e = int.from_bytes(d, "little") + if not (e >> 63) & 1: + return None + return (e & ((1 << 55) - 1)) << 12 + except OSError: + return None + +def main(): + for pid in pids(): + print("pid=%d" % pid) + try: + maps = open("/proc/%d/maps" % pid).read().splitlines() + except OSError: + continue + for line in maps: + p = line.split() + if len(p) < 6 or "nvidia" not in p[-1]: + continue + lo, hi = [int(x, 16) for x in p[0].split("-")] + if hi - lo > 0x1000000: + continue + # scan pages; look for RING_GPA + for va in range(lo, hi, 0x10000): + for off in range(0, 0x10000, 0x1000): + pa = read_pagemap(pid, va + off) + if pa == RING_GPA: + print("FOUND ring page: cpu_va=0x%x (map 0x%x-0x%x %s)" % (va+off, lo, hi, p[-1])) + delta_cpu = (va + off) - RING_GVA + push_cpu = PUSH_GVA + delta_cpu + print(" delta_cpu = 0x%x" % delta_cpu) + for gva in (PUSH_GVA, PUSH_GVA+0x1000, 0x4ca059c, 0x4ca0980, 0x4ca5870, 0x4903000): + cva = gva + delta_cpu + pa2 = read_pagemap(pid, cva & ~0xfff) + print(" gva=0x%x -> cpu=0x%x -> pa=%s" % (gva, cva, hex(pa2) if pa2 else None)) + return + print(" ring GPA 0x%x not found in this pid's mappings" % RING_GPA) + +main() diff --git a/tools/guest-diag/wall2/find_ring_va.py b/tools/guest-diag/wall2/find_ring_va.py new file mode 100644 index 0000000..c02ea40 --- /dev/null +++ b/tools/guest-diag/wall2/find_ring_va.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# Guest-side: read the nv_vm_map_pages vmap areas via /proc/kcore and find the +# GPFIFO entry pattern (entry0 raw 0x0005980004480000 for va=0x4480000 len=1432). +import struct, sys +pat = bytes.fromhex("0000480400980500") # 0x0005980004480000 LE +class Kcore: + def __init__(self): + self.fd = open("/proc/kcore", "rb") + hdr = self.fd.read(64) + e_phoff = struct.unpack_from("<Q", hdr, 32)[0] + e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54) + self.segs = [] + for i in range(e_phnum): + self.fd.seek(e_phoff + i * e_phentsize) + ph = self.fd.read(56) + p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from("<IIQQQQ", ph, 0) + if p_type == 1: + self.segs.append((p_va, p_va + p_sz, p_off)) + def read(self, va, n): + for lo, hi, off in self.segs: + if lo <= va and va + n <= hi: + self.fd.seek(off + (va - lo)) + return self.fd.read(n) + return None +kc = Kcore() +areas = [] +for line in open("/proc/vmallocinfo"): + p = line.split() + if len(p) < 2: continue + if "nv_vm_map_pages" not in line: continue + lo, hi = p[0].split("-") + areas.append((int(lo, 16), int(hi, 16))) +print("nv_vm_map_pages areas:", len(areas)) +for lo, hi in areas: + size = hi - lo + d = kc.read(lo, size) + if not d: + print(" 0x%x-0x%x unreadable" % (lo, hi)); continue + idx = d.find(pat) + print(" 0x%x-0x%x size=0x%x hit=%s" % (lo, hi, size, ("+0x%x -> VA 0x%x" % (idx, lo + idx)) if idx >= 0 else "no")) + if idx >= 0: + # dump the page around the hit + base = lo + (idx & ~0xfff) + page = kc.read(base, 0x100) + print(" page head: %s" % page.hex()) diff --git a/tools/guest-diag/wall2/gdb675.gdb b/tools/guest-diag/wall2/gdb675.gdb new file mode 100644 index 0000000..e4b73ce --- /dev/null +++ b/tools/guest-diag/wall2/gdb675.gdb @@ -0,0 +1,21 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc0d0b9f0 +condition 1 *(unsigned int*)($rdi+0x110) == 0xc46f +commands 1 +silent +printf "GRFILL ch=%p idx=%d len=%d\n", $rdi, $esi, $edx +printf " +0x048=%p +0x0b8=%p +0x0e8=%p\n", *(unsigned long*)($rdi+0x48), *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0xe8) +printf " +0x060=%p +0x0c0=%p +0x0c8=%p +0x0d0=%p +0x0d8=%p\n", *(unsigned long*)($rdi+0x60), *(unsigned long*)($rdi+0xc0), *(unsigned long*)($rdi+0xc8), *(unsigned long*)($rdi+0xd0), *(unsigned long*)($rdi+0xd8) +printf " +0x0e0=%p +0x0f0=%p +0x0f8=%p +0x108=%p +0x140=%p +0x148=%p\n", *(unsigned long*)($rdi+0xe0), *(unsigned long*)($rdi+0xf0), *(unsigned long*)($rdi+0xf8), *(unsigned long*)($rdi+0x108), *(unsigned long*)($rdi+0x140), *(unsigned long*)($rdi+0x148) +x/8gx $rdi+0x40 +x/8gx $rdi+0x80 +x/8gx $rdi+0xc0 +x/8gx $rdi+0xe0 +x/8gx $rdi+0x100 +x/8gx $rdi+0x140 +continue +end +continue diff --git a/tools/guest-diag/wall2/gdb675c.gdb b/tools/guest-diag/wall2/gdb675c.gdb new file mode 100644 index 0000000..25b5ca0 --- /dev/null +++ b/tools/guest-diag/wall2/gdb675c.gdb @@ -0,0 +1,12 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc0d0b9f0 +commands 1 +silent +printf "FILL ch=%p cls100=0x%x b8=0x%x stride48=0x%x idx=%d len=%d\n", $rdi, *(unsigned int*)($rdi+0x100), *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0x48), $esi, $edx +printf " d0=%p d8=%p e8=%p f8=%p 60=%p\n", *(unsigned long*)($rdi+0xd0), *(unsigned long*)($rdi+0xd8), *(unsigned long*)($rdi+0xe8), *(unsigned long*)($rdi+0xf8), *(unsigned long*)($rdi+0x60) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdb675d.gdb b/tools/guest-diag/wall2/gdb675d.gdb new file mode 100644 index 0000000..f9dc460 --- /dev/null +++ b/tools/guest-diag/wall2/gdb675d.gdb @@ -0,0 +1,11 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc1605fd0 +commands 1 +silent +printf "NVWRITE ch=%p idx=%u len=%u b8=0x%x f68=%p f50=0x%x fb0=0x%x fc0=0x%x f198=%p\n", $rdi, $esi, $edx, *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0x68), *(unsigned int*)($rdi+0x50), *(unsigned int*)($rdi+0xb0), *(unsigned int*)($rdi+0xc0), *(unsigned long*)($rdi+0x198) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbfill.gdb b/tools/guest-diag/wall2/gdbfill.gdb new file mode 100644 index 0000000..41d1ea3 --- /dev/null +++ b/tools/guest-diag/wall2/gdbfill.gdb @@ -0,0 +1,13 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +set $e = *(unsigned long*)($rdi+0x1428) +if *(unsigned short*)($e+0x2e) == 0xffff + set {unsigned short}($e+0x2e) = 0 + printf "FILLED slot at %p+0x2e -> 0\n", $e +end +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbgap3.sh b/tools/guest-diag/wall2/gdbgap3.sh new file mode 100644 index 0000000..bf19370 --- /dev/null +++ b/tools/guest-diag/wall2/gdbgap3.sh @@ -0,0 +1,28 @@ +#!/bin/bash +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk65.out 2>&1 &) +sleep 5 +PID=$(pgrep -x vk_frame | head -1) +cp /proc/$PID/maps /tmp/maps_d.txt +gdb -p $PID -batch -ex "set pagination off" -ex "bt 12" 2>&1 | grep -E "^#" > /tmp/bt_gap3.txt +python3 - <<'PY' +import re +maps=open('/tmp/maps_d.txt').read().splitlines() +segs=[] +for m in maps: + p=m.split() + if len(p)<6: continue + lo,hi=[int(x,16) for x in p[0].split('-')] + off=int(p[2],16) + segs.append((lo,hi,off,p[5])) +for line in open('/tmp/bt_gap3.txt'): + m=re.match(r'#(\d+)\s+0x([0-9a-f]+)', line) + if not m: continue + a=int(m.group(2),16) + for lo,hi,off,name in segs: + if lo<=a<hi: + print('#%s %s + 0x%x' % (m.group(1), name.split('/')[-1], off+(a-lo))) + break + else: + print('#%s 0x%x unmapped' % (m.group(1), a)) +PY diff --git a/tools/guest-diag/wall2/gdbparam.sh b/tools/guest-diag/wall2/gdbparam.sh new file mode 100644 index 0000000..3f30d49 --- /dev/null +++ b/tools/guest-diag/wall2/gdbparam.sh @@ -0,0 +1,16 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /dev/null 2>&1 & ) +sleep 0.5 +(timeout 60 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk79.out 2>&1 &) +sleep 5 +PID=$(pgrep -x vk_frame | head -1) +gdb -p $PID -batch \ + -ex "set pagination off" \ + -ex "catch syscall ioctl" \ + -ex "condition 1 \$rsi == 0xc020462a" \ + -ex "continue" \ + -ex "printf \"ioctl nr=0x2a arg=%p\n\", \$rdx" \ + -ex "x/40wx \$rdx" \ + -ex "detach" 2>/dev/null | tail -45 diff --git a/tools/guest-diag/wall2/gdbpoll6.gdb b/tools/guest-diag/wall2/gdbpoll6.gdb new file mode 100644 index 0000000..d7fccfd --- /dev/null +++ b/tools/guest-diag/wall2/gdbpoll6.gdb @@ -0,0 +1,9 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +printf "CHK rsi=%d cnt=%d e0=%p v=%x\n", $rsi, *(int*)($rdi+0x22efc), *(unsigned long*)($rdi+0x1428), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)+($rsi<<4)) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbspin.sh b/tools/guest-diag/wall2/gdbspin.sh new file mode 100644 index 0000000..989e584 --- /dev/null +++ b/tools/guest-diag/wall2/gdbspin.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk57.out 2>&1 &) +sleep 25 +PID=$(pgrep -x vk_frame | head -1) +echo "pid=$PID" +BASE=$(grep 'eglcore' /proc/$PID/maps | head -1 | cut -d- -f1) +echo "base=$BASE" +SPIN=$(printf '0x%x' $((0x$BASE + 0xa10e90))) +echo "spin=$SPIN" +gdb -p $PID -batch \ + -ex "set pagination off" \ + -ex "break *$SPIN" \ + -ex "continue" \ + -ex "printf \"HIT rip=%p rdi=%p rsi=%p rdx=%p rbx=%p rbp=%p\n\", \$rip, \$rdi, \$rsi, \$rdx, \$rbx, \$rbp" \ + -ex "bt" \ + -ex "info registers r12 r13 r14 r15" \ + -ex "x/12i \$rip-24" \ + -ex "detach" 2>&1 | tail -40 diff --git a/tools/guest-diag/wall2/gdbu16.gdb b/tools/guest-diag/wall2/gdbu16.gdb new file mode 100644 index 0000000..61f335b --- /dev/null +++ b/tools/guest-diag/wall2/gdbu16.gdb @@ -0,0 +1,9 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +printf "CHK rsi=%d cnt=%d e0=%p u16@2e=%x u32@20=%x u32@0=%x\n", $rsi, *(int*)($rdi+0x22efc), *(unsigned long*)($rdi+0x1428), *(unsigned short*)(*(unsigned long*)($rdi+0x1428)+0x2e), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)+0x20), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbwatch.gdb b/tools/guest-diag/wall2/gdbwatch.gdb new file mode 100644 index 0000000..36b3339 --- /dev/null +++ b/tools/guest-diag/wall2/gdbwatch.gdb @@ -0,0 +1,18 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +set $e = *(unsigned long*)($rdi+0x1428) +watch *(unsigned short*)($e+0x2e) +printf "WATCH set at %p+0x2e\n", $e +disable 1 +continue +end +commands 2 +silent +printf "WRITE to slot+0x2e: old=%x new=%x by pc=%p\n", *(unsigned short*)($e+0x2e), $eax, $pc +bt 3 +continue +end +continue diff --git a/tools/guest-diag/wall2/gr_walk.py b/tools/guest-diag/wall2/gr_walk.py new file mode 100644 index 0000000..cbdb02d --- /dev/null +++ b/tools/guest-diag/wall2/gr_walk.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# gr_walk.py - find the RM window memdesc records in the guest's own RAM +# (via /proc/kcore direct map) and walk their kernel page-list pointers in +# the SAME pass (addresses go stale otherwise). Prints VA->PA candidates. +import sys, struct +sys.path.insert(0, "/tmp") +import guest_pin_stream as gps + +PAT_SIZE = struct.pack("<Q", 0x20000) +PAT_FLAGS = struct.pack("<Q", 0x03008000) + +def main(): + kc = gps.Kcore() + if not kc.load(): + print("# kcore load failed"); return 1 + pov = kc.page_offset_base + print("# page_offset_base=0x%x" % pov) + found = [] + CH = 1 << 20 + pa = 0x100000000 + end = 0x180000000 + while pa < end and len(found) < 12: + va = pov + pa + d = kc.read(va, CH) + if d: + st = 0 + while True: + i = d.find(PAT_SIZE, st) + if i < 0: + break + st = i + 1 + if i + 16 <= len(d) and d[i+8:i+16] == PAT_FLAGS: + found.append((pa + i, d[i:i+0x60])) + pa += CH + print("# memdesc records: %d" % len(found)) + for (gpa, rec) in found: + size, flags = struct.unpack_from("<QQ", rec, 0) + wpa = struct.unpack_from("<Q", rec, 0x10)[0] + kptrs = [struct.unpack_from("<Q", rec, o)[0] for o in (0x18, 0x20, 0x28, 0x30, 0x38)] + print("rec@0x%x size=0x%x pa=0x%x kptrs=%s" % + (gpa, size, wpa, ["0x%x" % k for k in kptrs])) + # walk each kptr for PA-like lists + for kp in kptrs: + if not gps.is_kern_ptr(kp): + continue + dd = kc.read(kp, 0x80) + if not dd: + print(" kptr 0x%x: unreadable" % kp); continue + words = struct.unpack_from("<%dQ" % (len(dd)//8), dd, 0) + pas = [w for w in words if 0x100000000 <= w < 0x180000000] + npas = [w for w in words if 0x100000000 <= (w & ~0xfff) < 0x180000000] + print(" kptr 0x%x: %s" % (kp, " ".join("0x%x" % w for w in words[:10]))) + if len(npas) >= 2: + print(" PA-like: %s" % ["0x%x" % w for w in npas[:16]]) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/gr_walk2.py b/tools/guest-diag/wall2/gr_walk2.py new file mode 100644 index 0000000..7a50bc8 --- /dev/null +++ b/tools/guest-diag/wall2/gr_walk2.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# gr_walk2.py - deeper walk: from the window memdesc records, follow the +# kernel pointers 2-3 levels and search for page-array records +# {vmap_cpu, flags, PA, 0x1000} (2+ consecutive) to recover the page lists. +import sys, struct +sys.path.insert(0, "/tmp") +import guest_pin_stream as gps + +PAT_SIZE = struct.pack("<Q", 0x20000) +PAT_FLAGS = struct.pack("<Q", 0x03008000) + +def is_pa(v): + return 0x100000000 <= v < 0x180000000 + +def page_array_at(kc, va, n=0x400): + """Return [(off, [PA,...])] for runs of {ptr,flags,PA,0x1000} records.""" + d = kc.read(va, n) + if not d: + return [] + out = [] + i = 0 + while i + 32 <= len(d): + p, fl, pa, z = struct.unpack_from("<4Q", d, i) + if z == 0x1000 and is_pa(pa) and gps.is_kern_ptr(p): + run = [pa] + j = i + 32 + while j + 32 <= len(d): + p2, fl2, pa2, z2 = struct.unpack_from("<4Q", d, j) + if z2 == 0x1000 and is_pa(pa2): + run.append(pa2) + j += 32 + else: + break + if len(run) >= 2: + out.append((i, run)) + i = j + else: + i += 8 + return out + +def main(): + kc = gps.Kcore() + if not kc.load(): + print("# kcore load failed"); return 1 + pov = kc.page_offset_base + # find memdesc records (fast direct-map scan) + found = [] + CH = 1 << 20 + pa = 0x100000000 + while pa < 0x180000000 and len(found) < 12: + d = kc.read(pov + pa, CH) + if d: + st = 0 + while True: + i = d.find(PAT_SIZE, st) + if i < 0: + break + st = i + 1 + if i + 16 <= len(d) and d[i+8:i+16] == PAT_FLAGS: + found.append((pa + i, d[i:i+0x40])) + pa += CH + print("# memdesc records: %d" % len(found)) + for (gpa, rec) in found: + size, flags = struct.unpack_from("<QQ", rec, 0) + wpa = struct.unpack_from("<Q", rec, 0x10)[0] + print("rec@0x%x size=0x%x pa=0x%x" % (gpa, size, wpa)) + kptrs = [struct.unpack_from("<Q", rec, o)[0] for o in range(0x18, 0x40, 8)] + for kp in kptrs: + if not gps.is_kern_ptr(kp): + continue + # level 1: read the target, look for page arrays directly + for arr in page_array_at(kc, kp): + print(" L1 kptr 0x%x: page-array run at +0x%x: %s" % + (kp, arr[0], ["0x%x" % x for x in arr[1][:12]])) + # level 2: follow pointers in the target + d = kc.read(kp, 0x100) + if not d: + continue + for o in range(0, 0x100, 8): + p2 = struct.unpack_from("<Q", d, o)[0] + if not gps.is_kern_ptr(p2): + continue + for arr in page_array_at(kc, p2): + print(" L2 0x%x -> 0x%x: page-array at +0x%x: %s" % + (kp, p2, arr[0], ["0x%x" % x for x in arr[1][:12]])) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/guest_words.sh b/tools/guest-diag/wall2/guest_words.sh new file mode 100755 index 0000000..6ac200c --- /dev/null +++ b/tools/guest-diag/wall2/guest_words.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# guest_words.sh - print "VA PA" for the +0x80 word of every 16 KB rw-s +# /dev/nvidiactl mapping of the vk_frame client (pagemap PFN -> PA). +PID=$(pgrep -x vk_frame | head -1) +[ -z "$PID" ] && exit 0 +sudo grep -E 'rw-s' /proc/$PID/maps | grep nvidiactl | while read -r line; do + RNG=$(echo "$line" | awk '{print $1}') + LO=${RNG%-*}; HI=${RNG#*-} + SZ=$((0x$HI - 0x$LO)) + [ "$SZ" -ne 16384 ] && continue + VP=$(( (0x$LO + 0x80) / 4096 )) + ENT=$(sudo dd if=/proc/$PID/pagemap bs=8 skip=$VP count=1 2>/dev/null | xxd -e -g8 | awk '{print $2}') + [ -z "$ENT" ] && continue + PFN=$(( 0x$ENT & 0x7fffffffffffff )) + PA=$(( PFN * 4096 + 0x80 )) + echo "0x$LO 0x$(printf %x $PA)" +done diff --git a/tools/guest-diag/wall2/host_resolve.py b/tools/guest-diag/wall2/host_resolve.py new file mode 100644 index 0000000..258d994 --- /dev/null +++ b/tools/guest-diag/wall2/host_resolve.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# host_resolve.py - host-side push resolver: read the service's sema-pushall +# lines (pa=0x0, non-zero first8), find the content page in +# /dev/shm/vgpu-guest-ram by the push's first8 bytes, and inject the page +# mapping via sema_map_add. Replaces the guest-side kcore walker: the +# service already tells us push VA + content prefix, the host RAM file has +# the bytes. +import mmap, os, re, subprocess, sys, time + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +BASE = 0x80000000 # file offset = pa - BASE + +def inject(va_page, pa): + cmd = "echo 0x%x,0x%x,0x1000,1 > %s" % (va_page, pa, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + ok = r.returncode == 0 + print("inject 0x%x -> 0x%x : %s" % (va_page, pa, "ok" if ok else r.stderr.decode().strip())) + return ok + +def main(): + seen = set() + f = open(RAM, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + injected = 0 + deadline = time.time() + (float(sys.argv[1]) if len(sys.argv) > 1 else 600) + while time.time() < deadline and injected < 40: + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + lines = [l for l in out.splitlines() if "sema-pushall" in l] + for l in lines[-800:]: + m = re.search(r"va=(0x[0-9a-f]+) pa=0x0 len=\d+ allow=\d trusted=\d first8=([0-9a-f]{8}) ([0-9a-f]{8})", l) + if not m: + continue + va = int(m.group(1), 16) + w0, w1 = int(m.group(2), 16), int(m.group(3), 16) + if (w0, w1) == (0, 0) and (w0 == 0 or w1 == 0): + pass + if w0 == 0 and w1 == 0: + continue + key = (va & ~0xFFF, w0, w1) + if key in seen: + continue + pat = w0.to_bytes(4, "little") + w1.to_bytes(4, "little") + inpage = va & 0xFFF + # search with the in-page offset constraint + start = 0 + hit = None + while True: + i = mm.find(pat, start) + if i < 0: + break + if (i & 0xFFF) == inpage: + hit = i + break + start = i + 1 + if hit is None: + # fallback: any hit within the same page offset modulo + start = 0 + while True: + i = mm.find(pat, start) + if i < 0: + break + hit = i + break + # proceed only if the constraint is loose? no - skip + hit = None + if hit is None: + seen.add(key) + print("no match for va=0x%x first8=%08x %08x" % (va, w0, w1)) + continue + pa = (hit & ~0xFFF) + BASE + seen.add(key) + if inject(va & ~0xFFF, pa): + injected += 1 + time.sleep(0.5) + mm.close() + f.close() + print("done: injected=%d" % injected) + +if __name__ == "__main__": + main() diff --git a/tools/guest-diag/wall2/kc_dump_fn.py b/tools/guest-diag/wall2/kc_dump_fn.py new file mode 100644 index 0000000..e481d24 --- /dev/null +++ b/tools/guest-diag/wall2/kc_dump_fn.py @@ -0,0 +1,21 @@ +import struct, sys +VA = int(sys.argv[1], 16) +N = int(sys.argv[2], 16) if len(sys.argv) > 2 else 0x800 +f = open("/proc/kcore","rb") +hdr = f.read(64) +e_phoff = struct.unpack_from("<Q", hdr, 0x20)[0] +e_phentsize = struct.unpack_from("<H", hdr, 0x36)[0] +e_phnum = struct.unpack_from("<H", hdr, 0x38)[0] +segs = [] +for i in range(e_phnum): + f.seek(e_phoff + i*e_phentsize) + ph = f.read(e_phentsize) + p_type, p_flags, p_offset, p_vaddr, p_paddr, p_filesz, p_memsz = struct.unpack_from("<IIQQQQQ", ph, 0) + if p_type == 1: + segs.append((p_vaddr, p_memsz, p_offset)) +for v, sz, off in segs: + if v <= VA < v+sz: + f.seek(off + (VA - v)) + d = f.read(min(N, v+sz-VA)) + sys.stdout.write(d.hex()) + break diff --git a/tools/guest-diag/wall2/live_walk.py b/tools/guest-diag/wall2/live_walk.py new file mode 100644 index 0000000..48f131f --- /dev/null +++ b/tools/guest-diag/wall2/live_walk.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# live_walk.py - live PT-walk + inject loop for the running client. +# Polls dmesg for the published page directory (pdb) and for the VAs the +# service is failing to read (sema-entry: unread / entry[] lines / registered +# gpfifo), walks each VA through the pdb with the service's own GMMU math, +# and injects resolved pairs via sema_map_add (ap=1 sysmem). +import mmap, re, subprocess, sys, time + +F = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +SEMA_PA_MAX = 0x180000000 + +WINDOW_VAS = [0x44a0000, 0x4500000, 0x4560000, # rings + 0x4480000, 0x4485000, 0x4ca0000, 0x4ca5000, + 0x4d00000, 0x4b17000, 0x4020000, 0x4100000, 0x4160000, + 0x4213000, 0x45a0bb0, # ctx-init releases (deterministic) + 0x124935060, 0x102e34060, 0x108d4f060, 0x12a3a4060, 0x116c0b060, # notif families + 0x20400f000, 0x20400ffc0, 0x20480fef0, 0x204827f7c] # counter pages / tracker VAs + +mm = None +def off(pa): return pa if pa < 0x80000000 else pa - 0x80000000 +def rd64(pa): + if pa + 8 > SEMA_PA_MAX: return None + d = mm[off(pa):off(pa)+8] + return int.from_bytes(d, "little") if len(d) == 8 else None +def pde_ap(ap): return "FB" if ap == 1 else ("SYS" if ap in (2, 3) else None) +def pte_ap(ap): return "FB" if ap == 0 else (None if ap == 1 else "SYS") +def pte_addr(e, fb): return ((e >> 8) & 0x1FFFFFF) << 12 if fb else ((e >> 8) & 0x3FFFFFFFFFFF) << 12 +def pte_leaf(table, idx): + e = rd64(table + 8 * idx) + if e is None or not (e & 1): return None + ap = pte_ap((e >> 1) & 3) + if ap is None: return None + a = pte_addr(e, ap == "FB") + return (a, ap) if a + 8 <= SEMA_PA_MAX else None + +def walk(pdb, va, entries=4): + import math + root_bits = max(1, min(12, int(math.log2(entries or 1)))) + table = pdb + for level in range(3): + if level == 0: + bits, shift = root_bits, 49 - root_bits + else: + bits, shift = 9, 38 if level == 1 else 29 + idx = (va >> shift) & ((1 << bits) - 1) + e = rd64(table + 8 * idx) + if e is None or (e & 1): return None + ap = pde_ap((e >> 1) & 3) + if ap is None: return None + table = pte_addr(e, ap == "FB") + idx = (va >> 21) & 0xff + big = rd64(table + 16 * idx); small = rd64(table + 16 * idx + 8) + if big is None or small is None: return None + if not (small & 1): + sap = pde_ap((small >> 1) & 3) + if sap is not None: + r = pte_leaf(pte_addr(small, sap == "FB"), (va >> 12) & 0x1ff) + if r: return r[0] + (va & 0xfff), r[1] + if big & 1: + bap = pte_ap((big >> 1) & 3) + if bap is not None: + a = pte_addr(big, bap == "FB") + (va & ((1 << 21) - 1)) + if a + 8 <= SEMA_PA_MAX: return a, bap + else: + bap = pde_ap((big >> 1) & 3) + if bap is not None: + r = pte_leaf(pte_addr(big, bap == "FB"), (va >> 16) & 0x1f) + if r: return r[0] + (va & 0xffff), r[1] + return None + +def inject(va, pa, ln=0x1000): + cmd = "echo 0x%x,0x%x,0x%x,1 > %s" % (va & ~0xfff, pa & ~0xfff, ln, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print(" INJECT 0x%x -> 0x%x len=0x%x : %s" % + (va & ~0xfff, pa & ~0xfff, ln, "ok" if r.returncode == 0 else "FAIL"), flush=True) + return r.returncode == 0 + +def main(): + global mm + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 300 + end = time.time() + dur + pdbs = [] # (pdb_pa, entries) + done = set() # (pdb, va_page) + seen_va = set() + last_lines = 0 + print("# live_walk start (%.0fs)" % dur, flush=True) + while time.time() < end: + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + lines = out.splitlines()[-500:] + out = "\n".join(lines) + # pdb offers/registrations + for m in re.finditer(r"page directory (?:offered as candidate )?pdb=0x([0-9a-f]+) entries=(\d+)", out): + pdb, ent = int(m.group(1), 16), int(m.group(2)) + if (pdb, ent) not in pdbs: + pdbs.append((pdb, ent)) + print("# pdb 0x%x entries=%u" % (pdb, ent), flush=True) + # VAs the service is failing on + vas = [] + for m in re.finditer(r"sema-entry: unread h=0x[0-9a-f]+ idx=\d+ va=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"sema-chan: entry\[\d+\] h=0x[0-9a-f]+ eng=\d+ raw=[0-9a-f]+ va=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"registered gpfifo=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"GR ring h=0x[0-9a-f]+ rank=\d+ not at fb 0x[0-9a-f]+ yet \(e0=0x[0-9a-f]+ e1=0x[0-9a-f]+ va=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"rel=0x([0-9a-f]+) ", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"notif=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"counter page 0x([0-9a-f]+) unreadable", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"release (?:learned|derived) h=0x[0-9a-f]+ pa=0x[0-9a-f]+ \(from release va=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + vas = [v for v in vas if 0x4000000 <= v < 0x2000000000] + for va in WINDOW_VAS + vas: + seen_va.add(va) + # resolve + inject through every pdb + for (pdb, ent) in pdbs: + for va in list(seen_va): + key = (pdb, va & ~0xfff) + if key in done: + continue + r = walk(pdb, va, ent) + if not r: + continue + done.add(key) + pa, ap = r + if ap != "SYS": + print("# va=0x%x -> pa=0x%x ap=%s (skip non-sys)" % (va, pa, ap), flush=True) + continue + print("# walk ok pdb=0x%x va=0x%x -> pa=0x%x" % (pdb, va, pa), flush=True) + inject(va, pa) + time.sleep(0.4) + mm.close(); f.close() + print("# live_walk done", flush=True) + +if __name__ == "__main__": + main() diff --git a/tools/guest-diag/wall2/nudge.sh b/tools/guest-diag/wall2/nudge.sh new file mode 100755 index 0000000..143b1cd --- /dev/null +++ b/tools/guest-diag/wall2/nudge.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# nudge.sh - host-side readiness-word nudger for the vk_frame wait. +# Polls the guest for the 16 KB nvidiactl mapping words (guest_words.sh), +# and writes 1 to any that are still 0 in the guest RAM file. +# Usage: nudge.sh <duration_seconds> +DUR=${1:-300} +F=/dev/shm/vgpu-guest-ram +END=$(( $(date +%s) + DUR )) +sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/guest_words.sh ubuntu@127.0.0.1:/tmp/guest_words.sh >/dev/null 2>&1 +while [ "$(date +%s)" -lt "$END" ]; do + OUT=$(timeout 20 sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 "sudo bash /tmp/guest_words.sh" 2>/dev/null) + echo "[$(date +%H:%M:%S)] words:"; echo "$OUT" + echo "$OUT" | while read -r VA PA; do + [ -z "$PA" ] && continue + PAD=$((PA)); FOD=$((PAD - 0x80000000)) + [ $FOD -lt 0 ] && continue + CUR=$(sudo dd if=$F bs=4 skip=$((FOD/4)) count=1 2>/dev/null | xxd -p) + if [ "$CUR" = "00000000" ]; then + printf '\x01\x00\x00\x00' | sudo dd of=$F bs=1 seek=$FOD count=4 conv=notrunc 2>/dev/null + echo " NUDGE $PA (va=$VA) -> 1" + else + echo " skip $PA (va=$VA) = $CUR" + fi + done + sleep 2 +done +echo NUDGE-DONE diff --git a/tools/guest-diag/wall2/nudge2.sh b/tools/guest-diag/wall2/nudge2.sh new file mode 100755 index 0000000..16ebcd8 --- /dev/null +++ b/tools/guest-diag/wall2/nudge2.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# nudge2.sh - widened readiness-word nudger: ALL rw-s nvidia mappings <=256K, +# writes 1 to any +0x80 u32 that is 0. +DUR=${1:-300} +F=/dev/shm/vgpu-guest-ram +END=$(( $(date +%s) + DUR )) +sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/guest_words2.sh ubuntu@127.0.0.1:/tmp/guest_words2.sh >/dev/null 2>&1 +while [ "$(date +%s)" -lt "$END" ]; do + OUT=$(timeout 12 sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 "sudo bash /tmp/guest_words2.sh" 2>/dev/null) + [ -n "$OUT" ] && echo "[$(date +%H:%M:%S)] words:" && echo "$OUT" + echo "$OUT" | while read -r VA PA SZ; do + [ -z "$PA" ] && continue + PAD=$((PA)); FOD=$((PAD - 0x80000000)); [ $FOD -lt 0 ] && FOD=$((PAD)) + CUR=$(sudo dd if=$F bs=4 skip=$((FOD/4)) count=1 2>/dev/null | xxd -p) + if [ "$CUR" = "00000000" ]; then + printf '\x01\x00\x00\x00' | sudo dd of=$F bs=1 seek=$FOD count=4 conv=notrunc 2>/dev/null + echo " NUDGE va=$VA pa=$PA -> 1" + fi + done + sleep 1.5 +done +echo NUDGE2-DONE diff --git a/tools/guest-diag/wall2/pdbtick.py b/tools/guest-diag/wall2/pdbtick.py new file mode 100644 index 0000000..d65b79d --- /dev/null +++ b/tools/guest-diag/wall2/pdbtick.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# pdbtick.py - content-locate the GR0 ring page (deterministic first three +# GPFIFO entries) in the guest RAM file, then hand the verified +# {ring VA -> ring PA} pair to the service (sema_pdb_discover), which +# verifies the parked COPY_SERVER_PDES candidates and falls back to the +# vidmem walk-root sweep. Runs while the client is alive. +import mmap, struct, subprocess, sys, threading, time + +F = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_pdb_discover" +PAT = struct.pack("<QQQ", 0x0005980004ca0000, 0x8000040004ca0598, + 0x0003d00004ca059c) +RING_VA = 0x4cc0000 + +def gpa_of(x): + return x if x < 0x80000000 else x - 0x80000000 + 0x100000000 + +def main(): + fobj = open(F, "rb") + mm = mmap.mmap(fobj.fileno(), 0, access=mmap.ACCESS_READ) + size = len(mm) + nthr = 8 + step = size // nthr + res = [] + lock = threading.Lock() + def worker(a, b): + CH = 8 << 20 + off = a + while off < b: + d = mm[off:off + CH] + i = d.find(PAT) + while i >= 0: + with lock: + res.append(off + i) + i = d.find(PAT, i + 1) + off += CH - 64 + ts = [] + for k in range(nthr): + a = k * step + b = (k + 1) * step if k < nthr - 1 else size + t = threading.Thread(target=worker, args=(a, b)); t.start(); ts.append(t) + for t in ts: t.join() + if not res: + print("RING NOT FOUND") + return None + hits = [gpa_of(x) for x in res] + print("ring candidates:", [hex(h) for h in hits], flush=True) + for gpa in hits: + cmd = "echo 0x%x,0x%x > %s" % (RING_VA, gpa, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print("sema_pdb_discover 0x%x,0x%x rc=%d" % (RING_VA, gpa, r.returncode), flush=True) + return hits + + +def loop(): + import time as _t + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 120 + end = _t.time() + dur + fired = set() + while _t.time() < end: + hits = main() + if hits: + new = [h for h in hits if h not in fired] + if new: + fired.update(new) + _t.sleep(1.0) + print("# pdbtick end", flush=True) + +loop() diff --git a/tools/guest-diag/wall2/perfstall5.sh b/tools/guest-diag/wall2/perfstall5.sh new file mode 100644 index 0000000..ae1c161 --- /dev/null +++ b/tools/guest-diag/wall2/perfstall5.sh @@ -0,0 +1,9 @@ +#!/bin/sh +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk55.out 2>&1 &) +sleep 25 +PID=$(pgrep -x vk_frame | head -1) +cp /proc/$PID/maps /tmp/vk_maps2.txt +sudo perf record -p $PID -g -o /tmp/perf6.data -- sleep 3 2>/dev/null +sudo perf script -i /tmp/perf6.data -F ip,dso 2>/dev/null | grep eglcore | awk '{print $1}' | sort | uniq -c | sort -rn | head -8 > /tmp/spin_addrs.txt +cat /tmp/spin_addrs.txt diff --git a/tools/guest-diag/wall2/pt_bind.py b/tools/guest-diag/wall2/pt_bind.py new file mode 100644 index 0000000..126ed01 --- /dev/null +++ b/tools/guest-diag/wall2/pt_bind.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# pt_bind.py - bind GMMU PT pages to VA bases and read out VA->PA for the +# client's ring/window family. Strategy: +# 1. ring-page candidates: pages whose 8B-aligned u64 pairs look like +# GPFIFO entries (e0 = push VA & ~3 in the 0x44x-0x4ex families with +# va<4G; e1 top byte 0; length bits sane). +# 2. anchor: for each candidate PA and each family VA, check every 4KB +# page for entry idx=(VA>>12)&0x1ff whose PA part == candidate PA. +# A hit binds (PT page, VA, PA). +# 3. read out: from bound PT pages, decode entries for the whole VA list. +import mmap, struct, sys + +F = "/dev/shm/vgpu-guest-ram" +FAMILY = [ + 0x4020000, 0x4100000, 0x4160000, # r689 eng rings + 0x44a0000, 0x4500000, 0x4560000, # r690 ring VAs + 0x4480000, 0x4485000, 0x4480980, 0x4480f1c, + 0x4ca0000, 0x4ca5000, 0x4d00000, 0x4b17000, +] + +def gpa_to_off(pa): + return pa if pa < 0x80000000 else pa - 0x80000000 + +def off_to_gpa(off): + return off if off < 0x80000000 else off + 0x80000000 + +def pte_pa(v): + return (v >> 8) << 12 + +def main(): + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + size = len(mm) + print("# file size 0x%x" % size) + + # ---- pass 1: ring-page candidates by content ---- + cands = [] + pos = 0 + step = 4096 + while pos + step <= size: + d = mm[pos:pos+step] + hits = [] + for i in range(0, step, 8): + e0 = int.from_bytes(d[i:i+4], "little") + e1 = int.from_bytes(d[i+4:i+8], "little") + if 0x4400000 <= e0 <= 0x4e00000 and (e0 & 3) == 0 and (e1 & 0xff) == 0: + lw = (e1 >> 10) & 0x1fffff + if 0x40 <= lw <= 0x100000: + hits.append((i, e0, lw << 2)) + if len(hits) >= 3: + cands.append((off_to_gpa(pos), hits)) + if len(cands) >= 12: + break + pos += step + print("# ring-page candidates: %d" % len(cands)) + for pa, hits in cands: + print("cand pa=0x%x hits=%d first=%s" % + (pa, len(hits), ["0x%x/0x%x" % (h[1], h[2]) for h in hits[:4]])) + if not cands: + return 1 + + # ---- pass 2: anchor PT pages for candidate PAs x family VAs ---- + idxs = {} + for va in FAMILY: + idxs.setdefault((va >> 12) & 0x1ff, []).append(va) + cand_pas = set(pa for pa, _ in cands) + bound = [] + pos = 0 + while pos + step <= size: + pg = off_to_gpa(pos) + # skip scanning the candidate pages themselves + if pg not in cand_pas: + d = mm[pos:pos+step] + for idx, vas in idxs.items(): + o = idx * 8 + v = int.from_bytes(d[o:o+8], "little") + if v & 1: + pa = pte_pa(v) + if pa in cand_pas: + bound.append((pg, idx, vas, pa, v & 0xff)) + pos += step + print("# PT anchors found: %d" % len(bound)) + for pg, idx, vas, pa, fl in bound[:20]: + print("PT pa=0x%x entry[0x%x] flags=0x%x -> PA=0x%x (VA cands %s)" % + (pg, idx, fl, pa, ["0x%x" % x for x in vas])) + + # ---- pass 3: read out PT entries for the whole family from each bound PT page ---- + seen_pt = set() + for pg, idx, vas, pa, fl in bound: + if pg in seen_pt: + continue + seen_pt.add(pg) + off = gpa_to_off(pg) + d = mm[off:off+4096] + print("== PT page pa=0x%x family readout:" % pg) + for va in FAMILY: + i = (va >> 12) & 0x1ff + v = int.from_bytes(d[i*8:i*8+8], "little") + if v & 1: + print(" va=0x%x idx=0x%x -> pa=0x%x (raw=0x%x fl=0x%x)" % + (va, i, pte_pa(v), v, v & 0xff)) + else: + print(" va=0x%x idx=0x%x -> absent (raw=0x%x)" % (va, i, v)) + mm.close(); f.close() + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/run75g.sh b/tools/guest-diag/wall2/run75g.sh new file mode 100755 index 0000000..001a80f --- /dev/null +++ b/tools/guest-diag/wall2/run75g.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# run75g.sh - trace + record-word sweep on r680. +cd /tmp/opencode +rm -f /tmp/opencode/t75g-sweep.log +sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/t75g.py ubuntu@127.0.0.1:/tmp/t75g.py >/dev/null 2>&1 +( + sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 \ + 'sudo bash /tmp/slotfill2.sh > /tmp/slotfill75g.log 2>&1 & sleep 0.5; timeout 400 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk_t75g.out 2>&1 & sleep 8; PID=$(pgrep -x vk_frame | head -1); echo "PID=$PID"; sudo timeout 380 gdb -q -batch -p $PID -x /tmp/t75g.py > /tmp/t75g.out 2>&1; echo GDB_DONE rc=$?' +) > /tmp/opencode/t75g-ssh.log 2>&1 & +GPID=$! +sleep 35 +setvar() { echo "$2" | sudo tee /sys/module/vgsp/parameters/$1 >/dev/null; } +mark() { echo "[$(date +%s.%N)] $*" >> /tmp/opencode/t75g-sweep.log; } +# phase 1: baseline echo (client already polling) +mark "baseline echo" +sleep 20 +# phase 2: all-zero records +setvar wall2_a097_rec0 0; setvar wall2_a097_rec1 0; setvar wall2_a097_rec2 0; setvar wall2_a097_rec3 0 +setvar wall2_a097_rec4 0; setvar wall2_a097_rec5 0; setvar wall2_a097_rec6 0; setvar wall2_a097_rec7 0 +mark "records all zero"; sleep 20 +# phase 3: progress=100 +setvar wall2_a097_rec1 100 +mark "rec1=100"; sleep 20 +# phase 4: progress=1000 +setvar wall2_a097_rec1 1000 +mark "rec1=1000"; sleep 20 +# phase 5: progress=100000 +setvar wall2_a097_rec1 100000 +mark "rec1=100000"; sleep 20 +# phase 6: progress=1000000 + rec0=0 +setvar wall2_a097_rec1 1000000 +setvar wall2_a097_rec0 0 +mark "rec1=1000000 rec0=0"; sleep 20 +# phase 7: all ones +setvar wall2_a097_rec0 3735928559; setvar wall2_a097_rec1 3735928559; setvar wall2_a097_rec2 3735928559; setvar wall2_a097_rec3 3735928559 +mark "rec0-3=0xdeadbeef"; sleep 20 +# phase 8: back to echo +setvar wall2_a097_rec0 4294967295; setvar wall2_a097_rec1 4294967295; setvar wall2_a097_rec2 4294967295; setvar wall2_a097_rec3 4294967295 +mark "baseline echo again" +wait $GPID +echo SWEEP-DONE diff --git a/tools/guest-diag/wall2/runbook.py b/tools/guest-diag/wall2/runbook.py new file mode 100644 index 0000000..256d255 --- /dev/null +++ b/tools/guest-diag/wall2/runbook.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# runbook.py - watch for the GR0 channel ring registration, hand the pair to +# sema_pdb_discover, then (after the pdb anchor) issue bounded +# sema_chan_rewind calls so entries skipped before the anchor re-decode. +import re, subprocess, sys, time + +PDB = "/sys/module/vgsp/parameters/sema_pdb_discover" +REW = "/sys/module/vgsp/parameters/sema_chan_rewind" + +def dmesg_tail(n=150): + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + return out.splitlines()[-n:] + +def main(): + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 180 + end = time.time() + dur + fired = set() + rewinds = {} + print("# runbook start (%.0fs)" % dur, flush=True) + while time.time() < end: + for ln in dmesg_tail(): + m = re.search(r"sema-chan: h=0x([0-9a-f]+) eng=\d+ cid=\d+ gpfifo=0x([0-9a-f]+) ent=\d+ userd=0x[0-9a-f]+ as=\d+ put=\d+ get=\d+ ring rc=0 pa=0x([0-9a-f]+)", ln) + if not m: + continue + h = int(m.group(1), 16) + gv = int(m.group(2), 16) + rp = int(m.group(3), 16) + key = (h, gv, rp) + if key in fired: + # after the anchor has had time to land, rewind this channel + if h == 0xbeec0100: + t0 = rewinds.setdefault(key, time.time()) + n = int((time.time() - t0) // 8) + done = rewinds.get(("done", key), 0) + if n > done: + subprocess.run(["sudo", "sh", "-c", + "echo all > %s" % REW], capture_output=True) + rewinds[("done", key)] = n + print("[%s] REWIND all (h=0x%x, n=%d)" % ( + time.strftime("%H:%M:%S"), h, n), flush=True) + continue + fired.add(key) + r = subprocess.run(["sudo", "sh", "-c", + "echo 0x%x,0x%x > %s" % (gv, rp, PDB)], + capture_output=True) + print("[%s] TRIGGER h=0x%x va=0x%x pa=0x%x rc=%d" % ( + time.strftime("%H:%M:%S"), h, gv, rp, r.returncode), flush=True) + # report decodes/releases + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + for ln in out.splitlines()[-8:]: + if ("sema-push: h=0xbeec0100" in ln and "have_rel=1" in ln) or \ + "registered after verified" in ln or "rewind" in ln or \ + ("sema-push: h=0xbeec0100" in ln and "len=1432" in ln): + print("LOG %s" % ln.strip()[-170:], flush=True) + time.sleep(0.4) + print("# runbook end", flush=True) + +main() diff --git a/tools/guest-diag/wall2/scan_r673.py b/tools/guest-diag/wall2/scan_r673.py new file mode 100644 index 0000000..71a0128 --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# Analyze the live r673 guest RAM: locate every reference to the validated ring +# PA (0x12a783000) including flag-bit variants, and characterize the record +# arrays around each hit. Also dump the two pages named in the handoff. +import numpy as np, struct, sys, time + +RAM="/dev/shm/vgpu-guest-ram" +RING=0x12a783000 +MASK=~0xfff & ((1<<64)-1) + +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) + +img=open(RAM,"rb") +import os +fsz=os.path.getsize(RAM) +print("ram size 0x%x" % fsz) + +t0=time.time() +hits=[] +CH=64<<20 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero((a & np.uint64(MASK))==np.uint64(RING))[0] + for i in idx: + off=pos+int(i)*8 + val=int(a[i]) + hits.append((off,val)) +print("hits=%d scan=%.1fs" % (len(hits), time.time()-t0)) +for off,val in hits: + print(" off=0x%x gpa=0x%x val=0x%x (low=0x%x)" % (off,off2gpa(off),val,val&0xfff)) + +# characterize each hit page +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +def plausible_pa(v): return 0x100000000 <= (v & MASK) < 0x180000000 +def ptrish(v): + return (0xffff800000000000 <= v < 0xffffffffffffffff) + +for off,val in hits: + gpa=off2gpa(off) + page=gpa & MASK + rec_off=gpa-page + d=rd(page,0x1000) + print("=== hit gpa=0x%x page=0x%x rec_off=0x%x ===" % (gpa,page,rec_off)) + for stride in (0x20,0x40,0x60,0x30,0x50,0x28,0x48): + # record start such that the hit qword falls inside the record + for rel in range(0,stride,8): + start=rec_off-rel + if start<0: continue + # walk records; count plausible PA fields anywhere in each record + n_pa=0; n_ptr=0; n_1000=0 + for o in range(start,0x1000-stride,stride): + rec=d[o:o+stride] + qs=struct.unpack_from("<%dQ"%(stride//8),rec,0) + if any(plausible_pa(q) for q in qs): n_pa+=1 + if any(ptrish(q) for q in qs): n_ptr+=1 + if 0x1000 in qs: n_1000+=1 + if n_pa>=8: + print(" stride=0x%x start=0x%x: PA-records=%d ptr-records=%d 0x1000=%d" % (stride,start,n_pa,n_ptr,n_1000)) + # dump the first 0x100 bytes of the page and around the hit + print(" page head: %s" % d[:0x60].hex()) + s=max(0,rec_off-0x40); e=min(0x1000,rec_off+0x80) + print(" around hit [0x%x..0x%x]: %s" % (s,e,d[s:e].hex())) + +# dump the two named pages +for pa in (0x101499000,0x127cf2000): + d=rd(pa,0x1000) + print("=== page 0x%x ===" % pa) + if d is None: print(" unreadable"); continue + for o in range(0,0x120,0x20): + qs=struct.unpack_from("<4Q",d,o) + print(" +0x%03x: %016x %016x %016x %016x" % ((o,)+qs)) diff --git a/tools/guest-diag/wall2/scan_r673b.py b/tools/guest-diag/wall2/scan_r673b.py new file mode 100644 index 0000000..11613b1 --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673b.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# r673: dump the full record page 0x101499000 and search RAM for the ring/push +# VA structures to identify which record is the 16-page push (VA 0x4500000). +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +MASK=~0xfff & ((1<<64)-1) +img=open(RAM,"rb") +fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +print("== full record page 0x101499000 ==") +d=rd(0x101499000,0x1000) +nrec=0 +for o in range(0,0x1000,32): + p,f,pa,sz=struct.unpack_from("<QQQQ",d,o) + if p==0 and f==0 and pa==0 and sz==0: continue + nrec+=1 + print(" rec[%3d] vmap=%016x flags=%016x pa=%012x size=0x%x" % (o//32,p,f,pa,sz)) +print("nonempty records:",nrec) + +print() +print("== search RAM for qword 0x4520000 (ring VA) ==") +t0=time.time(); CH=64<<20 +for pat_name,pat in (("0x4520000",0x4520000),("0x4500000",0x4500000)): + print("-- pattern %s --" % pat_name) + hits=[] + for pos in range(0,fsz,CH): + img.seek(pos); dd=img.read(min(CH,fsz-pos)) + n=len(dd)//8 + a=np.frombuffer(dd[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(pat))[0] + for i in idx: + hits.append(pos+int(i)*8) + print(" hits=%d (%.1fs)" % (len(hits),time.time()-t0)) + for off in hits[:40]: + gpa=off2gpa(off); page=gpa&MASK + # show 32 bytes before/after + dd=rd(page,0x1000) + if not dd: continue + ro=gpa-page + s=max(0,ro-32); e=min(0x1000,ro+64) + print(" gpa=0x%x page=0x%x off=0x%x ctx=%s" % (gpa,page,ro,dd[s:e].hex())) + if len(hits)>40: print(" ... %d more" % (len(hits)-40)) diff --git a/tools/guest-diag/wall2/scan_r673c.py b/tools/guest-diag/wall2/scan_r673c.py new file mode 100644 index 0000000..db69b5d --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673c.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# r673: find the push page. Patterns: ring entry raw qwords, the sem VA, and +# method-stream signatures. Dump contexts. +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +pats=[ + ("entry0raw", 0x0005980004500000), + ("entry1raw", 0x0000040004500598), + ("entry2raw", 0x0003d0000450059c), + ("sem_va", 0x4500598), + ("sem_va2", 0x450059c), + ("m20038090", 0x20038090), + ("push_va", 0x4500000), +] +CH=64<<20 +for name,pat in pats: + hits=[] + t0=time.time() + for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(pat))[0] + for i in idx: hits.append(pos+int(i)*8) + print("== %s pat=0x%x hits=%d (%.1fs)" % (name,pat,len(hits),time.time()-t0)) + for off in hits[:12]: + gpa=off2gpa(off); page=gpa&~0xfff + d=rd(page,0x1000) + ro=gpa-page + s=max(0,ro-24); e=min(0x1000,ro+40) + print(" gpa=0x%x page=0x%x off=0x%x ctx=%s" % (gpa,page,ro,d[s:e].hex() if d else "?")) + if len(hits)>12: print(" ... %d more" % (len(hits)-12)) diff --git a/tools/guest-diag/wall2/scan_r673d.py b/tools/guest-diag/wall2/scan_r673d.py new file mode 100644 index 0000000..1fa89ca --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673d.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# r673 deep-dive: all ring-PA refs with low bits, ring page dump, RM record +# pages, and a RAM-wide scan for 0xa3bf20 VA-table records mentioning 0x4500000. +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +RING=0x12a783000; TAG=0xa3bf20 +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +MASK=~0xfff & ((1<<64)-1) +print("== all qwords (v & ~0xfff)==ring_pa ==") +CH=64<<20 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero((a & np.uint64(MASK))==np.uint64(RING))[0] + for i in idx: + off=pos+int(i)*8 + gpa=off2gpa(off); page=gpa&~0xfff + dd=rd(page,0x1000); ro=gpa-page + s=max(0,ro-32); e=min(0x1000,ro+48) + print(" gpa=0x%x low=0x%x ctx=%s" % (gpa,int(a[i])&0xfff,dd[s:e].hex() if dd else "?")) + +print() +print("== ring page 0x%x first 0x120 ==" % RING) +d=rd(RING,0x120) +for o in range(0,0x120,0x20): + print(" +0x%03x: %s" % (o,d[o:o+0x20].hex())) + +for pa in (0x1013a6000,0x104acb000,0x114760000,0x1202cd000,0x13e2c1000,0x13e62e000): + print() + print("== page 0x%x first 0x140 ==" % pa) + d=rd(pa,0x140) + for o in range(0,0x140,0x20): + q=struct.unpack_from("<4Q",d,o) + print(" +0x%03x: %016x %016x %016x %016x" % ((o,)+q)) + +print() +print("== RAM-wide: 32-byte records {va, cpu, 0xa3bf20, pages} with va==0x4500000 ==") +t0=time.time() +found=0 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(TAG))[0] + for i in idx: + off=pos+int(i)*8 + gpa=off2gpa(off); page=gpa&~0xfff + dd=rd(page,0x1000) + ro=gpa-page + if ro>=16 and ro+8<=0x1000: + va,cpu=struct.unpack_from("<QQ",dd,ro-16) + pgs=struct.unpack_from("<Q",dd,ro+8)[0] + if va==0x4500000 or (0x4400000<=va<=0x4600000): + print(" rec gpa=0x%x va=0x%x cpu=0x%x pages=0x%x" % (gpa,va,cpu,pgs)) + found+=1 +print("total", found, "(%.1fs)" % (time.time()-t0)) diff --git a/tools/guest-diag/wall2/session73-notes.md b/tools/guest-diag/wall2/session73-notes.md new file mode 100644 index 0000000..d2079d3 --- /dev/null +++ b/tools/guest-diag/wall2/session73-notes.md @@ -0,0 +1,51 @@ +# SESSION 73 notes (wall #2 live validation, r674 + r675) + +## r674 cycle results (8 client runs, all VKFAIL -4) +- Client vk_frame reproducibly fails at vkCreateDevice (-4) as before. +- wall2_resolver2 (content-first CE group 0x20038090 + tuple arrays): no candidates found + during the stall; tuple arrays contain the ring PA but not the push. +- livecap/livecap2/livecap3 (0x20038090 shape / 0x20050017 host-release shape): NO data + hits in guest RAM during the stall; only driver CODE pages match. +- Conclusion: the GR channel-init push content is NOT findable in /dev/shm/vgpu-guest-ram + by these patterns, even live. (The ring page content IS there and matches the service's + decode.) + +## Structural findings +- The push is RM-built (nvidia.ko), NOT UVM: kprobe on + uvm_hal_turing_host_semaphore_release never fired during the client run. +- nvidia.ko functions are `notrace` -> kprobe_events refuses them + ("Could not probe notrace function ... [nvidia]"). UVM module probes DO work. + QEMU gdbstub hardware breakpoints work fine (hbreak *addr). +- Service rescue matcher is CE-shaped only (hdr 0x20038090, hi==1, lo&0xfff==slot, + w5/v5 shape) -> can never match a GR push. Entries 0/1/2 are consumed as "stale" + after ~150 failed rescues each; gr_wedge=1. +- channelFillGpFifo(channel, idx, len) disasm: rdi=channel; +0x178=gpu; + +0xe8=cached ring CPU VA; +0xb8=push GPU VA base; +0x48=stride; + +0x110=class (0xc5b5 seen for a CE channel; GR = 0xc46f). + Entry encoding: [0]=va&~3 (u32), [1]=(va>>32 &0xff)|((len<<8)&0x7ffffc00). +- r674 ring PAs per run: 0x12a783000(?) no: run1 0x1050b6000, run2 0x10297d000, + later 0x102317000 (ring line format: "h=0x... gpfifo=0x44a0000 ent=32768 userd=0xffe00 as=2 put=3 get=0 ring rc=0 pa=0x... ap=1"). +- Push VA = 0x4480000, ring VA = 0x44a0000 (delta 0x20000); entry0 len=1432, + entry1 {va=0x4480598,len=4}, entry2 {va=0x448059c,len=976}. +- The pushbuffer (UVM) = 0x20000 (32 pages) aligned; UVM_PUSHBUFFER_SIZE=0x20000 + (UVM_MAX_PUSH_SIZE=0x400). UVM_RM_MEM_TYPE_SYS with CPU map. +- RM memdesc records {size=0x20000, flags=0x03008000, first_PA, ptr} exist in RAM + (11 found post-run) but their PAs did not contain the release group. +- RM window record {0x4480000, 0x20000} + head/tail list nodes exist in RAM. +- The r673 VA-table record {0x10, 0x4500000, 0x786fd9598ab0, 0xa3bf20} has a + "cpu" field that may be a user VA of the client (0x64..-0x78.. range). + +## Ops +- r674 wedged after the gdb-interrupted vk31 run (SSH + guest agent dead, vCPUs idle). + Fresh cycle r675 started. gdb674d still attached to the dead QEMU (never kill gdbs). +- /dev/shm 53%. + +## r675 plan +1. gdb675.gdb: hbreak channelFillGpFifo if class(+0x110)==0xc46f; on hit dump the + channel struct pointer fields (see gdb675.gdb) -> identify the push's CPU mapping + (direct-map or vmalloc) by content. +2. Compute PA (direct-map: VA - 0xffff8916c0000000 style base) and read the push + content from the RAM file; find the release group. +3. Inject 0x4480000 -> PA via sema_map_add BEFORE/DURING the stall; watch the service + apply the release and vkCreateDevice pass. +4. Durability: integrate into pin_gr_stream.py demand path or a service hook. diff --git a/tools/guest-diag/wall2/slotfill.sh b/tools/guest-diag/wall2/slotfill.sh new file mode 100644 index 0000000..a3416a9 --- /dev/null +++ b/tools/guest-diag/wall2/slotfill.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Watch the vk_frame client and fill the completion-slot u16 (base+0x2e) in +# every small /dev/nvidiactl rw-s mapping while it waits. +cd /tmp +rm -f /tmp/slotfill.log +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk76.out 2>&1 &) +PID="" +for i in $(seq 1 100); do PID=$(pgrep -x vk_frame | head -1); [ -n "$PID" ] && break; sleep 0.1; done +echo "pid=$PID" >> /tmp/slotfill.log +END=$((SECONDS+60)) +while [ $SECONDS -lt $END ]; do + [ -d /proc/$PID ] || break + for m in $(grep 'nvidiactl' /proc/$PID/maps | awk '$2 ~ /rw-s/ {print $1}'); do + lo=${m%-*}; hi=${m#*-} + size=$((0x$hi - 0x$lo)) + [ $size -gt 65536 ] && continue + off=$((0x$lo + 0x2e)) + v=$(dd if=/proc/$PID/mem bs=1 skip=$off count=2 2>/dev/null | xxd -p) + if [ "$v" = "ffff" ]; then + printf '\x00\x00' | dd of=/proc/$PID/mem bs=1 seek=$off count=2 conv=notrunc 2>/dev/null + echo "$(date +%s.%N) FILL $m+0x2e was ffff" >> /tmp/slotfill.log + fi + done + sleep 0.2 +done +echo "done" >> /tmp/slotfill.log +echo "=== client ===" +cat /tmp/vk76.out +echo "=== fills ===" +grep -c FILL /tmp/slotfill.log diff --git a/tools/guest-diag/wall2/slotfill2.sh b/tools/guest-diag/wall2/slotfill2.sh new file mode 100644 index 0000000..33ad702 --- /dev/null +++ b/tools/guest-diag/wall2/slotfill2.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cd /tmp +PID="" +for i in $(seq 1 200); do PID=$(pgrep -x vk_frame | head -1); [ -n "$PID" ] && break; sleep 0.05; done +END=$((SECONDS+45)) +while [ $SECONDS -lt $END ]; do + [ -d /proc/$PID ] || break + for m in $(grep 'nvidiactl' /proc/$PID/maps 2>/dev/null | awk '$2 ~ /rw-s/ {print $1}'); do + lo=${m%-*}; hi=${m#*-} + size=$((0x$hi - 0x$lo)) + [ $size -gt 65536 ] && continue + off=$((0x$lo + 0x2e)) + v=$(dd if=/proc/$PID/mem bs=1 skip=$off count=2 2>/dev/null | xxd -p) + if [ "$v" = "ffff" ]; then + printf '\x00\x00' | dd of=/proc/$PID/mem bs=1 seek=$off count=2 conv=notrunc 2>/dev/null + fi + done + sleep 0.1 +done diff --git a/tools/guest-diag/wall2/slots_probe.sh b/tools/guest-diag/wall2/slots_probe.sh new file mode 100644 index 0000000..71e6fb7 --- /dev/null +++ b/tools/guest-diag/wall2/slots_probe.sh @@ -0,0 +1,11 @@ +#!/bin/bash +PID=$1 +sudo grep -E 'rw-s' /proc/$PID/maps | grep nvidiactl | while read -r line; do + RNG=$(echo "$line" | awk '{print $1}') + LO=${RNG%-*}; HI=${RNG#*-} + SZ=$((0x$HI-0x$LO)) + [ $SZ -gt 262144 ] && continue + V2E=$(sudo dd if=/proc/$PID/mem bs=1 skip=$((0x$LO+0x2e)) count=2 2>/dev/null | xxd -p) + V0E=$(sudo dd if=/proc/$PID/mem bs=1 skip=$((0x$LO+0xe)) count=2 2>/dev/null | xxd -p) + echo "$RNG sz=$SZ w2e=$V2E w0e=$V0E" +done diff --git a/tools/guest-diag/wall2/stracevk.sh b/tools/guest-diag/wall2/stracevk.sh new file mode 100644 index 0000000..b2752c4 --- /dev/null +++ b/tools/guest-diag/wall2/stracevk.sh @@ -0,0 +1,7 @@ +#!/bin/sh +cd /tmp +sudo timeout 90 strace -f -T -tt -e trace=ioctl -o /tmp/vk_strace.out /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk58.out 2>&1 +echo "=== client out ===" +cat /tmp/vk58.out +echo "=== ioctl tail ===" +tail -30 /tmp/vk_strace.out diff --git a/tools/guest-diag/wall2/t101.py b/tools/guest-diag/wall2/t101.py new file mode 100644 index 0000000..436c609 --- /dev/null +++ b/tools/guest-diag/wall2/t101.py @@ -0,0 +1,55 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid)) +def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) +def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +state = {"chits":0, "cleared":0} +class BC(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i*8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception as e: + if state["chits"] <= 4: + print("chitfail %s" % e, flush=True) + if state["cleared"] >= 24: + self.enabled = False + print("BC-DISABLED", flush=True) + return False +try: + BC(); print("BPS-SET", flush=True) + gr("continue") +except Exception as e: + print("bpfail %s" % e, flush=True) +print("T101-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True) diff --git a/tools/guest-diag/wall2/t102.py b/tools/guest-diag/wall2/t102.py new file mode 100644 index 0000000..759d974 --- /dev/null +++ b/tools/guest-diag/wall2/t102.py @@ -0,0 +1,31 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid)) +try: + pc = int(gdb.parse_and_eval("$pc")) + print("PC 0x%x off=0x%x" % (pc, pc - base if base else 0)) +except Exception as e: + print("pcfail %s" % e) +try: + C = base + 0x2070B7C + print("A8COUNT %u" % int(gdb.parse_and_eval("*(unsigned int*)0x%x" % C))) +except Exception as e: + print("countfail %s" % e) +try: + print(gr("bt 6", to_string=True)) +except Exception as e: + print("btfail %s" % e) +gr("detach") +gr("quit") +print("T102-DONE") diff --git a/tools/guest-diag/wall2/t104.py b/tools/guest-diag/wall2/t104.py new file mode 100644 index 0000000..5a4c059 --- /dev/null +++ b/tools/guest-diag/wall2/t104.py @@ -0,0 +1,56 @@ +import gdb, time +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid), flush=True) +def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) +def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +state = {"chits":0, "cleared":0, "t0":time.time(), "last":time.time()} +class BC(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i*8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + state["last"] = time.time() + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + if state["cleared"] >= 8 or state["chits"] >= 40 or (time.time() - state["last"]) > 25.0: + self.enabled = False + print("BC-EXIT cleaned=%d chits=%d" % (state["cleared"], state["chits"]), flush=True) + return True + return False +try: + BC(); print("BPS-SET", flush=True) + gr("continue") +except Exception as e: + print("bpfail %s" % e, flush=True) +print("T104-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True) diff --git a/tools/guest-diag/wall2/t106.py b/tools/guest-diag/wall2/t106.py new file mode 100644 index 0000000..ceaac44 --- /dev/null +++ b/tools/guest-diag/wall2/t106.py @@ -0,0 +1,61 @@ +import gdb, time +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid), flush=True) +def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) +def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +state = {"chits":0, "cleared":0, "done":False} +class BC(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + if state["done"]: + return False + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i*8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + if state["cleared"] >= 5 or state["chits"] >= 14: + state["done"] = True + try: + gr("detach") + print("BC-DETACHED cleaned=%d chits=%d" % (state["cleared"], state["chits"]), flush=True) + except Exception as e: + print("detachfail %s" % e, flush=True) + return True + return False +try: + BC(); print("BPS-SET", flush=True) + gr("continue") +except Exception as e: + print("contend %s" % e, flush=True) +print("T106-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True) diff --git a/tools/guest-diag/wall2/t112c-gdbstub.py b/tools/guest-diag/wall2/t112c-gdbstub.py new file mode 100644 index 0000000..27d0b9d --- /dev/null +++ b/tools/guest-diag/wall2/t112c-gdbstub.py @@ -0,0 +1,42 @@ +import gdb + +gr = gdb.execute + +def u32(a): + return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +def u64(a): + return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) + +state = {"hits": {}} + +class HB(gdb.Breakpoint): + def __init__(self, pa, tag, quiet_after=8): + super().__init__("*0x%x" % pa, type=gdb.BP_BREAKPOINT, internal=False) + self.tag = tag + self.quiet_after = quiet_after + self.silent = True + def stop(self): + n = state["hits"].get(self.tag, 0) + 1 + state["hits"][self.tag] = n + try: + rdi = int(gdb.parse_and_eval("$rdi")) + rsi = int(gdb.parse_and_eval("$rsi")) + rdx = int(gdb.parse_and_eval("$rdx")) + rcx = int(gdb.parse_and_eval("$rcx")) + r8 = int(gdb.parse_and_eval("$r8")) + r9 = int(gdb.parse_and_eval("$r9")) + if n <= self.quiet_after or self.tag in ("PRED", "COMPR"): + print("HIT %s #%d rdi=0x%x rsi=0x%x rdx=0x%x rcx=0x%x r8=0x%x r9=0x%x" % + (self.tag, n, rdi, rsi, rdx, rcx, r8, r9), flush=True) + except Exception as e: + print("err", e, flush=True) + return False + +HB(0xffffffffc0c403b0, "VIDH") +HB(0xffffffffc0d667a0, "PMA", 4) +HB(0xffffffffc0d6ae30, "PRED") +HB(0xffffffffc0d57d60, "COMPR", 16) +HB(0xffffffffc0d607f0, "SCRUB", 4) +print("HB-SET", flush=True) +gr("continue") +print("T112B-DONE %s" % state["hits"], flush=True) diff --git a/tools/guest-diag/wall2/t120.py b/tools/guest-diag/wall2/t120.py new file mode 100644 index 0000000..68a1b8b --- /dev/null +++ b/tools/guest-diag/wall2/t120.py @@ -0,0 +1,148 @@ +import gdb +gr = gdb.execute +state = {"n": 0, "any": 0, "pend": None, "base": None, "chits": 0, "cleared": 0, + "whits": 0, "wrote": 0} + +def u32(a): + return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +def u64(a): + return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) + +def ioc(d, t, nr, size): + return (d << 30) | (size << 16) | (t << 8) | nr + +VIDHEAP = ioc(3, 0x46, 0x4a, 0xb8) + +def glbase(): + try: + pid = gdb.selected_inferior().pid + for line in open("/proc/%d/maps" % pid): + parts = line.split() + if "libnvidia-eglcore" in line and int(parts[2], 16) == 0: + return int(parts[0].split("-")[0], 16) + except Exception: + pass + return None + +class BC(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i * 8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + return False + +class BW(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["whits"] += 1 + try: + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx + 0x70) + obj2 = u64(sub + 0x58) + off = u32(obj2 + 0x60) + holder = u64(obj2 + 0x58) + base2 = u64(holder + 0x58) + word = base2 + off + val = u32(word) + if state["whits"] <= 6 or val == 0: + print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val), flush=True) + if val == 0: + gr("set {unsigned int}0x%x = 1" % word) + state["wrote"] += 1 + except Exception: + pass + return False + +class IOC(gdb.Breakpoint): + def __init__(self): + super().__init__("ioctl", internal=False) + self.silent = True + def stop(self): + if state["base"] is None: + b = glbase() + if b: + state["base"] = b + print("GLBASE 0x%x" % b, flush=True) + try: + BC(b + 0xB33D30) + BW(b + 0xDCEBD9) + print("BC-BW-SET", flush=True) + except Exception as e: + print("bpset-fail %s" % e, flush=True) + try: + req = int(gdb.parse_and_eval("$rsi")) & 0xFFFFFFFF + arg = int(gdb.parse_and_eval("$rdx")) + except Exception: + return False + if state["pend"] is not None: + try: + st = u32(state["pend"] + 0x14) + at = u32(state["pend"] + 0x38) + off = u64(state["pend"] + 0x68) + print("PREV149-OUT st=0x%x attr=0x%x off=0x%x" % (st, at, off), flush=True) + except Exception as e: + print("PREV149-STATUS unreadable %s" % e, flush=True) + state["pend"] = None + if req != VIDHEAP: + return False + state["any"] += 1 + try: + fn = u32(arg + 0x08) + size = u64(arg + 0x58) + attr = u32(arg + 0x38) + owner = u32(arg + 0x28) + flags = u32(arg + 0x34) + except Exception: + return False + if fn == 2 and size == 0x10000 and (attr & 0xf000) == 0x2000 and owner == 0x474C0000: + state["n"] += 1 + print("HIT149 n=%d attr=0x%x flags=0x%x owner=0x%x align=0x%x" % ( + state["n"], attr, flags, owner, u64(arg + 0x60)), flush=True) + if state["n"] == 1: + gr("bt 12") + newattr = attr & 0xFFFFCFFF + gr("set {unsigned int}0x%x = 0x%x" % (arg + 0x38, newattr)) + print("FLIP149 attr 0x%x -> 0x%x" % (attr, newattr), flush=True) + state["pend"] = arg + return False + +try: + IOC() + print("BPS-SET", flush=True) +except Exception as e: + print("bpset-fail %s" % e, flush=True) + +gr("set pagination off") +gr("set confirm off") +try: + gr("run") +except Exception as e: + print("runend %s" % e, flush=True) +print("T120-DONE n=%d any=%d chits=%d cleared=%d whits=%d wrote=%d" % ( + state["n"], state["any"], state["chits"], state["cleared"], + state["whits"], state["wrote"]), flush=True) +gr("quit") diff --git a/tools/guest-diag/wall2/t121.py b/tools/guest-diag/wall2/t121.py new file mode 100644 index 0000000..1ae25ed --- /dev/null +++ b/tools/guest-diag/wall2/t121.py @@ -0,0 +1,151 @@ +import gdb +gr = gdb.execute +state = {"n": 0, "any": 0, "pend": None, "base": None, "chits": 0, "cleared": 0, + "whits": 0, "wrote": 0} + +def u32(a): + return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +def u64(a): + return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) + +def ioc(d, t, nr, size): + return (d << 30) | (size << 16) | (t << 8) | nr + +VIDHEAP = ioc(3, 0x46, 0x4a, 0xb8) + +def glbase(): + try: + pid = gdb.selected_inferior().pid + for line in open("/proc/%d/maps" % pid): + parts = line.split() + if "libnvidia-eglcore" in line and int(parts[2], 16) == 0: + return int(parts[0].split("-")[0], 16) + except Exception: + pass + return None + +class BC(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i * 8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + return False + +class BW(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["whits"] += 1 + try: + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx + 0x70) + obj2 = u64(sub + 0x58) + off = u32(obj2 + 0x60) + holder = u64(obj2 + 0x58) + base2 = u64(holder + 0x58) + word = base2 + off + val = u32(word) + if state["whits"] <= 6 or val == 0: + print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val), flush=True) + if val == 0: + gr("set {unsigned int}0x%x = 1" % word) + state["wrote"] += 1 + except Exception: + pass + return False + +class IOC(gdb.Breakpoint): + def __init__(self): + super().__init__("ioctl", internal=False) + self.silent = True + def stop(self): + if state["base"] is None: + b = glbase() + if b: + state["base"] = b + print("GLBASE 0x%x" % b, flush=True) + try: + BC(b + 0xB33D30) + BW(b + 0xDCEBD9) + print("BC-BW-SET", flush=True) + except Exception as e: + print("bpset-fail %s" % e, flush=True) + try: + req = int(gdb.parse_and_eval("$rsi")) & 0xFFFFFFFF + arg = int(gdb.parse_and_eval("$rdx")) + except Exception: + return False + if state["pend"] is not None: + try: + st = u32(state["pend"] + 0x14) + at = u32(state["pend"] + 0x38) + off = u64(state["pend"] + 0x68) + print("PREV149-OUT st=0x%x attr=0x%x off=0x%x" % (st, at, off), flush=True) + except Exception as e: + print("PREV149-STATUS unreadable %s" % e, flush=True) + state["pend"] = None + if req != VIDHEAP: + return False + state["any"] += 1 + try: + fn = u32(arg + 0x08) + size = u64(arg + 0x58) + attr = u32(arg + 0x38) + owner = u32(arg + 0x28) + flags = u32(arg + 0x34) + except Exception: + return False + if fn == 2: + state["n"] += 1 + if state["n"] >= 120: + print("VH#%d size=0x%x align=0x%x attr=0x%x attr2=0x%x flags=0x%x owner=0x%x range=0x%x-0x%x" % ( + state["n"], size, u64(arg + 0x60), attr, u32(arg + 0x90), flags, owner, + u64(arg + 0x80), u64(arg + 0x88)), flush=True) + if size == 0x10000 and (attr & 0xf000) == 0x2000 and owner == 0x474C0000: + print("HIT149 attr=0x%x flags=0x%x" % (attr, flags), flush=True) + gr("bt 12") + newattr = attr & 0xFFFFCFFF + gr("set {unsigned int}0x%x = 0x%x" % (arg + 0x38, newattr)) + print("FLIP149 attr 0x%x -> 0x%x" % (attr, newattr), flush=True) + state["pend"] = arg + return False + +try: + IOC() + print("BPS-SET", flush=True) +except Exception as e: + print("bpset-fail %s" % e, flush=True) + +gr("set pagination off") +gr("set confirm off") +try: + gr("run") +except Exception as e: + print("runend %s" % e, flush=True) +print("T120-DONE n=%d any=%d chits=%d cleared=%d whits=%d wrote=%d" % ( + state["n"], state["any"], state["chits"], state["cleared"], + state["whits"], state["wrote"]), flush=True) +gr("quit") diff --git a/tools/guest-diag/wall2/t122.py b/tools/guest-diag/wall2/t122.py new file mode 100644 index 0000000..8c9c6fd --- /dev/null +++ b/tools/guest-diag/wall2/t122.py @@ -0,0 +1,151 @@ +import gdb +gr = gdb.execute +state = {"n": 0, "any": 0, "pend": None, "base": None, "chits": 0, "cleared": 0, + "whits": 0, "wrote": 0} + +def u32(a): + return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +def u64(a): + return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) + +def ioc(d, t, nr, size): + return (d << 30) | (size << 16) | (t << 8) | nr + +VIDHEAP = ioc(3, 0x46, 0x4a, 0xb8) + +def glbase(): + try: + pid = gdb.selected_inferior().pid + for line in open("/proc/%d/maps" % pid): + parts = line.split() + if "libnvidia-eglcore" in line and int(parts[2], 16) == 0: + return int(parts[0].split("-")[0], 16) + except Exception: + pass + return None + +class BC(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i * 8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + return False + +class BW(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["whits"] += 1 + try: + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx + 0x70) + obj2 = u64(sub + 0x58) + off = u32(obj2 + 0x60) + holder = u64(obj2 + 0x58) + base2 = u64(holder + 0x58) + word = base2 + off + val = u32(word) + if state["whits"] <= 6 or val == 0: + print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val), flush=True) + if val == 0: + gr("set {unsigned int}0x%x = 1" % word) + state["wrote"] += 1 + except Exception: + pass + return False + +class IOC(gdb.Breakpoint): + def __init__(self): + super().__init__("ioctl", internal=False) + self.silent = True + def stop(self): + if state["base"] is None: + b = glbase() + if b: + state["base"] = b + print("GLBASE 0x%x" % b, flush=True) + try: + BC(b + 0xB33D30) + BW(b + 0xDCEBD9) + print("BC-BW-SET", flush=True) + except Exception as e: + print("bpset-fail %s" % e, flush=True) + try: + req = int(gdb.parse_and_eval("$rsi")) & 0xFFFFFFFF + arg = int(gdb.parse_and_eval("$rdx")) + except Exception: + return False + if state["pend"] is not None: + try: + st = u32(state["pend"] + 0x14) + at = u32(state["pend"] + 0x38) + off = u64(state["pend"] + 0x68) + print("PREV149-OUT st=0x%x attr=0x%x off=0x%x" % (st, at, off), flush=True) + except Exception as e: + print("PREV149-STATUS unreadable %s" % e, flush=True) + state["pend"] = None + if req != VIDHEAP: + return False + state["any"] += 1 + try: + fn = u32(arg + 0x08) + size = u64(arg + 0x58) + attr = u32(arg + 0x38) + owner = u32(arg + 0x28) + flags = u32(arg + 0x34) + except Exception: + return False + if fn == 2: + state["n"] += 1 + if state["n"] >= 120: + print("VH#%d size=0x%x align=0x%x attr=0x%x attr2=0x%x flags=0x%x owner=0x%x range=0x%x-0x%x" % ( + state["n"], size, u64(arg + 0x60), attr, u32(arg + 0x90), flags, owner, + u64(arg + 0x80), u64(arg + 0x88)), flush=True) + if size >= 0x10000 and (attr & 0xf000) == 0x2000 and owner == 0x474C0000: + print("HIT149 size=0x%x attr=0x%x flags=0x%x" % (size, attr, flags), flush=True) + gr("bt 12") + newattr = attr & 0xFFFFCFFF + gr("set {unsigned int}0x%x = 0x%x" % (arg + 0x38, newattr)) + print("FLIP149 attr 0x%x -> 0x%x" % (attr, newattr), flush=True) + state["pend"] = arg + return False + +try: + IOC() + print("BPS-SET", flush=True) +except Exception as e: + print("bpset-fail %s" % e, flush=True) + +gr("set pagination off") +gr("set confirm off") +try: + gr("run") +except Exception as e: + print("runend %s" % e, flush=True) +print("T120-DONE n=%d any=%d chits=%d cleared=%d whits=%d wrote=%d" % ( + state["n"], state["any"], state["chits"], state["cleared"], + state["whits"], state["wrote"]), flush=True) +gr("quit") diff --git a/tools/guest-diag/wall2/t75g.py b/tools/guest-diag/wall2/t75g.py new file mode 100644 index 0000000..2a0881a --- /dev/null +++ b/tools/guest-diag/wall2/t75g.py @@ -0,0 +1,170 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid + +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found") + gdb.execute("detach") + sys.exit(1) +print("EGLBASE 0x%x PID %d" % (base, pid)) + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') +def u64(a): + return int.from_bytes(rd(a, 8), 'little') + +state = {"out": 0, "E": 0, "S": 0, "armed150": False, "armedS": False, + "bt_done": False, "post_done": False} + +class B(gdb.Breakpoint): + def __init__(self, spec, fn, limit=100000): + super().__init__(spec, internal=False) + self.fn = fn; self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + stop = bool(self.fn(self)) + except Exception as e: + print("ERR@0x%x %s" % (int(gdb.parse_and_eval("$pc")), e)) + stop = False + return stop or self.hits >= self.limit + +def a97(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + r14 = int(gdb.parse_and_eval("$r14")) + out = u64(rbp+0x10) + state["out"] = out + q = rd(out, 0x28) + rec_ptr = int.from_bytes(q[8:16], 'little') + print("A97 out=0x%x count=%d f10=%08x f20=%08x f24=%08x recp=0x%x" % + (out, int.from_bytes(q[0:4], 'little'), int.from_bytes(q[0x10:0x14], 'little'), + int.from_bytes(q[0x20:0x24], 'little'), int.from_bytes(q[0x24:0x28], 'little'), rec_ptr)) + if rec_ptr and rec_ptr > 0x1000: + try: + rec = rd(rec_ptr, 16) + print("A97 rec=%08x %08x %08x %08x" % + tuple(int.from_bytes(rec[i*4:i*4+4], 'little') for i in range(4))) + except Exception as e: + print("A97 rec-unreadable", e) + return False + +def post(bp): + if state["post_done"]: + return False + state["post_done"] = True + rsp = int(gdb.parse_and_eval("$rsp")) + q = rd(rsp, 0x40) + print("POST rsp=0x%x out=0x%x" % (rsp, state["out"])) + for i in range(0, 0x40, 8): + v = int.from_bytes(q[i:i+8], 'little') + print("POST rsp+%02x = 0x%016x" % (i, v)) + if state["out"]: + oq = rd(state["out"], 0x40) + for i in range(0, 0x40, 8): + v = int.from_bytes(oq[i:i+8], 'little') + print("POST out+%02x = 0x%016x" % (i, v)) + return False + +def t150(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + if not state.get("vt_done"): + state["vt_done"] = True + rdi = u64(rbp+0x148) + vt = u64(rdi) if rdi else 0 + m118 = u64(vt+0x118) if vt else 0 + m170 = u64(vt+0x170) if vt else 0 + print("VT E148=0x%x vt=0x%x m118=0x%x m170=0x%x" % (rdi, vt, m118, m170)) + if not state.get("eng_done"): + state["eng_done"] = True + n = u32(rbp+8) + ids = [u32(rbp+0x54+i*4) for i in range(min(n,8))] + print("ENG n=%d ids=%s" % (n, ",".join("0x%x" % x for x in ids))) + + # cfg area pointer is rip-relative 0x2019320 at runtime = base + (0x2019320 - 0) + # eglcore base == min load addr; file offset == vaddr for the .text; data seg present? + try: + vals = rd(base + 0x2019320 + 0xf80, 0xe0) + for i in range(0, 0xe0, 0x10): + v4 = [int.from_bytes(vals[i+j:i+j+4],'little') for j in (0,4,8,12)] + print("CFG +0x%03x = %s" % (0xf80+i, ["0x%x"%x for x in v4])) + except Exception as e: + print("CFG dump err", e) + v = u32(rbp+0x150) + b38 = rd(rbp+0x1b038, 1)[0]; b39 = rd(rbp+0x1b039, 1)[0] + p = u64(rbp+0x1b030); s8 = u64(p+8) if p else 0 + print("T150 E=0x%x v=0x%08x b038=%d b039=%d p=0x%x p8=0x%x" % + (rbp, v, b38, b39, p, s8)) + if not state["bt_done"]: + state["bt_done"] = True + print(gdb.execute("bt 8", to_string=True)) + if not state["armed150"]: + state["armed150"] = True + state["E"] = rbp + try: + gdb.execute("frame 1") + S = int(gdb.parse_and_eval("$rbp")) + gdb.execute("frame 0") + print("S=0x%x S120=%08x S128=%08x" % (S, u32(S+0x120), u32(S+0x128))) + state["S"] = S + except Exception as e: + print("S capture err", e) + S = 0 + W("*(unsigned int *)0x%x" % (rbp+0x150), rbp, 0x150, 4, "W150") + W("*(unsigned char *)0x%x" % (rbp+0x1b038), rbp, 0x1b038, 1, "W038") + W("*(unsigned char *)0x%x" % (rbp+0x1b039), rbp, 0x1b039, 1, "W039") + if S: + W("*(unsigned int *)0x%x" % (S+0x120), S, 0x120, 4, "W120") + print("WATCHES armed E=0x%x S=0x%x" % (rbp, S)) + return False + +def sl(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + print("SLOOP S=0x%x S120=%08x" % (rbp, u32(rbp+0x120))) + if not state["armedS"]: + state["armedS"] = True + W("*(unsigned int *)0x%x" % (rbp+0x120), rbp, 0x120, 4, "W120") + print("W120 armed on S+0x120") + return False + +class W(gdb.Breakpoint): + def __init__(self, spec, E, off, size, tag): + super().__init__(spec, type=gdb.BP_WATCHPOINT) + self.silent = True; self.E = E; self.off = off; self.size = size + self.tag = tag; self.hits = 0 + def stop(self): + self.hits += 1 + cur = int.from_bytes(rd(self.E+self.off, self.size), 'little') + pc = int(gdb.parse_and_eval("$pc")) + print("%s v=0x%x pc=0x%x (rel 0x%x)" % (self.tag, cur, pc, pc-base)) + if self.hits <= 8: + print(gdb.execute("bt 10", to_string=True)) + return self.hits >= 400 + +B("*0x%x" % (base+0xecd43d), a97, 220) +B("*0x%x" % (base+0xd5e4d3), post, 100000) +B("*0x%x" % (base+0xd5e449), t150, 220) +B("*0x%x" % (base+0xd6ef76), sl, 240) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +print("TRACE-END") +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/t79.py b/tools/guest-diag/wall2/t79.py new file mode 100644 index 0000000..c2fe381 --- /dev/null +++ b/tools/guest-diag/wall2/t79.py @@ -0,0 +1,99 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found"); gdb.execute("detach"); sys.exit(1) +print("EGLBASE 0x%x PID %d" % (base, pid)) + +maps = [] +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x, 16) for x in parts[0].split('-')] + maps.append((lo, hi, parts[1], parts[-1] if len(parts) > 5 else "anon")) + except Exception: + pass + +def resolve(addr): + for lo, hi, perm, name in maps: + if lo <= addr < hi: + return "0x%x-0x%x %s %s" % (lo, hi, perm, name) + return "?" + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') +def u64(a): + return int.from_bytes(rd(a, 8), 'little') + +state = {"done": False} + +class B(gdb.Breakpoint): + def __init__(self, spec, fn, limit=100000): + super().__init__(spec, internal=False) + self.fn = fn; self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + stop = bool(self.fn(self)) + except Exception as e: + print("ERR@0x%x %s" % (int(gdb.parse_and_eval("$pc")), e)) + stop = False + return stop or self.hits >= self.limit + +def loophead(bp): + if state["done"]: + return False + state["done"] = True + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx+0x70) + print("WAIT rbx=0x%x sub=0x%x" % (rbx, sub)) + try: + obj2 = u64(sub+0x58) + off = u32(obj2+0x60) + holder = u64(obj2+0x58) + base2 = u64(holder+0x58) + word_addr = base2 + off + val = u32(word_addr) + print("WAIT obj2=0x%x off=0x%x holder=0x%x base=0x%x word=0x%x val=0x%x" % + (obj2, off, holder, base2, word_addr, val)) + print("WAIT word map:", resolve(word_addr)) + print("WAIT holder obj2+0x50=0x%x obj2+0x64=0x%x" % (u64(obj2+0x50), u32(obj2+0x64))) + try: + hd = rd(holder, 0x30) + print("WAIT holder dump:", " ".join("%02x" % b for b in hd)) + except Exception as e: + print("WAIT holder dump err", e) + except Exception as e: + print("WAIT chain err", e) + # dump the client's nvidiactl rw-s mappings + print("WAIT nvidiactl maps:") + for lo, hi, perm, name in maps: + if "nvidiactl" in name and "rw-s" in perm: + print(" 0x%x-0x%x size=0x%x" % (lo, hi, hi-lo)) + return True + +B("*0x%x" % (base+0xdcebd9), loophead, 1) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +print("TRACE-END") +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/t80r.py b/tools/guest-diag/wall2/t80r.py new file mode 100644 index 0000000..ac2263b --- /dev/null +++ b/tools/guest-diag/wall2/t80r.py @@ -0,0 +1,32 @@ +import gdb, sys +inf = gdb.selected_inferior() +pid = inf.pid +cands = [] +for line in open("/proc/%d/maps" % pid): + if "nvidiactl" in line and "rw-s" in line: + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if hi - lo == 0x4000: + cands.append(lo + 0x80) +print("CANDIDATES %s" % ["0x%x" % c for c in cands]) +class W(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*(unsigned int*)0x%x" % addr, gdb.BP_WATCHPOINT, gdb.WP_READ, internal=False) + self.addr = addr + self.silent = True + def stop(self): + print("READWORD 0x%x" % self.addr) + return True +armed = 0 +for c in cands: + try: + W(c); armed += 1 + except Exception as e: + print("watchfail 0x%x: %s" % (c, e)) +print("ARMED %d" % armed) +if armed: + gdb.execute("continue") +print("T80R-DONE") diff --git a/tools/guest-diag/wall2/t83.py b/tools/guest-diag/wall2/t83.py new file mode 100644 index 0000000..5e3cce2 --- /dev/null +++ b/tools/guest-diag/wall2/t83.py @@ -0,0 +1,25 @@ +import gdb +from collections import Counter +gr = gdb.execute +pcs = Counter() +inf = gdb.selected_inferior() +for i in range(300): + try: + gr("stepi", to_string=True) + except Exception as e: + print("stepfail %s" % e); break + try: + pcs[int(gdb.parse_and_eval("$pc"))] += 1 + except Exception: + break +print("HOTPCS") +top = pcs.most_common(4) +for pc, n in top: + print("PC 0x%x hits=%d" % (pc, n)) +for pc, n in top[:1]: + try: + print(gr("x/10i 0x%x" % pc, to_string=True)) + print(gr("info registers rax rbx rcx rdx rsi rdi rbp r8 r9 r10 r11 r12 r13 r14 r15", to_string=True)) + except Exception as e: + print("fail %s" % e) +print("T83-DONE") diff --git a/tools/guest-diag/wall2/t89.py b/tools/guest-diag/wall2/t89.py new file mode 100644 index 0000000..92c1de3 --- /dev/null +++ b/tools/guest-diag/wall2/t89.py @@ -0,0 +1,17 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +print(gr("x/70i 0x%x" % (base + 0xB28519), to_string=True)) +print(gr("x/40i 0x%x" % (base + 0xB28560), to_string=True)) +print("T89-DONE") diff --git a/tools/guest-diag/wall2/t90.py b/tools/guest-diag/wall2/t90.py new file mode 100644 index 0000000..8518235 --- /dev/null +++ b/tools/guest-diag/wall2/t90.py @@ -0,0 +1,57 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found"); gdb.execute("detach"); sys.exit(1) +COUNT = base + 0x2070b7c +print("EGLBASE 0x%x PID %d COUNT_ADDR 0x%x" % (base, pid, COUNT)) + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') + +state = {"n": 0} + +class B(gdb.Breakpoint): + def __init__(self, spec, limit=100000): + super().__init__(spec, internal=False) + self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + v = u32(COUNT) + if v: + state["n"] += 1 + if state["n"] <= 8: + print("CLEAR count %u -> 0 (hit %d)" % (v, self.hits)) + gdb.execute("set {unsigned int}0x%x = 0" % COUNT) + except Exception as e: + print("ERR", e) + return self.hits >= 400 + +B("*0x%x" % (base+0xa85d24), 400) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +except KeyboardInterrupt: + pass +print("TRACE-END clears=%d" % state["n"]) +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/t91.py b/tools/guest-diag/wall2/t91.py new file mode 100644 index 0000000..5c84ba9 --- /dev/null +++ b/tools/guest-diag/wall2/t91.py @@ -0,0 +1,95 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found"); gdb.execute("detach"); sys.exit(1) +COUNT = base + 0x2070b7c +print("EGLBASE 0x%x PID %d COUNT 0x%x" % (base, pid, COUNT)) + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') +def u64(a): + return int.from_bytes(rd(a, 8), 'little') + +state = {"clears": 0, "mp": 0, "am": 0} + +class B(gdb.Breakpoint): + def __init__(self, spec, fn, limit=100000): + super().__init__(spec, internal=False) + self.fn = fn; self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + stop = bool(self.fn(self)) + except Exception as e: + print("ERR@0x%x %s" % (int(gdb.parse_and_eval("$pc")), e)) + stop = False + return stop or self.hits >= self.limit + +def clear(bp): + try: + v = u32(COUNT) + if v: + state["clears"] += 1 + if state["clears"] <= 5: + print("CLEAR count %u -> 0" % v) + gdb.execute("set {unsigned int}0x%x = 0" % COUNT) + except Exception as e: + print("CLEAR err", e) + return False + +def mp(bp): + rsi = int(gdb.parse_and_eval("$rsi")) + state["mp"] += 1 + if state["mp"] <= 3: + print("MEMPROPS ptr=0x%x" % rsi) + return False + +def am(bp): + rsi = int(gdb.parse_and_eval("$rsi")) + state["am"] += 1 + if state["am"] <= 10: + try: + st = u32(rsi) + size = u64(rsi+0x10) + tidx = u32(rsi+0x18) + print("ALLOC sType=%d size=0x%x typeIndex=%d" % (st, size, tidx)) + except Exception as e: + print("ALLOC err", e) + return False + +B("*0x%x" % (base+0xa85d24), clear, 100000) +try: + B("vkGetPhysicalDeviceMemoryProperties", mp, 5) +except Exception as e: + print("MP BP err", e) +try: + B("vkAllocateMemory", am, 20) +except Exception as e: + print("AM BP err", e) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +except KeyboardInterrupt: + pass +print("TRACE-END clears=%d" % state["clears"]) +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/t93.py b/tools/guest-diag/wall2/t93.py new file mode 100644 index 0000000..6a0c23d --- /dev/null +++ b/tools/guest-diag/wall2/t93.py @@ -0,0 +1,57 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +class B(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True; self.done = False + def stop(self): + if self.done: + return True + self.done = True + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + print("RETRYFN obj=0x%x n=0x%x" % (obj, n)) + cnt = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % (obj + 0x22efc))) + print("CNT %d" % cnt) + wrote = 0 + for i in range(0, min(cnt, 32)): + p = int(gdb.parse_and_eval("*(unsigned long*)0x%x" % (obj + 0x1428 + i*8))) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + v0 = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % tgt)) + print("E%d ptr=0x%x tgt=0x%x w0=0x%x w16e=0x%x" % (i, p, tgt, v0, v16)) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + wrote += 1 + except Exception as e: + print("E%d fail %s" % (i, e)) + print("CLEARED %d" % wrote) + gr("detach") + except Exception as e: + print("fail %s" % e) + return True +try: + B() + print("BP-SET") + gr("continue") +except Exception as e: + print("bpfail %s" % e) +print("T92-DONE") diff --git a/tools/guest-diag/wall2/t94.py b/tools/guest-diag/wall2/t94.py new file mode 100644 index 0000000..0d2aa27 --- /dev/null +++ b/tools/guest-diag/wall2/t94.py @@ -0,0 +1,54 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +state = {"hits":0, "cleared":0} +class B(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + state["hits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % (obj + 0x22efc))) + if state["hits"] <= 6: + print("HIT %d obj=0x%x n=0x%x cnt=%d" % (state["hits"], obj, n, cnt)) + for i in range(0, min(cnt, 32)): + p = int(gdb.parse_and_eval("*(unsigned long*)0x%x" % (obj + 0x1428 + i*8))) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + if state["cleared"] <= 12: + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt)) + except Exception as e: + if state["hits"] <= 6: + print("E%d fail %s" % (i, e)) + except Exception as e: + if state["hits"] <= 6: + print("hitfail %s" % e) + return state["hits"] >= 4000 +try: + B(); print("BP-SET") + gr("continue") +except Exception as e: + print("bpfail %s" % e) +print("T94-DONE hits=%d cleared=%d" % (state["hits"], state["cleared"])) diff --git a/tools/guest-diag/wall2/t95.py b/tools/guest-diag/wall2/t95.py new file mode 100644 index 0000000..8daf6ca --- /dev/null +++ b/tools/guest-diag/wall2/t95.py @@ -0,0 +1,19 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +print("--- wait site +0xA12D90..:") +print(gr("x/60i 0x%x" % (base + 0xA12D90), to_string=True)) +print("--- caller +0xD690D0..:") +print(gr("x/40i 0x%x" % (base + 0xD690D0), to_string=True)) +print("T95-DONE") diff --git a/tools/guest-diag/wall2/t96b.py b/tools/guest-diag/wall2/t96b.py new file mode 100644 index 0000000..cd3b7ab --- /dev/null +++ b/tools/guest-diag/wall2/t96b.py @@ -0,0 +1,40 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +class B(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xA12DDF), internal=False) + self.silent = True; self.done = False + def stop(self): + if self.done: + return True + self.done = True + try: + rdi = int(gdb.parse_and_eval("$rdi")) + rsi = int(gdb.parse_and_eval("$rsi")) + rdx = int(gdb.parse_and_eval("$rdx")) + print("POLL nfds=%d timeout=%d arr=0x%x" % (rsi, rdx, rdi)) + fd = int(gdb.parse_and_eval("*(int*)0x%x" % rdi)) + ev = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (rdi + 4))) + rv = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (rdi + 6))) + print("FDPOLL fd=%d events=0x%x revents=0x%x" % (fd, ev, rv)) + except Exception as e: + print("fail %s" % e) + return True +try: + B(); print("BP-SET") + gr("continue") +except Exception as e: + print("bpfail %s" % e) +print("T96-DONE") diff --git a/tools/guest-diag/wall2/t97.py b/tools/guest-diag/wall2/t97.py new file mode 100644 index 0000000..5a12348 --- /dev/null +++ b/tools/guest-diag/wall2/t97.py @@ -0,0 +1,22 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +C = base + 0x2070b7c +v = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % C)) +print("COUNT %u" % v) +print(gr("x/8wx 0x%x" % (C - 0x10), to_string=True)) +if v: + gr("set {unsigned int}0x%x = 0" % C) + print("WROTE 0") +print("T97-DONE") diff --git a/tools/guest-diag/wall2/t99.py b/tools/guest-diag/wall2/t99.py new file mode 100644 index 0000000..ed665fb --- /dev/null +++ b/tools/guest-diag/wall2/t99.py @@ -0,0 +1,79 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid)) +def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) +def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +state = {"chits":0, "cleared":0, "whits":0, "wrote":0} +class BC(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i*8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + if state["cleared"] <= 10: + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt)) + except Exception: + pass + except Exception as e: + if state["chits"] <= 4: + print("chitfail %s" % e) + return state["chits"] >= 6000 +class BW(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xDCEBD9), internal=False) + self.silent = True + def stop(self): + state["whits"] += 1 + try: + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx + 0x70) + obj2 = u64(sub + 0x58) + off = u32(obj2 + 0x60) + holder = u64(obj2 + 0x58) + base2 = u64(holder + 0x58) + word = base2 + off + val = u32(word) + if state["whits"] <= 6 or val == 0: + print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val)) + if val == 0: + gr("set {unsigned int}0x%x = 1" % word) + state["wrote"] += 1 + if state["wrote"] <= 10: + print("WORDWROTE 0x%x" % word) + except Exception as e: + if state["whits"] <= 4: + print("whitfail %s" % e) + return False +try: + BC(); BW(); print("BPS-SET") + gr("continue") +except Exception as e: + print("bpfail %s" % e) +print("T99-DONE chits=%d cleared=%d whits=%d wrote=%d" % (state["chits"], state["cleared"], state["whits"], state["wrote"])) diff --git a/tools/guest-diag/wall2/trace_push674.py b/tools/guest-diag/wall2/trace_push674.py new file mode 100644 index 0000000..ced4168 --- /dev/null +++ b/tools/guest-diag/wall2/trace_push674.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# r674: find every reference to the push VA 0x4480000 and follow RM structures +# (direct-map pointers: PA = ptr - page_offset_base) toward a page list. +import numpy as np, struct, os, time, sys + +RAM="/dev/shm/vgpu-guest-ram" +VA=0x4480000 +BASE=0xffff8916c0000000 # r674 page_offset_base (from dmesg rebuild #15) +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa<0x80000000: return pa + if pa<0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off<0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) +def ptr2pa(v): + if BASE<=v<BASE+0x180000000: + return v-BASE + return None + +CH=64<<20 +hits=[] +t0=time.time() +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(VA))[0] + for i in idx: hits.append(off2gpa(pos+int(i)*8)) +print("VA hits:", len(hits), "(%.1fs)" % (time.time()-t0)) + +def tag(v): + bs=struct.pack("<Q",v) + return bs.decode() if all(32<=c<127 for c in bs) else "" + +for gpa in hits: + page=gpa&~0xfff + d=rd(page,0x1000) + if not d: continue + ro=gpa-page + print("== hit gpa=0x%x page=0x%x off=0x%x ==" % (gpa,page,ro)) + # print 0x60 before, 0x80 after + s=max(0,ro-0x60); e=min(0x1000,ro+0x80) + for o in range(s,e,0x10): + a,b=struct.unpack_from("<QQ",d,o) + print(" +0x%03x: %016x %016x %s %s" % (o,a,b,tag(a),tag(b))) + # follow direct-map pointers in this page + for o in range(0,0x1000-8,8): + v=struct.unpack_from("<Q",d,o)[0] + pa=ptr2pa(v) + if pa is None: continue + dd=rd(pa&~0xfff,0x1000) + if not dd: continue + # search that page for PA runs (>=8 page-aligned RAM PAs ascending/descending) + runs=[]; cur=[] + for oo in range(0,0xff8,8): + vv=struct.unpack_from("<Q",dd,oo)[0] + if 0x100000000<=vv<0x180000000 and (vv&0xfff)==0: + cur.append((oo,vv)) + else: + if len(cur)>=8: runs.append(cur) + cur=[] + if len(cur)>=8: runs.append(cur) + if runs: + print(" ptr +0x%03x -> pa=0x%x has %d PA-run(s):" % (o,pa&~0xfff,len(runs))) + for r in runs[:4]: + print(" run at +0x%x: %d pages first=0x%x last=0x%x" % (r[0][0],len(r),r[0][1],r[-1][1])) diff --git a/tools/guest-diag/wall2/unwait.py b/tools/guest-diag/wall2/unwait.py new file mode 100644 index 0000000..94e677c --- /dev/null +++ b/tools/guest-diag/wall2/unwait.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# unwait.py - guest-side, zero-ptrace wait-word poller: writes 1 to the +0x80 +# u32 of every zero 16 KB rw-s nvidiactl mapping of the newest vk_frame. +import os, re, subprocess, sys, time +DUR = float(sys.argv[1]) if len(sys.argv) > 1 else 600 +def newest_pid(): + try: + out = subprocess.run(["pgrep", "-n", "-x", "vk_frame"], capture_output=True).stdout + return int(out.decode().strip()) + except Exception: + return None +def maps(pid): + try: + return open("/proc/%d/maps" % pid).read() + except Exception: + return "" +logged = set() +end = time.time() + DUR +while time.time() < end: + pid = newest_pid() + if pid: + for line in maps(pid).splitlines(): + p = line.split() + if len(p) < 6 or "nvidia" not in p[-1] or "rw-s" not in p[1]: + continue + try: + lo, hi = [int(x, 16) for x in p[0].split("-")] + except Exception: + continue + if hi - lo != 0x4000: + continue + wa = lo + 0x80 + try: + fd = os.open("/proc/%d/mem" % pid, os.O_RDWR) + os.lseek(fd, wa, 0) + v = os.read(fd, 4) + if len(v) == 4 and v == b"\x00\x00\x00\x00": + os.lseek(fd, wa, 0) + os.write(fd, b"\x01\x00\x00\x00") + key = (pid, wa) + if key not in logged: + logged.add(key) + print("WORDWROTE pid=%d va=0x%x" % (pid, wa), flush=True) + os.close(fd) + except Exception: + pass + time.sleep(0.4) +print("UNWAIT-DONE", flush=True) diff --git a/tools/guest-diag/wall2/unwait2.py b/tools/guest-diag/wall2/unwait2.py new file mode 100644 index 0000000..cd4169b --- /dev/null +++ b/tools/guest-diag/wall2/unwait2.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# unwait2.py - no-ptrace guest-side harness: +# (a) writes 1 to any zero u32 at +0x80 of every 0x4000-byte rw-s nvidia +# mapping of EVERY live vk_frame; +# (b) scans each live vk_frame's anonymous/heap rw-p mappings (16-byte +# aligned) for marker slots {u32@0==0, u16@+0xe==0xffff} and clears +# u16@+0xe to 0 (the instance-phase pending markers the t105 gdb used +# to clear). No ptrace anywhere. +import os, subprocess, sys, time +DUR = float(sys.argv[1]) if len(sys.argv) > 1 else 600 +def pids(): + try: + out = subprocess.run(["pgrep", "-x", "vk_frame"], capture_output=True).stdout + return [int(x) for x in out.split()] + except Exception: + return [] +def rd(fd, off, n): + try: + os.lseek(fd, off, 0) + return os.read(fd, n) + except Exception: + return b"" +def wr(fd, off, data): + try: + os.lseek(fd, off, 0) + os.write(fd, data) + return True + except Exception: + return False +wlog = set(); mlog = set() +end = time.time() + DUR +last_scan = 0.0 +while time.time() < end: + for pid in pids(): + try: + fd = os.open("/proc/%d/mem" % pid, os.O_RDWR) + except Exception: + continue + try: + for line in open("/proc/%d/maps" % pid): + p = line.split() + if len(p) < 6 or "nvidia" not in p[-1] or "rw-s" not in p[1]: + continue + lo, hi = [int(x, 16) for x in p[0].split("-")] + if hi - lo != 0x4000: + continue + wa = lo + 0x80 + v = rd(fd, wa, 4) + if len(v) == 4 and v == b"\x00\x00\x00\x00": + if wr(fd, wa, b"\x01\x00\x00\x00"): + k = (pid, wa) + if k not in wlog: + wlog.add(k) + print("WORDWROTE pid=%d va=0x%x" % (pid, wa), flush=True) + except Exception: + pass + if time.time() - last_scan > 1.5: + budget = 64 + try: + for line in open("/proc/%d/maps" % pid): + p = line.split() + if len(p) < 6 or p[1] != "rw-p": + continue + path = p[-1] + if path.startswith("/") and "nvidia" not in path and path != "[heap]": + continue + lo, hi = [int(x, 16) for x in p[0].split("-")] + if hi - lo > 0x10000000: + hi = lo + 0x10000000 + off = lo + while off < hi: + chunk = rd(fd, off, 0x100000) + if len(chunk) < 32: + break + j = 0 + while j + 16 <= len(chunk): + if chunk[j:j+4] == b"\x00\x00\x00\x00" and chunk[j+0xe:j+0x10] == b"\xff\xff": + wa = off + j + 0xe + if (pid, wa) not in mlog: + if wr(fd, wa, b"\x00\x00"): + mlog.add((pid, wa)) + print("MARKCLR pid=%d va=0x%x" % (pid, wa), flush=True) + j += 16 + off += 0x100000 + budget -= 1 + if budget <= 0: + break + if budget <= 0: + break + except Exception: + pass + last_scan = time.time() + os.close(fd) + time.sleep(0.4) +print("UNWAIT2-DONE", flush=True) diff --git a/tools/guest-diag/wall2/vk_alloc_test.py b/tools/guest-diag/wall2/vk_alloc_test.py new file mode 100644 index 0000000..0bcf35a --- /dev/null +++ b/tools/guest-diag/wall2/vk_alloc_test.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# vk_alloc_test.py - reproduce vk_frame's DEVICE_LOCAL image allocation: +# instance -> device -> OPTIMAL R8G8B8A8 storage image -> mem reqs -> +# pick first DEVICE_LOCAL type -> vkAllocateMemory -> report rc/type/size. +import ctypes, sys + +vk = ctypes.CDLL("libvulkan.so.1") + +class VkApplicationInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("pApplicationName", ctypes.c_char_p), ("applicationVersion", ctypes.c_uint32), + ("pEngineName", ctypes.c_char_p), ("engineVersion", ctypes.c_uint32), + ("apiVersion", ctypes.c_uint32)] + +class VkInstanceCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), ("pApplicationInfo", ctypes.c_void_p), + ("enabledLayerCount", ctypes.c_uint32), ("ppEnabledLayerNames", ctypes.c_void_p), + ("enabledExtensionCount", ctypes.c_uint32), ("ppEnabledExtensionNames", ctypes.c_void_p)] + +class VkMemoryType(ctypes.Structure): + _fields_ = [("propertyFlags", ctypes.c_uint32), ("heapIndex", ctypes.c_uint32)] +class VkMemoryHeap(ctypes.Structure): + _fields_ = [("size", ctypes.c_uint64), ("flags", ctypes.c_uint32)] +class VkPhysicalDeviceMemoryProperties(ctypes.Structure): + _fields_ = [("memoryTypeCount", ctypes.c_uint32), + ("memoryTypes", VkMemoryType * 32), + ("memoryHeapCount", ctypes.c_uint32), + ("memoryHeaps", VkMemoryHeap * 16)] + +class VkDeviceQueueCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), ("queueFamilyIndex", ctypes.c_uint32), + ("queueCount", ctypes.c_uint32), ("pQueuePriorities", ctypes.c_void_p)] +class VkDeviceCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), ("queueCreateInfoCount", ctypes.c_uint32), + ("pQueueCreateInfos", ctypes.c_void_p), + ("enabledLayerCount", ctypes.c_uint32), ("ppEnabledLayerNames", ctypes.c_void_p), + ("enabledExtensionCount", ctypes.c_uint32), ("ppEnabledExtensionNames", ctypes.c_void_p), + ("pEnabledFeatures", ctypes.c_void_p)] + +class VkExtent3D(ctypes.Structure): + _fields_ = [("width", ctypes.c_uint32), ("height", ctypes.c_uint32), ("depth", ctypes.c_uint32)] +class VkImageCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), ("flags", ctypes.c_uint32), + ("imageType", ctypes.c_uint32), ("format", ctypes.c_uint32), + ("extent", VkExtent3D), ("mipLevels", ctypes.c_uint32), ("arrayLayers", ctypes.c_uint32), + ("samples", ctypes.c_uint32), ("tiling", ctypes.c_uint32), ("usage", ctypes.c_uint32), + ("sharingMode", ctypes.c_uint32), ("queueFamilyIndexCount", ctypes.c_uint32), + ("pQueueFamilyIndices", ctypes.c_void_p), ("initialLayout", ctypes.c_uint32)] +class VkMemoryRequirements(ctypes.Structure): + _fields_ = [("size", ctypes.c_uint64), ("alignment", ctypes.c_uint64), ("memoryTypeBits", ctypes.c_uint32)] +class VkMemoryAllocateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("allocationSize", ctypes.c_uint64), ("memoryTypeIndex", ctypes.c_uint32)] + +app = VkApplicationInfo(0, None, b"alloctest", 1, b"alloctest", 1, 0x401000) +ci = VkInstanceCreateInfo(1, None, 0, ctypes.cast(ctypes.pointer(app), ctypes.c_void_p), 0, None, 0, None) +inst = ctypes.c_void_p() +print("vkCreateInstance rc=%d" % vk.vkCreateInstance(ctypes.byref(ci), None, ctypes.byref(inst))) + +n = ctypes.c_uint32(0) +vk.vkEnumeratePhysicalDevices(inst, ctypes.byref(n), None) +devs = (ctypes.c_void_p * n.value)() +vk.vkEnumeratePhysicalDevices(inst, ctypes.byref(n), devs) +phys = devs[0] +print("phys=0x%x" % (phys or 0)) + +prio = ctypes.c_float(1.0) +qci = VkDeviceQueueCreateInfo(2, None, 0, 0, 1, ctypes.cast(ctypes.pointer(prio), ctypes.c_void_p)) +dci = VkDeviceCreateInfo(3, None, 0, 1, ctypes.cast(ctypes.pointer(qci), ctypes.c_void_p), 0, None, 0, None, None) +dev = ctypes.c_void_p() +rc = vk.vkCreateDevice(phys, ctypes.byref(dci), None, ctypes.byref(dev)) +print("vkCreateDevice rc=%d dev=0x%x" % (rc, dev or 0)) +if rc != 0: + sys.exit(1) + +mp = VkPhysicalDeviceMemoryProperties() +vk.vkGetPhysicalDeviceMemoryProperties(phys, ctypes.byref(mp)) + +img_ci = VkImageCreateInfo(14, None, 0, 1, 37, # VK_IMAGE_TYPE_2D, R8G8B8A8_UNORM + VkExtent3D(64, 64, 1), 1, 1, 1, 0, # OPTIMAL + 0x2 | 0x1, 0, 0, None, 0) # STORAGE | TRANSFER_SRC, UNDEFINED +img = ctypes.c_void_p() +rc = vk.vkCreateImage(dev, ctypes.byref(img_ci), None, ctypes.byref(img)) +print("vkCreateImage rc=%d img=0x%x" % (rc, img or 0)) +if rc != 0: + sys.exit(1) + +mr = VkMemoryRequirements() +vk.vkGetImageMemoryRequirements(dev, img, ctypes.byref(mr)) +print("memreqs size=0x%x align=0x%x typeBits=0x%x" % (mr.size, mr.alignment, mr.memoryTypeBits)) + +tidx = 0 +found = False +for k in range(mp.memoryTypeCount): + if (mr.memoryTypeBits & (1 << k)) and (mp.memoryTypes[k].propertyFlags & 0x1): + tidx = k + found = True + break +print("selected type=%d found_device_local=%s (types: %s)" % + (tidx, found, ", ".join("%d:0x%x" % (k, mp.memoryTypes[k].propertyFlags) for k in range(mp.memoryTypeCount)))) + +ma = VkMemoryAllocateInfo(5, None, mr.size, tidx) +mem = ctypes.c_void_p() +rc = vk.vkAllocateMemory(dev, ctypes.byref(ma), None, ctypes.byref(mem)) +print("vkAllocateMemory size=0x%x type=%d rc=%d" % (mr.size, tidx, rc)) diff --git a/tools/guest-diag/wall2/vk_memprops.py b/tools/guest-diag/wall2/vk_memprops.py new file mode 100644 index 0000000..3cf6ba2 --- /dev/null +++ b/tools/guest-diag/wall2/vk_memprops.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# vk_memprops.py - enumerate Vulkan devices and dump their memory properties +# via ctypes (no vkCreateDevice needed; safe alongside other clients). +import ctypes, ctypes.util, sys + +vk = ctypes.CDLL("libvulkan.so.1") + +class VkApplicationInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("pApplicationName", ctypes.c_char_p), ("applicationVersion", ctypes.c_uint32), + ("pEngineName", ctypes.c_char_p), ("engineVersion", ctypes.c_uint32), + ("apiVersion", ctypes.c_uint32)] + +class VkInstanceCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), ("pApplicationInfo", ctypes.c_void_p), + ("enabledLayerCount", ctypes.c_uint32), ("ppEnabledLayerNames", ctypes.c_void_p), + ("enabledExtensionCount", ctypes.c_uint32), ("ppEnabledExtensionNames", ctypes.c_void_p)] + +class VkMemoryType(ctypes.Structure): + _fields_ = [("propertyFlags", ctypes.c_uint32), ("heapIndex", ctypes.c_uint32)] + +class VkMemoryHeap(ctypes.Structure): + _fields_ = [("size", ctypes.c_uint64), ("flags", ctypes.c_uint32)] + +class VkPhysicalDeviceMemoryProperties(ctypes.Structure): + _fields_ = [("memoryTypeCount", ctypes.c_uint32), + ("memoryTypes", VkMemoryType * 32), + ("memoryHeapCount", ctypes.c_uint32), + ("memoryHeaps", VkMemoryHeap * 16)] + +app = VkApplicationInfo(0, None, b"memprops", 1, b"memprops", 1, 0x401000) +ci = VkInstanceCreateInfo(1, None, 0, ctypes.cast(ctypes.pointer(app), ctypes.c_void_p), 0, None, 0, None) +inst = ctypes.c_void_p() +rc = vk.vkCreateInstance(ctypes.byref(ci), None, ctypes.byref(inst)) +print("vkCreateInstance rc=%d" % rc) +if rc != 0: + sys.exit(1) + +n = ctypes.c_uint32(0) +vk.vkEnumeratePhysicalDevices(inst, ctypes.byref(n), None) +print("devices=%d" % n.value) +devs = (ctypes.c_void_p * n.value)() +vk.vkEnumeratePhysicalDevices(inst, ctypes.byref(n), devs) + +for i in range(n.value): + props = VkPhysicalDeviceMemoryProperties() + vk.vkGetPhysicalDeviceMemoryProperties(devs[i], ctypes.byref(props)) + print("=== DEV %d phys=0x%x ===" % (i, devs[i] or 0)) + print(" memoryTypeCount=%d memoryHeapCount=%d" % (props.memoryTypeCount, props.memoryHeapCount)) + for t in range(props.memoryTypeCount): + mt = props.memoryTypes[t] + print(" type[%d] flags=0x%08x heap=%d" % (t, mt.propertyFlags, mt.heapIndex)) + for h in range(props.memoryHeapCount): + mh = props.memoryHeaps[h] + print(" heap[%d] size=0x%x (%u MiB) flags=0x%x" % (h, mh.size, mh.size >> 20, mh.flags)) diff --git a/tools/guest-diag/wall2/vmap_watch2.py b/tools/guest-diag/wall2/vmap_watch2.py new file mode 100644 index 0000000..194b0d6 --- /dev/null +++ b/tools/guest-diag/wall2/vmap_watch2.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# Guest-side live watcher v2: scan ALL vmallocinfo areas via /proc/kcore for +# (a) GR ring entry patterns, (b) pages starting with a small INC_METHOD header. +import struct, time +ENTRIES = [bytes.fromhex("0000480400980500"), bytes.fromhex("8000040004480598"), bytes.fromhex("0003d0000448059c")] +class Kcore: + def __init__(self): + self.fd = open("/proc/kcore", "rb") + hdr = self.fd.read(64) + e_phoff = struct.unpack_from("<Q", hdr, 32)[0] + e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54) + self.segs = [] + for i in range(e_phnum): + self.fd.seek(e_phoff + i * e_phentsize) + ph = self.fd.read(56) + p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from("<IIQQQQ", ph, 0) + if p_type == 1: + self.segs.append((p_va, p_va + p_sz, p_off)) + def read(self, va, n): + for lo, hi, off in self.segs: + if lo <= va and va + n <= hi: + self.fd.seek(off + (va - lo)) + return self.fd.read(n) + return None +kc = Kcore() +def areas(): + out = [] + for line in open("/proc/vmallocinfo"): + p = line.split() + if len(p) < 2 or "-" not in p[0]: continue + try: + lo, hi = p[0].split("-") + lo, hi = int(lo,16), int(hi,16) + except Exception: + continue + out.append((lo, hi, line.strip()[-60:])) + return out +def hdr_ok(page): + v = struct.unpack_from("<I", page, 0)[0] + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return op == 1 and 1 <= cnt <= 8 +print("vmap_watch2 start", flush=True) +found = set() +t0 = time.time() +it = 0 +while time.time() - t0 < 300: + it += 1 + nareas = 0 + for lo, hi, tag in areas(): + size = hi - lo + if size > 8 << 20: continue + nareas += 1 + d = kc.read(lo, size) + if not d: continue + for pat in ENTRIES: + i = d.find(pat) + if i >= 0: + key = (lo, i & ~0xfff, pat.hex()) + if key not in found: + found.add(key) + print("%.2f RING %s VA 0x%x (+0x%x) %s" % (time.time()-t0, pat.hex(), lo+(i&~0xfff), i&~0xfff, tag), flush=True) + for off in range(0, size - 0x1000, 0x1000): + page = d[off:off+0x1000] + if hdr_ok(page): + key = (lo, off, "h") + if key not in found: + found.add(key) + print("%.2f HDR VA 0x%x (+0x%x) %s head=%s" % (time.time()-t0, lo+off, off, tag, page[:0x30].hex()), flush=True) + if it % 10 == 0: + print("%.1f iter %d areas %d" % (time.time()-t0, it, nareas), flush=True) + time.sleep(0.2) +print("vmap_watch2 done", flush=True) diff --git a/tools/guest-diag/wall2/walk.py b/tools/guest-diag/wall2/walk.py new file mode 100644 index 0000000..3624007 --- /dev/null +++ b/tools/guest-diag/wall2/walk.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# walk.py - host-side replica of the service's svc_gmmu_walk + pd_discover, +# run against /dev/shm/vgpu-guest-ram. Discovers the page directory that +# reproduces the service's map[] sample pairs, then walks the ring/window +# VAs through it. +import mmap, struct, sys + +F = "/dev/shm/vgpu-guest-ram" +SEMA_PA_MAX = 0x180000000 + +# samples from the r690 dmesg map[] lines +SAMPLES = [ + (0x100000000, 0x100326000), + (0x100e2f000, 0x10096e000), + (0x100e33000, 0x10096b000), + (0x100e34000, 0x10096a000), + (0x100e35000, 0x100969000), + (0x100e37000, 0x102dcb000), + (0x100e38000, 0x102dca000), + (0x100e39000, 0x102dc9000), +] + +TARGET_VAS = [0x44a0000, 0x4500000, 0x4560000, 0x4480000, 0x4485000, + 0x4ca0000, 0x4ca5000, 0x4d00000, 0x4b17000, 0x4020000, + 0x4100000, 0x4160000] + +mm = None + +def off(pa): + return pa if pa < 0x80000000 else pa - 0x80000000 + +def rd(pa, n=8): + if pa + n > SEMA_PA_MAX: + return None + try: + d = mm[off(pa):off(pa)+n] + return d if len(d) == n else None + except Exception: + return None + +def rd64(pa): + d = rd(pa, 8) + return None if d is None else int.from_bytes(d, "little") + +def pde_aperture(ap): + if ap == 1: return "FB" + if ap in (2, 3): return "SYS" + return None + +def pte_aperture(ap): + if ap == 0: return "FB" + if ap == 1: return None + return "SYS" + +def pte_addr(e, fb): + if fb: + return ((e >> 8) & 0x1FFFFFF) << 12 + return ((e >> 8) & 0x3FFFFFFFFFFF) << 12 + +def pte_leaf(table, idx): + e = rd64(table + 8 * idx) + if e is None or not (e & 1): + return None + ap = pte_aperture((e >> 1) & 0x3) + if ap is None: + return None + a = pte_addr(e, ap == "FB") + if not fb_ok(a): + return None + return a, ap + +def fb_ok(a): + return a + 8 <= SEMA_PA_MAX + +def walk(va): + table = None + # root_bits = ilog2(entries=4) = 2; shift = 49-2 = 47 + e = rd64(table) if False else None + # level 0 + idx = (va >> 47) & 0x3 + # root table is the candidate itself (passed separately) + return None + +def walk_from(pdb, va, trace=False): + table = pdb + for level in range(3): + if level == 0: + bits, shift = 2, 47 + else: + bits, shift = 9, 38 if level == 1 else 29 + idx = (va >> shift) & ((1 << bits) - 1) + e = rd64(table + 8 * idx) + if e is None: + return None + if trace: + print(" L%d table=0x%x idx=%u e=0x%016x" % (level, table, idx, e)) + if e & 1: + return None + ap = pde_aperture((e >> 1) & 0x3) + if ap is None: + return None + table = pte_addr(e, ap == "FB") + if not fb_ok(table): + return None + idx = (va >> 21) & 0xff + big = rd64(table + 16 * idx) + small = rd64(table + 16 * idx + 8) + if big is None or small is None: + return None + if trace: + print(" PD0 table=0x%x idx=%u big=0x%016x small=0x%016x" % + (table, idx, big, small)) + if not (small & 1): + sap = pde_aperture((small >> 1) & 0x3) + if sap is not None: + t = pte_addr(small, sap == "FB") + r = pte_leaf(t, (va >> 12) & 0x1ff) + if r: + return r[0] + (va & 0xfff), r[1] + if big & 1: + bap = pte_aperture((big >> 1) & 0x3) + if bap is not None: + a = pte_addr(big, bap == "FB") + (va & ((1 << 21) - 1)) + if fb_ok(a): + return a, bap + else: + bap = pde_aperture((big >> 1) & 0x3) + if bap is not None: + t = pte_addr(big, bap == "FB") + r = pte_leaf(t, (va >> 16) & 0x1f) + if r: + return r[0] + (va & 0xffff), r[1] + return None + +def main(): + global mm + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + size = len(mm) + print("# scanning for directory candidates...") + cands = [] + for fo in range(0, size, 4096): + e = int.from_bytes(mm[fo:fo+8], "little") + if e and not (e & 1): + ap = (e >> 1) & 0x3 + if ap in (1, 2, 3): + pa = fo if fo < 0x80000000 else fo + 0x80000000 + if pa < SEMA_PA_MAX: + cands.append(pa) + print("# candidates: %d" % len(cands)) + best = [] + for c in cands: + hits = 0 + for (sva, spa) in SAMPLES: + r = walk_from(c, sva) + if r and (r[0] & ~0xfff) == (spa & ~0xfff): + hits += 1 + if hits >= 2: + best.append((hits, c)) + best.sort(reverse=True) + print("# candidates with >=2 hits: %d" % len(best)) + for hits, c in best[:10]: + print(" pdb=0x%x hits=%d/8" % (c, hits)) + if not best: + return 1 + pdb = best[0][1] + print("== walking targets through pdb=0x%x:" % pdb) + for va in TARGET_VAS: + r = walk_from(pdb, va) + if r: + print(" va=0x%x -> pa=0x%x ap=%s" % (va, r[0], r[1])) + else: + print(" va=0x%x -> no translation" % va) + mm.close(); f.close() + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_livecap.py b/tools/guest-diag/wall2/wall2_livecap.py new file mode 100644 index 0000000..af68c03 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# wall2_livecap.py - live capture of the GR channel-init push page. +# +# The release group in the push is {0x20038090, sem_va_hi, sem_va_lo, payload, +# continuation}: sem_va = (hi<<32)|lo, and for the channel-init entry the sem +# sits at push_va + entry_len (entry0: len=1432=0x598, sem at +0x598), with the +# group itself at push_va + entry_len - 24 = +0x580. +# +# This watcher scans guest RAM for 0x20038090 groups during the client's stall, +# classifies them, and injects push_va -> page_pa via sema_map_add when the +# group's sem VA and offset identify the push page. Prints full evidence. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap.log" +MASK = ~0xfff & ((1 << 64) - 1) + +ENTRY_LEN = int(os.environ.get("W2_LEN", "1432")) +WATCH_S = int(os.environ.get("W2_WATCH", "120")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def cont_ok(v): + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return (op in (1, 3, 5) and cnt != 0) or op in (4, 7) + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + g = d[i:i+24] + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, hi, lo, pay, w5, v5, g)) + pos += CH + return out + +def main(): + log("=== livecap start len=%d watch=%ds ===" % (ENTRY_LEN, WATCH_S)) + injected = {} + deadline = time.time() + WATCH_S + while time.time() < deadline: + hits = scan() + cands = [] + for (page, off, hi, lo, pay, w5, v5, g) in hits: + sem = (hi << 32) | lo + # classify + pool = (hi == 1 and lo < 0x10000 and 0 < pay < (1 << 24) + and (w5 >> 29) == 1 and ((w5 >> 13) & 7) == 4 + and ((w5 & 0x1fff) << 2) == 0x300 and (v5 & 3) == 0) + if pool: + continue # CE pool release family, not the push + if not (0x4000000 <= sem < 0x100000000): + continue + if not (0 < pay < (1 << 24)): + continue + if not cont_ok(w5): + continue + push_va = sem - ENTRY_LEN + cands.append((page, off, sem, pay, push_va, w5, v5, g)) + if cands: + # prefer the canonical group offset 0x580 / sem-consistent push VA + cands.sort(key=lambda c: (c[1] != (ENTRY_LEN - 24), -c[3])) + page, off, sem, pay, push_va, w5, v5, g = cands[0] + d = rd(page, 0x100) + log("livecap: %d cand(s); page=0x%x off=0x%x sem=0x%x pay=%u push_va=0x%x w5=0x%08x v5=0x%08x" + % (len(cands), page, off, sem, pay, push_va, w5, v5)) + log(" page head: %s" % d[:0x40].hex()) + if (push_va & 0xfff) == 0 and push_va not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (push_va, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x (off=0x%x)" % (push_va, page, off)) + injected[push_va] = page + except OSError as e: + log(" inject FAIL", e) + time.sleep(1) + log("=== livecap done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_livecap2.py b/tools/guest-diag/wall2/wall2_livecap2.py new file mode 100644 index 0000000..a33b25b --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap2.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# wall2_livecap2.py - capture the GR channel-init push page by its release +# group {0x20038090, hi, lo, payload, continuation} at page offset 0x598. +# The push VA comes from the service's own entry[0] decode in dmesg. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap2.log" +MASK = ~0xfff & ((1 << 64) - 1) +WATCH_S = int(os.environ.get("W2_WATCH", "180")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def prefix_ok(d): + off = 0 + while off + 4 <= len(d): + hdr = struct.unpack_from("<I", d, off)[0] + if hdr: break + off += 4 + if off + 4 > len(d): return False + op = (hdr >> 29) & 7 + if op in (4, 7): return True + if op not in (1, 3, 5): return False + return ((hdr >> 16) & 0xfff) != 0 + +def cont_ok(v): + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return (op in (1, 3, 5) and cnt != 0) or op in (4, 7) + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + g = d[i:i+24] + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, hi, lo, pay, w5, v5)) + pos += CH + return out + +def main(): + log("=== livecap2 start watch=%ds ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + seen_pages = {} + while time.time() < deadline: + hits = scan() + for (page, off, hi, lo, pay, w5, v5) in hits: + if not (0 < pay < (1 << 24)): + continue + if not cont_ok(w5): + continue + d = rd(page, 0x1000) + if not d: continue + pok = prefix_ok(d[:0x100]) + key = (page, off) + if key in seen_pages: + continue + seen_pages[key] = 1 + log("hit page=0x%x off=0x%x sem=0x%x pay=%u w5=0x%08x v5=0x%08x prefix_ok=%s" + % (page, off, (hi << 32) | lo, pay, w5, v5, pok)) + log(" head: %s" % d[:0x30].hex()) + log(" @0x590: %s" % d[0x590:0x5c0].hex()) + if pok and off in (0x598, 0x590, 0x580, 0x588): + va = 0x4480000 + if va not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (va, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x" % (va, page)) + injected.add(va) + except OSError as e: + log(" inject FAIL", e) + time.sleep(0.5) + log("=== livecap2 done: %d injections ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_livecap3.py b/tools/guest-diag/wall2/wall2_livecap3.py new file mode 100644 index 0000000..db28bc2 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap3.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# wall2_livecap3.py - capture the UVM GR channel-init push page via the host +# (C46F) semaphore release group emitted by uvm_hal_turing_host_semaphore_release: +# {0x20050017 (INC_METHOD count=5 addr=0x5c), sem_lo, sem_hi, payload, 0, exec} +# The group sits at push_va + entry_len - 24 = push_va + 0x580 (entry0 len 1432), +# with the header at +0x598 (entry1's 4-byte push). The push VA comes from the +# service's own entry[0] decode in dmesg. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap3.log" +MASK = ~0xfff & ((1 << 64) - 1) +WATCH_S = int(os.environ.get("W2_WATCH", "240")) +PUSH_VA_DEFAULT = 0x4480000 + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def push_va_now(): + # newest entry[0] line for the Vulkan client's channel + va = None + for ln in dmesg().splitlines(): + m = re.search(r"entry\[0\] h=0x([0-9a-f]+) eng=0 raw=\S+ va=0x([0-9a-f]+) len=(\d+)", ln) + if m: + va = int(m.group(2), 16) + return va + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<I", 0x20050017) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + lo, hi, pay, w4, w5 = struct.unpack_from("<5I", d, i + 4) + # data-word sanity: payload_hi==0, RELEASE op, sane payload + if w4 != 0: continue + if (w5 & 0x7) != 1: continue + if not (0 < pay < (1 << 24)): continue + sem = ((hi & 0xffffffff) << 32) | ((lo & 0x3fffffff) << 2) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, sem, pay, w5)) + pos += CH + return out + +def main(): + log("=== livecap3 start watch=%ds ===" % WATCH_S) + injected = {} + seen = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + hits = scan() + if hits: + for (page, off, sem, pay, w5) in hits: + key = (page, off) + if key in seen: continue + seen.add(key) + d = rd(page, 0x1000) + head = d[:0x20].hex() if d else "?" + log("hit page=0x%x off=0x%x sem=0x%x pay=%u exec=0x%08x head=%s" + % (page, off, sem, pay, w5, head)) + log(" @0x580: %s" % (d[0x580:0x5b0].hex() if d else "?")) + if off in (0x598, 0x580, 0x590): + pv = push_va_now() or PUSH_VA_DEFAULT + if pv not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (pv, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x" % (pv, page)) + injected[pv] = page + except OSError as e: + log(" inject FAIL", e) + time.sleep(0.5) + log("=== livecap3 done: %d injections ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_recscan.py b/tools/guest-diag/wall2/wall2_recscan.py new file mode 100644 index 0000000..7bc4e2c --- /dev/null +++ b/tools/guest-diag/wall2/wall2_recscan.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# wall2_recscan.py - scan the guest RAM for the RM's low-aperture window +# records and their memdesc PAs. +# +# Record layout (verified r685/r686): 0x40-byte stride +# {u64 tag; u64 VA; u64 SIZE; u64 0; u32 0x7461696c ("tail" as a u32 - +# on disk the BYTES are "liat"); u32 0} +# The tag is NOT the byte string "tail" - searching for b'tail' finds +# nothing; search for the 4-byte pattern 6c 69 61 74 (struct '<I' +# 0x7461696c) at record+0x20 and decode {tag,VA,SIZE} at -0x20/-0x18/-0x10. +# +# Usage: wall2_recscan.py [VA_prefix_or_full_VA] +# Prints every record and the size-matching memdesc {SIZE,0x03008000,PA} +# found in the same page, and dumps the first words of each memdesc page so +# semaphore windows can be identified by the release payload values. +import struct, sys + +RAM = "/dev/shm/vgpu-guest-ram" +TAG = struct.pack("<I", 0x7461696c) + +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) + +def gpa2off(pa): + if pa < 0x80000000: + return pa + if pa < 0x180000000: + return 0x80000000 + (pa - 0x100000000) + return None + +def main(): + want = None + if len(sys.argv) > 1: + want = int(sys.argv[1], 0) + img = open(RAM, "rb", buffering=0) + img.seek(0, 2); fsz = img.tell() + recs = [] + pos = 0; CH = 64 << 20 + while pos < fsz: + img.seek(pos); buf = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = buf.find(TAG, st) + if i < 0: + break + st = i + 1 + if i >= 0x20: + tag, va, sz, z = struct.unpack_from("<QQQQ", buf, i - 0x20) + if 0x4000000 <= va < 0x100000000 and \ + 0x1000 <= sz <= 0x1000000 and (sz & 0xfff) == 0 and z == 0: + if want is None or va == want or (va & ~0xfffff) == (want & ~0xfffff): + recs.append((off2gpa(pos + i - 0x20), tag, va, sz)) + pos += CH + for gpa, tag, va, sz in sorted(recs, key=lambda r: r[2]): + page = gpa & ~0xfff + img.seek(gpa2off(page)); d = img.read(0x1000) + descs = [] + for o in range(0, 0x1000 - 24, 8): + s2, f2 = struct.unpack_from("<QQ", d, o) + pa = struct.unpack_from("<Q", d, o + 16)[0] + if f2 == 0x03008000 and s2 == sz and 0x100000000 <= pa < 0x180000000: + descs.append(pa) + print("rec gpa=0x%x tag=0x%x va=0x%x size=0x%x" % (gpa, tag, va, sz)) + for pa in descs: + img.seek(gpa2off(pa)); pd = img.read(0x20) + print(" desc pa=0x%x head=%s" % (pa, pd.hex(" "))) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver2.py b/tools/guest-diag/wall2/wall2_resolver2.py new file mode 100644 index 0000000..785e5e2 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver2.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# wall2_resolver2.py - HOST-SIDE wall#2 resolver v2 (r674+). +# +# Why: the vk_frame client's channel-init push (va=0x4500000 len=1432) is +# kernel-built and kernel-only; the sema service cannot translate it +# ("false translation for push ... rescuing"), the client times out (~11 s) +# per attempt, retries, then VKFAIL -4. Guest RAM is host-readable, so the +# push page can be found and injected via sema_map_add. +# +# v2 recipe (per the r672/r673 structure analysis): +# 1. wait for the channel ring line in dmesg (validated RING_PA + handle) +# 2. scan guest RAM for the push page: +# a) CONTENT: the 24-byte release group the service's own rescue matches +# (hdr=0x20038090, hi=1, payload sane, w5/v5 shape). The channel-init +# entry (va=0x4500000 len=1432) puts that group at page offset 0x580. +# b) TUPLES: pages of 32-byte records {vmap_ptr; flags; PA; size} (both +# observed layouts) matched on (PA & ~0xfff) == RING_PA with strides +# 0x20/0x40/0x60; entries below the ring entry in VA order map +# 0x4500000.. (push VA = ring VA 0x4520000 - 0x20000 = 32 pages). +# 3. inject (va -> pa, len) through /sys/module/vgsp/parameters/sema_map_add +# 4. keep scanning/re-injecting for the watch window (retries allocate new +# pages) and log everything. +# +# Run this FIRST (foreground), then launch the client in the guest. +import subprocess, struct, time, sys, re, os, signal + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver2.log" +RING_VA = 0x4520000 +PUSH_VA = 0x4500000 +DELTA = (RING_VA - PUSH_VA) // 0x1000 # 32 pages +REL_OFF = 0x580 # group offset in the push page +MASK = ~0xfff & ((1 << 64) - 1) + +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "150")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,%d\n" % (va, pa, ln, ap)) + log("INJECT va=0x%x -> pa=0x%x len=0x%x" % (va, pa, ln)) + return True + except OSError as e: + log("inject FAIL", e) + return False + +def prefix_ok(d): + off = 0 + while off + 4 <= len(d): + hdr = struct.unpack_from("<I", d, off)[0] + if hdr: + break + off += 4 + if off + 4 > len(d): + return False + op = (hdr >> 29) & 7 + if op in (4, 7): + return True + if op not in (1, 3, 5): + return False + return ((hdr >> 16) & 0xfff) != 0 + +def rel_shape(g): + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + return (hdr == 0x20038090 and hi == 1 and 0 < pay < (1 << 24) + and (w5 >> 29) == 1 and ((w5 >> 13) & 7) == 4 + and ((w5 & 0x1fff) << 2) == 0x300 and (v5 & 3) == 0), lo & 0xfff, pay + +img = None +def open_ram(): + global img + if img is None: + img = open(RAM, "rb", buffering=0) + return img + +def gpa2off(pa): + if pa < 0x80000000: + return pa + if pa < 0x180000000: + return 0x80000000 + (pa - 0x100000000) + return None + +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) + +def rd(pa, n): + off = gpa2off(pa) + if off is None: + return None + f = open_ram() + try: + f.seek(off) + return f.read(n) + except OSError: + return None + +def find_content_candidates(): + """Pages containing the release group; returns [(page_pa, off, slot, pay)].""" + f = open_ram() + f.seek(0, 2) + fsz = f.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + f.seek(pos) + d = f.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: + break + st = i + 1 + if i + 24 > len(d): + continue + ok, slot, pay = rel_shape(d[i:i + 24]) + if not ok: + continue + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, slot, pay)) + pos += CH + return out + +def find_tuple_arrays(ring_pa): + """Find record pages containing the ring PA; returns list of + (page_gpa, stride, ring_index, entries[list of pa]).""" + f = open_ram() + f.seek(0, 2) + fsz = f.tell() + pat = struct.pack("<Q", ring_pa) + arrays = [] + CH = 64 << 20 + pos = 0 + hits = [] + while pos < fsz: + f.seek(pos) + d = f.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: + break + hits.append(off2gpa(pos + i)) + st = i + 1 + pos += CH + seen = set() + for gpa in hits: + page = gpa & MASK + if page in seen: + continue + seen.add(page) + d = rd(page, 0x1000) + if not d: + continue + rec_off = gpa - page + for stride in (0x20, 0x40, 0x60): + for rel in range(0, min(stride, rec_off + 1), 8): + start = rec_off - rel + ents = [] + for o in range(start, 0x1000 - stride + 1, stride): + # PA field anywhere in the record: try +0x10 first, then any qword + vals = struct.unpack_from("<%dQ" % (stride // 8), d, o) + pa = None + for cand_off in (0x10, 0x8, 0x18, 0x0, 0x20, 0x28): + if cand_off + 8 <= stride: + v = vals[cand_off // 8] + if (v & MASK) == ring_pa: + pa = v & MASK + break + if pa is None: + continue + ents.append((o, pa)) + if len(ents) >= 4: + idx = [e[1] for e in ents].index(ring_pa) + arrays.append((page, stride, start, idx, [e[1] for e in ents])) + break + if arrays and arrays[-1][0] == page: + break + return arrays + +def main(): + log("=== wall2_resolver2 start (wait_ring=%ss watch=%ss) ===" % (WAIT_RING_S, WATCH_S)) + ring_pa = None + ring_h = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + for ln in dmesg().splitlines(): + m = re.search(r"h=(0x[0-9a-f]+) .*ring rc=0 pa=0x([0-9a-f]+)", ln) + if m: + ring_h = m.group(1) + ring_pa = int(m.group(2), 16) + break + if ring_pa: + break + time.sleep(1) + if not ring_pa: + log("no ring line seen in %ss; exiting" % WAIT_RING_S) + return 1 + log("RING %s PA=0x%x" % (ring_h, ring_pa)) + + injected = set() + watch_deadline = time.time() + WATCH_S + last_pass = 0 + while time.time() < watch_deadline: + # ---- content pass + try: + cands = find_content_candidates() + except Exception as e: + log("content scan error:", e) + cands = [] + # prefer the canonical group offset 0x580, then highest payload + cands.sort(key=lambda c: (c[1] != REL_OFF, -c[3])) + if cands: + page, off, slot, pay = cands[0] + d = rd(page, 0x1000) + pok = bool(d) and prefix_ok(d[:0x100]) + log("content: %d candidate(s); best page=0x%x off=0x%x slot=0x%x pay=%u prefix_ok=%s" + % (len(cands), page, off, slot, pay, pok)) + key = ("C", PUSH_VA, page) + if key not in injected and pok: + if inject(PUSH_VA, page, 0x1000): + injected.add(key) + # also cover the following page (entry2 tail) if non-contiguous unknown; skip + else: + # ---- tuple fallback (only every ~10 s to save time) + if time.time() - last_pass > 10: + last_pass = time.time() + try: + arrs = find_tuple_arrays(ring_pa) + except Exception as e: + log("tuple scan error:", e) + arrs = [] + if arrs: + page, stride, start, idx, ents = arrs[0] + log("tuple: array page=0x%x stride=0x%x ring_index=%d entries=%d" + % (page, stride, idx, len(ents))) + for k in range(idx - DELTA, idx): + if k < 0 or k >= len(ents): + continue + va = PUSH_VA + (k - (idx - DELTA)) * 0x1000 + pa = ents[k] + key = ("T", va, pa) + if key in injected: + continue + d = rd(pa, 0x40) + if d and any(d): + inject(va, pa, 0x1000) + injected.add(key) + else: + log("tuple: no array with ring PA found") + # stop early when the service confirms our mapping + if len(injected) and "injected mapping va=0x%x" % PUSH_VA in dmesg(): + log("service confirmed the injected mapping; done") + break + time.sleep(1) + log("=== done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver3.py b/tools/guest-diag/wall2/wall2_resolver3.py new file mode 100644 index 0000000..4b16686 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver3.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# wall2_resolver3.py - HOST-SIDE wall#2 resolver v3: the RM window-memdesc recipe. +# +# Proven structure (r674/r675): the RM keeps per-window records +# {..., VA, SIZE, ...} (head/tail list nodes) +# and per-window memdescs in the same page: +# {SIZE, 0x03008000, FIRST_PA, ptr} +# where FIRST_PA is the PA of the window's FIRST VA page. Verified: the ring +# window {VA=0x44a0000, SIZE=0x40000} -> memdesc PA == the service's validated +# ring pa (0x1158ce000 r675). Therefore the push window +# {VA=0x4480000, SIZE=0x20000} -> memdesc PA == PA of VA 0x4480000 (the push +# page the sema service cannot translate). +# +# Recipe: wait for the ring line -> find the ring window memdesc +# {0x40000, 0x03008000, RING_PA} -> in the same page find the push window +# memdesc {0x20000, 0x03008000, PUSH_PA} -> inject 0x4480000 -> PUSH_PA. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver3.log" +PUSH_VA = 0x4480000 +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "150")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,%d\n" % (va, pa, ln, ap)) + log("INJECT va=0x%x -> pa=0x%x len=0x%x" % (va, pa, ln)) + return True + except OSError as e: + log("inject FAIL", e) + return False + +def find_memdescs(size): + """Find {size, 0x03008000, PA} patterns; returns [(page, off, pa)].""" + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", size, 0x03008000) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + pa = struct.unpack_from("<Q", d, i + 16)[0] + gpa = off2gpa(pos + i) + if 0x100000000 <= pa < 0x180000000: + out.append((gpa & ~0xfff, gpa - (gpa & ~0xfff), pa)) + pos += CH + return out + +def main(): + log("=== resolver3 start ===") + # count pre-existing ring lines; wait for a NEW one (fresh client kick) + def ring_lines(): + out = [] + for ln in dmesg().splitlines(): + m = re.search(r"ring rc=0 pa=0x([0-9a-f]+) ap=", ln) + if m: + out.append(int(m.group(1), 16)) + return out + base = ring_lines() + log("pre-existing ring lines: %d" % len(base)) + ring_pa = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + cur = ring_lines() + if len(cur) > len(base): + ring_pa = cur[-1] + break + time.sleep(1) + if not ring_pa: + log("no NEW ring line; exit"); return 1 + log("NEW RING PA = 0x%x" % ring_pa) + + injected = set() + watch = time.time() + WATCH_S + while time.time() < watch: + # ring window memdesc {0x40000, 0x03008000, ring_pa} + rd_hits = [x for x in find_memdescs(0x40000) if x[2] == ring_pa] + if not rd_hits: + time.sleep(0.5); continue + for (page, off, _pa) in rd_hits: + d = rd(page, 0x1000) + if not d: continue + # find sibling push memdesc in the same page + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if sz == 0x20000 and fl == 0x03008000: + pa = struct.unpack_from("<Q", d, o + 16)[0] + if not (0x100000000 <= pa < 0x180000000): + continue + key = (page, pa) + if key in injected: continue + log("memdesc page=0x%x ring_desc@0x%x push_desc@0x%x push_pa=0x%x" + % (page, off, o, pa)) + # content sanity: read the push page + pd = rd(pa, 0x40) + log(" push page head: %s" % (pd.hex() if pd else "?")) + if inject(PUSH_VA, pa, 0x1000): + injected.add(key) + if injected and ("injected mapping va=0x%x" % PUSH_VA) in dmesg(): + log("service confirmed; keep watching") + time.sleep(0.5) + log("=== resolver3 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver4.py b/tools/guest-diag/wall2/wall2_resolver4.py new file mode 100644 index 0000000..140aaa7 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver4.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# wall2_resolver4.py - HOST-SIDE wall#2 resolver v4 (final recipe). +# +# Association proven on r675: the RM keeps, in ONE page, the client's channel +# window records and their memdescs: +# push window record {VA=0x4480000, SIZE=0x20000} +# ring window record {VA=0x44a0000, SIZE=0x40000} +# push memdesc {0x20000, 0x03008000, FIRST_PA, ptr} +# ring memdesc {0x40000, 0x03008000, FIRST_PA, ptr} +# where FIRST_PA is the PA of the window's FIRST VA page. Verified: the ring +# memdesc's PA == the service's validated ring pa. So the push memdesc's PA == +# the PA of VA 0x4480000 (the push page the service cannot translate). +# +# Recipe: wait for a NEW ring line -> find the page containing the ring window +# record {0x44a0000, 0x40000} -> require the sibling push window record +# {0x4480000, 0x20000} and the memdescs in the same page -> cross-check the +# ring memdesc PA == ring_pa -> inject 0x4480000 -> push memdesc PA (sudo). +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver4.log" +PUSH_VA = 0x4480000 +RING_VA = 0x44a0000 +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "180")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x len=0x%x OK" % (va, pa, ln)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + """Find qword pairs {a,b}; returns [(page, off)].""" + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + gpa = off2gpa(pos + i) + out.append((gpa & ~0xfff, gpa - (gpa & ~0xfff))) + pos += CH + return out + +def scan_page_descs(page): + d = rd(page, 0x1000) + if not d: return [] + out = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if fl != 0x03008000: continue + if sz not in (0x20000, 0x40000): continue + pa = struct.unpack_from("<Q", d, o + 16)[0] + if 0x100000000 <= pa < 0x180000000: + out.append((sz, o, pa)) + return out + +def main(): + log("=== resolver4 start ===") + def ring_lines(): + return re.findall(r"ring rc=0 pa=0x([0-9a-f]+) ap=", dmesg()) + base = ring_lines() + log("pre-existing ring lines: %d" % len(base)) + ring_pa = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + cur = ring_lines() + if len(cur) > len(base): + ring_pa = int(cur[-1], 16) + break + time.sleep(1) + if not ring_pa: + log("no NEW ring line; exit"); return 1 + log("NEW RING PA = 0x%x" % ring_pa) + + injected = set() + LEARNED = [0x101e39000, 0x102a62000] # RM window-record pages (stable across runs) + def try_page(page): + descs = scan_page_descs(page) + rdesc = [x for x in descs if x[0] == 0x40000] + pdesc = [x for x in descs if x[0] == 0x20000] + if not rdesc or not pdesc: + return False + if rdesc[0][2] != ring_pa: + return False + ok = False + for (_sz, o, pa) in pdesc: + key = (page, o) + if key in injected: continue + pd = rd(pa, 0x40) + log("page=0x%x ring_desc@0x%x(pa=0x%x OK) push_desc@0x%x pa=0x%x head=%s" + % (page, rdesc[0][1], rdesc[0][2], o, pa, + pd.hex() if pd else "?")) + if inject(PUSH_VA, pa, 0x1000): + injected.add(key); ok = True + return ok + log("learned pages: %s" % [hex(p) for p in LEARNED]) + watch = time.time() + WATCH_S + last_full = 0 + while time.time() < watch: + hit = False + for page in LEARNED: + if try_page(page): + hit = True + if not hit and time.time() - last_full > 5: + last_full = time.time() + for (page, _off) in find_qpair(RING_VA, 0x40000): + if try_page(page): + if page not in LEARNED: + LEARNED.append(page) + hit = True + time.sleep(0.25) + log("=== resolver4 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver5.py b/tools/guest-diag/wall2/wall2_resolver5.py new file mode 100644 index 0000000..faec902 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver5.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# wall2_resolver5.py - final: continuous scan, all-candidate injection. +# +# Structure (r675, verified): an RM page holds the client channel's window +# records and memdescs: +# {0x4480000, 0x20000} push window record +# {0x44a0000, 0x40000} ring window record +# {0x20000, 0x03008000, PUSH_PA} push window memdesc +# {0x40000, 0x03008000, RING_PA} ring window memdesc (PA == validated ring pa) +# The resolver scans CONTINUOUSLY (no ring-line gate), finds pages holding the +# pair of window records, and injects 0x4480000 -> every 0x20000 memdesc PA in +# the same page (last write wins in the service; content check arbitrates). +import subprocess, struct, time, sys, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver5.log" +PUSH_VA = 0x4480000 +RING_VA = 0x44a0000 +WATCH_S = int(os.environ.get("W2_WATCH", "600")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x" % (va, pa)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + gpa = off2gpa(pos + i) + out.append(gpa & ~0xfff) + pos += CH + return out + +def main(): + log("=== resolver5 start (watch=%ds) ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + try: + push_pages = set(find_qpair(PUSH_VA, 0x20000)) + ring_pages = set(find_qpair(RING_VA, 0x40000)) + except Exception as e: + log("scan error:", e); time.sleep(2); continue + for page in sorted(push_pages & ring_pages): + d = rd(page, 0x1000) + if not d: continue + # collect 0x20000 descs in this page + cands = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if sz == 0x20000 and fl == 0x03008000: + pa = struct.unpack_from("<Q", d, o + 16)[0] + if 0x100000000 <= pa < 0x180000000: + cands.append((o, pa)) + for (o, pa) in cands: + key = (page, o) + if key in injected: continue + pd = rd(pa, 0x40) + nz = sum(1 for b in pd if b) if pd else -1 + log("page=0x%x desc@0x%x pa=0x%x live_nz=%d head=%s" + % (page, o, pa, nz, pd.hex()[:64] if pd else "?")) + if inject(PUSH_VA, pa): + injected.add(key) + time.sleep(1) + log("=== resolver5 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver6.py b/tools/guest-diag/wall2/wall2_resolver6.py new file mode 100644 index 0000000..4653efa --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver6.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# wall2_resolver6.py - resolver5 logic for BOTH current push windows: +# pair A: push 0x4480000 (0x20000) + ring 0x44a0000 (0x40000) +# pair B: push 0x4ca0000 (0x20000) + ring 0x4cc0000 (0x40000) +# Scans guest RAM continuously; injects PUSH_VA -> every 0x20000 memdesc PA +# found in a page holding both window records. +import subprocess, struct, time, sys, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver6.log" +WATCH_S = int(os.environ.get("W2_WATCH", "420")) +PAIRS = [(0x4480000, 0x44a0000), (0x4ca0000, 0x4cc0000)] + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x" % (va, pa)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + out.append(off2gpa(pos + i) & ~0xfff) + pos += CH + return out + +def main(): + log("=== resolver6 start (watch=%ds) ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + for (pva, rva) in PAIRS: + try: + pages = set(find_qpair(pva, 0x20000)) & set(find_qpair(rva, 0x40000)) + except Exception as e: + log("scan error:", e); continue + for page in sorted(pages): + d = rd(page, 0x1000) + if not d: continue + cands = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if sz == 0x20000 and fl == 0x03008000: + pa = struct.unpack_from("<Q", d, o + 16)[0] + if 0x100000000 <= pa < 0x180000000: + cands.append((o, pa)) + for (o, pa) in cands: + key = (pva, page, o) + if key in injected: continue + pd = rd(pa, 0x40) + nz = sum(1 for b in pd if b) if pd else -1 + log("pair=0x%x page=0x%x desc@0x%x pa=0x%x live_nz=%d head=%s" + % (pva, page, o, pa, nz, pd.hex()[:64] if pd else "?")) + if inject(pva, pa): + injected.add(key) + time.sleep(1) + log("=== resolver6 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver7.py b/tools/guest-diag/wall2/wall2_resolver7.py new file mode 100644 index 0000000..5d9e58e --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver7.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# wall2_resolver7.py - resolver6 + release-window injection + live-first ordering. +# Fixes the resolver6 zero-page overwrite: candidates are injected good-first +# (live_nz>0) and zero pages last, so the readable mapping wins by order if the +# service keeps the newest write. +# New: the GR init release windows (0x4b16000 size 0x1000 from the record page, +# and any {VA,0x1000} record alongside) are injected from the size-0x1000 +# memdescs of the same page. +import subprocess, struct, time, sys, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver7.log" +WATCH_S = int(os.environ.get("W2_WATCH", "420")) +PAIRS = [(0x4480000, 0x44a0000), (0x4ca0000, 0x4cc0000)] +REL_WINS = [0x4b16000] + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def off2gpa(off): return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x" % (va, pa)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + out.append(off2gpa(pos + i) & ~0xfff) + pos += CH + return out + +def collect(page): + d = rd(page, 0x1000) + if not d: return None + descs = [] + wins = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + pa = struct.unpack_from("<Q", d, o + 16)[0] + if fl == 0x03008000 and 0x1000 <= sz <= 0x400000 and \ + 0x100000000 <= pa < 0x180000000: + descs.append((o, sz, pa)) + va2 = sz + if 0x4000000 <= va2 < 0x100000000 and (va2 & 0xfff) == 0: + sz2 = fl + if 0x1000 <= sz2 <= 0x1000000 and (sz2 & 0xfff) == 0: + wins.append((va2, sz2)) + return descs, wins + +def main(): + log("=== resolver7 start (watch=%ds) ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + pages = set() + for (pva, rva) in PAIRS: + try: + pages |= set(find_qpair(pva, 0x20000)) & set(find_qpair(rva, 0x40000)) + except Exception as e: + log("scan error:", e) + for page in sorted(pages): + got = collect(page) + if not got: continue + descs, wins = got + # push windows: inject live-first (nz>0 first, zeros last) + for (pva, _rva) in PAIRS: + cands = [(o, sz, pa) for (o, sz, pa) in descs if sz == 0x20000] + order = [] + for (o, sz, pa) in cands: + pd = rd(pa, 0x40) + nz = sum(1 for b in pd if b) if pd else -1 + log("pair=0x%x page=0x%x desc@0x%x pa=0x%x live_nz=%d" % (pva, page, o, pa, nz)) + order.append((nz, o, pa)) + order.sort(key=lambda t: t[0]) # zeros first, live last + for (nz, o, pa) in order: + key = (pva, page, o) + if key in injected: continue + if inject(pva, pa): + injected.add(key) + # release windows: any {VA, 0x1000} record on the page -> size-0x1000 descs + for (va, sz) in wins: + if va not in REL_WINS and not (0x4b00000 <= va <= 0x4b20000): + continue + for (o, dsz, pa) in descs: + if dsz != sz: + continue + key = (va, page, o) + if key in injected: continue + if inject(va, pa): + injected.add(key) + time.sleep(1) + log("=== resolver7 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/xgrab.c b/tools/guest-diag/xgrab.c new file mode 100644 index 0000000..3f4ec76 --- /dev/null +++ b/tools/guest-diag/xgrab.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-3-Clause +// xgrab.c - capture the X screen (root window) into an append-only frame file. +// Pure client-side XGetImage on a software X server (Xvfb): no GL, no driver, +// no present-path hooks. Same record format as sdl_cap.c: +// [0] 8B "GZCAPV01" + zero pad to 4096 +// [4096..] records: 4B "GZF2" | u32 w | u32 h | u32 stride | u32 bpp | +// u64 ts_ns | u64 seq | stride*h bytes (top-down BGRA) +// Trailing record seq=0xFFFFFFFFFFFFFFFF on clean stop. +// Usage: xgrab <display> <out> <maxframes|0> <pace_us|0> +#include <X11/Xlib.h> +#include <X11/Xutil.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +int main(int argc, char **argv) +{ + const char *dpy = argc > 1 ? argv[1] : ":1"; + const char *out = argc > 2 ? argv[2] : "/dev/shm/gzframes.bin"; + long maxf = argc > 3 ? atol(argv[3]) : 0; + long pace = argc > 4 ? atol(argv[4]) : 0; + Display *d; + Window root; + XWindowAttributes wa; + FILE *f; + char hdr[4096]; + uint64_t seq = 0; + + d = XOpenDisplay(dpy); + if (!d) { + fprintf(stderr, "xgrab: cannot open display %s\n", dpy); + return 1; + } + root = DefaultRootWindow(d); + if (!XGetWindowAttributes(d, root, &wa)) { + fprintf(stderr, "xgrab: no root attrs\n"); + return 1; + } + fprintf(stderr, "xgrab: root %dx%d depth=%d\n", wa.width, wa.height, + wa.depth); + f = fopen(out, "wb"); + if (!f) { + perror("xgrab: open out"); + return 1; + } + memset(hdr, 0, sizeof(hdr)); + memcpy(hdr, "GZCAPV01", 8); + fwrite(hdr, 1, sizeof(hdr), f); + fflush(f); + + while (!maxf || (long)seq < maxf) { + XImage *im = XGetImage(d, root, 0, 0, wa.width, wa.height, + AllPlanes, ZPixmap); + struct timespec ts; + uint64_t tns; + uint32_t rh[4]; + char m[4] = { 'G', 'Z', 'F', '2' }; + int y; + + if (!im) { + usleep(10000); + continue; + } + if (im->bits_per_pixel != 32) { + fprintf(stderr, "xgrab: unexpected bpp %d\n", + im->bits_per_pixel); + XDestroyImage(im); + break; + } + clock_gettime(CLOCK_MONOTONIC, &ts); + tns = (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; + rh[0] = (uint32_t)wa.width; + rh[1] = (uint32_t)wa.height; + rh[2] = (uint32_t)wa.width * 4; + rh[3] = 32; + fwrite(m, 1, 4, f); + fwrite(rh, 4, 4, f); + fwrite(&tns, 8, 1, f); + fwrite(&seq, 8, 1, f); + for (y = 0; y < wa.height; y++) + fwrite(im->data + (size_t)y * im->bytes_per_line, 1, + (size_t)wa.width * 4, f); + fflush(f); + seq++; + XDestroyImage(im); + if (pace) + usleep(pace); + if ((seq & 63) == 0) + fprintf(stderr, "xgrab: frames=%llu\n", + (unsigned long long)seq); + } + { + char m[4] = { 'G', 'Z', 'F', '2' }; + uint64_t end = 0xFFFFFFFFFFFFFFFFull; + + fwrite(m, 1, 4, f); + fwrite(&end, 8, 1, f); + fflush(f); + } + fclose(f); + XCloseDisplay(d); + fprintf(stderr, "xgrab: done frames=%llu\n", (unsigned long long)seq); + return 0; +} diff --git a/tools/guest-diag/xorg-vgpu.conf b/tools/guest-diag/xorg-vgpu.conf new file mode 100644 index 0000000..4d4c07d --- /dev/null +++ b/tools/guest-diag/xorg-vgpu.conf @@ -0,0 +1,31 @@ +# Minimal Xorg config for the emulated vGPU (nvidia driver), headless. +# AllowEmptyInitialConfiguration lets the X server start with no connected +# display; UseDisplayDevice "None" keeps the server from demanding one. +Section "ServerLayout" + Identifier "Layout0" + Screen 0 "Screen0" +EndSection + +Section "Device" + Identifier "Device0" + Driver "nvidia" + BusID "PCI:1:0:0" + Option "AllowEmptyInitialConfiguration" "True" + Option "UseDisplayDevice" "None" + Option "ConnectToAcpid" "False" +EndSection + +Section "Screen" + Identifier "Screen0" + Device "Device0" + DefaultDepth 24 + SubSection "Display" + Depth 24 + Virtual 1280 720 + EndSubSection +EndSection + +Section "ServerFlags" + Option "AutoAddDevices" "False" + Option "AutoEnableDevices" "False" +EndSection
\ No newline at end of file diff --git a/tools/guest-diag/xresize.c b/tools/guest-diag/xresize.c new file mode 100644 index 0000000..877f0e9 --- /dev/null +++ b/tools/guest-diag/xresize.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause +// xresize.c - find a window by name substring and resize/move it (Xvfb has +// no WM, so a client-side XMoveResizeWindow is the only way to force the +// game window onto the screen). +// Usage: xresize <display> <name-substr> <w> <h> <x> <y> +#include <X11/Xlib.h> +#include <X11/Xutil.h> +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +static Window find_named(Display *d, Window root, const char *sub) +{ + Window r, parent, *kids = NULL; + unsigned int nkids = 0; + + if (!XQueryTree(d, root, &r, &parent, &kids, &nkids)) + return 0; + for (unsigned int i = 0; i < nkids; i++) { + char *name = NULL; + + if (XFetchName(d, kids[i], &name) && name) { + int hit = strstr(name, sub) != NULL; + + XFree(name); + if (hit) { + Window w = kids[i]; + + if (kids) + XFree(kids); + return w; + } + } + } + if (kids) + XFree(kids); + return 0; +} + +int main(int argc, char **argv) +{ + Display *d; + Window w; + + if (argc < 6) { + fprintf(stderr, "usage: xresize <display> <name> <w> <h> <x> <y>\n"); + return 2; + } + d = XOpenDisplay(argv[1]); + if (!d) { + fprintf(stderr, "no display\n"); + return 1; + } + w = find_named(d, DefaultRootWindow(d), argv[2]); + if (!w) { + fprintf(stderr, "window '%s' not found\n", argv[2]); + return 1; + } + XMoveResizeWindow(d, w, atoi(argv[5]), atoi(argv[6]), atoi(argv[3]), + atoi(argv[4])); + XFlush(d); + printf("resized 0x%lx to %sx%s at %s,%s\n", w, argv[3], argv[4], + argv[5], argv[6]); + XCloseDisplay(d); + return 0; +} diff --git a/tools/guest-diag/xshm_pub.c b/tools/guest-diag/xshm_pub.c new file mode 100644 index 0000000..98fb539 --- /dev/null +++ b/tools/guest-diag/xshm_pub.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * xshm_pub.c - capture an X11 screen and publish frames into the kvmfr + * ivshmem BAR so the host can display them (the Looking-Glass-style path: + * the guest renders with its real graphics stack - X on the passed GPU / + * GLX / Vulkan surface - and this tool copies the framebuffer into the + * shared BAR; the host reader decodes it). + * + * Layout in the BAR (same as tools/cuda-ab/frame_producer.cu): + * offset 0x100000: struct frame_hdr { magic, version, width, height, + * stride, format, seq } then width*height*4 RGBA8 bytes. + * + * Usage (guest, with DISPLAY set): + * ./xshm_pub [frames] [fps] [x] [y] [w] [h] + * Prints FRAME lines (first 5 and every 30th) then FRAME_DONE. + * Build (host, same glibc): gcc -O2 -o xshm_pub xshm_pub.c -lX11 -lXext + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <time.h> +#include <X11/Xlib.h> +#include <X11/Xutil.h> +#include <X11/extensions/XShm.h> +#include <sys/ipc.h> +#include <sys/shm.h> + +#define FRAME_OFFSET 0x100000UL +#define HDR_MAGIC 0x4b564d46u /* "FMVK" */ + +struct frame_hdr { + uint32_t magic; + uint32_t version; + uint32_t width; + uint32_t height; + uint32_t stride; + uint32_t format; /* 0 = RGBA8 */ + uint32_t seq; + uint32_t reserved; + uint64_t timestamp_ns; +}; + +static uint64_t now_ns(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + ts.tv_nsec; +} + +int main(int argc, char **argv) +{ + int frames = argc > 1 ? atoi(argv[1]) : 120; + int fps = argc > 2 ? atoi(argv[2]) : 10; + int gx = argc > 3 ? atoi(argv[3]) : 0; + int gy = argc > 4 ? atoi(argv[4]) : 0; + int gw = argc > 5 ? atoi(argv[5]) : 0; + int gh = argc > 6 ? atoi(argv[6]) : 0; + const char *bar_path = + "/sys/bus/pci/devices/0000:00:03.0/resource2"; + Display *dpy; + Window root; + XImage *img = NULL; + XShmSegmentInfo shm; + int use_shm = 1, fd, i, w, h; + unsigned char *bar; + struct frame_hdr *hdr; + volatile uint32_t *seqp; + uint64_t t_first = 0; + uint64_t t_last = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + + dpy = XOpenDisplay(NULL); + if (!dpy) { + printf("XOPEN_FAIL DISPLAY=%s\n", + getenv("DISPLAY") ? getenv("DISPLAY") : "(unset)"); + return 1; + } + root = DefaultRootWindow(dpy); + { + XWindowAttributes wa; + + XGetWindowAttributes(dpy, root, &wa); + w = gw > 0 ? gw : wa.width; + h = gh > 0 ? gh : wa.height; + printf("XSCREEN %dx%d root=%dx%d\n", w, h, wa.width, + wa.height); + } + fd = open(bar_path, O_RDWR | O_SYNC); + if (fd < 0) { + printf("OPEN_FAIL %s\n", bar_path); + return 1; + } + bar = (unsigned char *)mmap(NULL, 128UL << 20, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (bar == MAP_FAILED) { + printf("MMAP_FAIL\n"); + return 1; + } + hdr = (struct frame_hdr *)(bar + FRAME_OFFSET); + seqp = &hdr->seq; + + memset(&shm, 0, sizeof(shm)); + if (use_shm) { + img = XShmCreateImage(dpy, DefaultVisual(dpy, + DefaultScreen(dpy)), + DefaultDepth(dpy, DefaultScreen(dpy)), + ZPixmap, NULL, &shm, w, h); + if (!img) { + printf("XSHM_CREATE_FAIL\n"); + return 1; + } + shm.shmid = shmget(IPC_PRIVATE, + (size_t)img->bytes_per_line * h, + IPC_CREAT | 0600); + if (shm.shmid < 0) { + printf("SHMGET_FAIL\n"); + return 1; + } + shm.shmaddr = img->data = shmat(shm.shmid, NULL, 0); + if (shm.shmaddr == (char *)-1) { + printf("SHMAT_FAIL\n"); + return 1; + } + shm.readOnly = False; + if (!XShmAttach(dpy, &shm)) { + printf("XSHM_ATTACH_FAIL (falling back)\n"); + XDestroyImage(img); + img = NULL; + } else { + XSync(dpy, False); + printf("XSHM OK %dx%d bpl=%d\n", w, h, + img->bytes_per_line); + } + } + if (!img) { + img = XGetImage(dpy, root, gx, gy, w, h, AllPlanes, ZPixmap); + if (!img) { + printf("XGETIMAGE_FAIL\n"); + return 1; + } + printf("XGETIMAGE OK %dx%d bpl=%d\n", w, h, + img->bytes_per_line); + } + + hdr->magic = HDR_MAGIC; + hdr->version = 1; + hdr->width = (uint32_t)w; + hdr->height = (uint32_t)h; + hdr->stride = (uint32_t)w * 4; + hdr->format = 0; + + for (i = 0; i < frames; i++) { + uint64_t t0 = now_ns(); + unsigned char *dst = bar + FRAME_OFFSET + + sizeof(struct frame_hdr); + int x, y; + + if (shm.shmaddr) { + XShmGetImage(dpy, root, img, gx, gy, AllPlanes); + } else { + XImage *n = XGetImage(dpy, root, gx, gy, w, h, + AllPlanes, ZPixmap); + + if (!n) + break; + XDestroyImage(img); + img = n; + } + /* X ZPixmap: 0x00RRGGBB little-endian -> RGBA bytes */ + for (y = 0; y < h; y++) { + unsigned char *s = (unsigned char *)img->data + + (size_t)y * img->bytes_per_line; + unsigned char *d = dst + (size_t)y * w * 4; + + for (x = 0; x < w; x++) { + uint32_t p; + + memcpy(&p, s + 4 * x, 4); + d[4 * x + 0] = (unsigned char)(p >> 16); + d[4 * x + 1] = (unsigned char)(p >> 8); + d[4 * x + 2] = (unsigned char)(p); + d[4 * x + 3] = 0xff; + } + } + hdr->timestamp_ns = now_ns(); + if (!t_first) + t_first = t0; + t_last = t0; + __sync_synchronize(); + *seqp = (uint32_t)(i + 1); + __sync_synchronize(); + if (i < 5 || (i % 30) == 29) + printf("FRAME seq=%d ms=%.1f\n", i + 1, + (double)(now_ns() - t0) / 1e6); + if (fps > 0) + usleep((useconds_t)(1000000 / fps)); + } + if (t_first && t_last > t_first) + printf("XPUB_RATE fps=%.2f frames=%d\n", + (double)(i - 1) * 1e9 / + (double)(t_last - t_first), i); + printf("FRAME_DONE frames=%d %dx%d\n", i, w, h); + if (shm.shmaddr) + XShmDetach(dpy, &shm); + XCloseDisplay(dpy); + return 0; +}
\ No newline at end of file diff --git a/tools/guest-diag/xtest_move.c b/tools/guest-diag/xtest_move.c new file mode 100644 index 0000000..a156bf8 --- /dev/null +++ b/tools/guest-diag/xtest_move.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-3-Clause +// xtest_move.c - drive the game with XTEST: hold forward + fire and tap +// left/right periodically so the captured footage has constant motion. +// Usage: xtest_move <display> <seconds> +#include <X11/Xlib.h> +#include <X11/keysym.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +extern int XTestFakeKeyEvent(Display *, unsigned int, int, unsigned long); + +int main(int argc, char **argv) +{ + Display *d = XOpenDisplay(argc > 1 ? argv[1] : ":1"); + int secs = argc > 2 ? atoi(argv[2]) : 12; + KeyCode kFwd, kFire, kLeft, kRight; + int t; + + if (!d) { + fprintf(stderr, "xtest_move: no display\n"); + return 1; + } + /* No WM on Xvfb: give the game window input focus ourselves. */ + { + Window r, parent, *kids = NULL; + unsigned int nk = 0; + + if (XQueryTree(d, DefaultRootWindow(d), &r, &parent, &kids, &nk)) { + unsigned int i; + + for (i = 0; i < nk; i++) { + char *nm = NULL; + + if (XFetchName(d, kids[i], &nm) && nm) { + int hit = strstr(nm, "Freedoom") || strstr(nm, "Doom") || + strstr(nm, "gzdoom") || strstr(nm, "Outer"); + + XFree(nm); + if (hit) { + XSetInputFocus(d, kids[i], RevertToParent, CurrentTime); + fprintf(stderr, "xtest_move: focused 0x%lx\n", + (unsigned long)kids[i]); + break; + } + } + } + if (kids) + XFree(kids); + } + } + kFwd = XKeysymToKeycode(d, XK_Up); + kFire = XKeysymToKeycode(d, XK_space); + kLeft = XKeysymToKeycode(d, XK_Left); + kRight = XKeysymToKeycode(d, XK_Right); + fprintf(stderr, "xtest_move: fwd=%d fire=%d left=%d right=%d\n", + kFwd, kFire, kLeft, kRight); + XTestFakeKeyEvent(d, kFwd, 1, 0); + XTestFakeKeyEvent(d, kFire, 1, 0); + XFlush(d); + /* Continuous motion: rotate steadily (hold a turn key ~0.8 s, then + * the other) while forward+fire stay held - every frame changes. */ + for (t = 0; t < secs * 10; t++) { + KeyCode turn = (t & 8) ? kLeft : kRight; + Window wr, wpar, *wk = NULL; + unsigned int wnk = 0; + + /* Re-assert focus every tick: Xvfb has no WM, and any new X client + * (the capture tool) can leave the game unfocused - GZDoom pauses + * the game the moment SDL reports focus loss. */ + if (XQueryTree(d, DefaultRootWindow(d), &wr, &wpar, &wk, &wnk)) { + unsigned int i; + + for (i = 0; i < wnk; i++) { + char *nm = NULL; + + if (XFetchName(d, wk[i], &nm) && nm) { + int hit = strstr(nm, "Freedoom") || strstr(nm, "Outer") || + strstr(nm, "Doom"); + + XFree(nm); + if (hit) { + XSetInputFocus(d, wk[i], RevertToParent, + CurrentTime); + break; + } + } + } + if (wk) + XFree(wk); + } + XTestFakeKeyEvent(d, kLeft, 0, 0); + XTestFakeKeyEvent(d, kRight, 0, 0); + XTestFakeKeyEvent(d, turn, 1, 0); + XFlush(d); + usleep(100000); + } + XTestFakeKeyEvent(d, kLeft, 0, 0); + XTestFakeKeyEvent(d, kRight, 0, 0); + XTestFakeKeyEvent(d, kFwd, 0, 0); + XTestFakeKeyEvent(d, kFire, 0, 0); + XFlush(d); + XCloseDisplay(d); + fprintf(stderr, "xtest_move: done\n"); + return 0; +} |