diff options
| author | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-11-16 09:50:31 -0500 |
|---|---|---|
| committer | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-11-16 09:50:31 -0500 |
| commit | ed8a9c6b8e061fe772fc0050ab83c2dec2db8d1e (patch) | |
| tree | 458d2ea063d8a8a6996cd15891e3242cfe93ece5 /tools/guest-diag/run_cycles_long.sh | |
| parent | f2c73f4e52d24188a9edb0e8190c50f87f19181f (diff) | |
| download | open-iov-ed8a9c6b8e061fe772fc0050ab83c2dec2db8d1e.tar.gz open-iov-ed8a9c6b8e061fe772fc0050ab83c2dec2db8d1e.zip | |
vgsp: eng89 - live-derived GS map, draw-move anchor, raster enables, async capture
Kernel (deterministic fixes, each with an A/B knob):
- sema_gl_sph_gs_maps=6: derive the whole 216-bit GS IMAP/OMAP blocks
from the bound VTG image's OMAP and the PS image's IMAP at authoring
time (SPH-GS-MAP receipt; SPH-GS-SKIP names ps-invalid/gs-invalid/
vtg-overlap/vtg-short). maps=4 stays as the fallback.
- DS-MAP census line: dump stage-4/5 w5/w10/w13/w17 next to DS-IMAGE so
the fetched maps are visible without a new probe.
- MME pack carrier: reject maxa > 4096 (the IR pack array) instead of
64K, with MME-PACK-REJECT reason=bound.
- MME-DRAW-MOVE: anchor the slot-34 move after a 0x3810 VERTEX_STREAM
run. c848 moved CALL(34) to 0x204, ahead of the prepended stream bind,
so the 8x3 fetch used stale stream state.
- sema_gl_raster_enable=1: emit SET_RASTER_ENABLE=1 and
SET_GLOBAL_RENDER_ENABLE MODE_TRUE in the raster prelude (fresh replay
contexts may carry 0 for both, discarding every fragment).
Guest tooling (unblocks the render loop and capture):
- sdl_cap.c: asynchronous PBO+fence capture with a 5 s abandonment
bound; no glFinish/blocking readback (c899 wedged gzdoom in an
uninterruptible nv ioctl); clean sentinel from a destructor.
- gz_pbuf.c: request depth 24/stencil 8, log GL_DEPTH_BITS, and swap
with glFlush only.
- mission-gz.sh: gl_frames probe off by default; GZDoom launched
detached with a bounded wait, per-attempt X restart and retry;
capture in /dev/shm with GZCAP_MAXF.
- gzdoom_pbuf_run.sh: gl_pipeline_depth=2, gl_finishbeforeswap=0,
GZCAP_MAXF default, rc-file support for the detached run.
- run_cycles_long.sh: durable multi-cycle runner (the previous one lived
in /tmp and was lost on reboot); pushes tools, runs the mission, grabs
dmesg + guest logs + capture per cycle, stops on pixels.
Test matrix: params-r1624-maps{4,6}.txt.
Diffstat (limited to 'tools/guest-diag/run_cycles_long.sh')
| -rwxr-xr-x | tools/guest-diag/run_cycles_long.sh | 142 |
1 files changed, 142 insertions, 0 deletions
diff --git a/tools/guest-diag/run_cycles_long.sh b/tools/guest-diag/run_cycles_long.sh new file mode 100755 index 0000000..bf14ca2 --- /dev/null +++ b/tools/guest-diag/run_cycles_long.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Open-IOV contributors +# run_cycles_long.sh - durable multi-cycle batch runner for the GZDoom +# GPU-video campaign. For each tag: cycle-r1780 (rmmod/insmod + mdev + +# QEMU + GR-lane arm), push the GL/GZDoom guest tools, run mission-gz.sh +# detached in the guest, wait for /tmp/mission-done, then collect host +# dmesg + guest logs + any swap capture into the evidence dir. +# +# Usage: tools/guest-diag/run_cycles_long.sh c895 c896 c897 +# Env: GZ_SECS (default 120) guest GZDoom seconds +# GLF_FRAMES(default 2) gl_frames probe frames +# MODULE (default kernel/vgsp.ko) +# INSMOD_PARAMS (default params-r1624.txt) +# STOP_ON px|clean247|none (default px: stop at first RT pixel) +# MAX_FETCH_MB (default 512) skip gzframes.bin fetches above this +# +# The runner never reboots and never runs two QEMUs at once. Everything is +# appended to /tmp/opencode/batch-<stamp>.log and per-tag files under +# backups/evidence-20261002-victory/r1780/. +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +BACKUP="$REPO/backups/evidence-20261002-victory/r1780" +VMEXEC="$REPO/scripts/vgpu_vm.py" +MODULE="${MODULE:-$REPO/kernel/vgsp.ko}" +INSMOD_PARAMS="${INSMOD_PARAMS:-$HERE/params-r1624.txt}" +GZ_SECS="${GZ_SECS:-120}" +GLF_FRAMES="${GLF_FRAMES:-2}" +STOP_ON="${STOP_ON:-px}" +MAX_FETCH_MB="${MAX_FETCH_MB:-512}" +STAMP="$(date +%Y%m%d-%H%M%S)" +SUMMARY="/tmp/opencode/batch-$STAMP.log" +mkdir -p "$BACKUP" /tmp/opencode + +say() { echo "[$(date +%T)] $*" | tee -a "$SUMMARY"; } + +gexec() { # gexec <timeout> <cmd> + local t="$1"; shift + timeout $((t + 15)) sudo python3 "$VMEXEC" exec --timeout "$t" "$*" 2>/dev/null +} + +guest_done() { + local out + out=$(gexec 20 'cat /tmp/mission-done 2>/dev/null') + case "$out" in + *DONE*|*X-FAIL*) echo "$out" | tail -1; return 0 ;; + esac + return 1 +} + +analyze_tag() { # analyze_tag <tag> <dmesg-file> <gl-log> <gz-log> + local tag="$1" dm="$2" gl="$3" gz="$4" + local m247 fire busy clean ds poison px frames + m247=$(grep -c "MME-CALL-HOLD .*window=247" "$dm" 2>/dev/null) + fire=$(grep -c "MME-CARRY-FIRE" "$dm" 2>/dev/null) + busy=$(grep -c "CANCEL reason=request-busy" "$dm" 2>/dev/null) + poison=$(grep -c "ds_after=badf1201\|->badf1201" "$dm" 2>/dev/null) + clean=$(grep -E "DS-FAST2.*ds_after=8[01]" "$dm" 2>/dev/null | grep -v badf1201 | tail -3) + ds=$(grep -c "DS-FAST" "$dm" 2>/dev/null) + frames=$(grep -c "^FRAME seq" "$gl" 2>/dev/null) + px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -vc "px0=0* ") + # px0=00000000 means black; any other 8-hex value is non-black. + px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -Evc "px0=00000000") + { + echo "TAG=$tag" + echo "m247=$m247 fire=$fire busy=$busy poison=$poison ds=$ds frames=$frames px_nonblack=$px" + [ -n "$clean" ] && echo "CLEAN_DS: $clean" + grep -E "MME-CARRY-(ARM|FIRE|CANCEL|REARM)" "$dm" 2>/dev/null | tail -6 + grep -E "DS-FAST2" "$dm" 2>/dev/null | tail -3 + grep -E "^(FRAME seq|XOK|DONE|glf rc|gz rc|XORG-ALIVE)" "$gl" "$gz" 2>/dev/null | tail -8 + } > "$BACKUP/$tag-summary.txt" + say "$tag: m247=$m247 fire=$fire busy=$busy ds=$ds frames=$frames px_nonblack=$px" + grep -E "CLEAN_DS|DS-FAST2" "$BACKUP/$tag-summary.txt" | sed 's/^/ /' | tee -a "$SUMMARY" +} + +RC=0 +for TAG in "$@"; do + say "=== $TAG start (GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES params=$(basename "$INSMOD_PARAMS")) ===" + DM_START=$(sudo dmesg 2>/dev/null | wc -l) + if ! INSMOD_PARAMS="$INSMOD_PARAMS" bash "$HERE/cycle-r1780.sh" "$TAG" "$MODULE" \ + > "$BACKUP/$TAG-cycle.log" 2>&1; then + say "$TAG CYCLE FAILED (see $BACKUP/$TAG-cycle.log)" + tail -5 "$BACKUP/$TAG-cycle.log" | sed 's/^/ /' | tee -a "$SUMMARY" + RC=1 + continue + fi + say "$TAG cycle up" + bash "$HERE/push_gl_tools.sh" > "$BACKUP/$TAG-push.log" 2>&1 + if grep -q "PUSH-FAIL" "$BACKUP/$TAG-push.log"; then + say "$TAG push had failures (guest agent?) - continuing" + fi + + gexec 30 "rm -f /tmp/mission-done; nohup env GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES bash /tmp/mission-gz.sh >/tmp/mission-run.log 2>&1 & echo MISSION_STARTED" >/dev/null + WAITMAX=$(( (GZ_SECS + 300) / 5 )) + DONE="" + for i in $(seq 1 "$WAITMAX"); do + if ! pgrep -f qemu-system-x86_64 >/dev/null; then + say "$TAG QEMU died before mission-done" + break + fi + DONE=$(guest_done) && break + sleep 5 + done + if [ -z "$DONE" ]; then + say "$TAG mission-done timeout after ~$((WAITMAX * 5))s" + else + say "$TAG mission: $DONE" + fi + + sudo dmesg 2>/dev/null | tail -n +$((DM_START + 1)) > "$BACKUP/$TAG-dmesg.log" + gexec 30 'cat /tmp/mission-gl.log 2>/dev/null' > "$BACKUP/$TAG-gl.log" + gexec 30 'cat /tmp/mission-gz.log 2>/dev/null' > "$BACKUP/$TAG-gz.log" + gexec 30 'tail -c 3000 /tmp/mission-run.log 2>/dev/null' > "$BACKUP/$TAG-run.log" + + GZSZ=$(gexec 20 'stat -c%s /dev/shm/gzframes.bin 2>/dev/null' | tr -dc '0-9') + GZSZ="${GZSZ:-0}" + say "$TAG /dev/shm/gzframes.bin=$GZSZ bytes" + if [ "$GZSZ" -gt 4096 ] && [ "$GZSZ" -lt $((MAX_FETCH_MB * 1024 * 1024)) ]; then + if timeout 300 sudo python3 "$HERE/qga_fetch.py" /dev/shm/gzframes.bin \ + "$BACKUP/$TAG-gzframes.bin" > "$BACKUP/$TAG-fetch.log" 2>&1; then + say "$TAG captured $BACKUP/$TAG-gzframes.bin" + else + say "$TAG fetch FAILED (see $BACKUP/$TAG-fetch.log)" + fi + fi + + analyze_tag "$TAG" "$BACKUP/$TAG-dmesg.log" "$BACKUP/$TAG-gl.log" "$BACKUP/$TAG-gz.log" + + PXN=$(grep -E "^FRAME seq" "$BACKUP/$TAG-gl.log" 2>/dev/null | grep -Evc "px0=00000000") + if [ "$STOP_ON" = "px" ] && [ "$PXN" -gt 0 ]; then + say "STOP: $TAG produced non-black RT pixels (px_nonblack=$PXN)" + break + fi + if [ "$STOP_ON" = "clean247" ] && grep -q "CLEAN_DS" "$BACKUP/$TAG-summary.txt" 2>/dev/null; then + say "STOP: $TAG produced a clean maps DS read" + break + fi +done +say "=== batch finished; summary $SUMMARY ===" +exit "$RC"
\ No newline at end of file |
