summaryrefslogtreecommitdiffstats
path: root/tools/guest-diag/run_cycles_long.sh
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-11-16 09:50:31 -0500
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-11-16 09:50:31 -0500
commited8a9c6b8e061fe772fc0050ab83c2dec2db8d1e (patch)
tree458d2ea063d8a8a6996cd15891e3242cfe93ece5 /tools/guest-diag/run_cycles_long.sh
parentf2c73f4e52d24188a9edb0e8190c50f87f19181f (diff)
downloadopen-iov-ed8a9c6b8e061fe772fc0050ab83c2dec2db8d1e.tar.gz
open-iov-ed8a9c6b8e061fe772fc0050ab83c2dec2db8d1e.zip
vgsp: eng89 - live-derived GS map, draw-move anchor, raster enables, async capture
Kernel (deterministic fixes, each with an A/B knob): - sema_gl_sph_gs_maps=6: derive the whole 216-bit GS IMAP/OMAP blocks from the bound VTG image's OMAP and the PS image's IMAP at authoring time (SPH-GS-MAP receipt; SPH-GS-SKIP names ps-invalid/gs-invalid/ vtg-overlap/vtg-short). maps=4 stays as the fallback. - DS-MAP census line: dump stage-4/5 w5/w10/w13/w17 next to DS-IMAGE so the fetched maps are visible without a new probe. - MME pack carrier: reject maxa > 4096 (the IR pack array) instead of 64K, with MME-PACK-REJECT reason=bound. - MME-DRAW-MOVE: anchor the slot-34 move after a 0x3810 VERTEX_STREAM run. c848 moved CALL(34) to 0x204, ahead of the prepended stream bind, so the 8x3 fetch used stale stream state. - sema_gl_raster_enable=1: emit SET_RASTER_ENABLE=1 and SET_GLOBAL_RENDER_ENABLE MODE_TRUE in the raster prelude (fresh replay contexts may carry 0 for both, discarding every fragment). Guest tooling (unblocks the render loop and capture): - sdl_cap.c: asynchronous PBO+fence capture with a 5 s abandonment bound; no glFinish/blocking readback (c899 wedged gzdoom in an uninterruptible nv ioctl); clean sentinel from a destructor. - gz_pbuf.c: request depth 24/stencil 8, log GL_DEPTH_BITS, and swap with glFlush only. - mission-gz.sh: gl_frames probe off by default; GZDoom launched detached with a bounded wait, per-attempt X restart and retry; capture in /dev/shm with GZCAP_MAXF. - gzdoom_pbuf_run.sh: gl_pipeline_depth=2, gl_finishbeforeswap=0, GZCAP_MAXF default, rc-file support for the detached run. - run_cycles_long.sh: durable multi-cycle runner (the previous one lived in /tmp and was lost on reboot); pushes tools, runs the mission, grabs dmesg + guest logs + capture per cycle, stops on pixels. Test matrix: params-r1624-maps{4,6}.txt.
Diffstat (limited to 'tools/guest-diag/run_cycles_long.sh')
-rwxr-xr-xtools/guest-diag/run_cycles_long.sh142
1 files changed, 142 insertions, 0 deletions
diff --git a/tools/guest-diag/run_cycles_long.sh b/tools/guest-diag/run_cycles_long.sh
new file mode 100755
index 0000000..bf14ca2
--- /dev/null
+++ b/tools/guest-diag/run_cycles_long.sh
@@ -0,0 +1,142 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2026 Open-IOV contributors
+# run_cycles_long.sh - durable multi-cycle batch runner for the GZDoom
+# GPU-video campaign. For each tag: cycle-r1780 (rmmod/insmod + mdev +
+# QEMU + GR-lane arm), push the GL/GZDoom guest tools, run mission-gz.sh
+# detached in the guest, wait for /tmp/mission-done, then collect host
+# dmesg + guest logs + any swap capture into the evidence dir.
+#
+# Usage: tools/guest-diag/run_cycles_long.sh c895 c896 c897
+# Env: GZ_SECS (default 120) guest GZDoom seconds
+# GLF_FRAMES(default 2) gl_frames probe frames
+# MODULE (default kernel/vgsp.ko)
+# INSMOD_PARAMS (default params-r1624.txt)
+# STOP_ON px|clean247|none (default px: stop at first RT pixel)
+# MAX_FETCH_MB (default 512) skip gzframes.bin fetches above this
+#
+# The runner never reboots and never runs two QEMUs at once. Everything is
+# appended to /tmp/opencode/batch-<stamp>.log and per-tag files under
+# backups/evidence-20261002-victory/r1780/.
+set -u
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="$(cd "$HERE/../.." && pwd)"
+BACKUP="$REPO/backups/evidence-20261002-victory/r1780"
+VMEXEC="$REPO/scripts/vgpu_vm.py"
+MODULE="${MODULE:-$REPO/kernel/vgsp.ko}"
+INSMOD_PARAMS="${INSMOD_PARAMS:-$HERE/params-r1624.txt}"
+GZ_SECS="${GZ_SECS:-120}"
+GLF_FRAMES="${GLF_FRAMES:-2}"
+STOP_ON="${STOP_ON:-px}"
+MAX_FETCH_MB="${MAX_FETCH_MB:-512}"
+STAMP="$(date +%Y%m%d-%H%M%S)"
+SUMMARY="/tmp/opencode/batch-$STAMP.log"
+mkdir -p "$BACKUP" /tmp/opencode
+
+say() { echo "[$(date +%T)] $*" | tee -a "$SUMMARY"; }
+
+gexec() { # gexec <timeout> <cmd>
+ local t="$1"; shift
+ timeout $((t + 15)) sudo python3 "$VMEXEC" exec --timeout "$t" "$*" 2>/dev/null
+}
+
+guest_done() {
+ local out
+ out=$(gexec 20 'cat /tmp/mission-done 2>/dev/null')
+ case "$out" in
+ *DONE*|*X-FAIL*) echo "$out" | tail -1; return 0 ;;
+ esac
+ return 1
+}
+
+analyze_tag() { # analyze_tag <tag> <dmesg-file> <gl-log> <gz-log>
+ local tag="$1" dm="$2" gl="$3" gz="$4"
+ local m247 fire busy clean ds poison px frames
+ m247=$(grep -c "MME-CALL-HOLD .*window=247" "$dm" 2>/dev/null)
+ fire=$(grep -c "MME-CARRY-FIRE" "$dm" 2>/dev/null)
+ busy=$(grep -c "CANCEL reason=request-busy" "$dm" 2>/dev/null)
+ poison=$(grep -c "ds_after=badf1201\|->badf1201" "$dm" 2>/dev/null)
+ clean=$(grep -E "DS-FAST2.*ds_after=8[01]" "$dm" 2>/dev/null | grep -v badf1201 | tail -3)
+ ds=$(grep -c "DS-FAST" "$dm" 2>/dev/null)
+ frames=$(grep -c "^FRAME seq" "$gl" 2>/dev/null)
+ px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -vc "px0=0* ")
+ # px0=00000000 means black; any other 8-hex value is non-black.
+ px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -Evc "px0=00000000")
+ {
+ echo "TAG=$tag"
+ echo "m247=$m247 fire=$fire busy=$busy poison=$poison ds=$ds frames=$frames px_nonblack=$px"
+ [ -n "$clean" ] && echo "CLEAN_DS: $clean"
+ grep -E "MME-CARRY-(ARM|FIRE|CANCEL|REARM)" "$dm" 2>/dev/null | tail -6
+ grep -E "DS-FAST2" "$dm" 2>/dev/null | tail -3
+ grep -E "^(FRAME seq|XOK|DONE|glf rc|gz rc|XORG-ALIVE)" "$gl" "$gz" 2>/dev/null | tail -8
+ } > "$BACKUP/$tag-summary.txt"
+ say "$tag: m247=$m247 fire=$fire busy=$busy ds=$ds frames=$frames px_nonblack=$px"
+ grep -E "CLEAN_DS|DS-FAST2" "$BACKUP/$tag-summary.txt" | sed 's/^/ /' | tee -a "$SUMMARY"
+}
+
+RC=0
+for TAG in "$@"; do
+ say "=== $TAG start (GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES params=$(basename "$INSMOD_PARAMS")) ==="
+ DM_START=$(sudo dmesg 2>/dev/null | wc -l)
+ if ! INSMOD_PARAMS="$INSMOD_PARAMS" bash "$HERE/cycle-r1780.sh" "$TAG" "$MODULE" \
+ > "$BACKUP/$TAG-cycle.log" 2>&1; then
+ say "$TAG CYCLE FAILED (see $BACKUP/$TAG-cycle.log)"
+ tail -5 "$BACKUP/$TAG-cycle.log" | sed 's/^/ /' | tee -a "$SUMMARY"
+ RC=1
+ continue
+ fi
+ say "$TAG cycle up"
+ bash "$HERE/push_gl_tools.sh" > "$BACKUP/$TAG-push.log" 2>&1
+ if grep -q "PUSH-FAIL" "$BACKUP/$TAG-push.log"; then
+ say "$TAG push had failures (guest agent?) - continuing"
+ fi
+
+ gexec 30 "rm -f /tmp/mission-done; nohup env GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES bash /tmp/mission-gz.sh >/tmp/mission-run.log 2>&1 & echo MISSION_STARTED" >/dev/null
+ WAITMAX=$(( (GZ_SECS + 300) / 5 ))
+ DONE=""
+ for i in $(seq 1 "$WAITMAX"); do
+ if ! pgrep -f qemu-system-x86_64 >/dev/null; then
+ say "$TAG QEMU died before mission-done"
+ break
+ fi
+ DONE=$(guest_done) && break
+ sleep 5
+ done
+ if [ -z "$DONE" ]; then
+ say "$TAG mission-done timeout after ~$((WAITMAX * 5))s"
+ else
+ say "$TAG mission: $DONE"
+ fi
+
+ sudo dmesg 2>/dev/null | tail -n +$((DM_START + 1)) > "$BACKUP/$TAG-dmesg.log"
+ gexec 30 'cat /tmp/mission-gl.log 2>/dev/null' > "$BACKUP/$TAG-gl.log"
+ gexec 30 'cat /tmp/mission-gz.log 2>/dev/null' > "$BACKUP/$TAG-gz.log"
+ gexec 30 'tail -c 3000 /tmp/mission-run.log 2>/dev/null' > "$BACKUP/$TAG-run.log"
+
+ GZSZ=$(gexec 20 'stat -c%s /dev/shm/gzframes.bin 2>/dev/null' | tr -dc '0-9')
+ GZSZ="${GZSZ:-0}"
+ say "$TAG /dev/shm/gzframes.bin=$GZSZ bytes"
+ if [ "$GZSZ" -gt 4096 ] && [ "$GZSZ" -lt $((MAX_FETCH_MB * 1024 * 1024)) ]; then
+ if timeout 300 sudo python3 "$HERE/qga_fetch.py" /dev/shm/gzframes.bin \
+ "$BACKUP/$TAG-gzframes.bin" > "$BACKUP/$TAG-fetch.log" 2>&1; then
+ say "$TAG captured $BACKUP/$TAG-gzframes.bin"
+ else
+ say "$TAG fetch FAILED (see $BACKUP/$TAG-fetch.log)"
+ fi
+ fi
+
+ analyze_tag "$TAG" "$BACKUP/$TAG-dmesg.log" "$BACKUP/$TAG-gl.log" "$BACKUP/$TAG-gz.log"
+
+ PXN=$(grep -E "^FRAME seq" "$BACKUP/$TAG-gl.log" 2>/dev/null | grep -Evc "px0=00000000")
+ if [ "$STOP_ON" = "px" ] && [ "$PXN" -gt 0 ]; then
+ say "STOP: $TAG produced non-black RT pixels (px_nonblack=$PXN)"
+ break
+ fi
+ if [ "$STOP_ON" = "clean247" ] && grep -q "CLEAN_DS" "$BACKUP/$TAG-summary.txt" 2>/dev/null; then
+ say "STOP: $TAG produced a clean maps DS read"
+ break
+ fi
+done
+say "=== batch finished; summary $SUMMARY ==="
+exit "$RC" \ No newline at end of file