#!/bin/bash # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2026 Open-IOV contributors # run_cycles_long.sh - durable multi-cycle batch runner for the GZDoom # GPU-video campaign. For each tag: cycle-r1780 (rmmod/insmod + mdev + # QEMU + GR-lane arm), push the GL/GZDoom guest tools, run mission-gz.sh # detached in the guest, wait for /tmp/mission-done, then collect host # dmesg + guest logs + any swap capture into the evidence dir. # # Usage: tools/guest-diag/run_cycles_long.sh c895 c896 c897 # Env: GZ_SECS (default 120) guest GZDoom seconds # GLF_FRAMES(default 2) gl_frames probe frames # MODULE (default kernel/vgsp.ko) # INSMOD_PARAMS (default params-r1624.txt) # STOP_ON px|clean247|none (default px: stop at first RT pixel) # MAX_FETCH_MB (default 512) skip gzframes.bin fetches above this # # The runner never reboots and never runs two QEMUs at once. Everything is # appended to /tmp/opencode/batch-.log and per-tag files under # backups/evidence-20261002-victory/r1780/. set -u HERE="$(cd "$(dirname "$0")" && pwd)" REPO="$(cd "$HERE/../.." && pwd)" BACKUP="$REPO/backups/evidence-20261002-victory/r1780" VMEXEC="$REPO/scripts/vgpu_vm.py" MODULE="${MODULE:-$REPO/kernel/vgsp.ko}" INSMOD_PARAMS="${INSMOD_PARAMS:-$HERE/params-r1624.txt}" GZ_SECS="${GZ_SECS:-120}" GLF_FRAMES="${GLF_FRAMES:-2}" STOP_ON="${STOP_ON:-px}" MAX_FETCH_MB="${MAX_FETCH_MB:-512}" STAMP="$(date +%Y%m%d-%H%M%S)" SUMMARY="/tmp/opencode/batch-$STAMP.log" mkdir -p "$BACKUP" /tmp/opencode say() { echo "[$(date +%T)] $*" | tee -a "$SUMMARY"; } gexec() { # gexec local t="$1"; shift timeout $((t + 15)) sudo python3 "$VMEXEC" exec --timeout "$t" "$*" 2>/dev/null } guest_done() { local out out=$(gexec 20 'cat /tmp/mission-done 2>/dev/null') case "$out" in *DONE*|*X-FAIL*) echo "$out" | tail -1; return 0 ;; esac return 1 } analyze_tag() { # analyze_tag local tag="$1" dm="$2" gl="$3" gz="$4" local m247 fire busy clean ds poison px frames m247=$(grep -c "MME-CALL-HOLD .*window=247" "$dm" 2>/dev/null) fire=$(grep -c "MME-CARRY-FIRE" "$dm" 2>/dev/null) busy=$(grep -c "CANCEL reason=request-busy" "$dm" 2>/dev/null) poison=$(grep -c "ds_after=badf1201\|->badf1201" "$dm" 2>/dev/null) clean=$(grep -E "DS-FAST2.*ds_after=8[01]" "$dm" 2>/dev/null | grep -v badf1201 | tail -3) ds=$(grep -c "DS-FAST" "$dm" 2>/dev/null) frames=$(grep -c "^FRAME seq" "$gl" 2>/dev/null) px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -vc "px0=0* ") # px0=00000000 means black; any other 8-hex value is non-black. px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -Evc "px0=00000000") { echo "TAG=$tag" echo "m247=$m247 fire=$fire busy=$busy poison=$poison ds=$ds frames=$frames px_nonblack=$px" [ -n "$clean" ] && echo "CLEAN_DS: $clean" grep -E "MME-CARRY-(ARM|FIRE|CANCEL|REARM)" "$dm" 2>/dev/null | tail -6 grep -E "DS-FAST2" "$dm" 2>/dev/null | tail -3 grep -E "^(FRAME seq|XOK|DONE|glf rc|gz rc|XORG-ALIVE)" "$gl" "$gz" 2>/dev/null | tail -8 } > "$BACKUP/$tag-summary.txt" say "$tag: m247=$m247 fire=$fire busy=$busy ds=$ds frames=$frames px_nonblack=$px" grep -E "CLEAN_DS|DS-FAST2" "$BACKUP/$tag-summary.txt" | sed 's/^/ /' | tee -a "$SUMMARY" } RC=0 for TAG in "$@"; do say "=== $TAG start (GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES params=$(basename "$INSMOD_PARAMS")) ===" DM_START=$(sudo dmesg 2>/dev/null | wc -l) if ! INSMOD_PARAMS="$INSMOD_PARAMS" bash "$HERE/cycle-r1780.sh" "$TAG" "$MODULE" \ > "$BACKUP/$TAG-cycle.log" 2>&1; then say "$TAG CYCLE FAILED (see $BACKUP/$TAG-cycle.log)" tail -5 "$BACKUP/$TAG-cycle.log" | sed 's/^/ /' | tee -a "$SUMMARY" RC=1 continue fi say "$TAG cycle up" bash "$HERE/push_gl_tools.sh" > "$BACKUP/$TAG-push.log" 2>&1 if grep -q "PUSH-FAIL" "$BACKUP/$TAG-push.log"; then say "$TAG push had failures (guest agent?) - continuing" fi gexec 30 "rm -f /tmp/mission-done; nohup env GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES bash /tmp/mission-gz.sh >/tmp/mission-run.log 2>&1 & echo MISSION_STARTED" >/dev/null WAITMAX=$(( (GZ_SECS + 300) / 5 )) DONE="" for i in $(seq 1 "$WAITMAX"); do if ! pgrep -f qemu-system-x86_64 >/dev/null; then say "$TAG QEMU died before mission-done" break fi DONE=$(guest_done) && break sleep 5 done if [ -z "$DONE" ]; then say "$TAG mission-done timeout after ~$((WAITMAX * 5))s" else say "$TAG mission: $DONE" fi sudo dmesg 2>/dev/null | tail -n +$((DM_START + 1)) > "$BACKUP/$TAG-dmesg.log" gexec 30 'cat /tmp/mission-gl.log 2>/dev/null' > "$BACKUP/$TAG-gl.log" gexec 30 'cat /tmp/mission-gz.log 2>/dev/null' > "$BACKUP/$TAG-gz.log" gexec 30 'tail -c 3000 /tmp/mission-run.log 2>/dev/null' > "$BACKUP/$TAG-run.log" GZSZ=$(gexec 20 'stat -c%s /dev/shm/gzframes.bin 2>/dev/null' | tr -dc '0-9') GZSZ="${GZSZ:-0}" say "$TAG /dev/shm/gzframes.bin=$GZSZ bytes" if [ "$GZSZ" -gt 4096 ] && [ "$GZSZ" -lt $((MAX_FETCH_MB * 1024 * 1024)) ]; then if timeout 300 sudo python3 "$HERE/qga_fetch.py" /dev/shm/gzframes.bin \ "$BACKUP/$TAG-gzframes.bin" > "$BACKUP/$TAG-fetch.log" 2>&1; then say "$TAG captured $BACKUP/$TAG-gzframes.bin" else say "$TAG fetch FAILED (see $BACKUP/$TAG-fetch.log)" fi fi analyze_tag "$TAG" "$BACKUP/$TAG-dmesg.log" "$BACKUP/$TAG-gl.log" "$BACKUP/$TAG-gz.log" PXN=$(grep -E "^FRAME seq" "$BACKUP/$TAG-gl.log" 2>/dev/null | grep -Evc "px0=00000000") if [ "$STOP_ON" = "px" ] && [ "$PXN" -gt 0 ]; then say "STOP: $TAG produced non-black RT pixels (px_nonblack=$PXN)" break fi if [ "$STOP_ON" = "clean247" ] && grep -q "CLEAN_DS" "$BACKUP/$TAG-summary.txt" 2>/dev/null; then say "STOP: $TAG produced a clean maps DS read" break fi done say "=== batch finished; summary $SUMMARY ===" exit "$RC"