blob: 153be035d44d0d3f462440b6a69fc7a30326ff52 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
#!/bin/bash
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2026 Open-IOV contributors
# run_cycles_long.sh - durable multi-cycle batch runner for the GZDoom
# GPU-video campaign. For each tag: cycle-r1780 (rmmod/insmod + mdev +
# QEMU + GR-lane arm), push the GL/GZDoom guest tools, run mission-gz.sh
# detached in the guest, wait for /tmp/mission-done, then collect host
# dmesg + guest logs + any swap capture into the evidence dir.
#
# Usage: tools/guest-diag/run_cycles_long.sh c895 c896 c897
# Env: GZ_SECS (default 120) guest GZDoom seconds
# GLF_FRAMES(default 2) gl_frames probe frames
# MODULE (default kernel/vgsp.ko)
# INSMOD_PARAMS (default params-r1624.txt)
# STOP_ON px|clean247|none (default px: stop at first RT pixel)
# MAX_FETCH_MB (default 512) skip gzframes.bin fetches above this
#
# The runner never reboots and never runs two QEMUs at once. Everything is
# appended to /tmp/opencode/batch-<stamp>.log and per-tag files under
# backups/evidence-20261002-victory/r1780/.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
BACKUP="$REPO/backups/evidence-20261002-victory/r1780"
VMEXEC="$REPO/scripts/vgpu_vm.py"
MODULE="${MODULE:-$REPO/kernel/vgsp.ko}"
INSMOD_PARAMS="${INSMOD_PARAMS:-$HERE/params-r1624.txt}"
GZ_SECS="${GZ_SECS:-120}"
GLF_FRAMES="${GLF_FRAMES:-2}"
STOP_ON="${STOP_ON:-px}"
MAX_FETCH_MB="${MAX_FETCH_MB:-512}"
STAMP="$(date +%Y%m%d-%H%M%S)"
SUMMARY="/tmp/opencode/batch-$STAMP.log"
mkdir -p "$BACKUP" /tmp/opencode
say() { echo "[$(date +%T)] $*" | tee -a "$SUMMARY"; }
gexec() { # gexec <timeout> <cmd>
local t="$1"; shift
timeout $((t + 15)) sudo python3 "$VMEXEC" exec --timeout "$t" "$*" 2>/dev/null
}
guest_done() {
local out
out=$(gexec 20 'cat /tmp/mission-done 2>/dev/null')
case "$out" in
*DONE*|*X-FAIL*) echo "$out" | tail -1; return 0 ;;
esac
return 1
}
analyze_tag() { # analyze_tag <tag> <dmesg-file> <gl-log> <gz-log>
local tag="$1" dm="$2" gl="$3" gz="$4"
local m247 fire busy clean ds poison px frames
m247=$(grep -c "MME-CALL-HOLD .*window=247" "$dm" 2>/dev/null)
fire=$(grep -c "MME-CARRY-FIRE" "$dm" 2>/dev/null)
busy=$(grep -c "CANCEL reason=request-busy" "$dm" 2>/dev/null)
poison=$(grep -c "ds_after=badf1201\|->badf1201" "$dm" 2>/dev/null)
clean=$(grep -E "DS-FAST2.*ds_after=8[01]" "$dm" 2>/dev/null | grep -v badf1201 | tail -3)
ds=$(grep -c "DS-FAST" "$dm" 2>/dev/null)
frames=$(grep -c "^FRAME seq" "$gl" 2>/dev/null)
px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -vc "px0=0* ")
# px0=00000000 means black; any other 8-hex value is non-black.
px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -Evc "px0=00000000")
{
echo "TAG=$tag"
echo "m247=$m247 fire=$fire busy=$busy poison=$poison ds=$ds frames=$frames px_nonblack=$px"
[ -n "$clean" ] && echo "CLEAN_DS: $clean"
grep -E "MME-CARRY-(ARM|FIRE|CANCEL|REARM)" "$dm" 2>/dev/null | tail -6
grep -E "DS-FAST2" "$dm" 2>/dev/null | tail -3
grep -E "^(FRAME seq|XOK|DONE|glf rc|gz rc|XORG-ALIVE)" "$gl" "$gz" 2>/dev/null | tail -8
} > "$BACKUP/$tag-summary.txt"
say "$tag: m247=$m247 fire=$fire busy=$busy ds=$ds frames=$frames px_nonblack=$px"
grep -E "CLEAN_DS|DS-FAST2" "$BACKUP/$tag-summary.txt" | sed 's/^/ /' | tee -a "$SUMMARY"
}
RC=0
for TAG in "$@"; do
say "=== $TAG start (GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES params=$(basename "$INSMOD_PARAMS")) ==="
DM_SINCE=$(date "+%Y-%m-%d %H:%M:%S")
# Continuous kernel-log capture: the dmesg ring wraps in <4 min at the
# debug volume, which silently truncated per-cycle evidence (c921).
sudo dmesg -w > "$BACKUP/$TAG-dmesg-watch.log" 2>&1 &
DM_WATCH_PID=$!
if ! INSMOD_PARAMS="$INSMOD_PARAMS" bash "$HERE/cycle-r1780.sh" "$TAG" "$MODULE" \
> "$BACKUP/$TAG-cycle.log" 2>&1; then
say "$TAG CYCLE FAILED (see $BACKUP/$TAG-cycle.log)"
tail -5 "$BACKUP/$TAG-cycle.log" | sed 's/^/ /' | tee -a "$SUMMARY"
RC=1
continue
fi
say "$TAG cycle up"
bash "$HERE/push_gl_tools.sh" > "$BACKUP/$TAG-push.log" 2>&1
if grep -q "PUSH-FAIL" "$BACKUP/$TAG-push.log"; then
say "$TAG push had failures (guest agent?) - continuing"
fi
gexec 30 "rm -f /tmp/mission-done; nohup env GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES bash /tmp/mission-gz.sh >/tmp/mission-run.log 2>&1 & echo MISSION_STARTED" >/dev/null
WAITMAX=$(( (GZ_SECS + 120) / 5 ))
DONE=""
for i in $(seq 1 "$WAITMAX"); do
if ! pgrep -f qemu-system-x86_64 >/dev/null; then
say "$TAG QEMU died before mission-done"
break
fi
DONE=$(guest_done) && break
sleep 5
done
if [ -z "$DONE" ]; then
say "$TAG mission-done timeout after ~$((WAITMAX * 5))s"
else
say "$TAG mission: $DONE"
fi
kill "$DM_WATCH_PID" 2>/dev/null
sudo dmesg -T --since "$DM_SINCE" 2>/dev/null > "$BACKUP/$TAG-dmesg.log" || \
sudo dmesg 2>/dev/null | tail -n 4000 > "$BACKUP/$TAG-dmesg.log"
# Merge the streamed capture (it has the lines the ring dropped).
[ -s "$BACKUP/$TAG-dmesg-watch.log" ] && \
cat "$BACKUP/$TAG-dmesg-watch.log" >> "$BACKUP/$TAG-dmesg.log"
gexec 30 'cat /tmp/mission-gl.log 2>/dev/null' > "$BACKUP/$TAG-gl.log"
gexec 30 'cat /tmp/mission-gz.log 2>/dev/null' > "$BACKUP/$TAG-gz.log"
gexec 30 'tail -c 3000 /tmp/mission-run.log 2>/dev/null' > "$BACKUP/$TAG-run.log"
GZSZ=$(gexec 20 'stat -c%s /dev/shm/gzframes.bin 2>/dev/null' | tr -dc '0-9')
GZSZ="${GZSZ:-0}"
say "$TAG /dev/shm/gzframes.bin=$GZSZ bytes"
if [ "$GZSZ" -gt 4096 ] && [ "$GZSZ" -lt $((MAX_FETCH_MB * 1024 * 1024)) ]; then
if timeout 300 sudo python3 "$HERE/qga_fetch.py" /dev/shm/gzframes.bin \
"$BACKUP/$TAG-gzframes.bin" > "$BACKUP/$TAG-fetch.log" 2>&1; then
say "$TAG captured $BACKUP/$TAG-gzframes.bin"
else
say "$TAG fetch FAILED (see $BACKUP/$TAG-fetch.log)"
fi
fi
analyze_tag "$TAG" "$BACKUP/$TAG-dmesg.log" "$BACKUP/$TAG-gl.log" "$BACKUP/$TAG-gz.log"
PXN=$(grep -E "^FRAME seq" "$BACKUP/$TAG-gl.log" 2>/dev/null | grep -Evc "px0=00000000")
if [ "$STOP_ON" = "px" ] && [ "$PXN" -gt 0 ]; then
say "STOP: $TAG produced non-black RT pixels (px_nonblack=$PXN)"
break
fi
if [ "$STOP_ON" = "clean247" ] && grep -q "CLEAN_DS" "$BACKUP/$TAG-summary.txt" 2>/dev/null; then
say "STOP: $TAG produced a clean maps DS read"
break
fi
done
say "=== batch finished; summary $SUMMARY ==="
exit "$RC"
|