summaryrefslogtreecommitdiffstats
path: root/tools/guest-diag/run_cycles_long.sh
blob: bf14ca20d9798ffbacbbd2df08dde7e0418aa4bb (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
#!/bin/bash
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2026 Open-IOV contributors
# run_cycles_long.sh - durable multi-cycle batch runner for the GZDoom
# GPU-video campaign.  For each tag: cycle-r1780 (rmmod/insmod + mdev +
# QEMU + GR-lane arm), push the GL/GZDoom guest tools, run mission-gz.sh
# detached in the guest, wait for /tmp/mission-done, then collect host
# dmesg + guest logs + any swap capture into the evidence dir.
#
# Usage: tools/guest-diag/run_cycles_long.sh c895 c896 c897
# Env:   GZ_SECS   (default 120)  guest GZDoom seconds
#        GLF_FRAMES(default 2)    gl_frames probe frames
#        MODULE    (default kernel/vgsp.ko)
#        INSMOD_PARAMS (default params-r1624.txt)
#        STOP_ON   px|clean247|none  (default px: stop at first RT pixel)
#        MAX_FETCH_MB (default 512)  skip gzframes.bin fetches above this
#
# The runner never reboots and never runs two QEMUs at once.  Everything is
# appended to /tmp/opencode/batch-<stamp>.log and per-tag files under
# backups/evidence-20261002-victory/r1780/.
set -u

HERE="$(cd "$(dirname "$0")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
BACKUP="$REPO/backups/evidence-20261002-victory/r1780"
VMEXEC="$REPO/scripts/vgpu_vm.py"
MODULE="${MODULE:-$REPO/kernel/vgsp.ko}"
INSMOD_PARAMS="${INSMOD_PARAMS:-$HERE/params-r1624.txt}"
GZ_SECS="${GZ_SECS:-120}"
GLF_FRAMES="${GLF_FRAMES:-2}"
STOP_ON="${STOP_ON:-px}"
MAX_FETCH_MB="${MAX_FETCH_MB:-512}"
STAMP="$(date +%Y%m%d-%H%M%S)"
SUMMARY="/tmp/opencode/batch-$STAMP.log"
mkdir -p "$BACKUP" /tmp/opencode

say() { echo "[$(date +%T)] $*" | tee -a "$SUMMARY"; }

gexec() { # gexec <timeout> <cmd>
    local t="$1"; shift
    timeout $((t + 15)) sudo python3 "$VMEXEC" exec --timeout "$t" "$*" 2>/dev/null
}

guest_done() {
    local out
    out=$(gexec 20 'cat /tmp/mission-done 2>/dev/null')
    case "$out" in
        *DONE*|*X-FAIL*) echo "$out" | tail -1; return 0 ;;
    esac
    return 1
}

analyze_tag() { # analyze_tag <tag> <dmesg-file> <gl-log> <gz-log>
    local tag="$1" dm="$2" gl="$3" gz="$4"
    local m247 fire busy clean ds poison px frames
    m247=$(grep -c "MME-CALL-HOLD .*window=247" "$dm" 2>/dev/null)
    fire=$(grep -c "MME-CARRY-FIRE" "$dm" 2>/dev/null)
    busy=$(grep -c "CANCEL reason=request-busy" "$dm" 2>/dev/null)
    poison=$(grep -c "ds_after=badf1201\|->badf1201" "$dm" 2>/dev/null)
    clean=$(grep -E "DS-FAST2.*ds_after=8[01]" "$dm" 2>/dev/null | grep -v badf1201 | tail -3)
    ds=$(grep -c "DS-FAST" "$dm" 2>/dev/null)
    frames=$(grep -c "^FRAME seq" "$gl" 2>/dev/null)
    px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -vc "px0=0* ")
    # px0=00000000 means black; any other 8-hex value is non-black.
    px=$(grep -E "^FRAME seq" "$gl" 2>/dev/null | grep -Evc "px0=00000000")
    {
        echo "TAG=$tag"
        echo "m247=$m247 fire=$fire busy=$busy poison=$poison ds=$ds frames=$frames px_nonblack=$px"
        [ -n "$clean" ] && echo "CLEAN_DS: $clean"
        grep -E "MME-CARRY-(ARM|FIRE|CANCEL|REARM)" "$dm" 2>/dev/null | tail -6
        grep -E "DS-FAST2" "$dm" 2>/dev/null | tail -3
        grep -E "^(FRAME seq|XOK|DONE|glf rc|gz rc|XORG-ALIVE)" "$gl" "$gz" 2>/dev/null | tail -8
    } > "$BACKUP/$tag-summary.txt"
    say "$tag: m247=$m247 fire=$fire busy=$busy ds=$ds frames=$frames px_nonblack=$px"
    grep -E "CLEAN_DS|DS-FAST2" "$BACKUP/$tag-summary.txt" | sed 's/^/    /' | tee -a "$SUMMARY"
}

RC=0
for TAG in "$@"; do
    say "=== $TAG start (GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES params=$(basename "$INSMOD_PARAMS")) ==="
    DM_START=$(sudo dmesg 2>/dev/null | wc -l)
    if ! INSMOD_PARAMS="$INSMOD_PARAMS" bash "$HERE/cycle-r1780.sh" "$TAG" "$MODULE" \
            > "$BACKUP/$TAG-cycle.log" 2>&1; then
        say "$TAG CYCLE FAILED (see $BACKUP/$TAG-cycle.log)"
        tail -5 "$BACKUP/$TAG-cycle.log" | sed 's/^/    /' | tee -a "$SUMMARY"
        RC=1
        continue
    fi
    say "$TAG cycle up"
    bash "$HERE/push_gl_tools.sh" > "$BACKUP/$TAG-push.log" 2>&1
    if grep -q "PUSH-FAIL" "$BACKUP/$TAG-push.log"; then
        say "$TAG push had failures (guest agent?) - continuing"
    fi

    gexec 30 "rm -f /tmp/mission-done; nohup env GZ_SECS=$GZ_SECS GLF_FRAMES=$GLF_FRAMES bash /tmp/mission-gz.sh >/tmp/mission-run.log 2>&1 & echo MISSION_STARTED" >/dev/null
    WAITMAX=$(( (GZ_SECS + 300) / 5 ))
    DONE=""
    for i in $(seq 1 "$WAITMAX"); do
        if ! pgrep -f qemu-system-x86_64 >/dev/null; then
            say "$TAG QEMU died before mission-done"
            break
        fi
        DONE=$(guest_done) && break
        sleep 5
    done
    if [ -z "$DONE" ]; then
        say "$TAG mission-done timeout after ~$((WAITMAX * 5))s"
    else
        say "$TAG mission: $DONE"
    fi

    sudo dmesg 2>/dev/null | tail -n +$((DM_START + 1)) > "$BACKUP/$TAG-dmesg.log"
    gexec 30 'cat /tmp/mission-gl.log 2>/dev/null' > "$BACKUP/$TAG-gl.log"
    gexec 30 'cat /tmp/mission-gz.log 2>/dev/null' > "$BACKUP/$TAG-gz.log"
    gexec 30 'tail -c 3000 /tmp/mission-run.log 2>/dev/null' > "$BACKUP/$TAG-run.log"

    GZSZ=$(gexec 20 'stat -c%s /dev/shm/gzframes.bin 2>/dev/null' | tr -dc '0-9')
    GZSZ="${GZSZ:-0}"
    say "$TAG /dev/shm/gzframes.bin=$GZSZ bytes"
    if [ "$GZSZ" -gt 4096 ] && [ "$GZSZ" -lt $((MAX_FETCH_MB * 1024 * 1024)) ]; then
        if timeout 300 sudo python3 "$HERE/qga_fetch.py" /dev/shm/gzframes.bin \
                "$BACKUP/$TAG-gzframes.bin" > "$BACKUP/$TAG-fetch.log" 2>&1; then
            say "$TAG captured $BACKUP/$TAG-gzframes.bin"
        else
            say "$TAG fetch FAILED (see $BACKUP/$TAG-fetch.log)"
        fi
    fi

    analyze_tag "$TAG" "$BACKUP/$TAG-dmesg.log" "$BACKUP/$TAG-gl.log" "$BACKUP/$TAG-gz.log"

    PXN=$(grep -E "^FRAME seq" "$BACKUP/$TAG-gl.log" 2>/dev/null | grep -Evc "px0=00000000")
    if [ "$STOP_ON" = "px" ] && [ "$PXN" -gt 0 ]; then
        say "STOP: $TAG produced non-black RT pixels (px_nonblack=$PXN)"
        break
    fi
    if [ "$STOP_ON" = "clean247" ] && grep -q "CLEAN_DS" "$BACKUP/$TAG-summary.txt" 2>/dev/null; then
        say "STOP: $TAG produced a clean maps DS read"
        break
    fi
done
say "=== batch finished; summary $SUMMARY ==="
exit "$RC"