#!/bin/bash
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2026 Open-IOV contributors
# track_waits.sh - guest-side wait sampler: while a CUDA client (cdt) runs,
# repeatedly dump the per-channel completion counters (values), the tracker
# wait entries (trkinfo.py, every 2nd pass) and the push pages the host could
# not resolve (read_va.py), so the last signal before a failure is recorded.
#
# Written for the guest (no host paths). Run as root in the guest:
#   sudo /tmp/track_waits.sh <pid> [samples] [sleep]
# Logs: /tmp/trk.log, /tmp/cnt.log, /tmp/push.log
set -u

PID="${1:?usage: track_waits.sh <pid> [samples] [sleep]}"
N="${2:-600}"
SLEEP="${3:-1}"
PAGE_VA="${PAGE_VA:-0x20400f000}"
COUNT="${COUNT:-20}"
STRIDE="${STRIDE:-16}"
OFFSET="${OFFSET:-0xec0}"
PUSH="${PUSH:-1}"

: > /tmp/trk.log
: > /tmp/cnt.log
: > /tmp/push.log

python3 - "$PID" "$PAGE_VA" "$COUNT" "$STRIDE" "$OFFSET" <<'PY' >> /tmp/cnt.log 2>&1 &
import os, sys, time
pid = int(sys.argv[1]); page = int(sys.argv[2], 16)
count = int(sys.argv[3]); stride = int(sys.argv[4], 16); off = int(sys.argv[5], 16)
pm = os.open('/proc/%d/pagemap' % pid, os.O_RDONLY)
mem = os.open('/proc/%d/mem' % pid, os.O_RDONLY)
# Window dump start; slots live at [off, off+count*stride) inside the page.
WIN = 0x200
WIN_BASE = page + 0xe00
SLOT0 = off - 0xe00
while True:
    ts = time.strftime('%T')
    try:
        os.lseek(pm, (page // 4096) * 8, 0)
        e = int.from_bytes(os.read(pm, 8), 'little')
        if (e >> 63) & 1:
            # One read for both views: per-slot values and the raw dump come
            # from the same buffer, so they can never disagree. (The old
            # 20-single-word reader plus a later bulk read produced a
            # phantom "4 of 20 slots" pattern from a torn/remapped read.)
            raw = os.pread(mem, WIN, WIN_BASE)
            vals = []
            for i in range(count):
                o = SLOT0 + i * stride
                v = int.from_bytes(raw[o:o + 4], 'little')
                vals.append('%d:0x%x' % (i, v))
            print('%s %s' % (ts, ' '.join(vals)), flush=True)
            print('%s RAW %s' % (ts, raw.hex()), flush=True)
        else:
            print('%s %s' % (ts, ' '.join('%d:-' % i for i in range(count))),
                  flush=True)
    except OSError:
        pass
    time.sleep(0.5)
PY
CNTPID=$!

for i in $(seq 1 "$N"); do
    TS=$(date +%T.%3N)
    kill -0 "$PID" 2>/dev/null || { echo "PROCESS GONE at $TS (sample $i)" >> /tmp/trk.log; break; }

    if [ $((i % 2)) -eq 1 ] && [ "${GDB:-1}" = "1" ]; then
        echo "=== $TS sample $i ===" >> /tmp/trk.log
        timeout 8 gdb -p "$PID" -batch -x /tmp/trkinfo.py 2>&1 \
            | grep -vE "^\[|Reading symbols|^0x[0-9a-f]+ in |warning:" >> /tmp/trk.log
    fi

    if [ "$PUSH" = "1" ]; then
        {
            echo "=== $TS push pages ==="
            python3 /tmp/guest_pmap.py "$PID" 0x201000000 0x203400000 0x203400020 2>&1
            for va in 0x201000000 0x203400000 0x203400020; do
                echo "--- $va"
                timeout 5 python3 /tmp/read_va.py "$PID" "$va" 64 --u32 2>&1
            done
        } >> /tmp/push.log
    fi

    sleep "$SLEEP"
done
kill "$CNTPID" 2>/dev/null
echo "sampler done" >> /tmp/trk.log