userspace/cuda: libcuda analysis tree (debug-focused), guest ioctl tracer

Analysis tooling for debugging CUDA execution in Open-IOV guests:
- userspace/cuda/570.144/: fetch a library from a running guest (nc),
  objdump helpers, crash ip -> file vaddr + symbol mapping, and Ghidra
  headless import/analysis/decompile scripts (project kept so single
  functions decompile in seconds). The purpose is debugging correctness
  and performance of CUDA in Open-IOV VMs (not a reimplementation);
  extracted binaries and decompiled output are gitignored.
- tools/guest-diag/ioctl_trace.c: ptrace tracker for the NVIDIA driver
  ioctls of a child (libcuda issues raw syscalls, so LD_PRELOAD sees
  nothing). Decodes RM_ALLOC (class) and RM_CONTROL (cmd) arguments;
  this is how the cuCtxCreate crash sequence was captured.
- tools/test/cuda_drv_test.c: unbuffered + per-call markers so a crash
  identifies the CUDA call it happened in.

5 files changed, 238 insertions, 0 deletions

diff --git a/userspace/cuda/570.144/scripts/disasm.sh b/userspace/cuda/570.144/scripts/disasm.sh
new file mode 100755
index 0000000..01fea69
--- /dev/null
+++ b/userspace/cuda/570.144/scripts/disasm.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# disasm.sh - objdump helpers for a library under analysis.
+#
+# Usage:
+#   scripts/disasm.sh <file> <hex-vaddr> [before] [after]
+#       disassemble [vaddr-before, vaddr+after] (default 0x40 each side)
+#   scripts/disasm.sh --bytes <file> <hex-start> <hex-end>
+#       raw bytes in the range
+#   scripts/disasm.sh --symbol <file> <name>
+#       exported symbol address/size (readelf dyn-syms)
+#   scripts/disasm.sh --nearest <file> <hex-vaddr>
+#       nearest preceding exported symbol (FUNC) and the offset into it
+set -e
+
+cmd="${1:?usage: disasm.sh [--bytes|--symbol|--nearest] <file> ...}"
+case "$cmd" in
+--bytes)
+    file="$2"; start="$3"; end="$4"
+    objdump -s --start-address="$start" --stop-address="$end" "$file"
+    ;;
+--symbol)
+    file="$2"; name="$3"
+    readelf -sW --dyn-syms "$file" | awk -v n="$name" '$8 == n {print}'
+    ;;
+--nearest)
+    file="$2"; target="$3"
+    python3 - "$file" "$target" <<'PY'
+import subprocess, sys
+path, t = sys.argv[1], int(sys.argv[2], 16)
+out = subprocess.check_output(["readelf", "-sW", "--dyn-syms", path], text=True)
+syms = []
+for line in out.splitlines():
+    p = line.split()
+    if len(p) >= 8 and p[3] == "FUNC":
+        try:
+            syms.append((int(p[1], 16), int(p[2]), p[7]))
+        except ValueError:
+            pass
+syms.sort()
+prev = None
+for a, s, n in syms:
+    if a <= t:
+        prev = (a, s, n)
+    else:
+        break
+if prev:
+    print("nearest exported: %s at 0x%x size 0x%x (+0x%x)" %
+          (prev[2], prev[0], prev[1], t - prev[0]))
+else:
+    print("no exported symbol before 0x%x" % t)
+PY
+    ;;
+*)
+    file="$1"; vaddr="$2"
+    before="${3:-0x40}"; after="${4:-0x40}"
+    start=$(printf '0x%x' $(( vaddr - before )) 2>/dev/null || echo "$vaddr")
+    end=$(printf '0x%x' $(( vaddr + after )) 2>/dev/null || echo "$vaddr")
+    # allow hex arithmetic with shell
+    start=$(python3 -c "print(hex(int('$vaddr',16)-int('$before',16)))")
+    end=$(python3 -c "print(hex(int('$vaddr',16)+int('$after',16)))")
+    objdump -d -Mintel --start-address="$start" --stop-address="$end" "$file"
+    ;;
+esac
diff --git a/userspace/cuda/570.144/scripts/fetch_from_guest.sh b/userspace/cuda/570.144/scripts/fetch_from_guest.sh
new file mode 100755
index 0000000..6ed24f1
--- /dev/null
+++ b/userspace/cuda/570.144/scripts/fetch_from_guest.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# fetch_from_guest.sh - copy a file out of a running Open-IOV guest.
+#
+# The guest agent is used to start a netcat transfer from the guest to the
+# host (QEMU user networking: the host is 10.0.2.2), which is much faster and
+# more robust than pushing tens of megabytes through the agent in base64
+# chunks.
+#
+# Usage: scripts/fetch_from_guest.sh <guest-path> [guest-bind-ip]
+#        e.g. scripts/fetch_from_guest.sh \
+#               /usr/lib/x86_64-linux-gnu/libcuda.so.570.144
+set -e
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+RUN="$(cd "$HERE/.." && pwd)/run"
+VMEXEC="$HERE/../../../scripts/vgpu_vm.py"
+PORT="${PORT:-47311}"
+
+GUEST_PATH="${1:?usage: $0 <guest-path> [guest-bind-ip]}"
+HOST_IP="${2:-10.0.2.2}"
+OUT="$RUN/$(basename "$GUEST_PATH")"
+
+mkdir -p "$RUN"
+
+# Guest-side md5 for verification
+GUEST_MD5=$("$VMEXEC" exec "md5sum '$GUEST_PATH'" 2>/dev/null | awk '{print $1}')
+[ -n "$GUEST_MD5" ] || { echo "cannot stat $GUEST_PATH in guest"; exit 1; }
+
+echo "[*] guest md5: $GUEST_MD5"
+echo "[*] listening on :$PORT -> $OUT"
+rm -f "$OUT"
+
+nc -l "$PORT" > "$OUT" &
+NCPID=$!
+sleep 1
+
+"$VMEXEC" exec "setsid sh -c 'nc $HOST_IP $PORT < $GUEST_PATH' >/dev/null 2>&1 &" >/dev/null 2>&1 || true
+
+# Wait for the transfer to settle (size stable for two checks)
+prev=0
+for _ in $(seq 1 60); do
+    sleep 2
+    cur=$(stat -c%s "$OUT" 2>/dev/null || echo 0)
+    [ "$cur" -gt 0 ] && [ "$cur" = "$prev" ] && break
+    prev=$cur
+done
+wait $NCPID 2>/dev/null || true
+
+HOST_MD5=$(md5sum "$OUT" | awk '{print $1}')
+echo "[*] host  md5: $HOST_MD5"
+if [ "$GUEST_MD5" != "$HOST_MD5" ]; then
+    echo "[FAIL] md5 mismatch"
+    exit 1
+fi
+echo "[ok] $OUT ($(stat -c%s "$OUT") bytes)"
diff --git a/userspace/cuda/570.144/scripts/ghidra_analyze.sh b/userspace/cuda/570.144/scripts/ghidra_analyze.sh
new file mode 100755
index 0000000..b844300
--- /dev/null
+++ b/userspace/cuda/570.144/scripts/ghidra_analyze.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# ghidra_analyze.sh - import a library into a Ghidra project and run the full
+# auto-analysis once. The project is kept, so later decompilations are cheap:
+#
+#   scripts/ghidra_decompile_project.sh <hex-vaddr> [outfile]
+#
+# Full analysis of a ~70 MB library can take a long time; run it in the
+# background and log to run/ghidra-analyze.log.
+#
+# Usage: scripts/ghidra_analyze.sh <binary>
+#   env: GHIDRA_HOME (default /snap/ghidra/current/ghidra)
+#        PROJECT_DIR (default ../run/ghidra-proj)
+#        PROJECT_NAME (default cuda-analysis)
+set -e
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+RUN="$ROOT/run"
+BIN="${1:?usage: $0 <binary>}"
+
+GHIDRA_HOME="${GHIDRA_HOME:-/snap/ghidra/current/ghidra}"
+ANALYZE="$GHIDRA_HOME/support/analyzeHeadless"
+PROJECT_DIR="${PROJECT_DIR:-$RUN/ghidra-proj}"
+PROJECT_NAME="${PROJECT_NAME:-cuda-analysis}"
+
+[ -x "$ANALYZE" ] || { echo "analyzeHeadless not found at $ANALYZE"; exit 1; }
+mkdir -p "$PROJECT_DIR"
+
+echo "[*] importing + analyzing $BIN (this can take a long time)"
+"$ANALYZE" "$PROJECT_DIR" "$PROJECT_NAME" \
+    -import "$BIN" \
+    -overwrite \
+    -scriptPath "$ROOT/ghidra" \
+    -analysisTimeoutPerFile 0
+echo "[ok] analysis complete"
\ No newline at end of file
diff --git a/userspace/cuda/570.144/scripts/ghidra_decompile.sh b/userspace/cuda/570.144/scripts/ghidra_decompile.sh
new file mode 100755
index 0000000..bf95e8c
--- /dev/null
+++ b/userspace/cuda/570.144/scripts/ghidra_decompile.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# ghidra_decompile.sh - decompile the function containing a file vaddr.
+#
+# Runs Ghidra headless with -noanalysis: the program is imported, the crash
+# address is disassembled and turned into a function on demand, then
+# decompiled. That avoids a full auto-analysis pass over a ~70 MB library.
+#
+# Usage: scripts/ghidra_decompile.sh <binary> <hex-vaddr> [outfile]
+#   env: GHIDRA_HOME (default: /snap/ghidra/current/ghidra)
+#        PROJECT_DIR (default: ../run/ghidra-proj)
+#
+# Output: run/<basename>_<vaddr>.c unless outfile is given.
+set -e
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+RUN="$ROOT/run"
+BIN="${1:?usage: $0 <binary> <hex-vaddr> [outfile]}"
+VADDR="${2:?usage: $0 <binary> <hex-vaddr> [outfile]}"
+OUT="${3:-$RUN/$(basename "$BIN")_${VADDR}.c}"
+
+GHIDRA_HOME="${GHIDRA_HOME:-/snap/ghidra/current/ghidra}"
+ANALYZE="$GHIDRA_HOME/support/analyzeHeadless"
+PROJECT_DIR="${PROJECT_DIR:-$RUN/ghidra-proj}"
+PROJECT_NAME="${PROJECT_NAME:-cuda-analysis}"
+
+[ -x "$ANALYZE" ] || { echo "analyzeHeadless not found at $ANALYZE"; exit 1; }
+mkdir -p "$PROJECT_DIR" "$RUN"
+
+echo "[*] decompiling $BIN at $VADDR -> $OUT"
+"$ANALYZE" "$PROJECT_DIR" "$PROJECT_NAME" \
+    -import "$BIN" \
+    -overwrite \
+    -noanalysis \
+    -scriptPath "$ROOT/ghidra" \
+    -postScript DecompileAt.java "$VADDR" "$OUT" \
+    -deleteProject \
+    2>&1 | tail -5
+
+[ -f "$OUT" ] || { echo "[FAIL] no output produced"; exit 1; }
+echo "[ok] $OUT"
diff --git a/userspace/cuda/570.144/scripts/ghidra_decompile_project.sh b/userspace/cuda/570.144/scripts/ghidra_decompile_project.sh
new file mode 100755
index 0000000..66d7c90
--- /dev/null
+++ b/userspace/cuda/570.144/scripts/ghidra_decompile_project.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# ghidra_decompile_project.sh - decompile a function from an already-analyzed
+# Ghidra project (run scripts/ghidra_analyze.sh once first). No re-import and
+# no analysis, so this takes seconds.
+#
+# Usage: scripts/ghidra_decompile_project.sh <hex-vaddr> [outfile] [program]
+#   env: GHIDRA_HOME, PROJECT_DIR, PROJECT_NAME (same as ghidra_analyze.sh)
+set -e
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+RUN="$ROOT/run"
+VADDR="${1:?usage: $0 <hex-vaddr> [outfile] [program]}"
+OUT="${2:-$RUN/fn_$VADDR.c}"
+PROGRAM="${3:-libcuda.so.570.144}"
+
+GHIDRA_HOME="${GHIDRA_HOME:-/snap/ghidra/current/ghidra}"
+ANALYZE="$GHIDRA_HOME/support/analyzeHeadless"
+PROJECT_DIR="${PROJECT_DIR:-$RUN/ghidra-proj}"
+PROJECT_NAME="${PROJECT_NAME:-cuda-analysis}"
+
+[ -d "$PROJECT_DIR" ] || { echo "no project at $PROJECT_DIR (run ghidra_analyze.sh)"; exit 1; }
+mkdir -p "$RUN"
+
+echo "[*] decompiling $PROGRAM at $VADDR (project) -> $OUT"
+"$ANALYZE" "$PROJECT_DIR" "$PROJECT_NAME" \
+    -process "$PROGRAM" \
+    -noanalysis \
+    -scriptPath "$ROOT/ghidra" \
+    -postScript DecompileAt.java "$VADDR" "$OUT" \
+    2>&1 | tail -3
+
+[ -f "$OUT" ] || { echo "[FAIL] no output"; exit 1; }
+echo "[ok] $OUT"