tools: cuda_cycle.sh - one-command CUDA bring-up attempt

Restart the VM, push the guest tools, start cdt holding on failure, wait
for the 20 GR channels, register/pin the completion counters and arm
scanerr.py, so each iteration is a single command. scanerr now loops over
scanner hits until a hit carries a non-zero RM error record.

1 files changed, 27 insertions, 21 deletions

diff --git a/tools/guest-diag/scanerr.py b/tools/guest-diag/scanerr.py
index c3af2f5..96c4170 100755
--- a/tools/guest-diag/scanerr.py
+++ b/tools/guest-diag/scanerr.py
@@ -1,8 +1,8 @@
 # gdb script: at the libcuda error-scanner entry (FUN_0054fa00), walk the
 # context list and dump every non-zero RM error record (magic words,
-# errorClass at +0x8, shorts at +0xc/+0xe). This names the source of a
-# forced 999: the scanner returns 999 when param_2 != 0 and any record's
-# class fields are non-zero.
+# errorClass at +0x8, shorts at +0xc/+0xe). The scanner is called often with
+# a zeroed record; keep going until a hit has a non-zero record (that is the
+# one a forced 999 comes from) and print it with a backtrace.
 # Run: sudo gdb -p <cdt-pid> -batch -x /tmp/scanerr.py
 import gdb
 
@@ -20,11 +20,6 @@ if base is None:
 gdb.execute("set pagination off")
 gdb.execute("set confirm off")
 gdb.execute("break *0x%x" % (base + 0x44fa00))
-try:
-    gdb.execute("continue")
-except gdb.error as e:
-    print("CONTINUE %s" % e)
-print("HIT")
 
 
 def rd64(a):
@@ -41,12 +36,21 @@ def rd32(a):
         return None
 
 
-try:
-    p1 = int(gdb.parse_and_eval("$rdi")) & 0xffffffffffffffff
-    p2 = int(gdb.parse_and_eval("$rsi")) & 0xffffffff
-    print("param_1=0x%x param_2=%d" % (p1, p2))
+for hit in range(400):
+    try:
+        gdb.execute("continue")
+    except gdb.error as e:
+        print("CONTINUE_END %s" % e)
+        break
+    try:
+        p1 = int(gdb.parse_and_eval("$rdi")) & 0xffffffffffffffff
+        p2 = int(gdb.parse_and_eval("$rsi")) & 0xffffffff
+    except Exception:
+        print("REGS_GONE")
+        break
     pl = rd64(p1)
     n = 0
+    found = 0
     while pl and n < 64:
         mgr = rd64(pl + 0x1291 * 8)
         if mgr:
@@ -56,18 +60,20 @@ try:
                 if rec:
                     vals = [rd32(rec + i) or 0 for i in range(0, 0x28, 4)]
                     if any(vals):
-                        print("ENTRY %d ctx=0x%x mgr=0x%x rec=0x%x rec=%s" %
-                              (n, pl, mgr, rec,
+                        print("HIT %d param_2=%d ENTRY %d ctx=0x%x mgr=0x%x rec=0x%x rec=%s" %
+                              (hit, p2, n, pl, mgr, rec,
                                ' '.join('0x%08x' % v for v in vals)))
+                        found = 1
         pl = rd64(pl + 0x128e * 8)
         n += 1
-    print("WALKED %d" % n)
-except Exception as e:
-    print("dump_fail %s" % e)
-try:
-    gdb.execute("bt 8", to_string=True)
-except Exception:
-    pass
+    if found:
+        print("FOUND")
+        try:
+            print(gdb.execute("bt 12", to_string=True))
+        except Exception:
+            pass
+        break
+print("DONE")
 try:
     gdb.execute("detach")
 except Exception: