diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/README.md | 1 | ||||
| -rw-r--r-- | tools/gr-regs/README.md | 36 | ||||
| -rw-r--r-- | tools/gr-regs/analyze_regs.py | 38 | ||||
| -rw-r--r-- | tools/gr-regs/reg_sampler.py | 92 |
4 files changed, 167 insertions, 0 deletions
diff --git a/tools/README.md b/tools/README.md index f9e0018..811a46b 100644 --- a/tools/README.md +++ b/tools/README.md @@ -8,6 +8,7 @@ Collection of scripts and utilities for vGPU development, debugging, and testing tools/ ├── module_ops/ # Kernel module load/unload/reload utilities ├── debug/ # Debugging and diagnostic tools +├── gr-regs/ # Read-only GR/FECS/SKED register sampler + ground truth ├── test/ # Test runners and validation scripts ├── vm/ # VM/QEMU management for vGPU testing ├── firmware/ # GSP firmware management diff --git a/tools/gr-regs/README.md b/tools/gr-regs/README.md new file mode 100644 index 0000000..3294742 --- /dev/null +++ b/tools/gr-regs/README.md @@ -0,0 +1,36 @@ +# gr-regs - GR/FECS/SKED register ground truth + +Read-only BAR0 sampling of the physical GPU's GR state, used to answer +"what does a working GR launch do to the engine" and "what does our +forwarded launch actually reach" without instrumenting any driver. + +## Files + +- `reg_sampler.py` - high-rate read-only MMIO sampler (root required). + `sudo python3 reg_sampler.py <seconds> <Hz> <outfile> [label]` +- `analyze_regs.py` - value distribution + transition timeline for a sample + file: `python3 analyze_regs.py <outfile>` + +## What the registers mean (TU104) + +| Register | Meaning | +|---|---| +| `FECS_CURRENT_CTX` / `FECS_NEW_CTX` | FECS context load target: `PTR[27:0] = instance PA>>12`, `TARGET[29:28]`, `VALID[31]`. A channel is engine-current only while `VALID=1` on its instance. | +| `CWD_TABLE_0..3` | SKED context work-descriptor table. Cold/idle reads `CWD_TABLE_0=0x00002000`; a scheduled QMD shows its group + state (e.g. `0x3f002000`) and `CWD_TABLE_1` carries the QMD id (`QMD VA>>8`). | +| `SKED_ACTIVITY`, `SKED_HWW_ESR` | SKED processing / first-error latch. | +| `GRFIFO_STATUS`, `PGRAPH_STATUS` | GR FIFO and engine activity. | +| `FE_QMD_VIRT_*` | FE QMD virtual-address translation; reads 0 in native use. | +| `badfxxxx` samples | GPU "bad fence"/clock-gated pattern during driver handoff; filter them out. | + +## Ground truth recorded 2026-10-06 (distro 570.211.01, live CUDA) + +- CWD_TABLE_0 leaves the cold `0x00002000` and reads `0x3f002000`/`0x3f002400` + while CUDA runs. +- FECS_CURRENT_CTX/NEW_CTX read `0x801f3acb`/`0x801f3edf` (VALID=1) with the + active channel's instance PA>>12. +- FE_QMD_VIRT_CONTROL/BASE stay 0. + +These values are the reference for the campaign's forwarded-launch work +(see `backups/evidence-20261002-victory/`); a forwarded QMD that never +enqueues leaves `CWD_TABLE_0` at the cold value, one that enqueues shows the +same class of transitions as the real driver. diff --git a/tools/gr-regs/analyze_regs.py b/tools/gr-regs/analyze_regs.py new file mode 100644 index 0000000..b03aca4 --- /dev/null +++ b/tools/gr-regs/analyze_regs.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import collections, sys +path = sys.argv[1] if len(sys.argv) > 1 else '/tmp/opencode/regs-m63.txt' +cmp_mode = len(sys.argv) > 2 and sys.argv[2] == 'cmp' +hdr = None +rows = [] +with open(path) as f: + for line in f: + if line.startswith('#'): + if line.startswith('# ts '): + hdr = line.split()[2:] + continue + p = line.split() + if len(p) >= 2: + rows.append((float(p[0]), p)) +print("samples:", len(rows), "t range", rows[0][0], rows[-1][0]) +cols = collections.defaultdict(collections.Counter) +h = {n: i for i, n in enumerate(hdr)} +for t, p in rows: + for i, n in enumerate(hdr): + if 1 + i < len(p): + cols[n][p[1 + i]] += 1 +for n in ('FECS_CURRENT_CTX', 'FECS_NEW_CTX', 'CWD_TABLE_0', 'CWD_TABLE_1', + 'SKED_HWW_ESR', 'SKED_ACTIVITY', 'GRFIFO_STATUS', 'FECS_CPUCTL', + 'PGRAPH_STATUS', 'PMC_ENABLE'): + c = cols.get(n) + if c: + print('%-22s %s' % (n, ', '.join('%s x%d' % (k, v) for k, v in c.most_common(8)))) +# interesting transitions: where FECS_CURRENT_CTX or CWD_TABLE_0 or GRFIFO change +print("=== transitions (FECS_CURRENT_CTX / CWD_T0 / GRFIFO) ===") +last = {} +for t, p in rows: + key = (p[1 + h['FECS_CURRENT_CTX']], p[1 + h['CWD_TABLE_0']], p[1 + h['GRFIFO_STATUS']]) + if key != last.get('k'): + print('t=%8.2f FECS=%s NEW=%s CWD0=%s CWD1=%s SKED_ACT=%s GRFIFO=%s PGRAPH=%s' % ( + t, key[0], p[1 + h['FECS_NEW_CTX']], key[1], p[1 + h['CWD_TABLE_1']], + p[1 + h['SKED_ACTIVITY']], key[2], p[1 + h['PGRAPH_STATUS']])) + last['k'] = key diff --git a/tools/gr-regs/reg_sampler.py b/tools/gr-regs/reg_sampler.py new file mode 100644 index 0000000..af2cc97 --- /dev/null +++ b/tools/gr-regs/reg_sampler.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# reg_sampler.py - read-only BAR0 register sampler for GR/FECS/SKED ground truth. +# +# Purpose +# Sample the physical GPU's PGRAPH/FECS/SKED/PFIFO registers at high rate +# while a GR launch is in flight, so the engine-side state transitions +# (context load, CWD table occupancy, SKED activity, GRFIFO progress) are +# observable without the driver's help. Used to capture the ground-truth +# behaviour of a working CUDA channel (distro driver) and to inspect the +# ladder's forwarded launches. +# +# Safety +# Read-only MMIO. Safe to run while any driver owns the device. Needs root +# (resource0 mmap). Do not run concurrently with a driver rebind/FLR. +# +# Usage +# sudo python3 reg_sampler.py <seconds> <Hz> <outfile> [label] +# python3 analyze_regs.py <outfile> # value distribution + transitions +# +# Notes +# - The sampler writes one TSV line per sample: t and one u32 per register. +# - `badfxxxx` values are the GPU returning its "bad fence/clock-gated" +# pattern during driver handoff windows; filter them in analysis. +# - Offset source: kernel/gsp_phys_booter.c reg-probe table (TU104). +import mmap +import os +import struct +import sys +import time + +DEV = "/sys/bus/pci/devices/0000:01:00.0/resource0" +REGS = [ + (0x000200, "PMC_ENABLE"), + (0x400700, "PGRAPH_STATUS"), + (0x400504, "GRFIFO_STATUS"), + (0x400704, "TRAPPED_ADDR"), + (0x400708, "TRAPPED_DATA_LO"), + (0x404000, "FE_HWW_ESR"), + (0x407020, "SKED_HWW_ESR"), + (0x407040, "SKED_STATUS_0"), + (0x407054, "SKED_ACTIVITY"), + (0x407060, "CWD_TABLE_0"), + (0x407064, "CWD_TABLE_1"), + (0x407068, "CWD_TABLE_2"), + (0x40706C, "CWD_TABLE_3"), + (0x409008, "FECS_IRQSTAT"), + (0x409100, "FECS_CPUCTL"), + (0x409400, "FECS_CTXSW_STATUS_1"), + (0x409b00, "FECS_CURRENT_CTX"), + (0x409b04, "FECS_NEW_CTX"), + (0x409c00, "FECS_CTXSW_STATUS_FE_0"), + (0x409c18, "FECS_HOST_INT_STATUS"), + (0x409040, "FECS_MAILBOX0"), + (0x409044, "FECS_MAILBOX1"), + (0x40a300, "FE_QMD_VIRT_BASEA_0"), + (0x40a304, "FE_QMD_VIRT_BASEA_1"), + (0x40a400, "FE_QMD_VIRT_BASEB_0"), + (0x40a500, "FE_QMD_VIRT_CONTROL_0"), + (0x40a504, "FE_QMD_VIRT_CONTROL_1"), + (0x40a508, "FE_QMD_VIRT_CONTROL_2"), + (0x40a50c, "FE_QMD_VIRT_CONTROL_3"), + (0x41a040, "GPCCS_MAILBOX0"), + (0x41a100, "GPCCS_CPUCTL"), +] + + +def main(): + secs = float(sys.argv[1]) + hz = float(sys.argv[2]) + out = sys.argv[3] + label = sys.argv[4] if len(sys.argv) > 4 else "sample" + fd = os.open(DEV, os.O_RDONLY | os.O_SYNC) + m = mmap.mmap(fd, 0x1000000, mmap.MAP_SHARED, mmap.PROT_READ) + period = 1.0 / hz + t0 = time.monotonic() + n = 0 + with open(out, "w") as f: + f.write("# %s t=%.3f\n" % (label, t0)) + f.write("# ts " + " ".join(nm for _, nm in REGS) + "\n") + while time.monotonic() - t0 < secs: + ts = time.monotonic() - t0 + vals = [struct.unpack_from("<I", m, off)[0] for off, _ in REGS] + f.write("%.6f " % ts + " ".join("%08x" % v for v in vals) + "\n") + n += 1 + dt = period - (time.monotonic() - t0 - ts) + if dt > 0: + time.sleep(dt) + print("samples=%d" % n) + + +if __name__ == "__main__": + main() |