diff options
Diffstat (limited to 'tools/cuda-ab')
| -rw-r--r-- | tools/cuda-ab/launch_count.cu | 25 | ||||
| -rwxr-xr-x | tools/cuda-ab/push_battery.sh | 22 | ||||
| -rwxr-xr-x | tools/cuda-ab/run/launch_count | bin | 0 -> 820840 bytes | |||
| -rwxr-xr-x | tools/cuda-ab/run/streams2d | bin | 0 -> 829296 bytes | |||
| -rwxr-xr-x | tools/cuda-ab/soak.sh | 13 | ||||
| -rw-r--r-- | tools/cuda-ab/streams2d.cu | 197 |
6 files changed, 257 insertions, 0 deletions
diff --git a/tools/cuda-ab/launch_count.cu b/tools/cuda-ab/launch_count.cu new file mode 100644 index 0000000..02db1d6 --- /dev/null +++ b/tools/cuda-ab/launch_count.cu @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BSD-3-Clause +// launch_count.cu - single-launch execution-count repro (r342). +// +// One launch, one atomic increment. The engine-written page witness showed the +// increment applied TWICE (1 -> 2; the 100-increment variant -> 200): every +// forwarded launch executes its engine work twice (one submission, one ring +// publish, completion OK). This is the root cause of the multi-pass content +// class (victory_reduce exactly 2x, scale/saxpy/smiley/multikernel wrong). +// Run in the guest; expected LAUNCH_COUNT=1. +// for a single launch, the launch path executes work more than once. +#include <cstdio> +#include <cuda_runtime.h> +__global__ void inc(unsigned long long *c) { if (threadIdx.x == 0 && blockIdx.x == 0) atomicAdd(c, 1ULL); } +int main(void) { + unsigned long long *d; unsigned long long h = 0; + if (cudaMalloc(&d, 8) != cudaSuccess) { printf("CUDA_ERROR malloc\nSTATUS FAIL\n"); return 1; } + if (cudaMemset(d, 0, 8) != cudaSuccess) { printf("CUDA_ERROR memset\nSTATUS FAIL\n"); return 1; } + inc<<<1, 32>>>(d); + if (cudaGetLastError() != cudaSuccess) { printf("CUDA_ERROR launch\nSTATUS FAIL\n"); return 1; } + if (cudaDeviceSynchronize() != cudaSuccess) { printf("CUDA_ERROR sync\nSTATUS FAIL\n"); return 1; } + if (cudaMemcpy(&h, d, 8, cudaMemcpyDeviceToHost) != cudaSuccess) { printf("CUDA_ERROR d2h\nSTATUS FAIL\n"); return 1; } + printf("LAUNCH_COUNT=%llu (expect 1)\n", h); + printf("STATUS %s\n", h == 1 ? "OK" : "FAIL"); + return h == 1 ? 0 : 1; +} diff --git a/tools/cuda-ab/push_battery.sh b/tools/cuda-ab/push_battery.sh new file mode 100755 index 0000000..644beba --- /dev/null +++ b/tools/cuda-ab/push_battery.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# push_battery.sh - push every CUDA battery binary from tools/cuda-ab/run/ +# to the running guest via scp over the SSH hostfwd (port 2222). +# Usage: push_battery.sh [guest_dir] (default /tmp) +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" +RUN="$HERE/run" +GUEST_DIR="${1:-/tmp}" +CLIENTS="victory_mail smiley vecadd_loop streams2 memcpy_roundtrip multikernel victory_saxpy victory_reduce victory_matmul victory_strided victory_memset2d victory_atomics victory_streams_events victory_math vector_add" +export SSHPASS="${VGPU_GUEST_PASS:-ubuntu}" +FILES="" +for c in $CLIENTS; do [ -x "$RUN/$c" ] && FILES="$FILES $RUN/$c"; done +[ -n "$FILES" ] || { echo "no binaries in $RUN"; exit 1; } +# shellcheck disable=SC2086 +sshpass -e scp -P "${VGPU_GUEST_PORT:-2222}" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o PubkeyAuthentication=no -o PreferredAuthentications=password \ + -o ConnectTimeout=15 $FILES "ubuntu@127.0.0.1:$GUEST_DIR/" && \ +sshpass -e ssh -p "${VGPU_GUEST_PORT:-2222}" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o PubkeyAuthentication=no -o PreferredAuthentications=password \ + "ubuntu@127.0.0.1" "chmod +x $GUEST_DIR/* 2>/dev/null; ls $GUEST_DIR | wc -l; echo BATTERY_PUSHED" diff --git a/tools/cuda-ab/run/launch_count b/tools/cuda-ab/run/launch_count Binary files differnew file mode 100755 index 0000000..0c67f45 --- /dev/null +++ b/tools/cuda-ab/run/launch_count diff --git a/tools/cuda-ab/run/streams2d b/tools/cuda-ab/run/streams2d Binary files differnew file mode 100755 index 0000000..90b6df8 --- /dev/null +++ b/tools/cuda-ab/run/streams2d diff --git a/tools/cuda-ab/soak.sh b/tools/cuda-ab/soak.sh index f870fda..c150692 100755 --- a/tools/cuda-ab/soak.sh +++ b/tools/cuda-ab/soak.sh @@ -135,6 +135,19 @@ run_one() { r="$(gssh "cat $out 2>/dev/null")" || true [ -n "$r" ] && text="$r" printf '%s' "$text" | grep -q 'EXIT=' && break + # The guest wrapper's EXIT= marker can be lost when the guest's + # process reaping stalls (observed: the client reached STATUS OK + # but its zombie was never reaped and `timeout` never wrote the + # marker). A terminal STATUS line is proof of completion by + # itself - accept it after a short grace so the EXIT= marker + # still wins when it does arrive. + if printf '%s' "$text" | grep -qE 'STATUS (OK|FAIL)'; then + sleep 10 + r="$(gssh "cat $out 2>/dev/null")" || true + [ -n "$r" ] && text="$r" + printf '%s' "$text" | grep -q 'EXIT=' && break + break + fi tnow=$(date +%s) if [ $((tnow - t0)) -gt $((TMO + 120)) ]; then if client_alive "$bin"; then diff --git a/tools/cuda-ab/streams2d.cu b/tools/cuda-ab/streams2d.cu new file mode 100644 index 0000000..04a0866 --- /dev/null +++ b/tools/cuda-ab/streams2d.cu @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * streams2d.cu - streams2 with per-array / per-range diagnostic reporting. + * + * Same workload as streams2.cu (two streams, cross-stream event ordering) + * but instead of one "bad" count it reports, per array and per 4 KB block, + * how many elements mismatched, and prints the device pointers so a host + * log can be correlated with what the engine read/wrote. + * + * Output lines: + * DEVICE <name> cc=<major>.<minor> sm=<count> mem=<bytes> + * PTR <name>=0x<hex> (d_a1 d_b1 d_c1 d_a2 d_b2 d_c2) + * DIFF c1 total=<n> blocks=<first block index with mismatches> + * DIFF c2 total=<n> blocks=<...> + * STREAMS n=<count> bad=<mismatches> + * TIMING ms=<elapsed> + * STATUS OK|FAIL + * + * Build: nvcc -O2 --cudart=static -o streams2d streams2d.cu + */ + +#include <cstdio> +#include <cstdlib> +#include <cstdint> +#include <cuda_runtime.h> + +#define CHECK(x) \ + do { \ + cudaError_t e_ = (x); \ + if (e_ != cudaSuccess) { \ + printf("CUDA_ERROR %s at line %d\n", \ + cudaGetErrorString(e_), __LINE__); \ + printf("STATUS FAIL\n"); \ + return 1; \ + } \ + } while (0) + +__global__ void vadd(const float *a, const float *b, float *c, int n) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + + if (i < n) + c[i] = a[i] + b[i]; +} + +__global__ void scale(float *c, float s, int n) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + + if (i < n) + c[i] = c[i] * s; +} + +static void report_diff(const char *name, const float *got, + const float *exp, int n) +{ + int i, bad = 0, first = -1, last = -1; + int blk_hits = 0, blk_first = -1; + int nb = (n + 1023) / 1024; + int b; + + for (i = 0; i < n; i++) { + if (got[i] != exp[i]) { + bad++; + if (first < 0) + first = i; + last = i; + } + } + for (b = 0; b < nb; b++) { + int j, hit = 0; + + for (j = b * 1024; j < (b + 1) * 1024 && j < n; j++) + if (got[j] != exp[j]) { + hit = 1; + break; + } + if (hit) { + blk_hits++; + if (blk_first < 0) + blk_first = b; + } + } + printf("DIFF %s total=%d first=%d last=%d blocks=%d/%d blk_first=%d\n", + name, bad, first, last, blk_hits, nb, blk_first); + if (first >= 0) + printf("DIFF %s sample got[%d]=%a exp[%d]=%a got[%d]=%a exp[%d]=%a\n", + name, first, got[first], first, exp[first], last, + got[last], last, exp[last]); +} + +int main(void) +{ + cudaDeviceProp prop; + const int n = 1 << 15; + const float s2 = 2.0f; + size_t bytes = (size_t)n * sizeof(float); + float *h_a1, *h_b1, *h_c1, *h_a2, *h_b2, *h_c2; + float *d_a1, *d_b1, *d_c1, *d_a2, *d_b2, *d_c2; + cudaStream_t s1, s2s; + cudaEvent_t ev_add1, ev_scale2, t0, t1; + float ms = 0.0f; + int bad = 0, i; + const int grid = (n + 255) / 256; + + setvbuf(stdout, NULL, _IONBF, 0); + + CHECK(cudaGetDeviceProperties(&prop, 0)); + printf("DEVICE %s cc=%d.%d sm=%d mem=%llu\n", prop.name, + prop.major, prop.minor, prop.multiProcessorCount, + (unsigned long long)prop.totalGlobalMem); + + h_a1 = (float *)malloc(bytes); + h_b1 = (float *)malloc(bytes); + h_c1 = (float *)malloc(bytes); + h_a2 = (float *)malloc(bytes); + h_b2 = (float *)malloc(bytes); + h_c2 = (float *)malloc(bytes); + if (!h_a1 || !h_b1 || !h_c1 || !h_a2 || !h_b2 || !h_c2) + return 1; + + for (i = 0; i < n; i++) { + h_a1[i] = (float)(i % 1000) * 0.5f; + h_b1[i] = (float)(i % 997) * 0.25f; + h_a2[i] = (float)(i % 991) * 0.125f; + h_b2[i] = (float)(i % 983) * 0.0625f; + } + + CHECK(cudaMalloc(&d_a1, bytes)); + CHECK(cudaMalloc(&d_b1, bytes)); + CHECK(cudaMalloc(&d_c1, bytes)); + CHECK(cudaMalloc(&d_a2, bytes)); + CHECK(cudaMalloc(&d_b2, bytes)); + CHECK(cudaMalloc(&d_c2, bytes)); + printf("PTR d_a1=%p d_b1=%p d_c1=%p d_a2=%p d_b2=%p d_c2=%p\n", + (void *)d_a1, (void *)d_b1, (void *)d_c1, + (void *)d_a2, (void *)d_b2, (void *)d_c2); + + CHECK(cudaStreamCreate(&s1)); + CHECK(cudaStreamCreate(&s2s)); + CHECK(cudaEventCreate(&ev_add1)); + CHECK(cudaEventCreate(&ev_scale2)); + CHECK(cudaEventCreate(&t0)); + CHECK(cudaEventCreate(&t1)); + + CHECK(cudaEventRecord(t0)); + + CHECK(cudaMemcpyAsync(d_a1, h_a1, bytes, cudaMemcpyHostToDevice, s1)); + CHECK(cudaMemcpyAsync(d_b1, h_b1, bytes, cudaMemcpyHostToDevice, s1)); + CHECK(cudaMemcpyAsync(d_a2, h_a2, bytes, cudaMemcpyHostToDevice, s2s)); + CHECK(cudaMemcpyAsync(d_b2, h_b2, bytes, cudaMemcpyHostToDevice, s2s)); + vadd<<<grid, 256, 0, s1>>>(d_a1, d_b1, d_c1, n); + vadd<<<grid, 256, 0, s2s>>>(d_a2, d_b2, d_c2, n); + CHECK(cudaGetLastError()); + + CHECK(cudaEventRecord(ev_add1, s1)); + CHECK(cudaStreamWaitEvent(s2s, ev_add1, 0)); + scale<<<grid, 256, 0, s2s>>>(d_c2, s2, n); + CHECK(cudaGetLastError()); + + CHECK(cudaEventRecord(ev_scale2, s2s)); + CHECK(cudaStreamWaitEvent(s1, ev_scale2, 0)); + CHECK(cudaMemcpyAsync(h_c1, d_c1, bytes, cudaMemcpyDeviceToHost, s1)); + CHECK(cudaMemcpyAsync(h_c2, d_c2, bytes, cudaMemcpyDeviceToHost, s2s)); + CHECK(cudaEventRecord(t1, s1)); + + CHECK(cudaStreamSynchronize(s1)); + CHECK(cudaStreamSynchronize(s2s)); + CHECK(cudaEventElapsedTime(&ms, t0, t1)); + + report_diff("c1", h_c1, h_a1, n); + { + float *exp2 = (float *)malloc(bytes); + + for (i = 0; i < n; i++) + exp2[i] = (h_a2[i] + h_b2[i]) * s2; + report_diff("c2", h_c2, exp2, n); + for (i = 0; i < n; i++) { + if (h_c1[i] != h_a1[i] + h_b1[i]) + bad++; + if (h_c2[i] != exp2[i]) + bad++; + } + free(exp2); + } + + printf("STREAMS n=%d bad=%d\n", n, bad); + printf("TIMING ms=%.3f\n", ms); + + if (bad != 0) { + printf("STATUS FAIL\n"); + return 1; + } + + printf("STATUS OK\n"); + return 0; +} |