vgsp: host CUDA bridge + sema release recovery; greeneye simulator wave

- kernel: gsp_cuda_bridge (QMD launch capture, debugfs job/out/done, param+output resolution cached from inline writes), sema drain fixes (never hold/starve a channel), GR ring autopin + counter autoreg, per-page push reads, counter-derive gate; module builds clean
- tools/cuda-bridge: CUDA driver-API daemon executing mail/vadd kernels on the physical GPU, CUBB2/CBD2 v2 protocol, layout check + self-tests
- tools/guest-diag + tools/cuda-ab: scanner/pin fixes, victory push-only mode
- debug/risc-v-gsp-rm-vm: Phase S simulator wave (GSP device model, boot ladder, triage/re-demo harnesses, docs 30-41), GPGPU-sim scaffolding
- STATE-CURRENT/docs updates

1 files changed, 256 insertions, 0 deletions

diff --git a/tools/cuda-bridge/kernels_ptx.h b/tools/cuda-bridge/kernels_ptx.h
new file mode 100644
index 0000000..41f08fd
--- /dev/null
+++ b/tools/cuda-bridge/kernels_ptx.h
@@ -0,0 +1,256 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* generated by embed_ptx.py from kernels.ptx - do not edit */
+static const unsigned char kernels_ptx[] = {
+	0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61,
+	0x74, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x4e, 0x56, 0x49, 0x44, 0x49,
+	0x41, 0x20, 0x4e, 0x56, 0x56, 0x4d, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x69,
+	0x6c, 0x65, 0x72, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x6f,
+	0x6d, 0x70, 0x69, 0x6c, 0x65, 0x72, 0x20, 0x42, 0x75, 0x69, 0x6c, 0x64,
+	0x20, 0x49, 0x44, 0x3a, 0x20, 0x43, 0x4c, 0x2d, 0x33, 0x32, 0x32, 0x36,
+	0x37, 0x33, 0x30, 0x32, 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x75, 0x64, 0x61,
+	0x20, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x20, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x6c, 0x65,
+	0x61, 0x73, 0x65, 0x20, 0x31, 0x32, 0x2e, 0x30, 0x2c, 0x20, 0x56, 0x31,
+	0x32, 0x2e, 0x30, 0x2e, 0x31, 0x34, 0x30, 0x0a, 0x2f, 0x2f, 0x20, 0x42,
+	0x61, 0x73, 0x65, 0x64, 0x20, 0x6f, 0x6e, 0x20, 0x4e, 0x56, 0x56, 0x4d,
+	0x20, 0x37, 0x2e, 0x30, 0x2e, 0x31, 0x0a, 0x2f, 0x2f, 0x0a, 0x0a, 0x2e,
+	0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x38, 0x2e, 0x30, 0x0a,
+	0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x73, 0x6d, 0x5f, 0x37,
+	0x35, 0x0a, 0x2e, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x73,
+	0x69, 0x7a, 0x65, 0x20, 0x36, 0x34, 0x0a, 0x0a, 0x09, 0x2f, 0x2f, 0x20,
+	0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x6c, 0x09, 0x6d, 0x61, 0x69, 0x6c, 0x5f,
+	0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x0a, 0x2e, 0x67, 0x6c, 0x6f, 0x62,
+	0x61, 0x6c, 0x20, 0x2e, 0x61, 0x6c, 0x69, 0x67, 0x6e, 0x20, 0x31, 0x20,
+	0x2e, 0x62, 0x38, 0x20, 0x6b, 0x4d, 0x61, 0x69, 0x6c, 0x5b, 0x34, 0x30,
+	0x5d, 0x20, 0x3d, 0x20, 0x7b, 0x31, 0x31, 0x36, 0x2c, 0x20, 0x31, 0x30,
+	0x34, 0x2c, 0x20, 0x31, 0x30, 0x31, 0x2c, 0x20, 0x33, 0x32, 0x2c, 0x20,
+	0x31, 0x30, 0x38, 0x2c, 0x20, 0x31, 0x30, 0x35, 0x2c, 0x20, 0x31, 0x31,
+	0x36, 0x2c, 0x20, 0x31, 0x31, 0x36, 0x2c, 0x20, 0x31, 0x30, 0x38, 0x2c,
+	0x20, 0x31, 0x30, 0x31, 0x2c, 0x20, 0x33, 0x32, 0x2c, 0x20, 0x31, 0x30,
+	0x33, 0x2c, 0x20, 0x31, 0x31, 0x32, 0x2c, 0x20, 0x31, 0x31, 0x37, 0x2c,
+	0x20, 0x33, 0x32, 0x2c, 0x20, 0x31, 0x30, 0x39, 0x2c, 0x20, 0x39, 0x37,
+	0x2c, 0x20, 0x31, 0x31, 0x30, 0x2c, 0x20, 0x33, 0x32, 0x2c, 0x20, 0x31,
+	0x31, 0x35, 0x2c, 0x20, 0x39, 0x37, 0x2c, 0x20, 0x31, 0x32, 0x31, 0x2c,
+	0x20, 0x31, 0x31, 0x35, 0x2c, 0x20, 0x33, 0x32, 0x2c, 0x20, 0x31, 0x32,
+	0x31, 0x2c, 0x20, 0x31, 0x31, 0x31, 0x2c, 0x20, 0x31, 0x31, 0x37, 0x2c,
+	0x20, 0x33, 0x39, 0x2c, 0x20, 0x31, 0x31, 0x38, 0x2c, 0x20, 0x31, 0x30,
+	0x31, 0x2c, 0x20, 0x33, 0x32, 0x2c, 0x20, 0x31, 0x30, 0x33, 0x2c, 0x20,
+	0x31, 0x31, 0x31, 0x2c, 0x20, 0x31, 0x31, 0x36, 0x2c, 0x20, 0x33, 0x32,
+	0x2c, 0x20, 0x31, 0x30, 0x39, 0x2c, 0x20, 0x39, 0x37, 0x2c, 0x20, 0x31,
+	0x30, 0x35, 0x2c, 0x20, 0x31, 0x30, 0x38, 0x2c, 0x20, 0x30, 0x7d, 0x3b,
+	0x0a, 0x0a, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x20, 0x2e,
+	0x65, 0x6e, 0x74, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x6b,
+	0x65, 0x72, 0x6e, 0x65, 0x6c, 0x28, 0x0a, 0x09, 0x2e, 0x70, 0x61, 0x72,
+	0x61, 0x6d, 0x20, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x6d, 0x61, 0x69, 0x6c,
+	0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x70, 0x61, 0x72, 0x61,
+	0x6d, 0x5f, 0x30, 0x2c, 0x0a, 0x09, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x20, 0x2e, 0x75, 0x33, 0x32, 0x20, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x6b,
+	0x65, 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f,
+	0x31, 0x0a, 0x29, 0x0a, 0x7b, 0x0a, 0x09, 0x2e, 0x72, 0x65, 0x67, 0x20,
+	0x2e, 0x70, 0x72, 0x65, 0x64, 0x20, 0x09, 0x25, 0x70, 0x3c, 0x32, 0x3e,
+	0x3b, 0x0a, 0x09, 0x2e, 0x72, 0x65, 0x67, 0x20, 0x2e, 0x62, 0x31, 0x36,
+	0x20, 0x09, 0x25, 0x72, 0x73, 0x3c, 0x33, 0x3e, 0x3b, 0x0a, 0x09, 0x2e,
+	0x72, 0x65, 0x67, 0x20, 0x2e, 0x62, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72,
+	0x3c, 0x36, 0x3e, 0x3b, 0x0a, 0x09, 0x2e, 0x72, 0x65, 0x67, 0x20, 0x2e,
+	0x62, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x3c, 0x37, 0x3e, 0x3b,
+	0x0a, 0x0a, 0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x2e, 0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x31, 0x2c, 0x20,
+	0x5b, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c,
+	0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x30, 0x5d, 0x3b, 0x0a, 0x09,
+	0x6c, 0x64, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x2e, 0x75, 0x33, 0x32,
+	0x20, 0x09, 0x25, 0x72, 0x32, 0x2c, 0x20, 0x5b, 0x6d, 0x61, 0x69, 0x6c,
+	0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x70, 0x61, 0x72, 0x61,
+	0x6d, 0x5f, 0x31, 0x5d, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e, 0x75,
+	0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x33, 0x2c, 0x20, 0x25, 0x63, 0x74,
+	0x61, 0x69, 0x64, 0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e,
+	0x75, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x34, 0x2c, 0x20, 0x25, 0x6e,
+	0x74, 0x69, 0x64, 0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e,
+	0x75, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x35, 0x2c, 0x20, 0x25, 0x74,
+	0x69, 0x64, 0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x61, 0x64, 0x2e, 0x6c,
+	0x6f, 0x2e, 0x73, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x31, 0x2c, 0x20,
+	0x25, 0x72, 0x33, 0x2c, 0x20, 0x25, 0x72, 0x34, 0x2c, 0x20, 0x25, 0x72,
+	0x35, 0x3b, 0x0a, 0x09, 0x73, 0x65, 0x74, 0x70, 0x2e, 0x67, 0x65, 0x2e,
+	0x73, 0x33, 0x32, 0x20, 0x09, 0x25, 0x70, 0x31, 0x2c, 0x20, 0x25, 0x72,
+	0x31, 0x2c, 0x20, 0x25, 0x72, 0x32, 0x3b, 0x0a, 0x09, 0x40, 0x25, 0x70,
+	0x31, 0x20, 0x62, 0x72, 0x61, 0x20, 0x09, 0x24, 0x4c, 0x5f, 0x5f, 0x42,
+	0x42, 0x30, 0x5f, 0x32, 0x3b, 0x0a, 0x0a, 0x09, 0x63, 0x76, 0x74, 0x61,
+	0x2e, 0x74, 0x6f, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e, 0x75,
+	0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x32, 0x2c, 0x20, 0x25, 0x72,
+	0x64, 0x31, 0x3b, 0x0a, 0x09, 0x63, 0x76, 0x74, 0x2e, 0x73, 0x36, 0x34,
+	0x2e, 0x73, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x64, 0x33, 0x2c, 0x20,
+	0x25, 0x72, 0x31, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e, 0x75, 0x36,
+	0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x34, 0x2c, 0x20, 0x6b, 0x4d, 0x61,
+	0x69, 0x6c, 0x3b, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x2e, 0x73, 0x36, 0x34,
+	0x20, 0x09, 0x25, 0x72, 0x64, 0x35, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x34,
+	0x2c, 0x20, 0x25, 0x72, 0x64, 0x33, 0x3b, 0x0a, 0x09, 0x6c, 0x64, 0x2e,
+	0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e, 0x6e, 0x63, 0x2e, 0x75, 0x38,
+	0x20, 0x09, 0x25, 0x72, 0x73, 0x31, 0x2c, 0x20, 0x5b, 0x25, 0x72, 0x64,
+	0x35, 0x5d, 0x3b, 0x0a, 0x09, 0x63, 0x76, 0x74, 0x2e, 0x75, 0x31, 0x36,
+	0x2e, 0x75, 0x38, 0x20, 0x09, 0x25, 0x72, 0x73, 0x32, 0x2c, 0x20, 0x25,
+	0x72, 0x73, 0x31, 0x3b, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x2e, 0x73, 0x36,
+	0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x36, 0x2c, 0x20, 0x25, 0x72, 0x64,
+	0x32, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x33, 0x3b, 0x0a, 0x09, 0x73, 0x74,
+	0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e, 0x75, 0x38, 0x20, 0x09,
+	0x5b, 0x25, 0x72, 0x64, 0x36, 0x5d, 0x2c, 0x20, 0x25, 0x72, 0x73, 0x32,
+	0x3b, 0x0a, 0x0a, 0x24, 0x4c, 0x5f, 0x5f, 0x42, 0x42, 0x30, 0x5f, 0x32,
+	0x3a, 0x0a, 0x09, 0x72, 0x65, 0x74, 0x3b, 0x0a, 0x0a, 0x7d, 0x0a, 0x09,
+	0x2f, 0x2f, 0x20, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x6c, 0x09, 0x76, 0x61,
+	0x64, 0x64, 0x0a, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x20,
+	0x2e, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x20, 0x76, 0x61, 0x64, 0x64, 0x28,
+	0x0a, 0x09, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x20, 0x2e, 0x75, 0x36,
+	0x34, 0x20, 0x76, 0x61, 0x64, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x5f, 0x30, 0x2c, 0x0a, 0x09, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x20,
+	0x2e, 0x75, 0x36, 0x34, 0x20, 0x76, 0x61, 0x64, 0x64, 0x5f, 0x70, 0x61,
+	0x72, 0x61, 0x6d, 0x5f, 0x31, 0x2c, 0x0a, 0x09, 0x2e, 0x70, 0x61, 0x72,
+	0x61, 0x6d, 0x20, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x76, 0x61, 0x64, 0x64,
+	0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x32, 0x2c, 0x0a, 0x09, 0x2e,
+	0x70, 0x61, 0x72, 0x61, 0x6d, 0x20, 0x2e, 0x75, 0x33, 0x32, 0x20, 0x76,
+	0x61, 0x64, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x33, 0x0a,
+	0x29, 0x0a, 0x7b, 0x0a, 0x09, 0x2e, 0x72, 0x65, 0x67, 0x20, 0x2e, 0x70,
+	0x72, 0x65, 0x64, 0x20, 0x09, 0x25, 0x70, 0x3c, 0x32, 0x3e, 0x3b, 0x0a,
+	0x09, 0x2e, 0x72, 0x65, 0x67, 0x20, 0x2e, 0x66, 0x33, 0x32, 0x20, 0x09,
+	0x25, 0x66, 0x3c, 0x34, 0x3e, 0x3b, 0x0a, 0x09, 0x2e, 0x72, 0x65, 0x67,
+	0x20, 0x2e, 0x62, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x3c, 0x36, 0x3e,
+	0x3b, 0x0a, 0x09, 0x2e, 0x72, 0x65, 0x67, 0x20, 0x2e, 0x62, 0x36, 0x34,
+	0x20, 0x09, 0x25, 0x72, 0x64, 0x3c, 0x31, 0x31, 0x3e, 0x3b, 0x0a, 0x0a,
+	0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x2e, 0x75,
+	0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x31, 0x2c, 0x20, 0x5b, 0x76,
+	0x61, 0x64, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x30, 0x5d,
+	0x3b, 0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x2e,
+	0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x32, 0x2c, 0x20, 0x5b,
+	0x76, 0x61, 0x64, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x31,
+	0x5d, 0x3b, 0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x2e, 0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x33, 0x2c, 0x20,
+	0x5b, 0x76, 0x61, 0x64, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f,
+	0x32, 0x5d, 0x3b, 0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x70, 0x61, 0x72, 0x61,
+	0x6d, 0x2e, 0x75, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x32, 0x2c, 0x20,
+	0x5b, 0x76, 0x61, 0x64, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f,
+	0x33, 0x5d, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e, 0x75, 0x33, 0x32,
+	0x20, 0x09, 0x25, 0x72, 0x33, 0x2c, 0x20, 0x25, 0x63, 0x74, 0x61, 0x69,
+	0x64, 0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e, 0x75, 0x33,
+	0x32, 0x20, 0x09, 0x25, 0x72, 0x34, 0x2c, 0x20, 0x25, 0x6e, 0x74, 0x69,
+	0x64, 0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e, 0x75, 0x33,
+	0x32, 0x20, 0x09, 0x25, 0x72, 0x35, 0x2c, 0x20, 0x25, 0x74, 0x69, 0x64,
+	0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x61, 0x64, 0x2e, 0x6c, 0x6f, 0x2e,
+	0x73, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x31, 0x2c, 0x20, 0x25, 0x72,
+	0x33, 0x2c, 0x20, 0x25, 0x72, 0x34, 0x2c, 0x20, 0x25, 0x72, 0x35, 0x3b,
+	0x0a, 0x09, 0x73, 0x65, 0x74, 0x70, 0x2e, 0x67, 0x65, 0x2e, 0x73, 0x33,
+	0x32, 0x20, 0x09, 0x25, 0x70, 0x31, 0x2c, 0x20, 0x25, 0x72, 0x31, 0x2c,
+	0x20, 0x25, 0x72, 0x32, 0x3b, 0x0a, 0x09, 0x40, 0x25, 0x70, 0x31, 0x20,
+	0x62, 0x72, 0x61, 0x20, 0x09, 0x24, 0x4c, 0x5f, 0x5f, 0x42, 0x42, 0x31,
+	0x5f, 0x32, 0x3b, 0x0a, 0x0a, 0x09, 0x63, 0x76, 0x74, 0x61, 0x2e, 0x74,
+	0x6f, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e, 0x75, 0x36, 0x34,
+	0x20, 0x09, 0x25, 0x72, 0x64, 0x34, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x31,
+	0x3b, 0x0a, 0x09, 0x6d, 0x75, 0x6c, 0x2e, 0x77, 0x69, 0x64, 0x65, 0x2e,
+	0x73, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x64, 0x35, 0x2c, 0x20, 0x25,
+	0x72, 0x31, 0x2c, 0x20, 0x34, 0x3b, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x2e,
+	0x73, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x36, 0x2c, 0x20, 0x25,
+	0x72, 0x64, 0x34, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x35, 0x3b, 0x0a, 0x09,
+	0x63, 0x76, 0x74, 0x61, 0x2e, 0x74, 0x6f, 0x2e, 0x67, 0x6c, 0x6f, 0x62,
+	0x61, 0x6c, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x37,
+	0x2c, 0x20, 0x25, 0x72, 0x64, 0x32, 0x3b, 0x0a, 0x09, 0x61, 0x64, 0x64,
+	0x2e, 0x73, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x38, 0x2c, 0x20,
+	0x25, 0x72, 0x64, 0x37, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x35, 0x3b, 0x0a,
+	0x09, 0x6c, 0x64, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e, 0x66,
+	0x33, 0x32, 0x20, 0x09, 0x25, 0x66, 0x31, 0x2c, 0x20, 0x5b, 0x25, 0x72,
+	0x64, 0x38, 0x5d, 0x3b, 0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x67, 0x6c, 0x6f,
+	0x62, 0x61, 0x6c, 0x2e, 0x66, 0x33, 0x32, 0x20, 0x09, 0x25, 0x66, 0x32,
+	0x2c, 0x20, 0x5b, 0x25, 0x72, 0x64, 0x36, 0x5d, 0x3b, 0x0a, 0x09, 0x61,
+	0x64, 0x64, 0x2e, 0x66, 0x33, 0x32, 0x20, 0x09, 0x25, 0x66, 0x33, 0x2c,
+	0x20, 0x25, 0x66, 0x32, 0x2c, 0x20, 0x25, 0x66, 0x31, 0x3b, 0x0a, 0x09,
+	0x63, 0x76, 0x74, 0x61, 0x2e, 0x74, 0x6f, 0x2e, 0x67, 0x6c, 0x6f, 0x62,
+	0x61, 0x6c, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x39,
+	0x2c, 0x20, 0x25, 0x72, 0x64, 0x33, 0x3b, 0x0a, 0x09, 0x61, 0x64, 0x64,
+	0x2e, 0x73, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x31, 0x30, 0x2c,
+	0x20, 0x25, 0x72, 0x64, 0x39, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x35, 0x3b,
+	0x0a, 0x09, 0x73, 0x74, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e,
+	0x66, 0x33, 0x32, 0x20, 0x09, 0x5b, 0x25, 0x72, 0x64, 0x31, 0x30, 0x5d,
+	0x2c, 0x20, 0x25, 0x66, 0x33, 0x3b, 0x0a, 0x0a, 0x24, 0x4c, 0x5f, 0x5f,
+	0x42, 0x42, 0x31, 0x5f, 0x32, 0x3a, 0x0a, 0x09, 0x72, 0x65, 0x74, 0x3b,
+	0x0a, 0x0a, 0x7d, 0x0a, 0x09, 0x2f, 0x2f, 0x20, 0x2e, 0x67, 0x6c, 0x6f,
+	0x62, 0x6c, 0x09, 0x76, 0x61, 0x64, 0x64, 0x5f, 0x6d, 0x75, 0x74, 0x61,
+	0x74, 0x65, 0x64, 0x0a, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x62, 0x6c, 0x65,
+	0x20, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x20, 0x76, 0x61, 0x64, 0x64,
+	0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x65, 0x64, 0x28, 0x0a, 0x09, 0x2e,
+	0x70, 0x61, 0x72, 0x61, 0x6d, 0x20, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x76,
+	0x61, 0x64, 0x64, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x65, 0x64, 0x5f,
+	0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x30, 0x2c, 0x0a, 0x09, 0x2e, 0x70,
+	0x61, 0x72, 0x61, 0x6d, 0x20, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x76, 0x61,
+	0x64, 0x64, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70,
+	0x61, 0x72, 0x61, 0x6d, 0x5f, 0x31, 0x2c, 0x0a, 0x09, 0x2e, 0x70, 0x61,
+	0x72, 0x61, 0x6d, 0x20, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x76, 0x61, 0x64,
+	0x64, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61,
+	0x72, 0x61, 0x6d, 0x5f, 0x32, 0x2c, 0x0a, 0x09, 0x2e, 0x70, 0x61, 0x72,
+	0x61, 0x6d, 0x20, 0x2e, 0x75, 0x33, 0x32, 0x20, 0x76, 0x61, 0x64, 0x64,
+	0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72,
+	0x61, 0x6d, 0x5f, 0x33, 0x0a, 0x29, 0x0a, 0x7b, 0x0a, 0x09, 0x2e, 0x72,
+	0x65, 0x67, 0x20, 0x2e, 0x70, 0x72, 0x65, 0x64, 0x20, 0x09, 0x25, 0x70,
+	0x3c, 0x32, 0x3e, 0x3b, 0x0a, 0x09, 0x2e, 0x72, 0x65, 0x67, 0x20, 0x2e,
+	0x66, 0x33, 0x32, 0x20, 0x09, 0x25, 0x66, 0x3c, 0x35, 0x3e, 0x3b, 0x0a,
+	0x09, 0x2e, 0x72, 0x65, 0x67, 0x20, 0x2e, 0x62, 0x33, 0x32, 0x20, 0x09,
+	0x25, 0x72, 0x3c, 0x36, 0x3e, 0x3b, 0x0a, 0x09, 0x2e, 0x72, 0x65, 0x67,
+	0x20, 0x2e, 0x62, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x3c, 0x31,
+	0x31, 0x3e, 0x3b, 0x0a, 0x0a, 0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x70, 0x61,
+	0x72, 0x61, 0x6d, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64,
+	0x31, 0x2c, 0x20, 0x5b, 0x76, 0x61, 0x64, 0x64, 0x5f, 0x6d, 0x75, 0x74,
+	0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x30,
+	0x5d, 0x3b, 0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x2e, 0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x32, 0x2c, 0x20,
+	0x5b, 0x76, 0x61, 0x64, 0x64, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x65,
+	0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x31, 0x5d, 0x3b, 0x0a,
+	0x09, 0x6c, 0x64, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x2e, 0x75, 0x36,
+	0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x33, 0x2c, 0x20, 0x5b, 0x76, 0x61,
+	0x64, 0x64, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70,
+	0x61, 0x72, 0x61, 0x6d, 0x5f, 0x32, 0x5d, 0x3b, 0x0a, 0x09, 0x6c, 0x64,
+	0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x2e, 0x75, 0x33, 0x32, 0x20, 0x09,
+	0x25, 0x72, 0x32, 0x2c, 0x20, 0x5b, 0x76, 0x61, 0x64, 0x64, 0x5f, 0x6d,
+	0x75, 0x74, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x5f, 0x33, 0x5d, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e, 0x75, 0x33,
+	0x32, 0x20, 0x09, 0x25, 0x72, 0x33, 0x2c, 0x20, 0x25, 0x63, 0x74, 0x61,
+	0x69, 0x64, 0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e, 0x75,
+	0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x34, 0x2c, 0x20, 0x25, 0x6e, 0x74,
+	0x69, 0x64, 0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x2e, 0x75,
+	0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x35, 0x2c, 0x20, 0x25, 0x74, 0x69,
+	0x64, 0x2e, 0x78, 0x3b, 0x0a, 0x09, 0x6d, 0x61, 0x64, 0x2e, 0x6c, 0x6f,
+	0x2e, 0x73, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x31, 0x2c, 0x20, 0x25,
+	0x72, 0x33, 0x2c, 0x20, 0x25, 0x72, 0x34, 0x2c, 0x20, 0x25, 0x72, 0x35,
+	0x3b, 0x0a, 0x09, 0x73, 0x65, 0x74, 0x70, 0x2e, 0x67, 0x65, 0x2e, 0x73,
+	0x33, 0x32, 0x20, 0x09, 0x25, 0x70, 0x31, 0x2c, 0x20, 0x25, 0x72, 0x31,
+	0x2c, 0x20, 0x25, 0x72, 0x32, 0x3b, 0x0a, 0x09, 0x40, 0x25, 0x70, 0x31,
+	0x20, 0x62, 0x72, 0x61, 0x20, 0x09, 0x24, 0x4c, 0x5f, 0x5f, 0x42, 0x42,
+	0x32, 0x5f, 0x32, 0x3b, 0x0a, 0x0a, 0x09, 0x63, 0x76, 0x74, 0x61, 0x2e,
+	0x74, 0x6f, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e, 0x75, 0x36,
+	0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x34, 0x2c, 0x20, 0x25, 0x72, 0x64,
+	0x31, 0x3b, 0x0a, 0x09, 0x6d, 0x75, 0x6c, 0x2e, 0x77, 0x69, 0x64, 0x65,
+	0x2e, 0x73, 0x33, 0x32, 0x20, 0x09, 0x25, 0x72, 0x64, 0x35, 0x2c, 0x20,
+	0x25, 0x72, 0x31, 0x2c, 0x20, 0x34, 0x3b, 0x0a, 0x09, 0x61, 0x64, 0x64,
+	0x2e, 0x73, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x36, 0x2c, 0x20,
+	0x25, 0x72, 0x64, 0x34, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x35, 0x3b, 0x0a,
+	0x09, 0x63, 0x76, 0x74, 0x61, 0x2e, 0x74, 0x6f, 0x2e, 0x67, 0x6c, 0x6f,
+	0x62, 0x61, 0x6c, 0x2e, 0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64,
+	0x37, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x32, 0x3b, 0x0a, 0x09, 0x61, 0x64,
+	0x64, 0x2e, 0x73, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x38, 0x2c,
+	0x20, 0x25, 0x72, 0x64, 0x37, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x35, 0x3b,
+	0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e,
+	0x66, 0x33, 0x32, 0x20, 0x09, 0x25, 0x66, 0x31, 0x2c, 0x20, 0x5b, 0x25,
+	0x72, 0x64, 0x38, 0x5d, 0x3b, 0x0a, 0x09, 0x6c, 0x64, 0x2e, 0x67, 0x6c,
+	0x6f, 0x62, 0x61, 0x6c, 0x2e, 0x66, 0x33, 0x32, 0x20, 0x09, 0x25, 0x66,
+	0x32, 0x2c, 0x20, 0x5b, 0x25, 0x72, 0x64, 0x36, 0x5d, 0x3b, 0x0a, 0x09,
+	0x61, 0x64, 0x64, 0x2e, 0x66, 0x33, 0x32, 0x20, 0x09, 0x25, 0x66, 0x33,
+	0x2c, 0x20, 0x25, 0x66, 0x32, 0x2c, 0x20, 0x25, 0x66, 0x31, 0x3b, 0x0a,
+	0x09, 0x61, 0x64, 0x64, 0x2e, 0x66, 0x33, 0x32, 0x20, 0x09, 0x25, 0x66,
+	0x34, 0x2c, 0x20, 0x25, 0x66, 0x33, 0x2c, 0x20, 0x30, 0x66, 0x33, 0x46,
+	0x38, 0x30, 0x30, 0x30, 0x30, 0x30, 0x3b, 0x0a, 0x09, 0x63, 0x76, 0x74,
+	0x61, 0x2e, 0x74, 0x6f, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e,
+	0x75, 0x36, 0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x39, 0x2c, 0x20, 0x25,
+	0x72, 0x64, 0x33, 0x3b, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x2e, 0x73, 0x36,
+	0x34, 0x20, 0x09, 0x25, 0x72, 0x64, 0x31, 0x30, 0x2c, 0x20, 0x25, 0x72,
+	0x64, 0x39, 0x2c, 0x20, 0x25, 0x72, 0x64, 0x35, 0x3b, 0x0a, 0x09, 0x73,
+	0x74, 0x2e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x2e, 0x66, 0x33, 0x32,
+	0x20, 0x09, 0x5b, 0x25, 0x72, 0x64, 0x31, 0x30, 0x5d, 0x2c, 0x20, 0x25,
+	0x66, 0x34, 0x3b, 0x0a, 0x0a, 0x24, 0x4c, 0x5f, 0x5f, 0x42, 0x42, 0x32,
+	0x5f, 0x32, 0x3a, 0x0a, 0x09, 0x72, 0x65, 0x74, 0x3b, 0x0a, 0x0a, 0x7d,
+	0x0a, 0x0a,
+	0x00
+};
+static const unsigned int kernels_ptx_len = 2990;