diff options
| author | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-09-13 18:36:21 -0400 |
|---|---|---|
| committer | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-09-13 18:36:21 -0400 |
| commit | 00e54c13c7c9030c4d5f7151467f2d85a3730e29 (patch) | |
| tree | 17a45239df8d2e4912ff3697ed06e85a10b969de /userspace/sdks/nv570/nvalloc | |
| download | open-iov-00e54c13c7c9030c4d5f7151467f2d85a3730e29.tar.gz open-iov-00e54c13c7c9030c4d5f7151467f2d85a3730e29.zip | |
Initial commit: Open-IOV vGPU-IO complete stack
- Firmware decompilation: GSP 570.144W (28.5MB ELF, RV64IMAC), 132 decompiled C files
- Kernel modules: nvidia.ko (34MB, OpenRM), open-iov-vfio.ko (2.8MB, VFIO mdev), gsp-vgpu.ko (1.8MB, guest driver)
- Userspace: libiov.so/libvgpu.so (76KB), example_vgpu (28KB), rpc_emulator.c
- PCI config space emulation (256B with PM, MSI, MSI-X, PCIe caps)
- 12 RPC types fully mapped from firmware decompilation
- BAR trap, WPR, scheduling, notification, migration emulation
- Helper scripts: load_vgpu_stack, create_mdev, launch_qemu_vgpu, test_vgpu_stack, setup_guest_driver, unload_vgpu_stack
- Architecture documentation
- Wiki: 10 technical documents
- Git LFS for large binaries (.ko, .bin, .iso, .img, .o, ghidra repos)
- nvidia-open-gpu-kernel-modules as submodule
Diffstat (limited to 'userspace/sdks/nv570/nvalloc')
99 files changed, 64184 insertions, 0 deletions
diff --git a/userspace/sdks/nv570/nvalloc/common/inc/conf_compute/cc_keyrotation.h b/userspace/sdks/nv570/nvalloc/common/inc/conf_compute/cc_keyrotation.h new file mode 100644 index 0000000..473f294 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/conf_compute/cc_keyrotation.h @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef CC_KEYROTATION_H +#define CC_KEYROTATION_H + +// +// Default threshold value derived from SECURITY_POLICY_ATTACKER_ADVANTAGE_DEFAULT +// Minimum threshold defined based on minimum in confComputeSetKeyRotation. +// +#define KEY_ROTATION_MINIMUM_INTERNAL_THRESHOLD (134217727u) +#define KEY_ROTATION_DEFAULT_INTERNAL_THRESHOLD (24296003999ull) + +#endif // CC_KEYROTATION_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/conf_compute/cc_keystore.h b/userspace/sdks/nv570/nvalloc/common/inc/conf_compute/cc_keystore.h new file mode 100644 index 0000000..c629050 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/conf_compute/cc_keystore.h @@ -0,0 +1,487 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef CC_KEYSTORE_H +#define CC_KEYSTORE_H + +// +// Keys are typically grouped in two's so that the first key is used for CPU-to-GPU encryption +// and the second key is used for GPU-to-CPU encryption. +// + +// Keyspace identifiers. +enum +{ + CC_KEYSPACE_GSP, + CC_KEYSPACE_SEC2, + CC_KEYSPACE_LCE0, + CC_KEYSPACE_LCE1, + CC_KEYSPACE_LCE2, + CC_KEYSPACE_LCE3, + CC_KEYSPACE_LCE4, + CC_KEYSPACE_LCE5, + CC_KEYSPACE_LCE6, + CC_KEYSPACE_LCE7, + CC_KEYSPACE_LCE10, + CC_KEYSPACE_LCE11, + CC_KEYSPACE_LCE12, + CC_KEYSPACE_LCE13, + CC_KEYSPACE_LCE14, + CC_KEYSPACE_LCE15, + CC_KEYSPACE_LCE16, + CC_KEYSPACE_LCE17, + CC_KEYSPACE_SIZE // This is always the last element. +}; + +#define CC_KEYSPACE_SIZE_GH100 (CC_KEYSPACE_LCE7 + 1) + +enum +{ + CC_LKEYID_CPU_GSP_LOCKED_RPC, + CC_LKEYID_GSP_CPU_LOCKED_RPC, + CC_LKEYID_CPU_GSP_DMA, + CC_LKEYID_GSP_CPU_DMA, + CC_LKEYID_CPU_GSP_RESERVED1, + CC_LKEYID_GSP_CPU_REPLAYABLE_FAULT, + CC_LKEYID_CPU_GSP_RESERVED2, + CC_LKEYID_GSP_CPU_NON_REPLAYABLE_FAULT, + CC_LKEYID_SEC2_GSP_LOCKED_RPC, + CC_LKEYID_GSP_SEC2_LOCKED_RPC, + CC_LKEYID_CPU_GSP_NVLE_P2P_WRAPPING, + CC_LKEYID_GSP_CPU_RESERVED_1, + CC_KEYSPACE_GSP_SIZE // This is always the last element. +}; +// The fault buffers only support GPU-to-CPU encryption, so the CPU-to-GPU encryption slot +// is left reserved. + +#define CC_LKEYID_GSP_CPU_LOCKED_RPC_STR "gsp_cpu_locked_rpc" +#define CC_LKEYID_CPU_GSP_LOCKED_RPC_STR "cpu_gsp_locked_rpc" +#define CC_LKEYID_GSP_CPU_DMA_STR "gsp_cpu_dma" +#define CC_LKEYID_CPU_GSP_DMA_STR "cpu_gsp_dma" +#define CC_LKEYID_GSP_CPU_REPLAYABLE_FAULT_STR "gsp_cpu_replayable_fault" +#define CC_LKEYID_GSP_CPU_NON_REPLAYABLE_FAULT_STR "gsp_cpu_non_replayable_fault" +#define CC_LKEYID_CPU_GSP_NVLE_P2P_WRAPPING_STR "cpu_gsp_nvle_p2p_wrapping" + +enum +{ + CC_LKEYID_CPU_SEC2_DATA_USER, + CC_LKEYID_CPU_SEC2_HMAC_USER, + CC_LKEYID_CPU_SEC2_DATA_KERN, + CC_LKEYID_CPU_SEC2_HMAC_KERN, + CC_LKEYID_CPU_SEC2_DATA_SCRUBBER, + CC_LKEYID_CPU_SEC2_HMAC_SCRUBBER, + CC_KEYSPACE_SEC2_SIZE // This is always the last element. +}; + +#define CC_LKEYID_CPU_SEC2_DATA_USER_STR "cpu_sec2_data_user" +#define CC_LKEYID_CPU_SEC2_HMAC_USER_STR "cpu_sec2_hmac_user" +#define CC_LKEYID_CPU_SEC2_DATA_KERN_STR "cpu_sec2_data_kernel" +#define CC_LKEYID_CPU_SEC2_HMAC_KERN_STR "cpu_sec2_hmac_kernel" +#define CC_LKEYID_CPU_SEC2_DATA_SCRUBBER_STR "cpu_sec2_data_scrubber" +#define CC_LKEYID_CPU_SEC2_HMAC_SCRUBBER_STR "cpu_sec2_hmac_scrubber" + +enum +{ + CC_LKEYID_LCE_H2D_USER, + CC_LKEYID_LCE_D2H_USER, + CC_LKEYID_LCE_H2D_KERN, + CC_LKEYID_LCE_D2H_KERN, + CC_LKEYID_LCE_H2D_P2P, + CC_LKEYID_LCE_D2H_P2P, + CC_KEYSPACE_LCE_SIZE // This is always the last element. +}; + + +#define CC_KEYSPACE_TOTAL_SIZE (CC_KEYSPACE_GSP_SIZE + CC_KEYSPACE_SEC2_SIZE + (16 * CC_KEYSPACE_LCE_SIZE)) +#define CC_LKEYID_LCE0_H2D_USER_STR "Lce00_h2d_user" +#define CC_LKEYID_LCE0_D2H_USER_STR "Lce00_d2h_user" +#define CC_LKEYID_LCE0_H2D_KERN_STR "Lce00_h2d_kernel" +#define CC_LKEYID_LCE0_D2H_KERN_STR "Lce00_d2h_kernel" +#define CC_LKEYID_LCE0_H2D_P2P_STR "Lce00_h2d_p2p" +#define CC_LKEYID_LCE0_D2H_P2P_STR "Lce00_d2h_p2p" + +#define CC_LKEYID_LCE1_H2D_USER_STR "Lce01_h2d_user" +#define CC_LKEYID_LCE1_D2H_USER_STR "Lce01_d2h_user" +#define CC_LKEYID_LCE1_H2D_KERN_STR "Lce01_h2d_kernel" +#define CC_LKEYID_LCE1_D2H_KERN_STR "Lce01_d2h_kernel" +#define CC_LKEYID_LCE1_H2D_P2P_STR "Lce01_h2d_p2p" +#define CC_LKEYID_LCE1_D2H_P2P_STR "Lce01_d2h_p2p" + +#define CC_LKEYID_LCE2_H2D_USER_STR "Lce02_h2d_user" +#define CC_LKEYID_LCE2_D2H_USER_STR "Lce02_d2h_user" +#define CC_LKEYID_LCE2_H2D_KERN_STR "Lce02_h2d_kernel" +#define CC_LKEYID_LCE2_D2H_KERN_STR "Lce02_d2h_kernel" +#define CC_LKEYID_LCE2_H2D_P2P_STR "Lce02_h2d_p2p" +#define CC_LKEYID_LCE2_D2H_P2P_STR "Lce02_d2h_p2p" + +#define CC_LKEYID_LCE3_H2D_USER_STR "Lce03_h2d_user" +#define CC_LKEYID_LCE3_D2H_USER_STR "Lce03_d2h_user" +#define CC_LKEYID_LCE3_H2D_KERN_STR "Lce03_h2d_kernel" +#define CC_LKEYID_LCE3_D2H_KERN_STR "Lce03_d2h_kernel" +#define CC_LKEYID_LCE3_H2D_P2P_STR "Lce03_h2d_p2p" +#define CC_LKEYID_LCE3_D2H_P2P_STR "Lce03_d2h_p2p" + +#define CC_LKEYID_LCE4_H2D_USER_STR "Lce04_h2d_user" +#define CC_LKEYID_LCE4_D2H_USER_STR "Lce04_d2h_user" +#define CC_LKEYID_LCE4_H2D_KERN_STR "Lce04_h2d_kernel" +#define CC_LKEYID_LCE4_D2H_KERN_STR "Lce04_d2h_kernel" +#define CC_LKEYID_LCE4_H2D_P2P_STR "Lce04_h2d_p2p" +#define CC_LKEYID_LCE4_D2H_P2P_STR "Lce04_d2h_p2p" + +#define CC_LKEYID_LCE5_H2D_USER_STR "Lce05_h2d_user" +#define CC_LKEYID_LCE5_D2H_USER_STR "Lce05_d2h_user" +#define CC_LKEYID_LCE5_H2D_KERN_STR "Lce05_h2d_kernel" +#define CC_LKEYID_LCE5_D2H_KERN_STR "Lce05_d2h_kernel" +#define CC_LKEYID_LCE5_H2D_P2P_STR "Lce05_h2d_p2p" +#define CC_LKEYID_LCE5_D2H_P2P_STR "Lce05_d2h_p2p" + +#define CC_LKEYID_LCE6_H2D_USER_STR "Lce06_h2d_user" +#define CC_LKEYID_LCE6_D2H_USER_STR "Lce06_d2h_user" +#define CC_LKEYID_LCE6_H2D_KERN_STR "Lce06_h2d_kernel" +#define CC_LKEYID_LCE6_D2H_KERN_STR "Lce06_d2h_kernel" +#define CC_LKEYID_LCE6_H2D_P2P_STR "Lce06_h2d_p2p" +#define CC_LKEYID_LCE6_D2H_P2P_STR "Lce06_d2h_p2p" + +#define CC_LKEYID_LCE7_H2D_USER_STR "Lce07_h2d_user" +#define CC_LKEYID_LCE7_D2H_USER_STR "Lce07_d2h_user" +#define CC_LKEYID_LCE7_H2D_KERN_STR "Lce07_h2d_kernel" +#define CC_LKEYID_LCE7_D2H_KERN_STR "Lce07_d2h_kernel" +#define CC_LKEYID_LCE7_H2D_P2P_STR "Lce07_h2d_p2p" +#define CC_LKEYID_LCE7_D2H_P2P_STR "Lce07_d2h_p2p" + +#define CC_LKEYID_LCE10_H2D_USER_STR "Lce10_h2d_user" +#define CC_LKEYID_LCE10_D2H_USER_STR "Lce10_d2h_user" +#define CC_LKEYID_LCE10_H2D_KERN_STR "Lce10_h2d_kernel" +#define CC_LKEYID_LCE10_D2H_KERN_STR "Lce10_d2h_kernel" +#define CC_LKEYID_LCE10_H2D_P2P_STR "Lce10_h2d_p2p" +#define CC_LKEYID_LCE10_D2H_P2P_STR "Lce10_d2h_p2p" + +#define CC_LKEYID_LCE11_H2D_USER_STR "Lce11_h2d_user" +#define CC_LKEYID_LCE11_D2H_USER_STR "Lce11_d2h_user" +#define CC_LKEYID_LCE11_H2D_KERN_STR "Lce11_h2d_kernel" +#define CC_LKEYID_LCE11_D2H_KERN_STR "Lce11_d2h_kernel" +#define CC_LKEYID_LCE11_H2D_P2P_STR "Lce11_h2d_p2p" +#define CC_LKEYID_LCE11_D2H_P2P_STR "Lce11_d2h_p2p" + +#define CC_LKEYID_LCE12_H2D_USER_STR "Lce12_h2d_user" +#define CC_LKEYID_LCE12_D2H_USER_STR "Lce12_d2h_user" +#define CC_LKEYID_LCE12_H2D_KERN_STR "Lce12_h2d_kernel" +#define CC_LKEYID_LCE12_D2H_KERN_STR "Lce12_d2h_kernel" +#define CC_LKEYID_LCE12_H2D_P2P_STR "Lce12_h2d_p2p" +#define CC_LKEYID_LCE12_D2H_P2P_STR "Lce12_d2h_p2p" + +#define CC_LKEYID_LCE13_H2D_USER_STR "Lce13_h2d_user" +#define CC_LKEYID_LCE13_D2H_USER_STR "Lce13_d2h_user" +#define CC_LKEYID_LCE13_H2D_KERN_STR "Lce13_h2d_kernel" +#define CC_LKEYID_LCE13_D2H_KERN_STR "Lce13_d2h_kernel" +#define CC_LKEYID_LCE13_H2D_P2P_STR "Lce13_h2d_p2p" +#define CC_LKEYID_LCE13_D2H_P2P_STR "Lce13_d2h_p2p" + +#define CC_LKEYID_LCE14_H2D_USER_STR "Lce14_h2d_user" +#define CC_LKEYID_LCE14_D2H_USER_STR "Lce14_d2h_user" +#define CC_LKEYID_LCE14_H2D_KERN_STR "Lce14_h2d_kernel" +#define CC_LKEYID_LCE14_D2H_KERN_STR "Lce14_d2h_kernel" +#define CC_LKEYID_LCE14_H2D_P2P_STR "Lce14_h2d_p2p" +#define CC_LKEYID_LCE14_D2H_P2P_STR "Lce14_d2h_p2p" + +#define CC_LKEYID_LCE15_H2D_USER_STR "Lce15_h2d_user" +#define CC_LKEYID_LCE15_D2H_USER_STR "Lce15_d2h_user" +#define CC_LKEYID_LCE15_H2D_KERN_STR "Lce15_h2d_kernel" +#define CC_LKEYID_LCE15_D2H_KERN_STR "Lce15_d2h_kernel" +#define CC_LKEYID_LCE15_H2D_P2P_STR "Lce15_h2d_p2p" +#define CC_LKEYID_LCE15_D2H_P2P_STR "Lce15_d2h_p2p" + +#define CC_LKEYID_LCE16_H2D_USER_STR "Lce16_h2d_user" +#define CC_LKEYID_LCE16_D2H_USER_STR "Lce16_d2h_user" +#define CC_LKEYID_LCE16_H2D_KERN_STR "Lce16_h2d_kernel" +#define CC_LKEYID_LCE16_D2H_KERN_STR "Lce16_d2h_kernel" +#define CC_LKEYID_LCE16_H2D_P2P_STR "Lce16_h2d_p2p" +#define CC_LKEYID_LCE16_D2H_P2P_STR "Lce16_d2h_p2p" + +#define CC_LKEYID_LCE17_H2D_USER_STR "Lce17_h2d_user" +#define CC_LKEYID_LCE17_D2H_USER_STR "Lce17_d2h_user" +#define CC_LKEYID_LCE17_H2D_KERN_STR "Lce17_h2d_kernel" +#define CC_LKEYID_LCE17_D2H_KERN_STR "Lce17_d2h_kernel" +#define CC_LKEYID_LCE17_H2D_P2P_STR "Lce17_h2d_p2p" +#define CC_LKEYID_LCE17_D2H_P2P_STR "Lce17_d2h_p2p" + +// Generate a global key ID from a keyspace (a) and local key ID (b). +#define CC_GKEYID_GEN(a, b) (NvU32)(((a) << 16) | (b)) + +// Get the keyspace component from a global key ID. +#define CC_GKEYID_GET_KEYSPACE(a) (NvU16)((a) >> 16) + +// Get the local key ID from a global key ID. +#define CC_GKEYID_GET_LKEYID(a) (NvU16)((a) & 0xffff) + +// Decrement/increment the local key ID portion of a global key ID. +#define CC_GKEYID_DEC_LKEYID(a) CC_GKEYID_GEN(CC_GKEYID_GET_KEYSPACE((a)), CC_GKEYID_GET_LKEYID((a)) - 1) +#define CC_GKEYID_INC_LKEYID(a) CC_GKEYID_GEN(CC_GKEYID_GET_KEYSPACE((a)), CC_GKEYID_GET_LKEYID((a)) + 1) + +// Get the unqiue string from a global key ID. +#define CC_GKEYID_GET_STR(a) \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_GSP) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_GSP_CPU_LOCKED_RPC) ? \ + CC_LKEYID_GSP_CPU_LOCKED_RPC_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_GSP_LOCKED_RPC) ? \ + CC_LKEYID_CPU_GSP_LOCKED_RPC_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_GSP_CPU_DMA) ? \ + CC_LKEYID_GSP_CPU_DMA_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_GSP_DMA) ? \ + CC_LKEYID_CPU_GSP_DMA_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_GSP_CPU_REPLAYABLE_FAULT) ? \ + CC_LKEYID_GSP_CPU_REPLAYABLE_FAULT_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_GSP_CPU_NON_REPLAYABLE_FAULT) ? \ + CC_LKEYID_GSP_CPU_NON_REPLAYABLE_FAULT_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_GSP_NVLE_P2P_WRAPPING) ? \ + CC_LKEYID_CPU_GSP_NVLE_P2P_WRAPPING_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_SEC2) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_DATA_USER) ? \ + CC_LKEYID_CPU_SEC2_DATA_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_HMAC_USER) ? \ + CC_LKEYID_CPU_SEC2_HMAC_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_DATA_KERN) ? \ + CC_LKEYID_CPU_SEC2_DATA_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_HMAC_KERN) ? \ + CC_LKEYID_CPU_SEC2_HMAC_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_DATA_SCRUBBER) ? \ + CC_LKEYID_CPU_SEC2_DATA_SCRUBBER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_HMAC_SCRUBBER) ? \ + CC_LKEYID_CPU_SEC2_HMAC_SCRUBBER_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE0) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE0_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE0_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE0_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE0_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE0_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE0_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE1) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE1_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE1_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE1_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE1_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE1_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE1_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE2) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE2_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE2_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE2_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE2_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE2_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE2_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE3) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE3_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE3_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE3_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE3_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE3_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE3_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE4) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE4_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE4_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE4_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE4_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE4_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE4_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE5) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE5_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE5_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE5_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE5_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE5_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE5_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE6) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE6_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE6_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE6_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE6_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE6_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE6_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE7) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE7_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE7_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE7_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE7_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE7_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE7_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE10) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE10_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE10_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE10_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE10_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE10_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE10_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE11) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE11_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE11_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE11_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE11_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE11_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE11_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE12) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE12_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE12_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE12_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE12_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE12_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE12_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE13) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE13_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE13_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE13_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE13_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE13_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE13_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE14) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE14_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE14_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE14_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE14_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE14_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE14_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE15) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE15_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE15_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE15_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE15_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE15_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE15_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE16) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE16_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE16_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE16_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE16_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE16_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE16_D2H_P2P_STR : NULL : \ + (CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE17) ? \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \ + CC_LKEYID_LCE17_H2D_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_USER) ? \ + CC_LKEYID_LCE17_D2H_USER_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_KERN) ? \ + CC_LKEYID_LCE17_H2D_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_KERN) ? \ + CC_LKEYID_LCE17_D2H_KERN_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_P2P) ? \ + CC_LKEYID_LCE17_H2D_P2P_STR : \ + (CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_D2H_P2P) ? \ + CC_LKEYID_LCE17_D2H_P2P_STR : NULL : NULL + +#define CC_EXPORT_MASTER_KEY_SIZE_BYTES 32 + +#define CC_KEYID_PERCHANNEL_INITIAL_KEYSEED "ce_channel" +#define CC_KEYID_PERCHANNEL_UPDATE_KEYSEED "update_keyseeed" +#define CC_KEYID_PERCHANNEL_CPU_GPU_UPDATE_KEY "update_key1" +#define CC_KEYID_PERCHANNEL_GPU_CPU_UPDATE_KEY "update_key2" + +#endif // CC_KEYSTORE_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/dev_ctrl_defines.h b/userspace/sdks/nv570/nvalloc/common/inc/dev_ctrl_defines.h new file mode 100644 index 0000000..fa06bcf --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/dev_ctrl_defines.h @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @brief Defines for simplifying SW accesses to the dev_ctrl interrupt tree. + * These are generic defines ued in addition to architecure-specific + * defines in dev_vm_addendum.h + * + */ + +// +// Notes on the terms used below: +// Subtree: The HW tree is a 64-way tree that consists of 2 TOP level interrupt +// registers, 32 bits each. Each of these 64 is referred to as a subtree. +// Leaf: Each of these 64 subtrees are associated with a pair of LEAF registers +// giving us a total of 128 LEAF registers. +// GPU vector: The 128 LEAF registers give us a total of (128*32) GPU vectors +// giving us a total of 4096 GPU vectors +// + +// +// Given a subtree index, the below macros give us the index of the TOP level +// register and the bit within the TOP level register to program for that +// subtree. +// +#define NV_CTRL_INTR_SUBTREE_TO_TOP_IDX(i) (((NvU32)(i)) / 32) +#define NV_CTRL_INTR_SUBTREE_TO_TOP_BIT(i) (((NvU32)(i)) % 32) + +// +// Given a subtree index, the below macros give us the two LEAF register indices +// that correspond to that subtree. +// +#define NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_START(i) (((NvU32)(i)) * 2) +#define NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_END(i) ((((NvU32)(i)) * 2) + 1) + +#define NV_CTRL_INTR_LEAF_IDX_TO_SUBTREE(i) (((NvU32)(i)) / 2) + +// +// Given a LEAF register index, the below macros give us the range of GPU +// interrupt vectors that correspond to those leafs. +// +#define NV_CTRL_INTR_LEAF_IDX_TO_GPU_VECTOR_START(i) (((NvU32)(i)) * 32) +#define NV_CTRL_INTR_LEAF_IDX_TO_GPU_VECTOR_END(i) ((((NvU32)(i)) * 32) + 31) + +// +// Given a GPU interrupt vector, the below macros give us the index of the +// LEAF register and the bit within the LEAF register to program for that +// GPU interrupt vector. +// +#define NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(i) (((NvU32)(i)) / 32) +#define NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_BIT(i) (((NvU32)(i)) % 32) + +// +// Given a GPU interrupt vector, the below macro gives us the subtree in which +// it belongs. +// +#define NV_CTRL_INTR_GPU_VECTOR_TO_SUBTREE(i) \ + ((NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(i)) / 2) + +// First index of doorbell which is controlled by VF +#define NV_CTRL_INTR_GPU_DOORBELL_INDEX_VF_START 2048 + +// The max number of leaf registers we expect +#define NV_MAX_INTR_LEAVES 16 + +// In SW, this specifies an invalid interrupt vector +#define NV_INTR_VECTOR_INVALID (NV_U32_MAX) diff --git a/userspace/sdks/nv570/nvalloc/common/inc/flcnifcmn.h b/userspace/sdks/nv570/nvalloc/common/inc/flcnifcmn.h new file mode 100644 index 0000000..df4ba6c --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/flcnifcmn.h @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2014-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * flcn Command/Message Interfaces - Common + */ + +#ifndef FLCNIFCMN_H +#define FLCNIFCMN_H + +#include "nvtypes.h" +#include "flcnretval.h" + +/*! + * @ref NvU64_ALIGN32 , NvU64_ALIGN32_PACK, NvU64_ALIGN32_UNPACK + * TODO: Use NV types directly + */ +typedef NvU64_ALIGN32 RM_FLCN_U64; +typedef NvU64_ALIGN32 *PRM_FLCN_U64; + +#define RM_FLCN_U64_IS_ZERO NvU64_ALIGN32_IS_ZERO +#define RM_FLCN_U64_PACK NvU64_ALIGN32_PACK +#define RM_FLCN_U64_UNPACK NvU64_ALIGN32_UNPACK + +/*! + * @brief Header preceding each CMD/MSG exchanged through falcon's queues. + * + * @note Data package sent to falcon are referred as CMDs (commands). + * Data package sent by falcon are referred as MSGs (messages). + */ +typedef struct +{ + /*! + * Unit ID identifies falcon's task/unit receiving/issuing this message. + */ + NvU8 unitId; + /*! + * Total CMD/MSG size (including header). + */ + NvU8 size; + /*! + * Flags identifying state of CMD/MSG. + */ + NvU8 ctrlFlags; + /*! + * Sequence # ID to track each request sent to falcon (where applicable). + */ + NvU8 seqNumId; +} RM_FLCN_QUEUE_HDR, +*PRM_FLCN_QUEUE_HDR; + +/*! + * Convenience macro for determining the size of the falcon's queue header: + */ +#define RM_FLCN_QUEUE_HDR_SIZE sizeof(RM_FLCN_QUEUE_HDR) + +/*! + * Generic Falcon rewind unit ID. + */ +#define RM_FLCN_UNIT_ID_REWIND (0x00U) + +/*! + * Generic CMD structure to hold the header. + */ +typedef struct +{ + RM_FLCN_QUEUE_HDR hdr; + NvU32 cmd; +} RM_FLCN_CMD_GEN; + +/*! + * Generic MSG structure to hold the header. + */ +typedef struct +{ + RM_FLCN_QUEUE_HDR hdr; + NvU32 msg; +} RM_FLCN_MSG_GEN; + +/*! + * Convenience macros for determining the size of body for a command or message: + */ +#define RM_FLCN_CMD_BODY_SIZE(u,t) sizeof(RM_FLCN_##u##_CMD_##t) +#define RM_FLCN_MSG_BODY_SIZE(u,t) sizeof(RM_FLCN_##u##_MSG_##t) + +/*! + * Convenience macros for determining the size of a command or message: + */ +#define RM_FLCN_CMD_SIZE(u,t) \ + (RM_FLCN_QUEUE_HDR_SIZE + RM_FLCN_CMD_BODY_SIZE(u,t)) + +#define RM_FLCN_MSG_SIZE(u,t) \ + (RM_FLCN_QUEUE_HDR_SIZE + RM_FLCN_MSG_BODY_SIZE(u,t)) + +/*! + * Convenience macros for determining the type of a command or message + * (intended to be used symmetrically with the CMD and MSG _SIZE macros): + */ +#define RM_FLCN_CMD_TYPE(u,t) RM_FLCN_##u##_CMD_ID_##t +#define RM_FLCN_MSG_TYPE(u,t) RM_FLCN_##u##_MSG_ID_##t + +/*! + * @brief Falcons' queue header flags (@ref RM_FLCN_QUEUE_HDR::ctrlFlags). + * + * RM_FLCN_QUEUE_FLAGS_STATUS + * Set by command's sender to request back message confirming the completion of + * command's execution. In RM->FLCN communication response is required to free + * command related data tracked within RM (***_SEQ_INFO). + * + * RM_FLCN_QUEUE_HDR_FLAGS_EVENT + * Set by the falcon to distinguish messages sent to RM from command responses. + * + * RM_FLCN_QUEUE_HDR_FLAGS_UNIT_ACK + * Used internally within the falcon to track completed commands when updating + * get (tail) command queue pointers. + */ +#define RM_FLCN_QUEUE_HDR_FLAGS_STATUS NVBIT(0) +#define RM_FLCN_QUEUE_HDR_FLAGS_EVENT NVBIT(2) +#define RM_FLCN_QUEUE_HDR_FLAGS_UNIT_ACK NVBIT(5) + +#endif // FLCNIFCMN_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/flcnretval.h b/userspace/sdks/nv570/nvalloc/common/inc/flcnretval.h new file mode 100644 index 0000000..0e1efe5 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/flcnretval.h @@ -0,0 +1,305 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2008-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @file flcnretval.h + * @brief Defines various status codes that are convenient to relay status + * information in functions. + */ + +#ifndef FLCNRETVAL_H +#define FLCNRETVAL_H + +#include "nvtypes.h" +/*! + * Defines a generic type that may be used to convey status information. This + * is very similar to the RM_STATUS type but smaller in width to save DMEM and + * stack space. + */ +typedef NvU8 FLCN_STATUS; + +#define FLCN_OK (0x00U) +#define FLCN_ERR_BINARY_NOT_STARTED (0xFEU) +#define FLCN_ERROR (0xFFU) + +#define FLCN_ERR_NOT_SUPPORTED (0x01U) +#define FLCN_ERR_INVALID_ARGUMENT (0x02U) +#define FLCN_ERR_ILLEGAL_OPERATION (0x03U) +#define FLCN_ERR_TIMEOUT (0x04U) +#define FLCN_ERR_NO_FREE_MEM (0x05U) +#define FLCN_ERR_HDCP_INVALID_SRM (0x06U) +#define FLCN_ERR_HDCP_RECV_REVOKED (0x07U) +#define FLCN_ERR_RPC_INVALID_INPUT (0x08U) +#define FLCN_ERR_DMA_SUSPENDED (0x09U) +#define FLCN_ERR_MUTEX_ACQUIRED (0x10U) +#define FLCN_ERR_MUTEX_ID_NOT_AVAILABLE (0x11U) +#define FLCN_ERR_OBJECT_NOT_FOUND (0x12U) +#define FLCN_ERR_MSGBOX_TIMEOUT (0x13U) +#define FLCN_ERR_INVALID_INDEX (0x14U) +#define FLCN_ERR_INVALID_FUNCTION (0x15U) +#define FLCN_ERR_INSUFFICIENT_PMB_PLM_PROTECTION (0x16U) +#define FLCN_ERR_DMA_NACK (0x17U) +#define FLCN_ERR_CHIP_NOT_SUPPORTED_FOR_PR (0x18U) +#define FLCN_ERR_BAR0_PRIV_READ_ERROR (0x19U) +#define FLCN_ERR_BAR0_PRIV_WRITE_ERROR (0x1AU) +#define FLCN_ERR_HDCP22_ABORT_AUTHENTICATION (0x1BU) +#define FLCN_ERR_DPU_IS_BUSY (0x1CU) +#define FLCN_ERR_DPU_TIMEOUT_FOR_HDCP_TYPE1_LOCK_REQUEST (0x1DU) +#define FLCN_ERR_HDCP_TYPE1_LOCK_FAILED (0x1EU) +#define FLCN_ERR_HDCP_TYPE1_LOCK_UNKNOWN (0x1FU) +#define FLCN_ERR_WAIT_FOR_BAR0_IDLE_FAILED (0x20U) +#define FLCN_ERR_CSB_PRIV_READ_ERROR (0x21U) +#define FLCN_ERR_CSB_PRIV_WRITE_ERROR (0x22U) +#define FLCN_ERR_DMA_UNEXPECTED_DMAIDX (0x23U) +#define FLCN_ERR_PRIV_SEC_VIOLATION (0x24U) +#define FLCN_ERR_INVALID_VERSION (0x25U) +#define FLCN_ERR_PR_SHARED_STRUCT_INIT_FAILED (0x26U) +#define FLCN_ERR_GPU_IN_DEBUG_MODE (0x27U) +#define FLCN_ERR_HPD_UNPLUG (0x28U) +#define FLCN_ERR_HDCP22_DELAY_ABORT_AUTHENTICATION (0x29U) +#define FLCN_ERR_SECUREBUS_REGISTER_READ_ERROR (0x2AU) +#define FLCN_ERR_SECUREBUS_REGISTER_WRITE_ERROR (0x2BU) +#define FLCN_ERR_HDCP22_FLUSH_TYPE_LOCK_ACTIVE (0x2CU) +#define FLCN_ERR_HDCP22_FLUSH_TYPE_IN_PROGRESS (0x2DU) +#define FLCN_ERR_FEATURE_NOT_ENABLED (0x2EU) +#define FLCN_ERR_OUT_OF_RANGE (0x2FU) + +// I2C Errors +#define FLCN_ERR_I2C_BUSY (0x30U) +#define FLCN_ERR_I2C_NACK_ADDRESS (0x31U) +#define FLCN_ERR_I2C_NACK_BYTE (0x32U) +#define FLCN_ERR_I2C_SIZE (0x33U) +#define FLCN_ERR_I2C_BUS_INVALID (0x34U) + +#define FLCN_ERR_INVALID_STATE (0x35U) +#define FLCN_ERR_RECURSION_LIMIT_EXCEEDED (0x36U) +#define FLCN_ERR_INVALID_CAST (0x37U) + +// AUX Errors +#define FLCN_ERR_AUX_ERROR (0x3AU) +#define FLCN_ERR_AUX_SEMA_ACQUIRED (0x3BU) +#define FLCN_ERR_AUX_SEMA_INVALID_RELEASE (0x3CU) +#define FLCN_ERR_AUX_DEFER_TIMEOUT (0x3DU) +#define FLCN_ERR_MORE_PROCESSING_REQUIRED (0x3EU) +#define FLCN_ERR_DMA_ALIGN (0x3FU) + +// Power-Device Errors +#define FLCN_ERR_PWR_DEVICE_TAMPERED (0x40U) + +#define FLCN_ERR_ITERATION_END (0x41U) + +// Perf change sequence Errors +#define FLCN_ERR_LOCK_NOT_AVAILABLE (0x42U) + +#define FLCN_ERR_STATE_RESET_NEEDED (0x43U) +#define FLCN_ERR_DMA_GENERIC (0x44U) +#define FLCN_ERR_LS_CHK_UCODE_REVOKED (0x45U) +#define FLCN_ERR_ACC_SEQ_MISMATCH (0x46U) +#define FLCN_ERR_SSP_STACK_CHECK_FAILED (0x47U) +#define FLCN_ERR_SE_TRNG_FAILED (0x48U) +#define FLCN_ERR_PROD_MODE_NOT_YET_SUPPORTED (0x49U) + +// SHA HW errors +#define FLCN_ERR_SHA_HW_CHECK_INT_STATUS (0x4AU) +#define FLCN_ERR_SHA_HW_SOFTRESET_REQUIRED (0x4BU) +#define FLCN_ERR_SHA_HW_SOFTRESET_FAILED (0x4CU) +#define FLCN_ERR_SHA_HW_BUSY (0x4DU) + +// +// Add new generic error codes here, do not changes values of exiting error codes, +// because that will affect other binaries and their signatures +// +#define FLCN_ERR_CTXSW_ERROR (0x4EU) +#define FLCN_ERR_SE_SNPKA_HW_CRYPTO_OPERATION_FAILED (0x4FU) + +// VPR SEC2 task errors +#define FLCN_ERR_VPR_APP_INVALID_REQUEST_END_ADDR (0x51U) +#define FLCN_ERR_VPR_APP_INVALID_REQUEST_START_ADDR (0x52U) +#define FLCN_ERR_VPR_APP_SCRUB_VERIF_FAILED (0x53U) +#define FLCN_ERR_VPR_APP_MEMLOCK_ALREADY_SET (0x54U) +#define FLCN_ERR_VPR_APP_INVALID_INDEX (0x55U) +#define FLCN_ERR_VPR_APP_UNEXPECTED_VPR_HANDOFF_FROM_SCRUBBER (0x56U) +#define FLCN_ERR_VPR_APP_CBC_RANGE_CLASH (0x57U) +#define FLCN_ERR_VPR_APP_NOT_SUPPORTED_BY_HW (0x58U) +#define FLCN_ERR_VPR_APP_NOT_SUPPORTED_BY_SW (0x59U) +#define FLCN_ERR_VPR_APP_DISPLAY_VERSION_NOT_SUPPORTED (0x5AU) +#define FLCN_ERR_VPR_APP_VPR_WPR_WRITE_FAILED (0x5BU) +#define FLCN_ERR_VPR_APP_NOTHING_TO_DO (0x5CU) +#define FLCN_ERR_VPR_APP_DISPLAY_NOT_PRESENT (0x5DU) +#define FLCN_ERR_VPR_APP_PREVIOUS_CMD_FAILED_AS_MAX_VPR_IS_0 (0x5EU) +#define FLCN_ERR_VPR_APP_PLM_PROTECTION_NOT_RAISED (0x5FU) +#define FLCN_ERR_VPR_APP_PLM_PROTECTION_ALREADY_RAISED (0x60U) +#define FLCN_ERR_VPR_APP_DISP_FALCON_IS_NOT_IN_LS_MODE (0x61U) +#define FLCN_ERR_VPR_APP_VPR_IS_ALREADY_ENABLED (0x62U) +#define FLCN_ERR_VPR_APP_UNEXPECTEDLY_RUNNING_ON_RISCV (0x63U) +#define FLCN_ERR_INVALID_SR_CONFIGURATION (0x64U) +#define FLCN_ERR_VPR_CMD_NOT_SUPPORTED (0x65U) + +// Clocks Errors +#define FLCN_ERR_CYCLE_DETECTED (0x70U) +#define FLCN_ERR_INVALID_PATH (0x71U) +#define FLCN_ERR_MISMATCHED_TARGET (0x72U) +#define FLCN_ERR_FREQ_NOT_SUPPORTED (0x73U) +#define FLCN_ERR_INVALID_SOURCE (0x74U) +#define FLCN_ERR_NOT_INITIALIZED (0x75U) +#define FLCN_ERR_CLOCK_NOT_SETTLED (0x76U) + +// HDCP2.2 Errors +#define FLCN_ERR_HDCP22_GETDKEY_FAILED (0x80U) +#define FLCN_ERR_HDCP22_H_PRIME (0x81U) +#define FLCN_ERR_HDCP22_CERT_RX (0x82U) +#define FLCN_ERR_HDCP22_PAIRING (0x83U) +#define FLCN_ERR_HDCP22_L_PRIME (0x84U) +#define FLCN_ERR_HDCP22_V_PRIME (0x85U) +#define FLCN_ERR_HDCP22_INVALID_RXIDLIST (0x86U) +#define FLCN_ERR_HDCP22_M_PRIME (0x87U) +#define FLCN_ERR_HDCP22_SEQ_ROLLOVER (0x88U) +#define FLCN_ERR_HDCP22_RSA_HW (0x89U) +#define FLCN_ERR_HDCP22_ECF_TIMESLOT_MISMATCH (0x90U) + +// LibCCC Errors +#define FLCN_ERR_INIT_CRYPTO_DEVICE_FAILED (0x91U) +#define FLCN_ERR_NVPKA_SELECT_ENGINE_FAILED (0x92U) +#define FLCN_ERR_NVPKA_ACQUIRE_MUTEX_FAILED (0x93U) +#define FLCN_ERR_NVPKA_MODULAR_EXP_LOCK_FAILED (0x94U) +#define FLCN_ERR_NVRNG_INIT_CRYPTO_DEVICE_FAILED (0x95U) +#define FLCN_ERR_NVRNG_SELECT_ENGINE_FAILED (0x96U) +#define FLCN_ERR_NVRNG_GENERATE_FAILED (0x97U) +#define FLCN_ERR_NVPKA_ECDSA_INIT_FAILED (0x98U) +#define FLCN_ERR_NVPKA_ECDSA_SIGN_FAILED (0x99U) +#define FLCN_ERR_NVPKA_GET_ECC_KEY_PAIR_FAILED (0x9AU) + +// Heavy Secure Errors +#define FLCN_ERR_HS_CHK_INVALID_INPUT (0xA0U) +#define FLCN_ERR_HS_CHK_CHIP_NOT_SUPPORTED (0xA1U) +#define FLCN_ERR_HS_CHK_UCODE_REVOKED (0xA2U) +#define FLCN_ERR_HS_CHK_NOT_IN_LSMODE (0xA3U) +#define FLCN_ERR_HS_CHK_INVALID_LS_PRIV_LEVEL (0xA4U) +#define FLCN_ERR_HS_CHK_INVALID_REGIONCFG (0xA5U) +#define FLCN_ERR_HS_CHK_PRIV_SEC_DISABLED_ON_PROD (0xA6U) +#define FLCN_ERR_HS_CHK_SW_FUSING_ALLOWED_ON_PROD (0xA7U) +#define FLCN_ERR_HS_CHK_INTERNAL_SKU_ON_PROD (0xA8U) +#define FLCN_ERR_HS_CHK_DEVID_OVERRIDE_ENABLED_ON_PROD (0xA9U) +#define FLCN_ERR_HS_CHK_INCONSISTENT_PROD_MODES (0xAAU) +#define FLCN_ERR_HS_CHK_HUB_ENCRPTION_DISABLED (0xABU) +#define FLCN_ERR_HS_PR_ILLEGAL_LASSAHS_STATE_AT_HS_ENTRY (0xACU) +#define FLCN_ERR_HS_PR_ILLEGAL_LASSAHS_STATE_AT_MPK_DECRYPT (0xADU) +#define FLCN_ERR_HS_PR_ILLEGAL_LASSAHS_STATE_AT_HS_EXIT (0xAEU) +#define FLCN_ERR_HS_PROD_MODE_NOT_YET_SUPPORTED (0xAFU) +#define FLCN_ERR_HS_DEV_VERSION_ON_PROD (0xB0U) +#define FLCN_ERR_HS_PR_LASSAHS_LS_SIG_GRP_MISMATCH (0xB1U) +#define FLCN_ERR_HS_PR_LASSAHS_LS_SIG_GRP_OVERLAYS_CNT (0xB2U) +#define FLCN_ERR_HS_PR_LASSAHS_LS_SIG_GRP_INVALID_VA (0xB3U) +#define FLCN_ERR_HS_MUTEX_ACQUIRE_FAILED (0xB4U) +#define FLCN_ERR_HS_MUTEX_RELEASE_FAILED (0xB5U) +#define FLCN_ERR_HS_PR_MPK_DEC_NEEDS_NEWER_ACR_UDE_SCRUBBER (0xB6U) +#define FLCN_ERR_HS_CHK_ENGID_MISMATCH (0xB7U) +#define FLCN_ERR_HS_OPT_INTERNAL_SKU_CHECK_FAILED (0xB8U) +#define FLCN_ERR_HS_CHK_BOARD_MISMATCH (0xB9U) +#define FLCN_ERR_HS_CHK_DISP_ENG_DISABLED (0xBAU) +#define FLCN_ERR_HS_GEN_RANDOM (0xBBU) +#define FLCN_ERR_HS_CHK_IMPROPERLY_FUSED_BOARD (0xBCU) +#define FLCN_ERR_HS_CHK_HDCP_DISABLED (0xBDU) +#define FLCN_ERR_HS_CHK_HDCP_BLACKLISTED_SKU (0XBEU) +#define FLCN_ERR_HS_SECURE_ACTION_ARG_CHECK_FAILED (0xBFU) +#define FLCN_ERR_HS_CHK_RETURN_PC_AT_HS_ENTRY_IS_OF_HS (0xC0U) +#define FLCN_ERR_HS_CHK_HS_LIB_ENTRY_CALLED_BY_NON_HS (0xC1U) +#define FLCN_ERR_HS_DECODE_TRAP_ALREADY_IN_USE (0xC2U) +#define FLCN_ERR_HS_REGISTER_READ_WRITE_ERROR (0xC3U) +#define FLCN_ERR_HS_CHK_CPUCTL_ALIAS_FALSE (0xC4U) +#define FLCN_ERR_HS_UPDATE_RESET_PLM_ERROR (0xC5U) +#define FLCN_ERR_HS_RNG_CONFIG_ERROR (0xC6U) +#define FLCN_ERR_HS_CHK_NOT_IN_HSMODE (0xC7U) +#define FLCN_ERR_HS_CHK_GFW_CHAIN_OF_TRUST_BROKEN (0xC8U) +#define FLCN_ERR_HS_HDCP22_WRONG_SEQUENCE (0xC9U) +#define FLCN_ERR_HS_HDCP22_INTEGRITY_ERROR (0xCAU) +#define FLCN_ERR_HS_HDCP22_WRONG_TYPE (0xCBU) +#define FLCN_ERR_HS_APM_NOT_ENABLED (0xCCU) +#define FLCN_ERR_HS_APM_SMC_ENABLED (0xCDU) +#define FLCN_ERR_HS_APM_FECS_NOT_HALTED (0xCEU) +#define FLCN_ERR_HS_APM_SCRATCH_PLM_INVALID (0xCFU) +#define FLCN_ERR_HS_APM_SCRATCH_INIT_INVALID (0xD0U) +#define FLCN_ERR_HS_INVALID_KEY (0xD1U) +#define FLCN_ERR_HS_SWKG_INVALID_SIGNATURE (0xD2U) +#define FLCN_ERR_HS_SCP_LIB_DMHASH_FAILED (0xD3U) +#define FLCN_ERR_HS_SCP_LIB_AESKDF_FAILED (0xD4U) +#define FLCN_ERR_INSUFFICIENT_NOPS (0xD5U) + +// +// BAR0/CSB Priv Read/Write Error Handling Defines +// These need to be defined by HW - NV Bug 200198584 +// +#define FLCN_BAR0_PRIV_PRI_ERROR_MASK 0xFFF00000U +#define FLCN_BAR0_PRIV_PRI_ERROR_CODE 0xBAD00000U +#define FLCN_BAR0_PRIV_PRI_RETURN_VAL 0x00BADBADU + +#define FLCN_CSB_PRIV_PRI_ERROR_MASK 0xFFFF0000U +#define FLCN_CSB_PRIV_PRI_ERROR_CODE 0xBADF0000U + +// +// Macro to check FALCON return status +// +#define CHECK_FLCN_STATUS(expr) do { \ + flcnStatus = (expr); \ + if (flcnStatus != FLCN_OK) \ + { \ + goto ErrorExit; \ + } \ + } while (NV_FALSE) + +// Warnings. +#define FLCN_WARN_NOTHING_TO_DO (0xD0U) +#define FLCN_WARN_NOT_QUERIED (0xD1U) +#define FLCN_WARN_DATA_NOT_AVAILABLE (0xD2U) + +// Queue handling Errors +#define FLCN_ERR_QUEUE_MGMT_INVALID_UNIT_ID (0xE0U) +#define FLCN_ERR_QUEUE_MGMT_HEAP_MIRROR_ERR (0xE1U) +#define FLCN_ERR_QUEUE_TASK_INVALID_EVENT_TYPE (0xE2U) +#define FLCN_ERR_QUEUE_TASK_INVALID_UNIT_ID (0xE3U) +#define FLCN_ERR_QUEUE_TASK_INVALID_CMD_TYPE (0xE4U) +#define FLCN_ERR_QUEUE_TASK_SLOT_NOT_AVAILABLE (0xE5U) + +// Posted write errors +#define FLCN_ERR_POSTED_WRITE_FAILURE (0xF0U) +#define FLCN_ERR_POSTED_WRITE_INTERRUPTS_ENABLED (0xF1U) +#define FLCN_ERR_POSTED_WRITE_PRI_CLUSTER_COUNT_MISMATCH (0xF2U) +#define FLCN_ERR_POSTED_WRITE_INCORRECT_PARAMS (0xF3U) + +// Lane Margining errors +#define FLCN_ERR_LM_INVALID_RECEIVER_NUMBER (0xF5U) + +// APM errors +#define FLCN_ERR_APM_NOT_FUSED_FOR_EK (0xF6U) +#define FLCN_ERR_APM_BROM_SIGN_FAIL (0xF7U) +// Booter Reload on SEC2-RTOS errors +#define FLCN_ERR_AUTH_GSP_RM_HANDOFF_FAILED (0xF8U) +#define FLCN_ERR_INVALID_WPRMETA_MAGIC_OR_REVISION (0xF9U) + +// Arithmetic errors +#define FLCN_ERR_ARITHMETIC_OVERFLOW (0xFAU) + +// Pri errros +#define FLCN_ERR_OUTSTANDING_PRI_ERROR (0xFBU) + +#endif // FLCNRETVAL_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_caps_query_rpc.h b/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_caps_query_rpc.h new file mode 100644 index 0000000..dc4f887 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_caps_query_rpc.h @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef FSP_CAPS_QUERY_RPC_H +#define FSP_CAPS_QUERY_RPC_H + +#include "fsp/nvdm_payload_cmd_response.h" + +#pragma pack(1) + + /*! + * @biief Capability query payload command to FSP + */ +typedef struct +{ + NvU8 subMessageId; +} FSP_CAPS_QUERY_RPC_PAYLOAD_PARAMS; + +/*! + * @brief Capability query response payload for FSP capability query + */ +typedef struct +{ + NvU8 nvdmType; + NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse; + NvU8 rspPayload[4]; +} FSP_CAPS_QUERY_RESPONSE_PAYLOAD_PARAMS; + +#pragma pack() + +#endif // FSP_CAPS_QUERY_RPC_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_clock_boost_rpc.h b/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_clock_boost_rpc.h new file mode 100644 index 0000000..7564fb6 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_clock_boost_rpc.h @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef FSP_CLOCK_BOOST_RPC_H +#define FSP_CLOCK_BOOST_RPC_H + + +#define FSP_CLOCK_BOOST_FEATURE_DISABLE_SUBMESSAGE_ID 0x0 +#define FSP_CLOCK_BOOST_FEATURE_ENABLE_SUBMESSAGE_ID 0x1 +#define FSP_CLOCK_BOOST_TRIGGER_RESTORE_SUBMESSAGE_ID 0x2 + +#pragma pack(1) + + /*! + * @brief Clock Boost payload command to FSP + */ +typedef struct +{ + NvU8 subMessageId; +} FSP_CLOCK_BOOST_RPC_PAYLOAD_PARAMS; + +#pragma pack() + +#endif // FSP_CLOCK_BOOST_RPC_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_emem_channels.h b/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_emem_channels.h new file mode 100644 index 0000000..912eeae --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_emem_channels.h @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _FSP_EMEM_CHANNELS_H_ +#define _FSP_EMEM_CHANNELS_H_ + +// +// NVDM (NVIDIA Data Model) overlayed on MCTP (Management Component Transport +// Protocol) and sent over EMEM is the communication mechanism used between FSP +// management partition and CPU-RM/other uprocs. +// + +// RM uses channel 0 for FSP EMEM. +#define FSP_EMEM_CHANNEL_RM 0x0 + +// EMEM channel 0 (RM) is allocated 1K bytes. +#define FSP_EMEM_CHANNEL_RM_SIZE 1024 + +#endif // _FSP_EMEM_CHANNELS_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_tnvl_rpc.h b/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_tnvl_rpc.h new file mode 100644 index 0000000..73e3034 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/fsp/fsp_tnvl_rpc.h @@ -0,0 +1,137 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _FSP_TNVL_RPC_H_ +#define _FSP_TNVL_RPC_H_ + +#define TNVL_CAPS_SUBMESSAGE_ID 0xFF +#define TNVL_GET_ATT_CERTS_SUBMESSAGE_ID 0x0 +#define TNVL_GET_ATT_REPORT_SUBMESSAGE_ID 0x1 +#define TNVL_LOCK_CONFIG_SUBMESSAGE_ID 0x2 + +#define NVSWITCH_IK_HASH_LENGTH (48) +#define NVSWITCH_ATT_CERT_SIZE_FIELD_LENGTH (2) +#define NVSWITCH_ATT_RSVD1_FIELD_LENGTH (2) + +#pragma pack(1) + + /*! + * @brief TNVL payload to query FSP capabilities + */ +typedef struct +{ + NvU8 subMessageId; +} TNVL_RPC_CAPS_PAYLOAD; + +/*! + * @brief TNVL response payload for FSP capability query + */ +typedef struct +{ + NvU8 nvdmType; + NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse; + NvU8 rspPayload[40]; +} TNVL_RPC_CAPS_RSP_PAYLOAD; + + /*! + * @brief TNVL payload to query attestation cert chain + */ +typedef struct +{ + NvU8 subMessageId; + NvU8 rsvd; + NvU8 minorVersion; + NvU8 majorVersion; +} TNVL_GET_ATT_CERTS_CMD_PAYLOAD; + +/*! + * @brief TNVL response payload for attestation cert chain + */ +typedef struct +{ + NvU8 nvdmType; + NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse; + NvU8 subMessageId; + NvU8 rsvd0; + NvU8 minorVersion; + NvU8 majorVersion; + NvU16 certChainLength; + NvU16 rsvd1; + NvU8 devIkHash[NVSWITCH_IK_HASH_LENGTH]; + NvU8 certChain[NVSWITCH_ATTESTATION_CERT_CHAIN_MAX_SIZE]; +} TNVL_GET_ATT_CERTS_RSP_PAYLOAD; + + /*! + * @brief TNVL payload to query attestation report + */ +typedef struct +{ + NvU8 subMessageId; + NvU8 rsvd; + NvU8 minorVersion; + NvU8 majorVersion; + NvU8 nonce[NVSWITCH_NONCE_SIZE]; +} TNVL_GET_ATT_REPORT_CMD_PAYLOAD; + +/*! + * @brief TNVL response payload for attestation report + */ +typedef struct +{ + NvU8 nvdmType; + NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse; + NvU8 subMessageId; + NvU8 rsvd0; + NvU8 minorVersion; + NvU8 majorVersion; + NvU32 measurementSize; + NvU8 measurementBuffer[NVSWITCH_ATTESTATION_REPORT_MAX_SIZE]; +} TNVL_GET_ATT_REPORT_RSP_PAYLOAD; + +/*! + * @brief TNVL payload to send lock config + */ +typedef struct +{ + NvU8 subMessageId; + NvU8 rsvd; + NvU8 minorVersion; + NvU8 majorVersion; +} TNVL_LOCK_CONFIG_CMD_PAYLOAD; + +/*! + * @brief TNVL response payload for lock config + */ +typedef struct +{ + NvU8 nvdmType; + NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse; + NvU8 subMessageId; + NvU8 rsvd0; + NvU8 minorVersion; + NvU8 majorVersion; +} TNVL_LOCK_CONFIG_RSP_PAYLOAD; + +#pragma pack() + +#endif // _FSP_TNVL_RPC_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/fsp/nvdm_payload_cmd_response.h b/userspace/sdks/nv570/nvalloc/common/inc/fsp/nvdm_payload_cmd_response.h new file mode 100644 index 0000000..fa117aa --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/fsp/nvdm_payload_cmd_response.h @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/* + * @file nvdm_payload_cmd_response.h + * @brief Header file containing the definition for the + * NVDM_PAYLOAD_COMMAND_RESPONSE structure, which + * is required for SMBPBI-specific payloads and needs + * to be visible by bothe the PMU and the driver sides. + */ + +#ifndef _NVDM_PAYLOAD_CMD_RESPONSE_H +#define _NVDM_PAYLOAD_CMD_RESPONSE_H + +#pragma pack(1) +typedef struct +{ + NvU32 taskId; + NvU32 commandNvdmType; + NvU32 errorCode; +} NVDM_PAYLOAD_COMMAND_RESPONSE; +#pragma pack() + +#endif // _NVDM_PAYLOAD_CMD_RESPONSE_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fmc_error_code_formatting.h b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fmc_error_code_formatting.h new file mode 100644 index 0000000..2f0a51b --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fmc_error_code_formatting.h @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef GSP_FMC_ERROR_CODES_FORMATTING_H +#define GSP_FMC_ERROR_CODES_FORMATTING_H + +//================================================================================ +// DRF Field Definitions for NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR +// These define the bit ranges for the register fields. +//================================================================================ + +// Error code layout +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_VARIANT 31:28 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_PARTITION 27:24 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_PAYLOAD 23:0 + +// Fields within the PAYLOAD for Variant 0 (SK) +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_SK_ERROR 15:8 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_SK_PHASE 7:0 + +// Fields within the PAYLOAD for Variant 1 (Generic) +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_GENERIC_ADDITIONAL_INFO 23:16 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_GENERIC_ERROR_CODE 15:0 + +// Error code layout for safety +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_VARIANT_XMSB 31 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_VARIANT_XLSB 28 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_PARTITION_XMSB 27 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_PARTITION_XLSB 24 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_PAYLOAD_XMSB 23 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_PAYLOAD_XLSB 0 + +// Fields within the PAYLOAD for Variant 0 (SK) for safety +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_SK_ERROR_XMSB 15 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_SK_ERROR_XLSB 8 + +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_SK_PHASE_XMSB 7 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_SK_PHASE_XLSB 0 + +// Fields within the PAYLOAD for Variant 1 (Generic) for safety +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_GENERIC_ADDITIONAL_INFO_XMSB 23 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_GENERIC_ADDITIONAL_INFO_XLSB 16 + +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_GENERIC_ERROR_CODE_XMSB 15 +#define NV_PBUS_SW_SCRATCH_GSP_FMC_ERROR_GENERIC_ERROR_CODE_XLSB 0 + +//================================================================================ +// Constant Values +//================================================================================ + +// Partition ID Encoding Logic +#define GSP_FMC_PARTITION_ID_MASK 0xFU +#define GSP_FMC_PARTITION_ID_OFFSET 1U + +// Error Variants / Versions +#define GSP_FMC_ERROR_VARIANT_SK 0U +#define GSP_FMC_ERROR_VARIANT_GENERIC 1U + +#endif /* GSP_FMC_ERROR_CODES_FORMATTING_H */ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h new file mode 100644 index 0000000..22a0ef7 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h @@ -0,0 +1,74 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#ifndef GSP_FW_SR_META_H_ +#define GSP_FW_SR_META_H_ + +#define GSP_FW_SR_META_MAGIC 0x8a3bb9e6c6c39d93ULL +#define GSP_FW_SR_META_REVISION 2 + +#define GSP_FW_SR_META_INTERNAL_SIZE 128 + +/*! + * GSP firmware SR metadata + * + * Initialized by CPU-RM and kept in Sysmem. + * Consumed by secure ucode (Booter or ACR_RISCV). + */ +typedef struct +{ + // Magic for verification by secure ucode + NvU64 magic; // = GSP_FW_SR_META_MAGIC; + + // + // Revision number + // Bumped up when we change this interface so it is not backward compatible. + // + NvU64 revision; // = GSP_FW_SR_META_MAGIC_REVISION; + + // Members regarding data in SYSMEM + NvU64 sysmemAddrOfSuspendResumeData; + NvU64 sizeOfSuspendResumeData; + + // + // Internal members for use by secure ucode + // Must be exactly GSP_FW_SR_META_INTERNAL_SIZE bytes. + // + NvU32 internal[32]; + + // Same as flags of GspFwWprMeta + NvU32 flags; + + // Subrevision number used by secure ucode + NvU32 subrevision; + + // + // Pad structure to exactly 256 bytes (1 DMA chunk). + // Padding initialized to zero. + // + NvU32 padding[22]; +} GspFwSRMeta; + +#endif // GSP_FW_SR_META_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h new file mode 100644 index 0000000..35293d1 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h @@ -0,0 +1,242 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#ifndef GSP_FW_WPR_META_H_ +#define GSP_FW_WPR_META_H_ + +/*! + * GSP firmware WPR metadata + * + * Initialized by CPU-RM and DMA'd to FB, at the end of what will be WPR2. + * Verified, and locked in WPR2 by Booter. + * + * Firmware scrubs the last 256mb of FB, no memory outside of this region + * may be used until the FW RM has scrubbed the remainder of memory. + * + * ---------------------------- <- fbSize (end of FB, 1M aligned) + * | VGA WORKSPACE | + * ---------------------------- <- vbiosReservedOffset (64K? aligned) + * | (potential align. gap) | + * ---------------------------- <- gspFwWprEnd + frtsSize + pmuReservedSize + * | PMU mem reservation | + * ---------------------------- <- gspFwWprEnd (128K aligned) + frtsSize + * | FRTS data | (frtsSize is 0 on GA100) + * | ------------------------ | <- frtsOffset + * | BOOT BIN (e.g. SK + BL) | + * ---------------------------- <- bootBinOffset + * | GSP FW ELF | + * ---------------------------- <- gspFwOffset + * | GSP FW (WPR) HEAP | + * ---------------------------- <- gspFwHeapOffset + * | Booter-placed metadata | + * | (struct GspFwWprMeta) | + * ---------------------------- <- gspFwWprStart (128K aligned) + * | GSP FW (non-WPR) HEAP | + * ---------------------------- <- nonWprHeapOffset, gspFwRsvdStart + * (GSP_CARVEOUT_SIZE bytes from end of FB) + */ +typedef struct +{ + // Magic + // BL to use for verification (i.e. Booter locked it in WPR2) + NvU64 magic; // = 0xdc3aae21371a60b3; + + // Revision number of Booter-BL-Sequencer handoff interface + // Bumped up when we change this interface so it is not backward compatible. + // Bumped up when we revoke GSP-RM ucode + NvU64 revision; // = 1; + + // ---- Members regarding data in SYSMEM ---------------------------- + // Consumed by Booter for DMA + + NvU64 sysmemAddrOfRadix3Elf; + NvU64 sizeOfRadix3Elf; + + NvU64 sysmemAddrOfBootloader; + NvU64 sizeOfBootloader; + + // Offsets inside bootloader image needed by Booter + NvU64 bootloaderCodeOffset; + NvU64 bootloaderDataOffset; + NvU64 bootloaderManifestOffset; + + union + { + // Used only at initial boot + struct + { + NvU64 sysmemAddrOfSignature; + NvU64 sizeOfSignature; + }; + + // + // Used at suspend/resume to read GspFwHeapFreeList + // Offset relative to GspFwWprMeta FBMEM PA (gspFwWprStart) + // + struct + { + NvU32 gspFwHeapFreeListWprOffset; + NvU32 unused0; + NvU64 unused1; + }; + }; + + // ---- Members describing FB layout -------------------------------- + NvU64 gspFwRsvdStart; + + NvU64 nonWprHeapOffset; + NvU64 nonWprHeapSize; + + NvU64 gspFwWprStart; + + // GSP-RM to use to setup heap. + NvU64 gspFwHeapOffset; + NvU64 gspFwHeapSize; + + // BL to use to find ELF for jump + NvU64 gspFwOffset; + // Size is sizeOfRadix3Elf above. + + NvU64 bootBinOffset; + // Size is sizeOfBootloader above. + + NvU64 frtsOffset; + NvU64 frtsSize; + + NvU64 gspFwWprEnd; + + // GSP-RM to use for fbRegionInfo? + NvU64 fbSize; + + // ---- Other members ----------------------------------------------- + + // GSP-RM to use for fbRegionInfo? + NvU64 vgaWorkspaceOffset; + NvU64 vgaWorkspaceSize; + + // Boot count. Used to determine whether to load the firmware image. + NvU64 bootCount; + + // This union is organized the way it is to start at an 8-byte boundary and achieve natural + // packing of the internal struct fields. + union + { + struct + { + // TODO: the partitionRpc* fields below do not really belong in this + // structure. The values are patched in by the partition bootstrapper + // when GSP-RM is booted in a partition, and this structure was a + // convenient place for the bootstrapper to access them. These should + // be moved to a different comm. mechanism between the bootstrapper + // and the GSP-RM tasks. + + // Shared partition RPC memory (physical address) + NvU64 partitionRpcAddr; + + // Offsets relative to partitionRpcAddr + NvU16 partitionRpcRequestOffset; + NvU16 partitionRpcReplyOffset; + + // Code section and dataSection offset and size. + NvU32 elfCodeOffset; + NvU32 elfDataOffset; + NvU32 elfCodeSize; + NvU32 elfDataSize; + + // Used during GSP-RM resume to check for revocation + NvU32 lsUcodeVersion; + }; + + struct + { + // Pad for the partitionRpc* fields, plus 4 bytes + NvU32 partitionRpcPadding[4]; + + // CrashCat (contiguous) buffer size/location - occupies same bytes as the + // elf(Code|Data)(Offset|Size) fields above. + // TODO: move to GSP_FMC_INIT_PARAMS + NvU64 sysmemAddrOfCrashReportQueue; + NvU32 sizeOfCrashReportQueue; + + // Pad for the lsUcodeVersion field + NvU32 lsUcodeVersionPadding[1]; + }; + }; + + // Number of VF partitions allocating sub-heaps from the WPR heap + // Used during boot to ensure the heap is adequately sized + NvU8 gspFwHeapVfPartitionCount; + + // Flags to help decide GSP-FW flow. + NvU8 flags; + + // Pad structure to exactly 256 bytes. Can replace padding with additional + // fields without incrementing revision. Padding initialized to 0. + NvU8 padding[2]; + + // + // Starts at gspFwWprEnd+frtsSize b/c FRTS is positioned + // to end where this allocation starts (when RM requests FSP to create + // FRTS). + // + NvU32 pmuReservedSize; + + // BL to use for verification (i.e. Booter says OK to boot) + NvU64 verified; // 0x0 -> unverified, 0xa0a0a0a0a0a0a0a0 -> verified +} GspFwWprMeta; + +#define GSP_FW_WPR_META_VERIFIED 0xa0a0a0a0a0a0a0a0ULL +#define GSP_FW_WPR_META_REVISION 1 +#define GSP_FW_WPR_META_MAGIC 0xdc3aae21371a60b3ULL + +#define GSP_FW_WPR_HEAP_FREE_REGION_COUNT 128 + +typedef struct +{ + // + // offset relative to GspFwWprMeta FBMEM PA + // describes a region at [offs, offs + length) + // + NvU32 offs; // start, inclusive + NvU32 length; +} GspFwHeapFreeRegion; + +typedef struct +{ + NvU64 magic; + NvU32 nregions; + GspFwHeapFreeRegion regions[GSP_FW_WPR_HEAP_FREE_REGION_COUNT]; +} GspFwHeapFreeList; + +#define GSP_FW_HEAP_FREE_LIST_MAGIC 0x4845415046524545ULL + +#define GSP_FW_FLAGS 8:0 +#define GSP_FW_FLAGS_CLOCK_BOOST NVBIT(0) +#define GSP_FW_FLAGS_RECOVERY_MARGIN_PRESENT NVBIT(1) +#define GSP_FW_FLAGS_PPCIE_ENABLED NVBIT(2) +#define GSP_FW_FLAGS_MULTI_GPU_NVLE_ENABLED NVBIT(3) +#define GSP_FW_FLAGS_SCAN_RECOVERY_MARGIN NVBIT(4) + +#endif // GSP_FW_WPR_META_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_proxy_reg.h b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_proxy_reg.h new file mode 100644 index 0000000..28aad84 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gsp_proxy_reg.h @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGSP_PROXY_REG_H +#define NVGSP_PROXY_REG_H + +#if !defined(NVRISCV_LIBFSP_BUILD) || !NVRISCV_LIBFSP_BUILD + +#define NVGSP_PROXY_REG_CONFIDENTIAL_COMPUTE 0:0 +#define NVGSP_PROXY_REG_CONFIDENTIAL_COMPUTE_DISABLE 0x00000000 +#define NVGSP_PROXY_REG_CONFIDENTIAL_COMPUTE_ENABLE 0x00000001 + +#define NVGSP_PROXY_REG_CONF_COMPUTE_EARLY_INIT 1:1 +#define NVGSP_PROXY_REG_CONF_COMPUTE_EARLY_INIT_DISABLE 0x00000000 +#define NVGSP_PROXY_REG_CONF_COMPUTE_EARLY_INIT_ENABLE 0x00000001 + +#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE 2:2 +#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE_DISABLE 0x00000000 +#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE_ENABLE 0x00000001 + +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE 5:3 +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_NONE 0x00000000 +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_PROTECTED_PCIE 0x00000001 +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_NVLE 0x00000002 + +#define NVGSP_PROXY_REG_NVLINK_ENCRYPTION 6:6 +#define NVGSP_PROXY_REG_NVLINK_ENCRYPTION_DISABLE 0x00000000 +#define NVGSP_PROXY_REG_NVLINK_ENCRYPTION_ENABLE 0x00000001 + +#else +#define NVGSP_PROXY_REG_CONFIDENTIAL_COMPUTE 0U:0U +#define NVGSP_PROXY_REG_CONFIDENTIAL_COMPUTE_XLSB 0U +#define NVGSP_PROXY_REG_CONFIDENTIAL_COMPUTE_XMSB 0U +#define NVGSP_PROXY_REG_CONFIDENTIAL_COMPUTE_DISABLE 0x00000000U +#define NVGSP_PROXY_REG_CONFIDENTIAL_COMPUTE_ENABLE 0x00000001U + +#define NVGSP_PROXY_REG_CONF_COMPUTE_EARLY_INIT 1U:1U +#define NVGSP_PROXY_REG_CONF_COMPUTE_EARLY_INIT_XLSB 1U +#define NVGSP_PROXY_REG_CONF_COMPUTE_EARLY_INIT_XMSB 1U +#define NVGSP_PROXY_REG_CONF_COMPUTE_EARLY_INIT_DISABLE 0x00000000U +#define NVGSP_PROXY_REG_CONF_COMPUTE_EARLY_INIT_ENABLE 0x00000001U + +#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE 2U:2U +#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE_XLSB 2U +#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE_XMSB 2U +#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE_DISABLE 0x00000000U +#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE_ENABLE 0x00000001U + +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE 5U:3U +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_XLSB 3U +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_XMSB 5U +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_NONE 0x00000000U +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_PROTECTED_PCIE 0x00000001U +#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_NVLE 0x00000002U + +#define NVGSP_PROXY_REG_NVLINK_ENCRYPTION 6U:6U +#define NVGSP_PROXY_REG_NVLINK_ENCRYPTION_XLSB 6U +#define NVGSP_PROXY_REG_NVLINK_ENCRYPTION_XMSB 6U +#define NVGSP_PROXY_REG_NVLINK_ENCRYPTION_DISABLE 0x00000000U +#define NVGSP_PROXY_REG_NVLINK_ENCRYPTION_ENABLE 0x00000001U + +#endif // !defined(NVRISCV_LIBFSP_BUILD) || !NVRISCV_LIBFSP_BUILD +#endif // NVGSP_PROXY_REG_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/gsp/gspifpub.h b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gspifpub.h new file mode 100644 index 0000000..b366a9d --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/gsp/gspifpub.h @@ -0,0 +1,181 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef GSPIFPUB_H +#define GSPIFPUB_H + +#include <nvtypes.h> + +/*! + * @file gspifpub.h + * @brief GSP Command/Message Interfaces - Published + */ + +/*! + * Target physical memory apertures supported for DMA + */ +typedef enum { + GSP_DMA_TARGET_LOCAL_FB, + GSP_DMA_TARGET_COHERENT_SYSTEM, + GSP_DMA_TARGET_NONCOHERENT_SYSTEM, + GSP_DMA_TARGET_COUNT +} GSP_DMA_TARGET; + +/*! + * @brief GSP-CC Microcode Initialization Parameters + */ +typedef struct GSP_FMC_INIT_PARAMS +{ + // CC initialization "registry keys" + NvU32 regkeys; +} GSP_FMC_INIT_PARAMS; + +/*! + * @brief GSP-ACR BOOT_GSP_RM Command Parameters + * + * The wprCarveout fields have no effect in environments where the WPR can be allocated + * implicitly by ACR. + */ +typedef struct GSP_ACR_BOOT_GSP_RM_PARAMS +{ + // Physical memory aperture through which gspRmDescPa is accessed + GSP_DMA_TARGET target; + // Size in bytes of the GSP-RM descriptor structure + NvU32 gspRmDescSize; + // Physical offset in the target aperture of the GSP-RM descriptor structure + NvU64 gspRmDescOffset; + // Physical offset in FB to set the start of the WPR containing GSP-RM + NvU64 wprCarveoutOffset; + // Size in bytes of the WPR containing GSP-RM + NvU32 wprCarveoutSize; + // Whether to boot GSP-RM or GSP-Proxy through ACR + NvBool bIsGspRmBoot; + // Whether inst-in-sys mode is enabled or not + NvBool bInstInSysMode; +} GSP_ACR_BOOT_GSP_RM_PARAMS; + +/*! + * @brief GSP-RM Parameters + */ +typedef struct GSP_RM_PARAMS +{ + // Physical memory aperture through which bootArgsOffset is accessed + GSP_DMA_TARGET target; + // Physical offset in the memory aperture that will be passed to GSP-RM + NvU64 bootArgsOffset; +} GSP_RM_PARAMS; + +/*! + * @brief GSP-SPDM Parameters + * + * Parameters required to set up a communication mechanism between Kernel-RM + * and SPDM partition inside GSP. + */ +typedef struct GSP_SPDM_PARAMS +{ + // Physical Memory Aperture through which all addresses are accessed + GSP_DMA_TARGET target; + + // Physical offset in the memory aperture where SPDM payload is stored + NvU64 payloadBufferOffset; + + // Size of the above payload buffer + NvU32 payloadBufferSize; +} GSP_SPDM_PARAMS; + +typedef struct GSP_RM_MEM_PARAMS +{ + NvU32 flushSysmemAddrValLo; + NvU32 flushSysmemAddrValHi; +} GSP_RM_MEM_PARAMS; + +/*! + * @brief GSP-CC Microcode Parameters for Boot Partitions + */ +typedef struct GSP_FMC_BOOT_PARAMS +{ + GSP_FMC_INIT_PARAMS initParams; + GSP_ACR_BOOT_GSP_RM_PARAMS bootGspRmParams; + GSP_RM_PARAMS gspRmParams; + GSP_SPDM_PARAMS gspSpdmParams; + GSP_RM_MEM_PARAMS gspRmMemParams; +} GSP_FMC_BOOT_PARAMS; + +/*! + * @brief Inform KeyMgr about which slots needs to be refreshed + */ +typedef enum { + // + // KeyMgr will program the received rotated session key in spare (the once which is not being used for + // current crypto flow) Rx key slot of all ports + // + refreshRxKeySlot, + // + // KeyMgr will program the received rotated session key in Tx key slot of all ports and flip the epoch + // to start using new session key for crypto flow + // + refreshTxKeySlot +} sessionKeyRefreshStage; + +/*! + * @brief Definitions for messages to GSP which will convey NVLE programming keys + */ +#define RM_GSP_NVLE_CMD_ID_UPDATE_SESSION_KEYS (0x80) +#define RM_GSP_NVLE_CMD_ID_REFRESH_SESSION_KEYS (0x81) +#define RM_GSP_NVLE_SESSION_KEY_ENTRY_SET_COUNT (1) +#define RM_GSP_NVLE_AES_256_GCM_KEY_SIZE_BYTES (32) +#define RM_GSP_NVLE_AES_256_GCM_KEY_SIZE_DWORDS (RM_GSP_NVLE_AES_256_GCM_KEY_SIZE_BYTES / sizeof(NvU32)) +#define RM_GSP_NVLE_CMD_TAG_SIZE_BYTES (16) +#define RM_GSP_NVLE_CMD_TAG_SIZE_DWORDS (RM_GSP_NVLE_CMD_TAG_SIZE_BYTES / sizeof(NvU32)) + +typedef struct +{ + NvU32 remoteGpuClid; + NvU32 remoteGpuAlid; + NvU32 key[RM_GSP_NVLE_AES_256_GCM_KEY_SIZE_DWORDS]; + // + // Epoch will inform which keyslot<0,1> is spare in case of refreshRxKeySlot stage and the epoch to which + // the Tx key slot needs to switch to in case of refreshTxKeySlot stage. Epoch is provided only when + // keyslot needs to be refreshed and keymgr will implicitly use epoch = 0 during setup stage + // + NvU8 epoch; + NvBool bValid; +} RM_GSP_NVLE_SESSION_KEY_ENTRY; + +typedef struct +{ + NvU32 cmdId; + NvBool bForKeyRotation; // To be deprecated + RM_GSP_NVLE_SESSION_KEY_ENTRY wrappedKeyEntries[RM_GSP_NVLE_SESSION_KEY_ENTRY_SET_COUNT]; + NvU32 keyEntriesTag[RM_GSP_NVLE_CMD_TAG_SIZE_DWORDS]; +} RM_GSP_NVLE_UPDATE_SESSION_KEYS; + +typedef struct +{ + NvU32 cmdId; + sessionKeyRefreshStage stage; // Inform KeyMgr about which slots needs to be programmmed + RM_GSP_NVLE_SESSION_KEY_ENTRY wrappedKeyEntries[RM_GSP_NVLE_SESSION_KEY_ENTRY_SET_COUNT]; + NvU32 keyEntriesTag[RM_GSP_NVLE_CMD_TAG_SIZE_DWORDS]; +} RM_GSP_NVLE_REFRESH_SESSION_KEYS; + +#endif // GSPIFPUB_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrdem.h b/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrdem.h new file mode 100644 index 0000000..26395e8 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrdem.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) <year> NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _IFR_DEM_H_ +#define _IFR_DEM_H_ + +#define INFOROM_DEM_OBJECT_V1_00_PACKED_SIZE 4144 +#define INFOROM_DEM_OBJECT_V1_00_FIFO_SIZE 4096 + +struct INFOROM_DEM_OBJECT_V1_00 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U032 seqNumber; + inforom_U016 writeOffset; + inforom_U016 readOffset; + inforom_X008 fifoBuffer[INFOROM_DEM_OBJECT_V1_00_FIFO_SIZE]; + inforom_U032 reserved[8]; +}; + +#define INFOROM_DEM_OBJECT_V1_00_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d2w4096x8d" +typedef struct INFOROM_DEM_OBJECT_V1_00 INFOROM_DEM_OBJECT_V1_00; + +#endif // _IFR_DEM_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrecc.h b/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrecc.h new file mode 100644 index 0000000..309b47a --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrecc.h @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _IFRECC_H_ +#define _IFRECC_H_ + +#include "nvtypes.h" +#include "inforom/types.h" + +// NVSwitch ECC v6 object definition +#define INFOROM_ECC_OBJECT_V6_S0_PACKED_SIZE 3808 + +//Used to determine if the entry is empty or not +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_VALID 0:0 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_VALID_FALSE 0 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_VALID_TRUE 1 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_CORRUPT_TIMEDATA 1:1 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_CORRUPT_TIMEDATA_FALSE 0 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_CORRUPT_TIMEDATA_TRUE 1 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_ADDR_VALID 2:2 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_ADDR_VALID_FALSE 0 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_ADDR_VALID_TRUE 1 + +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_LOCATION_LINK_ID 7:0 + +typedef struct INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER +{ + inforom_U008 header; + inforom_U032 errId; + inforom_U032 lastErrorTimestamp; + inforom_U032 averageEventDelta; + inforom_U016 location; + inforom_U016 sublocation; + inforom_U032 correctedCount; + inforom_U032 uncorrectedCount; + inforom_U032 address; + +} INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER; + +#define INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_MAX_COUNT 128 + +typedef struct INFOROM_ECC_OBJECT_V6_S0 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + + NvU64_ALIGN32 uncorrectedTotal; + NvU64_ALIGN32 correctedTotal; + inforom_U032 lastClearedTimestamp; + + INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER errorEntries[INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_MAX_COUNT]; + + inforom_U008 padding[68]; +} INFOROM_ECC_OBJECT_V6_S0; + +#define INFOROM_ECC_OBJECT_V6_S0_HEADER_FMT INFOROM_OBJECT_HEADER_V1_00_FMT + +#define INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_FMT "b3d2w3d" + +#define INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_ARRAY_FMT \ + INFOROM_FMT_REP128(INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_FMT) + +#define INFOROM_ECC_OBJECT_V6_S0_PADDING_FMT "68b" + +#define INFOROM_ECC_OBJECT_V6_S0_FMT INFOROM_ECC_OBJECT_V6_S0_HEADER_FMT "qqd" \ + INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_ARRAY_FMT \ + INFOROM_ECC_OBJECT_V6_S0_PADDING_FMT + +// Error event structure for NVSwitch ECC errors +typedef struct +{ + NvU32 sxid; + NvU32 linkId; + NvBool bAddressValid; + NvU32 address; + + // The timestamp is filled in by the inforom ECC error logging API + NvU32 timestamp; + + NvBool bUncErr; + NvU32 errorCount; +} INFOROM_NVS_ECC_ERROR_EVENT; + +typedef union +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_ECC_OBJECT_V6_S0 v6s; +} INFOROM_ECC_OBJECT; + +typedef struct +{ + const char *pFmt; + NvU8 *pPackedObject; + INFOROM_ECC_OBJECT *pEcc; + + // Signals if there are pending updates to be flushed to InfoROM + NvBool bDirty; +} INFOROM_ECC_STATE, *PINFOROM_ECC_STATE; + +#endif // _IFRECC_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrnvl.h b/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrnvl.h new file mode 100644 index 0000000..f09740d --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrnvl.h @@ -0,0 +1,1561 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2017-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _IFRNVL_H_ +#define _IFRNVL_H_ + +// +// NVL - NVLINK Configuration and Error Data +// + +// Link Remote Endpoint Type +typedef enum +{ + INFOROM_NVLINK_REMOTE_TYPE_UNSUPPORTED = -1, + INFOROM_NVLINK_REMOTE_TYPE_NONE = 0, + INFOROM_NVLINK_REMOTE_TYPE_PEER = 1, + INFOROM_NVLINK_REMOTE_TYPE_SYSTEM = 2, + INFOROM_NVLINK_REMOTE_TYPE_SWITCH = 3, + INFOROM_NVLINK_REMOTE_TYPE_MAX = 3, + INFOROM_NVLINK_NUM_REMOTE_TYPES = 4 +} INFOROM_NVLINK_REMOTE_TYPE; + +// Fatal Errors +typedef enum +{ + INFOROM_NVLINK_FATAL_ERR_TYPE_UNSUPPORTED = -1, + + // + // NVLink 1.0 + // + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_DL_DATA_PARITY = 0, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_DL_CTRL_PARITY = 1, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_PROTOCOL = 2, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_OVERFLOW = 3, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_RAM_DATA_PARITY = 4, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_RAM_HDR_PARITY = 5, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_RESP = 6, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_POISON = 7, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_TX_RAM_DATA_PARITY = 8, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_TX_RAM_HDR_PARITY = 9, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_TX_CREDIT = 10, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_DL_FLOW_CTRL_PARITY = 11, + INFOROM_NVLINK_FATAL_ERR_TYPE_TL_DL_HDR_PARITY = 12, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_RECOVERY_LONG = 13, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_RAM = 14, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_INTERFACE = 15, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_SUBLINK_CHANGE = 16, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_FAULT_SUBLINK_CHANGE = 17, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_FAULT_DL_PROTOCOL = 18, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_LTSSM_FAULT = 19, + INFOROM_NVLINK_FATAL_ERR_TYPE_MAX_V1 = 19, + INFOROM_NVLINK_NUM_V1_FATAL_ERR_TYPES = 20, + + // + // NVLink 2.0 + // + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_DATA_PARITY = 0, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_CTRL_PARITY = 1, + // No direct equivalent to: _TL_RX_PROTOCOL = 2, + // No direct equivalent to: _TL_RX_OVERFLOW = 3, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RAM_DATA_PARITY = 4, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RAM_HDR_PARITY = 5, + // No direct equivalent to: _TL_RX_RESP = 6, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DATA_POISONED_PKT_RCVD = 7, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_RAM_DATA_PARITY = 8, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_RAM_HDR_PARITY = 9, + // No direct equivalent to: _TL_TX_CREDIT = 10, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DL_FLOW_CONTROL_PARITY = 11, + // No direct equivalent to: _TL_DL_HDR_PARITY = 12, + // Identical to NVLink 1.0: _DL_TX_RECOVERY_LONG = 13, + // Identical to NVLink 1.0: _DL_TX_FAULT_RAM = 14, + // Identical to NVLink 1.0: _DL_TX_FAULT_INTERFACE = 15, + // Identical to NVLink 1.0: _DL_TX_FAULT_SUBLINK_CHANGE = 16, + // Identical to NVLink 1.0: _DL_RX_FAULT_SUBLINK_CHANGE = 17, + // Identical to NVLink 1.0: _DL_RX_FAULT_DL_PROTOCOL = 18, + // Identical to NVLink 1.0: _DL_LTSSM_FAULT = 19, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_HDR_PARITY = 20, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_AE_FLIT_RCVD = 21, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_BE_FLIT_RCVD = 22, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_ADDR_ALIGN = 23, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_PKT_LEN = 24, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CMD_ENC = 25, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_DAT_LEN_ENC = 26, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_ADDR_TYPE = 27, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_RSP_STATUS = 28, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_PKT_STATUS = 29, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_REQ = 30, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_RESP = 31, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DAT_LEN_GT_ATOMIC_REQ_MAX_SIZE = 32, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DAT_LEN_GT_RMW_REQ_MAX_SIZE = 33, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DAT_LEN_LT_ATR_RESP_MIN_SIZE = 34, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_PO_FOR_CACHE_ATTR = 35, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_COMPRESSED_RESP = 36, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RESP_STATUS_TARGET = 37, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RESP_STATUS_UNSUPPORTED_REQUEST = 38, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_HDR_OVERFLOW = 39, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DATA_OVERFLOW = 40, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_STOMPED_PKT_RCVD = 41, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_CORRECTABLE_INTERNAL = 42, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_UNSUPPORTED_VC_OVERFLOW = 43, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_UNSUPPORTED_NVLINK_CREDIT_RELEASE = 44, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_UNSUPPORTED_NCISOC_CREDIT_RELEASE = 45, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_HDR_CREDIT_OVERFLOW = 46, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DATA_CREDIT_OVERFLOW = 47, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DL_REPLAY_CREDIT_OVERFLOW = 48, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_UNSUPPORTED_VC_OVERFLOW = 49, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_STOMPED_PKT_SENT = 50, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DATA_POISONED_PKT_SENT = 51, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_RESP_STATUS_TARGET = 52, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_RESP_STATUS_UNSUPPORTED_REQUEST = 53, + INFOROM_NVLINK_FATAL_ERR_TYPE_MIF_RX_RAM_DATA_PARITY = 54, + INFOROM_NVLINK_FATAL_ERR_TYPE_MIF_RX_RAM_HDR_PARITY = 55, + INFOROM_NVLINK_FATAL_ERR_TYPE_MIF_TX_RAM_DATA_PARITY = 56, + INFOROM_NVLINK_FATAL_ERR_TYPE_MIF_TX_RAM_HDR_PARITY = 57, + INFOROM_NVLINK_FATAL_ERR_TYPE_MAX_V2 = 57, + INFOROM_NVLINK_NUM_V2_FATAL_ERR_TYPES = 58, + + // NVLink 3.0 + + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_COLLAPSED_RESPONSE = 58, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_NCISOC_HDR_ECC_DBE = 59, + INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_NCISOC_PARITY = 60, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_LTSSM_FAULT_UP = 61, + INFOROM_NVLINK_FATAL_ERR_TYPE_DL_LTSSM_FAULT_DOWN = 62, + INFOROM_NVLINK_FATAL_ERR_TYPE_MAX_V3 = 62, + INFOROM_NVLINK_NUM_V3_FATAL_ERR_TYPES = 63, + + INFOROM_NVLINK_FATAL_ERR_TYPE_MAX = 62, + INFOROM_NVLINK_NUM_FATAL_ERR_TYPES = 63, +} INFOROM_NVLINK_FATAL_ERR_TYPE; + +typedef enum +{ + // DL RX ----------------------------------------- + // Fatal Counts + INFOROM_NVLINK_DL_RX_FAULT_DL_PROTOCOL_FATAL = 0, + INFOROM_NVLINK_DL_RX_FAULT_SUBLINK_CHANGE_FATAL, + + // Correctable Accumulated Counts + INFOROM_NVLINK_DL_RX_FLIT_CRC_CORR, + INFOROM_NVLINK_DL_RX_LANE0_CRC_CORR, + INFOROM_NVLINK_DL_RX_LANE1_CRC_CORR, + INFOROM_NVLINK_DL_RX_LANE2_CRC_CORR, + INFOROM_NVLINK_DL_RX_LANE3_CRC_CORR, + INFOROM_NVLINK_DL_RX_LINK_REPLAY_EVENTS_CORR, + + // DL TX ----------------------------------------- + // Fatal Counts + INFOROM_NVLINK_DL_TX_FAULT_RAM_FATAL, + INFOROM_NVLINK_DL_TX_FAULT_INTERFACE_FATAL, + INFOROM_NVLINK_DL_TX_FAULT_SUBLINK_CHANGE_FATAL, + + // Correctable Accumulated Counts + INFOROM_NVLINK_DL_TX_LINK_REPLAY_EVENTS_CORR, + + // DL Disassociated ------------------------------ + // Fatal Counts + INFOROM_NVLINK_DL_LTSSM_FAULT_UP_FATAL, + INFOROM_NVLINK_DL_LTSSM_FAULT_DOWN_FATAL, + + // Correctable Accumulated Counts + INFOROM_NVLINK_DL_LINK_RECOVERY_EVENTS_CORR, + + // TLC RX ---------------------------------------- + // Fatal Counts + INFOROM_NVLINK_TLC_RX_DL_HDR_PARITY_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_DL_DATA_PARITY_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_DL_CTRL_PARITY_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_INVALID_AE_FATAL, + INFOROM_NVLINK_TLC_RX_INVALID_BE_FATAL, + INFOROM_NVLINK_TLC_RX_INVALID_ADDR_ALIGN_FATAL, + INFOROM_NVLINK_TLC_RX_PKTLEN_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_RSVD_PACKET_STATUS_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_RSVD_CACHE_ATTR_PROBE_REQ_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_RSVD_CACHE_ATTR_PROBE_RSP_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_DATLEN_GT_RMW_REQ_MAX_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_DATLEN_LT_ATR_RSP_MIN_ERR_FATAL, + INFOROM_NVLINK_TLC_RX_INVALID_CR_FATAL, + INFOROM_NVLINK_TLC_RX_INVALID_COLLAPSED_RESPONSE_FATAL, + INFOROM_NVLINK_TLC_RX_HDR_OVERFLOW_FATAL, + INFOROM_NVLINK_TLC_RX_DATA_OVERFLOW_FATAL, + INFOROM_NVLINK_TLC_RX_STOMP_DETECTED_FATAL, + INFOROM_NVLINK_TLC_RX_RSVD_CMD_ENC_FATAL, + INFOROM_NVLINK_TLC_RX_RSVD_DAT_LEN_ENC_FATAL, + INFOROM_NVLINK_TLC_RX_INVALID_PO_FOR_CACHE_ATTR_FATAL, + // This is only needed for NCISOC support (not used outside SoC?) + // TLC_RX_NCISOC_CREDIT_PARITY_ERR_FATAL_COUNT, + + // Non-Fatal Counts + INFOROM_NVLINK_TLC_RX_RSP_STATUS_HW_ERR_NONFATAL, + INFOROM_NVLINK_TLC_RX_RSP_STATUS_UR_ERR_NONFATAL, + INFOROM_NVLINK_TLC_RX_RSP_STATUS_PRIV_ERR_NONFATAL, + INFOROM_NVLINK_TLC_RX_POISON_NONFATAL, + INFOROM_NVLINK_TLC_RX_AN1_HEARTBEAT_TIMEOUT_NONFATAL, + INFOROM_NVLINK_TLC_RX_ILLEGAL_PRI_WRITE_NONFATAL, + + // Correctable Accumulated Counts + // These should be logged in the ECC object instead + // TLC_RX_HDR_RAM_ECC_SBE_ACCUM, + // TLC_RX_DAT0_RAM_ECC_SBE_ACCUM, + // TLC_RX_DAT1_RAM_ECC_SBE_ACCUM, + + // TLC TX ---------------------------------------- + // Fatal Counts + INFOROM_NVLINK_TLC_TX_DL_CREDIT_PARITY_ERR_FATAL, + // These should be logged in the ECC object instead + // TLC_TX_CREQ_HDR_RAM_ECC_DBE_FATAL_COUNT, + // TLC_TX_RSP_HDR_RAM_ECC_DBE_FATAL_COUNT, + // TLC_TX_RSP1_HDR_RAM_ECC_DBE_FATAL_COUNT, + // TLC_TX_COM_HDR_RAM_ECC_DBE_FATAL_COUNT, + INFOROM_NVLINK_TLC_TX_NCISOC_HDR_ECC_DBE_FATAL, + INFOROM_NVLINK_TLC_TX_NCISOC_PARITY_ERR_FATAL, + + // Non-Fatal Counts + // This is only needed for NCISOC support (not used outside SoC?) + // TLC_TX_NCISOC_DAT_ECC_DBE_NONFATAL_COUNT, + INFOROM_NVLINK_TLC_TX_ILLEGAL_PRI_WRITE_NONFATAL, + INFOROM_NVLINK_TLC_TX_AN1_TIMEOUT_VC0_NONFATAL, + INFOROM_NVLINK_TLC_TX_AN1_TIMEOUT_VC1_NONFATAL, + INFOROM_NVLINK_TLC_TX_AN1_TIMEOUT_VC2_NONFATAL, + INFOROM_NVLINK_TLC_TX_AN1_TIMEOUT_VC3_NONFATAL, + INFOROM_NVLINK_TLC_TX_AN1_TIMEOUT_VC4_NONFATAL, + INFOROM_NVLINK_TLC_TX_AN1_TIMEOUT_VC5_NONFATAL, + INFOROM_NVLINK_TLC_TX_AN1_TIMEOUT_VC6_NONFATAL, + INFOROM_NVLINK_TLC_TX_AN1_TIMEOUT_VC7_NONFATAL, + INFOROM_NVLINK_TLC_TX_POISON_NONFATAL, + INFOROM_NVLINK_TLC_TX_RSP_STATUS_HW_ERR_NONFATAL, + INFOROM_NVLINK_TLC_TX_RSP_STATUS_UR_ERR_NONFATAL, + INFOROM_NVLINK_TLC_TX_RSP_STATUS_PRIV_ERR_NONFATAL, + + // Correctable Counts + // These should be logged in the ECC object instead + // TLC_TX_CREQ_RAMS_ECC_SBE_CORR_ACCUM, + // TLC_TX_RSP_RAMS_ECC_SBE_CORR_ACCUM, + // TLC_TX_RSP1_RAMS_ECC_SBE_CORR_ACCUM, + // TLC_TX_COM_RAMS_ECC_SBE_CORR_ACCUM, + // This is only needed for NCISOC support (not used outside SoC?) + // TLC_TX_NCISOC_ECC_LIMIT_ERR_CORR_ACCUM, + + // NVLIPT Disassociated -------------------------- + // Fatal Counts + INFOROM_NVLINK_NVLIPT_SLEEP_WHILE_ACTIVE_LINK_FATAL, + INFOROM_NVLINK_NVLIPT_RSTSEQ_PHYCTL_TIMEOUT_FATAL, + INFOROM_NVLINK_NVLIPT_RSTSEQ_CLKCTL_TIMEOUT_FATAL, + INFOROM_NVLINK_NVLIPT_CLKCTL_ILLEGAL_REQUEST_FATAL, + INFOROM_NVLINK_NVLIPT_RSTSEQ_PLL_TIMEOUT_FATAL, + INFOROM_NVLINK_NVLIPT_RSTSEQ_PHYARB_TIMEOUT_FATAL, + + // Non-Fatal Counts + INFOROM_NVLINK_NVLIPT_ILLEGAL_LINK_STATE_REQUEST_NONFATAL, + INFOROM_NVLINK_NVLIPT_FAILED_MINION_REQUEST_NONFATAL, + INFOROM_NVLINK_NVLIPT_RESERVED_REQUEST_VALUE_NONFATAL, + INFOROM_NVLINK_NVLIPT_LINK_STATE_WRITE_WHILE_BUSY_NONFATAL, + INFOROM_NVLINK_NVLIPT_WRITE_TO_LOCKED_SYSTEM_REG_NONFATAL, + INFOROM_NVLINK_NVLIPT_LINK_STATE_REQUEST_TIMEOUT_NONFATAL, + + // TODO 3014908 log these in the NVL object until we have ECC object support + INFOROM_NVLINK_TLC_RX_HDR_RAM_ECC_DBE_FATAL, + INFOROM_NVLINK_TLC_RX_DAT0_RAM_ECC_DBE_FATAL, + INFOROM_NVLINK_TLC_RX_DAT1_RAM_ECC_DBE_FATAL, + INFOROM_NVLINK_TLC_TX_CREQ_DAT_RAM_ECC_DBE_NONFATAL, + INFOROM_NVLINK_TLC_TX_RSP_DAT_RAM_ECC_DBE_NONFATAL, + INFOROM_NVLINK_TLC_TX_COM_DAT_RAM_ECC_DBE_NONFATAL, + INFOROM_NVLINK_TLC_TX_RSP1_DAT_RAM_ECC_DBE_FATAL, + + // DL Interrupts ----------------------------------------- + // Fatal Counts + INFOROM_NVLINK_DL_PHY_A_FATAL, + INFOROM_NVLINK_DL_RX_CRC_COUNTER_FATAL, + INFOROM_NVLINK_DL_TX_PL_ERROR_FATAL, + INFOROM_NVLINK_DL_RX_PL_ERROR_FATAL, + + INFOROM_NVLINK_MAX_ERROR_TYPE +} INFOROM_NVLINK_ERROR_TYPES; + +#define INFOROM_NVLINK_V1_FATAL_ERR_TYPES \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_DL_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_DL_CTRL_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_PROTOCOL) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_RAM_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_RAM_HDR_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_RESP) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_RX_POISON) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_TX_RAM_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_TX_RAM_HDR_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_TX_CREDIT) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_DL_FLOW_CTRL_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TL_DL_HDR_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_RECOVERY_LONG) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_RAM) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_INTERFACE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_SUBLINK_CHANGE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_FAULT_SUBLINK_CHANGE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_FAULT_DL_PROTOCOL) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_LTSSM_FAULT) + +#define INFOROM_NVLINK_V2_FATAL_ERR_TYPES \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_CTRL_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RAM_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RAM_HDR_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DATA_POISONED_PKT_RCVD) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_RAM_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_RAM_HDR_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DL_FLOW_CONTROL_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_RECOVERY_LONG) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_RAM) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_INTERFACE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_SUBLINK_CHANGE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_FAULT_SUBLINK_CHANGE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_FAULT_DL_PROTOCOL) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_LTSSM_FAULT) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_HDR_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_AE_FLIT_RCVD) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_BE_FLIT_RCVD) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_ADDR_ALIGN) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_PKT_LEN) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CMD_ENC) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_DAT_LEN_ENC) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_ADDR_TYPE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_RSP_STATUS) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_PKT_STATUS) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_REQ) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_RESP) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DAT_LEN_GT_ATOMIC_REQ_MAX_SIZE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DAT_LEN_GT_RMW_REQ_MAX_SIZE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DAT_LEN_LT_ATR_RESP_MIN_SIZE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_PO_FOR_CACHE_ATTR) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_COMPRESSED_RESP) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RESP_STATUS_TARGET) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RESP_STATUS_UNSUPPORTED_REQUEST) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_HDR_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DATA_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_STOMPED_PKT_RCVD) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_CORRECTABLE_INTERNAL) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_UNSUPPORTED_VC_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_UNSUPPORTED_NVLINK_CREDIT_RELEASE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_UNSUPPORTED_NCISOC_CREDIT_RELEASE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_HDR_CREDIT_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DATA_CREDIT_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DL_REPLAY_CREDIT_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_UNSUPPORTED_VC_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_STOMPED_PKT_SENT) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DATA_POISONED_PKT_SENT) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_RESP_STATUS_TARGET) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_RESP_STATUS_UNSUPPORTED_REQUEST) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_MIF_RX_RAM_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_MIF_RX_RAM_HDR_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_MIF_TX_RAM_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_MIF_TX_RAM_HDR_PARITY) + +#define INFOROM_NVLINK_V3_FATAL_ERR_TYPES \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_DATA_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_CTRL_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_DL_FLOW_CONTROL_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_RAM) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_INTERFACE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_FAULT_SUBLINK_CHANGE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_FAULT_SUBLINK_CHANGE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_FAULT_DL_PROTOCOL) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DL_HDR_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_AE_FLIT_RCVD) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_BE_FLIT_RCVD) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_ADDR_ALIGN) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_PKT_LEN) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CMD_ENC) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_DAT_LEN_ENC) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_PKT_STATUS) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_REQ) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_RESP) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DAT_LEN_GT_RMW_REQ_MAX_SIZE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DAT_LEN_LT_ATR_RESP_MIN_SIZE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_PO_FOR_CACHE_ATTR) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_COMPRESSED_RESP) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_HDR_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_DATA_OVERFLOW) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_STOMPED_PKT_RCVD) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_RX_INVALID_COLLAPSED_RESPONSE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_NCISOC_HDR_ECC_DBE) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_TLC_TX_NCISOC_PARITY) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_LTSSM_FAULT_UP) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_LTSSM_FAULT_DOWN) + +#define INFOROM_NVLINK_V4_FATAL_ERR_TYPES \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_PHY_A) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_CRC_COUNTER) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_TX_PL_ERROR) | \ + NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_DL_RX_PL_ERROR) + +#define INFOROM_ADD_NVLINK_FATAL_ERR_TYPE(mask, type) \ + mask |= NVBIT64(INFOROM_NVLINK_FATAL_ERR_TYPE_##type) + +typedef struct +{ + NvU32 errorsPerMinute; + NvU32 timestamp; +} INFOROM_NVLINK_ERROR_RATE; + +#define INFOROM_NVLINK_NUM_LINKS 6 + +typedef union INFOROM_NVL_OBJECT INFOROM_NVL_OBJECT, *PINFOROM_NVL_OBJECT; + +// Version 1.0 ---------------------------------------------------------------- + +// Link Remote Endpoint Type +typedef enum +{ + INFOROM_NVL_OBJECT_V1_REMOTE_TYPE_UNSUPPORTED = -1, + INFOROM_NVL_OBJECT_V1_REMOTE_TYPE_NONE = 0, + INFOROM_NVL_OBJECT_V1_REMOTE_TYPE_PEER = 1, + INFOROM_NVL_OBJECT_V1_REMOTE_TYPE_SYSTEM = 2, + INFOROM_NVL_OBJECT_V1_REMOTE_TYPE_SWITCH = 3, + INFOROM_NVL_OBJECT_V1_REMOTE_TYPE_MAX = 3, + INFOROM_NVL_OBJECT_V1_NUM_REMOTE_TYPES = 4, +} INFOROM_NVL_OBJECT_V1_REMOTE_TYPE; + +// Fatal Errors +typedef enum +{ + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_UNSUPPORTED = -1, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_RX_DL_DATA_PARITY = 0, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_RX_DL_CTRL_PARITY = 1, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_RX_PROTOCOL = 2, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_RX_OVERFLOW = 3, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_RX_RAM_DATA_PARITY = 4, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_RX_RAM_HDR_PARITY = 5, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_RX_RESP = 6, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_RX_POISON = 7, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_TX_RAM_DATA_PARITY = 8, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_TX_RAM_HDR_PARITY = 9, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_TX_CREDIT = 10, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_DL_FLOW_CTRL_PARITY = 11, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_TL_DL_HDR_PARITY = 12, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_DL_TX_RECOVERY_LONG = 13, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_DL_TX_FAULT_RAM = 14, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_DL_TX_FAULT_INTERFACE = 15, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_DL_TX_FAULT_SUBLINK_CHANGE = 16, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_DL_RX_FAULT_SUBLINK_CHANGE = 17, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_DL_RX_FAULT_DL_PROTOCOL = 18, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_DL_LTSSM_FAULT = 19, + INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX_MAX = 19, + INFOROM_NVL_OBJECT_V1_NUM_FATAL_ERR_INDICES = 20, +} INFOROM_NVL_OBJECT_V1_FATAL_ERR_INDEX; + +typedef struct INFOROM_NVL_OBJECT_V1_FATAL_ERR_DATA +{ + inforom_U008 counts[INFOROM_NVL_OBJECT_V1_NUM_FATAL_ERR_INDICES]; + inforom_U008 reserved[25]; +} INFOROM_NVL_OBJECT_V1_FATAL_ERR_DATA; + +#define INFOROM_NVL_OBJECT_V1_FATAL_ERR_DATA_FMT "45b" + +// Non-Fatal Errors +typedef struct INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_ENTRY +{ + inforom_U024 errRate; + inforom_U032 timestamp; +} INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_ENTRY; + +#define INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_FMT "td" + +#define INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_DAY_ENTRIES 5 +#define INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_DAY_INTERVAL (60*60*24) +#define INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_MONTH_ENTRIES 3 +#define INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_MONTH_INTERVAL (60*60*24*30) + +typedef struct INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_DATA +{ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_ENTRY maxNDaysAgo[ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_DAY_ENTRIES]; + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_ENTRY maxNMonthsAgo[ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_MONTH_ENTRIES]; +} INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_DATA; + +#define INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_DATA_FMT \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_FMT) + +typedef struct INFOROM_NVL_OBJECT_V1_ERR_DATA +{ + INFOROM_NVL_OBJECT_V1_FATAL_ERR_DATA fatal; + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_DATA nonfatal; +} INFOROM_NVL_OBJECT_V1_ERR_DATA; + +#define INFOROM_NVL_OBJECT_V1_ERR_DATA_FMT \ + INFOROM_NVL_OBJECT_V1_FATAL_ERR_DATA_FMT \ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_DATA_FMT + +#define INFOROM_NVL_OBJECT_V1_NUM_LINKS 6 + +typedef struct INFOROM_NVL_OBJECT_V1 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U004 lastErrorRemoteType[ + INFOROM_NVL_OBJECT_V1_NUM_LINKS]; + inforom_U008 reserved1[1]; + INFOROM_NVL_OBJECT_V1_ERR_DATA errData[INFOROM_NVL_OBJECT_V1_NUM_LINKS]; + inforom_U016 padding[1]; + inforom_U032 reserved2[13]; +} INFOROM_NVL_OBJECT_V1; + +#define INFOROM_NVL_OBJECT_V1_FMT \ + INFOROM_OBJECT_HEADER_V1_00_FMT \ + "6n" /* lastRemoteType */ \ + "b" /* reserved1 */ \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V1_ERR_DATA_FMT) \ + INFOROM_FMT_REP02(INFOROM_NVL_OBJECT_V1_ERR_DATA_FMT) \ + "w" /* padding */ \ + "13d" /* reserved2 */ + +#define INFOROM_NVL_OBJECT_V1_PACKED_SIZE 672 + +// Version 2.0 ---------------------------------------------------------------- + +// Link Remote Endpoint Type +typedef enum +{ + INFOROM_NVL_OBJECT_V2_REMOTE_TYPE_UNSUPPORTED = -1, + INFOROM_NVL_OBJECT_V2_REMOTE_TYPE_NONE = 0, + INFOROM_NVL_OBJECT_V2_REMOTE_TYPE_PEER = 1, + INFOROM_NVL_OBJECT_V2_REMOTE_TYPE_SYSTEM = 2, + INFOROM_NVL_OBJECT_V2_REMOTE_TYPE_SWITCH = 3, + INFOROM_NVL_OBJECT_V2_REMOTE_TYPE_MAX = 3, + INFOROM_NVL_OBJECT_V2_NUM_REMOTE_TYPES = 4, +} INFOROM_NVL_OBJECT_V2_REMOTE_TYPE; + +// Fatal Errors +typedef enum +{ + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_UNSUPPORTED = -1, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_DL_HDR_PARITY = 0, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_DL_DATA_PARITY = 1, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_DL_CTRL_PARITY = 2, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RAM_DATA_PARITY = 3, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RAM_HDR_PARITY = 4, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_INVALID_AE_FLIT_RCVD = 5, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_INVALID_BE_FLIT_RCVD = 6, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_INVALID_ADDR_ALIGN = 7, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_PKT_LEN = 8, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RSVD_CMD_ENC = 9, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RSVD_DAT_LEN_ENC = 10, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RSVD_ADDR_TYPE = 11, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RSVD_RSP_STATUS = 12, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RSVD_PKT_STATUS = 13, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_REQ = 14, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_RESP = 15, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_DAT_LEN_GT_ATOMIC_REQ_MAX_SIZE = 16, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_DAT_LEN_GT_RMW_REQ_MAX_SIZE = 17, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_DAT_LEN_LT_ATR_RESP_MIN_SIZE = 18, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_INVALID_PO_FOR_CACHE_ATTR = 19, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_INVALID_COMPRESSED_RESP = 20, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RESP_STATUS_TARGET = 21, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_RESP_STATUS_UNSUPPORTED_REQUEST = 22, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_HDR_OVERFLOW = 23, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_DATA_OVERFLOW = 24, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_STOMPED_PKT_RCVD = 25, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_DATA_POISONED_PKT_RCVD = 26, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_CORRECTABLE_INTERNAL = 27, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_UNSUPPORTED_VC_OVERFLOW = 28, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_UNSUPPORTED_NVLINK_CREDIT_RELEASE = 29, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_RX_UNSUPPORTED_NCISOC_CREDIT_RELEASE = 30, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_HDR_CREDIT_OVERFLOW = 31, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_DATA_CREDIT_OVERFLOW = 32, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_DL_REPLAY_CREDIT_OVERFLOW = 33, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_DL_FLOW_CONTROL_PARITY = 34, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_RAM_HDR_PARITY = 35, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_RAM_DATA_PARITY = 36, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_UNSUPPORTED_VC_OVERFLOW = 37, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_STOMPED_PKT_SENT = 38, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_DATA_POISONED_PKT_SENT = 39, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_RESP_STATUS_TARGET = 40, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_TLC_TX_RESP_STATUS_UNSUPPORTED_REQUEST = 41, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_MIF_RX_RAM_DATA_PARITY = 42, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_MIF_RX_RAM_HDR_PARITY = 43, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_MIF_TX_RAM_DATA_PARITY = 44, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_MIF_TX_RAM_HDR_PARITY = 45, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_DL_TX_RECOVERY_LONG = 46, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_DL_TX_FAULT_RAM = 47, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_DL_TX_FAULT_INTERFACE = 48, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_DL_TX_FAULT_SUBLINK_CHANGE = 49, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_DL_RX_FAULT_SUBLINK_CHANGE = 50, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_DL_RX_FAULT_DL_PROTOCOL = 51, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_DL_LTSSM_FAULT = 52, + INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX_MAX = 52, + INFOROM_NVL_OBJECT_V2_NUM_FATAL_ERR_INDICES = 53, +} INFOROM_NVL_OBJECT_V2_FATAL_ERR_INDEX; + +typedef struct INFOROM_NVL_OBJECT_V2_FATAL_ERR_DATA +{ + inforom_U008 counts[INFOROM_NVL_OBJECT_V2_NUM_FATAL_ERR_INDICES]; +} INFOROM_NVL_OBJECT_V2_FATAL_ERR_DATA; + +#define INFOROM_NVL_OBJECT_V2_FATAL_ERR_DATA_FMT "53b" + +// Non-Fatal Errors +typedef INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_ENTRY + INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_RATE_ENTRY; + +#define INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_RATE_FMT \ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_FMT + +#define INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_RATE_DAY_ENTRIES \ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_DAY_ENTRIES +#define INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_RATE_DAY_INTERVAL \ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_DAY_INTERVAL +#define INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_RATE_MONTH_ENTRIES \ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_MONTH_ENTRIES +#define INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_RATE_MONTH_INTERVAL \ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_RATE_MONTH_INTERVAL + +typedef INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_DATA + INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_DATA; + +#define INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_DATA_FMT \ + INFOROM_NVL_OBJECT_V1_NONFATAL_ERR_DATA_FMT + +typedef struct INFOROM_NVL_OBJECT_V2_ERR_DATA +{ + INFOROM_NVL_OBJECT_V2_FATAL_ERR_DATA fatal; + INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_DATA nonfatal; +} INFOROM_NVL_OBJECT_V2_ERR_DATA; + +#define INFOROM_NVL_OBJECT_V2_ERR_DATA_FMT \ + INFOROM_NVL_OBJECT_V2_FATAL_ERR_DATA_FMT \ + INFOROM_NVL_OBJECT_V2_NONFATAL_ERR_DATA_FMT + +#define INFOROM_NVL_OBJECT_V2_NUM_LINKS INFOROM_NVL_OBJECT_V1_NUM_LINKS + +typedef struct INFOROM_NVL_OBJECT_V2 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U004 lastErrorRemoteType[ + INFOROM_NVL_OBJECT_V2_NUM_LINKS]; + inforom_U008 reserved1[1]; + INFOROM_NVL_OBJECT_V2_ERR_DATA errData[INFOROM_NVL_OBJECT_V2_NUM_LINKS]; + inforom_U016 padding[1]; + inforom_U032 reserved2[1]; +} INFOROM_NVL_OBJECT_V2; + +#define INFOROM_NVL_OBJECT_V2_FMT \ + INFOROM_OBJECT_HEADER_V1_00_FMT \ + "6n" /* lastRemoteType */ \ + "b" /* reserved1 */ \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V2_ERR_DATA_FMT) \ + INFOROM_FMT_REP02(INFOROM_NVL_OBJECT_V2_ERR_DATA_FMT) \ + "w" /* padding */ \ + "d" /* reserved2 */ + +#define INFOROM_NVL_OBJECT_V2_PACKED_SIZE 672 + +// Version 3.G0 --------------------------------------------------------------- + +#define INFOROM_NVL_OBJECT_V3G_NUM_LINKS 12 +#define INFOROM_NVL_OBJECT_V3G_MAX_SEED_BUFFER_SIZE 7 + +typedef INFOROM_NVL_OBJECT_V2_REMOTE_TYPE INFOROM_NVL_OBJECT_V3_REMOTE_TYPE; + +typedef struct INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO +{ + inforom_U032 lastUpdated; + NvU64_ALIGN32 remoteDeviceSID; + inforom_U008 remoteDeviceType; + inforom_U008 remoteDeviceLink; +} INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO; + +#define INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO_FMT "dq2b" + +typedef struct INFOROM_NVL_OBJECT_V3_MINION_DATA +{ + inforom_U032 lastUpdated; + inforom_U032 data[INFOROM_NVL_OBJECT_V3G_MAX_SEED_BUFFER_SIZE]; +} INFOROM_NVL_OBJECT_V3_MINION_DATA; + +#define INFOROM_NVL_OBJECT_V3_MINION_DATA_FMT "8d" + +typedef struct INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE +{ + inforom_U032 lastUpdated; + inforom_U032 flitCrcErrorsPerMinute; + inforom_U016 laneCrcErrorsPerMinute[4]; +} INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE; + +#define INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE_FMT "2d4w" + +typedef INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE + INFOROM_NVL_OBJECT_V3G_CORRECTABLE_ERROR_RATE_ARRAY[INFOROM_NVL_OBJECT_V3G_NUM_LINKS]; + +#define INFOROM_NVL_OBJECT_V3G_CORRECTABLE_ERROR_RATE_ARRAY_FMT \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE_FMT) + +typedef struct INFOROM_NVL_OBJECT_V3G_MAX_CORRECTABLE_ERROR_RATES +{ + INFOROM_NVL_OBJECT_V3G_CORRECTABLE_ERROR_RATE_ARRAY dailyMaxCorrectableErrorRates[5]; + INFOROM_NVL_OBJECT_V3G_CORRECTABLE_ERROR_RATE_ARRAY monthlyMaxCorrectableErrorRates[3]; +} INFOROM_NVL_OBJECT_V3G_MAX_CORRECTABLE_ERROR_RATES; + +#define INFOROM_NVL_OBJECT_V3G_MAX_CORRECTABLE_ERROR_RATES_FMT \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V3G_CORRECTABLE_ERROR_RATE_ARRAY_FMT) + +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_NVLIPT_INSTANCE_ID 3:0 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID 7:4 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_DL0 0 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_DL1 1 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_DL2 2 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_DL3 3 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_DL4 4 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_DL5 5 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_DL(n) \ + (NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_DL0+(n)) +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_TLC0 6 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_TLC1 7 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_TLC2 8 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_TLC3 9 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_TLC4 10 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_TLC5 11 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_TLC(n) \ + (NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_TLC0+(n)) +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_MINION 12 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_BLOCK_ID_NVLIPT 13 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_DIRECTION 9:8 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_DIRECTION_NA 1 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_DIRECTION_RX 2 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_DIRECTION_TX 3 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_SEVERITY 11:10 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_SEVERITY_CORRECTABLE 1 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_SEVERITY_UNCORRECTABLE_NONFATAL 2 +#define NV_INFOROM_NVL_OBJECT_V3_ERROR_METADATA_SEVERITY_UNCORRECTABLE_FATAL 3 + +#define INFOROM_NVL_OBJECT_V3_NVLIPT_INSTANCE_MAX 15 +#define INFOROM_NVL_OBJECT_V3_BLOCK_ID_MAX 15 + +#define NV_INFOROM_NVL_OBJECT_V3_NVLIPT_ERROR_SUBTYPE 4:0 +#define NV_INFOROM_NVL_OBJECT_V3_NVLIPT_ERROR_LINK_ID 7:5 +#define NV_INFOROM_NVL_OBJECT_V3_NVLIPT_ERROR_LINK_ID_COMMON 7 + +// Format is <BLOCK>_<DIRECTION>_<SUBTYPE>_<SEVERITY>_<TYPE> +// Each <BLOCK>_<DIRECTION> has its own subtype space. +typedef enum +{ + // DL RX ----------------------------------------- + // Fatal Counts + DL_RX_FAULT_DL_PROTOCOL_FATAL_COUNT = 0, + DL_RX_FAULT_SUBLINK_CHANGE_FATAL_COUNT, + + // Correctable Accumulated Counts + DL_RX_FLIT_CRC_CORR_ACCUM, + DL_RX_LANE0_CRC_CORR_ACCUM, + DL_RX_LANE1_CRC_CORR_ACCUM, + DL_RX_LANE2_CRC_CORR_ACCUM, + DL_RX_LANE3_CRC_CORR_ACCUM, + DL_RX_LINK_REPLAY_EVENTS_CORR_ACCUM, + + // DL TX ----------------------------------------- + // Fatal Counts + DL_TX_FAULT_RAM_FATAL_COUNT = 0, + DL_TX_FAULT_INTERFACE_FATAL_COUNT, + DL_TX_FAULT_SUBLINK_CHANGE_FATAL_COUNT, + + // Correctable Accumulated Counts + DL_TX_LINK_REPLAY_EVENTS_CORR_ACCUM, + + // DL Disassociated ------------------------------ + // Fatal Counts + DL_NA_LTSSM_FAULT_UP_FATAL_COUNT = 0, + DL_NA_LTSSM_FAULT_DOWN_FATAL_COUNT, + + // Correctable Accumulated Counts + DL_NA_LINK_RECOVERY_EVENTS_CORR_ACCUM, + + DL_NA_PHY_A_FATAL_COUNT, + DL_RX_CRC_COUNTER_FATAL_COUNT, + DL_TX_PL_ERROR_FATAL_COUNT, + DL_RX_PL_ERROR_FATAL_COUNT, + + // TLC RX ---------------------------------------- + // Fatal Counts + TLC_RX_DL_HDR_PARITY_ERR_FATAL_COUNT = 0, + TLC_RX_DL_DATA_PARITY_ERR_FATAL_COUNT, + TLC_RX_DL_CTRL_PARITY_ERR_FATAL_COUNT, + TLC_RX_INVALID_AE_FATAL_COUNT, + TLC_RX_INVALID_BE_FATAL_COUNT, + TLC_RX_INVALID_ADDR_ALIGN_FATAL_COUNT, + TLC_RX_PKTLEN_ERR_FATAL_COUNT, + TLC_RX_RSVD_PACKET_STATUS_ERR_FATAL_COUNT, + TLC_RX_RSVD_CACHE_ATTR_PROBE_REQ_ERR_FATAL_COUNT, + TLC_RX_RSVD_CACHE_ATTR_PROBE_RSP_ERR_FATAL_COUNT, + TLC_RX_DATLEN_GT_RMW_REQ_MAX_ERR_FATAL_COUNT, + TLC_RX_DATLEN_LT_ATR_RSP_MIN_ERR_FATAL_COUNT, + TLC_RX_INVALID_CR_FATAL_COUNT, + TLC_RX_INVALID_COLLAPSED_RESPONSE_FATAL_COUNT, + TLC_RX_HDR_OVERFLOW_FATAL_COUNT, + TLC_RX_DATA_OVERFLOW_FATAL_COUNT, + TLC_RX_STOMP_DETECTED_FATAL_COUNT, + TLC_RX_RSVD_CMD_ENC_FATAL_COUNT, + TLC_RX_RSVD_DAT_LEN_ENC_FATAL_COUNT, + TLC_RX_INVALID_PO_FOR_CACHE_ATTR_FATAL_COUNT, + // This is only needed for NCISOC support (not used outside SoC?) + // TLC_RX_NCISOC_CREDIT_PARITY_ERR_FATAL_COUNT, + + // Non-Fatal Counts + TLC_RX_RSP_STATUS_HW_ERR_NONFATAL_COUNT, + TLC_RX_RSP_STATUS_UR_ERR_NONFATAL_COUNT, + TLC_RX_RSP_STATUS_PRIV_ERR_NONFATAL_COUNT, + TLC_RX_POISON_NONFATAL_COUNT, + TLC_RX_AN1_HEARTBEAT_TIMEOUT_NONFATAL_COUNT, + TLC_RX_ILLEGAL_PRI_WRITE_NONFATAL_COUNT, + + // Fatal Counts addendum + TLC_RX_HDR_RAM_ECC_DBE_FATAL_COUNT, + TLC_RX_DAT0_RAM_ECC_DBE_FATAL_COUNT, + TLC_RX_DAT1_RAM_ECC_DBE_FATAL_COUNT, + + // Correctable Accumulated Counts + // These should be logged in the ECC object instead + // TLC_RX_HDR_RAM_ECC_SBE_ACCUM, + // TLC_RX_DAT0_RAM_ECC_SBE_ACCUM, + // TLC_RX_DAT1_RAM_ECC_SBE_ACCUM, + + // TLC TX ---------------------------------------- + // Fatal Counts + TLC_TX_DL_CREDIT_PARITY_ERR_FATAL_COUNT = 0, + // These should be logged in the ECC object instead + // TLC_TX_CREQ_HDR_RAM_ECC_DBE_FATAL_COUNT, + // TLC_TX_RSP_HDR_RAM_ECC_DBE_FATAL_COUNT, + // TLC_TX_RSP1_HDR_RAM_ECC_DBE_FATAL_COUNT, + // TLC_TX_COM_HDR_RAM_ECC_DBE_FATAL_COUNT, + TLC_TX_NCISOC_HDR_ECC_DBE_FATAL_COUNT, + TLC_TX_NCISOC_PARITY_ERR_FATAL_COUNT, + + // Non-Fatal Counts + // This is only needed for NCISOC support (not used outside SoC?) + // TLC_TX_NCISOC_DAT_ECC_DBE_NONFATAL_COUNT, + TLC_TX_ILLEGAL_PRI_WRITE_NONFATAL_COUNT, + TLC_TX_AN1_TIMEOUT_VC0_NONFATAL_COUNT, + TLC_TX_AN1_TIMEOUT_VC1_NONFATAL_COUNT, + TLC_TX_AN1_TIMEOUT_VC2_NONFATAL_COUNT, + TLC_TX_AN1_TIMEOUT_VC3_NONFATAL_COUNT, + TLC_TX_AN1_TIMEOUT_VC4_NONFATAL_COUNT, + TLC_TX_AN1_TIMEOUT_VC5_NONFATAL_COUNT, + TLC_TX_AN1_TIMEOUT_VC6_NONFATAL_COUNT, + TLC_TX_AN1_TIMEOUT_VC7_NONFATAL_COUNT, + TLC_TX_POISON_NONFATAL_COUNT, + TLC_TX_RSP_STATUS_HW_ERR_NONFATAL_COUNT, + TLC_TX_RSP_STATUS_UR_ERR_NONFATAL_COUNT, + TLC_TX_RSP_STATUS_PRIV_ERR_NONFATAL_COUNT, + TLC_TX_CREQ_DAT_RAM_ECC_DBE_NONFATAL_COUNT, + TLC_TX_RSP_DAT_RAM_ECC_DBE_NONFATAL_COUNT, + TLC_TX_COM_DAT_RAM_ECC_DBE_NONFATAL_COUNT, + + // Correctable Counts + // These should be logged in the ECC object instead + // TLC_TX_CREQ_RAMS_ECC_SBE_CORR_ACCUM, + // TLC_TX_RSP_RAMS_ECC_SBE_CORR_ACCUM, + // TLC_TX_RSP1_RAMS_ECC_SBE_CORR_ACCUM, + // TLC_TX_COM_RAMS_ECC_SBE_CORR_ACCUM, + // This is only needed for NCISOC support (not used outside SoC?) + // TLC_TX_NCISOC_ECC_LIMIT_ERR_CORR_ACCUM, + + // TLC TX Fatal addendum + TLC_TX_RSP1_DAT_RAM_ECC_DBE_FATAL_COUNT, + + // NVLIPT Disassociated -------------------------- + // Fatal Counts + NVLIPT_NA_SLEEP_WHILE_ACTIVE_LINK_FATAL_COUNT = 0, + NVLIPT_NA_RSTSEQ_PHYCTL_TIMEOUT_FATAL_COUNT, + NVLIPT_NA_RSTSEQ_CLKCTL_TIMEOUT_FATAL_COUNT, + NVLIPT_NA_CLKCTL_ILLEGAL_REQUEST_FATAL_COUNT, + NVLIPT_NA_RSTSEQ_PLL_TIMEOUT_FATAL_COUNT, + NVLIPT_NA_RSTSEQ_PHYARB_TIMEOUT_FATAL_COUNT, + + // Non-Fatal Counts + NVLIPT_NA_ILLEGAL_LINK_STATE_REQUEST_NONFATAL_COUNT, + NVLIPT_NA_FAILED_MINION_REQUEST_NONFATAL_COUNT, + NVLIPT_NA_RESERVED_REQUEST_VALUE_NONFATAL_COUNT, + NVLIPT_NA_LINK_STATE_WRITE_WHILE_BUSY_NONFATAL_COUNT, + NVLIPT_NA_WRITE_TO_LOCKED_SYSTEM_REG_NONFATAL_COUNT, + NVLIPT_NA_LINK_STATE_REQUEST_TIMEOUT_NONFATAL_COUNT +} INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_SUBTYPE; + +#define INFOROM_NVL_OBJECT_V3_FATAL_ERR_INDEX_UNSUPPORTED ((NvU8) NV_U8_MAX) + +typedef enum +{ + INFOROM_NVL_ERROR_TYPE_INVALID = 0, + INFOROM_NVL_ERROR_TYPE_COUNT, + INFOROM_NVL_ERROR_TYPE_ACCUM +} INFOROM_NVL_OBJECT_V3_ERROR_TYPE; + +typedef enum +{ + INFOROM_NVL_ERROR_BLOCK_TYPE_DL = 0, + INFOROM_NVL_ERROR_BLOCK_TYPE_TLC, + INFOROM_NVL_ERROR_BLOCK_TYPE_NVLIPT +} INFOROM_NVL_ERROR_BLOCK_TYPE; + +typedef struct +{ + NvU8 nvliptInstance; + NvU8 localLinkIdx; + INFOROM_NVLINK_ERROR_TYPES error; + NvU32 count; // count is used only for correctable errors +} INFOROM_NVLINK_ERROR_EVENT; + +#define INFOROM_NVL_OBJECT_MAX_SUBLINK_WIDTH 8 +typedef struct +{ + NvU16 flitCrc; + NvU8 laneCrc[INFOROM_NVL_OBJECT_MAX_SUBLINK_WIDTH]; + NvU16 rxLinkReplay; + NvU16 txLinkReplay; + NvU16 linkRecovery; +} INFOROM_NVLINK_CORRECTABLE_ERROR_COUNTS; + +typedef struct INFOROM_NVL_OBJECT_V3_ERROR_ENTRY +{ + inforom_U008 header; + inforom_U016 metadata; + inforom_U008 errorSubtype; + union + { + struct + { + inforom_U032 lastError; + inforom_U032 avgEventDelta; + inforom_U032 totalCount; + } event; + struct + { + inforom_U032 lastUpdated; + NvU64_ALIGN32 totalCount; + } accum; + } data; +} INFOROM_NVL_OBJECT_V3_ERROR_ENTRY; + +#define INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT "bwb3d" + +#define INFOROM_NVL_OBJECT_V3G_NUM_ERROR_ENTRIES 240 + +typedef struct INFOROM_NVL_OBJECT_V3G +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO remoteDeviceInfo[INFOROM_NVL_OBJECT_V3G_NUM_LINKS]; + INFOROM_NVL_OBJECT_V3_MINION_DATA minionData[INFOROM_NVL_OBJECT_V3G_NUM_LINKS]; + INFOROM_NVL_OBJECT_V3G_MAX_CORRECTABLE_ERROR_RATES maxCorrectableErrorRates; + INFOROM_NVL_OBJECT_V3_ERROR_ENTRY errorLog[INFOROM_NVL_OBJECT_V3G_NUM_ERROR_ENTRIES]; +} INFOROM_NVL_OBJECT_V3G; + +#define INFOROM_NVL_OBJECT_V3G_REMOTE_DEVICE_INFO_ARRAY_FMT \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO_FMT) + +#define INFOROM_NVL_OBJECT_V3G_MINION_DATA_ARRAY_FMT \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V3_MINION_DATA_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V3_MINION_DATA_FMT) + +#define INFOROM_NVL_OBJECT_V3G_ERROR_LOG_FMT \ + INFOROM_FMT_REP128(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP64(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP32(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP16(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) + +#define INFOROM_NVL_OBJECT_V3G_FMT \ + INFOROM_OBJECT_HEADER_V1_00_FMT \ + INFOROM_NVL_OBJECT_V3G_REMOTE_DEVICE_INFO_ARRAY_FMT \ + INFOROM_NVL_OBJECT_V3G_MINION_DATA_ARRAY_FMT \ + INFOROM_NVL_OBJECT_V3G_MAX_CORRECTABLE_ERROR_RATES_FMT \ + INFOROM_NVL_OBJECT_V3G_ERROR_LOG_FMT + +#define INFOROM_NVL_OBJECT_V3G_PACKED_SIZE 5936 + +// Version 3.S0 --------------------------------------------------------------- +#define INFOROM_NVL_OBJECT_V3S_NUM_LINKS 36 +#define INFOROM_NVL_OBJECT_V3S_NUM_ERROR_ENTRIES 721 + +typedef INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE + INFOROM_NVL_OBJECT_V3S_CORRECTABLE_ERROR_RATE_ARRAY[INFOROM_NVL_OBJECT_V3S_NUM_LINKS]; + +#define INFOROM_NVL_OBJECT_V3S_CORRECTABLE_ERROR_RATE_ARRAY_FMT \ + INFOROM_FMT_REP32(INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE_FMT) + +typedef struct INFOROM_NVL_OBJECT_V3S_MAX_CORRECTABLE_ERROR_RATES +{ + INFOROM_NVL_OBJECT_V3S_CORRECTABLE_ERROR_RATE_ARRAY dailyMaxCorrectableErrorRates[5]; + INFOROM_NVL_OBJECT_V3S_CORRECTABLE_ERROR_RATE_ARRAY monthlyMaxCorrectableErrorRates[3]; +} INFOROM_NVL_OBJECT_V3S_MAX_CORRECTABLE_ERROR_RATES; + +#define INFOROM_NVL_OBJECT_V3S_MAX_CORRECTABLE_ERROR_RATES_FMT \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V3S_CORRECTABLE_ERROR_RATE_ARRAY_FMT) + +typedef struct INFOROM_NVL_OBJECT_V3S +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO remoteDeviceInfo[INFOROM_NVL_OBJECT_V3S_NUM_LINKS]; + INFOROM_NVL_OBJECT_V3_MINION_DATA minionData[INFOROM_NVL_OBJECT_V3S_NUM_LINKS]; + INFOROM_NVL_OBJECT_V3S_MAX_CORRECTABLE_ERROR_RATES maxCorrectableErrorRates; + INFOROM_NVL_OBJECT_V3_ERROR_ENTRY errorLog[INFOROM_NVL_OBJECT_V3S_NUM_ERROR_ENTRIES]; +} INFOROM_NVL_OBJECT_V3S; + +#define INFOROM_NVL_OBJECT_V3S_REMOTE_DEVICE_INFO_ARRAY_FMT \ + INFOROM_FMT_REP32(INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO_FMT) + +#define INFOROM_NVL_OBJECT_V3S_MINION_DATA_ARRAY_FMT \ + INFOROM_FMT_REP32(INFOROM_NVL_OBJECT_V3_MINION_DATA_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V3_MINION_DATA_FMT) + +#define INFOROM_NVL_OBJECT_V3S_ERROR_LOG_FMT \ + INFOROM_FMT_REP512(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP128(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP64(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP16(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT + +#define INFOROM_NVL_OBJECT_V3S_FMT \ + INFOROM_OBJECT_HEADER_V1_00_FMT \ + INFOROM_NVL_OBJECT_V3S_REMOTE_DEVICE_INFO_ARRAY_FMT \ + INFOROM_NVL_OBJECT_V3S_MINION_DATA_ARRAY_FMT \ + INFOROM_NVL_OBJECT_V3S_MAX_CORRECTABLE_ERROR_RATES_FMT \ + INFOROM_NVL_OBJECT_V3S_ERROR_LOG_FMT + +#define INFOROM_NVL_OBJECT_V3S_PACKED_SIZE 17808 + +// Version 4.G0 --------------------------------------------------------------- + +#define INFOROM_NVL_OBJECT_V4G_NUM_LINKS 18 + +typedef struct INFOROM_NVL_OBJECT_V4_L1_THRESHOLD_DATA +{ + inforom_U032 word1; + inforom_U032 word2; +} INFOROM_NVL_OBJECT_V4_L1_THRESHOLD_DATA; + +#define INFOROM_NVL_OBJECT_V4_L1_THRESHOLD_DATA_FMT "2d" + +typedef INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE + INFOROM_NVL_OBJECT_V4G_CORRECTABLE_ERROR_RATE_ARRAY[INFOROM_NVL_OBJECT_V4G_NUM_LINKS]; + +#define INFOROM_NVL_OBJECT_V4G_CORRECTABLE_ERROR_RATE_ARRAY_FMT \ + INFOROM_FMT_REP16(INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE_FMT) \ + INFOROM_FMT_REP02(INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE_FMT) + +typedef struct INFOROM_NVL_OBJECT_V4G_MAX_CORRECTABLE_ERROR_RATES +{ + INFOROM_NVL_OBJECT_V4G_CORRECTABLE_ERROR_RATE_ARRAY dailyMaxCorrectableErrorRates[5]; + INFOROM_NVL_OBJECT_V4G_CORRECTABLE_ERROR_RATE_ARRAY monthlyMaxCorrectableErrorRates[3]; +} INFOROM_NVL_OBJECT_V4G_MAX_CORRECTABLE_ERROR_RATES; + +#define INFOROM_NVL_OBJECT_V4G_MAX_CORRECTABLE_ERROR_RATES_FMT \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V4G_CORRECTABLE_ERROR_RATE_ARRAY_FMT) + +typedef struct INFOROM_NVL_OBJECT_V4G_RESERVED +{ + inforom_U032 reserved; +} INFOROM_NVL_OBJECT_V4G_RESERVED; + +#define INFOROM_NVL_OBJECT_V4G_RESERVED_FMT "d" + +#define INFOROM_NVL_OBJECT_V4G_NUM_ERROR_ENTRIES 364 + +typedef struct INFOROM_NVL_OBJECT_V4G +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_NVL_OBJECT_V4_L1_THRESHOLD_DATA l1ThresholdData; + INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO remoteDeviceInfo[INFOROM_NVL_OBJECT_V4G_NUM_LINKS]; + INFOROM_NVL_OBJECT_V4G_MAX_CORRECTABLE_ERROR_RATES maxCorrectableErrorRates; + INFOROM_NVL_OBJECT_V3_ERROR_ENTRY errorLog[INFOROM_NVL_OBJECT_V4G_NUM_ERROR_ENTRIES]; + INFOROM_NVL_OBJECT_V4G_RESERVED reserved; +} INFOROM_NVL_OBJECT_V4G; + +#define INFOROM_NVL_OBJECT_V4G_REMOTE_DEVICE_INFO_ARRAY_FMT \ + INFOROM_FMT_REP16(INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO_FMT) \ + INFOROM_FMT_REP02(INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO_FMT) + +#define INFOROM_NVL_OBJECT_V4G_ERROR_LOG_FMT \ + INFOROM_FMT_REP128(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP128(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP64(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP32(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) + +#define INFOROM_NVL_OBJECT_V4G_FMT \ + INFOROM_OBJECT_HEADER_V1_00_FMT \ + INFOROM_NVL_OBJECT_V4G_REMOTE_DEVICE_INFO_ARRAY_FMT \ + INFOROM_NVL_OBJECT_V4_L1_THRESHOLD_DATA_FMT \ + INFOROM_NVL_OBJECT_V4G_MAX_CORRECTABLE_ERROR_RATES_FMT \ + INFOROM_NVL_OBJECT_V4G_ERROR_LOG_FMT \ + INFOROM_NVL_OBJECT_V4G_RESERVED_FMT + +#define INFOROM_NVL_OBJECT_V4G_PACKED_SIZE 8400 + +// Version 4.S0 --------------------------------------------------------------- +#define INFOROM_NVL_OBJECT_V4S_NUM_LINKS 64 +#define INFOROM_NVL_OBJECT_V4S_NUM_ERROR_ENTRIES 1286 + +typedef INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE + INFOROM_NVL_OBJECT_V4S_CORRECTABLE_ERROR_RATE_ARRAY[INFOROM_NVL_OBJECT_V4S_NUM_LINKS]; + +#define INFOROM_NVL_OBJECT_V4S_CORRECTABLE_ERROR_RATE_ARRAY_FMT \ + INFOROM_FMT_REP64(INFOROM_NVL_OBJECT_V3_CORRECTABLE_ERROR_RATE_FMT) + +typedef struct INFOROM_NVL_OBJECT_V4S_MAX_CORRECTABLE_ERROR_RATES +{ + INFOROM_NVL_OBJECT_V4S_CORRECTABLE_ERROR_RATE_ARRAY dailyMaxCorrectableErrorRates[5]; + INFOROM_NVL_OBJECT_V4S_CORRECTABLE_ERROR_RATE_ARRAY monthlyMaxCorrectableErrorRates[3]; +} INFOROM_NVL_OBJECT_V4S_MAX_CORRECTABLE_ERROR_RATES; + +#define INFOROM_NVL_OBJECT_V4S_MAX_CORRECTABLE_ERROR_RATES_FMT \ + INFOROM_FMT_REP08(INFOROM_NVL_OBJECT_V4S_CORRECTABLE_ERROR_RATE_ARRAY_FMT) + +typedef struct INFOROM_NVL_OBJECT_V4S +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_NVL_OBJECT_V4_L1_THRESHOLD_DATA l1ThresholdData; + INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO remoteDeviceInfo[INFOROM_NVL_OBJECT_V4S_NUM_LINKS]; + INFOROM_NVL_OBJECT_V4S_MAX_CORRECTABLE_ERROR_RATES maxCorrectableErrorRates; + INFOROM_NVL_OBJECT_V3_ERROR_ENTRY errorLog[INFOROM_NVL_OBJECT_V4S_NUM_ERROR_ENTRIES]; +} INFOROM_NVL_OBJECT_V4S; + +#define INFOROM_NVL_OBJECT_V4S_REMOTE_DEVICE_INFO_ARRAY_FMT \ + INFOROM_FMT_REP64(INFOROM_NVL_OBJECT_V3_REMOTE_DEVICE_INFO_FMT) + +#define INFOROM_NVL_OBJECT_V4S_ERROR_LOG_FMT \ + INFOROM_FMT_REP512(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP512(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP256(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) \ + INFOROM_FMT_REP02(INFOROM_NVL_OBJECT_V3_ERROR_ENTRY_FMT) + +#define INFOROM_NVL_OBJECT_V4S_FMT \ + INFOROM_OBJECT_HEADER_V1_00_FMT \ + INFOROM_NVL_OBJECT_V4S_REMOTE_DEVICE_INFO_ARRAY_FMT \ + INFOROM_NVL_OBJECT_V4_L1_THRESHOLD_DATA_FMT \ + INFOROM_NVL_OBJECT_V4S_MAX_CORRECTABLE_ERROR_RATES_FMT \ + INFOROM_NVL_OBJECT_V4S_ERROR_LOG_FMT + +#define INFOROM_NVL_OBJECT_V4S_PACKED_SIZE 29680 + +#define INFOROM_NVL_OBJECT_V5_MAX_NUM_LINKS 18 +#define INFOROM_NVL_OBJECT_V5_MAX_NUM_LANES_PER_LINK 2 + +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_UPTIME_FMT "d" +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_LOCAL_MAX_RAW_BER_FMT "b" +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_LOCAL_MAX_EFF_BER_FMT "b" +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_CURR_EFF_MONITOR_FMT "w" +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_PLR_XMIT_RETRY_FMT "d" +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_TOTAL_RECOVERY_FMT "d" + +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_UPTIME_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_LOCAL_MAX_RAW_BER_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_LOCAL_MAX_EFF_BER_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_CURR_EFF_MONITOR_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_PLR_XMIT_RETRY_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_TOTAL_RECOVERY_FMT + +typedef struct INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA // u8[16] +{ + inforom_U032 uptime; // u8[4] + inforom_U008 localMaxRawBERExp; // u8[1] + inforom_U008 localMaxEffBERExp; // u8[1] + inforom_U016 currEffBERMonitor; // u8[2] + inforom_U032 plrXmitRetryWithinTSecMaxLo; // u8[4] + inforom_U032 totalSuccessfulRecoveryEvents; // u8[4] +} INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA; + +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_MFDE_FMT "43d" + +typedef struct INFOROM_NVL_OBJECT_V5_RECORD_METRIC_MFDE // u8[172] +{ + inforom_U032 data[43]; // u8[172] +} INFOROM_NVL_OBJECT_V5_RECORD_METRIC_MFDE; + +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_INITIAL_RAW_BER_FMT "18b" +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_INITIAL_EFF_BER_FMT "18b" +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_LAST_DOWN_REASON_FMT "18b" +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_PADDING_FMT "82b" + +#define INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_MFDE_FMT \ + INFOROM_FMT_REP16(INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_FMT) \ + INFOROM_FMT_REP02(INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA_FMT) \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_INITIAL_RAW_BER_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_INITIAL_EFF_BER_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_LAST_DOWN_REASON_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_PADDING_FMT + +typedef struct INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA // u8[596] +{ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_MFDE mfdeEvent; // u8[172] + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_LINK_DATA link[INFOROM_NVL_OBJECT_V5_MAX_NUM_LINKS]; // u8[288] + inforom_U008 initialRawBERExp[INFOROM_NVL_OBJECT_V5_MAX_NUM_LINKS]; // u8[18] + inforom_U008 initialEffBERExp[INFOROM_NVL_OBJECT_V5_MAX_NUM_LINKS]; // u8[18] + inforom_U008 lastDownReason[INFOROM_NVL_OBJECT_V5_MAX_NUM_LINKS]; // u8[18] + inforom_U008 padding[82]; // u8[82] +} INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA; + +#define INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_HISTOGRAM_FMT "12b" + +typedef struct INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_HISTOGRAM // u8[12] +{ + inforom_U008 fixedPoint[12]; // u8[12] +} INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_HISTOGRAM; + +#define INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_LINK_DATA_LINK_DOWN_COUNT_FMT "d" + +#define INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_LINK_DATA_FMT \ + "4b" \ + INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_LINK_DATA_LINK_DOWN_COUNT_FMT \ + INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_HISTOGRAM_FMT + +typedef struct INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_LINK_DATA // u8[20] +{ + inforom_U008 reserved[4]; // u8[4] + inforom_U032 linkDownCount; // u8[4] + INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_HISTOGRAM downReasonHistogram; // u8[12] +} INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_LINK_DATA; + +#define INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA_GLOBAL_MAX_RAW_BER_FMT "18b" +#define INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA_GLOBAL_MAX_EFF_BER_FMT "18b" + +#define INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA_FMT \ + INFOROM_FMT_REP16(INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_LINK_DATA_FMT) \ + INFOROM_FMT_REP02(INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_LINK_DATA_FMT) \ + INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA_GLOBAL_MAX_RAW_BER_FMT \ + INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA_GLOBAL_MAX_EFF_BER_FMT + +typedef struct INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA // u8[396] +{ + INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_LINK_DATA link[INFOROM_NVL_OBJECT_V5_MAX_NUM_LINKS]; // u8[360] + inforom_U008 globalMaxRawBERExp[INFOROM_NVL_OBJECT_V5_MAX_NUM_LINKS]; // u8[18] + inforom_U008 globalMaxEffBERExp[INFOROM_NVL_OBJECT_V5_MAX_NUM_LINKS]; // u8[18] +} INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA; + +#define INFOROM_NVL_OBJECT_V5_RESERVED_FMT "8b" + +#define INFOROM_NVL_OBJECT_V5_FMT \ + INFOROM_OBJECT_HEADER_V1_00_FMT \ + INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA_FMT \ + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA_FMT \ + INFOROM_NVL_OBJECT_V5_RESERVED_FMT + +#define INFOROM_NVL_OBJECT_V5_PACKED_SIZE 1008 + +typedef struct INFOROM_NVL_OBJECT_V5 // u8[1008] +{ + INFOROM_OBJECT_HEADER_V1_00 header; // u8[8] + INFOROM_NVL_OBJECT_V5_ACCUMULATIVE_METRIC_DATA accumulative; // u8[396] + INFOROM_NVL_OBJECT_V5_RECORD_METRIC_DATA record; // u8[596] + inforom_U008 reserved[8]; // u8[8] +} INFOROM_NVL_OBJECT_V5; + +#define INFOROM_NVL_OBJECT_FLUSH_INTERVAL (60 * 10) + +typedef struct +{ + struct + { + NvU32 flitCrc; + NvU32 laneCrc[4]; + } errorsPerMinute[INFOROM_NVL_OBJECT_V3G_NUM_LINKS]; + + struct + { + NvU32 flitCrc; + NvU32 laneCrc[4]; + NvU32 rxLinkReplay; + NvU32 txLinkReplay; + NvU32 linkRecovery; + } lastRead[INFOROM_NVL_OBJECT_V3G_NUM_LINKS]; +} INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V3G; + +typedef struct +{ + struct + { + NvU32 flitCrc; + NvU32 laneCrc[4]; + } errorsPerMinute[INFOROM_NVL_OBJECT_V3S_NUM_LINKS]; + + struct + { + NvU32 flitCrc; + NvU32 laneCrc[4]; + NvU32 rxLinkReplay; + NvU32 txLinkReplay; + NvU32 linkRecovery; + } lastRead[INFOROM_NVL_OBJECT_V3S_NUM_LINKS]; +} INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V3S; + +typedef struct +{ + struct + { + NvU32 flitCrc; + NvU32 laneCrc[4]; + } errorsPerMinute[INFOROM_NVL_OBJECT_V4G_NUM_LINKS]; + + struct + { + NvU32 flitCrc; + NvU32 laneCrc[4]; + NvU32 rxLinkReplay; + NvU32 txLinkReplay; + NvU32 linkRecovery; + } lastRead[INFOROM_NVL_OBJECT_V4G_NUM_LINKS]; +} INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V4G; + +typedef struct +{ + struct + { + NvU32 flitCrc; + NvU32 laneCrc[4]; + } errorsPerMinute[INFOROM_NVL_OBJECT_V4S_NUM_LINKS]; + + struct + { + NvU32 flitCrc; + NvU32 laneCrc[4]; + NvU32 rxLinkReplay; + NvU32 txLinkReplay; + NvU32 linkRecovery; + } lastRead[INFOROM_NVL_OBJECT_V4S_NUM_LINKS]; +} INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V4S; + +typedef struct INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V5 +{ + struct + { + void *pParams; + void *pAsyncHandle; + } request[2]; + NvU8 head; +} INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V5, INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V6; + +typedef union +{ + INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V3G v3g; + INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V3S v3s; + INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V4G v4g; + INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V4S v4s; + INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V5 v5; + INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE_V6 v6; +} INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE; + +typedef struct +{ + const char *pFmt; + NvU8 *pPackedObject; + INFOROM_NVL_OBJECT *pNvl; + + NvBool bDirty; + NvBool bDisableFatalErrorLogging; + NvBool bDisableCorrectableErrorLogging; + NvBool bCallbackPending; + + NvU32 timeSinceLastFlush; + + INFOROM_NVL_CORRECTABLE_ERROR_RATE_STATE correctableErrorRateState; +} INFOROM_NVLINK_STATE, *PINFOROM_NVLINK_STATE; + +// Version 6.0 ------------------------------------------------------------- +#define INFOROM_NVL_OBJECT_V6_MAX_NUM_LINKS 36 +#define INFOROM_NVL_OBJECT_V6_MAX_NUM_LANES_PER_LINK 2 + +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_PLR_TX_BW_LOSS_LOW_FREQUENCY_FMT "6w" +typedef struct INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_PLR_TX_BW_LOSS_LOW_FREQUENCY // u16[6] +{ + inforom_U016 lastPlrTxBwLossLf; // u16 + inforom_U016 meanPlrTxBwLossLf; // u16 + inforom_U016 maxPlrTxBwLossLf; // u16 + inforom_U016 consBelowPlrTxBwLossLfThreshold; // u16 + inforom_U016 numOfPlrTxBwLossLfWarnings; // u16 + inforom_U016 numOfPlrTxBwLossLfAlarms; // u16 +} INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_PLR_TX_BW_LOSS_LOW_FREQUENCY; + +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_RECOVERY_BW_LOSS_LOW_FREQUENCY_FMT "6w" +typedef struct INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_RECOVERY_BW_LOSS_LOW_FREQUENCY // u16[6] +{ + inforom_U016 lastRecoveryBwLossLf; // u16 + inforom_U016 meanRecoveryBwLossLf; // u16 + inforom_U016 maxRecoveryBwLossLf; // u16 + inforom_U016 consBelowRecoveryBwLossLfThreshold; // u16 + inforom_U016 numOfRecoveryBwLossLfWarnings; // u16 + inforom_U016 numOfRecoveryBwLossLfAlarms; // u16 +} INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_RECOVERY_BW_LOSS_LOW_FREQUENCY; + +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_EFFECTIVE_BER_LOW_FREQUENCY_FMT "3w3b6n" +typedef struct INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_EFFECTIVE_BER_LOW_FREQUENCY // u8[12] +{ + inforom_U016 consBelowEffBerLfThreshold; // u16 + inforom_U016 numOfEffBerLfWarnings; // u16 + inforom_U016 numOfEffBERLfAlarms; // u16 + inforom_U008 meanEffBerLfMagnitue; // u8 + inforom_U008 lastEffBerLfMagnitue; // u8 + inforom_U008 maxEffBerLfMagnitue; // u8 + inforom_U004 lastEffBerLfCoef; // u4 + inforom_U004 meanEffBerLfCoef; // u4 + inforom_U004 maxEffBerLfCoef; // u4 + inforom_U004 padding[3]; // u4[3] +} INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_EFFECTIVE_BER_LOW_FREQUENCY; + +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_UPTIME_FMT "d" +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_TOTAL_RECOVERY_FMT "d" + +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_UPTIME_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_PLR_TX_BW_LOSS_LOW_FREQUENCY_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_RECOVERY_BW_LOSS_LOW_FREQUENCY_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_EFFECTIVE_BER_LOW_FREQUENCY_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_TOTAL_RECOVERY_FMT + +typedef struct INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA // u8[44] +{ + inforom_U032 uptime; // u8[4] + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_PLR_TX_BW_LOSS_LOW_FREQUENCY plrTxBwLossLf; // u16[6] + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_RECOVERY_BW_LOSS_LOW_FREQUENCY recoveryBwLossLf; // u16[6] + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_EFFECTIVE_BER_LOW_FREQUENCY effBerLf; // u8[12] + inforom_U032 totalSuccessfulRecoveryEvents; // u8[4] +} INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA; + +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_MFDE_FMT "43w" + +typedef struct INFOROM_NVL_OBJECT_V6_RECORD_METRIC_MFDE // u8[172] +{ + inforom_U032 data[43]; // u8[172] +} INFOROM_NVL_OBJECT_V6_RECORD_METRIC_MFDE; + +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_INITIAL_RAW_BER_FMT "36b" +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_INITIAL_EFF_BER_FMT "36b" +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_LAST_DOWN_REASON_FMT "36b" +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_LAST_DOWN_ADVANCED_REASON_FMT "36w" + +#define INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_MFDE_FMT \ + INFOROM_FMT_REP32(INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA_FMT) \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_INITIAL_RAW_BER_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_INITIAL_EFF_BER_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_LAST_DOWN_REASON_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_LAST_DOWN_ADVANCED_REASON_FMT + +typedef struct INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA // u8[1936] +{ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_MFDE mfdeEvent; // u8[172] + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_LINK_DATA link[INFOROM_NVL_OBJECT_V6_MAX_NUM_LINKS]; // u8[1584] + inforom_U008 initialRawBERExp[INFOROM_NVL_OBJECT_V6_MAX_NUM_LINKS]; // u8[36] + inforom_U008 initialEffBERExp[INFOROM_NVL_OBJECT_V6_MAX_NUM_LINKS]; // u8[36] + inforom_U008 lastDownReason[INFOROM_NVL_OBJECT_V6_MAX_NUM_LINKS]; // u8[36] + inforom_U016 lastDownAdvancedReason[INFOROM_NVL_OBJECT_V6_MAX_NUM_LINKS]; // u8[72] +} INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA; + +#define INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_FMT "16b" + +typedef struct INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM // u8[24] +{ + inforom_U008 fixedPoint[24]; // u8[24] +} INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM; + +#define INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_LINK_FMT "5b" + +typedef struct INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_LINK // u8[5] +{ + inforom_U008 fixedPoint[5]; +} INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_LINK; + +#define INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_GPU_FMT "d3b" + +typedef struct INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_GPU // u8[7] +{ + inforom_U032 xidCount; // u8[4] + inforom_U008 fixedPoint[3]; // u8[3] 3 per GPU XIDs +} INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_GPU; + +#define INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA_LINK_DOWN_COUNT_FMT "d" +#define INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA_PADDING_FMT "3b" + +#define INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA_FMT \ + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA_LINK_DOWN_COUNT_FMT \ + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_FMT \ + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_LINK_FMT \ + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA_PADDING_FMT + +typedef struct INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA // u8[36] +{ + inforom_U032 linkDownCount; // u8[4] + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM downReasonHistogram; // u8[24] + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_LINK xidHistogram; // u8[5] + inforom_U008 padding[3]; // u8[3] +} INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA; + +#define INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_DATA_PADDING_FMT "b" + +#define INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_DATA_FMT \ + INFOROM_FMT_REP32(INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA_FMT) \ + INFOROM_FMT_REP04(INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA_FMT) \ + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_GPU_FMT \ + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_DATA_PADDING_FMT + +typedef struct INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_DATA // u8[1304] +{ + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_LINK_DATA link[INFOROM_NVL_OBJECT_V6_MAX_NUM_LINKS]; // u8[1296] + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_HISTOGRAM_XID_GPU xidHistogram; // u8[7] + inforom_U008 padding; // u8 +} INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_DATA; + +#define INFOROM_NVL_OBJECT_V6_RESERVED_FMT "784b" + +#define INFOROM_NVL_OBJECT_V6_FMT \ + INFOROM_OBJECT_HEADER_V1_00_FMT \ + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_DATA_FMT \ + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA_FMT \ + INFOROM_NVL_OBJECT_V6_RESERVED_FMT + +#define INFOROM_NVL_OBJECT_V6_PACKED_SIZE 4032 + +typedef struct INFOROM_NVL_OBJECT_V6 // u8[] +{ + INFOROM_OBJECT_HEADER_V1_00 header; // u8[8] + INFOROM_NVL_OBJECT_V6_ACCUMULATIVE_METRIC_DATA accumulative; // u8[1304] + INFOROM_NVL_OBJECT_V6_RECORD_METRIC_DATA record; // u8[1936] + inforom_U008 reserved[784]; // u8[784] +} INFOROM_NVL_OBJECT_V6; + +union INFOROM_NVL_OBJECT +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_NVL_OBJECT_V1 v1; + INFOROM_NVL_OBJECT_V2 v2; + INFOROM_NVL_OBJECT_V3G v3g; + INFOROM_NVL_OBJECT_V3S v3s; + INFOROM_NVL_OBJECT_V4G v4g; + INFOROM_NVL_OBJECT_V4S v4s; + INFOROM_NVL_OBJECT_V5 v5; + INFOROM_NVL_OBJECT_V6 v6; +}; + +#endif /* _IFRNVL_H_ */ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrstruct.h b/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrstruct.h new file mode 100644 index 0000000..5c1caed --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/inforom/ifrstruct.h @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _IFRSTRUCT_H_ +#define _IFRSTRUCT_H_ + +#include "inforom/types.h" + +#include "inforom/ifrnvl.h" +#include "inforom/ifrecc.h" +#include "inforom/ifrdem.h" +#include "inforom/omsdef.h" + +#define INFOROM_OBD_OBJECT_V1_XX_PACKED_SIZE 128 + +struct INFOROM_OBD_OBJECT_V1_XX +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U032 buildDate; + inforom_U008 marketingName[24]; + inforom_U008 serialNumber[16]; + inforom_U008 memoryManufacturer; + inforom_U008 memoryPartID[20]; + inforom_U008 memoryDateCode[6]; + inforom_U008 productPartNumber[20]; + inforom_U008 boardRevision[3]; + inforom_U008 boardType; + inforom_U008 board699PartNumber[20]; + inforom_U008 reserved[5]; +}; +#define INFOROM_OBD_OBJECT_V1_XX_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d116b" +typedef struct INFOROM_OBD_OBJECT_V1_XX INFOROM_OBD_OBJECT_V1_XX; + +#define INFOROM_OBD_OBJECT_V2_XX_PACKED_SIZE 224 + +struct INFOROM_OBD_OBJECT_V2_XX +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U032 buildDate; + inforom_U008 marketingName[24]; + inforom_U008 serialNumber[16]; + inforom_U008 memoryManufacturer; + inforom_U008 memoryPartID[20]; + inforom_U008 memoryDateCode[6]; + inforom_U008 productPartNumber[20]; + inforom_U008 boardRevision[3]; + inforom_U008 boardType; + inforom_U008 board699PartNumber[20]; + inforom_U032 productLength; + inforom_U032 productWidth; + inforom_U032 productHeight; + inforom_U008 vbiosFactoryVersion[5]; + inforom_U008 board965PartNumber[20]; + inforom_U008 reserved[64]; +}; +#define INFOROM_OBD_OBJECT_V2_XX_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d111b3d89b" +typedef struct INFOROM_OBD_OBJECT_V2_XX INFOROM_OBD_OBJECT_V2_XX; + +// +// OEM 1.0 +// + +#define INFOROM_OEM_OBJECT_V1_00_PACKED_SIZE 512 +#define INFOROM_OEM_OBJECT_HEADER_VERSION 1 + +struct INFOROM_OEM_OBJECT_V1_00 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U008 oemInfo[504]; +}; +#define INFOROM_OEM_OBJECT_V1_00_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "504b" +typedef struct INFOROM_OEM_OBJECT_V1_00 INFOROM_OEM_OBJECT_V1_00; + +#define INFOROM_IMG_OBJECT_V1_00_PACKED_SIZE 64 +#define INFOROM_IMG_OBJECT_V1_00_VERSION_LENGTH 16 + +struct INFOROM_IMG_OBJECT_V1_00 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U008 version[INFOROM_IMG_OBJECT_V1_00_VERSION_LENGTH]; + inforom_U016 pciDeviceId; + inforom_U016 pciVendorId; + inforom_U016 pciSubsystemId; + inforom_U016 pciSubsystemVendorId; + inforom_U008 reserved[32]; +}; +#define INFOROM_IMG_OBJECT_V1_00_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "16b4w32b" +typedef struct INFOROM_IMG_OBJECT_V1_00 INFOROM_IMG_OBJECT_V1_00; + +#endif // _IFRSTRUCT_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/inforom/omsdef.h b/userspace/sdks/nv570/nvalloc/common/inc/inforom/omsdef.h new file mode 100644 index 0000000..9509a7b --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/inforom/omsdef.h @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _INFOROM_OMSDEF_H_ +#define _INFOROM_OMSDEF_H_ + +#include "inforom/types.h" + +#define INFOROM_OMS_OBJECT_V1_PACKED_SIZE 112 + +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_ENTRY_AVAILABLE 0:0 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_ENTRY_AVAILABLE_NO 0 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_ENTRY_AVAILABLE_YES 1 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_FORCE_DEVICE_DISABLE 1:1 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_FORCE_DEVICE_DISABLE_NO 0 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_FORCE_DEVICE_DISABLE_YES 1 + +#define INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY_DATA_RESERVED 23:2 +#define INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY_DATA_ENTRY_CHECKSUM 31:24 + +typedef struct INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY +{ + inforom_U032 data; + inforom_U032 clockLimitMin; + inforom_U032 clockLimitMax; +} INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY; + +#define INFOROM_OMS_OBJECT_V1G_NUM_SETTINGS_ENTRIES 8 + +typedef struct INFOROM_OMS_OBJECT_V1G +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U032 lifetimeRefreshCount; + INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY settings[ + INFOROM_OMS_OBJECT_V1G_NUM_SETTINGS_ENTRIES]; + inforom_U032 reserved; +} INFOROM_OMS_OBJECT_V1G; + +#define INFOROM_OMS_OBJECT_V1G_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "26d" + +#define INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY_DATA_RESERVED 7:2 +#define INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY_DATA_ENTRY_CHECKSUM 15:8 + +typedef struct INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY +{ + inforom_U016 data; +} INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY; + +#define INFOROM_OMS_OBJECT_V1S_NUM_SETTINGS_ENTRIES 50 + +typedef struct INFOROM_OMS_OBJECT_V1S +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U032 lifetimeRefreshCount; + INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY settings[ + INFOROM_OMS_OBJECT_V1S_NUM_SETTINGS_ENTRIES]; +} INFOROM_OMS_OBJECT_V1S; + +#define INFOROM_OMS_OBJECT_V1S_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d50w" + +typedef struct INFOROM_OMS_V1S_DATA +{ + INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY *pIter; + INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY prev; + INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY *pNext; +} INFOROM_OMS_V1S_DATA; + +typedef struct INFOROM_OMS_V1G_DATA +{ + INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY *pIter; + INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY prev; + INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY *pNext; +} INFOROM_OMS_V1G_DATA; + +typedef union +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_OMS_OBJECT_V1S v1s; + INFOROM_OMS_OBJECT_V1G v1g; +} INFOROM_OMS_OBJECT; + +typedef union +{ + INFOROM_OMS_V1S_DATA v1s; + INFOROM_OMS_V1G_DATA v1g; +} INFOROM_OMS_DATA; + +typedef struct +{ + const char *pFmt; + NvU8 *pPackedObject; + INFOROM_OMS_OBJECT *pOms; + INFOROM_OMS_DATA omsData; +} INFOROM_OMS_STATE; + +#endif /* _INFOROM_OMSDEF_H_ */ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/inforom/types.h b/userspace/sdks/nv570/nvalloc/common/inc/inforom/types.h new file mode 100644 index 0000000..c062cb0 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/inforom/types.h @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _INFOROM_TYPES_H_ +#define _INFOROM_TYPES_H_ + +/*! + * @file types.h + * @brief Common types and definitions used by InfoROM objects + */ + +typedef NvS32 inforom_S008; +typedef NvU32 inforom_U004; +typedef NvU32 inforom_U008; +typedef NvU32 inforom_U016; +typedef NvU32 inforom_U024; +typedef NvU32 inforom_U032; +typedef NvU64 inforom_U064; +typedef NvU8 inforom_X008; + +#define INFOROM_FMT_S08 's' +#define INFOROM_FMT_U04 'n' +#define INFOROM_FMT_U08 'b' +#define INFOROM_FMT_U16 'w' +#define INFOROM_FMT_U24 't' +#define INFOROM_FMT_U32 'd' +#define INFOROM_FMT_U64 'q' +#define INFOROM_FMT_BINARY 'x' + +// Helper macros for generating repeating format sequences +#define INFOROM_FMT_REP02(fmt) fmt fmt +#define INFOROM_FMT_REP04(fmt) INFOROM_FMT_REP02(fmt) INFOROM_FMT_REP02(fmt) +#define INFOROM_FMT_REP08(fmt) INFOROM_FMT_REP04(fmt) INFOROM_FMT_REP04(fmt) +#define INFOROM_FMT_REP16(fmt) INFOROM_FMT_REP08(fmt) INFOROM_FMT_REP08(fmt) +#define INFOROM_FMT_REP32(fmt) INFOROM_FMT_REP16(fmt) INFOROM_FMT_REP16(fmt) +#define INFOROM_FMT_REP64(fmt) INFOROM_FMT_REP32(fmt) INFOROM_FMT_REP32(fmt) +#define INFOROM_FMT_REP128(fmt) INFOROM_FMT_REP64(fmt) INFOROM_FMT_REP64(fmt) +#define INFOROM_FMT_REP256(fmt) INFOROM_FMT_REP128(fmt) INFOROM_FMT_REP128(fmt) +#define INFOROM_FMT_REP512(fmt) INFOROM_FMT_REP256(fmt) INFOROM_FMT_REP256(fmt) + +#define INFOROM_OBJECT_SUBVERSION_SUPPORTS_NVSWITCH(sv) \ + ((((sv) & 0xF0) == 0) || (((sv) & 0xF0) == 0x20)) + +#define INFOROM_OBJECT_HEADER_V1_00_SIZE_OFFSET 0x05 +#define INFOROM_OBJECT_HEADER_V1_00_CHECKSUM_OFFSET 0x07 +#define INFOROM_OBJECT_HEADER_V1_00_PACKED_SIZE 8 + +typedef struct INFOROM_OBJECT_HEADER_V1_00 +{ + inforom_S008 type[3]; + inforom_U008 version; + inforom_U008 subversion; + inforom_U016 size; + inforom_U008 checksum; +} INFOROM_OBJECT_HEADER_V1_00; + +#define INFOROM_OBJECT_HEADER_V1_00_FMT "3s2bwb" + +#endif // _INFOROM_TYPES_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/mctp_format.h b/userspace/sdks/nv570/nvalloc/common/inc/mctp_format.h new file mode 100644 index 0000000..dd2052a --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/mctp_format.h @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef MCTP_FORMAT_H +#define MCTP_FORMAT_H + +/** + * MCTP (Management Component Transport Protocol) format definitions + * + * Based on DMTF DSP0236 - MCTP Base Specification. + * This is a common header that will be used across multiple RM subsystems + * like FSP, GSP, SEC2, etc. for standardized inter-processor communication. + * + * MCTP provides a transport-agnostic protocol for message exchange between + * platform management components. RM uses MCTP's basic transport header + * with vendor-defined message types (NVDM) for internal communication. + */ + +// MCTP Transport Header Fields (First DWORD) +#define MCTP_HEADER_VERSION 3:0 +#define MCTP_HEADER_RSVD 7:4 +#define MCTP_HEADER_DEID 15:8 +#define MCTP_HEADER_SEID 23:16 +#define MCTP_HEADER_TAG 26:24 +#define MCTP_HEADER_TO 27:27 +#define MCTP_HEADER_SEQ 29:28 +#define MCTP_HEADER_EOM 30:30 +#define MCTP_HEADER_SOM 31:31 + +// MCTP Message Header Fields (Second DWORD) +#define MCTP_MSG_HEADER_TYPE 6:0 +#define MCTP_MSG_HEADER_IC 7:7 +#define MCTP_MSG_HEADER_VENDOR_ID 23:8 +#define MCTP_MSG_HEADER_NVDM_TYPE 31:24 + +// MCTP Constants from specification +#define MCTP_MSG_HEADER_TYPE_VENDOR_PCI 0x7e +#define MCTP_MSG_HEADER_VENDOR_ID_NV 0x10de + +/** + * @brief Create MCTP transport header + * + * Generic helper function to construct the MCTP transport header (first DWORD). + * Can be used by any subsystem (FSP, GSP, SEC2, etc.). + * + * Note: TAG and TO fields are set to 0. These are only needed for complex + * scenarios with concurrent interleaved messages or request/response correlation, + * which are not currently used. Header version is hard-coded to 0x1 (MCTP 1.0). + * + * @param[in] som Start of Message flag (1 for first packet) + * @param[in] eom End of Message flag (1 for last packet) + * @param[in] seid Source Endpoint ID + * @param[in] deid Destination Endpoint ID + * @param[in] seq Packet sequence number (0-3, modulo 4) + * + * @return Constructed MCTP transport header (32-bit DWORD) + */ +static NV_INLINE NvU32 +mctpCreateTransportHeader +( + NvU8 som, + NvU8 eom, + NvU8 seid, + NvU8 deid, + NvU8 seq +) +{ + return REF_NUM(MCTP_HEADER_VERSION, 0x1) | + REF_NUM(MCTP_HEADER_SEID, seid) | + REF_NUM(MCTP_HEADER_DEID, deid) | + REF_NUM(MCTP_HEADER_SEQ, seq) | + REF_NUM(MCTP_HEADER_EOM, eom) | + REF_NUM(MCTP_HEADER_SOM, som); +} + +/** + * @brief Create NVDM over MCTP header + * + * Generic helper function to construct the NVDM message header (second DWORD). + * Sets the MCTP message type to Vendor Defined PCI (0x7E) and includes + * NVIDIA's vendor ID (0x10DE) along with the subsystem-specific NVDM type. + * Can be used by any subsystem (FSP, GSP, SEC2, etc.). + * + * @param[in] nvdmType NVDM message type (subsystem/communication-path specific) + * + * @return Constructed NVDM header (32-bit DWORD) + */ +static NV_INLINE NvU32 +mctpCreateNvdmHeader +( + NvU8 nvdmType +) +{ + return REF_DEF(MCTP_MSG_HEADER_TYPE, _VENDOR_PCI) | + REF_NUM(MCTP_MSG_HEADER_IC, 0) | + REF_DEF(MCTP_MSG_HEADER_VENDOR_ID, _NV) | + REF_NUM(MCTP_MSG_HEADER_NVDM_TYPE, nvdmType); +} + +#endif // MCTP_FORMAT_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/nv-firmware-chip-family-select.h b/userspace/sdks/nv570/nvalloc/common/inc/nv-firmware-chip-family-select.h new file mode 100644 index 0000000..962776a --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/nv-firmware-chip-family-select.h @@ -0,0 +1,230 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NV_FIRMWARE_CHIP_FAMILY_SELECT_H +#define NV_FIRMWARE_CHIP_FAMILY_SELECT_H + + + +#include <published/nv_arch.h> +#include <nv-firmware.h> +#include <nvctassert.h> + +static inline nv_firmware_chip_family_t nv_firmware_get_chip_family( + NvU32 gpuArch, + NvU32 gpuImpl +) +{ + switch (gpuArch) + { + case GPU_ARCHITECTURE_TURING: + if (gpuImpl <= GPU_IMPLEMENTATION_TU106) + return NV_FIRMWARE_CHIP_FAMILY_TU10X; + else + return NV_FIRMWARE_CHIP_FAMILY_TU11X; + + case GPU_ARCHITECTURE_AMPERE: + if (gpuImpl == GPU_IMPLEMENTATION_GA100) + return NV_FIRMWARE_CHIP_FAMILY_GA100; + else + return NV_FIRMWARE_CHIP_FAMILY_GA10X; + + case GPU_ARCHITECTURE_ADA: + return NV_FIRMWARE_CHIP_FAMILY_AD10X; + + case GPU_ARCHITECTURE_HOPPER: + return NV_FIRMWARE_CHIP_FAMILY_GH100; + + case GPU_ARCHITECTURE_BLACKWELL_GB1XX: + if (gpuImpl == GPU_IMPLEMENTATION_GB10B) + return NV_FIRMWARE_CHIP_FAMILY_GB10Y; + else + return NV_FIRMWARE_CHIP_FAMILY_GB10X; + + case GPU_ARCHITECTURE_BLACKWELL_GB2XX: + if (gpuImpl >= GPU_IMPLEMENTATION_GB20B) + return NV_FIRMWARE_CHIP_FAMILY_GB20Y; + else + return NV_FIRMWARE_CHIP_FAMILY_GB20X; + + case GPU_ARCHITECTURE_RUBIN_GR1XX: + return NV_FIRMWARE_CHIP_FAMILY_GR10X; + } + + return NV_FIRMWARE_CHIP_FAMILY_NULL; +} + +typedef enum +{ + NV_FIRMWARE_TASK_INIT, + NV_FIRMWARE_TASK_RM, + NV_FIRMWARE_TASK_INTR, + NV_FIRMWARE_TASK_VGPU, + NV_FIRMWARE_TASK_MNOC, + NV_FIRMWARE_TASK_DEBUG, + NV_FIRMWARE_TASK_ROOT, + NV_FIRMWARE_TASK_RM_STATE_MONITOR, + + NV_FIRMWARE_TASK_COUNT +} nv_firmware_task_t; + +typedef enum { + NV_FIRMWARE_LIBOS_NONE_SUPPORTED = 0x0, + NV_FIRMWARE_LIBOS2_SUPPORTED = 0x1, + NV_FIRMWARE_LIBOS3_SUPPORTED = 0x2, +} libosMappingType; + +typedef struct +{ + const char *memory_id; + const char *prefix; + const char *elf_section_name; + const NvU8 supported_os; // bitmask +} nv_firmware_task_log_info_t; + +static inline +const nv_firmware_task_log_info_t *nv_firmware_get_task_log_infos(unsigned *num_infos) +{ + const static nv_firmware_task_log_info_t infos[] = + { + {"LOGINIT", "INIT", ".fwlogging_init", NV_FIRMWARE_LIBOS2_SUPPORTED | NV_FIRMWARE_LIBOS3_SUPPORTED}, + {"LOGRM", "RM", ".fwlogging_rm", NV_FIRMWARE_LIBOS2_SUPPORTED | NV_FIRMWARE_LIBOS3_SUPPORTED}, + {"LOGINTR", "INTR", ".fwlogging_rm", NV_FIRMWARE_LIBOS3_SUPPORTED}, + {"LOGVGPU", "VGPU", ".fwlogging_vgpu", NV_FIRMWARE_LIBOS2_SUPPORTED | NV_FIRMWARE_LIBOS3_SUPPORTED}, + {"LOGMNOC", "MNOC", ".fwlogging_mnoc", NV_FIRMWARE_LIBOS3_SUPPORTED}, + {"LOGDBG", "DBG", ".fwlogging_debug", NV_FIRMWARE_LIBOS3_SUPPORTED}, + {"LOGROOT", "ROOT", ".fwlogging_start", NV_FIRMWARE_LIBOS3_SUPPORTED}, + {"LOGRMON", "RMON", ".fwlogging_rm", NV_FIRMWARE_LIBOS3_SUPPORTED}, + }; + + *num_infos = NV_ARRAY_ELEMENTS(infos); + ct_assert(NV_ARRAY_ELEMENTS(infos) == NV_FIRMWARE_TASK_COUNT); + + return infos; +} + +static inline +const nv_firmware_task_log_info_t* nv_firmware_log_info_for_task(nv_firmware_task_t task_id) +{ + unsigned num_infos = 0; + const nv_firmware_task_log_info_t* infos = nv_firmware_get_task_log_infos(&num_infos); + + if ((unsigned)task_id >= num_infos) + { + return NULL; + } + + return &infos[task_id]; +} + +typedef struct +{ + const char *elf_section_name; + + struct + { + NvU32 lo, hi; + } gpuArch; + + struct + { + NvU32 lo, hi; + } gpuImpl; +} nv_firmware_kernel_log_info_t; + +#define NV_FIRMWARE_KERNEL_LOG_MEMORY_ID "LOGKRNL" +#define NV_FIRMWARE_KERNEL_LOG_PREFIX "KRNL" +#define NV_FIRMWARE_GPU_ARCH_SHIFT 0x4 + +// +// HAL for selecting .fwlogging section. Chip ID is checked to fall within {gpuArch, gpuImpl}::{lo, hi} +// ranges. Optional gpuImpl range can be used to specify sub-range of an architecture. +// If `hi` is not set, the range is treated as open. Ranges are checked in order of specification. +// Range overlaps are allowed and resolved by the order. +// +static inline +const nv_firmware_kernel_log_info_t *nv_firmware_get_kernel_log_infos(unsigned *num_infos) +{ + const static nv_firmware_kernel_log_info_t infos[] = + { + {".fwlogging_kernel_gr10x", + {GPU_ARCHITECTURE_RUBIN_GR1XX}, // gr10x... + }, + {".fwlogging_kernel_gb20y", + {GPU_ARCHITECTURE_BLACKWELL_GB2XX, GPU_ARCHITECTURE_BLACKWELL_GB2XX}, // gb20y + {GPU_IMPLEMENTATION_GB20B, 0} + }, + {".fwlogging_kernel_gb20x", + {GPU_ARCHITECTURE_BLACKWELL_GB2XX}, // gb20x... + }, + {".fwlogging_kernel_gb10y", + {GPU_ARCHITECTURE_BLACKWELL_GB1XX, GPU_ARCHITECTURE_BLACKWELL_GB1XX}, // gb10y + {GPU_IMPLEMENTATION_GB10B, 0} + }, + + {".fwlogging_kernel_gb10x", + {GPU_ARCHITECTURE_BLACKWELL_GB1XX}, // gb10x... + }, + + {".fwlogging_kernel_gh100", + {GPU_ARCHITECTURE_HOPPER, GPU_ARCHITECTURE_HOPPER}, // gh100 + }, + + {".fwlogging_kernel_ga10x", + {GPU_ARCHITECTURE_AMPERE, GPU_ARCHITECTURE_ADA}, // ga10x...ad10x + }, + }; + + *num_infos = NV_ARRAY_ELEMENTS(infos); + + return infos; +} + +static inline +const nv_firmware_kernel_log_info_t *nv_firmware_kernel_log_info_for_gpu(NvU32 gpuArch, NvU32 gpuImpl) +{ + unsigned num_infos = 0; + const nv_firmware_kernel_log_info_t* infos = nv_firmware_get_kernel_log_infos(&num_infos); + + for (unsigned i = 0; i < num_infos; i++) + { + const nv_firmware_kernel_log_info_t* info = &infos[i]; + + NvBool bMatchingArch = info->gpuArch.lo <= gpuArch; + if (info->gpuArch.hi != 0) + bMatchingArch &= gpuArch <= info->gpuArch.hi; + + NvBool bMatchingImpl = info->gpuImpl.lo <= gpuImpl; + if (info->gpuImpl.hi != 0) + bMatchingImpl &= gpuImpl <= info->gpuImpl.hi; + + if (bMatchingArch && bMatchingImpl) + { + return info; + } + } + + return NULL; +} + +#endif // NV_FIRMWARE_CHIP_FAMILY_SELECT_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/nv-firmware.h b/userspace/sdks/nv570/nvalloc/common/inc/nv-firmware.h new file mode 100644 index 0000000..000aa38 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/nv-firmware.h @@ -0,0 +1,145 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NV_FIRMWARE_H +#define NV_FIRMWARE_H + + + +#include <nvtypes.h> +#include <nvmisc.h> + +typedef enum +{ + NV_FIRMWARE_TYPE_GSP, + NV_FIRMWARE_TYPE_GSP_LOG, + NV_FIRMWARE_TYPE_UCODES +} nv_firmware_type_t; + +typedef enum +{ + NV_FIRMWARE_CHIP_FAMILY_NULL = 0, + NV_FIRMWARE_CHIP_FAMILY_TU10X = 1, + NV_FIRMWARE_CHIP_FAMILY_TU11X = 2, + NV_FIRMWARE_CHIP_FAMILY_GA100 = 3, + NV_FIRMWARE_CHIP_FAMILY_GA10X = 4, + NV_FIRMWARE_CHIP_FAMILY_AD10X = 5, + NV_FIRMWARE_CHIP_FAMILY_GH100 = 6, + NV_FIRMWARE_CHIP_FAMILY_GB10X = 8, + NV_FIRMWARE_CHIP_FAMILY_GB20X = 9, + NV_FIRMWARE_CHIP_FAMILY_GR10X = 10, + NV_FIRMWARE_CHIP_FAMILY_GB10Y = 11, + NV_FIRMWARE_CHIP_FAMILY_GB20Y = 12, + NV_FIRMWARE_CHIP_FAMILY_END, +} nv_firmware_chip_family_t; + +static inline const char *nv_firmware_chip_family_to_string( + nv_firmware_chip_family_t fw_chip_family +) +{ + switch (fw_chip_family) { + case NV_FIRMWARE_CHIP_FAMILY_GR10X: return "gr10x"; + case NV_FIRMWARE_CHIP_FAMILY_GB10X: return "gb10x"; + case NV_FIRMWARE_CHIP_FAMILY_GB10Y: return "gb10y"; + case NV_FIRMWARE_CHIP_FAMILY_GB20Y: return "gb20y"; + case NV_FIRMWARE_CHIP_FAMILY_GB20X: return "gb20x"; + case NV_FIRMWARE_CHIP_FAMILY_GH100: return "gh100"; + case NV_FIRMWARE_CHIP_FAMILY_AD10X: return "ad10x"; + case NV_FIRMWARE_CHIP_FAMILY_GA10X: return "ga10x"; + case NV_FIRMWARE_CHIP_FAMILY_GA100: return "ga100"; + case NV_FIRMWARE_CHIP_FAMILY_TU11X: return "tu11x"; + case NV_FIRMWARE_CHIP_FAMILY_TU10X: return "tu10x"; + + case NV_FIRMWARE_CHIP_FAMILY_END: // fall through + case NV_FIRMWARE_CHIP_FAMILY_NULL: + return ""; + } + return ""; +} + +// The includer may optionally define +// NV_FIRMWARE_FOR_NAME(name) +// to return a platform-defined string for a given a firmware base name. +// +// The function nv_firmware_for_chip_family will then be available. +#if defined(NV_FIRMWARE_FOR_NAME) +static inline const char *nv_firmware_for_chip_family( + nv_firmware_type_t fw_type, + nv_firmware_chip_family_t fw_chip_family +) +{ + switch (fw_chip_family) + { + case NV_FIRMWARE_CHIP_FAMILY_GR10X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GB10X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GB10Y: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GB20Y: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GB20X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through + case NV_FIRMWARE_CHIP_FAMILY_AD10X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GA10X: + { + switch (fw_type) + { + case NV_FIRMWARE_TYPE_GSP: return NV_FIRMWARE_FOR_NAME("gsp_ga10x"); + case NV_FIRMWARE_TYPE_GSP_LOG: return NV_FIRMWARE_FOR_NAME("gsp_log_ga10x"); + case NV_FIRMWARE_TYPE_UCODES: return NV_FIRMWARE_FOR_NAME("ucodes_ga10x"); + } + return ""; + } + + case NV_FIRMWARE_CHIP_FAMILY_GA100: // fall through + case NV_FIRMWARE_CHIP_FAMILY_TU11X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_TU10X: + { + switch (fw_type) + { + case NV_FIRMWARE_TYPE_GSP: return NV_FIRMWARE_FOR_NAME("gsp_tu10x"); + case NV_FIRMWARE_TYPE_GSP_LOG: return NV_FIRMWARE_FOR_NAME("gsp_log_tu10x"); + case NV_FIRMWARE_TYPE_UCODES: return NV_FIRMWARE_FOR_NAME("ucodes_tu10x"); + } + return ""; + } + + case NV_FIRMWARE_CHIP_FAMILY_END: // fall through + case NV_FIRMWARE_CHIP_FAMILY_NULL: + return ""; + } + + return ""; +} +#endif // defined(NV_FIRMWARE_FOR_NAME) + +// The includer may optionally define +// NV_FIRMWARE_DECLARE_GSP(name) +// which will then be invoked (at the top-level) for each +// gsp_* and ucodes_* (but not gsp_log_*) +#if defined(NV_FIRMWARE_DECLARE_GSP) +NV_FIRMWARE_DECLARE_GSP("gsp_ga10x") +NV_FIRMWARE_DECLARE_GSP("gsp_tu10x") +NV_FIRMWARE_DECLARE_GSP("ucodes_ga10x") +NV_FIRMWARE_DECLARE_GSP("ucodes_tu10x") +#endif // defined(NV_FIRMWARE_DECLARE_GSP) + +#endif // NV_FIRMWARE_DECLARE_GSP + diff --git a/userspace/sdks/nv570/nvalloc/common/inc/nvcst.h b/userspace/sdks/nv570/nvalloc/common/inc/nvcst.h new file mode 100644 index 0000000..1b7239a --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/nvcst.h @@ -0,0 +1,460 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2004-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVCST_H +#define NVCST_H + +#include <platform/chipset/chipset.h> +#include <platform/chipset/chipset_info.h> +#include <nvpcie.h> +#include <nvdevid.h> + +#define CHIPSET_SETUP_FUNC(name) static NV_STATUS name(OBJCL *pCl); + +CHIPSET_SETUP_FUNC(Intel_25XX_setupFunc) +CHIPSET_SETUP_FUNC(Intel_27XX_setupFunc) +CHIPSET_SETUP_FUNC(Intel_2A40_setupFunc) +CHIPSET_SETUP_FUNC(Intel_0040_setupFunc) +CHIPSET_SETUP_FUNC(Intel_2E00_setupFunc) +CHIPSET_SETUP_FUNC(Intel_25E0_setupFunc) +CHIPSET_SETUP_FUNC(Intel_29XX_setupFunc) +CHIPSET_SETUP_FUNC(Intel_29X0_setupFunc) +CHIPSET_SETUP_FUNC(Intel_29E0_setupFunc) +CHIPSET_SETUP_FUNC(Intel_359E_setupFunc) +CHIPSET_SETUP_FUNC(Intel_4000_setupFunc) +CHIPSET_SETUP_FUNC(Intel_4003_setupFunc) +CHIPSET_SETUP_FUNC(Intel_3400_setupFunc) +CHIPSET_SETUP_FUNC(Intel_3B42_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1C46_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1C10_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1C4B_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1C49_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1D40_setupFunc) +CHIPSET_SETUP_FUNC(Intel_8D47_setupFunc) +CHIPSET_SETUP_FUNC(Intel_8D44_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1E10_setupFunc) +CHIPSET_SETUP_FUNC(Intel_8C4B_setupFunc) +CHIPSET_SETUP_FUNC(Intel_8CC4_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A145_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A2C5_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A242_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A2D2_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A2C9_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A301_setupFunc) +CHIPSET_SETUP_FUNC(Intel_0685_setupFunc) +CHIPSET_SETUP_FUNC(Intel_4381_setupFunc) +CHIPSET_SETUP_FUNC(Intel_7A82_setupFunc) +CHIPSET_SETUP_FUNC(Intel_7A04_setupFunc) +CHIPSET_SETUP_FUNC(Intel_7D06_setupFunc) +CHIPSET_SETUP_FUNC(SiS_656_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RS400_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RS480_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RD870_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RD890_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RX780_setupFunc) +CHIPSET_SETUP_FUNC(ATI_FX990_setupFunc) +CHIPSET_SETUP_FUNC(AMD_RS780_setupFunc) +CHIPSET_SETUP_FUNC(AMD_FX790_setupFunc) +CHIPSET_SETUP_FUNC(AMD_FX890_setupFunc) +CHIPSET_SETUP_FUNC(AMD_X370_setupFunc) +CHIPSET_SETUP_FUNC(VIA_VX900_setupFunc) +CHIPSET_SETUP_FUNC(APM_Storm_setupFunc) +CHIPSET_SETUP_FUNC(ARMV8_generic_setupFunc) +CHIPSET_SETUP_FUNC(Marvell_ThunderX2_setupFunc) +CHIPSET_SETUP_FUNC(QEMU_setupFunc) +CHIPSET_SETUP_FUNC(Ampere_eMag_setupFunc) +CHIPSET_SETUP_FUNC(Huawei_Kunpeng920_setupFunc) +CHIPSET_SETUP_FUNC(Mellanox_BlueField_setupFunc) +CHIPSET_SETUP_FUNC(Mellanox_BlueField3_setupFunc) +CHIPSET_SETUP_FUNC(Amazon_Gravitron2_setupFunc) +CHIPSET_SETUP_FUNC(Fujitsu_A64FX_setupFunc) +CHIPSET_SETUP_FUNC(Ampere_Altra_setupFunc) +CHIPSET_SETUP_FUNC(Arm_NeoverseN1_setupFunc) +CHIPSET_SETUP_FUNC(Ampere_AmpereOne_setupFunc) +CHIPSET_SETUP_FUNC(Qualcomm_Snapdragon_setupFunc) +CHIPSET_SETUP_FUNC(Nvidia_T210_setupFunc) +CHIPSET_SETUP_FUNC(Nvidia_T194_setupFunc) +CHIPSET_SETUP_FUNC(Nvidia_TH500_setupFunc) +CHIPSET_SETUP_FUNC(PLDA_XpressRichAXI_setupFunc) +CHIPSET_SETUP_FUNC(Riscv_generic_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A70D_setupFunc) +CHIPSET_SETUP_FUNC(AMD_14D8_setupFunc) +CHIPSET_SETUP_FUNC(Nvidia_TB500_setupFunc) + + +// Keep string length <=32 (including termination) to avoid string copy overflow +CSINFO chipsetInfo[] = +{ + // PCI Express chipset + + {PCI_VENDOR_ID_INTEL, 0x2580, CS_INTEL_2580, "Grantsdale", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2584, CS_INTEL_2584, "Alderwood", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2588, CS_INTEL_2588, "Intel2588", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2590, CS_INTEL_2590, "Alviso", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x25C0, CS_INTEL_25E0, "Greencreek", Intel_25E0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x25E0, CS_INTEL_25E0, "Greencreek", Intel_25E0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x29B0, CS_INTEL_29X0, "IntelQ35", Intel_29X0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x29C0, CS_INTEL_29X0, "BearlakeB", Intel_29X0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x29D0, CS_INTEL_29X0, "IntelQ33", Intel_29X0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x29E0, CS_INTEL_29E0, "BearlakeX", Intel_29E0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x359E, CS_INTEL_359E, "Tumwater", Intel_359E_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x4000, CS_INTEL_4000, "Stoakley", Intel_4000_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x4003, CS_INTEL_4003, "SkullTrail", Intel_4003_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3400, CS_INTEL_3400, "IntelX58", Intel_3400_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3403, CS_INTEL_3400, "IntelX58", Intel_3400_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3405, CS_INTEL_3400, "IntelX58", Intel_3400_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3406, CS_INTEL_3400, "Tylersburg", Intel_3400_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2770, CS_INTEL_2770, "Lakeport", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2774, CS_INTEL_2774, "Glenwood", Intel_27XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x277C, CS_INTEL_277C, "Glenwood", Intel_27XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2A40, CS_INTEL_2A40, "Montevina", Intel_2A40_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2E00, CS_INTEL_2E00, "Eaglelake", Intel_2E00_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2E10, CS_INTEL_2E00, "Eaglelake", Intel_2E00_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2E20, CS_INTEL_2E00, "Eaglelake", Intel_2E00_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2E30, CS_INTEL_2E00, "Eaglelake", Intel_2E00_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0044, CS_INTEL_0040, "Arrandale/Auburndale", Intel_0040_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0062, CS_INTEL_0040, "Arrandale/Auburndale", Intel_0040_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xD130, CS_INTEL_3B42, "Clarksfield", Intel_3B42_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xD132, CS_INTEL_3B42, "Clarksfield", Intel_3B42_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3B42, CS_INTEL_3B42, "P55/PM55/H57", Intel_3B42_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C46, CS_INTEL_1C46, "IntelP67-CougarPoint", Intel_1C46_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C4B, CS_INTEL_1C46, "HuronRiver-HM67", Intel_1C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C4F, CS_INTEL_1C46, "HuronRiver-QM67", Intel_1C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C49, CS_INTEL_1C49, "HuronRiver-HM65", Intel_1C49_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C44, CS_INTEL_1C46, "IntelZ68", Intel_1C46_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C10, CS_INTEL_1C10, "IntelP67", Intel_1C10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1D40, CS_INTEL_1D40, "IntelX79", Intel_1D40_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1D41, CS_INTEL_1D40, "IntelX79", Intel_1D40_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1E10, CS_INTEL_1E10, "IntelZ75", Intel_1E10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0150, CS_INTEL_1E10, "IntelZ77A-GD55", Intel_1E10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0151, CS_INTEL_1E10, "IntelZ77A-GD55", Intel_1E10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0100, CS_INTEL_1E10, "IntelZ77A-GD55", Intel_1E10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C4B, CS_INTEL_8C4B, "SharkBay-HM87", Intel_8C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C44, CS_INTEL_8C4B, "SharkBay-Z87", Intel_8C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C41, CS_INTEL_8C4B, "SharkBay-H8x/P8x", Intel_8C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C49, CS_INTEL_8C4B, "SharkBay-HM86", Intel_8C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C52, CS_INTEL_8C4B, "SharkBay-E3", Intel_8C4B_setupFunc}, // Does not support SLI + {PCI_VENDOR_ID_INTEL, 0x8CC4, CS_INTEL_8CC4, "IntelZ97", Intel_8CC4_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8CC3, CS_INTEL_8CC4, "IntelHM97", Intel_8CC4_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA145, CS_INTEL_A145, "IntelZ170", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA14E, CS_INTEL_A145, "IntelHM170", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA150, CS_INTEL_A145, "IntelHM170", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA149, CS_INTEL_A145, "SkyLake C236", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA14A, CS_INTEL_A145, "SkyLake C232", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA14D, CS_INTEL_A145, "SkyLake-H", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA244, CS_INTEL_A145, "SkyLake C620", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C8, CS_INTEL_A145, "SkyLake C620", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8D47, CS_INTEL_8D47, "IntelX99", Intel_8D47_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8D44, CS_INTEL_8D47, "IntelC612", Intel_8D44_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA2C5, CS_INTEL_A2C5, "IntelZ270", Intel_A2C5_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA154, CS_INTEL_A2C5, "IntelZ270", Intel_A2C5_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA152, CS_INTEL_A2C5, "IntelRX9S", Intel_A2C5_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA242, CS_INTEL_A242, "IntelC422", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA241, CS_INTEL_A242, "IntelC422", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA243, CS_INTEL_A242, "IntelC422", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA2D2, CS_INTEL_A2D2, "IntelX299", Intel_A2D2_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA2D3, CS_INTEL_A242, "IntelC422", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C1, CS_INTEL_A242, "IntelC621", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C2, CS_INTEL_A242, "IntelC622", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C3, CS_INTEL_A242, "IntelC624", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C4, CS_INTEL_A242, "IntelC625", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C5, CS_INTEL_A242, "IntelC626", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C6, CS_INTEL_A242, "IntelC627", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C7, CS_INTEL_A242, "IntelC628", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA2C9, CS_INTEL_A2C9, "IntelZ370", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA310, CS_INTEL_A2C9, "IntelZ370", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA30E, CS_INTEL_A2C9, "IntelZ370", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA305, CS_INTEL_A2C9, "IntelZ390", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA30D, CS_INTEL_A2C9, "IntelH370", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA301, CS_INTEL_A301, "Intel-CannonLake", Intel_A301_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0685, CS_INTEL_0685, "Intel-CometLake", Intel_0685_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1CB, CS_INTEL_C620, "Intel-IceLake", NULL}, + {PCI_VENDOR_ID_INTEL, 0x4381, CS_INTEL_4381, "Intel-RocketLake", Intel_4381_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x4385, CS_INTEL_4381, "Intel-RocketLake", Intel_4381_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7A82, CS_INTEL_7A82, "Intel-AlderLake", Intel_7A82_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7A84, CS_INTEL_7A82, "Intel-AlderLake", Intel_7A82_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1B81, CS_INTEL_1B81, "Intel-SapphireRapids", NULL}, + {PCI_VENDOR_ID_INTEL, 0x7A8A, CS_INTEL_1B81, "Intel-SapphireRapids", NULL}, + {PCI_VENDOR_ID_INTEL, 0x18DC, CS_INTEL_18DC, "Intel-IceLake", NULL}, + {PCI_VENDOR_ID_INTEL, 0x7A04, CS_INTEL_7A04, "Intel-RaptorLake", Intel_7A04_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x5795, CS_INTEL_5795, "Intel-GraniteRapids", NULL}, + {PCI_VENDOR_ID_INTEL, 0xA70D, CS_INTEL_B660, "Intel-B660", Intel_A70D_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D06, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D1A, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D1B, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D1C, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D2A, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D2D, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D2E, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D2F, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D35, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D67, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7D30, CS_INTEL_7D06, "Intel-Arrowlake", Intel_7D06_setupFunc}, + + {PCI_VENDOR_ID_NVIDIA, 0x0FAE, CS_NVIDIA_T210, "T210", Nvidia_T210_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x0FAF, CS_NVIDIA_T210, "T210", Nvidia_T210_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x10E5, CS_NVIDIA_T186, "T186", Nvidia_T210_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x10E6, CS_NVIDIA_T186, "T186", Nvidia_T210_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x1AD0, CS_NVIDIA_T194, "T194", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x1AD1, CS_NVIDIA_T194, "T194", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x1AD2, CS_NVIDIA_T194, "T194", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x229A, CS_NVIDIA_T234, "T234", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x229C, CS_NVIDIA_T234, "T234", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x229E, CS_NVIDIA_T234, "T234", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22C2, CS_NVIDIA_T23x, "T23x", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22C3, CS_NVIDIA_T23x, "T23x", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22B1, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22B2, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22B3, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22B4, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22B8, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22B9, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22D6, CS_NVIDIA_T264, "T264", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22D8, CS_NVIDIA_T264, "T264", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22DA, CS_NVIDIA_T264, "T264", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x2B00, CS_NVIDIA_T264, "T264", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x2F95, CS_NVIDIA_TB500, "TB500", Nvidia_TB500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x2F96, CS_NVIDIA_TB500, "TB500", Nvidia_TB500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x2F97, CS_NVIDIA_TB500, "TB500", Nvidia_TB500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x2F98, CS_NVIDIA_TB500, "TB500", Nvidia_TB500_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x2F99, CS_NVIDIA_TB500, "TB500", Nvidia_TB500_setupFunc}, + + {PCI_VENDOR_ID_SIS, 0x0649, CS_SIS_649, "649", SiS_656_setupFunc}, + {PCI_VENDOR_ID_SIS, 0x0656, CS_SIS_656, "656", SiS_656_setupFunc}, + + {PCI_VENDOR_ID_ATI, 0x5A31, CS_ATI_RS400, "RS400", ATI_RS400_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5A33, CS_ATI_RS400, "RS400", ATI_RS400_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5950, CS_ATI_RS480, "RS480", ATI_RS480_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5951, CS_ATI_RS480, "RS480", ATI_RS480_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5956, CS_ATI_FX790, "FX790" ,AMD_FX790_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5A11, CS_ATI_FX890, "FX890" ,AMD_FX890_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5a13, CS_ATI_RD850, "RD850" ,ATI_RD870_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5a12, CS_ATI_RD870, "RD870" ,ATI_RD870_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5a10, CS_ATI_RD890, "RD890" ,ATI_RD890_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5957, CS_ATI_RX780, "RX780" ,ATI_RX780_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5A14, CS_ATI_FX990, "FX990/X990/970",ATI_FX990_setupFunc}, + + {PCI_VENDOR_ID_AMD, 0x9601, CS_AMD_GX890, "GX890" ,AMD_FX890_setupFunc}, + {PCI_VENDOR_ID_AMD, 0x9600, CS_AMD_RS780, "RS780" ,AMD_RS780_setupFunc}, + {PCI_VENDOR_ID_AMD, 0x790e, CS_AMD_X370, "X370/X399/X470/ TRX40/X570/WRX80", AMD_X370_setupFunc }, + {PCI_VENDOR_ID_AMD, 0x14d8, CS_AMD_RPH, "AMD-Raphael", AMD_14D8_setupFunc}, + + {PCI_VENDOR_ID_VIA, 0x0308, CS_VIA_VT8369B, "VT8369B", NULL}, + {PCI_VENDOR_ID_VIA, 0x0410, CS_VIA_VX900, "VX900", VIA_VX900_setupFunc}, + {PCI_VENDOR_ID_APM, 0xe004, CS_APM_STORM, "X-Gene Storm", APM_Storm_setupFunc}, + {PCI_VENDOR_ID_IBM, 0x03DC, CS_IBM_VENICE, "Venice", NULL}, + {PCI_VENDOR_ID_MARVELL, 0xAF00, CS_MARVELL_THUNDERX2, "Marvell ThunderX2", Marvell_ThunderX2_setupFunc}, + {PCI_VENDOR_ID_REDHAT, 0x0008, CS_REDHAT_QEMU, "QEMU Redhat", QEMU_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE005, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE006, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE007, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE008, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE009, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00A, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00B, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00C, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_HUAWEI, 0xA120, CS_HUAWEI_KUNPENG920, "Huawei Kunpeng920", Huawei_Kunpeng920_setupFunc}, + {PCI_VENDOR_ID_MELLANOX, 0xA2D0, CS_MELLANOX_BLUEFIELD, "Mellanox BlueField", Mellanox_BlueField_setupFunc}, + {PCI_VENDOR_ID_MELLANOX, 0xA2D4, CS_MELLANOX_BLUEFIELD2, "Mellanox BlueField 2", NULL}, + {PCI_VENDOR_ID_MELLANOX, 0xA2D5, CS_MELLANOX_BLUEFIELD2, "Mellanox BlueField 2 Crypto disabled", NULL}, + {PCI_VENDOR_ID_MELLANOX, 0xA2DA, CS_MELLANOX_BLUEFIELD3, "Mellanox BlueField 3 Crypto enabled", Mellanox_BlueField3_setupFunc}, + {PCI_VENDOR_ID_MELLANOX, 0xA2DB, CS_MELLANOX_BLUEFIELD3, "Mellanox BlueField 3 Crypto disabled", Mellanox_BlueField3_setupFunc}, + {PCI_VENDOR_ID_AMAZON, 0x0200, CS_AMAZON_GRAVITRON2, "Amazon Gravitron2", Amazon_Gravitron2_setupFunc}, + {PCI_VENDOR_ID_FUJITSU, 0x1952, CS_FUJITSU_A64FX, "Fujitsu A64FX", Fujitsu_A64FX_setupFunc}, + {PCI_VENDOR_ID_CADENCE, 0xDC01, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xDC08, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xDC16, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xFC01, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xFC08, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xFC16, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_AMPERE, 0xE000, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00D, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00E, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE010, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE100, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE110, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_ARM, 0x0100, CS_ARM_NEOVERSEN1, "Arm Neoverse N1", Arm_NeoverseN1_setupFunc}, + {PCI_VENDOR_ID_HYGON, 0x790E, CS_HYGON_C86, "Hygon-C86-7151", NULL}, + {PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX, "Marvell Octeon CN96xx", ARMV8_generic_setupFunc}, + {PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX, "Marvell Octeon CN98xx", ARMV8_generic_setupFunc}, + {PCI_VENDOR_ID_QUALCOMM, 0x010E, CS_SNAPDRAGON_8CX, "Qualcomm Snapdragon 8cx Gen3", Qualcomm_Snapdragon_setupFunc}, + {PCI_VENDOR_ID_QUALCOMM, 0x0111, CS_SNAPDRAGON_8CX, "Qualcomm Snapdragon 8cx Gen4", Qualcomm_Snapdragon_setupFunc}, + {PCI_VENDOR_ID_SIFIVE, 0x0000, CS_SIFIVE_FU740_C000, "SiFive FU740-000", Riscv_generic_setupFunc}, + {PCI_VENDOR_ID_PLDA, 0x1111, CS_PLDA_XPRESSRICH_AXI_REF, "XpressRich-AXI Ref Design", PLDA_XpressRichAXI_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE200, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE201, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE202, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE203, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE204, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE205, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE206, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE207, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_PHYTIUM, 0x5C16, CS_PHYTIUM_S5000, "Phytium S5000", NULL}, + {PCI_VENDOR_ID_AMPERE, 0xE208, CS_AMPERE_AMPEREONE192, "Ampere AmpereOne-192", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE209, CS_AMPERE_AMPEREONE192, "Ampere AmpereOne-192", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE20A, CS_AMPERE_AMPEREONE192, "Ampere AmpereOne-192", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE20B, CS_AMPERE_AMPEREONE192, "Ampere AmpereOne-192", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE20C, CS_AMPERE_AMPEREONE192, "Ampere AmpereOne-192", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE20D, CS_AMPERE_AMPEREONE192, "Ampere AmpereOne-192", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE20E, CS_AMPERE_AMPEREONE192, "Ampere AmpereOne-192", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE20F, CS_AMPERE_AMPEREONE192, "Ampere AmpereOne-192", Ampere_AmpereOne_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22D1, CS_NVIDIA_T254, "T254", Nvidia_T194_setupFunc}, + +/////////////////////////////////////////////////////////////////////////////////////////////////// + + // last element must have chipset CS_UNKNOWN (zero) + {0, 0, CS_UNKNOWN, "Unknown", NULL} +}; + + +VENDORNAME vendorName[] = +{ + {PCI_VENDOR_ID_NVIDIA, "NVIDIA"}, + {PCI_VENDOR_ID_INTEL, "Intel"}, + {PCI_VENDOR_ID_VIA, "VIA"}, + {PCI_VENDOR_ID_RCC, "ServerWorks"}, + {PCI_VENDOR_ID_MICRON_1, "Micron"}, + {PCI_VENDOR_ID_MICRON_2, "Micron"}, + {PCI_VENDOR_ID_APPLE, "Apple"}, + {PCI_VENDOR_ID_SIS, "SiS"}, + {PCI_VENDOR_ID_ATI, "ATI"}, + {PCI_VENDOR_ID_TRANSMETA, "Transmeta"}, + {PCI_VENDOR_ID_HP, "HP"}, + {PCI_VENDOR_ID_AMD, "AMD"}, + {PCI_VENDOR_ID_ALI, "ALi"}, + {PCI_VENDOR_ID_APM, "AppliedMicro"}, + {PCI_VENDOR_ID_IBM, "IBM"}, + {PCI_VENDOR_ID_MARVELL, "MarvellThunderX2"}, + {PCI_VENDOR_ID_REDHAT, "QemuRedhat"}, + {PCI_VENDOR_ID_AMPERE, "AmpereComputing"}, + {PCI_VENDOR_ID_HUAWEI, "Huawei"}, + {PCI_VENDOR_ID_MELLANOX, "Mellanox"}, + {PCI_VENDOR_ID_AMAZON, "Amazon"}, + {PCI_VENDOR_ID_FUJITSU, "Fujitsu"}, + {PCI_VENDOR_ID_CADENCE, "Cadence"}, + {PCI_VENDOR_ID_ARM, "ARM"}, + {PCI_VENDOR_ID_ALIBABA, "Alibaba"}, + {PCI_VENDOR_ID_QUALCOMM, "Qualcomm"}, + {PCI_VENDOR_ID_SIFIVE, "SiFive"}, + {PCI_VENDOR_ID_PLDA, "PLDA"}, + {PCI_VENDOR_ID_PHYTIUM, "Phytium"}, + {0, "Unknown"} // Indicates end of the table +}; + + +// +// Allowlist all chipsets with which dGPU over PCIe is supported on ARM +// (both v7 and v8) platforms. Also include IGPU specific chipsets. +// +ARMCSALLOWLISTINFO armChipsetAllowListInfo[] = +{ + {PCI_VENDOR_ID_NVIDIA, 0x0FAE, CS_NVIDIA_T210}, // NVIDIA Tegra X1 RP0 + {PCI_VENDOR_ID_NVIDIA, 0x0FAF, CS_NVIDIA_T210}, // NVIDIA Tegra X1 RP1 + {PCI_VENDOR_ID_NVIDIA, 0x10E5, CS_NVIDIA_T186}, // NVIDIA Tegra P1 RP0 + {PCI_VENDOR_ID_NVIDIA, 0x10E6, CS_NVIDIA_T186}, // NVIDIA Tegra P1 RP1 + {PCI_VENDOR_ID_NVIDIA, 0x1AD0, CS_NVIDIA_T194}, // NVIDIA Tegra V1 RP0 + {PCI_VENDOR_ID_NVIDIA, 0x1AD1, CS_NVIDIA_T194}, // NVIDIA Tegra V1 RP1 + {PCI_VENDOR_ID_NVIDIA, 0x1AD2, CS_NVIDIA_T194}, // NVIDIA Tegra V1 RP2 + {PCI_VENDOR_ID_NVIDIA, 0x229A, CS_NVIDIA_T234}, // NVIDIA Tegra Orin RP0 + {PCI_VENDOR_ID_NVIDIA, 0x229C, CS_NVIDIA_T234}, // NVIDIA Tegra Orin RP1 + {PCI_VENDOR_ID_NVIDIA, 0x229E, CS_NVIDIA_T234}, // NVIDIA Tegra Orin RP2 + {PCI_VENDOR_ID_NVIDIA, 0x22C2, CS_NVIDIA_T23x}, // NVIDIA Tegra RP0 + {PCI_VENDOR_ID_NVIDIA, 0x22C3, CS_NVIDIA_T23x}, // NVIDIA Tegra RP1 + {PCI_VENDOR_ID_NVIDIA, 0x22B1, CS_NVIDIA_TH500}, // NVIDIA TH500 RP for GH100 GPU in GH180. + {PCI_VENDOR_ID_NVIDIA, 0x22B2, CS_NVIDIA_TH500}, // NVIDIA TH500 RP x16 + {PCI_VENDOR_ID_NVIDIA, 0x22B3, CS_NVIDIA_TH500}, // NVIDIA TH500 RP x4 + {PCI_VENDOR_ID_NVIDIA, 0x22B4, CS_NVIDIA_TH500}, // NVIDIA TH500 RP x1 + {PCI_VENDOR_ID_NVIDIA, 0x22B8, CS_NVIDIA_TH500}, // NVIDIA TH500 RP x8 + {PCI_VENDOR_ID_NVIDIA, 0x22B9, CS_NVIDIA_TH500}, // NVIDIA TH500 RP x2 + {PCI_VENDOR_ID_NVIDIA, 0x22D6, CS_NVIDIA_T264}, // NVIDIA Tegra RP x8 + {PCI_VENDOR_ID_NVIDIA, 0x22D8, CS_NVIDIA_T264}, // NVIDIA Tegra RP x4 + {PCI_VENDOR_ID_NVIDIA, 0x22DA, CS_NVIDIA_T264}, // NVIDIA Tegra RP x2 + {PCI_VENDOR_ID_NVIDIA, 0x22E6, CS_NVIDIA_T264}, // NVIDIA Tegra iGPU RP + {PCI_VENDOR_ID_NVIDIA, 0x2B00, CS_NVIDIA_T264}, // NVIDIA Tegra iGPU + {PCI_VENDOR_ID_NVIDIA, 0x2F95, CS_NVIDIA_TB500}, // NVIDIA TB500 RP x16 + {PCI_VENDOR_ID_NVIDIA, 0x2F96, CS_NVIDIA_TB500}, // NVIDIA TB500 RP x8 + {PCI_VENDOR_ID_NVIDIA, 0x2F97, CS_NVIDIA_TB500}, // NVIDIA TB500 RP x4 + {PCI_VENDOR_ID_NVIDIA, 0x2F98, CS_NVIDIA_TB500}, // NVIDIA TB500 RP x2 + {PCI_VENDOR_ID_NVIDIA, 0x2F99, CS_NVIDIA_TB500}, // NVIDIA TB500 RP x1 + + {PCI_VENDOR_ID_APM, 0xe004, CS_APM_STORM}, // Applied Micro X-Gene "Storm" + {PCI_VENDOR_ID_MARVELL, 0xAF00, CS_MARVELL_THUNDERX2}, // Marvell ThunderX2 + {PCI_VENDOR_ID_REDHAT, 0x0008, CS_REDHAT_QEMU}, // Redhat QEMU + {PCI_VENDOR_ID_AMPERE, 0xE005, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE006, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE007, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE008, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE009, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE00A, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE00B, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE00C, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_HUAWEI, 0xA120, CS_HUAWEI_KUNPENG920}, // Huawei Kunpeng 920 + {PCI_VENDOR_ID_MELLANOX, 0xA2D0, CS_MELLANOX_BLUEFIELD}, // Mellanox BlueField + {PCI_VENDOR_ID_MELLANOX, 0xA2D4, CS_MELLANOX_BLUEFIELD2},// Mellanox BlueField 2 + {PCI_VENDOR_ID_MELLANOX, 0xA2D5, CS_MELLANOX_BLUEFIELD2},// Mellanox BlueField 2 Crypto disabled + {PCI_VENDOR_ID_MELLANOX, 0xA2DB, CS_MELLANOX_BLUEFIELD3},// Mellanox BlueField 3 Crypto disabled + {PCI_VENDOR_ID_MELLANOX, 0xA2DA, CS_MELLANOX_BLUEFIELD3},// Mellanox BlueField 3 Crypto enabled + {PCI_VENDOR_ID_AMAZON, 0x0200, CS_AMAZON_GRAVITRON2}, // Amazon Gravitron2 + {PCI_VENDOR_ID_FUJITSU, 0x1952, CS_FUJITSU_A64FX}, // Fujitsu A64FX + {PCI_VENDOR_ID_CADENCE, 0xDC01, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xDC08, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xDC16, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xFC01, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xFC08, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xFC16, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_AMPERE, 0xE000, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE00D, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE00E, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE010, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE100, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE110, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_ARM, 0x0100, CS_ARM_NEOVERSEN1}, // Arm Neoverse N1 + {PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX}, // Marvell OCTEON CN96xx + {PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX}, // Marvell OCTEON CN98xx + {PCI_VENDOR_ID_ALIBABA, 0x8000, CS_ALIBABA_YITIAN}, // Alibaba Yitian + {PCI_VENDOR_ID_QUALCOMM, 0x010E, CS_SNAPDRAGON_8CX}, // Qualcomm Snapdragon 8cx Gen3 - 8280 dev platform + {PCI_VENDOR_ID_QUALCOMM, 0x0111, CS_SNAPDRAGON_8CX}, // Qualcomm Snapdragon 8cx Gen4 - 8380 dev(?) platform + {PCI_VENDOR_ID_AMPERE, 0xE200, CS_AMPERE_AMPEREONE160}, // Ampere AmpereOne-160 + {PCI_VENDOR_ID_AMPERE, 0xE201, CS_AMPERE_AMPEREONE160}, // Ampere AmpereOne-160 + {PCI_VENDOR_ID_AMPERE, 0xE202, CS_AMPERE_AMPEREONE160}, // Ampere AmpereOne-160 + {PCI_VENDOR_ID_AMPERE, 0xE203, CS_AMPERE_AMPEREONE160}, // Ampere AmpereOne-160 + {PCI_VENDOR_ID_AMPERE, 0xE204, CS_AMPERE_AMPEREONE160}, // Ampere AmpereOne-160 + {PCI_VENDOR_ID_AMPERE, 0xE205, CS_AMPERE_AMPEREONE160}, // Ampere AmpereOne-160 + {PCI_VENDOR_ID_AMPERE, 0xE206, CS_AMPERE_AMPEREONE160}, // Ampere AmpereOne-160 + {PCI_VENDOR_ID_AMPERE, 0xE207, CS_AMPERE_AMPEREONE160}, // Ampere AmpereOne-160 + {PCI_VENDOR_ID_PHYTIUM, 0x5C16, CS_PHYTIUM_S5000}, // Phytium S5000 + {PCI_VENDOR_ID_AMPERE, 0xE208, CS_AMPERE_AMPEREONE192}, // Ampere AmpereOne-192 + {PCI_VENDOR_ID_AMPERE, 0xE209, CS_AMPERE_AMPEREONE192}, // Ampere AmpereOne-192 + {PCI_VENDOR_ID_AMPERE, 0xE20A, CS_AMPERE_AMPEREONE192}, // Ampere AmpereOne-192 + {PCI_VENDOR_ID_AMPERE, 0xE20B, CS_AMPERE_AMPEREONE192}, // Ampere AmpereOne-192 + {PCI_VENDOR_ID_AMPERE, 0xE20C, CS_AMPERE_AMPEREONE192}, // Ampere AmpereOne-192 + {PCI_VENDOR_ID_AMPERE, 0xE20D, CS_AMPERE_AMPEREONE192}, // Ampere AmpereOne-192 + {PCI_VENDOR_ID_AMPERE, 0xE20E, CS_AMPERE_AMPEREONE192}, // Ampere AmpereOne-192 + {PCI_VENDOR_ID_AMPERE, 0xE20F, CS_AMPERE_AMPEREONE192}, // Ampere AmpereOne-192 + {PCI_VENDOR_ID_NVIDIA, 0x22D1, CS_NVIDIA_T254}, + + // last element must have chipset CS_UNKNOWN (zero) + {0, 0, CS_UNKNOWN} +}; + +#endif /* NVCST_H */ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/nvdm_format.h b/userspace/sdks/nv570/nvalloc/common/inc/nvdm_format.h new file mode 100644 index 0000000..db3baa0 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/nvdm_format.h @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVDM_FORMAT_H +#define NVDM_FORMAT_H + +/** + * NVDM (NVIDIA Data Model) overlaid on MCTP (Management Component Transport + * Protocol) format definitions. + * + * This header consolidates NVDM type definitions from multiple RM subsystems + * including FSP, SEC2, GSP, and others. NVDM is a vendor-defined message layer + * overlaid on MCTP that provides a simple way for different subsystems to + * communicate. + */ + +// +// List of NVDM Message Types +// +// Values are allocated from a shared namespace across all subsystems. +// Add new types here and update the NVDM Message Type allocation page. +// +#define NVDM_TYPE_RESET 0x4 +#define NVDM_TYPE_HULK 0x11 +#define NVDM_TYPE_FIRMWARE_UPDATE 0x12 +#define NVDM_TYPE_PRC 0x13 +#define NVDM_TYPE_COT 0x14 +#define NVDM_TYPE_FSP_RESPONSE 0x15 +#define NVDM_TYPE_SEC2_RESPONSE NVDM_TYPE_FSP_RESPONSE +#define NVDM_TYPE_CAPS_QUERY 0x16 +#define NVDM_TYPE_INFOROM 0x17 +#define NVDM_TYPE_SMBPBI 0x18 +#define NVDM_TYPE_ROMREAD 0x1A +#define NVDM_TYPE_UEFI_RM 0x1C +#define NVDM_TYPE_UEFI_XTL_DEBUG_INTR 0x1D +#define NVDM_TYPE_TNVL 0x1F +#define NVDM_TYPE_CLOCK_BOOST 0x20 +#define NVDM_TYPE_FSP_GSP_COMM 0x21 +#define NVDM_TYPE_SEC2_GSP_COMM NVDM_TYPE_FSP_GSP_COMM + +#define NVDM_TYPE_RM_RPC 0x25 // GSP-RM RPC messages + +#endif // NVDM_FORMAT_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/nvpcie.h b/userspace/sdks/nv570/nvalloc/common/inc/nvpcie.h new file mode 100644 index 0000000..c4ee983 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/nvpcie.h @@ -0,0 +1,545 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2000-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVPCIE_H +#define NVPCIE_H + + +#include "nvtypes.h" + +/**************** Resource Manager Defines and Structures ******************\ +* * +* Private PCI Express related defines and structures. * +* * +\***************************************************************************/ + +typedef struct NBADDR +{ + NvU32 domain; + NvU8 bus; + NvU8 device; + NvU8 func; + NvU8 valid; + void *handle; +} NBADDR; + +typedef struct PORTDATA +{ + NBADDR addr; + void *vAddr; // virtual address of the port, if it has been mapped . Not used starting with Win10 BuildXXXXX + NvU32 PCIECapPtr; // offset of the PCIE capptr in the NB + // Capability register set in enhanced configuration space + // + NvU32 PCIEErrorCapPtr; // offset of the Advanced Error Reporting Capability register set + NvU32 PCIEVCCapPtr; // offset of the Virtual Channel (VC) Capability register set + NvU32 PCIEL1SsCapPtr; // Offset of the L1 Substates Capabilities + NvU32 PCIEAcsCapPtr; // Offset of the ACS redirect Capabilities + NvU16 DeviceID, VendorID; // device and vendor ID for port +} PORTDATA; + +#define PCI_VENDOR_ID 0x00 +#ifndef PCI_DEVICE_ID +#define PCI_DEVICE_ID 0x02 +#endif +#define PCI_BASE_ADDRESS_1 0x14 /* Aperture Base */ +#define PCI_BASE_ADDRESS_2 0x18 /* Aperture Base */ +#define PCI_CAPABILITY_LIST 0x34 +#define PCI_DEVICE_SPECIFIC 0x40 + +#define NV_PCI_ID 0x0 +#define NV_PCI_ID_VENDOR 15:0 +#define NV_PCI_ID_VENDOR_NVIDIA 0x10DE +#define NV_PCI_ID_DEVICE 31:16 + +#define PCI_MAX_SLOTS 255 +#define PCI_MAX_LANE_WIDTH 32 + +#define PCI_MAX_FUNCTION 8 +#define PCI_INVALID_VENDORID 0xFFFF +#define PCI_INVALID_DEVICEID 0xFFFF +#define PCI_INVALID_SUBVENDORID 0xFFFF +#define PCI_INVALID_SUBDEVICEID 0xFFFF + +#define PCI_IS_VENDORID_VALID(id) (((id) != 0x0000) && ((id) != 0xFFFF)) + +#define PCI_CLASS_BRIDGE_DEV 0x06 +#define PCI_SUBCLASS_BR_HOST 0x00 +#define PCI_MULTIFUNCTION 0x80 + +// From PCI Local Bus Specification, Revision 3.0 +// and PCI Express Base Specification 6.0 +// numbers in comments to right of values indicate +// the referenced section in the PCIE spec + + +#define CAP_ID_MASK 0xFF + +#define CAP_ID_NULL 0x00 // 7.9.28.1 +#define CAP_ID_PMI 0x01 // 7.5.2.1 +#define CAP_ID_AGP 0x02 +#define CAP_ID_VPD 0x03 // 7.9.18.1 +#define CAP_ID_SLOT_ID 0x04 +#define CAP_ID_MSI 0x05 // 7.7.1.1 +#define CAP_ID_HOT_SWAP 0x06 +#define CAP_ID_PCI_X 0x07 +#define CAP_ID_HYPER_TRANSPORT 0x08 +#define CAP_ID_VENDOR_SPECIFIC 0x09 +#define CAP_ID_DEBUG_PORT 0x0A +#define CAP_ID_CRC 0x0B +#define CAP_ID_HOT_PLUG 0x0C +#define CAP_ID_SUBSYSTEM_ID 0x0D // 7.9.23.1 +#define CAP_ID_AGP8X 0x0E +#define CAP_ID_SECURE 0x0F +#define CAP_ID_PCI_EXPRESS 0x10 // 7.5.3.1 +#define CAP_ID_MSI_X 0x11 // 7.7.2.1 +#define CAP_ID_ENHANCED_ALLOCATION 0x14 // 7.8.5.1 +#define CAP_ID_FPB 0x15 // 7.8.11.1 +#define CAP_ID_AF 0x13 // 7.9.21.1 + +// +// sizes for static PCI capabilities structure +// +#define CAP_NULL_SIZE 0x04 // 7.9.28 +#define CAP_PMI_SIZE 0x08 // 7.5.2 +#define CAP_VPD_SIZE 0x08 // 7.9.18 +#define CAP_PCI_X_SIZE 0x3C // 7.5.3 +#define CAP_PCI_EXPRESS_SIZE 0x3C // 7.5.3 +#define CAP_FPB_SIZE 0x08 // 7.8.11.1 +#define CAP_AF_SIZE 0x08 // 7.9.21 +#define CAP_SUBSYSTEM_ID_SIZE 0x08 // 7.9.23 + +// MSI capability size related fields +#define PCI_MSI_CONTROL 0x02 // 7.7.1.2 +#define PCI_MSI_CONTROL_64BIT_CAPABLE 7:7 +#define PCI_MSI_CONTROL_64BIT_CAPABLE_FALSE 0 +#define PCI_MSI_CONTROL_64BIT_CAPABLE_TRUE 1 +#define PCI_MSI_CONTROL_PVM_CAPABLE 8:8 +#define PCI_MSI_CONTROL_PVM_CAPABLE_FALSE 0 +#define PCI_MSI_CONTROL_PVM_CAPABLE_TRUE 1 +#define PCI_MSI_BASE_SIZE 0x0C // 7.7.1 +#define PCI_MSI_64BIT_ADDR_CAPABLE_ADJ_SIZE 0x04 // 7.7.1 +#define PCI_MSI_PVM_CAPABLE_ADJ_SIZE 0x08 // 7.7.1 + +// MSI-X capability size related fields +#define PCI_MSI_X_BASE_SIZE 0x0C // 7.7.2 +#define PCI_MSI_X_CONTROL 0x02 // 7.7.2.2 +#define PCI_MSI_X_CONTROL_TABLE_SIZE 10:0 +#define PCI_MSI_X_TABLE_OFFSET_BIR 0x04 // 7.7.2.3 +#define PCI_MSI_X_TABLE_OFFSET 31:3 +#define PCI_MSI_X_PBR_OFFSET_BIR 0x08 // 7.7.2.4 +#define PCI_MSI_X_PBR_OFFSET 31:3 +#define PCI_MSI_X_TABLE_ENTRY_SIZE 0x10 // 7.7.2 +#define PCI_MSI_X_PBR_ENTRY_SIZE 0x10 // 7.7.2 + +// Enhanced Allocation Capability size related fields +#define PCI_ENHANCED_ALLOCATION_FIRST_DW 0x00 // 7.8.5.1 +#define PCI_ENHANCED_ALLOCATION_FIRST_DW_NUM_ENTRIES 21:16 +#define PCI_ENHANCED_ALLOCATION_TYPE_0_BASE_SIZE 0x04 // 7.8.5.1 +#define PCI_ENHANCED_ALLOCATION_TYPE_1_BASE_SIZE 0x08 // 7.8.5.2 +#define PCI_ENHANCED_ALLOCATION_ENTRY_HEADER 0x00 // 7.8.5.3 +#define PCI_ENHANCED_ALLOCATION_ENTRY_HEADER_ENTRY_SIZE 2:0 + +// PCI Vendor Specific Capability size related fields +#define PCI_VENDOR_SPECIFIC_CAP_HEADER 0x00 // 7.9.4 +#define PCI_VENDOR_SPECIFIC_CAP_HEADER_LENGTH 23:16 + +// +// Extended config space size is 4096 bytes. +// +#define PCI_EXTENDED_CONFIG_SPACE_LENGTH 4096 + +// +// From PCI Local Bus Specification, Revision 3.0 +// HEADER TYPE0 Definitions - Byte offsets +// +#define PCI_HEADER_TYPE0_VENDOR_ID 0x00 +#define PCI_HEADER_TYPE0_DEVICE_ID 0x02 +#define PCI_HEADER_TYPE0_COMMAND 0x04 +#define PCI_HEADER_TYPE0_STATUS 0x06 +#define PCI_HEADER_TYPE0_REVISION_ID 0x08 +#define PCI_HEADER_TYPE0_PROGIF 0x09 +#define PCI_HEADER_TYPE0_SUBCLASS 0x0A +#define PCI_HEADER_TYPE0_BASECLASS 0x0B +#define PCI_HEADER_TYPE0_CACHE_LINE_SIZE 0x0C +#define PCI_HEADER_TYPE0_LATENCY_TIMER 0x0D +#define PCI_HEADER_TYPE0_HEADER_TYPE 0x0E +#define PCI_HEADER_TYPE0_HEADER_TYPE_0 0 +#define PCI_HEADER_TYPE0_HEADER_TYPE_1 1 +#define PCI_HEADER_TYPE0_BIST 0x0F +#define PCI_HEADER_TYPE0_BAR0 0x10 +#define PCI_HEADER_TYPE0_BAR1 0x14 +#define PCI_HEADER_TYPE0_BAR2 0x18 +#define PCI_HEADER_TYPE0_BAR3 0x1C +#define PCI_HEADER_TYPE0_BAR4 0x20 +#define PCI_HEADER_TYPE0_BAR5 0x24 +#define PCI_HEADER_TYPE0_CBCIS_PTR 0x28 +#define PCI_HEADER_TYPE0_SUBSYS_VEN_ID 0x2C +#define PCI_HEADER_TYPE0_SUBSYS_ID 0x2E +#define PCI_HEADER_TYPE0_ROMBAR 0x30 +#define PCI_HEADER_TYPE0_CAP_PTR 0x34 +#define PCI_HEADER_TYPE0_INT_LINE 0x3C +#define PCI_HEADER_TYPE0_INT_PIN 0x3D +#define PCI_HEADER_TYPE0_MIN_GNT 0x3E +#define PCI_HEADER_TYPE0_MAX_LAT 0x3F + +// +// From PCI Express Base Specification Revision 2.0 +// HEADER TYPE1 Definitions +#define PCI_HEADER_TYPE1_BRIDGE_CONTROL 0x3E +#define PCI_HEADER_TYPE1_BRIDGE_CONTROL_VGA_EN 0x08 + +#define PCIE_LINK_CAPABILITIES_2 0x000000A4 /* R--4R */ +#define PCIE_LINK_CAPABILITIES_2_RSVD 0:0 /* C--VF */ +#define PCIE_LINK_CAPABILITIES_2_RSVD_INIT 0x00000000 /* C---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED 7:1 /* R-EVF */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1_GEN2_GEN3_GEN4_GEN5 0x0000001F /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_HIDDEN 0x00000000 /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1 0x00000001 /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1_GEN2 0x00000003 /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1_GEN2_GEN3 0x00000007 /* R-E-V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1_GEN2_GEN3_GEN4 0x0000000F /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_CROSS_LINK_SUPPORT 8:8 /* C--VF */ +#define PCIE_LINK_CAPABILITIES_2_CROSS_LINK_SUPPORT_DISABLED 0x00000000 /* C---V */ +#define PCIE_LINK_CAPABILITIES_2_RET_PRESENCE_DET_SUPP 23:23 /* R-EVF */ +#define PCIE_LINK_CAPABILITIES_2_RET_PRESENCE_DET_SUPP_UNSET 0x00000000 /* R-E-V */ +#define PCIE_LINK_CAPABILITIES_2_2RET_PRESENCE_DET_SUPP 24:24 /* R-EVF */ +#define PCIE_LINK_CAPABILITIES_2_2RET_PRESENCE_DET_SUPP_UNSET 0x00000000 /* R-E-V */ +#define PCIE_LINK_CAPABILITIES_2_RSVD1 31:25 /* C--VF */ +#define PCIE_LINK_CAPABILITIES_2_RSVD1_INIT 0x00000000 /* C---V */ + +// +// PCI Express Virtual Peer-to-Peer Approval Definition +// +// These offsets are unused in hardware on existing chips and are reserved on +// future chips. Software has defined a virtual PCI capability that may be +// emulated by hypervisors at these offsets, and this capability is not tied +// to any specific hardware. +// +// +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0 0x000000C8 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_ID 7:0 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_NEXT 15:8 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_LENGTH 23:16 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_SIG_LO 31:24 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1 0x000000CC +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_SIG_HI 15:0 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_VERSION 18:16 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_PEER_CLIQUE_ID 22:19 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_RSVD 31:23 + +#define NV_PCI_VIRTUAL_P2P_APPROVAL_SIGNATURE 0x00503250 + +#define NV_PCI_VIRTUAL_CONFIG_BITS_CAP_0 0x000000D4 +#define NV_PCI_VIRTUAL_CONFIG_BITS_CAP_0_ID 7:0 +#define NV_PCI_VIRTUAL_CONFIG_BITS_CAP_0_NEXT 15:8 +#define NV_PCI_VIRTUAL_CONFIG_BITS_CAP_0_LENGTH 23:16 +#define NV_PCI_VIRTUAL_CONFIG_BITS_CAP_0_SIG_LO 31:24 +#define NV_PCI_VIRTUAL_CONFIG_BITS_CAP_1 0x000000D8 +#define NV_PCI_VIRTUAL_CONFIG_BITS_CAP_1_SIG_HI 15:0 +#define NV_PCI_VIRTUAL_CONFIG_BITS_CAP_1_VALUE 31:16 + +#define NV_PCI_VIRTUAL_CONFIG_BITS_SIGNATURE 0x00535442 //"BTS" + +// Allocation of _VALUE bits +#define NV_PCI_VIRTUAL_CONFIG_BITS_PCI_EGRESS_POISON_ENABLE 0:0 + +// Chipset-specific definitions. +// Intel SantaRosa definitions +#define INTEL_2A00_CONFIG_SPACE_BASE 0x60 + +// Intel Montevina definitions +#define INTEL_2A40_CONFIG_SPACE_BASE 0x60 +#define INTEL_2A40_ASLM_CAPABLE_REVID 0x05 + +// Intel EagleLake definitions +#define INTEL_2E00_CONFIG_SPACE_BASE 0x60 + +// Intel Bearlake definitions +#define INTEL_29XX_CONFIG_SPACE_BASE 0x60 + +// Intel BroadWater definitions +#define INTEL_29A0_CONFIG_SPACE_BASE 0x60 + +// Intel Grantsdale definitions +#define INTEL_25XX_CONFIG_SPACE_BASE 0x48 + +// Intel Tumwater definitions +#define INTEL_359E_CONFIG_SPACE_BASE 0xCC + +// Intel Greencreek definitions +#define INTEL_25E0_CONFIG_SPACE_BASE_ADDRESS 0xE0000000 + +// Intel Stoakley definitions +#define INTEL_4000_CONFIG_SPACE_BASE_ADDRESS 0xE0000000 + +// Intel SkullTrail definitions +#define INTEL_4003_CONFIG_SPACE_BASE_ADDRESS_F 0xF0000000 +#define INTEL_4003_CONFIG_SPACE_BASE_ADDRESS_E 0xE0000000 +#define INTEL_4003_CONFIG_SPACE_BASE_ADDRESS INTEL_4003_CONFIG_SPACE_BASE_ADDRESS_F +#define INTEL_4003_CONFIG_SPACE_BASE 0x64 + +// SiS 656 +#define SIS_656_CONFIG_SPACE_BASE 0xE0 +#define SIS_656_CONFIG_SPACE_BASE_ADDRESS 3:0 // mapped to 31:28 + +// PCI/PCIE definitions +#define PCI_MAX_CAPS 20 // max caps to parse +#define PCI_MAX_DEVICES 32 // max devices on bus +#define PCI_MAX_FUNCTIONS 8 // max functions for a device +#define PCI_CAP_HEADER_ID 7:0 // PCI cap header id +#define PCI_CAP_HEADER_NEXT 15:8 // PCI cap header next +#define PCI_COMMON_CLASS_SUBCLASS 0x0a // PCI class/subclass (word) +#define PCI_COMMON_CLASS_SUBBASECLASS_HOST 0x0600 // Host bridge (connect PCI to CPU) [00] + Bridge Device [06] +#define PCI_COMMON_CLASS_SUBBASECLASS_P2P 0x0604 // PCI-to-PCI bridge (connects PCI buses) [04] + Bridge Device [06] +#define PCI_COMMON_CLASS_SUBBASECLASS_VGA 0x0300 +#define PCI_COMMON_CLASS_SUBBASECLASS_3DCTRL 0x0302 +#define PCI_COMMON_CAP_PTR 0x34 // PCI common cap ptr (byte) +#define PCI_TYPE_1_SECONDARY_BUS_NUMBER 0x19 // PCI type 1 sec bus (byte) +#define PCI_TYPE_1_SUBORDINATE_BUS_NUMBER 0x1a // PCI type 1 sub bus (byte) +#define PCIE_CAP_HEADER_ID 15:0 // PCIE cap header id +#define PCIE_CAP_HEADER_ID_INVALID 0xffff +#define PCIE_CAP_HEADER_NEXT 31:20 // PCIE cap header next +#define PCIE_BUS_SHIFT 20 // PCIE cfg space bus shift +#define PCIE_DEVICE_SHIFT 15 // PCIE cfg space dev shift +#define PCIE_FUNC_SHIFT 12 // PCIE cfg space func shift +#define PCIE_CAP_VERSION 19:16 // PCIE cap version +#define PCIE_CAP_VERSION_2P0 2 // PCIE 2.0 version +#define PCIE_LINK_CNTRL_STATUS_2_OFFSET 0x30 // PCIE Link Control/Status 2 offset +#define PCIE_LINK_STATUS_2 31:16 // PCIE Link Status 2 Register +#define PCIE_LINK_STATUS_2_DE_EMPHASIS 0:0 // PCIE De-Emphasis Level +#define PCI_COMMON_SUBSYSTEM_VENDOR_ID 0x2c // PCI subsystem Vendor Id +#define PCI_COMMON_SUBSYSTEM_ID 0x2e // PCI subsystem Id +#define PCIE_CAPABILITY_BASE 0x100 // 1st PCIE capability. + +// PCI Express Capability ID in the enhanced configuration space +#define PCIE_CAP_ID_NULL 0x00 // 7.9.28.1 +#define PCIE_CAP_ID_ERROR 0x01 // 7.8.4.1 +#define PCIE_CAP_ID_VC 0x02 // 7.9.1.1 +#define PCIE_CAP_ID_SERIAL 0x03 // 7.9.3.1 +#define PCIE_CAP_ID_POWER 0x04 // 7.8.1.1 +#define PCIE_CAP_ID_ROOT_COMPLEX 0x05 // 7.9.8.1 +#define PCIE_CAP_ID_ROOT_COMPLEX_INTERNAL_LINK_CTRL 0x06 // 7.9.9.1 +#define PCIE_CAP_ID_ROOT_COMPLEX_EVENT_COLLECTOR_ENDPOINT 0x07 // 7.9.10.1 +#define PCIE_CAP_ID_PCIE_CAP_ID_MFVC 0x08 // 7.9.2.1 +#define PCIE_CAP_ID_RCRB 0x0A // 7.9.7.1 +#define PCIE_CAP_ID_ACS 0x0D // 7.7.11.1 +#define PCIE_CAP_ID_ARI 0x0E // 7.8.8.1 +#define PCIE_CAP_ID_MULTICAST 0x12 // 7.9.11.1 +#define PCIE_CAP_ID_RESIZABLE_BAR 0x15 // 7.8.6.1 +#define PCIE_CAP_ID_DYNAMIC_POWER_ALLOCATION 0x16 // 7.9.12.1 +#define PCIE_CAP_ID_TPH 0x17 // 7.9.13.1 +#define PCIE_CAP_ID_LATENCY_TOLERANCE 0x18 // 7.8.2.1 +#define PCIE_CAP_ID_SECONDARY_PCIE_CAPABILITY 0x19 // 7.7.3.1 +#define PCIE_CAP_ID_PASID 0x1B // 7.8.9.1 +#define PCIE_CAP_ID_DPC 0x1D // 7.9.14.1 +#define PCIE_CAP_ID_L1_PM_SUBSTATES 0x1E // 7.8.3.1 +#define PCIE_CAP_ID_PTM 0x1F // 7.9.15.1 +#define PCIE_CAP_ID_FRS_QUEUING 0x21 // 7.8.10.1 +#define PCIE_CAP_ID_READINESS_TIME_REPORTING 0x22 // 7.9.16.1 +#define PCIE_CAP_ID_VENDOR_SPECIFIC 0x23 // 7.9.6.1 +#define PCIE_CAP_ID_VF_RESIZABLE_BAR 0x24 // 7.8.7.1 +#define PCIE_CAP_ID_DATA_LINK 0x25 // 7.7.4.1 +#define PCIE_CAP_ID_PHYSLAYER_16_GT 0x26 // 7.7.5.1 +#define PCIE_CAP_ID_LANE_MARGINING_AT_RECEVER 0x27 // 7.7.10.1 +#define PCIE_CAP_ID_HIERARCHY_ID 0x28 // 7.9.17.1 +#define PCIE_CAP_ID_NPEM 0x29 // 7.9.19.1 +#define PCIE_CAP_ID_PHYSLAYER_32_GT 0x2A // 7.7.6.1 +#define PCIE_CAP_ID_ALTERNATE_PROTOCOL 0x2B // 7.9.20.1 +#define PCIE_CAP_ID_SFI 0x2C // 7.9.22.1 +#define PCIE_CAP_ID_SHADOW_FUNCTIONS 0x2D // 7.9.25.1 +#define PCIE_CAP_ID_DATA_OBJECT_EXCHANGE 0x2E // 7.9.24.1 +#define PCIE_CAP_ID_DEVICE_3 0x2F // 7.7.9.1 +#define PCIE_CAP_ID_IDE 0x30 // 7.9.26.1 +#define PCIE_CAP_ID_PHYSLAYER_64_GT 0x31 // 7.7.7.1 +#define PCIE_CAP_ID_FLT_LOGGING 0x32 // 7.7.8.1 +#define PCIE_CAP_ID_FLIT_PERF_MEASURMENT 0x33 // 7.8.12.1 +#define PCIE_CAP_ID_FLIT_ERROR_INJECTION 0x34 // 7.8.13.1 + +// static sized structure sizes +#define PCIE_CAP_HEADER_SIZE 0x04 // 7.6.3 +#define PCIE_CAP_NULL_SIZE 0x04 // 7.9.28 +#define PCIE_CAP_ERROR_SIZE 0x48 // 7.8.4 +#define PCIE_CAP_POWER_SIZE 0x10 // 7.8.1 +#define PCIE_CAP_ROOT_COMPLEX_INTERNAL_LINK_CTRL_SIZE 0x0C // 7.9.9 +#define PCIE_CAP_ROOT_COMPLEX_EVENT_COLLECTOR_ENDPOINT_SIZE 0x0C // 7.9.10 +#define PCIE_CAP_SECONDARY_PCIE_SIZE 0x4C // 7.7.3.1 +#define PCIE_CAP_DATA_LINK_SIZE 0x0C // 7.7.4 +#define PCIE_CAP_PHYSLAYER_16_GT_SIZE 0x40 // 7.7.5 +#define PCIE_CAP_PHYSLAYER_32_GT_SIZE 0x40 // 7.7.6 +#define PCIE_CAP_PHYSLAYER_64_GT_SIZE 0x20 // 7.7.7 +#define PCIE_CAP_FLT_LOGGING_SIZE 0x3C // 7.7.8 +#define PCIE_CAP_DEVICE_3_SIZE 0x10 // 7.7.9 +#define PCIE_CAP_LANE_MARGINING_AT_RECEVER_SIZE 0x88 // 7.7.10 +#define PCIE_CAP_ACS_SIZE 0x10 // 7.7.11 +#define PCIE_CAP_LATENCY_TOLERANCE_SIZE 0x08 // 7.8.2 +#define PCIE_CAP_L1_PM_SUBSTATE_SIZE 0x14 // 7.8.3 +#define PCIE_CAP_RESIZABLE_BAR_SIZE 0x34 // 7.8.6 +#define PCIE_CAP_VF_RESIZABLE_BAR_SIZE 0x34 // 7.8.7 +#define PCIE_CAP_ARI_SIZE 0x08 // 7.8.8 +#define PCIE_CAP_PASID_SIZE 0x08 // 7.8.9 +#define PCIE_CAP_FRS_QUEUING_SIZE 0x10 // 7.8.10 +#define PCIE_CAP_FPB_SIZE 0x24 // 7.8.11 +#define PCIE_CAP_FLIT_PERF_MEASURMENT_SIZE 0x24 // 7.8.12 +#define PCIE_CAP_FLIT_ERROR_INJECTION_SIZE 0x24 // 7.8.13 +#define PCIE_CAP_DEV_SERIAL_SIZE 0x0C // 7.9.3 +#define PCIE_CAP_RCRB_SIZE 0x14 // 7.9.7 +#define PCIE_CAP_MULTICAST_SIZE 0x30 // 7.9.11 +#define PCIE_CAP_DYNAMIC_POWER_ALLOCATION_SIZE 0x30 // 7.9.12 +#define PCIE_CAP_DPC_SIZE 0x5C // 7.9.14 +#define PCIE_CAP_PTM_SIZE 0x0C // 7.9.15 +#define PCIE_CAP_READINESS_TIME_REPORTING_SIZE 0x0C // 7.9.16 +#define PCIE_CAP_HIERARCHY_ID_SIZE 0x0C // 7.9.17 +#define PCIE_CAP_NPEM_SIZE 0x10 // 7.9.19 +#define PCIE_CAP_ALTERNATE_PROTOCOL_SIZE 0x14 // 7.9.20 +#define PCIE_CAP_SFI_SIZE 0x14 // 7.9.22 +#define PCIE_CAP_DATA_OBJECT_EXCHANGE_SIZE 0x18 // 7.9.24 +#define PCIE_CAP_SHADOW_FUNCTIONS_SIZE 0x1C // 7.9.25 +#define PCIE_CAP_IDE_SIZE 0x34 // 7.9.26 + +// Virtual Channel Capability size related fields +#define PCIE_VC_REGISTER_1 0x04 // 7.9.1.2 +#define PCIE_VC_REGISTER_1_EXTENDED_VC_COUNT 2:0 +#define PCIE_VIRTUAL_CHANNELS_BASE_SIZE 0x18 // 7.9.1 +#define PCIE_VIRTUAL_CHANNELS_EXTENDED_VC_ENTRY_SIZE 0x10 // 7.9.1 + +// Multi Function Virtual Channel Capability size related fields +#define PCIE_MFVC_REGISTER_1 0x04 // 7.9.2.2 +#define PCIE_MFVC_REGISTER_1_EXTENDED_VC_COUNT 2:0 +#define PCIE_PCIE_CAP_ID_MFVC_BASE_SIZE 0x18 // 7.9.2 +#define PCIE_PCIE_CAP_ID_MFVC_EXTENDED_VC_ENTRY_SIZE 0x10 // 7.9.2 + +// Vendor Specific Capability size related fields +#define PCIE_VENDOR_SPECIFIC_HEADER_1 0x04 // 7.9.6.2 +#define PCIE_VENDOR_SPECIFIC_HEADER_1_LENGTH 31:20 + +// Root Complex Capability size related fields +#define PCIE_ROOT_COMPLEX_SELF_DESC_REGISTER 0x04 // 7.9.8.2 +#define PCIE_ROOT_COMPLEX_SELF_DESC_REGISTER_NUM_LINK_ENTRIES 15:8 +#define PCIE_ROOT_COMPLEX_BASE_SIZE 0x0C // 7.9.8 +#define PCIE_ROOT_COMPLEX_LINK_ENTRY_SIZE 0x10 // 7.9.8.3 + +// TPH capability size related fields +#define PCIE_TPH_REQUESTOR_REGISTER 0x04 // 7.9.13.2 +#define PCIE_TPH_REQUESTOR_REGISTER_ST_TABLE_SIZE 26:16 +#define PCIE_TPH_BASE_SIZE 0x0C // 7.9.13 +#define PCIE_TPH_ST_ENTRY_SIZE 0x02 // 7.9.13.4 + +// Intel CPU family. +#define INTEL_CPU_FAMILY_06 0x06 +#define INTEL_CPU_FAMILY_15 0x0f +#define INTEL_CPU_FAMILY_16 0x10 +#define INTEL_CPU_FAMILY_21 0x15 + +// Intel CPU Model. Calculated as Model += (extModel << 4). +#define INTEL_CPU_MODEL_2A 0x2a +#define INTEL_CPU_MODEL_2D 0x2d +#define INTEL_CPU_MODEL_3A 0x3a +#define INTEL_CPU_MODEL_3F 0x3f + +// Symbolic defines for each possible virtual channel +enum +{ + RM_PCIE_VIRTUAL_CHANNEL_0 = 0, + RM_PCIE_VIRTUAL_CHANNEL_1, + RM_PCIE_VIRTUAL_CHANNEL_2, + RM_PCIE_VIRTUAL_CHANNEL_3, + RM_PCIE_VIRTUAL_CHANNEL_4, + RM_PCIE_VIRTUAL_CHANNEL_5, + RM_PCIE_VIRTUAL_CHANNEL_6, + RM_PCIE_VIRTUAL_CHANNEL_7, + RM_PCIE_VIRTUAL_CHANNEL_INVALID +}; + +// Diagnostic collection actions. +#define RM_PCIE_ACTION_NOP 0 +#define RM_PCIE_ACTION_COLLECT_CONFIG_SPACE 1 +#define RM_PCIE_ACTION_COLLECT_PCI_CAP_STRUCT 2 +#define RM_PCIE_ACTION_COLLECT_PCIE_CAP_STRUCT 3 +#define RM_PCIE_ACTION_COLLECT_ALL_PCI_CAPS 4 +#define RM_PCIE_ACTION_COLLECT_ALL_PCIE_CAPS 5 +#define RM_PCIE_ACTION_REPORT_PCI_CAPS_COUNT 6 +#define RM_PCIE_ACTION_REPORT_PCIE_CAPS_COUNT 7 +#define RM_PCIE_ACTION_EOS 0xff + + +// Diagnostic collection device Type ids +#define RM_PCIE_DEVICE_TYPE_NONE 0xff +#define RM_PCIE_DEVICE_TYPE_GPU 0 +#define RM_PCIE_DEVICE_TYPE_UPSTREAM_BRIDGE 1 +#define RM_PCIE_DEVICE_COUNT 2 + +// Diagnostic collection capability Type ids +#define RM_PCIE_DC_CAP_TYPE_NONE 0xff +#define RM_PCIE_DC_CAP_TYPE_PCI 0 +#define RM_PCIE_DC_CAP_TYPE_PCIE 1 +#define RM_PCIE_DC_CAP_TYPE_COUNT 2 + +typedef struct _def_bif_dc_diagnostic_collection_command +{ + NvU8 action; + NvU8 deviceType; + NvU16 locator; + NvU16 length; +} CL_PCIE_DC_DIAGNOSTIC_COLLECTION_ENTRY; + +typedef struct _def_cl_pcie_dc_capability_map_entry +{ + NvU16 id; + NvU16 blkOffset; +} CL_PCIE_DC_CAPABILITY_MAP_ENTRY; +typedef struct +{ + NvU16 count; + CL_PCIE_DC_CAPABILITY_MAP_ENTRY entries[PCI_MAX_CAPS]; +} CL_PCIE_DC_CAPABILITY_MAP; + +typedef struct OBJCL OBJCL; +typedef struct OBJGPU OBJGPU; +// root port setup functions +NV_STATUS Broadcom_HT2100_setupFunc(OBJGPU *, OBJCL*); + +NV_STATUS Intel_RP25XX_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_RP81XX_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_RP3C0X_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_RP2F0X_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_RP0C0X_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Broadwell_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Skylake_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Skylake_U_Pch_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Skylake_H_Pch_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Kabylake_Y_setupFunc(OBJGPU *, OBJCL*); + +NV_STATUS Nvidia_RPC19_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Nvidia_RPC51_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Nvidia_RPC55_setupFunc(OBJGPU *, OBJCL*); + +NV_STATUS AMD_RP1480_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS AMD_RP1630_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS AMD_RP1483_setupFunc(OBJGPU *, OBJCL*); + +NV_STATUS Qualcomm_Snapdragon8cx_RP_setupFunc(OBJGPU *, OBJCL*); + +// Determines if the GPU is in a multi-GPU board based on devid checks +NvBool gpuIsMultiGpuBoard(OBJGPU *); + +#endif // NVPCIE_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/nvrangetypes.h b/userspace/sdks/nv570/nvalloc/common/inc/nvrangetypes.h new file mode 100644 index 0000000..1ff7df6 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/nvrangetypes.h @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @file nvrangetypes.h + * @brief Range types and operator macros + * @note #include a header to define NvUxx and NvSxx before sourcing this file. + */ + +#ifndef _NVRANGETYPES_H_ +#define _NVRANGETYPES_H_ + + +// +// Define range types by convention +// +#define __NV_DEFINE_RANGE_TYPE(T) \ +typedef struct NvRange ## T \ +{ \ + Nv ## T min; \ + Nv ## T max; \ +} NvRange ## T; + + +__NV_DEFINE_RANGE_TYPE(U64) // NvRangeU64 +__NV_DEFINE_RANGE_TYPE(S64) // NvRangeS64 +__NV_DEFINE_RANGE_TYPE(U32) // NvRangeU32 +__NV_DEFINE_RANGE_TYPE(S32) // NvRangeS32 +__NV_DEFINE_RANGE_TYPE(U16) // NvRangeU16 +__NV_DEFINE_RANGE_TYPE(S16) // NvRangeS16 +__NV_DEFINE_RANGE_TYPE(U8) // NvRangeU8 +__NV_DEFINE_RANGE_TYPE(S8) // NvRangeS8 + + +// +// Operator macros +// +// Macros are named xxx_RANGE (rather than xxx_RANGEU32, etc.) since they work +// properly on ranges with any number of bits, signed or unsigned. +// + +#define NV_EQUAL_RANGE(r1, r2) ((r1).min == (r2).min && (r1).max == (r2).max) +#define NV_EMPTY_INCLUSIVE_RANGE(r) ((r).min > (r).max) +#define NV_EMPTY_EXCLUSIVE_RANGE(r) ((r).min + 1 > (r).max - 1) +#define NV_WITHIN_INCLUSIVE_RANGE(r, x) ((r).min <= (x) && (x) <= (r).max) +#define NV_WITHIN_EXCLUSIVE_RANGE(r, x) ((r).min < (x) && (x) < (r).max) +#define NV_IS_SUBSET_RANGE(r1, r2) ((r1).min >= (r2).min && (r2).max >= (r1).max) +#define NV_IS_SUPERSET_RANGE(r1, r2) ((r1).min <= (r2).min && (r2).max <= (r1).max) +#define NV_CENTER_OF_RANGE(r) ((r).min / 2 + ((r).max + 1) / 2) // Avoid overflow and rounding anomalies. +#define NV_IS_OVERLAPPING_RANGE(r1, r2) \ + (NV_WITHIN_INCLUSIVE_RANGE((r1), (r2).min) || \ + NV_WITHIN_INCLUSIVE_RANGE((r1), (r2).max)) + +#define NV_DISTANCE_FROM_RANGE(r, x) ((x) < (r).min? (r).min - (x): ((x) > (r).max? (x) - (r).max: 0)) +#define NV_VALUE_WITHIN_INCLUSIVE_RANGE(r, x) ((x) < (r).min? (r).min : ((x) > (r).max? (r).max : (x))) +#define NV_VALUE_WITHIN_EXCLUSIVE_RANGE(r, x) ((x) <= (r).min? (r).min + 1 : ((x) >= (r).max? (r).max - 1 : (x))) + +#define NV_INIT_RANGE(r, x, y) \ +do \ +{ \ + (r).min = (x); \ + (r).max = (y); \ +} while(0) + +#define NV_ASSIGN_DELTA_RANGE(r, x, d) \ +do \ +{ \ + (r).min = (x) - (d); \ + (r).max = (x) + (d); \ +} while(0) + +#define NV_ASSIGN_INTERSECTION_RANGE(r1, r2) \ +do \ +{ \ + if ((r1).min < (r2).min) \ + (r1).min = (r2).min; \ + if ((r1).max > (r2).max) \ + (r1).max = (r2).max; \ +} while(0) + +#define NV_ASSIGN_UNION_RANGE(r1, r2) \ +do \ +{ \ + if ((r1).min > (r2).min) \ + (r1).min = (r2).min; \ + if ((r1).max < (r2).max) \ + (r1).max = (r2).max; \ +} while(0) + +#define NV_MULTIPLY_RANGE(r, x) \ +do \ +{ \ + (r).min *= (x); \ + (r).max *= (x); \ +} while(0) + +#define NV_DIVIDE_FLOOR_RANGE(r, x) \ +do \ +{ \ + (r).min /= (x); \ + (r).max /= (x); \ +} while(0) + +#define NV_DIVIDE_CEILING_RANGE(r, x) \ +do \ +{ \ + (r).min = ((r).min + (x) - 1) / (x); \ + (r).max = ((r).max + (x) - 1) / (x); \ +} while(0) + +#define NV_DIVIDE_ROUND_RANGE(r, x) \ +do \ +{ \ + (r).min = ((r).min + (x) / 2) / (x); \ + (r).max = ((r).max + (x) / 2) / (x); \ +} while(0) + +#define NV_DIVIDE_WIDE_RANGE(r, x) \ +do \ +{ \ + (r).min /= (x); \ + (r).max = ((r).max + (x) - 1) / (x); \ +} while(0) + +#define NV_DIVIDE_NARROW_RANGE(r, x) \ +do \ +{ \ + (r).min = ((r).min + (x) - 1) / (x); \ + (r).max /= (x); \ +} while(0) + +#define NV_VALUE_WITHIN_INCLUSIVE_RANGE(r, x) \ + ((x) < (r).min? (r).min : ((x) > (r).max? (r).max : (x))) + +#define NV_WITHIN_INCLUSIVE_RANGE(r, x) \ + ((r).min <= (x) && (x) <= (r).max) + +#define NV_DISTANCE_FROM_RANGE(r, x) \ + ((x) < (r).min? (r).min - (x): ((x) > (r).max? (x) - (r).max: 0)) + +#endif // _NVRANGETYPES_H_ + diff --git a/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi.h b/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi.h new file mode 100644 index 0000000..aefad5e --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi.h @@ -0,0 +1,2925 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2010-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _SMBPBI_H +#define _SMBPBI_H + +#include "oob/smbpbi_impl.h" + +/*! + * This file contains definitions for SMBPBI - SMBUS Out-Of-Band (OOB) interface + */ +/*! + * NV_MSGBOX_CMD() macro below and its derivatives depend on DRF_DEF() and DRF_NUM() + * macros, that are defined in "nvdrf.h" header file, which is a part of + * smbus/msgbox SDK. That file needs to be #included before this (smbpbi.h) + * header file. + */ + +/******************************************************************************** + * * + * GPU COMMANDS * + * * + ********************************************************************************/ + +// +// Encoding: +// 30 29 23 16 15 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |I|C|R| STATUS | ARG2 | ARG1 | OPCODE | COMMAND +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// 31 28 24 +// + +#define NV_MSGBOX_CMD_OPCODE 7:0 +#define NV_MSGBOX_CMD_OPCODE_NULL_CMD 0x00000000 +#define NV_MSGBOX_CMD_OPCODE_GET_CAP_DWORD 0x00000001 +#define NV_MSGBOX_CMD_OPCODE_GET_TEMP 0x00000002 +#define NV_MSGBOX_CMD_OPCODE_GET_EXT_TEMP 0x00000003 /* With 8 fractional bits! */ +#define NV_MSGBOX_CMD_OPCODE_GET_POWER 0x00000004 +#define NV_MSGBOX_CMD_OPCODE_GET_SYS_ID_DATA 0x00000005 +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V1 0x00000006 +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V2 0x00000007 +#define NV_MSGBOX_CMD_OPCODE_GPU_PCONTROL 0x00000008 +#define NV_MSGBOX_CMD_OPCODE_GPU_SYSCONTROL 0x00000009 +#define NV_MSGBOX_CMD_OPCODE_SET_MASTER_CAPS 0x0000000a +#define NV_MSGBOX_CMD_OPCODE_SET_PRIMARY_CAPS 0x0000000a // Duplicate value till bug 3190552 is addressed. +#define NV_MSGBOX_CMD_OPCODE_GPU_REQUEST_CPL 0x0000000b +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V3 0x0000000c +#define NV_MSGBOX_CMD_OPCODE_SCRATCH_READ 0x0000000d +#define NV_MSGBOX_CMD_OPCODE_SCRATCH_WRITE 0x0000000e +#define NV_MSGBOX_CMD_OPCODE_SCRATCH_COPY 0x0000000f +#define NV_MSGBOX_CMD_OPCODE_ASYNC_REQUEST 0x00000010 +#define NV_MSGBOX_CMD_OPCODE_REGISTER_ACCESS 0x00000011 +#define NV_MSGBOX_CMD_OPCODE_GET_POWER_CONNECTOR_STATE 0x00000012 +#define NV_MSGBOX_CMD_OPCODE_GET_PAGE_RETIREMENT_STATS 0x00000013 +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V4 0x00000014 +#define NV_MSGBOX_CMD_OPCODE_GET_THERM_PARAM 0x00000015 +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V5 0x00000016 +#define NV_MSGBOX_CMD_OPCODE_ACCESS_WP_MODE 0x00000017 + +// +// TODO: Get rid of MISC_GPU_FLAGS once all SMBPBI implementations have made +// the switch to MISC_DEVICE_FLAGS. +// +#define NV_MSGBOX_CMD_OPCODE_GET_MISC_GPU_FLAGS 0x00000018 +#define NV_MSGBOX_CMD_OPCODE_GET_MISC_DEVICE_FLAGS 0x00000018 +#define NV_MSGBOX_CMD_OPCODE_GPU_UTIL_COUNTERS 0x00000019 +#define NV_MSGBOX_CMD_OPCODE_GET_NVLINK_INFO 0x0000001a +#define NV_MSGBOX_CMD_OPCODE_GET_CLOCK_FREQ_INFO 0x0000001b +#define NV_MSGBOX_CMD_OPCODE_BUNDLE_LAUNCH 0x0000001c +#define NV_MSGBOX_CMD_OPCODE_GET_DRIVER_EVENT_MSG 0x0000001d +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V6 0x0000001e +#define NV_MSGBOX_CMD_OPCODE_REMAP_ROW_STATS 0x00000020 +#define NV_MSGBOX_CMD_OPCODE_GET_PCIE_LINK_INFO 0x00000021 +#define NV_MSGBOX_CMD_OPCODE_GET_ENERGY_COUNTER 0x00000022 +#define NV_MSGBOX_CMD_OPCODE_RESERVED_0 0x00000024 +#define NV_MSGBOX_CMD_OPCODE_RESERVED_1 0x00000025 +#define NV_MSGBOX_CMD_OPCODE_GET_POWER_HINT_INFO 0x00000026 +#define NV_MSGBOX_CMD_OPCODE_DYNAMIC_SYSTEM_INFORMATION 0x00000027 +#define NV_MSGBOX_CMD_OPCODE_GPU_PERFORMANCE_MONITORING 0x00000028 + +#define NV_MSGBOX_CMD_ARG1 15:8 +#define NV_MSGBOX_CMD_ARG1_NULL 0x00000000 +#define NV_MSGBOX_CMD_ARG1_TEMP_GPU_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_TEMP_NVSWITCH_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_TEMP_GPU_1 0x00000001 /* For Gemini boards */ +#define NV_MSGBOX_CMD_ARG1_TEMP_BOARD 0x00000004 +#define NV_MSGBOX_CMD_ARG1_TEMP_MEMORY 0x00000005 +#define NV_MSGBOX_CMD_ARG1_TEMP_PWR_SUPPLY 0x00000006 +#define NV_MSGBOX_CMD_ARG1_TEMP_T_LIMIT 0x00000007 +#define NV_MSGBOX_CMD_ARG1_TEMP_NUM_SENSORS 8 +#define NV_MSGBOX_CMD_ARG1_POWER_TOTAL 0x00000000 +#define NV_MSGBOX_CMD_ARG1_SMBPBI_POWER 0x00000001 +#define NV_MSGBOX_CMD_ARG1_POWER_FB 0x00000002 +#define NV_MSGBOX_CMD_ARG1_POWER_MODULE 0x00000003 +/* SysId info type encodings for opcode NV_MSGBOX_CMD_OPCODE_GET_SYS_ID_DATA (0x05) */ +#define NV_MSGBOX_CMD_ARG1_BOARD_PART_NUM_V1 0x00000000 +#define NV_MSGBOX_CMD_ARG1_OEM_INFO_V1 0x00000001 +#define NV_MSGBOX_CMD_ARG1_SERIAL_NUM_V1 0x00000002 +#define NV_MSGBOX_CMD_ARG1_MARKETING_NAME_V1 0x00000003 +#define NV_MSGBOX_CMD_ARG1_GPU_PART_NUM_V1 0x00000004 +#define NV_MSGBOX_CMD_ARG1_MEMORY_VENDOR_V1 0x00000005 +#define NV_MSGBOX_CMD_ARG1_MEMORY_PART_NUM_V1 0x00000006 +#define NV_MSGBOX_CMD_ARG1_BUILD_DATE_V1 0x00000007 +#define NV_MSGBOX_CMD_ARG1_FIRMWARE_VER_V1 0x00000008 +#define NV_MSGBOX_CMD_ARG1_VENDOR_ID_V1 0x00000009 +#define NV_MSGBOX_CMD_ARG1_DEV_ID_V1 0x0000000a +#define NV_MSGBOX_CMD_ARG1_SUB_VENDOR_ID_V1 0x0000000b +#define NV_MSGBOX_CMD_ARG1_SUB_ID_V1 0x0000000c +#define NV_MSGBOX_CMD_ARG1_GPU_GUID_V1 0x0000000d +#define NV_MSGBOX_CMD_ARG1_INFOROM_VER_V1 0x0000000e +#define NV_MSGBOX_CMD_ARG1_PRODUCT_LENGTH_V1 0x0000000f +#define NV_MSGBOX_CMD_ARG1_PRODUCT_WIDTH_V1 0x00000010 +#define NV_MSGBOX_CMD_ARG1_PRODUCT_HEIGHT_V1 0x00000011 +#define NV_MSGBOX_CMD_ARG1_PCIE_SPEED_V1 0x00000012 +#define NV_MSGBOX_CMD_ARG1_PCIE_WIDTH_V1 0x00000013 +#define NV_MSGBOX_CMD_ARG1_TGP_LIMIT_V1 0x00000014 +#define NV_MSGBOX_CMD_ARG1_FRU_PART_NUMBER_V1 0x00000015 +#define NV_MSGBOX_CMD_ARG1_MODULE_POWER_LIMIT_V1 0x00000016 +#define NV_MSGBOX_CMD_ARG1_MAX_DRAM_CAPACITY_V1 0x00000017 +#define NV_MSGBOX_CMD_ARG1_SYS_ID_DATA_TYPE_MAX 0x00000017 /* Adjust, when adding new types */ +#define NV_MSGBOX_CMD_ARG1_REGISTER_ACCESS_WRITE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_REGISTER_ACCESS_READ 0x00000001 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_TARGET 0x00000000 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_HW_SLOWDN 0x00000001 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_SHUTDN 0x00000002 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_HBM_SLOWDN 0x00000003 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_SW_SLOWDN 0x00000004 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_TARGET_TLIMIT 0x00000005 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_HW_SLOWDN_TLIMIT 0x00000006 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_SHUTDN_TLIMIT 0x00000007 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_HBM_SLOWDN_TLIMIT 0x00000008 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_SW_SLOWDN_TLIMIT 0x00000009 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_ECC_ENABLED_STATE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_RESET_REQUIRED 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_1 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_DEVICE_FLAGS_PAGE_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_DEVICE_FLAGS_PAGE_1 0x00000001 + +#define NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_CONTEXT_TIME 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_SM_TIME 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_RESET_COUNTERS 0x000000FF +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_COUNT 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_STATE_V1 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_BANDWIDTH 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER_REPLAY 0x00000003 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER_RECOVERY 0x00000004 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER_FLIT_CRC 0x00000005 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER_DATA_CRC 0x00000006 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_STATE_V2 0x00000007 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_SUBLINK_WIDTH 0x00000008 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_DATA_TX 0x00000009 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_DATA_RX 0x0000000a +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_RAW_TX 0x0000000b +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_RAW_RX 0x0000000c +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_TRAINING_ERROR_STATE 0x0000000d +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_RUNTIME_ERROR_STATE 0x0000000e +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_AVAILABILITY 0x0000000f +#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_CURRENT 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_MIN 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_MAX 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_PAGE_3 0x00000003 +#define NV_MSGBOX_CMD_ARG1_GET_SUPPORTED_CLOCK_THROTTLE_REASONS 0x00000004 +#define NV_MSGBOX_CMD_ARG1_GET_CURRENT_CLOCK_THROTTLE_REASONS 0x00000005 +#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_RAW_COUNTS 0x00000000 +#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_STATE_FLAGS 0x00000001 +#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_1 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_2 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_3 0x00000003 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_4 0x00000004 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_6 0x00000006 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_8 0x00000008 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_9 0x00000009 + +/* Async requests */ +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_GET \ + 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_SET \ + 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_INFO_GET \ + 0x00000002 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_COUNT_GET \ + 0x00000003 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_INFO_GET \ + 0x00000004 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_STATUS_GET \ + 0x00000005 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OVERCLOCKING_LIMIT_CONTROL_GET \ + 0x00000006 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OVERCLOCKING_LIMIT_CONTROL_SET \ + 0x00000007 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_ENERGY_COUNTER_STATUS_GET \ + 0x00000008 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_VIOLATION_COUNTERS_STATUS_GET \ + 0x00000009 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_UTILIZATION_RATE_GET \ + 0x0000000a +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OOB_CLOCK_LIMIT_SET \ + 0x0000000b +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_DEVICE_MODE_CONTROL \ + 0x0000000c +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_TEST_MESSAGE_SEND \ + 0x0000000e +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_CLOCK_LIMIT_GET \ + 0x0000000d +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_FAN_CURVE_POINTS_GET_SET \ + 0x0000000f +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POWER_HINT_GET \ + 0x00000010 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_CONFIGURE_PROGRAMMABLE_EDPP \ + 0x00000011 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_MODULE_LIMIT_CONTROL_GET \ + 0x00000012 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_MODULE_LIMIT_CONTROL_SET \ + 0x00000013 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_MODULE_LIMIT_INFO_GET \ + 0x00000014 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_MEMORY_CAPACITY_UTILIZATION_GET \ + 0x00000015 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POLL 0x000000ff + + +#define NV_MSGBOX_CMD_ARG1_ECC_V2_SEL 15:14 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_SEL_CNT 0x00000000 /* counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_SEL_FBA 0x00000001 /* FB addresses */ +/* Counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_DEV 13:12 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_DEV_FB 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_DEV_GR 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_FB_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_FB_SRC_LTC 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_FB_SRC_FB 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_GR_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_GR_SRC_L1 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_GR_SRC_RF 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_TYPE 9:8 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_TYPE_SBE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_TYPE_DBE 0x00000001 +/* FB addresses */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_SC 13:13 /* scope */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_SC_IND 0x00000000 /* individual addresses */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_SC_CUMUL 0x00000001 /* cumulative counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_IND_FLD 12:12 /* field */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_IND_FLD_CTRS 0x00000000 /* counters: sbe, dbe */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_IND_FLD_ADDR 0x00000001 /* address value */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_IND_INDEX 11:8 /* address array index: 0-13 */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V3_SEL 15:14 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_SEL_CNT 0x00000000 /* counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_SEL_FBA 0x00000001 /* FB addresses */ +/* Counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_DEV 13:12 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_DEV_FB 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_DEV_GR 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_FB_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_FB_SRC_LTC 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_FB_SRC_FB 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_GR_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_GR_SRC_L1 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_GR_SRC_RF 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_GR_SRC_TEX 0x00000002 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_TYPE 9:8 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_TYPE_SBE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_TYPE_DBE 0x00000001 +/* FB addresses */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_SC 13:13 /* scope */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_SC_IND 0x00000000 /* individual addresses */ +// These are not present in V3 +//#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_SC_CUMUL 0x00000001 /* cumulative counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_IND_FLD 12:12 /* field */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_IND_FLD_CTRS 0x00000000 /* counters: sbe, dbe */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_IND_FLD_ADDR 0x00000001 /* address value */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_IND_INDEX 11:8 /* address array index: 0-13 */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V4_SEL 15:14 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_SEL_CNT 0x00000000 /* counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_SEL_FBA 0x00000001 /* FB addresses */ +/* Counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_DEV 13:12 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_DEV_FB 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_DEV_GR 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_FB_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_FB_SRC_LTC 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_FB_SRC_FB 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_GR_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_GR_SRC_SHM 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_GR_SRC_RF 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_GR_SRC_TEX 0x00000002 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_TYPE 9:8 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_TYPE_SBE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_TYPE_DBE 0x00000001 +/* FB addresses */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_SC 13:13 /* scope */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_SC_IND 0x00000000 /* individual addresses */ +// These are not present in V4 +//#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_SC_CUMUL 0x00000001 /* cumulative counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_IND_FLD 12:12 /* field */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_IND_FLD_CTRS 0x00000000 /* counters: sbe, dbe */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_IND_FLD_ADDR 0x00000001 /* address value */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_IND_INDEX 11:8 /* address array index: 0-13 */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V5_SEL 15:15 +#define NV_MSGBOX_CMD_ARG1_ECC_V5_SEL_COUNTS 0x00000000 /* address/region counts */ +#define NV_MSGBOX_CMD_ARG1_ECC_V5_SEL_ERR_BUFFER 0x00000001 /* SRAM error buffer */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V5_COUNTS_INDEX_LO 14:8 /* 7 least significant index bits */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V5_ERR_BUF_INDEX 11:8 /* buffer entry index */ + +#define NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION 8:8 +#define NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_GET 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_SET 0x00000001 + +#define NV_MSGBOX_CMD_ARG1_BUNDLE_REQUEST_COUNT 11:8 +#define NV_MSGBOX_CMD_ARG1_BUNDLE_DISP_RULE_COUNT 15:12 + +#define NV_MSGBOX_CMD_ARG1_GET_DEM_OLDEST 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_DEM_BY_SEQ_NUMBER 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_DEM_BY_TIMESTAMP 0x00000002 + +#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE 15:8 +#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE_CORRECTABLE_ERROR 0 +#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE_UNCORRECTABLE_ERROR 1 +#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE_ECC_STATE_FLAGS 2 + +#define NV_MSGBOX_CMD_ARG1_ENERGY_COUNTER_GPU 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ENERGY_COUNTER_MODULE 0x00000003 + +// Query type of _GET_POWER_HINT_INFO +#define NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_CLK 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_TEMP 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_PROFILES 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_NUM \ + (NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_PROFILES + 1) + +// Arg1 for _GPU_PERFORMANCE_MONITORING +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION 15:14 +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION_GET_INSTANTANEOUS_METRIC 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION_GET_SNAPSHOT_METRIC 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION_CAPTURE_SNAPSHOT 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION_SET_MULTIPLIER 0x00000003 + +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC 13:8 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_GRAPHICS_ENGINE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_SM_ACTIVITY 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_SM_OCCUPANCY 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_TENSOR_CORE_ACTIVITY 0x00000003 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_DRAM_USAGE 0x00000004 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_FP64_ACTIVITY 0x00000005 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_FP32_ACTIVITY 0x00000006 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_FP16_ACTIVITY 0x00000007 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_PCIE_TX_BANDWIDTH 0x00000008 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_PCIE_RX_BANDWIDTH 0x00000009 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_TX_BANDWIDTH 0x0000000A +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_TX_BANDWIDTH 0x0000000B +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_RX_BANDWIDTH 0x0000000C +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_RX_BANDWIDTH 0x0000000D +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVDEC_UTILIZATION 0x0000000E +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVJPG_UTILIZATION 0x0000000F +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVOFA_UTILIZATION 0x00000010 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_TX_BW_PER_LINK 0x00000011 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_TX_BW_PER_LINK 0x00000012 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_RX_BW_PER_LINK 0x00000013 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_RX_BW_PER_LINK 0x00000014 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVDEC_UTIL_PER_INSTANCE 0x00000015 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVJPG_UTIL_PER_INSTANCE 0x00000016 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_INTEGER_UTILIZATION 0x00000017 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_DMMA_UTILIZATION 0x00000018 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_HMMA_UTILIZATION 0x00000019 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_IMMA_UTILIZATION 0x0000001A + +#define NV_MSGBOX_CMD_ARG1_DYN_SYS_INFO_DRIVER_VERSION_V1 0x00000000 + +#define NV_MSGBOX_CMD_ARG2 23:16 +#define NV_MSGBOX_CMD_ARG2_NULL 0x00000000 + +#define NV_MSGBOX_CMD_ARG2_ECC_V2_FB_PARTITION 23:20 +#define NV_MSGBOX_CMD_ARG2_ECC_V2_FB_SLICE 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V2_GR_GPC 23:20 +#define NV_MSGBOX_CMD_ARG2_ECC_V2_GR_TPC 19:16 + +#define NV_MSGBOX_CMD_ARG2_ECC_V3_FB_PARTITION 23:20 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_FB_SLICE 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_FB_SUBPARTITION 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_GR_TEX 23:22 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_GR_GPC 21:19 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_GR_TPC 18:16 + +#define NV_MSGBOX_CMD_ARG2_ECC_V4_FB_PARTITION 23:20 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_FB_SLICE 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_FB_SUBPARTITION 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_GR_TEX 23:22 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_GR_GPC 21:19 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_GR_TPC 18:16 + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_INDEX_HI 23:21 +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE 18:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_HDR_META 0x0 /* header and metadata */ + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_ADDR_ADDR 0x1 /* addr: address */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_ADDR_UNCORRECTED_COUNTS 0x2 /* addr: uncorrectedCounts */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_ADDR_CORRECTED_TOTAL 0x3 /* addr: correctedTotal */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_ADDR_CORRECTED_UNIQUE 0x4 /* addr: correctedUnique */ + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_REGN_CORRECTED_TOTAL 0x1 /* region: correctedTotal */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_REGN_CORRECTED_UNIQUE 0x2 /* region: correctedUnique */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_REGN_UNCORRECTED_TOTAL 0x3 /* region: uncorrectedTotal */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_REGN_UNCORRECTED_UNIQUE 0x4 /* region: uncorrectedUnique */ + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_MAX 0x4 + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE 17:16 /* entry element type */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE_ERR_TYPE_META 0x0 /* errorType and metadata */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE_TIME_STAMP 0x1 /* time stamp */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE_ADDR 0x2 /* address */ + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE_MAX 0x2 + +#define NV_MSGBOX_CMD_ARG2_REG_INDEX_SCRATCH_PAGE 0x00000000 +#define NV_MSGBOX_CMD_ARG2_REG_INDEX_EVENTS_PENDING 0x00000001 +#define NV_MSGBOX_CMD_ARG2_REG_INDEX_EVENT_MASK 0x00000002 +#define NV_MSGBOX_CMD_ARG2_REG_INDEX_MAX 0x00000002 /* increase when adding + new registers */ + +#define NV_MSGBOX_CMD_ARG2_NVLINK_INFO_AGGREGATE 0x000000FF + +#define NV_MSGBOX_CMD_ARG2_ACCESS_WP_MODE_SET_STATE 23:16 +#define NV_MSGBOX_CMD_ARG2_ACCESS_WP_MODE_SET_STATE_DISABLED 0x0000005A +#define NV_MSGBOX_CMD_ARG2_ACCESS_WP_MODE_SET_STATE_ENABLED 0x000000A5 + +#define NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_GPCCLK 0x00000000 +#define NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_MEMCLK 0x00000001 + +#define NV_MSGBOX_CMD_ARG2_ECC_V6_COUNTER_TYPE 23:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V6_COUNTER_TYPE_SRAM 0 +#define NV_MSGBOX_CMD_ARG2_ECC_V6_COUNTER_TYPE_DRAM 1 + +#define NV_MSGBOX_CMD_ARG2_REMAP_ROW_RAW_CNT_COMBINED 0x00000000 +#define NV_MSGBOX_CMD_ARG2_REMAP_ROW_RAW_CNT_UNCORR 0x00000001 +#define NV_MSGBOX_CMD_ARG2_REMAP_ROW_RAW_CNT_CORR 0x00000002 + +#define NV_MSGBOX_CMD_ARG2_REMAP_ROWS_STATE_FLAGS_PAGE0 0x00000000 + +/*! + * Arg2 for _GET_PCIE_LINK_INFO + * Arg1 == _GET_PCIE_LINK_INFO_PAGE_8 + * Return TX EQ parameters + */ +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_8_LANE_IDX 3:0 +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_8_SPEED_SELECT 5:4 +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_8_SPEED_SELECT_GEN_3 \ + 0x00000000 +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_8_SPEED_SELECT_GEN_4 \ + 0x00000001 +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_8_SPEED_SELECT_GEN_5 \ + 0x00000002 + +/*! + * Arg2 for _GET_PCIE_LINK_INFO + * Arg1 == _GET_PCIE_LINK_INFO_PAGE_9 + * Return RX EQ parameters + */ +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_9_LANE_IDX 3:0 +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_9_SPEED_SELECT 5:4 +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_9_SPEED_SELECT_GEN_3 \ + 0x00000000 +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_9_SPEED_SELECT_GEN_4 \ + 0x00000001 +#define NV_MSGBOX_CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_9_SPEED_SELECT_GEN_5 \ + 0x00000002 + +/*! + * Arg2 for _GET_POWER_HINT_INFO + * ARG1 == _GET_POWER_HINT_INFO_CLK + * Return Min/Max frequency in MHz + */ +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_CLK_GR 0x00000000 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_CLK_MEM 0x00000001 + +/*! + * Arg2 for _GET_POWER_HINT_INFO + * ARG1 == _GET_POWER_HINT_INFO_PROFILES + * Return bits represents which profiles are available + */ +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_0 0x00000000 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_1 0x00000001 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_2 0x00000002 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_3 0x00000003 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_TOTAL_PAGES \ + (NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_3 + 1) + +// Arg2 for _GPU_PERFORMANCE_MONITORING +#define NV_MSGBOX_CMD_ARG2_GPM_PARTITION_AGGREGATE 0x000000FF +#define NV_MSGBOX_CMD_ARG2_GPM_PARTITION_INDEX 21:16 +#define NV_MSGBOX_CMD_ARG2_GPM_CI_METRICS_REQUESTED 23:23 +#define NV_MSGBOX_CMD_ARG2_GPM_CI_METRICS_REQUESTED_YES 0x00000001 +#define NV_MSGBOX_CMD_ARG2_GPM_CI_METRICS_REQUESTED_NO 0x00000000 + +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER 23:16 +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_1X 0x00000001 +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_2X 0x00000002 +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_5X 0x00000005 +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_10X 0x0000000A +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_100X 0x00000064 + + +#define NV_MSGBOX_CMD_STATUS 28:24 +#define NV_MSGBOX_CMD_STATUS_NULL 0x00000000 +#define NV_MSGBOX_CMD_STATUS_ERR_REQUEST 0x00000001 +#define NV_MSGBOX_CMD_STATUS_ERR_OPCODE 0x00000002 +#define NV_MSGBOX_CMD_STATUS_ERR_ARG1 0x00000003 +#define NV_MSGBOX_CMD_STATUS_ERR_ARG2 0x00000004 +#define NV_MSGBOX_CMD_STATUS_ERR_DATA 0x00000005 +#define NV_MSGBOX_CMD_STATUS_ERR_MISC 0x00000006 +#define NV_MSGBOX_CMD_STATUS_ERR_I2C_ACCESS 0x00000007 +#define NV_MSGBOX_CMD_STATUS_ERR_NOT_SUPPORTED 0x00000008 +#define NV_MSGBOX_CMD_STATUS_ERR_NOT_AVAILABLE 0x00000009 +#define NV_MSGBOX_CMD_STATUS_ERR_BUSY 0x0000000a +#define NV_MSGBOX_CMD_STATUS_ERR_AGAIN 0x0000000b +#define NV_MSGBOX_CMD_STATUS_ERR_SENSOR_DATA 0x0000000c +#define NV_MSGBOX_CMD_STATUS_ERR_DISPOSITION 0x0000000d +#define NV_MSGBOX_CMD_STATUS_PARTIAL_FAILURE 0x0000001b +#define NV_MSGBOX_CMD_STATUS_ACCEPTED 0x0000001c +#define NV_MSGBOX_CMD_STATUS_INACTIVE 0x0000001d +#define NV_MSGBOX_CMD_STATUS_READY 0x0000001e +#define NV_MSGBOX_CMD_STATUS_SUCCESS 0x0000001f + +#define NV_MSGBOX_CMD_RSVD 29:29 +#define NV_MSGBOX_CMD_RSVD_INIT 0x00000000 + +#define NV_MSGBOX_CMD_COPY_DATA 30:30 +#define NV_MSGBOX_CMD_COPY_DATA_OFF 0x00000000 +#define NV_MSGBOX_CMD_COPY_DATA_ON 0x00000001 + +#define NV_MSGBOX_CMD_EVENT_PENDING 30:30 +#define NV_MSGBOX_CMD_EVENT_PENDING_OFF 0x00000000 +#define NV_MSGBOX_CMD_EVENT_PENDING_ON 0x00000001 + +#define NV_MSGBOX_CMD_INTR 31:31 +#define NV_MSGBOX_CMD_INTR_NOT_PENDING 0x00000000 +#define NV_MSGBOX_CMD_INTR_PENDING 0x00000001 +#define NV_MSGBOX_CMD_INTR_CLEAR 0x00000000 + +/* For individual requests in a bundle */ + +#define NV_MSGBOX_CMD_ON_BUNDLE_FAILURE 31:31 +#define NV_MSGBOX_CMD_ON_BUNDLE_FAILURE_CONTINUE 0x00000000 +#define NV_MSGBOX_CMD_ON_BUNDLE_FAILURE_STOP 0x00000001 + +#define NV_MSGBOX_CMD_DATA_COPY 23:0 +#define NV_MSGBOX_CMD_DATA_COPY_INIT 0x00000000 +#define NV_MSGBOX_CMD_EXT_STATUS 23:0 + +// NV_MSGBOX_CMD_COPY_SIZE_ENCODING used for certain commands to embed size +// indicators in the output when copy-bit is set in request + +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_DATA_OUT 0:0 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_DATA_OUT_SET 1 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_DATA_OUT_NOT_SET 0 + +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_EXT_DATA_OUT 1:1 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_EXT_DATA_OUT_SET 1 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_EXT_DATA_OUT_NOT_SET 0 + +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_DATA 23:2 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_DATA_INIT 0x00000000 + +/* Response to NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM */ +#define NV_MSGBOX_CMD_REMAP_ROW_HISTOGRAM_NONE_AVAILABILITY 15:0 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_* */ +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY 1:0 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY_KIB 0x00000000 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY_MIB 0x00000001 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY_GIB 0x00000002 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY_TIB 0x00000003 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_NEW_SAMPLE 2:2 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_NEW_SAMPLE_FALSE 0x00000000 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_NEW_SAMPLE_TRUE 0x00000001 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_INVALID_DELTA 3:3 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_INVALID_DELTA_FALSE 0x00000000 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_INVALID_DELTA_TRUE 0x00000001 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_DELTA 23:4 + +/* + * Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_TRAINING_ERR_STATE + * NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_RUNTIME_ERR_STATE + */ +#define NV_MSGBOX_CMD_NVLINK_INFO_TRAINING_ERROR_COUNT 11:0 +#define NV_MSGBOX_CMD_NVLINK_INFO_RUNTIME_ERROR_COUNT 23:12 +#define NV_MSGBOX_CMD_NVLINK_INFO_ERROR_COUNT_MAX \ + DRF_MASK(NV_MSGBOX_CMD_NVLINK_INFO_TRAINING_ERROR_COUNT) + +/** + * Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_AVAILABILITY + */ +#define NV_MSGBOX_CMD_NVLINK_INFO_GET_NVLINK_INFO_AVAILABILTY_PAGE_0 0x00000000 + +/** + * This field is populated as response to the following metrics: + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_DRAM_USAGE + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_PCIE_TX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_PCIE_RX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_TX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_TX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_RX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_RX_BANDWIDTH + */ +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY 2:0 +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY_B 0x00000000 +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY_KIB 0x00000001 +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY_MIB 0x00000002 +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY_GIB 0x00000003 + +/* MSGBOX data, capability dword structure */ + +#define NV_MSGBOX_DATA_REG 31:0 +#define NV_MSGBOX_DATA_CAP_COUNT 6 + +#define NV_MSGBOX_DATA_CAP_0 0 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_0 0:0 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_0_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_0_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_NVSWITCH_0 0:0 +#define NV_MSGBOX_DATA_CAP_0_TEMP_NVSWITCH_0_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_NVSWITCH_0_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_1 1:1 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_BOARD 4:4 +#define NV_MSGBOX_DATA_CAP_0_TEMP_BOARD_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_BOARD_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_MEMORY 5:5 +#define NV_MSGBOX_DATA_CAP_0_TEMP_MEMORY_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_MEMORY_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_PWR_SUPPLY 6:6 +#define NV_MSGBOX_DATA_CAP_0_TEMP_PWR_SUPPLY_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_PWR_SUPPLY_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_T_LIMIT 7:7 +#define NV_MSGBOX_DATA_CAP_0_TEMP_T_LIMIT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_T_LIMIT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS 11:8 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS_ZERO 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS_ADT7473 0x00000002 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS_SFXP11_5 0x00000005 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS_SFXP24_8 0x00000008 +#define NV_MSGBOX_DATA_CAP_0_GET_ENERGY_COUNTER_MODULE 12:12 +#define NV_MSGBOX_DATA_CAP_0_GET_ENERGY_COUNTER_MODULE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_GET_ENERGY_COUNTER_MODULE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_CONTROL_GET 13:13 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_CONTROL_GET_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_CONTROL_GET_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_CONTROL_SET 14:14 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_CONTROL_SET_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_CONTROL_SET_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_INFO_GET 15:15 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_INFO_GET_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_MODULE_LIMIT_INFO_GET_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_POWER_TOTAL 16:16 +#define NV_MSGBOX_DATA_CAP_0_POWER_TOTAL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_POWER_TOTAL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_GPU_PCONTROL 17:17 +#define NV_MSGBOX_DATA_CAP_0_GPU_PCONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_GPU_PCONTROL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_GPU_SYSCONTROL 18:18 +#define NV_MSGBOX_DATA_CAP_0_GPU_SYSCONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_GPU_SYSCONTROL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_BITS 28:19 // Adjust when adding new bits +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_TLIMIT_BITS 23:19 // Adjust when adding new bits +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_BITS 28:24 // Adjust when adding new bits +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC_TLIMIT 19:19 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC_TLIMIT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC_TLIMIT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN_TLIMIT 20:20 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN_TLIMIT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN_TLIMIT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN_TLIMIT 21:21 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN_TLIMIT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN_TLIMIT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY_TLIMIT 22:22 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY_TLIMIT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY_TLIMIT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN_TLIMIT 23:23 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN_TLIMIT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN_TLIMIT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC 24:24 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN 25:25 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN 26:26 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY 27:27 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN 28:28 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS 29:29 +#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_POWER_FB 30:30 +#define NV_MSGBOX_DATA_CAP_0_POWER_FB_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_POWER_FB_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_POWER_MODULE 31:31 +#define NV_MSGBOX_DATA_CAP_0_POWER_MODULE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_POWER_MODULE_AVAILABLE 0x00000001 + +#define NV_MSGBOX_DATA_CAP_1 1 +#define NV_MSGBOX_DATA_CAP_1_BOARD_PART_NUM_V1 0:0 +#define NV_MSGBOX_DATA_CAP_1_BOARD_PART_NUM_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_BOARD_PART_NUM_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_OEM_INFO_V1 1:1 +#define NV_MSGBOX_DATA_CAP_1_OEM_INFO_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_OEM_INFO_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_SERIAL_NUM_V1 2:2 +#define NV_MSGBOX_DATA_CAP_1_SERIAL_NUM_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_SERIAL_NUM_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_MARKETING_NAME_V1 3:3 +#define NV_MSGBOX_DATA_CAP_1_MARKETING_NAME_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_MARKETING_NAME_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GPU_PART_NUM_V1 4:4 +#define NV_MSGBOX_DATA_CAP_1_GPU_PART_NUM_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GPU_PART_NUM_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_VENDOR_V1 5:5 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_VENDOR_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_VENDOR_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_PART_NUM_V1 6:6 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_PART_NUM_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_PART_NUM_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_BUILD_DATE_V1 7:7 +#define NV_MSGBOX_DATA_CAP_1_BUILD_DATE_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_BUILD_DATE_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_FIRMWARE_VER_V1 8:8 +#define NV_MSGBOX_DATA_CAP_1_FIRMWARE_VER_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_FIRMWARE_VER_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_VENDOR_ID_V1 9:9 +#define NV_MSGBOX_DATA_CAP_1_VENDOR_ID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_VENDOR_ID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_DEV_ID_V1 10:10 +#define NV_MSGBOX_DATA_CAP_1_DEV_ID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_DEV_ID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_SUB_VENDOR_ID_V1 11:11 +#define NV_MSGBOX_DATA_CAP_1_SUB_VENDOR_ID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_SUB_VENDOR_ID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_SUB_ID_V1 12:12 +#define NV_MSGBOX_DATA_CAP_1_SUB_ID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_SUB_ID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GPU_GUID_V1 13:13 +#define NV_MSGBOX_DATA_CAP_1_GPU_GUID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GPU_GUID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_INFOROM_VER_V1 14:14 +#define NV_MSGBOX_DATA_CAP_1_INFOROM_VER_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_INFOROM_VER_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_MODULE_POWER_LIMIT_V1 15:15 +#define NV_MSGBOX_DATA_CAP_1_MODULE_POWER_LIMIT_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_MODULE_POWER_LIMIT_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V1 16:16 +#define NV_MSGBOX_DATA_CAP_1_ECC_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V2 17:17 +#define NV_MSGBOX_DATA_CAP_1_ECC_V2_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V2_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V3 18:18 +#define NV_MSGBOX_DATA_CAP_1_ECC_V3_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V3_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_RET_PG_CNT 19:19 +#define NV_MSGBOX_DATA_CAP_1_RET_PG_CNT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_RET_PG_CNT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V4 20:20 +#define NV_MSGBOX_DATA_CAP_1_ECC_V4_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V4_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V5 21:21 +#define NV_MSGBOX_DATA_CAP_1_ECC_V5_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V5_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ACCESS_WP_MODE 22:22 +#define NV_MSGBOX_DATA_CAP_1_ACCESS_WP_MODE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ACCESS_WP_MODE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GET_ECC_ENABLED_STATE 23:23 +#define NV_MSGBOX_DATA_CAP_1_GET_ECC_ENABLED_STATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GET_ECC_ENABLED_STATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GET_GPU_RESET_REQUIRED 24:24 +#define NV_MSGBOX_DATA_CAP_1_GET_GPU_RESET_REQUIRED_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GET_GPU_RESET_REQUIRED_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GPU_UTIL_COUNTERS 25:25 +#define NV_MSGBOX_DATA_CAP_1_GPU_UTIL_COUNTERS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GPU_UTIL_COUNTERS_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_INFO_STATE_V1 26:26 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_INFO_STATE_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_INFO_STATE_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_ERROR_COUNTERS 27:27 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_ERROR_COUNTERS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_ERROR_COUNTERS_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_FREQ_INFO 28:28 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_FREQ_INFO_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_FREQ_INFO_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GET_MIG_ENABLED_STATE 29:29 +#define NV_MSGBOX_DATA_CAP_1_GET_MIG_ENABLED_STATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GET_MIG_ENABLED_STATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V6 30:30 +#define NV_MSGBOX_DATA_CAP_1_ECC_V6_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V6_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_THROTTLE_REASON 31:31 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_THROTTLE_REASON_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_THROTTLE_REASON_AVAILABLE 0x00000001 + +#define NV_MSGBOX_DATA_CAP_2 2 +#define NV_MSGBOX_DATA_CAP_2_GPU_DRIVER 0:0 +#define NV_MSGBOX_DATA_CAP_2_GPU_DRIVER_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GPU_DRIVER_NOT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVSWITCH_DRIVER 0:0 +#define NV_MSGBOX_DATA_CAP_2_NVSWITCH_DRIVER_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVSWITCH_DRIVER_NOT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GPU_REQUEST 1:1 +#define NV_MSGBOX_DATA_CAP_2_GPU_REQUEST_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GPU_REQUEST_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NUM_SCRATCH_BANKS 4:2 +#define NV_MSGBOX_DATA_CAP_2_NUM_SCRATCH_BANKS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_FAN_V1_CONTROL 5:5 +#define NV_MSGBOX_DATA_CAP_2_FAN_V1_CONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_FAN_V1_CONTROL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_LENGTH_V1 6:6 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_LENGTH_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_LENGTH_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_WIDTH_V1 7:7 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_WIDTH_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_WIDTH_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_HEIGHT_V1 8:8 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_HEIGHT_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_HEIGHT_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PCIE_SPEED_V1 9:9 +#define NV_MSGBOX_DATA_CAP_2_PCIE_SPEED_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PCIE_SPEED_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PCIE_WIDTH_V1 10:10 +#define NV_MSGBOX_DATA_CAP_2_PCIE_WIDTH_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PCIE_WIDTH_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_TGP_LIMIT_V1 11:11 +#define NV_MSGBOX_DATA_CAP_2_TGP_LIMIT_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_TGP_LIMIT_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_SCRATCH_PAGE_SIZE 12:12 +#define NV_MSGBOX_DATA_CAP_2_SCRATCH_PAGE_SIZE_1024B 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_SCRATCH_PAGE_SIZE_256B 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_STATS 13:13 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_STATS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_STATS_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_INFO 14:14 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_INFO_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_INFO_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GET_GPU_DRAIN_AND_RESET_RECOMMENDED 15:15 +#define NV_MSGBOX_DATA_CAP_2_GET_GPU_DRAIN_AND_RESET_RECOMMENDED_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GET_GPU_DRAIN_AND_RESET_RECOMMENDED_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_THROUGHPUT 16:16 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_THROUGHPUT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_THROUGHPUT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_LINK_STATE_V2 17:17 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_LINK_STATE_V2_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_LINK_STATE_V2_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_SUBLINK_WIDTH 18:18 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_SUBLINK_WIDTH_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_SUBLINK_WIDTH_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GET_ENERGY_COUNTER 19:19 +#define NV_MSGBOX_DATA_CAP_2_GET_ENERGY_COUNTER_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GET_ENERGY_COUNTER_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_PENDING 20:20 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_PENDING_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_PENDING_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_HISTOGRAM 21:21 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_HISTOGRAM_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_HISTOGRAM_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_TRAINING_ERROR_STATE 22:22 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_TRAINING_ERROR_STATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_TRAINING_ERROR_STATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_RUNTIME_ERROR_STATE 23:23 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_RUNTIME_ERROR_STATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_RUNTIME_ERROR_STATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_TARGET_SPEED 25:25 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_TARGET_SPEED_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_TARGET_SPEED_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_CURRENT_PSTATE 26:26 +#define NV_MSGBOX_DATA_CAP_2_CURRENT_PSTATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_CURRENT_PSTATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_AVAILABILITY 27:27 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_AVAILABILITY_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_AVAILABILITY_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_FRU_PART_NUMBER_V1 31:31 +#define NV_MSGBOX_DATA_CAP_2_FRU_PART_NUMBER_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_FRU_PART_NUMBER_V1_AVAILABLE 0x00000001 + +#define NV_MSGBOX_DATA_CAP_4 4 +#define NV_MSGBOX_DATA_CAP_4_HW_VIOLATION_TIME 0:0 +#define NV_MSGBOX_DATA_CAP_4_HW_VIOLATION_TIME_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_HW_VIOLATION_TIME_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SW_VIOLATION_TIME 1:1 +#define NV_MSGBOX_DATA_CAP_4_SW_VIOLATION_TIME_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SW_VIOLATION_TIME_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SW_POWER_VIOLATION_TIME 2:2 +#define NV_MSGBOX_DATA_CAP_4_SW_POWER_VIOLATION_TIME_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SW_POWER_VIOLATION_TIME_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SW_THERMAL_VIOLATION_TIME 3:3 +#define NV_MSGBOX_DATA_CAP_4_SW_THERMAL_VIOLATION_TIME_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SW_THERMAL_VIOLATION_TIME_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_UTILIZATION_RATE 4:4 +#define NV_MSGBOX_DATA_CAP_4_UTILIZATION_RATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_UTILIZATION_RATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_GET_DRIVER_EVENT_MSG 5:5 +#define NV_MSGBOX_DATA_CAP_4_GET_DRIVER_EVENT_MSG_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_GET_DRIVER_EVENT_MSG_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_REQUEST_BUNDLING 6:6 +#define NV_MSGBOX_DATA_CAP_4_REQUEST_BUNDLING_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_REQUEST_BUNDLING_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SET_DEVICE_DISABLE 7:7 +#define NV_MSGBOX_DATA_CAP_4_SET_DEVICE_DISABLE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SET_DEVICE_DISABLE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SET_ECC_MODE 8:8 +#define NV_MSGBOX_DATA_CAP_4_SET_ECC_MODE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SET_ECC_MODE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SET_MIG_MODE 10:10 +#define NV_MSGBOX_DATA_CAP_4_SET_MIG_MODE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SET_MIG_MODE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_FAN_CURVE_POINTS_GET_SET 11:11 +#define NV_MSGBOX_DATA_CAP_4_FAN_CURVE_POINTS_GET_SET_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_FAN_CURVE_POINTS_GET_SET_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_POWER_HINT 22:22 +#define NV_MSGBOX_DATA_CAP_4_POWER_HINT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_POWER_HINT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_DRIVER_VERSION_V1 23:23 +#define NV_MSGBOX_DATA_CAP_4_DRIVER_VERSION_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_DRIVER_VERSION_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_GPU_PERFORMANCE_MONITORING 24:24 +#define NV_MSGBOX_DATA_CAP_4_GPU_PERFORMANCE_MONITORING_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_GPU_PERFORMANCE_MONITORING_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_DEM_BY_SEQNUM_TIMESTAMP 26:26 +#define NV_MSGBOX_DATA_CAP_4_DEM_BY_SEQNUM_TIMESTAMP_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_DEM_BY_SEQNUM_TIMESTAMP_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_CONFIGURE_PROGRAMMABLE_EDPP 30:30 +#define NV_MSGBOX_DATA_CAP_4_CONFIGURE_PROGRAMMABLE_EDPP_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_CONFIGURE_PROGRAMMABLE_EDPP_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_MAX_DRAM_CAPACITY_V1 31:31 +#define NV_MSGBOX_DATA_CAP_4_MAX_DRAM_CAPACITY_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_MAX_DRAM_CAPACITY_V1_AVAILABLE 0x00000001 + +#define NV_MSGBOX_DATA_CAP_5 5 +#define NV_MSGBOX_DATA_CAP_5_MEMORY_CAPACITY_UTILIZATION 6:6 +#define NV_MSGBOX_DATA_CAP_5_MEMORY_CAPACITY_UTILIZATION_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_5_MEMORY_CAPACITY_UTILIZATION_AVAILABLE 0x00000001 + +#define NV_MSGBOX_DATA_CAP_5_SRAM_ERROR_THRESHOLD_EXCEEDED 9:9 +#define NV_MSGBOX_DATA_CAP_5_SRAM_ERROR_THRESHOLD_EXCEEDED_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_5_SRAM_ERROR_THRESHOLD_EXCEEDED_AVAILABLE 0x00000001 + +/* ECC counters */ +#define NV_MSGBOX_DATA_ECC_CNT_16BIT_DBE 31:16 +#define NV_MSGBOX_DATA_ECC_CNT_16BIT_SBE 16:0 +#define NV_MSGBOX_DATA_ECC_CNT_8BIT_DBE 23:16 +#define NV_MSGBOX_DATA_ECC_CNT_8BIT_SBE 7:0 + +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HEADER 7:0 + +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HDR_ID 1:0 +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HDR_ID_INVAL 0 +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HDR_ID_ADDR 1 +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HDR_ID_REGN 2 + +#define NV_MSGBOX_DATA_ECC_V5_COUNT_METADATA 31:16 + +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_ERR_TYPE 7:0 + +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_ERR_TYPE_ID 0:0 +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_ERR_TYPE_ID_UNCORR 0 +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_ERR_TYPE_ID_CORR 1 + +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_METADATA 31:16 + +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID 21:16 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_LRF 0 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_L1DATA 1 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_L1TAG 2 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_CBU 3 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_LTC 4 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_DRAM 5 + +#define NV_MSGBOX_DATA_ECC_V5_METADATA_LOCATION_ID 26:22 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_SUBLOCATION_ID 31:27 + +/* ECC state flags */ +#define NV_MSGBOX_DATA_ECC_V6_STATE_FLAGS 31:0 + +#define NV_MSGBOX_DATA_ECC_V6_STATE_FLAGS_SRAM_ERROR_THRESHOLD_EXCEEDED 0:0 +#define NV_MSGBOX_DATA_ECC_V6_STATE_FLAGS_SRAM_ERROR_THRESHOLD_EXCEEDED_FALSE 0 +#define NV_MSGBOX_DATA_ECC_V6_STATE_FLAGS_SRAM_ERROR_THRESHOLD_EXCEEDED_TRUE 1 + +/* NV_MSGBOX_CMD_OPCODE_SCRATCH_COPY src offset argument */ +#define NV_MSGBOX_DATA_COPY_SRC_OFFSET 7:0 + +/* NV_MSGBOX_CMD_ARG2_REG_INDEX_SCRATCH_PAGE register layout */ +#define NV_MSGBOX_DATA_SCRATCH_PAGE_DST 7:0 +#define NV_MSGBOX_DATA_SCRATCH_PAGE_SRC 15:8 + +/* Async request status codes returned in the data register */ +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS 7:0 + +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_SUCCESS 0x00000000 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_CARD_NOT_PRESENT 0x00000001 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_DUAL_LINK_INUSE 0x00000002 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_GENERIC 0x00000003 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_GPU_NOT_FULL_POWER 0x00000004 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_IN_USE 0x00000005 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INSUFFICIENT_RESOURCES 0x00000006 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_ACCESS_TYPE 0x00000007 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_ARGUMENT 0x00000008 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_BASE 0x00000009 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_CHANNEL 0x0000000A +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_CLASS 0x0000000B +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_CLIENT 0x0000000C +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_COMMAND 0x0000000D +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_DATA 0x0000000E +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_DEVICE 0x0000000F +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_DMA_SPECIFIER 0x00000010 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_EVENT 0x00000011 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_FLAGS 0x00000012 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_FUNCTION 0x00000013 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_HEAP 0x00000014 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_INDEX 0x00000015 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_LIMIT 0x00000016 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_METHOD 0x00000017 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_BUFFER 0x00000018 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_ERROR 0x00000019 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_HANDLE 0x0000001A +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_NEW 0x0000001B +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_OLD 0x0000001C +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_PARENT 0x0000001D +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OFFSET 0x0000001E +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OWNER 0x0000001F +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_PARAM_STRUCT 0x00000020 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_PARAMETER 0x00000021 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_POINTER 0x00000022 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_REGISTRY_KEY 0x00000023 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_STATE 0x00000024 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_STRING_LENGTH 0x00000025 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_XLATE 0x00000026 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_IRQ_NOT_FIRING 0x00000027 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_MULTIPLE_MEMORY_TYPES 0x00000028 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_NOT_SUPPORTED 0x00000029 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_OPERATING_SYSTEM 0x0000002A +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_PROTECTION_FAULT 0x0000002B +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_TIMEOUT 0x0000002C +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_TOO_MANY_PRIMARIES 0x0000002D +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_IRQ_EDGE_TRIGGERED 0x0000002E +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OPERATION 0x0000002F +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_NOT_COMPATIBLE 0x00000030 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_MORE_PROCESSING_REQUIRED 0x00000031 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INSUFFICIENT_PERMISSIONS 0x00000032 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_TIMEOUT_RETRY 0x00000033 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_NOT_READY 0x00000034 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_GPU_IS_LOST 0x00000035 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_IN_FULLCHIP_RESET 0x00000036 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_LOCK_STATE 0x00000037 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_ADDRESS 0x00000038 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_IRQ_LEVEL 0x00000039 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_MEMORY_TRAINING_FAILED 0x00000040 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_BUSY_RETRY 0x00000041 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INSUFFICIENT_POWER 0x00000042 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_OBJECT_NOT_FOUND 0x00000043 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_BUFFER_TOO_SMALL 0x00000044 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_RESET_REQUIRED 0x00000045 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_PRIV_SEC_VIOLATION 0x00000046 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_DEFERRED 0x00000047 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_FREQ_NOT_SUPPORTED 0x00000048 + +/* Response to NV_MSGBOX_CMD_OPCODE_GET_POWER_CONNECTOR_STATE */ +#define NV_MSGBOX_DATA_POWER_CONNECTED 0:0 +#define NV_MSGBOX_DATA_POWER_CONNECTED_SUFFICIENT 0x00000000 +#define NV_MSGBOX_DATA_POWER_CONNECTED_INSUFFICIENT 0x00000001 + +/* Response to NV_MSGBOX_CMD_OPCODE_GET_PAGE_RETIREMENT_STATS */ +#define NV_MSGBOX_DATA_RETIRED_PAGES_CNT_SBE 7:0 +#define NV_MSGBOX_DATA_RETIRED_PAGES_CNT_DBE 15:8 + +/* + * Response to + * NV_MSGBOX_CMD_ARG1_GET_MISC_ECC_ENABLED_STATE + * NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_0 + */ +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_SUPPORTED 0:0 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_SUPPORTED_OFF 0x00000000 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_SUPPORTED_ON 0x00000001 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_CURRENT 1:1 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_CURRENT_OFF 0x00000000 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_CURRENT_ON 0x00000001 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_PENDING 2:2 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_PENDING_OFF 0x00000000 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_PENDING_ON 0x00000001 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_0 */ +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_SUPPORTED 3:3 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_SUPPORTED_OFF 0x00000000 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_SUPPORTED_ON 0x00000001 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_CURRENT 4:4 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_CURRENT_OFF 0x00000000 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_CURRENT_ON 0x00000001 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_PENDING 5:5 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_PENDING_OFF 0x00000000 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_PENDING_ON 0x00000001 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_MISC_DEVICE_FLAGS_PAGE_0 */ +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_STATUS 6:6 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_STATUS_NOT_RUNNING 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_STATUS_RUNNING 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DEVICE_STATUS 7:7 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DEVICE_STATUS_ENABLED 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DEVICE_STATUS_DISABLED 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS 10:8 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS_NOT_RUNNING 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS_RUNNING 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS_TIMEOUT 0x00000002 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS_ERROR 0x00000003 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE 13:11 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_NONE 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_LOCAL_DEVICE 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_PEER_DEVICE 0x00000002 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_IN_BAND 0x00000003 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_OUT_OF_BAND 0x00000004 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON 16:14 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_NONE 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_CLIENT_OVERRIDE 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_TRUNK_LINK_FAILED 0x00000002 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_ACCESS_LINK_FAILED 0x00000003 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_UNSPECIFIED_FAILURE 0x00000004 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_RELOAD 17:17 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_RELOAD_NOT_PENDING 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_RELOAD_PENDING 0x00000001 + +/* Response to NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_GET */ +#define NV_MSGBOX_DATA_ACCESS_WP_MODE_GET_STATE 7:0 +#define NV_MSGBOX_DATA_ACCESS_WP_MODE_GET_STATE_DISABLED 0x0000005A +#define NV_MSGBOX_DATA_ACCESS_WP_MODE_GET_STATE_ENABLED 0x000000A5 + +/* + * Response to + * NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_RESET_REQUIRED + * NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_1 + */ +#define NV_MSGBOX_DATA_GPU_RESET_REQUIRED 0:0 +#define NV_MSGBOX_DATA_GPU_RESET_REQUIRED_OFF 0x00000000 +#define NV_MSGBOX_DATA_GPU_RESET_REQUIRED_ON 0x00000001 +#define NV_MSGBOX_DATA_GPU_DRAIN_AND_RESET_RECOMMENDED 1:1 +#define NV_MSGBOX_DATA_GPU_DRAIN_AND_RESET_RECOMMENDED_NO 0x00000000 +#define NV_MSGBOX_DATA_GPU_DRAIN_AND_RESET_RECOMMENDED_YES 0x00000001 + +/** + * Response to + * NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_PAGE_3 + */ +#define NV_MSGBOX_DATA_GET_CLOCK_FREQ_INFO_PAGE_3_CURRENT_PSTATE 3:0 +#define NV_MSGBOX_DATA_GET_CLOCK_FREQ_INFO_PAGE_3_CURRENT_PSTATE_INVALID 0x0000000F + +/** + * Response to + * NV_MSGBOX_CMD_ARG1_GET_CLOCK_THROTTLE_REASON + */ +#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON 31:0 +#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_NONE 0x00000000 +#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_SW_POWER_CAP 0x00000001 +#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_HW_SLOWDOWN 0x00000002 +#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_HW_THERMAL_SLOWDOWN 0x00000004 +#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_HW_POWER_BREAK_SLOWDOWN 0x00000008 +#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_SYNC_BOOST 0x00000010 +#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_SW_THERMAL_SLOWDOWN 0x00000020 + +/* + * Number of Nvlink data outputs (dataOut, extData) for + * NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK queries + */ +#define NV_MSGBOX_DATA_NVLINK_INFO_DATA_WIDTH 2 + +/* + * Encoding for Invalid data + */ +#define NV_MSGBOX_DATA_NVLINK_INFO_DATA_INVALID 0xFFFFFFFF + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_STATE_V1 */ +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V1__SIZE 32 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V1_DOWN 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V1_UP 0x00000001 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V1_INVALID 0x000000ff + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_STATE_V2 */ +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2(i) (3+(i)*4):(0+(i)*4) +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2__SIZE 8 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_OFF 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_SAFE 0x00000001 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_ACTIVE 0x00000002 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_ERROR 0x00000003 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_L1_LOW_POWER 0x00000004 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_NVLINK_DISABLED 0x00000005 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_INVALID 0x000000ff + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_BANDWIDTH (in Mps) */ +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_BANDWIDTH__SIZE 1 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_INVALID 0xFFFFFFFF +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_00000_MBPS 0 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_16000_MBPS 16000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_19200_MBPS 19200 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_20000_MBPS 20000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_25000_MBPS 25000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_25781_MBPS 25781 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_32000_MBPS 32000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_40000_MBPS 40000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_50000_MBPS 50000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_53125_MBPS 53125 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_SUBLINK_WIDTH */ +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_WIDTH(i) (5+(i)*6):(0+(i)*6) +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_WIDTH__SIZE 5 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_WIDTH_INVALID 0x000000ff +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH 2:0 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_0 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_1 0x00000001 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_2 0x00000002 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_4 0x00000003 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_8 0x00000004 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_INVALID 0x00000007 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH 5:3 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_0 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_1 0x00000001 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_2 0x00000002 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_4 0x00000003 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_8 0x00000004 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_INVALID 0x00000007 + +/* + * Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_TRAINING_ERR_STATE + * NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_RUNTIME_ERR_STATE + */ +#define NV_MSGBOX_DATA_NVLINK_INFO_ERROR_STATE__SIZE 32 +#define NV_MSGBOX_DATA_NVLINK_INFO_ERROR_STATE_INVALID 0x000000ff +#define NV_MSGBOX_DATA_NVLINK_INFO_ERROR_STATE_NO_ERROR 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_ERROR_STATE_ERROR 0x00000001 + +/* Respones to NV_MSGBOX_CMD_ARG2_REMAP_ROW_RAW_CNT_COMBINED */ +#define NV_MSGBOX_DATA_REMAP_ROW_UNCORR_CNT 10:0 +#define NV_MSGBOX_DATA_REMAP_ROW_UNCORR_CNT_EXCESS 11:11 +#define NV_MSGBOX_DATA_REMAP_ROW_UNCORR_CNT_EXCESS_FALSE 0 +#define NV_MSGBOX_DATA_REMAP_ROW_UNCORR_CNT_EXCESS_TRUE 1 +#define NV_MSGBOX_DATA_REMAP_ROW_CORR_CNT 22:12 +#define NV_MSGBOX_DATA_REMAP_ROW_CORR_CNT_EXCESS 23:23 +#define NV_MSGBOX_DATA_REMAP_ROW_CORR_CNT_EXCESS_FALSE 0 +#define NV_MSGBOX_DATA_REMAP_ROW_CORR_CNT_EXCESS_TRUE 1 + +/* Response to NV_MSGBOX_CMD_ARG1_REMAP_ROWS_STATE_FLAGS */ +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_FAILED_REMAPPING 0:0 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_FAILED_REMAPPING_FALSE 0 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_FAILED_REMAPPING_TRUE 1 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_PENDING_REMAPPING 1:1 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_PENDING_REMAPPING_FALSE 0 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_PENDING_REMAPPING_TRUE 1 + +/* Response to NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM */ +#define NV_MSGBOX_DATA_REMAP_ROW_HISTOGRAM_LOW_AVAILABILITY 15:0 +#define NV_MSGBOX_DATA_REMAP_ROW_HISTOGRAM_PARTIAL_AVAILABILITY 31:16 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_0 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED 2:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_UNKNOWN 0x00000000 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_2500_MTPS 0x00000001 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_5000_MTPS 0x00000002 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_8000_MTPS 0x00000003 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_16000_MTPS 0x00000004 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_32000_MTPS 0x00000005 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH 6:4 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_UNKNOWN 0x00000000 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X1 0x00000001 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X2 0x00000002 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X4 0x00000003 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X8 0x00000004 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X16 0x00000005 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X32 0x00000006 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_NONFATAL_ERROR_COUNT 15:8 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_FATAL_ERROR_COUNT 23:16 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_UNSUPP_REQ_COUNT 31:24 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_1 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_1_L0_TO_RECOVERY_COUNT 31:0 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_2 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_2_REPLAY_ROLLOVER_COUNT 15:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_2_NAKS_RCVD_COUNT 31:16 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_3 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED 2:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_UNKNOWN 0x00000000 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_2500_MTPS 0x00000001 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_5000_MTPS 0x00000002 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_8000_MTPS 0x00000003 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_16000_MTPS 0x00000004 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_32000_MTPS 0x00000005 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_RESERVED 2:0 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_4 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_4_TX_COUNT 31:0 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_6 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE 4:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_DETECT 0x00000000 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_POLLING 0x00000001 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_CONFIGURATION 0x00000002 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_RECOVERY 0x00000003 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_RECOVERY_EQZN 0x00000004 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_L0 0x00000005 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_L0S 0x00000006 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_L1 0x00000007 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_L1_PLL_PD 0x00000008 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_L2 0x00000009 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_L1_CPM 0x0000000a +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_L1_1 0x0000000b +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_L1_2 0x0000000c +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_HOT_RESET 0x0000000d +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_LOOPBACK 0x0000000e +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_DISABLED 0x0000000f +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_LINK_DOWN 0x00000010 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_LINK_READY 0x00000011 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_LANES_IN_SLEEP 0x00000012 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_6_LTSSM_STATE_ILLEGAL 0x0000001f + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_8 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_8_EQ_TX_LOCAL_PRESET 3:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_8_EQ_TX_LOCAL_USE_PRESET 4:4 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_8_EQ_TX_LOCAL_FS 10:5 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_8_EQ_TX_LOCAL_LF 16:11 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_9 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_9_EQ_RX_REMOTE_PRESET 3:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_9_EQ_RX_REMOTE_USE_PRESET 4:4 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_9_EQ_RX_REMOTE_FS 10:5 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_9_EQ_RX_REMOTE_LF 16:11 + +/* + * Input for NV_MSGBOX_CMD_OPCODE_GPU_PERFORMANCE_MONITORING. Value is valid + * only if Arg2 != GPM_PARTITION_AGGREGATE and Arg2.Bit7 == 1 + */ +#define NV_MSGBOX_DATA_GPM_COMPUTE_INSTANCE_INDEX 15:8 + +/* + * The following three fields correspond to the data which is interpreted + * differently depending on GPM Metric specified in Arg1. + */ +#define NV_MSGBOX_DATA_GPM_NVLINK_INDEX 7:0 +#define NV_MSGBOX_DATA_GPM_NVDEC_INSTANCE 7:0 +#define NV_MSGBOX_DATA_GPM_NVJPG_INSTANCE 7:0 + +/* MSGBOX Extended Data Register */ +#define NV_MSGBOX_EXT_DATA_REG 31:0 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_0 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_0_CORRECTABLE_ERROR_COUNT 15:0 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_1 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_1_REPLAY_COUNT 31:0 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_2 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_2_NAKS_SENT_COUNT 15:0 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_4 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_4_RX_COUNT 31:0 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_8 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_8_EQ_TX_LOCAL_PRECUR 5:0 +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_8_EQ_TX_LOCAL_MAINCUR 11:6 +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_8_EQ_TX_LOCAL_POSTCUR 17:12 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_9 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_9_EQ_RX_REMOTE_PRECUR 5:0 +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_9_EQ_RX_REMOTE_MAINCUR 11:6 +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_9_EQ_RX_REMOTE_POSTCUR 17:12 + +/* Response to NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM */ +#define NV_MSGBOX_EXT_DATA_REMAP_ROW_HISTOGRAM_MAX_AVAILABILITY 31:16 +#define NV_MSGBOX_EXT_DATA_REMAP_ROW_HISTOGRAM_HIGH_AVAILABILITY 15:0 + +/* SysId info per type sizes */ +#define NV_MSGBOX_SYSID_DATA_SIZE_BOARD_PART_NUM_V1 24 +#define NV_MSGBOX_SYSID_DATA_SIZE_OEM_INFO_V1 24 +#define NV_MSGBOX_SYSID_DATA_SIZE_SERIAL_NUM_V1 16 +#define NV_MSGBOX_SYSID_DATA_SIZE_MARKETING_NAME_V1 24 +#define NV_MSGBOX_SYSID_DATA_SIZE_GPU_PART_NUM_V1 16 +#define NV_MSGBOX_SYSID_DATA_SIZE_MEMORY_VENDOR_V1 1 +#define NV_MSGBOX_SYSID_DATA_SIZE_MEMORY_PART_NUM_V1 20 +#define NV_MSGBOX_SYSID_DATA_SIZE_BUILD_DATE_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_FIRMWARE_VER_V1 14 +#define NV_MSGBOX_SYSID_DATA_SIZE_VENDOR_ID_V1 2 +#define NV_MSGBOX_SYSID_DATA_SIZE_DEV_ID_V1 2 +#define NV_MSGBOX_SYSID_DATA_SIZE_SUB_VENDOR_ID_V1 2 +#define NV_MSGBOX_SYSID_DATA_SIZE_SUB_ID_V1 2 +#define NV_MSGBOX_SYSID_DATA_SIZE_GPU_GUID_V1 16 +#define NV_MSGBOX_SYSID_DATA_SIZE_INFOROM_VER_V1 16 +#define NV_MSGBOX_SYSID_DATA_SIZE_PRODUCT_LENGTH_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_PRODUCT_WIDTH_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_PRODUCT_HEIGHT_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_PCIE_SPEED_V1 1 +#define NV_MSGBOX_SYSID_DATA_SIZE_PCIE_WIDTH_V1 1 +#define NV_MSGBOX_SYSID_DATA_SIZE_TGP_LIMIT_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_FRU_PART_NUMBER_V1 20 +#define NV_MSGBOX_SYSID_DATA_SIZE_MODULE_POWER_LIMIT_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_MAX_DRAM_CAPACITY_V1 4 + +/*! + * Response to NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_CLK + * 16-bit frequency in MHz + */ +#define NV_MSGBOX_DATA_POWER_HINT_INFO_CLK_GR_MIN 15:0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_CLK_GR_MAX 31:16 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_CLK_MEM_MIN 15:0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_CLK_MEM_MAX 31:16 + +/*! + * Response to NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_TEMP + * 16-bit frequency in MHz + */ +#define NV_MSGBOX_DATA_POWER_HINT_INFO_TEMP_MIN 15:0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_TEMP_MAX 31:16 + +/*! + * Response to _ARG1_GET_POWER_HINT_INFO_PROFILES, 1 page consists of data and + * ext data, then we can support up to 256 profiles + * Bit index represents profile ID + */ +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_PERF 0:0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_PERF_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_PERF_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_HIGH_K 1:1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_HIGH_K_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_HIGH_K_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_LOW_K 2:2 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_LOW_K_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_LOW_K_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_HMMA 3:3 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_HMMA_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_HMMA_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_IMMA 4:4 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_IMMA_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_IMMA_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_SGEMM 5:5 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_SGEMM_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_SGEMM_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_TRANSFORMER 6:6 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_TRANSFORMER_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_TRANSFORMER_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_0 7:7 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_0_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_0_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_1 8:8 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_1_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_1_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_2 9:9 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_2_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_2_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_3 10:10 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_3_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_3_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_4 11:11 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_4_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_4_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_5 12:12 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_5_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_5_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_6 13:13 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_6_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_6_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_7 14:14 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_7_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_7_AVAILABLE 1 + +#define NV_MSGBOX_DATA_DYN_SYS_INFO_SIZE_DRIVER_VERSION 64 + +/* Event types */ +typedef enum +{ + NV_MSGBOX_EVENT_TYPE_SERVER_RESTART_COLD = 0, + NV_MSGBOX_EVENT_TYPE_GPU_RESET_REQUIRED, + NV_MSGBOX_EVENT_TYPE_DRIVER_ERROR_MESSAGE_OLDEST, + NV_MSGBOX_EVENT_TYPE_TGP_LIMIT_SET_SUCCESS, + NV_MSGBOX_EVENT_TYPE_CLOCK_LIMIT_SET_SUCCESS, + NV_MSGBOX_EVENT_TYPE_ECC_TOGGLE_SUCCESS, + NV_MSGBOX_EVENT_TYPE_MIG_TOGGLE_SUCCESS, + NV_MSGBOX_EVENT_TYPE_SERVER_RESTART_WARM, + NV_MSGBOX_EVENT_TYPE_DRIVER_ERROR_MESSAGE_NEW, + NV_MSGBOX_EVENT_TYPE_MODULE_LIMIT_SET_SUCCESS, + NV_MSGBOX_NUM_EVENTS, /* insert new event types before this line */ +} NvMsgboxEventType; + +/* Legacy event names for compatipility */ +#define NV_MSGBOX_EVENT_TYPE_SERVER_RESTART NV_MSGBOX_EVENT_TYPE_SERVER_RESTART_COLD +#define NV_MSGBOX_EVENT_TYPE_DRIVER_ERROR_MESSAGE NV_MSGBOX_EVENT_TYPE_DRIVER_ERROR_MESSAGE_OLDEST + +/* Bit mask of all defined events */ +#define NV_MSGBOX_EVENT_TYPE__ALL (NVBIT(NV_MSGBOX_NUM_EVENTS) - 1) + +/* ----------------------- Set Master Capabilities ---------------------------- */ + +// +// SET_MASTER_CAPS +// +// Descripiton: +// Set/declare SMBPBI Master capabilities. +// +// Common Encoding: +// 7 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | CLASS | ARG1 | CAP_INDEX | ARG2 +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +// The SMBPBI Master capabilities are logically grouped into classes (specified +// in arg1). Capabilities are sent 32 at a time (dword index stored in arg2) +// per-class (typically once during system-initialization). +// +// Classes: +// _GPU_REQ - Class of capability bits corresponding 1:1 with the GPU- +// Request opcodes the master supports. +// +#define NV_MSGBOX_CMD_SET_MASTER_CAPS_ARG1_CLASS 7:0 +#define NV_MSGBOX_CMD_SET_MASTER_CAPS_ARG1_CLASS_GPU_REQ 0x00000000 + +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_COUNT 1 + +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0 31:0 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_CPU_PCONTROL 0:0 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_CPU_PCONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_CPU_PCONTROL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_SYS_PCONTROL 1:1 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_SYS_PCONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_SYS_PCONTROL_AVAILABLE 0x00000001 + +/* ------------------ GPU Performance Control (PCONTROL) ---------------------- */ + +// +// GPU_PCONTROL arg1, arg2, and data encoding (fields are relative to zero, not +// relative to their position in the COMMAND/DATA registers). +// + +// +// ARG1/ARG2 Encoding +// +// 7 3 2 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | TARGET | ACT | ARG1 |0 0 0 0 0 0 0 0| ARG2 (reserved) +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION 2:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION_GET_INFO 0x00000000 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION_GET_LIMIT 0x00000001 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION_SET_LIMIT 0x00000002 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION_GET_STATUS 0x00000003 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_TARGET 7:3 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_TARGET_VPSTATE 0x00000000 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_TARGET_POWER_SMBIDX 0x00000001 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG2_RSVD 7:0 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_SET_ACTION(arg1, action) \ + (arg1) = FLD_SET_DRF(_MSGBOX_CMD, _GPU_PCONTROL_ARG1, _ACTION, \ + action, (arg1)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_GET_ACTION(arg1) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_ARG1, _ACTION, (arg1)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_SET_TARGET(arg1, target) \ + (arg1) = FLD_SET_DRF(_MSGBOX_CMD, _GPU_PCONTROL_ARG1, _TARGET, \ + target, (arg1)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_GET_TARGET(arg1) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_ARG1, _TARGET, (arg1)) + +// +// Getting GPU vPstate Information +// +// Inputs: +// ARG1 - {TARGET=VPSTATE, ACTION=GET_INFO} +// ARG2 - Unused/reserved (must be zero) +// +// Outputs: +// DATA - Current min/max range of externally available vpstates +// (vPmin=fastest, vPmax=slowest) +// +// Encoding: +// 31 16 15 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0| MIN | MAX | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_MAX 7:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_MIN 15:8 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_RSVD 31:16 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_SET_MAX(data, max) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_GET_INFO_MAX, (max), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_GET_MAX(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_GET_INFO_MAX, (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_SET_MIN(data, min) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_GET_INFO_MIN, (min), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_GET_MIN(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_GET_INFO_MIN, (data)) + +// +// Getting GPU vPstate Limits +// +// Inputs: +// ARG1 - {TARGET=VPSTATE, ACTION=GET_LIMIT} +// ARG2 - Unused/reserved (must be zero) +// +// Outputs: +// DATA - Last requested vpstate limit/value, zero when no limit has been set. +// +// Encoding: +// 31 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0| VALUE | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_LIMIT_VALUE 7:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_LIMIT_RSVD 31:8 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_LIMIT_SET_VALUE(data, value) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_GET_LIMIT_VALUE, (value), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_LIMIT_GET_VALUE(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_GET_LIMIT_VALUE, (data)) + +// +// Setting GPU vPstate Limits: +// +// Inputs: +// ARG1 - {TARGET=VPSTATE, ACTION=SET_LIMIT} +// ARG2 - Unused/reserved (must be zero) +// DATA - Desired vpstate limit/value between vPmin and vPmax (inclusive), +// zero to clear active limits. +// Outputs: +// none +// +// Encoding: +// 31 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0| VALUE | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_VALUE 7:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_VALUE_CLEAR 0x00000000 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_VALUE_INIT 0x00000000 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_RSVD 31:8 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_SET_VALUE(data, limit) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_SET_LIMIT_VALUE, (limit), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_GET_VALUE(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_SET_LIMIT_VALUE, (data)) + +// +// Getting GPU vPstate Status +// +// Inputs: +// ARG1 - {TARGET=VPSTATE, ACTION=GET_STATUS} +// ARG2 - Unused/reserved (must be zero) +// +// Outputs: +// DATA - Current vPstate +// +// Encoding: +// 31 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0| CURRENT | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_STATUS_CURRENT 7:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_STATUS_RSVD 31:8 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_STATUS_SET_CURRENT(data, curr) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_GET_STATUS_CURRENT, (curr), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_STATUS_GET_CURRENT(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_GET_STATUS_CURRENT, (data)) + + +/* ------------------------- GPU System Control ------------------------------- */ + +// +// GPU_SYSCONTROL +// +// Description: +// Command sent to the GPU to set/remove a system-imposed operating limit +// on a specific system-parameter. +// +// Common Encoding: +// 7 2 1 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | TARGET |ACT| ARG1 |0 0 0 0 0 0 0 0| ARG2 (reserved) +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +// Notes: +// - TARGET accepts NV0000_CTRL_SYSTEM_PARAM_* definitions +// - arg1, arg2, and data encoding (fields are relative to zero, not relative +// to their position in the DATA_OUT register). +// +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_ACTION 1:0 +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_ACTION_CLEAR_LIMIT 0x00000000 +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_ACTION_SET_LIMIT 0x00000001 +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_TARGET 7:2 +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG2_RSVD 7:0 + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_SET_ACTION(arg1, action) \ + (arg1) = FLD_SET_DRF(_MSGBOX_CMD, _GPU_SYSCONTROL_ARG1, _ACTION, \ + action, (arg1)) + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_GET_ACTION(arg1) \ + DRF_VAL(_MSGBOX_CMD, _GPU_SYSCONTROL_ARG1, _ACTION, (arg1)) + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_SET_TARGET(arg1, target) \ + (arg1) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_SYSCONTROL_ARG1, _TARGET, \ + target, (arg1)) + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_GET_TARGET(arg1) \ + DRF_VAL(_MSGBOX_CMD, _GPU_SYSCONTROL_ARG1, _TARGET, (arg1)) + +// +// Setting a System-Parameter Operating-Limit: +// +// Inputs: +// ARG1 - {TARGET=NV_MSGBOX_SYS_PARAM_*, ACTION=SET_LIMIT} +// ARG2 - Unused/reserved (must be zero) +// DATA - Desired limit/value. +// +// Outputs: +// none +// +// Encoding: +// 31 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// | VALUE | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_DATA_SET_LIMIT_VALUE 31:0 + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_DATA_SET_LIMIT_SET_VALUE(data, limit) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_SYSCONTROL_DATA, \ + _SET_LIMIT_VALUE, (limit), (data)) + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_DATA_PARAM_SET_LIMIT_GET_VALUE(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_SYSCONTROL_DATA, _SET_LIMIT_VALUE, (data)) + +// +// Host side scratch memory buffer +// This memory buffer is allocated by the host CPU in its system memory +// and is used for passing additional arguments and results, that don't +// fit in the command/status and data registers due to their size. +// This memory can be accessed by the BMC SMBus master through +// dedicated requests. +// +// Page size is various on different SMBPBI servers. +// Capability dword[2] bit 12 shows size in bytes +// +// _1024 --> 0xFF dwords +// _256 --> 0x40 dowrds +// +#define NV_MSGBOX_SCRATCH_PAGE_SIZE_1024_D 0xFF +#define NV_MSGBOX_SCRATCH_PAGE_SIZE_256_D 0x40 + +// +// TODO Once switch soc/pmu/oobtest remove the reference to this, this will be +// removed. +// +#define NV_MSGBOX_SCRATCH_PAGE_SIZE 1024 +#define NV_MSGBOX_SCRATCH_NUM_PAGES_P 2 // expressed as a power of 2 + // must be >= 2 + +// This cap code plugs into NV_MSGBOX_DATA_CAP_2_NUM_SCRATCH_BANKS +#define NV_MSGBOX_SCRATCH_NUM_PAGES_CAP_CODE \ + (NV_MSGBOX_SCRATCH_NUM_PAGES_P > 0 ? \ + NV_MSGBOX_SCRATCH_NUM_PAGES_P - 1 : 0) + +#define NV_MSGBOX_SCRATCH_NUM_PAGES (1 << NV_MSGBOX_SCRATCH_NUM_PAGES_P) +#define NV_MSGBOX_SCRATCH_BUFFER_SIZE (NV_MSGBOX_SCRATCH_NUM_PAGES \ + * NV_MSGBOX_SCRATCH_PAGE_SIZE) + +// Auxiliary definitions related to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_STATE_SPEED +#define NV_MSGBOX_NVLINK_STATE_BITS_PER_NVLINK 4 +#define NV_MSGBOX_NVLINK_STATE_NVLINKS_PER_REGISTER (32 / NV_MSGBOX_NVLINK_STATE_BITS_PER_NVLINK) +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_VOLTA 6 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_TURING 2 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_GA100 3 +#define NV_MSGBOX_NVLINK_STATE_NUM_LINKS_PER_NVLIPT_NVL30 4 +#define NV_MSGBOX_NVLINK_SUBLINK_WIDTH_MAX_NVL30 4 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_GA100 \ + (NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_GA100 * \ + NV_MSGBOX_NVLINK_STATE_NUM_LINKS_PER_NVLIPT_NVL30) +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_GA10X 1 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_GA10X \ + (NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_GA10X * \ + NV_MSGBOX_NVLINK_STATE_NUM_LINKS_PER_NVLIPT_NVL30) +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_LR10 9 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_LR10 \ + (NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_LR10 * \ + NV_MSGBOX_NVLINK_STATE_NUM_LINKS_PER_NVLIPT_NVL30) +#define NV_MSGBOX_NVLINK_STATE_NUM_REGISTERS(arch) \ + ((NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS##arch + NV_MSGBOX_NVLINK_STATE_NVLINKS_PER_REGISTER - 1) \ + / NV_MSGBOX_NVLINK_STATE_NVLINKS_PER_REGISTER) + +// Auxiliary definitions related to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_* +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_DATA_TX 0x00000000 +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_DATA_RX 0x00000001 +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_RAW_TX 0x00000002 +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_RAW_RX 0x00000003 +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_MAX 0x00000004 + +#if !defined(NV_MSGBOX_NO_PARAM_STRUCTS) +// +// Parameter structures for asynchronous requests +// passed in scratch space +// + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_GET, + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_SET, + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_MODULE_LIMIT_CONTROL_GET and + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_MODULE_LIMIT_CONTROL_SET + */ +typedef struct +{ + NvU32 flags; + /*! + * If flags:_PERSIST is _ON, make limitCurrInput persistent, + * surviving the driver reload and the system reboot. + * + * If flags:_CLEAR is _ON, it will clear the TGP limit. The + * persistence still depends on persist flag. + */ +#define NV_MSGBOX_PMGR_PWR_POWER_LIMIT_CONTROL_PARAMS_FLAGS_PERSIST 0:0 +#define NV_MSGBOX_PMGR_PWR_POWER_LIMIT_CONTROL_PARAMS_FLAGS_PERSIST_OFF 0x00000000 +#define NV_MSGBOX_PMGR_PWR_POWER_LIMIT_CONTROL_PARAMS_FLAGS_PERSIST_ON 0x00000001 +#define NV_MSGBOX_PMGR_PWR_POWER_LIMIT_CONTROL_PARAMS_FLAGS_CLEAR 1:1 +#define NV_MSGBOX_PMGR_PWR_POWER_LIMIT_CONTROL_PARAMS_FLAGS_CLEAR_OFF 0x00000000 +#define NV_MSGBOX_PMGR_PWR_POWER_LIMIT_CONTROL_PARAMS_FLAGS_CLEAR_ON 0x00000001 + + /*! + * Current total GPU power limit value to enforce, requested by the + * SMBPBI client, expressed in milliwatts. + * Must always be within range imposed by the current policy. + * 0xffffffff is returned, if the limit has not been set by the SMBPBI client. + */ + NvU32 limitCurrInput; + + /*! + * Currently arbitrated total GPU power limit value after taking + * into account limits, requested by all clients, expressed + * in milliwatts. + */ + NvU32 limitCurrOutput; +} NV_MSGBOX_PMGR_PWR_POWER_LIMIT_CONTROL_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_INFO_GET and + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_MODULE_LIMIT_INFO_GET + */ +typedef struct +{ + /*! + * Current total GPU power limit lower and upper bounds and the + * default setting, expressed in milliwatts. + * These constraints must be observed, when the limit + * is being set with either + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_SET or + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_MODULE_LIMIT_CONTROL_SET. + */ + NvU32 limitMin; + NvU32 limitMax; + NvU32 limitDefault; +} NV_MSGBOX_PMGR_PWR_POWER_LIMIT_INFO_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_COUNT_GET + */ +typedef struct +{ + NvU32 fanCount; //<! number of fans on the board +} NV_MSGBOX_THERMAL_FAN_V1_COUNT_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_INFO_GET + */ +typedef struct +{ + NvU32 fanIndex; //<! fan ID to be filled by the caller + NvU32 fanProperties; //<! fan properties, detailed below + +//! Fan level metric +#define NV_MSGBOX_FAN_V1_PROP_LEVEL_METRIC 0:0 +#define NV_MSGBOX_FAN_V1_PROP_LEVEL_METRIC_PWM 0x00000000 +#define NV_MSGBOX_FAN_V1_PROP_LEVEL_METRIC_RPM 0x00000001 + +//! Fan tachometer availability +#define NV_MSGBOX_FAN_V1_PROP_TACH 1:1 +#define NV_MSGBOX_FAN_V1_PROP_TACH_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_FAN_V1_PROP_TACH_AVAILABLE 0x00000001 + +//! Fan tachometer bounds (min/max) availability +#define NV_MSGBOX_FAN_V1_PROP_TACH_BOUNDS 2:2 +#define NV_MSGBOX_FAN_V1_PROP_TACH_BOUNDS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_FAN_V1_PROP_TACH_BOUNDS_AVAILABLE 0x00000001 + + NvU16 fanLevelMin; //<! minimum fan level + NvU16 fanLevelMax; //<! maximum fan level + NvU16 fanTachMin; //<! tachometer minimum + NvU16 fanTachMax; //<! tachometer maximum +} NV_MSGBOX_THERMAL_FAN_V1_INFO_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_STATUS_GET + */ +typedef struct +{ + NvU32 fanIndex; //<! fan ID to be filled by the caller + /*! + * Current fan level and speed + */ + NvU8 levelCurr; //<! current fan level, 0 if n/a + NvU16 speedCurr; //<! current fan speed (RPM), 0 if no tach + +} NV_MSGBOX_THERMAL_FAN_V1_STATUS_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OVERCLOCKING_LIMIT_CONTROL + */ +typedef struct +{ + NvU32 limitType; //<! Argument representing VPSTATE/CLOCK + +//! CLOCK/VPSTATE types - add types below +#define NV_MSGBOX_OVERCLOCKING_LIMIT_TYPE_MAX_CUSTOMER_BOOST_VPSTATE 0x00000001 + + NvU32 maxClkLimitMHz; //<! Clock limit + +} NV_MSGBOX_OVERCLOCKING_LIMIT_CONTROL_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_ENERGY_COUNTER_STATUS_GET + */ +typedef struct +{ + NV_DECLARE_ALIGNED(NvU64 energyCountermJ, 8); //<! Energy counter value + +} NV_MSGBOX_ENERGY_COUNTER_STATUS_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_VIOLATION_COUNTERS_STATUS_GET + */ +typedef struct +{ + NvU32 policyMask; + /*! + * Argument representing policies whose violation times + * are being requested/provided + */ + +//! NVBIT OFFSET <> POLICY - add types below +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_ALL_SUPPORTED_POLICIES 0xFFFFFFFF //-1 represents All Supported policies +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_HW_VIOLATION 0 +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_SW_GLOBAL_VIOLATION 1 +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_SW_POWER_VIOLATION 2 +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_SW_THERMAL_VIOLATION 3 + + NV_DECLARE_ALIGNED(NvU64 violationTime[32], 8); //<! Violation counter value + +} NV_MSGBOX_VIOLATION_COUNTERS_STATUS_PARAMS; + +//! Number of clock domains to track for gathering utilization data. +#define NV_MSGBOX_UTILIZATION_RATE_CLOCK_DOMAIN_COUNT 2 + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_UTILIZATION_RATE_GET + */ +typedef struct +{ + NvU32 gpuUtil; //<! Percent of sample that the Graphics/SM unit remains busy. + NvU32 memUtil; //<! Percent of sample that the memory unit remains busy. + +} NV_MSGBOX_UTILIZATION_RATE_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OOB_CLOCK_LIMIT_SET + * + * Masks of NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_<XYZ>. + */ +typedef struct +{ + NvU32 flags; + /*! + * If flags: _CLEAR is _ON, it will clear the limits. Otherwise, + * set the clock limits. + * + * If flags: _PERSIST is _ON, set/clear limits persistently, + * surviving the driver reload and the system reboot. + * + */ +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_PERSIST 0:0 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_PERSIST_OFF 0x00000000 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_PERSIST_ON 0x00000001 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_CLEAR 1:1 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_CLEAR_OFF 0x00000000 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_CLEAR_ON 0x00000001 + + /*! + * Graphics clock limit in MHz + */ + NvU32 clkMinFreqMHz; + NvU32 clkMaxFreqMHz; +} NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_MEMORY_CAPACITY_UTILIZATION_GET + */ +typedef struct +{ + NvU32 capacityReservedMiB; //<! Device memory (in MiB) reserved for system use. + NvU32 capacityUsedMiB; //<! Allocated device memory (in MiB). +} NV_MSGBOX_MEMORY_CAPACITY_UTILIZATION_PARAMS; + +/* + * Parameters for NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_TEST_MESSAGE_SEND + */ +#define NV_MSGBOX_TEST_MESSAGE_SEND_STRING_SIZE_MAX 32 +/* + * If the message starts with this character, the message + * will not be sent to the kernel. + */ +#define NV_MSGBOX_TEST_MESSAGE_SEND_SILENT_CHAR '@' +typedef struct +{ + NvU8 string[NV_MSGBOX_TEST_MESSAGE_SEND_STRING_SIZE_MAX]; +} NV_MSGBOX_TEST_MESSAGE_SEND_PARAMS; + +/* + * Individual request structure definition for bundled requests + */ +typedef struct +{ + NvU32 cmdStatus; //<! Command/Status + NvU32 dataIn; //<! Data In + NvU32 dataOut; //<! Data Out + NvU32 extData; //<! Extended Data +} NV_MSGBOX_IND_REQUEST; + +/*! + * Driver Event Message record + */ +typedef struct +{ + NvU8 recordSize; //<! in 32-bit words + NvU8 xidId; //<! event type Id (Xid) + NvU8 flags; //<! bit 0: some DEMs after this one + //<! have been lost + //<! bit 1: the text message has been + //<! truncated + NvU8 xidIdExt; //<! event type Id (Xid) extension + NvU32 seqNumber; //<! record sequential number + NvU32 timeStamp; //<! seconds since the epoch UTC + + /* + * Some Windows builds include this file, and older Microsoft + * compilers won't accept a zero size array in this context: + * "C4200 nonstandard extension used: zero-sized array in struct/union", + * despite this being a legitimate C99. + * We need to find a way to fix this, if we ever decide to + * implement DEM/Xid logging for Windows. + */ + NvU8 textMessage[]; //<! NUL terminated + +} NV_MSGBOX_DEM_RECORD; +#define NV_MSGBOX_DEM_RECORD_FLAGS_OVFL 0:0 +#define NV_MSGBOX_DEM_RECORD_FLAGS_OVFL_NOT_SET 0x00000000 +#define NV_MSGBOX_DEM_RECORD_FLAGS_OVFL_SET 0x00000001 +#define NV_MSGBOX_DEM_RECORD_FLAGS_TRUNC 1:1 +#define NV_MSGBOX_DEM_RECORD_FLAGS_TRUNC_NOT_SET 0x00000000 +#define NV_MSGBOX_DEM_RECORD_FLAGS_TRUNC_SET 0x00000001 + +/*! + * Result Disposition Rule word layout + */ +#define NV_MSGBOX_DISP_RULE_REQ_IDX 2:0 +#define NV_MSGBOX_DISP_RULE_SRC_REG 4:3 +#define NV_MSGBOX_DISP_RULE_SRC_REG_STATUS 0x00000000 +#define NV_MSGBOX_DISP_RULE_SRC_REG_DATA 0x00000001 +#define NV_MSGBOX_DISP_RULE_SRC_REG_EXT_DATA 0x00000002 +#define NV_MSGBOX_DISP_RULE_SRC_REG_MAX 0x00000002 +#define NV_MSGBOX_DISP_RULE_SRC_RIGHT_BIT 9:5 +#define NV_MSGBOX_DISP_RULE_SRC_WIDTH 14:10 +#define NV_MSGBOX_DISP_RULE_DST_REG 16:15 +#define NV_MSGBOX_DISP_RULE_DST_REG_STATUS 0x00000000 +#define NV_MSGBOX_DISP_RULE_DST_REG_DATA 0x00000001 +#define NV_MSGBOX_DISP_RULE_DST_REG_EXT_DATA 0x00000002 +#define NV_MSGBOX_DISP_RULE_DST_REG_MAX 0x00000002 +#define NV_MSGBOX_DISP_RULE_DST_RIGHT_BIT 21:17 + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_CLOCK_LIMIT_GET + * Masks of NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_<XYZ>. + */ +typedef struct { + /*! + * Graphics clock limit in MHz + * if there is no cap, return + * NV_MSGBOX_CLOCK_LIMIT_GET_PARAMS_CLOCK_LIMIT_NOT_SET + */ + NvU16 oobClientMin; + NvU16 oobClientMax; + NvU16 enforcedMin; + NvU16 enforcedMax; +#define NV_MSGBOX_CLOCK_LIMIT_GET_PARAMS_CLOCK_LIMIT_NOT_SET 0xffff +} NV_MSGBOX_CLOCK_LIMIT_GET_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_DEVICE_MODE_CONTROL + */ +typedef struct +{ + NvU16 modeType; //<! Mode type +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_MODE_TYPE_ECC 0x00000000 +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_MODE_TYPE_MIG 0x00000001 +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_MODE_TYPE_DISABLE 0x00000002 + + NvU16 flags; //<! flags +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_FLAGS_CONTROL 0:0 +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_FLAGS_CONTROL_CLEAR 0x00000000 +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_FLAGS_CONTROL_SET 0x00000001 + +} NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS; + +// +// This is the maximum number of curve points this API will be able to carry. +// With the understanding that the maximum number of points currently +// implemented in the RM is 3, we are leaving 2 additional slots as a room +// for possible future growth, so that we don't have to spin the API definition +// in case we ever want to implement more than 3 points. +// +#define NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_MAX_POINTS 5 + +typedef struct +{ + NvS16 temperature; //<! degrees C, signed 16 bit value + + union { + NvU16 level; //<! Fan PWM setting in the range 0..100 + //<! 0: no rotation + //<! 100: full speed + NvU16 rpmValue; //<! tach speed in RPM + } fanSetting; +} NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINT; + +#define NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_GET 0 +#define NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_GET_DEFAULT 1 +#define NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_SET 2 +typedef struct +{ + NvU8 action; //<! [in] + //<! NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_GET: + //<! get currently active point values + //<! NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_GET_DEFAULT: + //<! get VBIOS default point values + //<! NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_SET: + //<! set new active point values + + NvU8 fanIndex; //<! [in] + + NvU8 numEffectivePoints; //<! [out] for fanGetCurvePoints. This is the + //<! number of points actually implemented + //<! and filled in the array below + //<! [in] for fanSetCurvePoints. This is how + //<! many points the caller is sending to + //<! the API. Must not exceed what was + //<! returned by fanGetCurvePoints. + + NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINT + curvePoints[NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_MAX_POINTS]; + //<! [out] for fanGetCurvePoints. + //<! [in] for fanSetCurvePoints. +} NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_PARAMS; + +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_DMMA_PERF 0 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_DMMA_HIGH_K 1 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_DMMA_LOW_K 2 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_HMMA 3 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_IMMA 4 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_SGEMM 5 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_TRANSFORMER 6 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_0 7 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_1 8 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_2 9 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_3 10 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_4 11 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_5 12 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_6 13 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_7 14 +/*! + * This param is used for NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POWER_HINT_GET + */ +typedef struct +{ + /*! + * [out] Power Hint in mW + */ + NvU32 powermW; + /*! + * [in] Graphics Clock in MHz + */ + NvU16 grClkMHz; + /*! + * [in] Memory Clock + */ + NvU16 memClkMHz; + /*! + * [in] GPU Temperature + */ + NvS16 temperature; + /*! + * [in] Profile ID + */ + NvU8 profile; + // new param starting from here. +} NV_MSGBOX_POWER_HINT_PARAMS; + +/*! + * This param is used for NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_CONFIGURE_PROGRAMMABLE_EDPP + */ +typedef struct +{ + /*! + * [out] Power Hint in mW + */ + NvU32 data; +} NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS; + +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA 31:0 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_OPERATION 1:0 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_OPERATION_GET 0x0 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_OPERATION_SET 0x1 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_OPERATION_CLEAR 0x2 + +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_MODE 3:2 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_MODE_PERSISTENT 0x0 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_MODE_ONESHOT 0x1 + +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_LIMIT_CURRENT 10:4 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_LIMIT_RATED 17:11 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_LIMIT_MAXIMUM 24:18 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_LIMIT_MINIMUM 31:25 + +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_LIMIT_NOT_SET 0x7F + +/*! + * @brief Union of all possible parameter struct. Used to determine the maximum + * amount of space parameter blocks can take. + */ +typedef union { + NV_MSGBOX_PMGR_PWR_POWER_LIMIT_CONTROL_PARAMS tgpLimitControl; + NV_MSGBOX_THERMAL_FAN_V1_COUNT_PARAMS fanCountV1Get; + NV_MSGBOX_THERMAL_FAN_V1_INFO_PARAMS fanCountV1Info; + NV_MSGBOX_THERMAL_FAN_V1_STATUS_PARAMS fanCountV1Status; + NV_MSGBOX_OVERCLOCKING_LIMIT_CONTROL_PARAMS overclockingLimitControl; + NV_MSGBOX_ENERGY_COUNTER_STATUS_PARAMS energyCounterStatus; + NV_MSGBOX_VIOLATION_COUNTERS_STATUS_PARAMS violationCountersStatus; + NV_MSGBOX_UTILIZATION_RATE_PARAMS utilizationRate; + NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS oobClockLimitCtrlParams; + NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS deviceModeControlParams; + NV_MSGBOX_TEST_MESSAGE_SEND_PARAMS testMessageSend; + NV_MSGBOX_CLOCK_LIMIT_GET_PARAMS clockLimitGet; + NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_PARAMS fanCurvePointsV3; + NV_MSGBOX_POWER_HINT_PARAMS powerHintParams; + NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS programmableEdppParams; +} NV_MSGBOX_ASYNC_REQ_PARAMS_UNION; + +#endif // !NV_MSGBOX_NO_PARAM_STRUCTS + +/* Utility command constructor macros */ + +#define NV_MSGBOX_CMD(opcode, arg1, arg2) \ + ( \ + DRF_DEF(_MSGBOX, _CMD, _OPCODE, opcode) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1, (arg1)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2, (arg2)) | \ + DRF_DEF(_MSGBOX, _CMD, _STATUS, _NULL) | \ + DRF_DEF(_MSGBOX, _CMD, _RSVD, _INIT) | \ + DRF_DEF(_MSGBOX, _CMD, _INTR, _PENDING) \ + ) +#define NV_MSGBOX_CMD_NULL_CMD() NV_MSGBOX_CMD(_NULL_CMD, 0, 0) +#define NV_MSGBOX_CMD_GET_CAP_DWORD(index) NV_MSGBOX_CMD(_GET_CAP_DWORD, (index), 0) +#define NV_MSGBOX_CMD_GET_TEMP(source) NV_MSGBOX_CMD(_GET_TEMP, (source), 0) +#define NV_MSGBOX_CMD_GET_EXT_TEMP(source) NV_MSGBOX_CMD(_GET_EXT_TEMP, (source), 0) +#define NV_MSGBOX_CMD_GET_POWER(source) NV_MSGBOX_CMD(_GET_POWER, (source), 0) +#define NV_MSGBOX_CMD_GET_SYS_ID_DATA(type, offset) \ + NV_MSGBOX_CMD(_GET_SYS_ID_DATA, (type), (offset) / 4) +#define NV_MSGBOX_CMD_SCRATCH_READ(off) NV_MSGBOX_CMD(_SCRATCH_READ, (off), 0) +#define NV_MSGBOX_CMD_SCRATCH_WRITE(off, cnt) NV_MSGBOX_CMD(_SCRATCH_WRITE, (off), (cnt) - 1) +#define NV_MSGBOX_CMD_SCRATCH_COPY(off, cnt) NV_MSGBOX_CMD(_SCRATCH_COPY, (off), (cnt) - 1) +#define NV_MSGBOX_CMD_ASYNC_REQUEST(req_type, scratch_offset) \ + NV_MSGBOX_CMD(_ASYNC_REQUEST, (req_type), (scratch_offset) / 4) +#define NV_MSGBOX_CMD_ASYNC_REQUEST_POLL(req_id) \ + NV_MSGBOX_CMD(_ASYNC_REQUEST, \ + NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POLL, \ + (req_id)) +#define NV_MSGBOX_CMD_REGISTER_READ(addr) NV_MSGBOX_CMD(_REGISTER_ACCESS, \ + NV_MSGBOX_CMD_ARG1_REGISTER_ACCESS_READ, \ + (addr)) +#define NV_MSGBOX_CMD_REGISTER_WRITE(addr) NV_MSGBOX_CMD(_REGISTER_ACCESS, \ + NV_MSGBOX_CMD_ARG1_REGISTER_ACCESS_WRITE, \ + (addr)) +#define NV_MSGBOX_CMD_GET_THERM_PARAM(param) NV_MSGBOX_CMD(_GET_THERM_PARAM, \ + NV_MSGBOX_CMD_ARG1_THERM_PARAM##param, 0) + +#define NV_MSGBOX_CMD_ACCESS_WP_MODE(action, state) \ + NV_MSGBOX_CMD(_ACCESS_WP_MODE, (action), (state)) + +#define NV_MSGBOX_CMD_GET_WP_MODE() \ + NV_MSGBOX_CMD_ACCESS_WP_MODE(NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_GET, 0) + +#define NV_MSGBOX_CMD_SET_WP_MODE(state) \ + NV_MSGBOX_CMD_ACCESS_WP_MODE(NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_SET, (state)) + +#define NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(subtype, arg2) \ + NV_MSGBOX_CMD(_GET_MISC_GPU_FLAGS, (subtype), (arg2)) + +#define NV_MSGBOX_CMD_GET_ECC_ENABLED_STATE() \ + NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(NV_MSGBOX_CMD_ARG1_GET_MISC_ECC_ENABLED_STATE, 0) + +#define NV_MSGBOX_CMD_GET_GPU_RESET_REQUIRED() \ + NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_RESET_REQUIRED, 0) + +#define NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS_PAGE_0() \ + NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_0, 0) + +#define NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS_PAGE_1() \ + NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_1, 0) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO(subtype, arg2) \ + NV_MSGBOX_CMD(_GET_NVLINK_INFO, (subtype), (arg2)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(errtype, link_index) \ + NV_MSGBOX_CMD(_GET_NVLINK_INFO, \ + NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER##errtype,\ + (link_index)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_STATE(errtype, link_page) \ + NV_MSGBOX_CMD(_GET_NVLINK_INFO, \ + NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO##errtype##_ERROR_STATE, \ + (link_page)) + +#define NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS(subtype) \ + NV_MSGBOX_CMD(_GPU_UTIL_COUNTERS, (subtype), 0) + +#define NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS_CONTEXT_TIME() \ + NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS(NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_CONTEXT_TIME) + +#define NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS_SM_TIME() \ + NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS(NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_SM_TIME) + +#define NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS_RESET_COUNTERS() \ + NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS(NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_RESET_COUNTERS) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT_REPLAY(link_index) \ + NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(_REPLAY, (link_index)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT_RECOVERY(link_index) \ + NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(_RECOVERY, (link_index)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT_FLIT_CRC(link_index) \ + NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(_FLIT_CRC, (link_index)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT_DATA_CRC(link_index) \ + NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(_DATA_CRC, (link_index)) + +#define NV_MSGBOX_CMD_GET_CLOCK_FREQ_INFO(subtype, clkdomain) \ + NV_MSGBOX_CMD(_GET_CLOCK_FREQ_INFO, (subtype), (clkdomain)) + +#define NV_MSGBOX_CMD_GET_CLOCK_FREQ_INFO_GPC_CURRENT() \ + NV_MSGBOX_CMD_GET_CLOCK_FREQ_RANGE(NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_RANGE_GPC_CURRENT, \ + NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_GPCCLK) + +#define NV_MSGBOX_CMD_GET_CLOCK_FREQ_INFO_GPC_MIN() \ + NV_MSGBOX_CMD_GET_CLOCK_FREQ_RANGE(NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_GPC_MIN, \ + NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_GPCCLK) + +#define NV_MSGBOX_CMD_GET_CLOCK_FREQ_INFO_GPC_MAX() \ + NV_MSGBOX_CMD_GET_CLOCK_FREQ_RANGE(NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_GPC_MAX, \ + NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_GPCCLK) + +#define NV_MSGBOX_CMD_GET_DRIVER_EVENT_MSG(ptr) NV_MSGBOX_CMD(_GET_DRIVER_EVENT_MSG, 0, (ptr)) + +// +// ECC V2 +// +#define NV_MSGBOX_CMD_ECC_V2_CNT_FB(src, type, partition, slice) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_FB_SRC, src) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V2_CNT_GR(src, type, gpc, tpc) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_GR_SRC, src) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_GR_TPC, (tpc)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V2_FBA_CUMUL(partition, slice) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_SC, _CUMUL) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V2_FBA_IND_ADDR(partition, slice, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_IND_FLD, _ADDR) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V2_FBA_IND_CTRS(partition, slice, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_IND_FLD, _CTRS) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_SLICE, (slice)) \ + ) + +// +// ECC V3 +// +#define NV_MSGBOX_CMD_ECC_V3_CNT_FB_LTC(type, partition, slice) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_FB_SRC, _LTC) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_CNT_FB_FB(type, partition, subpart) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_FB_SRC, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_SUBPARTITION, (subpart)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_CNT_GR(src, type, gpc, tpc) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_GR_SRC, src) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_TPC, (tpc)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_CNT_GR_TEX(type, gpc, tpc, tex) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_GR_SRC, _TEX) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_TPC, (tpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_TEX, (tex)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_FBA_IND_ADDR(partition, subpart, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_IND_FLD, _ADDR) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_SUBPARTITION, (subpart)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_FBA_IND_CTRS(partition, subpart, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_IND_FLD, _CTRS) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_SUBPARTITION, (subpart)) \ + ) + +// +// ECC V4 +// +#define NV_MSGBOX_CMD_ECC_V4_CNT_FB_LTC(type, partition, slice) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_FB_SRC, _LTC) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_CNT_FB_FB(type, partition, subpart) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_FB_SRC, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_SUBPARTITION, (subpart)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_CNT_GR(src, type, gpc, tpc) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_GR_SRC, src) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_TPC, (tpc)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_CNT_GR_TEX(type, gpc, tpc, tex) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_GR_SRC, _TEX) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_TPC, (tpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_TEX, (tex)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_FBA_IND_ADDR(partition, subpart, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_IND_FLD, _ADDR) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_SUBPARTITION, (subpart)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_FBA_IND_CTRS(partition, subpart, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_IND_FLD, _CTRS) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_SUBPARTITION, (subpart)) \ + ) + +// +// ECC V5 +// +#define NV_MSGBOX_CMD_ECC_V5_CNT(index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V5, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V5_SEL, _COUNTS) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V5_COUNTS_INDEX_LO, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_INDEX_HI, (index) >> 7) \ + ) +#define NV_MSGBOX_ECC_V5_COUNTS_NUM_ENTRIES 600 + +#define NV_MSGBOX_CMD_ECC_V5_CNT_HDR_META(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _HDR_META) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_REGN_CORRECTED_TOTAL(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _REGN_CORRECTED_TOTAL) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_REGN_CORRECTED_UNIQUE(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _REGN_CORRECTED_UNIQUE) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_REGN_UNCORRECTED_TOTAL(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _REGN_UNCORRECTED_TOTAL) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_REGN_UNCORRECTED_UNIQUE(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _REGN_UNCORRECTED_UNIQUE) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_ADDR_ADDR(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _ADDR_ADDR) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_ADDR_UNCORRECTED_COUNTS(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _ADDR_UNCORRECTED_COUNTS) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_ADDR_CORRECTED_TOTAL(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _ADDR_CORRECTED_TOTAL) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_ADDR_CORRECTED_UNIQUE(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _ADDR_CORRECTED_UNIQUE) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_ERR_BUF(index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V5, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V5_SEL, _ERR_BUFFER) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V5_ERR_BUF_INDEX, (index)) \ + ) +#define NV_MSGBOX_ECC_V5_ERR_BUF_NUM_ENTRIES 16 + +#define NV_MSGBOX_CMD_ECC_V5_ERR_BUF_ERR_TYPE_META(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_ERR_BUF(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_ERR_BUF_TYPE, _ERR_TYPE_META) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_ERR_BUF_TIME_STAMP(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_ERR_BUF(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_ERR_BUF_TYPE, _TIME_STAMP) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_ERR_BUF_ADDR(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_ERR_BUF(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_ERR_BUF_TYPE, _ADDR) \ + ) + +#define NV_MSGBOX_CMD_SET_COPY_DATA(cmd) \ + ( \ + FLD_SET_DRF(_MSGBOX, _CMD, _COPY_DATA, _ON, (cmd)) \ + ) + +#define NV_MSGBOX_CMD_GET_PCIE_LINK_INFO_1(page) \ + ( \ + NV_MSGBOX_CMD(_GET_PCIE_LINK_INFO, \ + (page), \ + 0) \ + ) + +// We use the fact here that for pages ## 8 and 9 Arg2 definitions are similar +#define NV_MSGBOX_CMD_GET_PCIE_LINK_INFO_2(page, lane, speed) \ + ( \ + NV_MSGBOX_CMD(_GET_PCIE_LINK_INFO, \ + (page), \ + DRF_NUM(_MSGBOX, _CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_8, \ + _LANE_IDX, (lane)) | \ + DRF_NUM(_MSGBOX, _CMD_ARG2_GET_PCIE_LINK_INFO_PAGE_8, \ + _SPEED_SELECT, (speed))) \ + ) + +#define NV_MSGBOX_CMD_GPM_GET_METRIC(type, metric, req_ci_metrics, partition) \ + ( \ + NV_MSGBOX_CMD(_GPU_PERFORMANCE_MONITORING, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_GPM_ACTION, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_GPM_METRIC, metric) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_GPM_CI_METRICS_REQUESTED, req_ci_metrics) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_GPM_PARTITION_INDEX, partition) \ + ) + +#define NV_MSGBOX_CMD_GPM_GET_METRIC_AGGREGATE(type, metric) \ + ( \ + NV_MSGBOX_CMD(_GPU_PERFORMANCE_MONITORING, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_GPM_ACTION, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_GPM_METRIC, metric) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2, _GPM_PARTITION_AGGREGATE) \ + ) + +#define NV_MSGBOX_DATA_IN_GPM(instance_id, ci_index) \ + ( \ + DRF_NUM(_MSGBOX, _DATA, _GPM_NVDEC_INSTANCE, instance_id) | \ + DRF_NUM(_MSGBOX, _DATA, _GPM_COMPUTE_INSTANCE_INDEX, ci_index) \ + ) + +#define NV_MSGBOX_CMD_GPM_SET_MULTIPLIER(multiplier) \ + ( \ + NV_MSGBOX_CMD(_GPU_PERFORMANCE_MONITORING, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_GPM_ACTION, _SET_MULTIPLIER) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_GPM_MULTIPLIER, (multiplier)) \ + ) + +#define NV_MSGBOX_GET_CMD_OPCODE(cmd) DRF_VAL(_MSGBOX, _CMD, _OPCODE, (cmd)) +#define NV_MSGBOX_GET_CMD_ARG1(cmd) DRF_VAL(_MSGBOX, _CMD, _ARG1 , (cmd)) +#define NV_MSGBOX_GET_CMD_ARG2(cmd) DRF_VAL(_MSGBOX, _CMD, _ARG2 , (cmd)) +#define NV_MSGBOX_GET_CMD_STATUS(cmd) DRF_VAL(_MSGBOX, _CMD, _STATUS, (cmd)) +#define NV_MSGBOX_CMD_IS_COPY_DATA_SET(cmd) \ + FLD_TEST_DRF(_MSGBOX, _CMD, _COPY_DATA, _ON, (cmd)) + +#define NV_MSGBOX_CAP_IS_AVAILABLE(caps, idx, name) \ + !(FLD_TEST_DRF(_MSGBOX, _DATA_CAP_##idx, name, _NOT_AVAILABLE, (caps)[idx])) + +#define NV_MSGBOX_CAP_SET_AVAILABLE(caps, idx, name) \ + (caps)[idx] = FLD_SET_DRF(_MSGBOX, _DATA_CAP_##idx, name, _AVAILABLE, \ + (caps)[idx]) + +/******************************************************************************** + * * + * GPU REQUESTS * + * * + ********************************************************************************/ + +// +// Encoding: +// 31 24 23 16 15 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// | REQUEST ID | ARG2 | ARG1 | OPCODE | DATA_OUT +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_REQ_OPCODE 7:0 +#define NV_MSGBOX_REQ_OPCODE_CPU_PCONTROL 0x00000000 +#define NV_MSGBOX_REQ_OPCODE_SYS_PCONTROL 0x00000001 +#define NV_MSGBOX_REQ_ARG1 15:8 +#define NV_MSGBOX_REQ_ARG2 23:16 +#define NV_MSGBOX_REQ_REQUEST_ID 31:24 +#define NV_MSGBOX_REQ(opcode, arg1, arg2, requestId) \ + ( \ + DRF_NUM(_MSGBOX, _REQ, _OPCODE , (opcode)) | \ + DRF_NUM(_MSGBOX, _REQ, _ARG1 , (arg1)) | \ + DRF_NUM(_MSGBOX, _REQ, _ARG2 , (arg2)) | \ + DRF_NUM(_MSGBOX, _REQ, _REQUEST_ID, (requestId)) \ + ) + +/* ---------------- CPU Performance Control (PCONTROL) ------------------------ */ + +// +// CPU_PCONTROL +// +// Description: +// GPU-Request used to retrieve information and control various aspects of +// CPU performance. +// +// Common Encoding: +// 7 3 2 0 +// .-+-+-+-+-+-+-+-. +// | TARGET | ACT | ARG1 +// `-+-+-+-+-+-+-+-' +// +// Notes: +// arg1, arg2, and data encoding (fields are relative to zero, not relative +// to their position in the DATA_OUT register). +// +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_ACTION 2:0 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_ACTION_RSVD1 0x00000000 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_ACTION_RSVD2 0x00000001 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_ACTION_SET_LIMIT 0x00000002 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_TARGET 7:3 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_TARGET_PSTATE 0x00000000 + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_SET_ACTION(arg1, action) \ + (arg1) = FLD_SET_DRF(_MSGBOX_REQ, _CPU_PCONTROL_ARG1, _ACTION, \ + action, (arg1)) + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_GET_ACTION(arg1) \ + DRF_VAL(_MSGBOX_REQ, _CPU_PCONTROL_ARG1, _ACTION, (arg1)) + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_SET_TARGET(arg1, target) \ + (arg1) = FLD_SET_DRF(_MSGBOX_REQ, _CPU_PCONTROL_ARG1, _TARGET, \ + target, (arg1)) + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_GET_TARGET(arg1) \ + DRF_VAL(_MSGBOX_REQ, _CPU_PCONTROL_ARG1, _TARGET, (arg1)) + +// +// Setting CPU Pstate Limits: +// +// Inputs: +// ARG1 - {TARGET=PSTATE, ACTION=SET_LIMIT} +// ARG2 - Stores desired CPU Pstate limit +// +// Outputs: +// none +// +// Encoding: +// 7 3 2 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | TARGET | ACT | ARG1 | VALUE | ARG2 +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG2_PSTATE_SET_LIMIT_VALUE 7:0 + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG2_PSTATE_SET_LIMIT_SET_VALUE(arg2, limit) \ + (arg2) = FLD_SET_DRF_NUM(_MSGBOX_REQ, _CPU_PCONTROL_ARG2, \ + _PSTATE_SET_LIMIT_VALUE, (limit), (arg2)) + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG2_PSTATE_SET_LIMIT_GET_VALUE(arg2) \ + DRF_VAL(_MSGBOX_REQ, _CPU_PCONTROL_ARG2, _PSTATE_SET_LIMIT_VALUE, (arg2)) + +/* ----------------------- System Performance Control ------------------------- */ + +// +// SYS_PCONTROL +// +// Description: +// GPU-Request used to retrieve information and control various aspects of +// system-performance. +// +// Common Encoding: +// 7 3 2 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | TARGET | ACT | ARG1 |0 0 0 0 0 0 0 0| ARG2 (reserved) +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +// Notes: +// - TARGET accepts NV0000_CTRL_SYSTEM_PARAM_* definitions +// - arg1, arg2, and data encoding (fields are relative to zero, not relative +// to their position in the DATA_OUT register). +// +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_ACTION 2:0 +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_ACTION_GET_STATUS 0x00000000 +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_ACTION_GET_LIMIT 0x00000001 +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_TARGET 7:3 +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG2_RSVD 7:0 + +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_SET_ACTION(arg1, action) \ + (arg1) = FLD_SET_DRF(_MSGBOX_REQ, _SYS_PCONTROL_ARG1, _ACTION, \ + action, (arg1)) + +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_GET_ACTION(arg1) \ + DRF_VAL(_MSGBOX_REQ, _SYS_PCONTROL_ARG1, _ACTION, (arg1)) + +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_SET_TARGET(arg1, target) \ + (arg1) = FLD_SET_DRF_NUM(_MSGBOX_REQ, _SYS_PCONTROL_ARG1, _TARGET, \ + (target), (arg1)) + +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_GET_TARGET(arg1) \ + DRF_VAL(_MSGBOX_REQ, _SYS_PCONTROL_ARG1, _TARGET, (arg1)) + +#endif /* _SMBPBI_H */ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi_impl.h b/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi_impl.h new file mode 100644 index 0000000..e7052c8 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi_impl.h @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef _SMBPBI_IMPL_H +#define _SMBPBI_IMPL_H +/*! + * This header file stores implementation dependent parameters of the SMBPBI server. + */ + +/*! + * Maximum number of individual requests in a bundle + */ +#define NV_MSGBOX_PARAM_MAX_BUNDLE_SIZE 4 + +/*! + * Maximum number of Result Disposition Rules + */ +#define NV_MSGBOX_PARAM_MAX_DISP_RULES 10 + +/*! + * Maximum length of the Driver Event Message text string is 80, including + * the terminating NUL character. + */ +#define NV_MSGBOX_MAX_DRIVER_EVENT_MSG_TXT_SIZE 80 + +#endif // _SMBPBI_IMPL_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi_priv.h b/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi_priv.h new file mode 100644 index 0000000..53b48da --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/oob/smbpbi_priv.h @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2010-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _SMBPBI_PRIV_H_ +#define _SMBPBI_PRIV_H_ + +/*! + * This file contains NVIDIA private defines for the SMBPBI + * interface. + */ + +#include "oob/smbpbi.h" + +// MSGBOX command word structure + +// +// Reserving opcodes above 0xC0 for internal/private functionationality. +// +// These opcodes should not be included in any documentation we release outside +// of NVIDIA! +// + +// +// Only for internal use (should not be written to the command register). Used +// for internal tracking when commands are redirected to the RM from the PMU. +// +#define NV_MSGBOX_CMD_ERR_MORE_PROCESSING_REQUIRED 0x000000F0 + +// +// Alternative encodings of the command word. +// These were distinguished by a non-zero value in the 29:29 bit. +// Bit 29 is now reserved and must be 0 i.e. only standard requests will be processed +// and debug requests would fail. +// +#define NV_MSGBOX_CMD_ENCODING 29:29 +#define NV_MSGBOX_CMD_ENCODING_STANDARD 0x00000000 + +#endif // _SMBPBI_PRIV_H_ + diff --git a/userspace/sdks/nv570/nvalloc/common/inc/rmRiscvUcode.h b/userspace/sdks/nv570/nvalloc/common/inc/rmRiscvUcode.h new file mode 100644 index 0000000..d2e4259 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/rmRiscvUcode.h @@ -0,0 +1,95 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/* + * << DO NOT EDIT >> + * + * This file describes the format of generated ucode binary. Please do not change the + * content unless the same change is applied to the target ucode builds. + */ + +#ifndef RM_RISCV_UCODE_H +#define RM_RISCV_UCODE_H + +#include "nvtypes.h" + +typedef struct { + // + // Version 1 + // Version 2 + // Version 3 = for Partition boot + // Version 4 = for eb riscv boot + // Version 5 = Support signing entire RISC-V image as "code" in code section for hopper and later. + // + NvU32 version; // structure version + NvU32 bootloaderOffset; + NvU32 bootloaderSize; + NvU32 bootloaderParamOffset; + NvU32 bootloaderParamSize; + NvU32 riscvElfOffset; + NvU32 riscvElfSize; + NvU32 appVersion; // Changelist number associated with the image + // + // Manifest contains information about Monitor and it is + // input to BR + // + NvU32 manifestOffset; + NvU32 manifestSize; + // + // Monitor Data offset within RISCV image and size + // + NvU32 monitorDataOffset; + NvU32 monitorDataSize; + // + // Monitor Code offset withtin RISCV image and size + // + NvU32 monitorCodeOffset; + NvU32 monitorCodeSize; + NvU32 bIsMonitorEnabled; + // + // Swbrom Code offset within RISCV image and size + // + NvU32 swbromCodeOffset; + NvU32 swbromCodeSize; + // + // Swbrom Data offset within RISCV image and size + // + NvU32 swbromDataOffset; + NvU32 swbromDataSize; + // + // Total size of FB carveout (image and reserved space). + // + NvU32 fbReservedSize; + // + // Indicates whether the entire RISC-V image is signed as "code" in code section. + // + NvU32 bSignedAsCode; + // + // SMP (multihart) and PLIC (split interrupts) + // enablement options + // + NvU32 bIsSmp; + NvU32 bIsPlicEnabled; +} RM_RISCV_UCODE_DESC; + +#endif // RM_RISCV_UCODE_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/rm_state_monitor.h b/userspace/sdks/nv570/nvalloc/common/inc/rm_state_monitor.h new file mode 100644 index 0000000..e89fd16 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/rm_state_monitor.h @@ -0,0 +1,72 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef RM_STATE_MONITOR_H +#define RM_STATE_MONITOR_H + +// run every 100ms to prevent excessive CPU usage +// as RM dump task runs as a higher priority task +// than RM main task +#define RM_STATE_MONITOR_WAIT_PERIOD (100000000) // 100ms + +typedef enum { + RM_STATE_MONITOR_HEADER_REQUEST_EXIT, + RM_STATE_MONITOR_HEADER_REQUEST_RM_STATE, + RM_STATE_MONITOR_HEADER_RESPONSE +} RM_STATE_MONITOR_HEADER; + +typedef struct { + RM_STATE_MONITOR_HEADER header; +} RM_STATE_MONITOR_REQUEST_MESSAGE; + +typedef struct { + NV_STATUS status; +} RM_STATE_MONITOR_RESPONSE_MESSAGE; + +typedef union { + RM_STATE_MONITOR_REQUEST_MESSAGE request; + RM_STATE_MONITOR_RESPONSE_MESSAGE response; +} RM_STATE_MONITOR_MESSAGE; + +#define RM_STATE_MONITOR_MAX_DPC_QUEUE_ITEMS 16 +#define RM_STATE_MONITOR_MAX_WORK_ITEMS 32 +#define RM_STATE_MONITOR_MAX_INTERRUPTS 32 +#define RM_STATE_MONITOR_MAX_ENINGE_BYTE_ARRAY (4) + +typedef struct { + NvU64 dpcQueue[RM_STATE_MONITOR_MAX_DPC_QUEUE_ITEMS][RM_STATE_MONITOR_MAX_ENINGE_BYTE_ARRAY]; + NvU64 workItemsQueue[RM_STATE_MONITOR_MAX_WORK_ITEMS]; + NvU64 lastDpcExecutionTime; + NvU64 totalDpcExecutionTime; + NvU64 lastWorkItemsExecutionTime; + NvU64 totalWorkItemsExecutionTime; + NvU64 interrupts[RM_STATE_MONITOR_MAX_INTERRUPTS]; //TODO: should this get set? will affect perf +} RM_STATE_MONITOR_DATA; + +typedef struct { + NvU64 requestSequenceNumber; + NvU64 responseSequenceNumber; + RM_STATE_MONITOR_DATA data; +} RM_STATE_MONITOR_BUFFER_LAYOUT; + +#endif diff --git a/userspace/sdks/nv570/nvalloc/common/inc/rmflcnbl.h b/userspace/sdks/nv570/nvalloc/common/inc/rmflcnbl.h new file mode 100644 index 0000000..7a49306 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/rmflcnbl.h @@ -0,0 +1,147 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2013-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * Data structures and interfaces used for generic falcon boot-loader. + * + * This generic boot-loader is designed to load both non-secure and + * secure code taking care of signature as well. This bootloader + * should be loaded at the end of the IMEM so that it doesnt overwrite + * itself when it tries to load the code into IMEM starting at blk 0. + * The data will be loaded into DMEM offset 0. + */ + +#ifndef RMFLCNBL_H +#define RMFLCNBL_H + +#include "flcnifcmn.h" + +/*! + * @brief Structure used by the boot-loader to load the rest of the code. + * + * This has to be filled by the GPU driver and copied into DMEM at offset + * provided in the @ref _def_rm_flcn_bl_desc.blDmemDescLoadOff. + */ +typedef struct _def_rm_flcn_bl_dmem_desc +{ + /*! + * reserved should be always first element + */ + NvU32 reserved[4]; + /*! + * signature should follow reserved 16B signature for secure code. + * 0s if no secure code + */ + NvU32 signature[4]; + /*! + * ctxDma is used by the bootloader while loading code/data. + */ + NvU32 ctxDma; + /*! + * 256B aligned physical FB address where code is located. + */ + RM_FLCN_U64 codeDmaBase; + /*! + * Offset from codeDmaBase where the nonSecure code is located. + * The offset must be multiple of 256 to help performance. + */ + NvU32 nonSecureCodeOff; + /*! + * The size of the nonSecure code part. + */ + NvU32 nonSecureCodeSize; + /*! + * Offset from codeDmaBase where the secure code is located. + * The offset must be multiple of 256 to help performance. + */ + NvU32 secureCodeOff; + /*! + * The size of the ecure code part. + */ + NvU32 secureCodeSize; + /*! + * Code entry point which will be invoked by BL after code is loaded. + */ + NvU32 codeEntryPoint; + /*! + * 256B aligned Physical FB Address where data is located. + */ + RM_FLCN_U64 dataDmaBase; + /*! + * Size of data block. Should be multiple of 256B. + */ + NvU32 dataSize; + /*! + * Arguments to be passed to the target firmware being loaded. + */ + NvU32 argc; + /*! + * Number of arguments to be passed to the target firmware being loaded. + */ + NvU32 argv; +} RM_FLCN_BL_DMEM_DESC, *PRM_FLCN_BL_DMEM_DESC; + +/*! + * @brief The header used by the GPU driver to figure out code and data + * sections of bootloader. + */ +typedef struct _def_rm_flcn_bl_img_header +{ + /*! + * Offset of code section in the image. + */ + NvU32 blCodeOffset; + /*! + * Size of code section in the image. + */ + NvU32 blCodeSize; + /*! + * Offset of data section in the image. + */ + NvU32 blDataOffset; + /*! + * Size of data section in the image. + */ + NvU32 blDataSize; +} RM_FLCN_BL_IMG_HEADER, *PRM_FLCN_BL_IMG_HEADER; + +/*! + * @brief The descriptor used by RM to figure out the requirements of boot loader. + */ +typedef struct _def_rm_flcn_bl_desc +{ + /*! + * Starting tag of bootloader + */ + NvU32 blStartTag; + /*! + * Dmem offset where _def_rm_flcn_bl_dmem_desc to be loaded + */ + NvU32 blDmemDescLoadOff; + /*! + * Description of the image + */ + RM_FLCN_BL_IMG_HEADER blImgHeader; +} RM_FLCN_BL_DESC, *PRM_FLCN_BL_DESC; + +#endif // RMFLCNBL_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/rmgspseq.h b/userspace/sdks/nv570/nvalloc/common/inc/rmgspseq.h new file mode 100644 index 0000000..5d130d6 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/rmgspseq.h @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * Top-level header-file that defines the GSP data structures used to request + * firmware load/execution by the CPU. + */ + +#ifndef _RMGSPSEQ_H_ +#define _RMGSPSEQ_H_ + +#include "rmflcnbl.h" + +/*! + * Structure for loading/executing generic bootloader + */ +typedef struct GspLoadExecGenericBootloaderParams +{ + RM_FLCN_BL_DMEM_DESC dmemDesc; + NvU32 dmemDescSize; + NvU32 addrSpace; + NvU32 cpuCacheAttrib; + NvU32 reserved[4]; +} GspLoadExecGenericBootloaderParams; + +/*! + * Structure for loading/executing HS binary + */ +typedef struct GspLoadExecHsBinaryParams +{ + NvU64 imemPhysAddr; + NvU64 dmemPhysAddr; + NvU64 reserved64[2]; + NvU32 ucodeImemVA; + NvU32 ucodeImemPA; + NvU32 ucodeImemSize; + NvU32 ucodeDmemVA; + NvU32 ucodeDmemPA; + NvU32 ucodeDmemSize; + NvU32 hsSigDmemAddr; + NvU32 engineIdMask; + NvU32 ucodeId; + NvU32 reserved32[3]; +} GspLoadExecHsBinaryParams; + +#endif // _RMGSPSEQ_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/rmifrif.h b/userspace/sdks/nv570/nvalloc/common/inc/rmifrif.h new file mode 100644 index 0000000..b774d02 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/rmifrif.h @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2012 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _RMIFRIF_H_ +#define _RMIFRIF_H_ + +/*! + * @file rmifrif.h + * @brief Defines structures and interfaces common between RM and + * Init-From-Rom (IFR). + * + * For systems supporting GC6 that have on-board VBIOS ROMs, IFR is used + * to expedite several parts of GC6 exit in parallel with PEX init. + * + * After running devinit using a PMU ucode image loaded from the ROM itself, + * parts of RM stateLoad can be done using RM's ucode image. This is + * achieved by loading RM PMU ucode directly from FB. The primary difficulties + * are how to find RM's PMU ucode and how to bootstrap it. + * + * We use the simple approach of allocating a fixed buffer near the + * top of FB that contains the information required to bootstrap RM's PMU + * image. This buffer is called the RM_IFR_GC6_CTX. + * + * The buffer is allocated within RM's reserved memory space, directly before + * the VBIOS workspace (if any is present). Since the VBIOS workspace is + * always a multiple of 64K, RM enforces that the offset between top of memory + * and the end of the buffer is 64K. This way the IFR code can start + * from the top of memory and search downwards in 64K decrements. + * + * A small header is placed at the end of the buffer which contains a + * string signature identifying the buffer and other data needed to find the + * remaining context data. + * + * Top_Of-FB /---------------------\ <- + * | | \ + * | (VBIOS_Workspace) | | END_OFFSET + * | | / + * |---------------------| <- + | | \ + * | GSP FW (if present) | | pFbHalPvtInfo->gspFwSizeBytes + * | | / + * |---------------------| <- + * | RM_IFR_GC6_CTX_HDR | \ + * |---------------------| | + * | (Padding) | | RM_IFR_GC6_CTX_HDR.bufferSize + * |---------------------| | + * | Sequence Data | / + * |---------------------| <- + * | | + * | | + * | | + * | | + * 0x00000000 \---------------------/ + * + * To simplify the RM PMU bootstrap process and decrease IFR maintainence + * cost, the bootstrap process is encoded as a sequence script, leveraging + * a small subset of RM's PMU_SEQ_INST interface (see pmuseqinst.h). + * Register writes are captured during the initial (CPU-driven) RM PMU bootstrap + * and saved into a sequence for replay during GC6 exit. + * + * Only the following opcodes are supported currently: + * NV_PMU_SEQ_WRITE_REG_OPC - (multi-)register write + * NV_PMU_SEQ_EXIT_OPC - sequence done + * + */ + +/*! + * Header structure which identifies the GC6 context buffer. + */ +typedef struct +{ + NvU8 signature[12]; // RM_IFR_GC6_CTX_SIGNATURE + NvU32 bufferSize; // Size of the entire context buffer in bytes + NvU32 seqSizeWords; // Number of 32-bit words of sequence data + NvU32 checksum; // 32-bit chunk checksum of the sequence data +} RM_IFR_GC6_CTX_HDR, *PRM_IFR_GC6_CTX_HDR; + +/*! + * String signature that IFR searches for to find the GC6 context buffer. + */ +#define RM_IFR_GC6_CTX_SIGNATURE "GC6_CTX_HDR" // 12 bytes + +/*! + * Alignment of the offset between top of memory and the end of the + * GC6 context buffer (which is also the end of the header). + */ +#define RM_IFR_GC6_CTX_END_OFFSET_ALIGNMENT 0x10000 // 64KB + +/*! + * Maximum offset between top of memory and the end of the + * GC6 context buffer. This is meant to be a loose upper bound preventing + * scanning of the whole of memory (e.g. when something goes wrong). + */ +#define RM_IFR_GC6_CTX_END_OFFSET_MAX 0x1000000 // 16MB + +/*! + * Maximum size of the context data in bytes. + * This is limited by FECS falcon DMEM size (4K on Kepler). + * The buffer must fit within DMEM together with stack and other global data. + */ +#define RM_IFR_GC6_CTX_DATA_MAX_SIZE 2048 // 2KB + +#endif // _RMIFRIF_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/rmlsfm.h b/userspace/sdks/nv570/nvalloc/common/inc/rmlsfm.h new file mode 100644 index 0000000..ca91911 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/rmlsfm.h @@ -0,0 +1,579 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2011-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef RMLSFM_H +#define RMLSFM_H + +/*****************************************************************************/ +/* This file is shared between ACR, SEC2 Binaries */ +/* Do not update this file without updating ACR/SEC2 */ +/*****************************************************************************/ + +/*! + * @file rmlsfm.h + * @brief Top-level header-file that defines Light Secure Falcon Managment + SW shared interfaces. + */ + +/*! + * READ/WRITE masks for WPR region + */ +#define LSF_WPR_REGION_RMASK (0xCU) // Readable only from level 2 and 3 client +#define LSF_WPR_REGION_WMASK (0xCU) // Writable only from level 2 and 3 client +#define LSF_WPR_REGION_RMASK_SUB_WPR_ENABLED (0x8) // Readable only from level 3 client +#define LSF_WPR_REGION_WMASK_SUB_WPR_ENABLED (0x8) // Writable only from level 3 client +#define LSF_WPR_REGION_ALLOW_READ_MISMATCH_NO (0x0) // Disallow read mis-match for all clients +#define LSF_WPR_REGION_ALLOW_WRITE_MISMATCH_NO (0x0) // Disallow write mis-match for all clients + +/*! + * READ mask for WPR region on Tegra + * This is required until we update tegra binaries, Bug 200281517 + * TODO: dgoyal - Remove this once tegra binaries are updated + */ +#define LSF_WPR_REGION_RMASK_FOR_TEGRA (0xFU) + +/*! + * Expected REGION ID to be used for the unprotected FB region (region that + * does not have read or write protections) + */ +#define LSF_UNPROTECTED_REGION_ID (0x0U) + +/*! + * Expected REGION ID to be used for the WPR region for the falcon microcode (includes data). + * ACR allots client requests to each region based on read/write masks and it is supposed + * to provide first priority to requests from LSFM. Providing first priority will naturally assign + * region ID 1 to LSFM and this define will provide a way for different parties to sanity check + * this fact. Also there are other falcons (FECS/video falcons) which depends on this define, so please + * be aware while modifying this. + */ +#define LSF_WPR_EXPECTED_REGION_ID (0x1U) + +/*! + * Expected REGION ID to be used for the unused WPR region. + */ +#define LSF_WPR_UNUSED_REGION_ID (0x2U) + +/*! + * Invalid LS falcon subWpr ID + */ +#define LSF_SUB_WPR_ID_INVALID (0xFFFFFFFFU) + +/*! + * Expected REGION ID to be used for the VPR region. + */ +#define LSF_VPR_REGION_ID (0x3U) + +/*! + * Expected REGION ID to be used for the CPR region with conf compute. + */ +#define LSF_CPR_REGION_ID (0x3U) + +/*! + * Size of the separate bootloader data that could be present in WPR region. + */ +#define LSF_LS_BLDATA_EXPECTED_SIZE (0x100U) + +/*! + * since we dont check signatures in GC6 exit, we need to hardcode the WPR offset + */ +#define LSF_WPR_EXPECTED_OFFSET (0x0U) + +/*! + * CTXDMA to be used while loading code/data in target falcons + */ +#define LSF_BOOTSTRAP_CTX_DMA_FECS (0x0) + +/*! + * Context DMA ID 6 is reserved for Video UCODE + */ +#define LSF_BOOTSTRAP_CTX_DMA_VIDEO (0x6) +#define LSF_BOOTSTRAP_CTX_DMA_BOOTSTRAP_OWNER (0x0) +#define LSF_BOOTSTRAP_CTX_DMA_FBFLCN (0x0) + +/*! + * Falcon Id Defines + * Defines a common Light Secure Falcon identifier. + * Codesigning infra. assumes LSF_FALCON_ID_ prefix for units, + * Changes to the define needs to be reflected in path [1] + * For new Falcon Id adding, we need to append to the end; + * don't insert the new falcon Id in the middle. + * + * @note If a newly added Falcon has multiple instances sharing + * the same Falcon Id, the LSF_FALCON_USES_INSTANCE macro + * need to be updated. + */ +#define LSF_FALCON_ID_PMU (0U) +#define LSF_FALCON_ID_DPU (1U) +#define LSF_FALCON_ID_GSPLITE LSF_FALCON_ID_DPU +#define LSF_FALCON_ID_FECS (2U) +#define LSF_FALCON_ID_GPCCS (3U) +#define LSF_FALCON_ID_NVDEC (4U) +#define LSF_FALCON_ID_NVENC (5U) +#define LSF_FALCON_ID_NVENC0 (5U) +#define LSF_FALCON_ID_NVENC1 (6U) +#define LSF_FALCON_ID_SEC2 (7U) +#define LSF_FALCON_ID_NVENC2 (8U) +#define LSF_FALCON_ID_MINION (9U) +#define LSF_FALCON_ID_FBFALCON (10U) +#define LSF_FALCON_ID_XUSB (11U) +#define LSF_FALCON_ID_GSP_RISCV (12U) +#define LSF_FALCON_ID_PMU_RISCV (13U) +#define LSF_FALCON_ID_SOE (14U) +#define LSF_FALCON_ID_NVDEC1 (15U) +#define LSF_FALCON_ID_OFA (16U) +#define LSF_FALCON_ID_SEC2_RISCV (17U) +#define LSF_FALCON_ID_NVDEC_RISCV (18U) +#define LSF_FALCON_ID_NVDEC_RISCV_EB (19U) +#define LSF_FALCON_ID_NVJPG (20U) +#define LSF_FALCON_ID_FECS_RISCV (21U) +#define LSF_FALCON_ID_GPCCS_RISCV (22U) +#define LSF_FALCON_ID_NVJPG_RISCV_EB (23U) +#define LSF_FALCON_ID_OFA_RISCV_EB (24U) +#define LSF_FALCON_ID_NVENC_RISCV_EB (25U) +#define LSF_FALCON_ID_PMU_RISCV_EB (26U) +#define LSF_FALCON_ID_NVDEC0_RISCV_EB (27U) +#define LSF_FALCON_ID_GSPLITE_RISCV_EB (28U) +#define LSF_FALCON_ID_DISPLAY_RISCV_EB (29U) +#define LSF_FALCON_ID_FBFALCON_RISCV_EB (30U) +#define LSF_FALCON_ID_LPSEC (31U) +#define LSF_FALCON_ID_END (32U) + +#define LSF_FALCON_ID_INVALID (0xFFFFFFFFU) + +// +// TODO: Remove below Alias and add _EB Patching to macro LSF_FALCON_ID_FECS_RISCV, similarly for GPCCS, +// and similar cleanups in RM since RISCV based CTXSW engines are to be booted externally. +// Tracking in Bug 3808599 +// +#define LSF_FALCON_ID_FECS_RISCV_EB (LSF_FALCON_ID_FECS_RISCV) +#define LSF_FALCON_ID_GPCCS_RISCV_EB (LSF_FALCON_ID_GPCCS_RISCV) + +// +// ************************ NOTIFICATION ********************************* +// In case anyone needs to add new LSF falconId, please must calculate +// WPR header size per LSF_FALCON_ID_END. RM needs to call lsfmGetWprHeaderSizeMax_HAL +// to align with acrReadSubWprHeader_HAL in ACR. Otherwise, ACR can't get correct +// address to read sub wpr headers. +// We observer in case LSF_FALCON_ID_END > 32 will cause SEC2 IMEM tag missing error; +// but don't get the root cause yet. +// +#define LSF_FALCON_ID_END_15 (15U) +#define LSF_FALCON_ID_END_17 (17U) +#define LSF_FALCON_ID_END_18 (18U) +#define LSF_FALCON_ID_END_21 (21U) +#define LSF_FALCON_ID_END_30 (30U) + +#define LSF_FALCON_INSTANCE_DEFAULT_0 (0x0U) +#define LSF_FALCON_INSTANCE_DEFAULT_3 (0x3U) +#define LSF_FALCON_INSTANCE_DEFAULT_4 (0x4U) +#define LSF_FALCON_INSTANCE_COUNT_DEFAULT_1 (0x1U) + +// Currently max supported instance is 24 for FECS/GPCCS SMC +#define LSF_FALCON_INSTANCE_FECS_GPCCS_MAX (0x18U) +#define LSF_FALCON_INSTANCE_INVALID (0xFFFFFFFFU) +#define LSF_FALCON_INDEX_MASK_DEFAULT_0 (0x0U) + +/*! + * Checks if the LSF Falcon specified by falconId uses a falconInstance to uniquely identify itself. + * Some Falcons (eg: NVENC) use separate FalconId for each instance while some (eg: NVJPG) + * shares the same falconId across all instances of that engine. Those engines require a falconInstance + * to uniquely identify it. + * @note this macro should be updated as needed whenever LSF_FALCON_ID* defines are added. See Bug: 3833461 + */ +#define LSF_FALCON_USES_INSTANCE(falconId) ((falconId == LSF_FALCON_ID_NVDEC_RISCV_EB) || \ + (falconId == LSF_FALCON_ID_NVJPG) || \ + (falconId == LSF_FALCON_ID_NVJPG_RISCV_EB) || \ + (falconId == LSF_FALCON_ID_NVENC_RISCV_EB) || \ + (falconId == LSF_FALCON_ID_FECS_RISCV_EB) || \ + (falconId == LSF_FALCON_ID_GPCCS_RISCV_EB) || \ + (falconId == LSF_FALCON_ID_GSPLITE_RISCV_EB) || \ + (falconId == LSF_FALCON_ID_OFA_RISCV_EB) || \ + (falconId == LSF_FALCON_ID_LPSEC)) + + +/*! + * Size in entries of the ucode descriptor's dependency map. + * This used to be LSF_FALCON_ID_END, but since that had to grow and we did not want to break any + * existing binaries, they had to be split. + * + * Increasing this number should be done with care. + */ +#define LSF_FALCON_DEPMAP_SIZE (11U) + +/*! + * Falcon Binaries version defines + */ +#define LSF_FALCON_BIN_VERSION_INVALID (0xFFFFFFFFU) + + +/*! + * Light Secure Falcon Ucode Description Defines + * This stucture is prelim and may change as the ucode signing flow evolves. + */ +typedef struct +{ + NvU8 prdKeys[2][16]; + NvU8 dbgKeys[2][16]; + NvU32 bPrdPresent; + NvU32 bDbgPresent; + NvU32 falconId; + NvU32 bSupportsVersioning; + NvU32 version; + NvU32 depMapCount; + NvU8 depMap[LSF_FALCON_DEPMAP_SIZE * 2 * 4]; + NvU8 kdf[16]; +} LSF_UCODE_DESC, *PLSF_UCODE_DESC; + +/*! + * Light Secure WPR Header + * Defines state allowing Light Secure Falcon bootstrapping. + * + * falconId - LS falcon ID + * lsbOffset - Offset into WPR region holding LSB header + * bootstrapOwner - Bootstrap OWNER (either PMU or SEC2) + * bLazyBootstrap - Skip bootstrapping by ACR + * status - Bootstrapping status + */ +typedef struct +{ + NvU32 falconId; + NvU32 lsbOffset; + NvU32 bootstrapOwner; + NvU32 bLazyBootstrap; + NvU32 binVersion; + NvU32 status; +} LSF_WPR_HEADER, *PLSF_WPR_HEADER; + +/*! + * LSF shared SubWpr Header + * + * useCaseId - Shared SubWpr se case ID (updated by RM) + * startAddr - start address of subWpr (updated by RM) + * size4K - size of subWpr in 4K (updated by RM) + */ +typedef struct +{ + NvU32 useCaseId; + NvU32 startAddr; + NvU32 size4K; +} LSF_SHARED_SUB_WPR_HEADER, *PLSF_SHARED_SUB_WPR_HEADER; + +// Shared SubWpr use case IDs +typedef enum +{ + LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_FRTS_VBIOS_TABLES = 1, + LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_PLAYREADY_SHARED_DATA = 2, + LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_APM_RTS = 3 +} LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_ENUM; + +#define LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_APM_RTS +#define LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_INVALID (0xFFFFFFFFU) + +#define MAX_SUPPORTED_SHARED_SUB_WPR_USE_CASES LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX + +// +// Static sizes of shared subWPRs +// Minimum granularity supported is 4K +// +#define LSF_SHARED_DATA_SUB_WPR_FRTS_VBIOS_TABLES_SIZE_IN_4K (0x100) // 1MB in 4K +#define LSF_SHARED_DATA_SUB_WPR_PLAYREADY_SHARED_DATA_SIZE_IN_4K (0x1) // 4K +#define LSF_SHARED_DATA_SUB_WPR_APM_RTS_SIZE_IN_4K (0x1) // 4K + +/*! + * Bootstrap Owner Defines + */ +#define LSF_BOOTSTRAP_OWNER_PMU (LSF_FALCON_ID_PMU) +#define LSF_BOOTSTRAP_OWNER_SEC2 (LSF_FALCON_ID_SEC2) +#define LSF_BOOTSTRAP_OWNER_GSPLITE (LSF_FALCON_ID_GSPLITE) +#define LSF_BOOTSTRAP_OWNER_DEFAULT LSF_BOOTSTRAP_OWNER_PMU + +/*! + * Image Status Defines + */ +#define LSF_IMAGE_STATUS_NONE (0U) +#define LSF_IMAGE_STATUS_COPY (1U) +#define LSF_IMAGE_STATUS_VALIDATION_CODE_FAILED (2U) +#define LSF_IMAGE_STATUS_VALIDATION_DATA_FAILED (3U) +#define LSF_IMAGE_STATUS_VALIDATION_DONE (4U) +#define LSF_IMAGE_STATUS_VALIDATION_SKIPPED (5U) +#define LSF_IMAGE_STATUS_BOOTSTRAP_READY (6U) +#define LSF_IMAGE_STATUS_REVOCATION_CHECK_FAILED (7U) + +/*! + * Light Secure Bootstrap Header + * Defines state allowing Light Secure Falcon bootstrapping. + * + * signature - Code/data signature details for this LS falcon + * ucodeOffset - Offset into WPR region where UCODE is located + * ucodeSize - Size of ucode + * dataSize - Size of ucode data + * blCodeSize - Size of bootloader that needs to be loaded by bootstrap owner + * blImemOffset - BL starting virtual address. Need for tagging. + * blDataOffset - Offset into WPR region holding the BL data + * blDataSize - Size of BL data + * appCodeOffset - Offset into WPR region where Application UCODE is located + * appCodeSize - Size of Application UCODE + * appDataOffset - Offset into WPR region where Application DATA is located + * appDataSize - Size of Application DATA + * blLoadCodeAt0 - Load BL at 0th IMEM offset + * bSetVACtx - Make sure to set the code/data loading CTX DMA to be virtual before exiting + * bDmaReqCtx - This falcon requires a ctx before issuing DMAs + * bForcePrivLoad- Use priv loading method instead of bootloader/DMAs + */ + +#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0 0:0 +#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0_FALSE 0 +#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0_TRUE 1 +#define NV_FLCN_ACR_LSF_FLAG_SET_VA_CTX 1:1 +#define NV_FLCN_ACR_LSF_FLAG_SET_VA_CTX_FALSE 0 +#define NV_FLCN_ACR_LSF_FLAG_SET_VA_CTX_TRUE 1 +#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX 2:2 +#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_FALSE 0 +#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_TRUE 1 +#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD 3:3 +#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_FALSE 0 +#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_TRUE 1 +typedef struct +{ + LSF_UCODE_DESC signature; + NvU32 ucodeOffset; + NvU32 ucodeSize; + NvU32 dataSize; + NvU32 blCodeSize; + NvU32 blImemOffset; + NvU32 blDataOffset; + NvU32 blDataSize; + NvU32 appCodeOffset; + NvU32 appCodeSize; + NvU32 appDataOffset; + NvU32 appDataSize; + NvU32 flags; + /* + * TODO: Uncomment this once Sanket's changes + * of HALifying is done + NvU32 monitorCodeOffset; + NvU32 monitorDataOffset; + NvU32 manifestOffset; + */ +} LSF_LSB_HEADER, *PLSF_LSB_HEADER; + + +/*! + * Light Secure WPR Content Alignments + */ +#define LSF_WPR_HEADER_ALIGNMENT (256U) +#define LSF_SUB_WPR_HEADER_ALIGNMENT (256U) +#define LSF_LSB_HEADER_ALIGNMENT (256U) +#define LSF_BL_DATA_ALIGNMENT (256U) +#define LSF_BL_DATA_SIZE_ALIGNMENT (256U) +#define LSF_BL_CODE_SIZE_ALIGNMENT (256U) +#define LSF_DATA_SIZE_ALIGNMENT (256U) +#define LSF_CODE_SIZE_ALIGNMENT (256U) + +// MMU excepts subWpr sizes in units of 4K +#define SUB_WPR_SIZE_ALIGNMENT (4096U) + +/*! + * Maximum WPR Header size + */ +#define LSF_WPR_HEADERS_TOTAL_SIZE_MAX (NV_ALIGN_UP((sizeof(LSF_WPR_HEADER) * LSF_FALCON_ID_END), LSF_WPR_HEADER_ALIGNMENT)) +#define LSF_LSB_HEADER_TOTAL_SIZE_MAX (NV_ALIGN_UP(sizeof(LSF_LSB_HEADER), LSF_LSB_HEADER_ALIGNMENT)) + +// +// PMU OBJACR_ALIGNED_256 size will vary with LSF_FALCON_ID_END. +// PMU could run out of DMEM in case we increase LSF_FALCON_ID_END more and more. +// The PMU supports the ACR task on GM20X_thru_VOLTA profiles only. +// In order to prevent LSF_FALCON_ID_END changes to affect older / shipped PMU ucodes (increase of DMEM footprint) +// adding PMU specific ***_END define capturing value covering all PMU profiles that this with the ACR task. +// +#define LSF_FALCON_ID_END_PMU (LSF_FALCON_ID_FBFALCON + 1) +#define LSF_WPR_HEADERS_TOTAL_SIZE_MAX_PMU (NV_ALIGN_UP((sizeof(LSF_WPR_HEADER) * LSF_FALCON_ID_END_PMU), LSF_WPR_HEADER_ALIGNMENT)) + +// +// In order to prevent LSF_FALCON_ID_END changes to affect older / shipped SEC2/ACR ucodes (increase of DMEM footprint) +// adding SEC2/ACR specific ***_END define covering all supported falcons in pre-hopper SEC2-RTOS/ACR ucode. +// +#define LSF_FALCON_ID_END_ACR_ON_SEC2 (LSF_FALCON_ID_NVJPG + 1) +#define LSF_WPR_HEADERS_TOTAL_SIZE_MAX_ACR_ON_SEC2 (NV_ALIGN_UP((sizeof(LSF_WPR_HEADER) * LSF_FALCON_ID_END_ACR_ON_SEC2), LSF_WPR_HEADER_ALIGNMENT)) + +// Maximum SUB WPR header size +#define LSF_SUB_WPR_HEADERS_TOTAL_SIZE_MAX (NV_ALIGN_UP((sizeof(LSF_SHARED_SUB_WPR_HEADER) * LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX), LSF_SUB_WPR_HEADER_ALIGNMENT)) + +/*! + * For the ucode surface alignment, We align to RM_PAGE_SIZE because of + * busMapRmAperture issues, not because of Falcon ucode alignment requirements + * which currently are that it be at least 256. + */ +#define LSF_UCODE_DATA_ALIGNMENT RM_PAGE_SIZE + +/*! + * ACR Descriptors used by ACR UC + */ + +/*! + * Supporting maximum of 2 regions. + * This is needed to pre-allocate space in DMEM + */ +#define RM_FLCN_ACR_MAX_REGIONS (2) +#define LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE (0x200) + +/*! + * startAddress - Starting address of region + * endAddress - Ending address of region + * regionID - Region ID + * readMask - Read Mask + * writeMask - WriteMask + * clientMask - Bit map of all clients currently using this region + * shadowMemStartAddress- FB location from where contents need to be copied to startAddress + */ +typedef struct _def_acr_dmem_region_prop +{ + NvU32 startAddress; + NvU32 endAddress; + NvU32 regionID; + NvU32 readMask; + NvU32 writeMask; + NvU32 clientMask; + NvU32 shadowMemStartAddress; +} RM_FLCN_ACR_REGION_PROP, *PRM_FLCN_ACR_REGION_PROP; + + +/*! + * noOfRegions - Number of regions used by RM. + * regionProps - Region properties + */ +typedef struct _def_acr_regions +{ + NvU32 noOfRegions; + RM_FLCN_ACR_REGION_PROP regionProps[RM_FLCN_ACR_MAX_REGIONS]; +} RM_FLCN_ACR_REGIONS, *PRM_FLCN_ACR_REGIONS; + +/*! + * bVprEnabled : When set, ACR_LOCKDOWN phase programs VPR range. Needs to be + : NvU32 because of alignment + * vprStartAddress : Start address of VPR region. SEC2 binary updates this value + * vprEndAddress : End address of VPR region. SEC2 binary updates this value + * hdcpPolicies : VPR display policies. SEC2 binary updates this value + */ +typedef struct _def_acr_vpr_dmem_desc +{ + NvU32 bVprEnabled; + NvU32 vprStartAddress; + NvU32 vprEndAddress; + NvU32 hdcpPolicies; +} ACR_BSI_VPR_DESC, *PACR_BSI_VPR_DESC; + +/*! + * reservedDmem - When the bootstrap owner has done bootstrapping other falcons, + * and need to switch into LS mode, it needs to have its own actual + * DMEM image copied into DMEM as part of LS setup. If ACR desc is at + * location 0, it will definitely get overwritten causing data corruption. + * Hence we are reserving 0x200 bytes to give room for any loading data. + * NOTE: This has to be the first member always + * signature - Signature of ACR ucode. + * wprRegionID - Region ID holding the WPR header and its details + * wprOffset - Offset from the WPR region holding the wpr header + * regions - Region descriptors + * ucodeBlobBase- Used for Tegra, stores non-WPR start address where kernel stores ucode blob + * ucodeBlobSize- Used for Tegra, stores the size of the ucode blob + */ +typedef struct _def_acr_dmem_desc +{ + NvU32 signatures[4]; + NvU32 wprRegionID; + NvU32 wprOffset; + NvU32 mmuMemoryRange; + RM_FLCN_ACR_REGIONS regions; + NvU32 ucodeBlobSize; + // uCodeBlobBase is moved after ucodeBlobSize to inherently align to qword (8 bytes) + NvU64 NV_DECLARE_ALIGNED(ucodeBlobBase, 8); + + /*! + * Do not change the offset of this descriptor as it shared between + * ACR_REGION_LOCKDOWN HS binary and SEC2. Any change in this structure + * need recompilation of SEC2 and ACR_LOCKDOWN HS binary + */ + ACR_BSI_VPR_DESC vprDesc; +} RM_FLCN_ACR_DESC, *PRM_FLCN_ACR_DESC; + +/*! +* Hub keys/nonce Structure in BSI +*/ +#define MAX_SFBHUB_ENCRYPTION_REGION_KEY_SIZE 4 + +typedef struct _def_acr_hub_scratch_data +{ + NvU32 key[MAX_SFBHUB_ENCRYPTION_REGION_KEY_SIZE]; + NvU32 nonce[MAX_SFBHUB_ENCRYPTION_REGION_KEY_SIZE]; +} ACR_BSI_HUB_DESC, *PACR_BSI_HUB_DESC; + +#define MAX_HUB_ENCRYPTION_REGION_COUNT 3 +typedef struct _def_acr_hub_scratch_array +{ + ACR_BSI_HUB_DESC entries[MAX_HUB_ENCRYPTION_REGION_COUNT]; +} ACR_BSI_HUB_DESC_ARRAY, *PACR_BSI_HUB_DESC_ARRAY; + +typedef struct _def_acr_reserved_dmem +{ + NvU32 reservedDmem[(LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE/4)]; // Always first.. +} ACR_RESERVED_DMEM, *PACR_RESERVED_DMEM; + +typedef struct _def_booter_reserved_dmem +{ + NvU32 reservedDmem[(LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE/4)]; // Always first.. +} BOOTER_RESERVED_DMEM; + +#define NV_FLCN_ACR_DESC_FLAGS_SIG_VERIF 0:0 +#define NV_FLCN_ACR_DESC_FLAGS_SIG_VERIF_DISABLE 0 +#define NV_FLCN_ACR_DESC_FLAGS_SIG_VERIF_ENABLE 1 + +// Macro defines to be consumed by RM to get GH100 GSP Inst_in_sys FMC boot status. +#define GSP_INST_IN_SYS_COMPLETION_STATUS_OK 0x55 +#define GSP_INST_IN_SYS_COMPLETION_STATUS_ERROR 0xAA +#define GSP_INST_IN_SYS_COMPLETION_STATUS_IN_PROGRESS 0x00 + +/*! + * Size of ACR phase in dword + */ +#define ACR_PHASE_SIZE_DWORD sizeof(RM_FLCN_ACR_DESC)/sizeof(NvU32) + +/*! + * Falcon Mode Tokens + * This is the value logged to a mailbox register to indicate that the + * falcon isn't booted in secure mode. + */ +#define LSF_FALCON_MODE_TOKEN_FLCN_INSECURE (0xDEADDEADU) + + +// LS encryption context, to store data to decrypt LS images. +#define LS_ENCRYPTION_AES128_CBC_IV_SIZE_BYTE (16) + +typedef struct +{ + NvU8 bLsEncrypted; + NvU8 rsvd[3]; + NvU8 lsEncIV[LS_ENCRYPTION_AES128_CBC_IV_SIZE_BYTE]; +} LSF_ENCRYPTION_CONTEXT, *PLSF_ENCRYPTION_CONTEXT; + +#endif // RMLSFM_H + diff --git a/userspace/sdks/nv570/nvalloc/common/inc/rmpbicmdif.h b/userspace/sdks/nv570/nvalloc/common/inc/rmpbicmdif.h new file mode 100644 index 0000000..cd4d092 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/rmpbicmdif.h @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2009-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * Shared postbox interface defines for RM and PMU. + */ + +#ifndef RMPBICMDIF_H +#define RMPBICMDIF_H + +/* + * Define the status of postbox interface at different instances. + * + * The values should be in accordance to the spec and must not be changed. + * A new PBI command must be submitted with the status NV_PBI_COMMAND_STATUS_UNDEFINED. + */ + +#define NV_PBI_COMMAND_STATUS 7:0 +#define NV_PBI_COMMAND_STATUS_UNDEFINED 0x00 // command submitted to PMU +#define NV_PBI_COMMAND_STATUS_SUCCESS 0x01 // command successfully completed by PMU +#define NV_PBI_COMMAND_STATUS_PENDING 0x02 // command accepted by PMU +#define NV_PBI_COMMAND_STATUS_BUSY 0x03 // command processing in PMU +#define NV_PBI_COMMAND_STATUS_UNSPECIFIED_FAILURE 0x04 // unknown failure or hang +#define NV_PBI_COMMAND_STATUS_INVALID_ADDRESS 0x05 // invalid address submitted to PMU +#define NV_PBI_COMMAND_STATUS_MORE_DATA 0x06 // user needs to send more data to PMU +#define NV_PBI_COMMAND_STATUS_INVALID_COMMAND 0x07 // invalid command submitted + +/* + * This corresponds to reserved bits of command register + */ +#define NV_PBI_COMMAND_RSVD_0 15:8 +#define NV_PBI_COMMAND_RSVD_0_VAL 0x00 + +/* + * This specifies the dword index if client is sending multiple dwords + * for single Dword this should be zero + */ +#define NV_PBI_COMMAND_BUFFER_INDEX 19:16 + +/* + * This specifies the total no. of dwords passed by client + */ +#define NV_PBI_COMMAND_BUFFER_SIZE 23:20 + +/* + * These corresponds the different function ID's supported out of PBI + */ +#define NV_PBI_COMMAND_FUNC_ID 27:24 +#define NV_PBI_COMMAND_FUNC_ID_GET_CAPABILITIES (0) +#define NV_PBI_COMMAND_FUNC_ID_EXECUTE_ROUTINE (11) + +// +// Definitions for common 'Execute Routine' calls that are +// shared across all GPUs. +// +// Common routine IDs must be in the range 0x80 to 0xff. +// +// Get GID: +#define NV_PBI_EXECUTE_ROUTINE_GET_GID 0x80 +// Get Feature: +#define NV_PBI_EXECUTE_ROUTINE_GET_FEATURE 0x81 +#define NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION 5:0 +#define NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION_ALLOWED 0x2 + +/* + * This corresponds to reserved field of command register + */ +#define NV_PBI_COMMAND_RSVD_1 28:28 +#define NV_PBI_COMMAND_RSVD_1_VAL 0x00 + +/* + * If this bit is set system will be notified on command completion + */ +#define NV_PBI_COMMAND_SYS_NOTIFY 29:29 +#define NV_PBI_COMMAND_SYS_NOTIFY_TRUE 0x01 +#define NV_PBI_COMMAND_SYS_NOTIFY_FALSE 0x00 + +/* + * If this bit is set driver will be notified of command completion status + */ +#define NV_PBI_COMMAND_DRV_NOTIFY 30:30 +#define NV_PBI_COMMAND_DRV_NOTIFY_TRUE 0x01 +#define NV_PBI_COMMAND_DRV_NOTIFY_FALSE 0x00 + +/* + * Defines the interrupt state of the PBI command + */ +#define NV_PBI_COMMAND_INTERRUPT 31:31 +#define NV_PBI_COMMAND_INTERRUPT_TRUE 0x01 +#define NV_PBI_COMMAND_INTERRUPT_FALSE 0x00 + +/* + * This sets the different fields of command register + */ +#define PBI_SET_COMMAND_PARAMS(status, r0, index, sz, cmd, r1, sys, \ + drv, intr, val) \ +{ \ + val = DRF_NUM(_PBI, _COMMAND, _STATUS, status) | \ + DRF_NUM(_PBI, _COMMAND, _RSVD_0, r0) | \ + DRF_NUM(_PBI, _COMMAND, _BUFFER_INDEX, index) | \ + DRF_NUM(_PBI, _COMMAND, _BUFFER_SIZE, sz) | \ + DRF_NUM(_PBI, _COMMAND, _FUNC_ID, cmd) | \ + DRF_NUM(_PBI, _COMMAND, _RSVD_1, r1) | \ + DRF_NUM(_PBI, _COMMAND, _SYS_NOTIFY, sys) | \ + DRF_NUM(_PBI, _COMMAND, _DRV_NOTIFY, drv) | \ + DRF_NUM(_PBI, _COMMAND, _INTERRUPT, intr); \ +} + +/* + * postbox_clients_mutex_id + * + * Define the Mutex ID for different PBI clients + */ +enum postbox_clients_mutex_id +{ + PBI_CLIENT_NONE = 0x00, + PBI_CLIENT_DRIVER_PCIPBI_SHIM = 0x15 +}; + +#endif /* RMPBICMDIF_H */ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/sec2/nvdm_payload_cmd_response.h b/userspace/sdks/nv570/nvalloc/common/inc/sec2/nvdm_payload_cmd_response.h new file mode 100644 index 0000000..fa117aa --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/sec2/nvdm_payload_cmd_response.h @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/* + * @file nvdm_payload_cmd_response.h + * @brief Header file containing the definition for the + * NVDM_PAYLOAD_COMMAND_RESPONSE structure, which + * is required for SMBPBI-specific payloads and needs + * to be visible by bothe the PMU and the driver sides. + */ + +#ifndef _NVDM_PAYLOAD_CMD_RESPONSE_H +#define _NVDM_PAYLOAD_CMD_RESPONSE_H + +#pragma pack(1) +typedef struct +{ + NvU32 taskId; + NvU32 commandNvdmType; + NvU32 errorCode; +} NVDM_PAYLOAD_COMMAND_RESPONSE; +#pragma pack() + +#endif // _NVDM_PAYLOAD_CMD_RESPONSE_H diff --git a/userspace/sdks/nv570/nvalloc/common/inc/sec2/sec2_emem_channels.h b/userspace/sdks/nv570/nvalloc/common/inc/sec2/sec2_emem_channels.h new file mode 100644 index 0000000..3681f24 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/sec2/sec2_emem_channels.h @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _SEC2_EMEM_CHANNELS_H_ +#define _SEC2_EMEM_CHANNELS_H_ + +// +// NVDM (NVIDIA Data Model) overlayed on MCTP (Management Component Transport +// Protocol) and sent over EMEM is the communication mechanism used between SEC2 +// management partition and CPU-RM/other uprocs. +// + +// RM uses channel 0 for SEC2 EMEM. +#define SEC2_EMEM_CHANNEL_RM 0x0 + +// EMEM channel 0 (RM) is allocated 1K bytes. +#define SEC2_EMEM_CHANNEL_RM_SIZE 1024 + +#endif // _SEC2_EMEM_CHANNELS_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmrsakeys.h b/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmrsakeys.h new file mode 100644 index 0000000..882497b --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmrsakeys.h @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @file rmspdmrsakeys.h + * @brief SPDM Interfaces - RSA keys + */ + +#ifndef _RM_SPDM_RSA_KEYS_H_ +#define _RM_SPDM_RSA_KEYS_H_ + +// +// Note !! +// All key components order must be big-endian(network order). +// If any change for these key components, need to rebuild GSP/RM code as well. +// +#ifndef USE_MBEDTLS + +const NvU8 g_rsa3k_modulus_mutual_authentication_prod[] = +{ + 0xcd, 0x65, 0xd2, 0xca, 0x74, 0xd1, 0x76, 0x76, 0x9f, 0x18, 0x23, 0x64, 0x55, 0x84, 0x98, 0x60, + 0x71, 0xeb, 0x42, 0x6d, 0xa4, 0x98, 0xf9, 0x92, 0xea, 0x83, 0x5b, 0x9b, 0xe2, 0x66, 0x8b, 0x43, + 0x14, 0x4d, 0x5a, 0xb8, 0xdb, 0x68, 0x2b, 0xfa, 0x5f, 0xdf, 0x7f, 0xf4, 0xfd, 0x5e, 0x42, 0x34, + 0x09, 0x98, 0xa1, 0x21, 0x98, 0x4c, 0x8d, 0xbc, 0x99, 0xdb, 0xea, 0xc1, 0xe3, 0x42, 0xe6, 0x67, + 0x26, 0x86, 0x2c, 0xd0, 0xdb, 0xf3, 0x9c, 0x12, 0xad, 0xb3, 0x82, 0x93, 0x9c, 0xb9, 0xae, 0x98, + 0x82, 0xeb, 0x59, 0xb6, 0x5c, 0x09, 0x9f, 0xa6, 0x15, 0x30, 0xa0, 0xc6, 0x77, 0xd5, 0xae, 0xa2, + 0x91, 0x65, 0x24, 0xc3, 0x7d, 0x9b, 0xa4, 0x2c, 0x31, 0x73, 0x41, 0x26, 0x72, 0xe7, 0x2f, 0xb8, + 0x60, 0xd8, 0xce, 0xb8, 0xd8, 0x4b, 0x90, 0x6c, 0xa3, 0x19, 0x7e, 0x2b, 0xd5, 0xf6, 0x05, 0x8a, + 0x2b, 0xb9, 0x9e, 0x27, 0xba, 0x2e, 0x16, 0x81, 0x9a, 0x9e, 0xf5, 0x6c, 0x38, 0x0a, 0x01, 0xea, + 0xd6, 0xe1, 0xa4, 0x83, 0x76, 0xd0, 0x68, 0xbb, 0x90, 0x63, 0xce, 0x1c, 0x8c, 0x6f, 0x0f, 0x6b, + 0x65, 0x5b, 0x12, 0xe2, 0x92, 0x60, 0x79, 0x88, 0x99, 0x5e, 0x17, 0x89, 0x4d, 0x54, 0xb1, 0x87, + 0x9c, 0xe6, 0x6d, 0x3c, 0x56, 0x1c, 0x3a, 0x3e, 0xb9, 0x16, 0x2c, 0xc8, 0xda, 0x1a, 0xfb, 0x5c, + 0xd7, 0x16, 0x3c, 0x74, 0xda, 0x78, 0xf1, 0x53, 0xd7, 0x66, 0x97, 0x8a, 0x57, 0x0b, 0x86, 0x57, + 0x2c, 0x45, 0xa1, 0x5e, 0xae, 0x39, 0x39, 0xde, 0xe0, 0x40, 0x6f, 0xdb, 0x4c, 0xd8, 0xc1, 0x8c, + 0xdb, 0xce, 0xea, 0x05, 0xc0, 0xbc, 0x89, 0x72, 0x15, 0xfd, 0xbf, 0xb2, 0xb7, 0xf2, 0x5d, 0x05, + 0xdd, 0x2b, 0x53, 0xa4, 0x03, 0x1e, 0x11, 0x67, 0xa7, 0x0f, 0x87, 0xfb, 0x57, 0x48, 0x91, 0xc8, + 0x02, 0xb7, 0x46, 0x3d, 0x82, 0xcd, 0x06, 0x4a, 0x79, 0x0f, 0xa5, 0x8a, 0xac, 0xfb, 0xb7, 0xc2, + 0xf0, 0x95, 0x19, 0x4c, 0x78, 0x7a, 0xc7, 0xd7, 0x70, 0xee, 0x6e, 0x59, 0xaf, 0x51, 0x9b, 0x11, + 0x03, 0xd3, 0x56, 0xb3, 0x05, 0x5a, 0xbb, 0x1f, 0xbd, 0xc2, 0x0e, 0x89, 0x77, 0xb4, 0xc1, 0x02, + 0xf9, 0x97, 0x56, 0x07, 0x2e, 0x4e, 0x2d, 0x01, 0x73, 0x89, 0x7d, 0xf3, 0xc9, 0x8c, 0x88, 0x2c, + 0x79, 0xd9, 0x47, 0x34, 0x9e, 0x32, 0x51, 0xd4, 0xa6, 0x7e, 0xd1, 0x08, 0xda, 0xc0, 0x76, 0x24, + 0x8e, 0x25, 0x73, 0x14, 0x30, 0xd2, 0x17, 0x37, 0xbc, 0xe0, 0x3e, 0xa2, 0x47, 0xff, 0xe2, 0x4e, + 0x9b, 0x31, 0x6c, 0xe6, 0x54, 0xaf, 0x62, 0x3a, 0xcd, 0xfa, 0x2f, 0xaf, 0x73, 0x2e, 0x73, 0x4a, + 0x3a, 0x60, 0xa8, 0xa9, 0xfc, 0x77, 0xb2, 0x57, 0xdd, 0x3a, 0xfa, 0xce, 0x35, 0xc3, 0xea, 0xa9 +}; + +const NvU8 g_rsa3k_public_exponent_mutual_authentication_prod[] = {0x00, 0x01, 0x00, 0x01}; + +const NvU8 g_rsa3k_private_exponent_mutual_authentication_prod[] = +{ + 0x04, 0x85, 0xc0, 0x6d, 0x6a, 0xc3, 0x0d, 0xeb, 0xb0, 0xb7, 0x14, 0x58, 0x6a, 0x35, 0xa0, 0x31, + 0x47, 0x70, 0xd9, 0xa6, 0x96, 0x60, 0x33, 0xe6, 0x93, 0x39, 0x4f, 0x34, 0x10, 0x79, 0x17, 0x89, + 0xe3, 0x91, 0x8c, 0x74, 0xd3, 0x87, 0xe8, 0xa7, 0xfb, 0xa2, 0x6d, 0x2b, 0xd4, 0xc4, 0x55, 0x8a, + 0xd5, 0xc7, 0x41, 0x8c, 0xfe, 0xd0, 0x78, 0xb2, 0x75, 0x64, 0xcd, 0x37, 0x75, 0xac, 0x8c, 0x6c, + 0x2b, 0x01, 0xfb, 0x4c, 0xa1, 0xfb, 0x2c, 0x2d, 0x91, 0x1b, 0x89, 0xbd, 0x7c, 0x4e, 0xee, 0x54, + 0x37, 0x55, 0xbd, 0x44, 0xf1, 0xa3, 0xd8, 0x9b, 0x0c, 0x8e, 0x64, 0x8b, 0xda, 0x29, 0x9f, 0x4d, + 0xbf, 0x0b, 0xce, 0x12, 0x6b, 0xda, 0x98, 0x73, 0xcc, 0xa6, 0xf0, 0x01, 0x4b, 0xb8, 0x61, 0x69, + 0x9a, 0xe0, 0x4b, 0x34, 0xf0, 0xb6, 0x41, 0x38, 0xd3, 0x22, 0x85, 0x8f, 0xab, 0x87, 0x5e, 0x39, + 0xdb, 0x9a, 0x9b, 0xab, 0xde, 0x42, 0x93, 0x86, 0x49, 0x04, 0x44, 0xfd, 0x1c, 0x02, 0xc4, 0x66, + 0x0a, 0x53, 0x2f, 0x8f, 0x21, 0x77, 0x97, 0x46, 0xc1, 0xf8, 0x20, 0x9a, 0xaa, 0x50, 0xeb, 0xb2, + 0xfe, 0xa9, 0x51, 0xb2, 0x21, 0x6c, 0xf4, 0x60, 0x18, 0x98, 0x64, 0xc6, 0x46, 0x28, 0x9e, 0x3c, + 0x11, 0x2d, 0x55, 0xac, 0x65, 0x6c, 0xfb, 0xaf, 0x53, 0xdd, 0xf4, 0x20, 0x7c, 0x04, 0xea, 0x11, + 0xe8, 0x25, 0x65, 0x15, 0x32, 0x8e, 0x08, 0x84, 0xc7, 0x41, 0x13, 0x58, 0x73, 0x61, 0x63, 0x1f, + 0xef, 0x30, 0x1d, 0x3d, 0x36, 0x08, 0x3a, 0xb2, 0xf7, 0x25, 0x56, 0xfc, 0x03, 0x78, 0xb5, 0x45, + 0xf4, 0x60, 0x47, 0x95, 0x1c, 0x6a, 0x2b, 0xfa, 0xe3, 0x1c, 0x2f, 0x0d, 0x2a, 0x90, 0x43, 0x65, + 0x09, 0xbe, 0x63, 0x71, 0x33, 0xd3, 0x29, 0x31, 0xd5, 0x29, 0x26, 0xa3, 0x15, 0xc2, 0x46, 0x70, + 0xb4, 0x3e, 0x23, 0xaf, 0xb6, 0xfb, 0x87, 0x8f, 0x39, 0xf0, 0xd2, 0x3e, 0x35, 0x3c, 0xec, 0x7a, + 0xd2, 0x0f, 0xd8, 0xa4, 0x0c, 0x19, 0xc8, 0xee, 0x47, 0x7c, 0x1e, 0xd6, 0x67, 0x31, 0xe2, 0x9d, + 0xc0, 0x65, 0x64, 0x60, 0xe7, 0xd2, 0xeb, 0xe1, 0x02, 0xd5, 0x92, 0x7c, 0x51, 0xf1, 0x3b, 0x12, + 0x00, 0x65, 0xfd, 0x2b, 0x13, 0x15, 0xfa, 0x6d, 0x99, 0x1d, 0xd3, 0x03, 0x77, 0xb1, 0xb0, 0xf0, + 0x39, 0x7c, 0x27, 0x13, 0x30, 0xba, 0xff, 0x4d, 0x2e, 0xda, 0xe0, 0x37, 0xad, 0xf4, 0x49, 0x0a, + 0xdd, 0x1e, 0x87, 0x8c, 0xc9, 0x6b, 0xf8, 0xc6, 0xb3, 0x05, 0xeb, 0x6c, 0x5f, 0x84, 0x64, 0x62, + 0x1c, 0xf6, 0x04, 0x6f, 0xd7, 0xa9, 0xbc, 0x22, 0x97, 0xdb, 0x8d, 0xa5, 0xe1, 0x3a, 0x5c, 0x0d, + 0x7b, 0x78, 0x25, 0x98, 0x04, 0x7f, 0x2b, 0x59, 0x5b, 0x7c, 0xf7, 0x73, 0x37, 0x7f, 0x92, 0x8d +}; + +#else + +const NvU8 g_rsa3k_modulus_mutual_authentication_prod[] = +{ + 0xa9, 0xea, 0xc3, 0x35, 0xce, 0xfa, 0x3a, 0xdd, 0x57, 0xb2, 0x77, 0xfc, 0xa9, 0xa8, 0x60, 0x3a, + 0x4a, 0x73, 0x2e, 0x73, 0xaf, 0x2f, 0xfa, 0xcd, 0x3a, 0x62, 0xaf, 0x54, 0xe6, 0x6c, 0x31, 0x9b, + 0x4e, 0xe2, 0xff, 0x47, 0xa2, 0x3e, 0xe0, 0xbc, 0x37, 0x17, 0xd2, 0x30, 0x14, 0x73, 0x25, 0x8e, + 0x24, 0x76, 0xc0, 0xda, 0x08, 0xd1, 0x7e, 0xa6, 0xd4, 0x51, 0x32, 0x9e, 0x34, 0x47, 0xd9, 0x79, + 0x2c, 0x88, 0x8c, 0xc9, 0xf3, 0x7d, 0x89, 0x73, 0x01, 0x2d, 0x4e, 0x2e, 0x07, 0x56, 0x97, 0xf9, + 0x02, 0xc1, 0xb4, 0x77, 0x89, 0x0e, 0xc2, 0xbd, 0x1f, 0xbb, 0x5a, 0x05, 0xb3, 0x56, 0xd3, 0x03, + 0x11, 0x9b, 0x51, 0xaf, 0x59, 0x6e, 0xee, 0x70, 0xd7, 0xc7, 0x7a, 0x78, 0x4c, 0x19, 0x95, 0xf0, + 0xc2, 0xb7, 0xfb, 0xac, 0x8a, 0xa5, 0x0f, 0x79, 0x4a, 0x06, 0xcd, 0x82, 0x3d, 0x46, 0xb7, 0x02, + 0xc8, 0x91, 0x48, 0x57, 0xfb, 0x87, 0x0f, 0xa7, 0x67, 0x11, 0x1e, 0x03, 0xa4, 0x53, 0x2b, 0xdd, + 0x05, 0x5d, 0xf2, 0xb7, 0xb2, 0xbf, 0xfd, 0x15, 0x72, 0x89, 0xbc, 0xc0, 0x05, 0xea, 0xce, 0xdb, + 0x8c, 0xc1, 0xd8, 0x4c, 0xdb, 0x6f, 0x40, 0xe0, 0xde, 0x39, 0x39, 0xae, 0x5e, 0xa1, 0x45, 0x2c, + 0x57, 0x86, 0x0b, 0x57, 0x8a, 0x97, 0x66, 0xd7, 0x53, 0xf1, 0x78, 0xda, 0x74, 0x3c, 0x16, 0xd7, + 0x5c, 0xfb, 0x1a, 0xda, 0xc8, 0x2c, 0x16, 0xb9, 0x3e, 0x3a, 0x1c, 0x56, 0x3c, 0x6d, 0xe6, 0x9c, + 0x87, 0xb1, 0x54, 0x4d, 0x89, 0x17, 0x5e, 0x99, 0x88, 0x79, 0x60, 0x92, 0xe2, 0x12, 0x5b, 0x65, + 0x6b, 0x0f, 0x6f, 0x8c, 0x1c, 0xce, 0x63, 0x90, 0xbb, 0x68, 0xd0, 0x76, 0x83, 0xa4, 0xe1, 0xd6, + 0xea, 0x01, 0x0a, 0x38, 0x6c, 0xf5, 0x9e, 0x9a, 0x81, 0x16, 0x2e, 0xba, 0x27, 0x9e, 0xb9, 0x2b, + 0x8a, 0x05, 0xf6, 0xd5, 0x2b, 0x7e, 0x19, 0xa3, 0x6c, 0x90, 0x4b, 0xd8, 0xb8, 0xce, 0xd8, 0x60, + 0xb8, 0x2f, 0xe7, 0x72, 0x26, 0x41, 0x73, 0x31, 0x2c, 0xa4, 0x9b, 0x7d, 0xc3, 0x24, 0x65, 0x91, + 0xa2, 0xae, 0xd5, 0x77, 0xc6, 0xa0, 0x30, 0x15, 0xa6, 0x9f, 0x09, 0x5c, 0xb6, 0x59, 0xeb, 0x82, + 0x98, 0xae, 0xb9, 0x9c, 0x93, 0x82, 0xb3, 0xad, 0x12, 0x9c, 0xf3, 0xdb, 0xd0, 0x2c, 0x86, 0x26, + 0x67, 0xe6, 0x42, 0xe3, 0xc1, 0xea, 0xdb, 0x99, 0xbc, 0x8d, 0x4c, 0x98, 0x21, 0xa1, 0x98, 0x09, + 0x34, 0x42, 0x5e, 0xfd, 0xf4, 0x7f, 0xdf, 0x5f, 0xfa, 0x2b, 0x68, 0xdb, 0xb8, 0x5a, 0x4d, 0x14, + 0x43, 0x8b, 0x66, 0xe2, 0x9b, 0x5b, 0x83, 0xea, 0x92, 0xf9, 0x98, 0xa4, 0x6d, 0x42, 0xeb, 0x71, + 0x60, 0x98, 0x84, 0x55, 0x64, 0x23, 0x18, 0x9f, 0x76, 0x76, 0xd1, 0x74, 0xca, 0xd2, 0x65, 0xcd, + 0x00 +}; + +const NvU8 g_rsa3k_public_exponent_mutual_authentication_prod[] = {0x01, 0x00, 0x01, 0x00}; + +const NvU8 g_rsa3k_private_exponent_mutual_authentication_prod[] = +{ + 0x8d, 0x92, 0x7f, 0x37, 0x73, 0xf7, 0x7c, 0x5b, 0x59, 0x2b, 0x7f, 0x04, 0x98, 0x25, 0x78, 0x7b, + 0x0d, 0x5c, 0x3a, 0xe1, 0xa5, 0x8d, 0xdb, 0x97, 0x22, 0xbc, 0xa9, 0xd7, 0x6f, 0x04, 0xf6, 0x1c, + 0x62, 0x64, 0x84, 0x5f, 0x6c, 0xeb, 0x05, 0xb3, 0xc6, 0xf8, 0x6b, 0xc9, 0x8c, 0x87, 0x1e, 0xdd, + 0x0a, 0x49, 0xf4, 0xad, 0x37, 0xe0, 0xda, 0x2e, 0x4d, 0xff, 0xba, 0x30, 0x13, 0x27, 0x7c, 0x39, + 0xf0, 0xb0, 0xb1, 0x77, 0x03, 0xd3, 0x1d, 0x99, 0x6d, 0xfa, 0x15, 0x13, 0x2b, 0xfd, 0x65, 0x00, + 0x12, 0x3b, 0xf1, 0x51, 0x7c, 0x92, 0xd5, 0x02, 0xe1, 0xeb, 0xd2, 0xe7, 0x60, 0x64, 0x65, 0xc0, + 0x9d, 0xe2, 0x31, 0x67, 0xd6, 0x1e, 0x7c, 0x47, 0xee, 0xc8, 0x19, 0x0c, 0xa4, 0xd8, 0x0f, 0xd2, + 0x7a, 0xec, 0x3c, 0x35, 0x3e, 0xd2, 0xf0, 0x39, 0x8f, 0x87, 0xfb, 0xb6, 0xaf, 0x23, 0x3e, 0xb4, + 0x70, 0x46, 0xc2, 0x15, 0xa3, 0x26, 0x29, 0xd5, 0x31, 0x29, 0xd3, 0x33, 0x71, 0x63, 0xbe, 0x09, + 0x65, 0x43, 0x90, 0x2a, 0x0d, 0x2f, 0x1c, 0xe3, 0xfa, 0x2b, 0x6a, 0x1c, 0x95, 0x47, 0x60, 0xf4, + 0x45, 0xb5, 0x78, 0x03, 0xfc, 0x56, 0x25, 0xf7, 0xb2, 0x3a, 0x08, 0x36, 0x3d, 0x1d, 0x30, 0xef, + 0x1f, 0x63, 0x61, 0x73, 0x58, 0x13, 0x41, 0xc7, 0x84, 0x08, 0x8e, 0x32, 0x15, 0x65, 0x25, 0xe8, + 0x11, 0xea, 0x04, 0x7c, 0x20, 0xf4, 0xdd, 0x53, 0xaf, 0xfb, 0x6c, 0x65, 0xac, 0x55, 0x2d, 0x11, + 0x3c, 0x9e, 0x28, 0x46, 0xc6, 0x64, 0x98, 0x18, 0x60, 0xf4, 0x6c, 0x21, 0xb2, 0x51, 0xa9, 0xfe, + 0xb2, 0xeb, 0x50, 0xaa, 0x9a, 0x20, 0xf8, 0xc1, 0x46, 0x97, 0x77, 0x21, 0x8f, 0x2f, 0x53, 0x0a, + 0x66, 0xc4, 0x02, 0x1c, 0xfd, 0x44, 0x04, 0x49, 0x86, 0x93, 0x42, 0xde, 0xab, 0x9b, 0x9a, 0xdb, + 0x39, 0x5e, 0x87, 0xab, 0x8f, 0x85, 0x22, 0xd3, 0x38, 0x41, 0xb6, 0xf0, 0x34, 0x4b, 0xe0, 0x9a, + 0x69, 0x61, 0xb8, 0x4b, 0x01, 0xf0, 0xa6, 0xcc, 0x73, 0x98, 0xda, 0x6b, 0x12, 0xce, 0x0b, 0xbf, + 0x4d, 0x9f, 0x29, 0xda, 0x8b, 0x64, 0x8e, 0x0c, 0x9b, 0xd8, 0xa3, 0xf1, 0x44, 0xbd, 0x55, 0x37, + 0x54, 0xee, 0x4e, 0x7c, 0xbd, 0x89, 0x1b, 0x91, 0x2d, 0x2c, 0xfb, 0xa1, 0x4c, 0xfb, 0x01, 0x2b, + 0x6c, 0x8c, 0xac, 0x75, 0x37, 0xcd, 0x64, 0x75, 0xb2, 0x78, 0xd0, 0xfe, 0x8c, 0x41, 0xc7, 0xd5, + 0x8a, 0x55, 0xc4, 0xd4, 0x2b, 0x6d, 0xa2, 0xfb, 0xa7, 0xe8, 0x87, 0xd3, 0x74, 0x8c, 0x91, 0xe3, + 0x89, 0x17, 0x79, 0x10, 0x34, 0x4f, 0x39, 0x93, 0xe6, 0x33, 0x60, 0x96, 0xa6, 0xd9, 0x70, 0x47, + 0x31, 0xa0, 0x35, 0x6a, 0x58, 0x14, 0xb7, 0xb0, 0xeb, 0x0d, 0xc3, 0x6a, 0x6d, 0xc0, 0x85, 0x04 +}; + +#endif + +#endif // _RM_SPDM_RSA_KEYS_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmtransport.h b/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmtransport.h new file mode 100644 index 0000000..9440ed8 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmtransport.h @@ -0,0 +1,97 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _RM_SPDM_TRANSPORT_H_ +#define _RM_SPDM_TRANSPORT_H_ + +/* ------------------------- Macros and Defines ----------------------------- */ +// +// The following defines and macros are shared for any message-related constants +// shared between RM and endpoint. +// + +// Defines used for polling-based communication between Kernel-RM and GSP. +#define NV_SPDM_PARTITION_BOOT_SUCCESS (0x59D3B007) +#define NV_SPDM_REQUESTER_SECRETS_DERIVED (0x59D35DED) +#define NV_SPDM_NO_MESSAGE_PENDING_TOKEN (0x00000000) +#define NV_SPDM_REQUESTER_MESSAGE_PENDING_TOKEN (0xAFAFAFAF) +#define NV_SPDM_RESPONDER_MESSAGE_PENDING_TOKEN (0xFAFAFAFA) +#define NV_SPDM_SECRET_TEARDOWN_SUCCESS (0x59D35EDE) +#define NV_SPDM_SECRET_TEARDOWN_FAILURE (0x59D35EFA) +#define NV_SPDM_SECRET_TEARDOWN_ACK (0x59D35EAC) + +// TDISP Defines for polling-based communication between Kernel-RM and GSP. +#define NV_TDISP_PARTITION_BOOT_SUCCESS (0x59D4B001) + +// +// The SPDM communication between Kernel-RM and GSP utilizes a buffer in sysmem accessible +// to both endpoints. The sysmem buffer will contain both a custom NV-defined header +// describing the message type and size, and the SPDM payload itself. +// + +// +// Size of the full transportation buffer (header + SPDM payload) allocated in sysmem. +// We need to align sysmem allocations with page size, but need less space. So we +// over-allocate and tell GSP we only have the size we actually need. +// +#define NV_SPDM_SYSMEM_SURFACE_SIZE_PAGE_ALIGNED (0x2000) +#define NV_SPDM_SYSMEM_SURFACE_SIZE_IN_BYTES (0x1200) +#define NV_SPDM_SYSMEM_SURFACE_ALIGNMENT_IN_BYTES (256) + +// Struct and defines for the header which prepends the SPDM payload. +#pragma pack(1) +typedef struct _NV_SPDM_DESC_HEADER +{ + NvU32 msgType; + NvU32 msgSizeByte; + NvU32 version; +} NV_SPDM_DESC_HEADER, *PNV_SPDM_DESC_HEADER; +#pragma pack() + +#define NV_SPDM_MESSAGE_TYPE_NORMAL (0) +#define NV_SPDM_MESSAGE_TYPE_SECURED (1) +#define NV_SPDM_MESSAGE_TYPE_APPLICATION (2) +#define NV_SPDM_DESC_HEADER_VERSION_1_0 (0x10) +#define NV_SPDM_DESC_HEADER_VERSION_CURRENT (NV_SPDM_DESC_HEADER_VERSION_1_0) +#define NV_SPDM_DESC_HEADER_ALIGNMENT_IN_BYTES (NV_SPDM_SYSMEM_SURFACE_ALIGNMENT_IN_BYTES) + +// Define utilized to determine size available for SPDM payload +#define NV_SPDM_MAX_SPDM_PAYLOAD_SIZE (NV_SPDM_SYSMEM_SURFACE_SIZE_IN_BYTES - sizeof(NV_SPDM_DESC_HEADER)) +#define NV_SPDM_MAX_TRANSCRIPT_BUFFER_SIZE (2 * NV_SPDM_MAX_SPDM_PAYLOAD_SIZE) + +// Limited by the transport size, do not increase without increasing transport buffer. +#define NV_SPDM_MAX_RANDOM_MSG_BYTES (0x0) + +#ifdef NVRM +#include "gpu/mem_mgr/mem_desc.h" +#else +#ifndef ADDR_SYSMEM +// System memory (PCI) +#define ADDR_SYSMEM (1) +#endif // ADDR_SYSMEM +#endif //NVRM + +#define NV_SPDM_DMA_ADDR_SPACE_DEFAULT (ADDR_SYSMEM) +#define NV_SPDM_DMA_REGION_ID_DEFAULT (0) + +#endif // _RM_SPDM_TRANSPORT_H_ diff --git a/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmvendordef.h b/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmvendordef.h new file mode 100644 index 0000000..76eec37 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/common/inc/spdm/rmspdmvendordef.h @@ -0,0 +1,161 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _RMSPDMVENDORDEF_H_ +#define _RMSPDMVENDORDEF_H_ +#include <nvtypes.h> + +/*! + * In NVIDIA's implementation of SPDM, we will support application messages + * which are payloads delivered encrypted in SPDM secured messages, + * sent across the SPDM channel, but do not contain SPDM messages. + * Instead, these messages are for NVIDIA-application purposes. + * These messages and relevant structs are defined here. + */ + +/* ------------------------- Application Payloads ------------------------ */ +/*! + * All application messages will follow a simple header format in order to + * simplify the parsing and forwarding of application messages to the + * relevant recipient endpoint. This format is defined as follows: + * + * +----------------------------------+ + * | RM_SPDM_NV_CMD | + * | Command Type | Command Payload | + * | (4 bytes) (n - 4 - 4 bytes) | + * +----------------------------------+ + * + * The Command Type value defines the expected structure of the payload, + * and how it should be interpreted. + * + * There are two major categories of commands - requests and responses. + * The values of 0x00000000-0x7FFFFFFF are reserved for requests. + * The values of 0x80000000-0xFFFFFFFF are reserved for responses. + */ + +typedef enum +{ + // Request commands. + RM_SPDM_NV_CMD_TYPE_REQ_KEYMGR_NVLE = 0x00000000, + // 0x00000001 - 0x7FFFFFFF reserved. + + // Response commands. + RM_SPDM_NV_CMD_TYPE_RSP_SUCCESS = 0x80000000, + // 0x80000001 - 0xFFFFFFFF reserved. + RM_SPDM_NV_CMD_TYPE_RSP_ERROR = 0xFFFFFFFF, +} RM_SPDM_NV_CMD_TYPE; + +typedef struct +{ + NvU32 cmdType; +} RM_SPDM_NV_CMD_HDR; + +/* ------------------------- NVIDIA Requests -------------------------------- */ +/*! + * Request Command: Key Manager NVLE Command + * Command Type: 0x00000000 + * + * Fields: + * Payload - Variable size opaque data blob containing command meant for Key manager + * + * This command is a generic wrapper around all commands that must be forwarded to Key Manager for NvLink Encryption. + * The command structure itself is opaque, as SPDM need not process any of the message itself. This limits coupling + * between the two command interfaces. + */ +typedef struct +{ + RM_SPDM_NV_CMD_HDR hdr; + + NvU8 payload[]; +} RM_SPDM_NV_CMD_REQ_KEYMGR_NVLE; + +/*! + * Union of all known requests, so responder can allocate enough space for all possible requests. + */ +typedef union { + RM_SPDM_NV_CMD_HDR hdr; + + RM_SPDM_NV_CMD_REQ_KEYMGR_NVLE keyMgrNvle; +} RM_SPDM_NV_CMD_REQ; + +/* ------------------------- NVIDIA Responses ------------------------------- */ +/*! + * Response Command: Success + * Command Type: 0x80000000 + * + * This command is a simple payload for the responding endpoint to + * return success. It has no Command Payload. + */ +typedef struct +{ + RM_SPDM_NV_CMD_HDR hdr; +} RM_SPDM_NV_CMD_RSP_SUCCESS; + +/*! + * Response Command: Error + * Command Type: 0xFFFFFFFF + * + * This command is a generic placeholder error response to return specific + * failure code to application endpoint. This is used to return an error + * back to the original application which sent the message, rather than + * suggesting that there was some SPDM or transport layer error. + */ +typedef struct +{ + RM_SPDM_NV_CMD_HDR hdr; + NV_STATUS status; +} RM_SPDM_NV_CMD_RSP_ERROR; + +/*! + * Union of all known responses, so requester can allocate enough space for all possible responses. + */ +typedef union { + RM_SPDM_NV_CMD_HDR hdr; + + RM_SPDM_NV_CMD_RSP_SUCCESS success; + RM_SPDM_NV_CMD_RSP_ERROR error; +} RM_SPDM_NV_CMD_RSP; + +#pragma pack() + +/* ------------------------- Macros ----------------------------------------- */ +#define SPDM_MAX_MEASUREMENT_BLOCK_COUNT (64) +#define SPDM_CERT_DEFAULT_SLOT_ID (0) +#define SPDM_CAPABILITIES_CT_EXPONENT_MAX (0x1F) +#define SPDM_DEFAULT_HEARTBEAT_PERIOD_IN_SEC (4) + +#define NV_BYTE_TO_BIT_OVERFLOW_MASK_UINT32 (0xE0000000) +#define IS_BYTE_TO_BIT_OVERFLOW_UINT32(a) \ + ((a & NV_BYTE_TO_BIT_OVERFLOW_MASK_UINT32) != 0) + +/* ---------------------- Public defines ------------------------------------- */ +#define NV_KEY_UPDATE_TRIGGER_THRESHOLD (0x10000000) + +#define NV_KEY_UPDATE_TRIGGER_ID_INVALID (0) +#define NV_KEY_UPDATE_TRIGGER_ID_GET_MEASUREMENTS (1) +#define NV_KEY_UPDATE_TRIGGER_ID_HEARTBEAT (2) +#define NV_KEY_UPDATE_TRIGGER_ID_LAST (3) + +#define NV_SPDM_ENCAP_CERT_SIZE_IN_BYTE (578) + +#endif // _RMSPDMVENDORDEF_H_ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-caps.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-caps.h new file mode 100644 index 0000000..35bbf7c --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-caps.h @@ -0,0 +1,94 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_CAPS_H_ +#define _NV_CAPS_H_ + +#include <nv-kernel-interface-api.h> + +/* + * Opaque OS-specific struct; on Linux, this has member + * 'struct proc_dir_entry'. + */ +typedef struct nv_cap nv_cap_t; + +/* + * Creates directory named "capabilities" under the provided path. + * + * @param[in] path Absolute path + * + * Returns a valid nv_cap_t upon success. Otherwise, returns NULL. + */ +nv_cap_t* NV_API_CALL nv_cap_init(const char *path); + +/* + * Creates capability directory entry + * + * @param[in] parent_cap Parent capability directory + * @param[in] name Capability directory's name + * @param[in] mode Capability directory's access mode + * + * Returns a valid nv_cap_t upon success. Otherwise, returns NULL. + */ +nv_cap_t* NV_API_CALL nv_cap_create_dir_entry(nv_cap_t *parent_cap, const char *name, int mode); + +/* + * Creates capability file entry + * + * @param[in] parent_cap Parent capability directory + * @param[in] name Capability file's name + * @param[in] mode Capability file's access mode + * + * Returns a valid nv_cap_t upon success. Otherwise, returns NULL. + */ +nv_cap_t* NV_API_CALL nv_cap_create_file_entry(nv_cap_t *parent_cap, const char *name, int mode); + +/* + * Destroys capability entry + * + * @param[in] cap Capability entry + */ +void NV_API_CALL nv_cap_destroy_entry(nv_cap_t *cap); + +/* + * Validates and duplicates the provided file descriptor + * + * @param[in] cap Capability entry + * @param[in] fd File descriptor to be validated + * + * Returns duplicate fd upon success. Otherwise, returns -1. + */ +int NV_API_CALL nv_cap_validate_and_dup_fd(const nv_cap_t *cap, int fd); + +/* + * Closes file descriptor + * + * This function should be used to close duplicate file descriptors + * returned by nv_cap_validate_and_dup_fd. + * + * @param[in] fd File descriptor to be validated + * + */ +void NV_API_CALL nv_cap_close_fd(int fd); + +#endif /* _NV_CAPS_H_ */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-chardev-numbers.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-chardev-numbers.h new file mode 100644 index 0000000..54ca547 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-chardev-numbers.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef _NV_CHARDEV_NUMBERS_H_ +#define _NV_CHARDEV_NUMBERS_H_ + +// NVIDIA's reserved major character device number (Linux). +#define NV_MAJOR_DEVICE_NUMBER 195 + +// Minor numbers 0 to 247 reserved for regular devices +#define NV_MINOR_DEVICE_NUMBER_REGULAR_MAX 247 + +// Minor numbers 248 to 253 currently unused + +// Minor number 254 reserved for the modeset device (provided by NVKMS) +#define NV_MINOR_DEVICE_NUMBER_MODESET_DEVICE 254 + +// Minor number 255 reserved for the control device +#define NV_MINOR_DEVICE_NUMBER_CONTROL_DEVICE 255 + +#endif // _NV_CHARDEV_NUMBERS_H_ + diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-gpu-info.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-gpu-info.h new file mode 100644 index 0000000..1b79320 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-gpu-info.h @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2015 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_GPU_INFO_H_ +#define _NV_GPU_INFO_H_ + +typedef struct { + NvU32 gpu_id; + + struct { + NvU32 domain; + NvU8 bus, slot, function; + } pci_info; + + NvBool needs_numa_setup; + NvBool is_soc_disp; + + /* + * opaque OS-specific pointer; on Linux, this is a pointer to the + * 'struct device' for the GPU. + */ + void *os_device_ptr; +} nv_gpu_info_t; + +#define NV_MAX_GPUS 32 + +#endif /* _NV_GPU_INFO_H_ */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-lockless-diag.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-lockless-diag.h new file mode 100644 index 0000000..84e3fa7 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-lockless-diag.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#if !defined(NV_IOCTL_LOCKLESS_DIAG) +#define NV_IOCTL_LOCKLESS_DIAG + +#include <nvtypes.h> +#include "ctrl/ctrl0000/ctrl0000nvd.h" + +typedef struct +{ + NvU32 cmd; // in + NvU32 status; // out + union // in/out + { + NV0000_CTRL_NVD_GET_NVLOG_INFO_PARAMS getNvlogInfo; + NV0000_CTRL_NVD_GET_NVLOG_BUFFER_INFO_PARAMS getNvlogBufferInfo; + NV0000_CTRL_NVD_GET_NVLOG_PARAMS getNvlog; + } params; +} NV_LOCKLESS_DIAGNOSTIC_PARAMS; + +#endif + diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-numa.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-numa.h new file mode 100644 index 0000000..0af5267 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-numa.h @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef NV_IOCTL_NUMA_H +#define NV_IOCTL_NUMA_H + +#include <nv-ioctl-numbers.h> + +#if defined(NV_KERNEL_INTERFACE_LAYER) && defined(NV_LINUX) +#include <linux/types.h> +#elif defined (NV_KERNEL_INTERFACE_LAYER) && defined(NV_BSD) +#include <sys/stdint.h> +#else + +#include <stdint.h> + +#if !defined(__aligned) +#define __aligned(n) __attribute__((aligned(n))) +#endif + +#endif + +#define NV_ESC_NUMA_INFO (NV_IOCTL_BASE + 15) +#define NV_ESC_SET_NUMA_STATUS (NV_IOCTL_BASE + 16) + +#define NV_IOCTL_NUMA_INFO_MAX_OFFLINE_ADDRESSES 64 +typedef struct offline_addresses +{ + uint64_t addresses[NV_IOCTL_NUMA_INFO_MAX_OFFLINE_ADDRESSES] __aligned(8); + uint32_t numEntries; +} nv_offline_addresses_t; + + +/* per-device NUMA memory info as assigned by the system */ +typedef struct nv_ioctl_numa_info +{ + int32_t nid; + int32_t status; + uint64_t memblock_size __aligned(8); + uint64_t numa_mem_addr __aligned(8); + uint64_t numa_mem_size __aligned(8); + uint8_t use_auto_online; + nv_offline_addresses_t offline_addresses __aligned(8); +} nv_ioctl_numa_info_t; + +/* set the status of the device NUMA memory */ +typedef struct nv_ioctl_set_numa_status +{ + int32_t status; +} nv_ioctl_set_numa_status_t; + +#define NV_IOCTL_NUMA_STATUS_DISABLED 0 +#define NV_IOCTL_NUMA_STATUS_OFFLINE 1 +#define NV_IOCTL_NUMA_STATUS_ONLINE_IN_PROGRESS 2 +#define NV_IOCTL_NUMA_STATUS_ONLINE 3 +#define NV_IOCTL_NUMA_STATUS_ONLINE_FAILED 4 +#define NV_IOCTL_NUMA_STATUS_OFFLINE_IN_PROGRESS 5 +#define NV_IOCTL_NUMA_STATUS_OFFLINE_FAILED 6 + +#endif diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-numbers.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-numbers.h new file mode 100644 index 0000000..e266368 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl-numbers.h @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef NV_IOCTL_NUMBERS_H +#define NV_IOCTL_NUMBERS_H + +/* NOTE: using an ioctl() number > 55 will overflow! */ +/* + * NOTE: Any new ioctls added here should also be added to nv_validate_ioctls() in nv.c or + * rm_validate_ioctls() in osapi.c + */ +#define NV_IOCTL_MAGIC 'F' +#define NV_IOCTL_BASE 200 +#define NV_ESC_CARD_INFO (NV_IOCTL_BASE + 0) +#define NV_ESC_REGISTER_FD (NV_IOCTL_BASE + 1) +#define NV_ESC_ALLOC_OS_EVENT (NV_IOCTL_BASE + 6) +#define NV_ESC_FREE_OS_EVENT (NV_IOCTL_BASE + 7) +#define NV_ESC_STATUS_CODE (NV_IOCTL_BASE + 9) +#define NV_ESC_CHECK_VERSION_STR (NV_IOCTL_BASE + 10) +#define NV_ESC_IOCTL_XFER_CMD (NV_IOCTL_BASE + 11) +#define NV_ESC_ATTACH_GPUS_TO_FD (NV_IOCTL_BASE + 12) +#define NV_ESC_QUERY_DEVICE_INTR (NV_IOCTL_BASE + 13) +#define NV_ESC_SYS_PARAMS (NV_IOCTL_BASE + 14) +#define NV_ESC_EXPORT_TO_DMABUF_FD (NV_IOCTL_BASE + 17) +#define NV_ESC_WAIT_OPEN_COMPLETE (NV_IOCTL_BASE + 18) + +#endif diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl.h new file mode 100644 index 0000000..8ca1642 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-ioctl.h @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef NV_IOCTL_H +#define NV_IOCTL_H + +#include <nv-ioctl-numbers.h> +#include <nv-ioctl-numa.h> +#include <nvtypes.h> + + +typedef struct { + NvU32 domain; /* PCI domain number */ + NvU8 bus; /* PCI bus number */ + NvU8 slot; /* PCI slot number */ + NvU8 function; /* PCI function number */ + NvU16 vendor_id; /* PCI vendor ID */ + NvU16 device_id; /* PCI device ID */ +} nv_pci_info_t; + +/* + * ioctl()'s with parameter structures too large for the + * _IOC cmd layout use the nv_ioctl_xfer_t structure + * and the NV_ESC_IOCTL_XFER_CMD ioctl() to pass the actual + * size and user argument pointer into the RM, which + * will then copy it to/from kernel space in separate steps. + */ +typedef struct nv_ioctl_xfer +{ + NvU32 cmd; + NvU32 size; + NvP64 ptr NV_ALIGN_BYTES(8); +} nv_ioctl_xfer_t; + +typedef struct nv_ioctl_card_info +{ + NvBool valid; + nv_pci_info_t pci_info; /* PCI config information */ + NvU32 gpu_id; + NvU16 interrupt_line; + NvU64 reg_address NV_ALIGN_BYTES(8); + NvU64 reg_size NV_ALIGN_BYTES(8); + NvU64 fb_address NV_ALIGN_BYTES(8); + NvU64 fb_size NV_ALIGN_BYTES(8); + NvU32 minor_number; + NvU8 dev_name[10]; /* device names such as vmgfx[0-32] for vmkernel */ +} nv_ioctl_card_info_t; + +/* alloc event */ +typedef struct nv_ioctl_alloc_os_event +{ + NvHandle hClient; + NvHandle hDevice; + NvU32 fd; + NvU32 Status; +} nv_ioctl_alloc_os_event_t; + +/* free event */ +typedef struct nv_ioctl_free_os_event +{ + NvHandle hClient; + NvHandle hDevice; + NvU32 fd; + NvU32 Status; +} nv_ioctl_free_os_event_t; + +/* status code */ +typedef struct nv_ioctl_status_code +{ + NvU32 domain; + NvU8 bus; + NvU8 slot; + NvU32 status; +} nv_ioctl_status_code_t; + +/* check version string */ +#define NV_RM_API_VERSION_STRING_LENGTH 64 + +typedef struct nv_ioctl_rm_api_version +{ + NvU32 cmd; + NvU32 reply; + char versionString[NV_RM_API_VERSION_STRING_LENGTH]; +} nv_ioctl_rm_api_version_t; + +#define NV_RM_API_VERSION_CMD_STRICT 0 +#define NV_RM_API_VERSION_CMD_RELAXED '1' +#define NV_RM_API_VERSION_CMD_QUERY '2' + +#define NV_RM_API_VERSION_REPLY_UNRECOGNIZED 0 +#define NV_RM_API_VERSION_REPLY_RECOGNIZED 1 + +typedef struct nv_ioctl_query_device_intr +{ + NvU32 intrStatus NV_ALIGN_BYTES(4); + NvU32 status; +} nv_ioctl_query_device_intr_t; + +/* system parameters that the kernel driver may use for configuration */ +typedef struct nv_ioctl_sys_params +{ + NvU64 memblock_size NV_ALIGN_BYTES(8); +} nv_ioctl_sys_params_t; + +typedef struct nv_ioctl_register_fd +{ + int ctl_fd; +} nv_ioctl_register_fd_t; + +#define NV_DMABUF_EXPORT_MAX_HANDLES 128 + +#define NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT 0 +#define NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE 1 + +typedef struct nv_ioctl_export_to_dma_buf_fd +{ + int fd; + NvHandle hClient; + NvU32 totalObjects; + NvU32 numObjects; + NvU32 index; + NvU64 totalSize NV_ALIGN_BYTES(8); + NvU8 mappingType; + NvBool bAllowMmap; + NvHandle handles[NV_DMABUF_EXPORT_MAX_HANDLES]; + NvU64 offsets[NV_DMABUF_EXPORT_MAX_HANDLES] NV_ALIGN_BYTES(8); + NvU64 sizes[NV_DMABUF_EXPORT_MAX_HANDLES] NV_ALIGN_BYTES(8); + NvU32 status; +} nv_ioctl_export_to_dma_buf_fd_t; + +typedef struct nv_ioctl_wait_open_complete +{ + int rc; + NvU32 adapterStatus; +} nv_ioctl_wait_open_complete_t; + +#endif diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-kernel-rmapi-ops.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-kernel-rmapi-ops.h new file mode 100644 index 0000000..d50a1e4 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-kernel-rmapi-ops.h @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2015-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_KERNEL_RMAPI_OPS_H_ +#define _NV_KERNEL_RMAPI_OPS_H_ + +/* + * Define the RMAPI provided to kernel-level RM clients. + * + * Kernel-level RM clients should populate nvidia_kernel_rmapi_ops_t + * by assigning nvidia_kernel_rmapi_ops_t::op and the corresponding + * parameter structure in nvidia_kernel_rmapi_ops_t's params union. + * Then, pass a pointer to the nvidia_kernel_rmapi_ops_t to + * rm_kernel_rmapi_op(). + */ + +#include "nvtypes.h" +#include "nvos.h" + +typedef struct { + NvU32 op; /* One of the NV0[14]_XXXX operations listed below. */ + + union { + NVOS00_PARAMETERS free; /* NV01_FREE */ + NVOS02_PARAMETERS allocMemory64; /* NV01_ALLOC_MEMORY */ + NVOS64_PARAMETERS alloc; /* NV04_ALLOC */ + NVOS32_PARAMETERS *pVidHeapControl; /* NV04_VID_HEAP_CONTROL */ + NVOS33_PARAMETERS mapMemory; /* NV04_MAP_MEMORY */ + NVOS34_PARAMETERS unmapMemory; /* NV04_UNMAP_MEMORY */ + NVOS39_PARAMETERS allocContextDma2; /* NV04_ALLOC_CONTEXT_DMA */ + NVOS46_PARAMETERS mapMemoryDma; /* NV04_MAP_MEMORY_DMA */ + NVOS47_PARAMETERS unmapMemoryDma; /* NV04_UNMAP_MEMORY_DMA */ + NVOS49_PARAMETERS bindContextDma; /* NV04_BIND_CONTEXT_DMA */ + NVOS54_PARAMETERS control; /* NV04_CONTROL*/ + NVOS55_PARAMETERS dupObject; /* NV04_DUP_OBJECT */ + NVOS57_PARAMETERS share; /* NV04_SHARE */ + NVOS61_PARAMETERS addVblankCallback; /* NV04_ADD_VBLANK_CALLBACK */ + } params; +} nvidia_kernel_rmapi_ops_t; + +#endif /* _NV_KERNEL_RMAPI_OPS_H_ */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-nb-regs.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-nb-regs.h new file mode 100644 index 0000000..6282b35 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-nb-regs.h @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2007-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_NB_REGS_H_ +#define _NV_NB_REGS_H_ + +#include "nvdevid.h" + +typedef struct +{ + NvU32 subsystem_vendor_id; + NvU32 subsystem_device_id; + NvU16 gpu_device_id; +} nv_nb_id_t; + +typedef struct +{ + NvU32 vendor_id; + const char *name; + NvU32 data; +} nv_nb_reg_t; + +/* + * nb_id_table contains the OEM vendor ID, the subsystem ID and the + * GPU device ID of the notebooks for which we need to enable + * vendor specific registry keys. nb_reg_table contains the vendor + * specific registry key values. The initVendorSpecificRegistry() + * function compares the present notebooks OEM subsystem ID and the + * GPU device ID with the values present in id_tables. If a match + * is found, initVendorSpecificRegistry() extracts the vendor + * ID and sets any associated registry key listed in nb_reg_table. + */ + +static nv_nb_id_t nb_id_table[] = { + { PCI_VENDOR_ID_PC_PARTNER, 0x0620, 0x1284 }, // Acer GT 630 + { PCI_VENDOR_ID_PC_PARTNER, 0x0620, 0x124b }, // Acer GT 640 + { 0, 0, 0 } +}; + +static nv_nb_reg_t nb_reg_table[] = { + { PCI_VENDOR_ID_PC_PARTNER, "RmOverrideSupportChipsetAspm", 2 }, + { 0, NULL, 0 } +}; + +#endif //_NV_NB_REGS_H_ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-priv.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-priv.h new file mode 100644 index 0000000..a3dcebf --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-priv.h @@ -0,0 +1,357 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_PRIV_H_ +#define _NV_PRIV_H_ + +#include <nv.h> +#include <os/os.h> +#include <ctrl/ctrl402c.h> +#include <gpu/disp/kern_disp_max.h> +#include <gpu/disp/kern_disp_type.h> + +#define NV_PRIV_REG_WR08(b,o,d) (*((volatile NvV8*)&(b)->Reg008[(o)/1])=(NvV8)(d)) +#define NV_PRIV_REG_WR16(b,o,d) (*((volatile NvV16*)&(b)->Reg016[(o)/2])=(NvV16)(d)) +#define NV_PRIV_REG_WR32(b,o,d) (*((volatile NvV32*)&(b)->Reg032[(o)/4])=(NvV32)(d)) + +#define NV_PRIV_REG_RD08(b,o) ((b)->Reg008[(o)/1]) +#define NV_PRIV_REG_RD16(b,o) ((b)->Reg016[(o)/2]) +#define NV_PRIV_REG_RD32(b,o) ((b)->Reg032[(o)/4]) + +struct OBJGPU; + +typedef struct +{ + NvBool baseValid; + VGAADDRDESC base; + NvBool workspaceBaseValid; + VGAADDRDESC workspaceBase; + NvU32 vesaMode; +} nv_vga_t; + +/* +* device state during Power Management +*/ +typedef struct nv_pm_state_s +{ + NvU32 IntrEn; + NvBool InHibernate; +} nv_pm_state_t; + +/* +* data structure for the UNIX workqueues +*/ +typedef struct nv_work_item_s +{ + OsQueueWorkItemFlags flags; + NvU32 gpuInstance; + union + { + OSWorkItemFunction *pGpuFunction; + OSSystemWorkItemFunction *pSystemFunction; + } func; + void *pData; +} nv_work_item_t; + +#define INVALID_DISP_ID 0xFFFFFFFF +#define MAX_DISP_ID_PER_ADAPTER 0x2 + +typedef struct nv_i2c_adapter_entry_s +{ + void *pOsAdapter; + NvU32 port; + NvU32 displayId[MAX_DISP_ID_PER_ADAPTER]; +} nv_i2c_adapter_entry_t; + +#define NV_INIT_FLAG_HAL 0x0001 +#define NV_INIT_FLAG_HAL_COMPONENTS 0x0002 +#define NV_INIT_FLAG_GPU_STATE 0x0004 +#define NV_INIT_FLAG_GPU_STATE_LOAD 0x0008 +#define NV_INIT_FLAG_FIFO_WATCHDOG 0x0010 +#define NV_INIT_FLAG_CORE_LOGIC 0x0020 +#define NV_INIT_FLAG_GPUMGR_ATTACH 0x0040 +#define NV_INIT_FLAG_PUBLIC_I2C 0x0080 +#define NV_INIT_FLAG_SCALABILITY 0x0100 +#define NV_INIT_FLAG_DMA 0x0200 + +#define MAX_I2C_ADAPTERS NV402C_CTRL_NUM_I2C_PORTS + +/* + * GPU dynamic power state machine. + * + * The GPU is in exactly one of these states at at time. Only certain state + * transitions are valid, as documented by the DAGs below. + * + * When in "instant idle" or COARSE mode: + * + * +----------------------+ + * v | + * +---------+ +----------------+ +--------+ + * | UNKNOWN | --> | IDLE_INDICATED | --> | IN_USE | + * +---------+ +----------------+ +--------+ + * + * The transition from UNKNOWN to IDLE_INDICATED happens in + * rm_init_dynamic_power_management(). + * + * Thereafter, transitions from IDLE_INDICATED to IN_USE happen when + * os_ref_dynamic_power() is called and the refcount transitions from 0 to 1; + * transitions from IN_USE to IDLE_INDICATED happen when + * os_unref_dynamic_power() is called and the refcount transitions from 1 to 0. + * Note that only calls to os_(un)ref_dynamic_power() with the mode == COARSE + * are considered in this mode; calls with mode == FINE are ignored. Since + * COARSE calls are placed only in rm_init_adapter/rm_shutdown_adapter, the GPU + * effectively stays in the IN_USE state any time any client has initialized + * it. + * + * + * When in "deferred idle" or FINE mode: + * + * +----------------------------------------------------------------+ + * | | + * | | + * | +-------------------------------------------+----------------------+ + * | | | v + * | +---------+ +----------------+ +--------------+ +----------------+ +--------+ + * | | UNKNOWN | --> | IDLE_INDICATED | --> | | --> | IDLE_SUSTAINED | --> | IN_USE | -+ + * | +---------+ +----------------+ | | +----------------+ +--------+ | + * | ^ | | | ^ | + * +--------------------+ | IDLE_INSTANT | ------+----------------------+ | + * | | | | + * | | | | + * | | <-----+ | + * +--------------+ | + * ^ | + * +-----------------------------------------------------+ + * + * As before, the transition from UNKNOWN to IDLE_INDICATED happens in + * rm_init_dynamic_power_management(). This is not ideal: it means the GPU may + * be powered down immediately upon loading the RM module, even if + * rm_init_adapter() is going to be called soon thereafter. However, we can't + * rely on deferred idle callbacks yet, since those currently rely on core RM + * being initialized. + * + * At the beginning of rm_init_adapter(), the GPU transitions to the IN_USE + * state; during the rm_init_adapter() sequence, + * RmInitDeferredDynamicPowerManagement() will be called which will schedule + * timer callbacks and set the "deferred_idle_enabled" boolean. + * + * While in "deferred idle" mode, one of the callbacks + * timerCallbackForIdlePreConditions(), timerCallbackToIndicateIdle(), or + * RmIndicateIdle() should be scheduled when in the states: + * - IN_USE + * - IDLE_INSTANT + * - IDLE_SUSTAINED + * Note that since we may transition from IN_USE to IDLE_INSTANT rapidly (e.g., + * for a series of RM calls), we don't attempt to schedule the callbacks and + * cancel them on each of these transitions. The + * timerCallbackForIdlePreConditions() callback will simply exit early if in + * the IN_USE state. + * + * As before, the GPU will remain in the IN_USE state until + * os_unref_dynamic_power() is called and the count transitions from 1 to 0 + * (calls with mode == FINE are honored, in this mode, and these transitions + * can happen frequently). When the refcount reaches 0, rather than going + * directly to the IDLE_INDICATED state, it transitions to the IDLE_INSTANT + * state. + * + * Then, when the next timerCallbackForIdlePreConditions() callback executes, + * if all preconditions are met, the state will transition to IDLE_SUSTAINED. + * + * If, when in the IDLE_SUSTAINED state, os_ref_dynamic_power() is called, the + * GPU will transition back to the IN_USE state and return to the IDLE_INSTANT + * state. This ensures that there is a suitable delay between any activity + * that requires bumping the refcount and indicating idleness. + * + * If the timerCallbackForIdlePreConditions() callback executes again and the + * GPU is still in the IDLE_SUSTAINED state, userspace mappings will be revoked + * and the timerCallbackToIndicateIdle() callback will be scheduled. + * + * If, before the timerCallbackToIndicateIdle() callback executes, either + * os_ref_dynamic_power() is called or a mapping which has been revoked is + * accessed (which triggers the RmForceGpuNotIdle() callback), the GPU will + * transition back to the IN_USE or IDLE_INSTANT state, respectively. + * + * Then, when the timerCallbackToIndicateIdle() callback executes, if all + * mappings are still revoked, and the GPU is still in the IDLE_SUSTAINED + * state, and all GPU idleness preconditions remain satisfied, the + * RmIndicateIdle() work item will be enqueued. (Else, the GPU will transition + * back to the IDLE_INSTANT state and the callback for preconditions is + * scheduled again.) + * + * Finally, once the RmIndicateIdle() work item is called, if all of the same + * conditions still hold, the state will transition to IDLE_INDICATED. No + * callbacks will be scheduled from here; the callbacks for preconditions + * should be re-scheduled when transitioning out of the IDLE_INDICATED state. + * + * Once in the IDLE_INDICATED state, the kernel is free to call the RM to + * perform the GC6 entry sequence then turn off power to the GPU (although it + * may not, if the audio function is being used for example). + * + * There are two paths to exit the IDLE_INDICATED state: + * (a) If os_ref_dynamic_power() is called, in which case it transitions + * directly to the IN_USE state; + * (b) If RmForceGpuNotIdle() is called, in which case it transitions back to + * the IDLE_INSTANT state. + */ +typedef enum +{ + NV_DYNAMIC_POWER_STATE_UNKNOWN = 0, + + NV_DYNAMIC_POWER_STATE_IN_USE, + + NV_DYNAMIC_POWER_STATE_IDLE_INSTANT, + NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED, + NV_DYNAMIC_POWER_STATE_IDLE_INDICATED, +} nv_dynamic_power_state_t; + +typedef struct nv_dynamic_power_s +{ + /* + * mode is read without the mutex -- should be read-only outside of + * rm_init_dynamic_power_management, called during probe only. + */ + nv_dynamic_power_mode_t mode; + /* + * Whether to indicate idle immediately when the refcount reaches 0, or + * only go to the IDLE_INSTANT state, and expect timer callbacks to + * transition through IDLE_SUSTAINED -> IDLE_INDICATED. + */ + NvBool deferred_idle_enabled; + + PORT_ATOMIC nv_dynamic_power_state_t state; + NvS32 refcount; + + /* + * A word on lock ordering. These locks must be taken in the order: + * + * RM API lock > this dynamic_power mutex > RM GPUs lock + * + * Skipping any of those locks is fine (if they aren't required to protect + * whatever state is being accessed or modified), so long as the order is + * not violated. + */ + PORT_MUTEX *mutex; + + /* + * callback handles for deferred dynamic power management. + */ + NvP64 idle_precondition_check_event; + NvP64 indicate_idle_event; + NvBool idle_precondition_check_callback_scheduled; + + /* + * callback handle for kernel initiated gc6 entry/exit. + * these will be protected by the gpu lock. + */ + NvP64 remove_idle_holdoff; + NvBool b_idle_holdoff; + + /* + * flag set if the platform does not support fine grain dynamic power + * management. + */ + NvBool b_fine_not_supported; + + /* + * This flag is used to check if a workitem is queued for + * RmQueueIdleSustainedWorkitem(). + */ + NvBool b_idle_sustained_workitem_queued; + + /* + * Counter to track clients disallowing GCOFF. + */ + NvU32 clients_gcoff_disallow_refcount; + + /* + * Maximum FB allocation size which can be saved in system memory + * while doing GCOFF based dynamic PM. + */ + NvU64 gcoff_max_fb_size; + + /* + * NVreg_DynamicPowerManagement regkey value set by the user + */ + NvU32 dynamic_power_regkey; +} nv_dynamic_power_t; + +typedef struct +{ + OBJGPU *pGpu; + + NvU32 pmc_boot_0; + + nv_vga_t vga; + + NvU32 flags; + NvU32 status; + + nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS]; + + void *pVbiosCopy; + NvU32 vbiosSize; + + nv_pm_state_t pm_state; + + nv_reg_entry_t *pRegistry; + + nv_dynamic_power_t dynamic_power; + + /* Flag to check if GPU mobile config is enabled */ + NvBool b_mobile_config_enabled; + + /* Flag to check if S0ix-based power management is enabled. */ + NvBool s0ix_pm_enabled; + + /* Variable to track Dynamic Boost support */ + int db_supported; + + /* + * Maximum FB allocation size which can be saved in system memory + * during system supened with S0ix-based power management. + */ + NvU64 s0ix_gcoff_max_fb_size; + + NvU32 pmc_boot_1; + NvU32 pmc_boot_42; + + /* + * This flag is set if the upstream port has been configured ("D3cold Aux Power Limit" and + * "PERST# Assertion Delay") for GC6. This configuration is needed only for desktops. + */ + NvBool gc6_upstream_port_configured; + + /* This flag is set if _PR3 ACPI method is available to support RTD3. */ + NvBool pr3_acpi_method_present; +} nv_priv_t; + +#define NV_SET_NV_PRIV(nv,p) ((nv)->priv = (p)) +#define NV_GET_NV_PRIV(nv) ((nv) ? (nv)->priv : NULL) + +/* + * Make sure that your stack has taken API Lock before using this macro. + */ +#define NV_GET_NV_PRIV_PGPU(nv) \ + (NV_GET_NV_PRIV(nv) ? ((nv_priv_t *)NV_GET_NV_PRIV(nv))->pGpu : NULL) + +#endif // _NV_PRIV_H_ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-reg.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-reg.h new file mode 100644 index 0000000..93999d4 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-reg.h @@ -0,0 +1,1137 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2006-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +// +// This file holds Unix-specific NVIDIA driver options +// + +#ifndef _RM_REG_H_ +#define _RM_REG_H_ + +#include "nvtypes.h" +#include "nv-firmware-registry.h" + +/* + * use NV_REG_STRING to stringify a registry key when using that registry key + */ + +#define __NV_REG_STRING(regkey) #regkey +#define NV_REG_STRING(regkey) __NV_REG_STRING(regkey) + +/* + * use NV_DEFINE_REG_ENTRY and NV_DEFINE_PARAMS_TABLE_ENTRY to simplify definition + * of registry keys in the kernel module source code. + */ + +#define __NV_REG_VAR(regkey) NVreg_##regkey + +#if defined(NV_MODULE_PARAMETER) +#define NV_DEFINE_REG_ENTRY(regkey, default_value) \ + static NvU32 __NV_REG_VAR(regkey) = (default_value); \ + NV_MODULE_PARAMETER(__NV_REG_VAR(regkey)) +#define NV_DEFINE_REG_ENTRY_GLOBAL(regkey, default_value) \ + NvU32 __NV_REG_VAR(regkey) = (default_value); \ + NV_MODULE_PARAMETER(__NV_REG_VAR(regkey)) +#else +#define NV_DEFINE_REG_ENTRY(regkey, default_value) \ + static NvU32 __NV_REG_VAR(regkey) = (default_value) +#define NV_DEFINE_REG_ENTRY_GLOBAL(regkey, default_value) \ + NvU32 __NV_REG_VAR(regkey) = (default_value) +#endif + +#if defined(NV_MODULE_STRING_PARAMETER) +#define NV_DEFINE_REG_STRING_ENTRY(regkey, default_value) \ + char *__NV_REG_VAR(regkey) = (default_value); \ + NV_MODULE_STRING_PARAMETER(__NV_REG_VAR(regkey)) +#else +#define NV_DEFINE_REG_STRING_ENTRY(regkey, default_value) \ + char *__NV_REG_VAR(regkey) = (default_value) +#endif + +#define NV_DEFINE_PARAMS_TABLE_ENTRY(regkey) \ + { NV_REG_STRING(regkey), &__NV_REG_VAR(regkey) } + +/* + * Like NV_DEFINE_PARMS_TABLE_ENTRY, but allows a mismatch between the name of + * the regkey and the name of the module parameter. When using this macro, the + * name of the parameter is passed to the extra "parameter" argument, and it is + * this name that must be used in the NV_DEFINE_REG_ENTRY() macro. + */ + +#define NV_DEFINE_PARAMS_TABLE_ENTRY_CUSTOM_NAME(regkey, parameter) \ + { NV_REG_STRING(regkey), &__NV_REG_VAR(parameter)} + +/* + *----------------- registry key definitions-------------------------- + */ + +/* + * Option: ModifyDeviceFiles + * + * Description: + * + * When this option is enabled, the NVIDIA driver will verify the validity + * of the NVIDIA device files in /dev and attempt to dynamically modify + * and/or (re-)create them, if necessary. If you don't wish for the NVIDIA + * driver to touch the device files, you can use this registry key. + * + * This module parameter is only honored by the NVIDIA GPU driver and NVIDIA + * capability driver. Furthermore, the NVIDIA capability driver provides + * modifiable /proc file entry (DeviceFileModify=0/1) to alter the behavior of + * this module parameter per device file. + * + * Possible Values: + * 0 = disable dynamic device file management + * 1 = enable dynamic device file management (default) + */ + +#define __NV_MODIFY_DEVICE_FILES ModifyDeviceFiles +#define NV_REG_MODIFY_DEVICE_FILES NV_REG_STRING(__NV_MODIFY_DEVICE_FILES) + +/* + * Option: DeviceFileUID + * + * Description: + * + * This registry key specifies the UID assigned to the NVIDIA device files + * created and/or modified by the NVIDIA driver when dynamic device file + * management is enabled. + * + * This module parameter is only honored by the NVIDIA GPU driver. + * + * The default UID is 0 ('root'). + */ + +#define __NV_DEVICE_FILE_UID DeviceFileUID +#define NV_REG_DEVICE_FILE_UID NV_REG_STRING(__NV_DEVICE_FILE_UID) + +/* + * Option: DeviceFileGID + * + * Description: + * + * This registry key specifies the GID assigned to the NVIDIA device files + * created and/or modified by the NVIDIA driver when dynamic device file + * management is enabled. + * + * This module parameter is only honored by the NVIDIA GPU driver. + * + * The default GID is 0 ('root'). + */ + +#define __NV_DEVICE_FILE_GID DeviceFileGID +#define NV_REG_DEVICE_FILE_GID NV_REG_STRING(__NV_DEVICE_FILE_GID) + +/* + * Option: DeviceFileMode + * + * Description: + * + * This registry key specifies the device file mode assigned to the NVIDIA + * device files created and/or modified by the NVIDIA driver when dynamic + * device file management is enabled. + * + * This module parameter is only honored by the NVIDIA GPU driver. + * + * The default mode is 0666 (octal, rw-rw-rw-). + */ + +#define __NV_DEVICE_FILE_MODE DeviceFileMode +#define NV_REG_DEVICE_FILE_MODE NV_REG_STRING(__NV_DEVICE_FILE_MODE) + +/* + * Option: ResmanDebugLevel + * + * Default value: ~0 + */ + +#define __NV_RESMAN_DEBUG_LEVEL ResmanDebugLevel +#define NV_REG_RESMAN_DEBUG_LEVEL NV_REG_STRING(__NV_RESMAN_DEBUG_LEVEL) + +/* + * Option: RmLogonRC + * + * Default value: 1 + */ + +#define __NV_RM_LOGON_RC RmLogonRC +#define NV_REG_RM_LOGON_RC NV_REG_STRING(__NV_RM_LOGON_RC) + +/* + * Option: InitializeSystemMemoryAllocations + * + * Description: + * + * The NVIDIA Linux driver normally clears system memory it allocates + * for use with GPUs or within the driver stack. This is to ensure + * that potentially sensitive data is not rendered accessible by + * arbitrary user applications. + * + * Owners of single-user systems or similar trusted configurations may + * choose to disable the aforementioned clears using this option and + * potentially improve performance. + * + * Possible values: + * + * 1 = zero out system memory allocations (default) + * 0 = do not perform memory clears + */ + +#define __NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS \ + InitializeSystemMemoryAllocations +#define NV_REG_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS \ + NV_REG_STRING(__NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS) + +/* + * Option: RegistryDwords + * + * Description: + * + * This option accepts a semicolon-separated list of key=value pairs. Each + * key name is checked against the table of static options; if a match is + * found, the static option value is overridden, but invalid options remain + * invalid. Pairs that do not match an entry in the static option table + * are passed on to the RM directly. + * + * Format: + * + * NVreg_RegistryDwords="<key=value>;<key=value>;..." + */ + +#define __NV_REGISTRY_DWORDS RegistryDwords +#define NV_REG_REGISTRY_DWORDS NV_REG_STRING(__NV_REGISTRY_DWORDS) + +/* + * Option: RegistryDwordsPerDevice + * + * Description: + * + * This option allows to specify registry keys per GPU device. It helps to + * control registry at GPU level of granularity. It accepts a semicolon + * separated list of key=value pairs. The first key value pair MUST be + * "pci=DDDD:BB:DD.F;" where DDDD is Domain, BB is Bus Id, DD is device slot + * number and F is the Function. This PCI BDF is used to identify which GPU to + * assign the registry keys that follows next. + * If a GPU corresponding to the value specified in "pci=DDDD:BB:DD.F;" is NOT + * found, then all the registry keys that follows are skipped, until we find next + * valid pci identified "pci=DDDD:BB:DD.F;". Following are the valid formats for + * the value of the "pci" string: + * 1) bus:slot : Domain and function defaults to 0. + * 2) domain:bus:slot : Function defaults to 0. + * 3) domain:bus:slot.func : Complete PCI dev id string. + * + * For each of the registry keys that follows, key name is checked against the + * table of static options; if a match is found, the static option value is + * overridden, but invalid options remain invalid. Pairs that do not match an + * entry in the static option table are passed on to the RM directly. + * + * Format: + * + * NVreg_RegistryDwordsPerDevice="pci=DDDD:BB:DD.F;<key=value>;<key=value>;..; \ + * pci=DDDD:BB:DD.F;<key=value>;..;" + */ + +#define __NV_REGISTRY_DWORDS_PER_DEVICE RegistryDwordsPerDevice +#define NV_REG_REGISTRY_DWORDS_PER_DEVICE NV_REG_STRING(__NV_REGISTRY_DWORDS_PER_DEVICE) + +#define __NV_RM_MSG RmMsg +#define NV_RM_MSG NV_REG_STRING(__NV_RM_MSG) + +/* + * Option: EnableMSI + * + * Description: + * + * When this option is enabled and the host kernel supports the MSI feature, + * the NVIDIA driver will enable the PCI-E MSI capability of GPUs with the + * support for this feature instead of using PCI-E wired interrupt. + * + * Possible Values: + * + * 0 = disable MSI interrupt + * 1 = enable MSI interrupt (default) + * + */ + +#define __NV_ENABLE_MSI EnableMSI +#define NV_REG_ENABLE_MSI NV_REG_STRING(__NV_ENABLE_MSI) + +/* + * Option: EnablePCIeGen3 + * + * Description: + * + * Due to interoperability problems seen with Kepler PCIe Gen3 capable GPUs + * when configured on SandyBridge E desktop platforms, NVIDIA feels that + * delivering a reliable, high-quality experience is not currently possible in + * PCIe Gen3 mode on all PCIe Gen3 platforms. Therefore, Quadro, Tesla and + * NVS Kepler products operate in PCIe Gen2 mode by default. You may use this + * option to enable PCIe Gen3 support. + * + * This is completely unsupported! + * + * Possible Values: + * + * 0: disable PCIe Gen3 support (default) + * 1: enable PCIe Gen3 support + */ + +#define __NV_ENABLE_PCIE_GEN3 EnablePCIeGen3 +#define NV_REG_ENABLE_PCIE_GEN3 NV_REG_STRING(__NV_ENABLE_PCIE_GEN3) + +/* + * Option: MemoryPoolSize + * + * Description: + * + * When set to a non-zero value, this option specifies the size of the + * memory pool, given as a multiple of 1 GB, created on VMware ESXi to + * satisfy any system memory allocations requested by the NVIDIA kernel + * module. + */ + +#define __NV_MEMORY_POOL_SIZE MemoryPoolSize +#define NV_REG_MEMORY_POOL_SIZE NV_REG_STRING(__NV_MEMORY_POOL_SIZE) + +/* + * Option: KMallocHeapMaxSize + * + * Description: + * + * When set to a non-zero value, this option specifies the maximum size of the + * heap memory space reserved for kmalloc operations. Given as a + * multiple of 1 MB created on VMware ESXi to satisfy any system memory + * allocations requested by the NVIDIA kernel module. + */ + +#define __NV_KMALLOC_HEAP_MAX_SIZE KMallocHeapMaxSize +#define NV_KMALLOC_HEAP_MAX_SIZE NV_REG_STRING(__NV_KMALLOC_HEAP_MAX_SIZE) + +/* + * Option: VMallocHeapMaxSize + * + * Description: + * + * When set to a non-zero value, this option specifies the maximum size of the + * heap memory space reserved for vmalloc operations. Given as a + * multiple of 1 MB created on VMware ESXi to satisfy any system memory + * allocations requested by the NVIDIA kernel module. + */ + +#define __NV_VMALLOC_HEAP_MAX_SIZE VMallocHeapMaxSize +#define NV_VMALLOC_HEAP_MAX_SIZE NV_REG_STRING(__NV_VMALLOC_HEAP_MAX_SIZE) + +/* + * Option: IgnoreMMIOCheck + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will ignore + * MMIO limit check during device probe on VMWare ESXi kernel. This is + * typically necessary when VMware ESXi MMIO limit differs between any + * base version and its updates. Customer using updates can set regkey + * to avoid probe failure. + */ + +#define __NV_IGNORE_MMIO_CHECK IgnoreMMIOCheck +#define NV_REG_IGNORE_MMIO_CHECK NV_REG_STRING(__NV_IGNORE_MMIO_CHECK) + +/* + * Option: pci + * + * Description: + * + * On Unix platforms, per GPU based registry key can be specified as: + * NVreg_RegistryDwordsPerDevice="pci=DDDD:BB:DD.F,<per-gpu registry keys>". + * where DDDD:BB:DD.F refers to Domain:Bus:Device.Function. + * We need this key "pci" to identify what follows next is a PCI BDF identifier, + * for which the registry keys are to be applied. + * + * This define is not used on non-UNIX platforms. + * + * Possible Formats for value: + * + * 1) bus:slot : Domain and function defaults to 0. + * 2) domain:bus:slot : Function defaults to 0. + * 3) domain:bus:slot.func : Complete PCI BDF identifier string. + */ +#define __NV_PCI_DEVICE_BDF pci +#define NV_REG_PCI_DEVICE_BDF NV_REG_STRING(__NV_PCI_DEVICE_BDF) + +/* + * Option: EnableStreamMemOPs + * + * Description: + * + * When this option is enabled, the CUDA driver will enable support for + * CUDA Stream Memory Operations in user-mode applications, which are so + * far required to be disabled by default due to limited support in + * devtools. + * + * Note: this is treated as a hint. MemOPs may still be left disabled by CUDA + * driver for other reasons. + * + * Possible Values: + * + * 0 = disable feature (default) + * 1 = enable feature + */ +#define __NV_ENABLE_STREAM_MEMOPS EnableStreamMemOPs +#define NV_REG_ENABLE_STREAM_MEMOPS NV_REG_STRING(__NV_ENABLE_STREAM_MEMOPS) + +/* + * Option: EnableUserNUMAManagement + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will require the + * user-mode NVIDIA Persistence daemon to manage the onlining and offlining + * of its NUMA device memory. + * + * This option has no effect on platforms that do not support onlining + * device memory to a NUMA node (this feature is only supported on certain + * POWER9 systems). + * + * Possible Values: + * + * 0: disable user-mode NUMA management + * 1: enable user-mode NUMA management (default) + */ +#define __NV_ENABLE_USER_NUMA_MANAGEMENT EnableUserNUMAManagement +#define NV_REG_ENABLE_USER_NUMA_MANAGEMENT NV_REG_STRING(__NV_ENABLE_USER_NUMA_MANAGEMENT) + +/* + * Option: CoherentGPUMemoryMode + * + * Description: + * + * This option can be set to control how GPU Memory is accessed through + * the coherent link. + * + * This option has no effect on platforms that do not support onlining + * device memory to a NUMA node. + * + * Possible string values: + * + * "driver" : disable onlining coherent memory to the OS as a NUMA node. The driver + * will manage it in this case + * "numa" (or unset) : enable onlining coherent memory to the OS as a NUMA node (default) + */ +#define __NV_COHERENT_GPU_MEMORY_MODE CoherentGPUMemoryMode +#define NV_REG_COHERENT_GPU_MEMORY_MODE NV_REG_STRING(__NV_COHERENT_GPU_MEMORY_MODE) + +/* + * Option: GpuBlacklist + * + * Description: + * + * This option accepts a list of blacklisted GPUs, separated by commas, that + * cannot be attached or used. Each blacklisted GPU is identified by a UUID in + * the ASCII format with leading "GPU-". An exact match is required; no partial + * UUIDs. This regkey is deprecated and will be removed in the future. Use + * NV_REG_EXCLUDED_GPUS instead. + */ +#define __NV_GPU_BLACKLIST GpuBlacklist +#define NV_REG_GPU_BLACKLIST NV_REG_STRING(__NV_GPU_BLACKLIST) + +/* + * Option: ExcludedGpus + * + * Description: + * + * This option accepts a list of excluded GPUs, separated by commas, that + * cannot be attached or used. Each excluded GPU is identified by a UUID in + * the ASCII format with leading "GPU-". An exact match is required; no partial + * UUIDs. + */ +#define __NV_EXCLUDED_GPUS ExcludedGpus +#define NV_REG_EXCLUDED_GPUS NV_REG_STRING(__NV_EXCLUDED_GPUS) + +/* + * Option: NVreg_ExcludeAllGpus + * + * Description: + * + * This option allows the user to exclude all GPUs. If used in conjunction with + * UUID based GPU exclusion, all GPUs will be excluded + * + * Possible values: + * 1 - Excludes all GPUs from being attached or used + * 0 - No effect (default) + */ + #define __NV_EXCLUDE_ALL_GPUS ExcludeAllGpus + #define NV_EXCLUDE_ALL_GPUS NV_REG_STRING(__NV_EXCLUDE_ALL_GPUS) + #define NV_EXCLUDE_ALL_GPUS_DEFAULT 0 + +/* + * Option: NvLinkDisable + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will not attempt to + * initialize or train NVLink connections for any GPUs. System reboot is required + * for changes to take affect. + * + * This option has no effect if no GPUs support NVLink. + * + * Possible Values: + * + * 0: Do not disable NVLink (default) + * 1: Disable NVLink + */ +#define __NV_NVLINK_DISABLE NvLinkDisable +#define NV_REG_NVLINK_DISABLE NV_REG_STRING(__NV_NVLINK_DISABLE) + +/* + * Option: RestrictProfilingToAdminUsers + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will prevent users + * without administrative access (i.e., the CAP_SYS_ADMIN capability) from + * using GPU performance counters. + * + * Possible Values: + * + * 0: Do not restrict GPU counters (default) + * 1: Restrict GPU counters to system administrators only + */ + +#define __NV_RM_PROFILING_ADMIN_ONLY RmProfilingAdminOnly +#define __NV_RM_PROFILING_ADMIN_ONLY_PARAMETER RestrictProfilingToAdminUsers +#define NV_REG_RM_PROFILING_ADMIN_ONLY NV_REG_STRING(__NV_RM_PROFILING_ADMIN_ONLY) + + +/* +* Option: EnableDebuggerInterface +* +* Description: +* +* When this option is enabled, the NVIDIA kernel module will allow non-administrator users to +* create a non-preemptable debugger session for non-CILP contexts. +* +* Possible Values: +* +* 0: Do not allow non-administrator users to create non-preemptable debugger sessions (default) +* 1: Allow non-administrator users to create non-preemptable debugger sessions +*/ +#define __NV_ENABLE_NON_PREEMTABLE_DEBUGGER_SESSION EnableDebuggerInterface +#define NV_REG_ENABLE_NON_PREEMTABLE_DEBUGGER_SESSION NV_REG_STRING(__NV_ENABLE_NON_PREEMTABLE_DEBUGGER_SESSION) + +/* + * Option: TemporaryFilePath + * + * Description: + * + * When specified, this option changes the location in which the + * NVIDIA kernel module will create unnamed temporary files (e.g. to + * save the contents of video memory in). The indicated file must + * be a directory. By default, temporary files are created in /tmp. + */ +#define __NV_TEMPORARY_FILE_PATH TemporaryFilePath +#define NV_REG_TEMPORARY_FILE_PATH NV_REG_STRING(__NV_TEMPORARY_FILE_PATH) + +/* + * Option: PreserveVideoMemoryAllocations + * + * If enabled, this option prompts the NVIDIA kernel module to save and + * restore all video memory allocations across system power management + * cycles, i.e. suspend/resume and hibernate/restore. Otherwise, + * only select allocations are preserved. + * + * Possible Values: + * + * 0: Preserve only select video memory allocations + * 1: Preserve all video memory allocations + * 2: Automatically enable video memory allocations when + * NVreg_UseKernelSuspendNotifiers is enabled (default) + */ +#define __NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS PreserveVideoMemoryAllocations +#define NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS \ + NV_REG_STRING(__NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS) + +#define NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_DISABLED 0 +#define NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_ENABLED 1 +#define NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_AUTO 2 + +/* + * Option: UseKernelSuspendNotifiers + * + * If enabled, this option prompts the NVIDIA kernel module to register a + * notifier that saves and restores all video memory allocations across system + * power management cycles if PreserveVideoMemoryAllocations is enabled. + * + * Possible Values: + * + * 0: Suspend notifiers are not used (default) + * 1: Suspend notifiers are used when available + */ + +#define __NV_USE_KERNEL_SUSPEND_NOTIFIERS UseKernelSuspendNotifiers +#define NV_REG_USE_KERNEL_SUSPEND_NOTIFIERS \ + NV_REG_STRING(__NV_USE_KERNEL_SUSPEND_NOTIFIERS) + +/* + * Option: EnableS0ixPowerManagement + * + * When this option is enabled, the NVIDIA driver will use S0ix-based + * power management for system suspend/resume, if both the platform and + * the GPU support S0ix. + * + * During system suspend, if S0ix is enabled and + * video memory usage is above the threshold configured by + * 'S0ixPowerManagementVideoMemoryThreshold', video memory will be kept + * in self-refresh mode while the rest of the GPU is powered down. + * + * Otherwise, the driver will copy video memory contents to system memory + * and power off the video memory along with the GPU. + * + * Possible Values: + * + * 0: Disable S0ix based power management (default) + * 1: Enable S0ix based power management + */ + +#define __NV_ENABLE_S0IX_POWER_MANAGEMENT EnableS0ixPowerManagement +#define NV_REG_ENABLE_S0IX_POWER_MANAGEMENT \ + NV_REG_STRING(__NV_ENABLE_S0IX_POWER_MANAGEMENT) + +/* + * Option: S0ixPowerManagementVideoMemoryThreshold + * + * This option controls the threshold that the NVIDIA driver will use during + * S0ix-based system power management. + * + * When S0ix is enabled and the system is suspended, the driver will + * compare the amount of video memory in use with this threshold, + * to decide whether to keep video memory in self-refresh or copy video + * memory content to system memory. + * + * See the 'EnableS0ixPowerManagement' option. + * + * Values are expressed in Megabytes (1048576 bytes). + * + * Default value for this option is 256MB. + * + */ +#define __NV_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD \ + S0ixPowerManagementVideoMemoryThreshold +#define NV_REG_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD \ + NV_REG_STRING(__NV_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD) + +/* + * Option: DynamicPowerManagement + * + * This option controls how aggressively the NVIDIA kernel module will manage + * GPU power through kernel interfaces. + * + * Possible Values: + * + * 0: Never allow the GPU to be powered down (default). + * 1: Power down the GPU when it is not initialized. + * 2: Power down the GPU after it has been inactive for some time. + * 3: (Default) Power down the GPU after a period of inactivity (i.e., + * mode 2) on Ampere or later notebooks. Otherwise, do not power down + * the GPU. + */ +#define __NV_DYNAMIC_POWER_MANAGEMENT DynamicPowerManagement +#define NV_REG_DYNAMIC_POWER_MANAGEMENT \ + NV_REG_STRING(__NV_DYNAMIC_POWER_MANAGEMENT) + +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER 0 +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_COARSE 1 +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_FINE 2 +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT 3 + +/* + * Option: DynamicPowerManagementVideoMemoryThreshold + * + * This option controls the threshold that the NVIDIA driver will use + * when selecting the dynamic power management scheme. + * + * When the driver detects that the GPU is idle, it will compare the amount + * of video memory in use with this threshold. + * + * If the current video memory usage is less than the threshold, the + * driver may preserve video memory contents in system memory and power off + * the video memory along with the GPU itself, if supported. Otherwise, + * the video memory will be kept in self-refresh mode while powering down + * the rest of the GPU, if supported. + * + * Values are expressed in Megabytes (1048576 bytes). + * + * If the requested value is greater than 200MB (the default), then it + * will be capped to 200MB. + */ +#define __NV_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD \ + DynamicPowerManagementVideoMemoryThreshold +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD \ + NV_REG_STRING(__NV_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD) + +/* + * Option: TegraGpuPgMask + * + * This option controls the TPC/GPC/FBP power-gating mask for Tegra iGPU. + * + */ +#define __NV_TEGRA_GPU_PG_MASK TegraGpuPgMask +#define NV_REG_TEGRA_GPU_PG_MASK \ + NV_REG_STRING(__NV_TEGRA_GPU_PG_MASK) + +/* + * Option: RegisterPCIDriver + * + * Description: + * + * When this option is enabled, the NVIDIA driver will register with + * PCI subsystem. + * + * Possible values: + * + * 1 - register as PCI driver (default) + * 0 - do not register as PCI driver + */ + +#define __NV_REGISTER_PCI_DRIVER RegisterPCIDriver +#define NV_REG_REGISTER_PCI_DRIVER NV_REG_STRING(__NV_REGISTER_PCI_DRIVER) + +/* + * Option: RegisterPlatformDeviceDriver + * + * Description: + * + * When this option is enabled, the NVIDIA driver will register with + * platform subsystem. + * + * Possible values: + * + * 1 - register as platform driver (default) + * 0 - do not register as platform driver + */ + +#define __NV_REGISTER_PLATFORM_DEVICE_DRIVER RegisterPlatformDeviceDriver +#define NV_REG_REGISTER_PLATFORM_DEVICE_DRIVER NV_REG_STRING(__NV_REGISTER_PLATFORM_DEVICE_DRIVER) + +/* + * Option: EnablePCIERelaxedOrderingMode + * + * Description: + * + * When this option is enabled, the registry key RmSetPCIERelaxedOrdering will + * be set to NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING_FORCE_ENABLE, causing + * every device to set the relaxed ordering bit to 1 in all outbound MWr + * transaction-layer packets. This is equivalent to setting the regkey to + * FORCE_ENABLE as a non-per-device registry key. + * + * Possible values: + * 0 - Do not enable PCIe TLP relaxed ordering bit-setting (default) + * 1 - Enable PCIe TLP relaxed ordering bit-setting + */ +#define __NV_ENABLE_PCIE_RELAXED_ORDERING_MODE EnablePCIERelaxedOrderingMode +#define NV_REG_ENABLE_PCIE_RELAXED_ORDERING_MODE \ + NV_REG_STRING(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE) + +/* + * Option: EnableResizableBar + * + * Description: + * + * When this option is enabled, the NVIDIA driver will attempt to resize + * BAR1 to match framebuffer size, or the next largest available size on + * supported machines. This is currently only implemented for Linux. + * + * Possible values: + * 0 - Do not enable PCI BAR resizing + * 1 - Enable PCI BAR resizing + */ +#define __NV_ENABLE_RESIZABLE_BAR EnableResizableBar +#define NV_REG_ENABLE_RESIZABLE_BAR NV_REG_STRING(__NV_ENABLE_RESIZABLE_BAR) + +/* + * Option: EnableGpuFirmware + * + * Description: + * + * When this option is enabled, the NVIDIA driver will enable use of GPU + * firmware. + * + * If this key is set globally to the system, the driver may still attempt + * to apply some policies to maintain uniform firmware modes across all + * GPUS. This may result in the driver failing initialization on some GPUs + * to maintain such a policy. + * + * If this key is set using NVreg_RegistryDwordsPerDevice, then the driver + * will attempt to honor whatever configuration is specified without applying + * additional policies. This may also result in failed GPU initialzations if + * the configuration is not possible (for example if the firmware is missing + * from the filesystem, or the GPU is not capable). + * + * NOTE: More details for this regkey can be found in nv-firmware-registry.h + */ +#define __NV_ENABLE_GPU_FIRMWARE EnableGpuFirmware +#define NV_REG_ENABLE_GPU_FIRMWARE NV_REG_STRING(__NV_ENABLE_GPU_FIRMWARE) + +/* + * Option: EnableGpuFirmwareLogs + * + * When this option is enabled, the NVIDIA driver will send GPU firmware logs + * to the system log, when possible. + * + * NOTE: More details for this regkey can be found in nv-firmware-registry.h + */ +#define __NV_ENABLE_GPU_FIRMWARE_LOGS EnableGpuFirmwareLogs +#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS NV_REG_STRING(__NV_ENABLE_GPU_FIRMWARE_LOGS) + +/* + * Option: EnableDbgBreakpoint + * + * When this option is set to a non-zero value, and the kernel is configured + * appropriately, assertions within resman will trigger a CPU breakpoint (e.g., + * INT3 on x86_64), assumed to be caught by an attached debugger. + * + * When this option is set to the value zero (the default), assertions within + * resman will print to the system log, but no CPU breakpoint will be triggered. + */ +#define __NV_ENABLE_DBG_BREAKPOINT EnableDbgBreakpoint + + +/* + * Option: OpenRmEnableUnsupportedGpus + * + * This option to require opt in for use of Open RM on non-Data Center + * GPUs is deprecated and no longer required. The kernel module parameter + * is left here, though ignored, for backwards compatibility. + */ +#define __NV_OPENRM_ENABLE_UNSUPPORTED_GPUS OpenRmEnableUnsupportedGpus + +/* + * Option: NVreg_DmaRemapPeerMmio + * + * Description: + * + * When this option is enabled, the NVIDIA driver will use device driver + * APIs provided by the Linux kernel for DMA-remapping part of a device's + * MMIO region to another device, creating e.g., IOMMU mappings as necessary. + * When this option is disabled, the NVIDIA driver will instead only apply a + * fixed offset, which may be zero, to CPU physical addresses to produce the + * DMA address for the peer's MMIO region, and no IOMMU mappings will be + * created. + * + * This option only affects peer MMIO DMA mappings, and not system memory + * mappings. + * + * Possible Values: + * 0 = disable dynamic DMA remapping of peer MMIO regions + * 1 = enable dynamic DMA remapping of peer MMIO regions (default) + */ +#define __NV_DMA_REMAP_PEER_MMIO DmaRemapPeerMmio +#define NV_DMA_REMAP_PEER_MMIO NV_REG_STRING(__NV_DMA_REMAP_PEER_MMIO) +#define NV_DMA_REMAP_PEER_MMIO_DISABLE 0x00000000 +#define NV_DMA_REMAP_PEER_MMIO_ENABLE 0x00000001 + +/* + * Option: NVreg_RmNvlinkBandwidthLinkCount + * + * Description: + * + * This option allows user to reduce the GPU nvlink bandwidth to save power. + * + * This option is only for Blackwell+ GPU with NVLINK version 5.0. + */ +#define __NV_RM_NVLINK_BW_LINK_COUNT RmNvlinkBandwidthLinkCount +#define NV_RM_NVLINK_BW_LINK_COUNT NV_REG_STRING(__NV_RM_NVLINK_BW_LINK_COUNT) + +/* + * Option: NVreg_RmNvlinkBandwidth + * + * Description: + * + * This option allows user to reduce the NVLINK P2P bandwidth to save power. + * The option is in the string format. + * + * Possible string values: + * OFF: 0% bandwidth + * MIN: 15%-25% bandwidth depending on the system's NVLink topology + * HALF: 50% bandwidth + * 3QUARTER: 75% bandwidth + * FULL: 100% bandwidth (default) + * + * This option is only for Hopper+ GPU with NVLINK version 4.0. + */ +#define __NV_RM_NVLINK_BW RmNvlinkBandwidth +#define NV_RM_NVLINK_BW NV_REG_STRING(__NV_RM_NVLINK_BW) + +/* + * Option: NVreg_EnableNonblockingOpen + * + * Description: + * + * When this option is enabled, the NVIDIA driver will try to perform any + * required device initialization in the background when /dev/nvidiaN devices + * are opened with the flag O_NONBLOCK. + * + * Possible Values: + * 0 = O_NONBLOCK flag when opening devices is ignored + * 1 = O_NONBLOCK flag when opening devices results in background device + * initialization (default) + */ +#define __NV_ENABLE_NONBLOCKING_OPEN EnableNonblockingOpen +#define NV_ENABLE_NONBLOCKING_OPEN NV_REG_STRING(__NV_ENABLE_NONBLOCKING_OPEN) + +/* + * Option: NVreg_ImexChannelCount + * + * Description: + * + * This option allows users to specify the number of IMEX (import/export) + * channels. Within an IMEX domain, the channels allow sharing memory + * securely in a multi-user environment using the CUDA driver's fabric handle + * based APIs. + * + * An IMEX domain is either an OS instance or a group of securely + * connected OS instances using the NVIDIA IMEX daemon. The option must + * be set to the same value on each OS instance within the IMEX domain. + * + * An IMEX channel is a logical entity that is represented by a /dev node. + * The IMEX channels are global resources within the IMEX domain. When + * exporter and importer CUDA processes have been granted access to the + * same IMEX channel, they can securely share memory. + * + * Note that the NVIDIA driver will not attempt to create the /dev nodes. Thus, + * the related CUDA APIs will fail with an insufficient permission error until + * the /dev nodes are set up. The creation of these /dev nodes, + * /dev/nvidia-caps-imex-channels/channelN, must be handled by the + * administrator, where N is the minor number. The major number can be + * queried from /proc/devices. + * + * nvidia-modprobe CLI support is available to set up the /dev nodes. + * NVreg_ModifyDeviceFiles, NVreg_DeviceFileGID, NVreg_DeviceFileUID + * and NVreg_DeviceFileMode will be honored by nvidia-modprobe. + * + * Also, refer to the NVreg_CreateImexChannel0 option. + * + * Possible values: + * 0 - Disable IMEX using CUDA driver's fabric handles. + * N - N IMEX channels will be enabled in the driver to facilitate N + * concurrent users. Default value is 2048 channels, and the current + * maximum value is 20-bit, same as Linux dev_t's minor number limit. + */ +#define __NV_IMEX_CHANNEL_COUNT ImexChannelCount +#define NV_REG_IMEX_CHANNEL_COUNT NV_REG_STRING(__NV_IMEX_CHANNEL_COUNT) + +/* + * Option: NVreg_CreateImexChannel0 + * + * Description: + * + * This option allows users to specify whether the NVIDIA driver must create + * the IMEX channel 0 by default. The channel will be created automatically + * when the NVIDIA open GPU kernel module is loaded. + * + * Note that users are advised to enable this option only in trusted + * environments where it is acceptable for applications to share the same + * IMEX channel. + * + * For more details on IMEX channels, refer to the NVreg_ImexChannelCount + * option. + * + * Possible values: + * 0 - Do not create IMEX channel 0 (default). + * 1 - Create IMEX channel 0. + */ +#define __NV_CREATE_IMEX_CHANNEL_0 CreateImexChannel0 +#define NV_CREATE_IMEX_CHANNEL_0 NV_REG_STRING(__CREATE_IMEX_CHANNEL_0) + +/* + * Option: NVreg_EnableSystemMemoryPools + * + * Description: + * + * This option controls system memory page pools creation for different page sizes + * Pool for pageSize is enabled by setting bit (pageSize >> NV_ENABLE_SYSTEM_MEMORY_POOLS_SHIFT) + * The pools keep memory cached once freed to speed-up reallocation + * Pools are shared by all adapters + * + * This feature is only supported by OpenRM driver + * + * By default 4K, 64K, 2M page size pools are enabled + */ +#define __NV_ENABLE_SYSTEM_MEMORY_POOLS EnableSystemMemoryPools +#define NV_ENABLE_SYSTEM_MEMORY_POOLS NV_REG_STRING(__NV_ENABLE_SYSTEM_MEMORY_POOLS) +#define NV_ENABLE_SYSTEM_MEMORY_POOLS_DEFAULT 0x00000211 +#define NV_ENABLE_SYSTEM_MEMORY_POOLS_SHIFT 12 + +/* + * Option: NVreg_OsEnableCxlSupport + * + * Description: + * + * When this option is enabled, the driver will enable support for + * CXL at the OS layer. + * + * This option has no effect on platforms that do not support CXL or on + * GPUs without CXL capability. + * + * Possible Values: + * + * 0: disable CXL support + * 1: enable CXL support (default) + */ +#define __NV_OS_ENABLE_CXL_SUPPORT OsEnableCxlSupport +#define NV_REG_OS_ENABLE_CXL_SUPPORT NV_REG_STRING(__NV_OS_ENABLE_CXL_SUPPORT) + +/* + * Option: NVreg_GpuInitOnProbe + * + * Description: + * + * This option allows the user to control initialization of the GPU at PCI probe time + * + * Possible values: + * 0 - Disable initializing the GPUs on PCI probe + * 1 - Initialize GPUs on PCI probe + */ + #define __NV_GPU_INIT_ON_PROBE GpuInitOnProbe + #define NV_GPU_INIT_ON_PROBE NV_REG_STRING(__NV_GPU_INIT_ON_PROBE) + #define NV_GPU_INIT_ON_PROBE_DEFAULT 0 + +#if defined(NV_DEFINE_REGISTRY_KEY_TABLE) + +/* + *---------registry key parameter declarations-------------- + */ + +NV_DEFINE_REG_ENTRY(__NV_RESMAN_DEBUG_LEVEL, ~0); +NV_DEFINE_REG_ENTRY(__NV_RM_LOGON_RC, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_MODIFY_DEVICE_FILES, 1); +NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_UID, 0); +NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_GID, 0); +NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_MODE, 0666); +NV_DEFINE_REG_ENTRY(__NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, 1); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_PCIE_GEN3, 0); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_MSI, 1); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_STREAM_MEMOPS, 0); +NV_DEFINE_REG_ENTRY(__NV_RM_PROFILING_ADMIN_ONLY_PARAMETER, 1); +NV_DEFINE_REG_ENTRY(__NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS, + NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_AUTO); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_S0IX_POWER_MANAGEMENT, 0); +NV_DEFINE_REG_ENTRY(__NV_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD, 256); +NV_DEFINE_REG_ENTRY(__NV_DYNAMIC_POWER_MANAGEMENT, 3); +NV_DEFINE_REG_ENTRY(__NV_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD, 200); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_GPU_FIRMWARE, NV_REG_ENABLE_GPU_FIRMWARE_DEFAULT_VALUE); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_GPU_FIRMWARE_LOGS, NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG); +NV_DEFINE_REG_ENTRY(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS, 1); + +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_USER_NUMA_MANAGEMENT, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_MEMORY_POOL_SIZE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_KMALLOC_HEAP_MAX_SIZE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_VMALLOC_HEAP_MAX_SIZE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_IGNORE_MMIO_CHECK, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_NVLINK_DISABLE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PCI_DRIVER, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PLATFORM_DEVICE_DRIVER, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_RESIZABLE_BAR, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_DBG_BREAKPOINT, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_TEGRA_GPU_PG_MASK, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_NONBLOCKING_OPEN, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_EXCLUDE_ALL_GPUS, NV_EXCLUDE_ALL_GPUS_DEFAULT); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_GPU_INIT_ON_PROBE, NV_GPU_INIT_ON_PROBE_DEFAULT); + +NV_DEFINE_REG_STRING_ENTRY(__NV_COHERENT_GPU_MEMORY_MODE, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS_PER_DEVICE, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_RM_MSG, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_GPU_BLACKLIST, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_TEMPORARY_FILE_PATH, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_EXCLUDED_GPUS, NULL); +NV_DEFINE_REG_ENTRY(__NV_DMA_REMAP_PEER_MMIO, NV_DMA_REMAP_PEER_MMIO_ENABLE); +NV_DEFINE_REG_STRING_ENTRY(__NV_RM_NVLINK_BW, NULL); +NV_DEFINE_REG_ENTRY(__NV_RM_NVLINK_BW_LINK_COUNT, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_IMEX_CHANNEL_COUNT, 2048); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_CREATE_IMEX_CHANNEL_0, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_SYSTEM_MEMORY_POOLS, NV_ENABLE_SYSTEM_MEMORY_POOLS_DEFAULT); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_OS_ENABLE_CXL_SUPPORT, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_USE_KERNEL_SUSPEND_NOTIFIERS, 0); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_NON_PREEMTABLE_DEBUGGER_SESSION, 0); +/* + *----------------registry database definition---------------------- + */ + +/* + * You can enable any of the registry options disabled by default by + * editing their respective entries in the table below. The last field + * determines if the option is considered valid - in order for the + * changes to take effect, you need to recompile and reload the NVIDIA + * kernel module. + */ +nv_parm_t nv_parms[] = { + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RESMAN_DEBUG_LEVEL), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RM_LOGON_RC), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_MODIFY_DEVICE_FILES), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_UID), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_GID), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_MODE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_MSI), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_PCIE_GEN3), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_MEMORY_POOL_SIZE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_KMALLOC_HEAP_MAX_SIZE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_VMALLOC_HEAP_MAX_SIZE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_IGNORE_MMIO_CHECK), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_STREAM_MEMOPS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_USER_NUMA_MANAGEMENT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_NVLINK_DISABLE), + NV_DEFINE_PARAMS_TABLE_ENTRY_CUSTOM_NAME(__NV_RM_PROFILING_ADMIN_ONLY, + __NV_RM_PROFILING_ADMIN_ONLY_PARAMETER), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_USE_KERNEL_SUSPEND_NOTIFIERS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_S0IX_POWER_MANAGEMENT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DYNAMIC_POWER_MANAGEMENT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_TEGRA_GPU_PG_MASK), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_REGISTER_PCI_DRIVER), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_RESIZABLE_BAR), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_GPU_FIRMWARE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_GPU_FIRMWARE_LOGS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RM_NVLINK_BW_LINK_COUNT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_DBG_BREAKPOINT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DMA_REMAP_PEER_MMIO), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_IMEX_CHANNEL_COUNT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_CREATE_IMEX_CHANNEL_0), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_SYSTEM_MEMORY_POOLS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OS_ENABLE_CXL_SUPPORT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_NON_PREEMTABLE_DEBUGGER_SESSION), + {NULL, NULL} +}; + +#elif defined(NVRM) + +extern nv_parm_t nv_parms[]; + +#endif /* NV_DEFINE_REGISTRY_KEY_TABLE */ + +#endif /* _RM_REG_H_ */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv-unix-nvos-params-wrappers.h b/userspace/sdks/nv570/nvalloc/unix/include/nv-unix-nvos-params-wrappers.h new file mode 100644 index 0000000..02b0156 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv-unix-nvos-params-wrappers.h @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_UNIX_NVOS_PARAMS_WRAPPERS_H_ +#define _NV_UNIX_NVOS_PARAMS_WRAPPERS_H_ + +#include <nvos.h> + +/* + * This is a wrapper for NVOS02_PARAMETERS with file descriptor + */ + +typedef struct +{ + NVOS02_PARAMETERS params; + int fd; +} nv_ioctl_nvos02_parameters_with_fd; + +/* + * This is a wrapper for NVOS33_PARAMETERS with file descriptor + */ +typedef struct +{ + NVOS33_PARAMETERS params; + int fd; +} nv_ioctl_nvos33_parameters_with_fd; + +#endif // _NV_UNIX_NVOS_PARAMS_WRAPPERS_H_ + diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv.h b/userspace/sdks/nv570/nvalloc/unix/include/nv.h new file mode 100644 index 0000000..d8db166 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv.h @@ -0,0 +1,1367 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef _NV_H_ +#define _NV_H_ + + + +#include <nvlimits.h> + +#if defined(NV_KERNEL_INTERFACE_LAYER) && defined(__FreeBSD__) + #include <sys/stddef.h> // NULL +#elif defined(NV_KERNEL_INTERFACE_LAYER) && defined(NV_LINUX) + #include <linux/stddef.h> // NULL +#else + #include <stddef.h> // NULL +#endif + +#include <nvstatus.h> +#include "nv_stdarg.h" +#include <nv-caps.h> +#include <nv-firmware.h> +#include <nv-ioctl.h> +#include <nv-ioctl-numa.h> +#include <nvmisc.h> +#include <os/nv_memory_area.h> + +#if !defined(NV_KERNEL_INTERFACE_LAYER) && defined(NVRM) && !defined(NV_DRM_FREEBSD_LKPI) +#include <nvport/nvport.h> +#else +#define PORT_ATOMIC +#endif + +extern nv_cap_t *nvidia_caps_root; + +extern const NvBool nv_is_rm_firmware_supported_os; + +#include <nvi2c.h> +#include <nvimpshared.h> + +#include <nv-kernel-interface-api.h> + +#define GPU_UUID_LEN (16) + +/* + * Buffer size for an ASCII UUID: We need 2 digits per byte, plus space + * for "GPU", 5 dashes, and '\0' termination: + */ +#define GPU_UUID_ASCII_LEN (GPU_UUID_LEN * 2 + 9) + +/* + * #define an absolute maximum used as a sanity check for the + * NV_ESC_IOCTL_XFER_CMD ioctl() size argument. + */ +#define NV_ABSOLUTE_MAX_IOCTL_SIZE 16384 + +/* + * Solaris provides no more than 8 bits for the argument size in + * the ioctl() command encoding; make sure we don't exceed this + * limit. + */ +#define __NV_IOWR_ASSERT(type) ((sizeof(type) <= NV_PLATFORM_MAX_IOCTL_SIZE) ? 1 : -1) +#define __NV_IOWR(nr, type) ({ \ + typedef char __NV_IOWR_TYPE_SIZE_ASSERT[__NV_IOWR_ASSERT(type)]; \ + _IOWR(NV_IOCTL_MAGIC, (nr), type); \ +}) + +#define NV_PCI_DEV_FMT "%04x:%02x:%02x.%x" +#define NV_PCI_DEV_FMT_ARGS(nv) (nv)->pci_info.domain, (nv)->pci_info.bus, \ + (nv)->pci_info.slot, (nv)->pci_info.function + +#define NV_RM_DEVICE_INTR_ADDRESS 0x100 + +#define NV_TEGRA_PCI_IGPU_PG_MASK_DEFAULT 0xFFFFFFFF + +/* + * Clock domain identifier, which is used for fetching the engine + * load backed by the specified clock domain for Tegra platforms + * conforming linux devfreq framework to realize dynamic frequency + * scaling. + */ +typedef enum _TEGRASOC_DEVFREQ_CLK +{ + TEGRASOC_DEVFREQ_CLK_GPC, + TEGRASOC_DEVFREQ_CLK_NVD, +} TEGRASOC_DEVFREQ_CLK; + +/*! + * @brief The order of the display clocks in the below defined enum + * should be synced with below mapping array and macro. + * All four should be updated simultaneously in case + * of removal or addition of clocks in below order. + * Also, TEGRASOC_WHICH_CLK_MAX is used in various places + * in below mentioned files. + * arch/nvalloc/unix/Linux/nv-linux.h + * + * arch/nvalloc/unix/src/os.c + * dispClkMapRmToOsArr[] = {...}; + * + * arch/nvalloc/unix/Linux/nv-clk.c + * osMapClk[] = {...}; + * + */ +typedef enum _TEGRASOC_WHICH_CLK +{ + TEGRASOC_WHICH_CLK_NVDISPLAYHUB, + TEGRASOC_WHICH_CLK_NVDISPLAY_DISP, + TEGRASOC_WHICH_CLK_NVDISPLAY_P0, + TEGRASOC_WHICH_CLK_NVDISPLAY_P1, + TEGRASOC_WHICH_CLK_NVDISPLAY_P2, + TEGRASOC_WHICH_CLK_NVDISPLAY_P3, + TEGRASOC_WHICH_CLK_NVDISPLAY_P4, + TEGRASOC_WHICH_CLK_NVDISPLAY_P5, + TEGRASOC_WHICH_CLK_NVDISPLAY_P6, + TEGRASOC_WHICH_CLK_NVDISPLAY_P7, + TEGRASOC_WHICH_CLK_DPAUX0, + TEGRASOC_WHICH_CLK_FUSE, + TEGRASOC_WHICH_CLK_DSIPLL_VCO, + TEGRASOC_WHICH_CLK_DSIPLL_CLKOUTPN, + TEGRASOC_WHICH_CLK_DSIPLL_CLKOUTA, + TEGRASOC_WHICH_CLK_SPPLL0_VCO, + TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTA, + TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTB, + TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTPN, + TEGRASOC_WHICH_CLK_SPPLL1_CLKOUTPN, + TEGRASOC_WHICH_CLK_SPPLL0_DIV27, + TEGRASOC_WHICH_CLK_SPPLL1_DIV27, + TEGRASOC_WHICH_CLK_SPPLL0_DIV10, + TEGRASOC_WHICH_CLK_SPPLL0_DIV25, + TEGRASOC_WHICH_CLK_SPPLL1_VCO, + TEGRASOC_WHICH_CLK_VPLL0_REF, + TEGRASOC_WHICH_CLK_VPLL0, + TEGRASOC_WHICH_CLK_VPLL1, + TEGRASOC_WHICH_CLK_VPLL2, + TEGRASOC_WHICH_CLK_VPLL3, + TEGRASOC_WHICH_CLK_VPLL4, + TEGRASOC_WHICH_CLK_VPLL5, + TEGRASOC_WHICH_CLK_VPLL6, + TEGRASOC_WHICH_CLK_VPLL7, + TEGRASOC_WHICH_CLK_NVDISPLAY_P0_REF, + TEGRASOC_WHICH_CLK_RG0, + TEGRASOC_WHICH_CLK_RG1, + TEGRASOC_WHICH_CLK_RG2, + TEGRASOC_WHICH_CLK_RG3, + TEGRASOC_WHICH_CLK_RG4, + TEGRASOC_WHICH_CLK_RG5, + TEGRASOC_WHICH_CLK_RG6, + TEGRASOC_WHICH_CLK_RG7, + TEGRASOC_WHICH_CLK_DISPPLL, + TEGRASOC_WHICH_CLK_DISPHUBPLL, + TEGRASOC_WHICH_CLK_DSI_LP, + TEGRASOC_WHICH_CLK_DSI_CORE, + TEGRASOC_WHICH_CLK_DSI_PIXEL, + TEGRASOC_WHICH_CLK_PRE_SOR0, + TEGRASOC_WHICH_CLK_PRE_SOR1, + TEGRASOC_WHICH_CLK_PRE_SOR2, + TEGRASOC_WHICH_CLK_PRE_SOR3, + TEGRASOC_WHICH_CLK_DP_LINKA_REF, + TEGRASOC_WHICH_CLK_DP_LINKB_REF, + TEGRASOC_WHICH_CLK_DP_LINKC_REF, + TEGRASOC_WHICH_CLK_DP_LINKD_REF, + TEGRASOC_WHICH_CLK_SOR_LINKA_INPUT, + TEGRASOC_WHICH_CLK_SOR_LINKB_INPUT, + TEGRASOC_WHICH_CLK_SOR_LINKC_INPUT, + TEGRASOC_WHICH_CLK_SOR_LINKD_INPUT, + TEGRASOC_WHICH_CLK_SOR_LINKA_AFIFO, + TEGRASOC_WHICH_CLK_SOR_LINKB_AFIFO, + TEGRASOC_WHICH_CLK_SOR_LINKC_AFIFO, + TEGRASOC_WHICH_CLK_SOR_LINKD_AFIFO, + TEGRASOC_WHICH_CLK_SOR_LINKA_AFIFO_M, + TEGRASOC_WHICH_CLK_RG0_M, + TEGRASOC_WHICH_CLK_RG1_M, + TEGRASOC_WHICH_CLK_SOR0_M, + TEGRASOC_WHICH_CLK_SOR1_M, + TEGRASOC_WHICH_CLK_PLLHUB, + TEGRASOC_WHICH_CLK_SOR0, + TEGRASOC_WHICH_CLK_SOR1, + TEGRASOC_WHICH_CLK_SOR2, + TEGRASOC_WHICH_CLK_SOR3, + TEGRASOC_WHICH_CLK_SOR_PADA_INPUT, + TEGRASOC_WHICH_CLK_SOR_PADB_INPUT, + TEGRASOC_WHICH_CLK_SOR_PADC_INPUT, + TEGRASOC_WHICH_CLK_SOR_PADD_INPUT, + TEGRASOC_WHICH_CLK_SOR0_PAD, + TEGRASOC_WHICH_CLK_SOR1_PAD, + TEGRASOC_WHICH_CLK_SOR2_PAD, + TEGRASOC_WHICH_CLK_SOR3_PAD, + TEGRASOC_WHICH_CLK_PRE_SF0, + TEGRASOC_WHICH_CLK_SF0, + TEGRASOC_WHICH_CLK_SF1, + TEGRASOC_WHICH_CLK_SF2, + TEGRASOC_WHICH_CLK_SF3, + TEGRASOC_WHICH_CLK_SF4, + TEGRASOC_WHICH_CLK_SF5, + TEGRASOC_WHICH_CLK_SF6, + TEGRASOC_WHICH_CLK_SF7, + TEGRASOC_WHICH_CLK_DSI_PAD_INPUT, + TEGRASOC_WHICH_CLK_PRE_SOR0_REF, + TEGRASOC_WHICH_CLK_PRE_SOR1_REF, + TEGRASOC_WHICH_CLK_SOR0_PLL_REF, + TEGRASOC_WHICH_CLK_SOR1_PLL_REF, + TEGRASOC_WHICH_CLK_SOR2_PLL_REF, + TEGRASOC_WHICH_CLK_SOR3_PLL_REF, + TEGRASOC_WHICH_CLK_SOR0_REF, + TEGRASOC_WHICH_CLK_SOR1_REF, + TEGRASOC_WHICH_CLK_SOR2_REF, + TEGRASOC_WHICH_CLK_SOR3_REF, + TEGRASOC_WHICH_CLK_OSC, + TEGRASOC_WHICH_CLK_DSC, + TEGRASOC_WHICH_CLK_MAUD, + TEGRASOC_WHICH_CLK_AZA_2XBIT, + TEGRASOC_WHICH_CLK_AZA_BIT, + TEGRASOC_WHICH_CLK_MIPI_CAL, + TEGRASOC_WHICH_CLK_UART_FST_MIPI_CAL, + TEGRASOC_WHICH_CLK_SOR0_DIV, + TEGRASOC_WHICH_CLK_DISP_ROOT, + TEGRASOC_WHICH_CLK_HUB_ROOT, + TEGRASOC_WHICH_CLK_PLLA_DISP, + TEGRASOC_WHICH_CLK_PLLA_DISPHUB, + TEGRASOC_WHICH_CLK_PLLA, + TEGRASOC_WHICH_CLK_VPLLX_SOR0_MUXED, + TEGRASOC_WHICH_CLK_VPLLX_SOR1_MUXED, + TEGRASOC_WHICH_CLK_VPLLX_SOR2_MUXED, + TEGRASOC_WHICH_CLK_VPLLX_SOR3_MUXED, + TEGRASOC_WHICH_CLK_SF0_SOR, + TEGRASOC_WHICH_CLK_SF1_SOR, + TEGRASOC_WHICH_CLK_SF2_SOR, + TEGRASOC_WHICH_CLK_SF3_SOR, + TEGRASOC_WHICH_CLK_SF4_SOR, + TEGRASOC_WHICH_CLK_SF5_SOR, + TEGRASOC_WHICH_CLK_SF6_SOR, + TEGRASOC_WHICH_CLK_SF7_SOR, + TEGRASOC_WHICH_CLK_EMC, + TEGRASOC_WHICH_CLK_GPU_FIRST, + TEGRASOC_WHICH_CLK_GPU_SYS = TEGRASOC_WHICH_CLK_GPU_FIRST, + TEGRASOC_WHICH_CLK_GPU_NVD, + TEGRASOC_WHICH_CLK_GPU_UPROC, + TEGRASOC_WHICH_CLK_GPU_GPC0, + TEGRASOC_WHICH_CLK_GPU_GPC1, + TEGRASOC_WHICH_CLK_GPU_GPC2, + TEGRASOC_WHICH_CLK_GPU_LAST = TEGRASOC_WHICH_CLK_GPU_GPC2, + TEGRASOC_WHICH_CLK_MAX, // TEGRASOC_WHICH_CLK_MAX is defined for boundary checks only. +} TEGRASOC_WHICH_CLK; + +#ifdef NVRM + +extern const char *pNVRM_ID; + +/* + * ptr arithmetic convenience + */ + +typedef union +{ + volatile NvV8 Reg008[1]; + volatile NvV16 Reg016[1]; + volatile NvV32 Reg032[1]; +} nv_hwreg_t, * nv_phwreg_t; + + +#define NVRM_PCICFG_NUM_BARS 6 +#define NVRM_PCICFG_BAR_OFFSET(i) (0x10 + (i) * 4) +#define NVRM_PCICFG_BAR_REQTYPE_MASK 0x00000001 +#define NVRM_PCICFG_BAR_REQTYPE_MEMORY 0x00000000 +#define NVRM_PCICFG_BAR_MEMTYPE_MASK 0x00000006 +#define NVRM_PCICFG_BAR_MEMTYPE_64BIT 0x00000004 +#define NVRM_PCICFG_BAR_ADDR_MASK 0xfffffff0 + +#define NVRM_PCICFG_NUM_DWORDS 16 + +#define NV_GPU_NUM_BARS 3 +#define NV_GPU_BAR_INDEX_REGS 0 +#define NV_GPU_BAR_INDEX_FB 1 +#define NV_GPU_BAR_INDEX_IMEM 2 + +typedef struct +{ + NvU64 cpu_address; + NvU64 size; + NvU32 offset; + NvU32 *map; + nv_phwreg_t map_u; +} nv_aperture_t; + +typedef struct +{ + char *name; + NvU32 *data; +} nv_parm_t; + +#define NV_RM_PAGE_SHIFT 12 +#define NV_RM_PAGE_SIZE (1 << NV_RM_PAGE_SHIFT) +#define NV_RM_PAGE_MASK (NV_RM_PAGE_SIZE - 1) + +#define NV_RM_TO_OS_PAGE_SHIFT (os_page_shift - NV_RM_PAGE_SHIFT) +#define NV_RM_PAGES_TO_OS_PAGES(count) \ + ((((NvUPtr)(count)) >> NV_RM_TO_OS_PAGE_SHIFT) + \ + ((((count) & ((1 << NV_RM_TO_OS_PAGE_SHIFT) - 1)) != 0) ? 1 : 0)) + +// for dynamic granularity page array feature. +// Notice: dst_granu should be larger than src_granu. +#define NV_SRC_TO_DST_PAGE_SHIFT(src_granu, dst_granu) (GET_PAGE_SHIFT(dst_granu) - GET_PAGE_SHIFT(src_granu)) +#define NV_SRC_TO_DST_PAGE_COUNT(src_page_count,src_granu, dst_granu) \ + ((((NvUPtr)(src_page_count)) >> NV_SRC_TO_DST_PAGE_SHIFT(src_granu, dst_granu)) + \ + ((((src_page_count) & ((1 << NV_SRC_TO_DST_PAGE_SHIFT(src_granu, dst_granu)) - 1)) != 0) ? 1 : 0)) + +#if defined(NVCPU_X86_64) +#define NV_STACK_SIZE (NV_RM_PAGE_SIZE * 3) +#else +#define NV_STACK_SIZE (NV_RM_PAGE_SIZE * 2) +#endif + +typedef struct nvidia_stack_s +{ + NvU32 size; + void *top; + NvU8 stack[NV_STACK_SIZE-16] __attribute__ ((aligned(16))); +} nvidia_stack_t; + +/* + * TODO: Remove once all UNIX layers have been converted to use nvidia_stack_t + */ +typedef nvidia_stack_t nv_stack_t; + +typedef struct nv_file_private_t nv_file_private_t; + +/* + * this is a wrapper for unix events + * unlike the events that will be returned to clients, this includes + * kernel-specific data, such as file pointer, etc.. + */ +typedef struct nv_event_s +{ + NvHandle hParent; + NvHandle hObject; + NvU32 index; + NvU32 info32; + NvU16 info16; + nv_file_private_t *nvfp; /* per file-descriptor data pointer */ + NvU32 fd; + PORT_ATOMIC NvS32 active; /* whether the event should be signaled */ + NvU32 refcount; /* count of associated RM events */ + struct nv_event_s *next; +} nv_event_t; + +typedef struct nv_kern_mapping_s +{ + void *addr; + NvU64 size; + NvU32 modeFlag; + struct nv_kern_mapping_s *next; +} nv_kern_mapping_t; + +typedef struct nv_usermap_access_params_s +{ + NvU64 addr; + NvU64 size; + NvU64 offset; + NvU64 *page_array; + NvU64 num_pages; + MemoryArea memArea; + NvU64 access_start; + NvU64 access_size; + NvBool contig; + NvU32 caching; +} nv_usermap_access_params_t; + +/* + * It stores mapping context per mapping + */ +typedef struct nv_alloc_mapping_context_s { + void *alloc; + NvU64 page_index; + NvU64 *page_array; + NvU64 num_pages; + MemoryArea memArea; + NvU64 access_start; + NvU64 access_size; + NvU32 prot; + NvBool valid; + NvU32 caching; +} nv_alloc_mapping_context_t; + +struct nv_alloc_mapping_list_node_s; + +typedef struct nv_alloc_mapping_list_node_s { + nv_alloc_mapping_context_t context; + MemoryRange fileRange; + struct nv_alloc_mapping_list_node_s *pNext; + struct nv_alloc_mapping_list_node_s *pPrev; +} nv_alloc_mapping_list_node_t; + +typedef enum +{ + NV_SOC_IRQ_DISPLAY_TYPE = 0x1, + NV_SOC_IRQ_DPAUX_TYPE, + NV_SOC_IRQ_GPIO_TYPE, + NV_SOC_IRQ_HDACODEC_TYPE, + NV_SOC_IRQ_TCPC2DISP_TYPE, + NV_SOC_IRQ_INVALID_TYPE +} nv_soc_irq_type_t; + +/* + * It stores interrupt numbers and interrupt type and private data + */ +typedef struct nv_soc_irq_info_s { + NvU32 irq_num; + nv_soc_irq_type_t irq_type; + NvBool bh_pending; + union { + NvU32 gpio_num; + NvU32 dpaux_instance; + } irq_data; + NvS32 ref_count; +} nv_soc_irq_info_t; + +#define NV_MAX_SOC_IRQS 10 +#define NV_MAX_DPAUX_NUM_DEVICES 4 +#define NV_MAX_DPAUX_DEV_NAME_SIZE 10 + +#define NV_MAX_SOC_DPAUX_NUM_DEVICES 4 + +/* + * per device state + */ + +/* DMA-capable device data, defined by kernel interface layer */ +typedef struct nv_dma_device nv_dma_device_t; + +typedef struct nv_phys_addr_range +{ + NvU64 addr; + NvU64 len; +} nv_phys_addr_range_t; + +typedef struct +{ + char vbios_version[15]; + char firmware_version[64]; +} nv_cached_gpu_info_t; + +typedef struct nv_state_t +{ + void *priv; /* private data */ + void *os_state; /* os-specific device state */ + + int flags; + + /* PCI config info */ + nv_pci_info_t pci_info; + NvU16 subsystem_id; + NvU16 subsystem_vendor; + NvU32 gpu_id; + NvU32 iovaspace_id; + struct + { + NvBool valid; + NvU8 uuid[GPU_UUID_LEN]; + NvBool pci_uuid_read_attempted; + NV_STATUS pci_uuid_status; + } nv_uuid_cache; + void *handle; + + NvU32 pci_cfg_space[NVRM_PCICFG_NUM_DWORDS]; + + /* physical characteristics */ + nv_aperture_t bars[NV_GPU_NUM_BARS]; + nv_aperture_t *regs; + nv_aperture_t *dpaux[NV_MAX_DPAUX_NUM_DEVICES]; + nv_aperture_t *hdacodec_regs; + nv_aperture_t *mipical_regs; + nv_aperture_t *fb, ud; + nv_aperture_t *simregs; + + NvU32 num_dpaux_instance; + NvU32 interrupt_line; + NvU32 dpaux_irqs[NV_MAX_DPAUX_NUM_DEVICES]; + char dpaux_devname[NV_MAX_DPAUX_NUM_DEVICES][NV_MAX_DPAUX_DEV_NAME_SIZE]; + nv_soc_irq_info_t soc_irq_info[NV_MAX_SOC_IRQS]; + NvS32 current_soc_irq; + NvU32 num_soc_irqs; + NvU32 hdacodec_irq; + NvU32 tcpc2disp_irq; + NvU8 *soc_dcb_blob; + NvU32 soc_dcb_size; + NvU32 disp_sw_soc_chip_id; + NvBool soc_is_dpalt_mode_supported; + NvBool soc_is_hfrp_supported; + NvBool soc_is_external_phy_supported; + NvU64 dma_mask; + + NvBool is_tegra_pci_igpu; + NvBool supports_tegra_igpu_rg; + NvBool is_tegra_pci_igpu_rg_enabled; + NvU32 tegra_pci_igpu_pg_mask; + + NvBool primary_vga; + + NvU32 sim_env; + + NvU32 rc_timer_enabled; + + /* list of events allocated for this device */ + nv_event_t *event_list; + + /* lock to protect event_list */ + void *event_spinlock; + + nv_kern_mapping_t *kern_mappings; + + /* Kernel interface DMA device data */ + nv_dma_device_t *dma_dev; + nv_dma_device_t *niso_dma_dev; + + /* + * Per-GPU queue. The actual queue object is usually allocated in the + * arch-specific parent structure (e.g. nv_linux_state_t), and this + * pointer just points to it. + */ + struct os_work_queue *queue; + + /* For loading RM as a firmware (DCE or GSP) client */ + NvBool request_firmware; /* request firmware from the OS */ + NvBool request_fw_client_rm; /* attempt to init RM as FW a client */ + NvBool allow_fallback_to_monolithic_rm; /* allow fallback to monolithic RM if FW client RM doesn't work out */ + NvBool enable_firmware_logs; /* attempt to enable firmware log decoding/printing */ + + /* Variable to track, if nvidia_remove is called */ + NvBool removed; + + NvBool console_device; + + /* Variable to track, if GPU is external GPU */ + NvBool is_external_gpu; + + /* Variable to track, if regkey PreserveVideoMemoryAllocations is set */ + NvBool preserve_vidmem_allocations; + + /* Variable to force allocation of 32-bit addressable memory */ + NvBool force_dma32_alloc; + + /* PCI power state should be D0 during system suspend */ + NvBool d0_state_in_suspend; + + /* Current cyclestats client and context */ + NvU32 profiler_owner; + void *profiler_context; + + /* + * RMAPI objects to use in the OS layer to talk to core RM. + * + * Note that we only need to store one subdevice handle: in SLI, we will + * have a separate nv_state_t per physical GPU. + */ + struct { + NvHandle hClient; + NvHandle hDevice; + NvHandle hSubDevice; + NvHandle hI2C; + NvHandle hDisp; + } rmapi; + + /* Bool to check if dma-buf is supported */ + NvBool dma_buf_supported; + + /* Bool to check if the device received a shutdown notification */ + NvBool is_shutdown; + + /* Bool to check if the GPU has a coherent sysmem link */ + NvBool coherent; + + /* + * Bool to check if GPU memory is backed by struct page. + * False for non-coherent platforms. May also be false + * on coherent platforms if GPU memory is not onlined to the kernel. + */ + NvBool mem_has_struct_page; + + /* + * Coherent GPU Memory Mode, one of: + * NV_COHERENT_GPU_MEM_MODE_NONE + * NV_COHERENT_GPU_MEM_MODE_NUMA + * NV_COHERENT_GPU_MEM_MODE_DRIVER + */ + NvBool coherent_gpu_mem_mode; + + /* OS detected GPU has ATS capability */ + NvBool ats_support; + /* + * NUMA node ID of the CPU to which the GPU is attached. + * Holds NUMA_NO_NODE on platforms that don't support NUMA configuration. + */ + NvS32 cpu_numa_node_id; + + struct { + /* Bool to check if ISO iommu enabled */ + NvBool iso_iommu_present; + /* Bool to check if NISO iommu enabled */ + NvBool niso_iommu_present; + /* Display SMMU Stream IDs */ + NvU32 dispIsoStreamId; + NvU32 dispNisoStreamId; + } iommus; + + struct { + NvU32 max_dispclk_rate_using_disppllkhz; + NvU32 max_dispclk_rate_using_sppll0clkoutakhz; + NvU32 max_hubclk_rate_using_sppll0clkoutbkhz; + } clocks; + + /* Console is managed by drm drivers or NVKMS */ + NvBool client_managed_console; + + /* Console is mapped to sysmem instead of GPU PCIe BAR1 or BAR2 */ + NvBool sysmem_mapped_console; + + /* Struct to cache the gpu info details */ + nv_cached_gpu_info_t cached_gpu_info; + + /* Bool to check if power management is supported */ + NvBool is_pm_unsupported; + + /* Bool to check if the GPU is a CXL device */ + NvBool is_cxl_dev; +} nv_state_t; + +#define NVFP_TYPE_NONE ((NvU32)0x0) +#define NVFP_TYPE_REFCOUNTED ((NvU32)0x1) +#define NVFP_TYPE_REGISTERED ((NvU32)0x2) + +struct nv_file_private_t +{ + NvHandle *handles; + NvU16 maxHandles; + NvU32 deviceInstance; + NvU32 gpuInstanceId; + NvU8 metadata[64]; + + nv_file_private_t * PORT_ATOMIC ctl_nvfp; + void *ctl_nvfp_priv; + PORT_ATOMIC NvU32 register_or_refcount; + // In-flight nv_post_event() calls using this file private. + PORT_ATOMIC NvS32 event_posting_refcount; + + // + // True if a client or an event was ever allocated on this fd. + // If false, RMAPI cleanup is skipped. + // + NvBool bCleanupRmapi; +}; + +static inline NvS32 nv_event_posting_refcount_read(nv_file_private_t *nvfp) +{ + return *((volatile NvS32 *)&nvfp->event_posting_refcount); +} + +// Forward define the gpu ops structures +typedef struct gpuSession *nvgpuSessionHandle_t; +typedef struct gpuDevice *nvgpuDeviceHandle_t; +typedef struct gpuAddressSpace *nvgpuAddressSpaceHandle_t; +typedef struct gpuTsg *nvgpuTsgHandle_t; +typedef struct UvmGpuTsgAllocParams_tag nvgpuTsgAllocParams_t; +typedef struct gpuChannel *nvgpuChannelHandle_t; +typedef struct UvmGpuChannelInfo_tag *nvgpuChannelInfo_t; +typedef struct UvmGpuChannelAllocParams_tag nvgpuChannelAllocParams_t; +typedef struct UvmGpuCaps_tag *nvgpuCaps_t; +typedef struct UvmGpuCopyEnginesCaps_tag *nvgpuCesCaps_t; +typedef struct UvmGpuAddressSpaceInfo_tag *nvgpuAddressSpaceInfo_t; +typedef struct UvmGpuAllocInfo_tag *nvgpuAllocInfo_t; +typedef struct UvmGpuP2PCapsParams_tag *nvgpuP2PCapsParams_t; +typedef struct UvmGpuFbInfo_tag *nvgpuFbInfo_t; +typedef struct UvmGpuNvlinkInfo_tag *nvgpuNvlinkInfo_t; +typedef struct UvmGpuEccInfo_tag *nvgpuEccInfo_t; +typedef struct UvmGpuFaultInfo_tag *nvgpuFaultInfo_t; +typedef struct UvmGpuAccessCntrInfo_tag *nvgpuAccessCntrInfo_t; +typedef struct UvmGpuAccessCntrConfig_tag nvgpuAccessCntrConfig_t; +typedef struct UvmGpuInfo_tag nvgpuInfo_t; +typedef struct UvmGpuClientInfo_tag nvgpuClientInfo_t; +typedef struct UvmPmaAllocationOptions_tag *nvgpuPmaAllocationOptions_t; +typedef struct UvmPmaStatistics_tag *nvgpuPmaStatistics_t; +typedef struct UvmGpuMemoryInfo_tag *nvgpuMemoryInfo_t; +typedef struct UvmGpuExternalMappingInfo_tag *nvgpuExternalMappingInfo_t; +typedef struct UvmGpuExternalPhysAddrInfo_tag *nvgpuExternalPhysAddrInfo_t; +typedef struct UvmGpuChannelResourceInfo_tag *nvgpuChannelResourceInfo_t; +typedef struct UvmGpuChannelInstanceInfo_tag *nvgpuChannelInstanceInfo_t; +typedef struct UvmGpuChannelResourceBindParams_tag *nvgpuChannelResourceBindParams_t; +typedef struct UvmGpuPagingChannelAllocParams_tag nvgpuPagingChannelAllocParams_t; +typedef struct UvmGpuPagingChannel_tag *nvgpuPagingChannelHandle_t; +typedef struct UvmGpuPagingChannelInfo_tag *nvgpuPagingChannelInfo_t; +typedef enum UvmPmaGpuMemoryType_tag nvgpuGpuMemoryType_t; +typedef NV_STATUS (*nvPmaEvictPagesCallback)(void *, NvU64, NvU64 *, NvU32, NvU64, NvU64, nvgpuGpuMemoryType_t); +typedef NV_STATUS (*nvPmaEvictRangeCallback)(void *, NvU64, NvU64, nvgpuGpuMemoryType_t); +typedef struct UvmGpuAccessBitsBufferAlloc_tag *nvgpuAccessBitBufferAlloc_t; +/* + * flags + */ + +#define NV_FLAG_INITIALIZED 0x0001 +#define NV_FLAG_EXCLUDE 0x0002 +#define NV_FLAG_CONTROL 0x0004 +#define NV_FLAG_PCI_P2P_UNSUPPORTED_CHIPSET 0x0008 +#define NV_FLAG_SOC_DISPLAY 0x0010 +#define NV_FLAG_USES_MSI 0x0020 +#define NV_FLAG_USES_MSIX 0x0040 +#define NV_FLAG_PASSTHRU 0x0080 +#define NV_FLAG_SUSPENDED 0x0100 +#define NV_FLAG_HAS_CONSOLE_IN_SYSMEM_CARVEOUT 0x0200 +/* To be set when an FLR needs to be triggered after device shut down. */ +#define NV_FLAG_TRIGGER_FLR 0x0400 +#define NV_FLAG_PERSISTENT_SW_STATE 0x0800 +#define NV_FLAG_IN_RECOVERY 0x1000 +// Unused 0x2000 +#define NV_FLAG_UNBIND_LOCK 0x4000 +/* To be set when GPU is not present on the bus, to help device teardown */ +#define NV_FLAG_IN_SURPRISE_REMOVAL 0x8000 + +typedef enum +{ + NV_PM_ACTION_HIBERNATE, + NV_PM_ACTION_STANDBY, + NV_PM_ACTION_RESUME +} nv_pm_action_t; + +typedef enum +{ + NV_PM_ACTION_DEPTH_DEFAULT, + NV_PM_ACTION_DEPTH_MODESET, + NV_PM_ACTION_DEPTH_UVM +} nv_pm_action_depth_t; + +typedef enum +{ + NV_DYNAMIC_PM_NEVER, + NV_DYNAMIC_PM_COARSE, + NV_DYNAMIC_PM_FINE +} nv_dynamic_power_mode_t; + +typedef enum +{ + NV_POWER_STATE_IN_HIBERNATE, + NV_POWER_STATE_IN_STANDBY, + NV_POWER_STATE_RUNNING +} nv_power_state_t; + +typedef struct +{ + const char *vidmem_power_status; + const char *dynamic_power_status; + const char *gc6_support; + const char *gcoff_support; + const char *s0ix_status; + const char *db_support; +} nv_power_info_t; + +typedef enum +{ + NV_MEMORY_TYPE_SYSTEM, /* Memory mapped for ROM, SBIOS and physical RAM. */ + NV_MEMORY_TYPE_REGISTERS, + NV_MEMORY_TYPE_FRAMEBUFFER, + NV_MEMORY_TYPE_DEVICE_MMIO, /* All kinds of MMIO referred by NVRM e.g. BARs and MCFG of device */ +} nv_memory_type_t; + +#define NV_PRIMARY_VGA(nv) ((nv)->primary_vga) + +#define NV_IS_CTL_DEVICE(nv) ((nv)->flags & NV_FLAG_CONTROL) +#define NV_IS_SOC_DISPLAY_DEVICE(nv) \ + ((nv)->flags & NV_FLAG_SOC_DISPLAY) + +#define NV_IS_DEVICE_IN_SURPRISE_REMOVAL(nv) \ + (((nv)->flags & NV_FLAG_IN_SURPRISE_REMOVAL) != 0) + +#define NV_HAS_CONSOLE_IN_SYSMEM_CARVEOUT(nv) \ + (((nv)->flags & NV_FLAG_HAS_CONSOLE_IN_SYSMEM_CARVEOUT) != 0) + +/* + * For console setup by EFI GOP, the base address is BAR1. + * For console setup by VBIOS, the base address is BAR2 + 16MB. + */ +#define NV_IS_CONSOLE_MAPPED(nv, addr) \ + (((addr) == (nv)->bars[NV_GPU_BAR_INDEX_FB].cpu_address) || \ + ((addr) == ((nv)->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000))) + +#define NV_SOC_IS_ISO_IOMMU_PRESENT(nv) \ + ((nv)->iommus.iso_iommu_present) + +#define NV_SOC_IS_NISO_IOMMU_PRESENT(nv) \ + ((nv)->iommus.niso_iommu_present) +/* + * GPU add/remove events + */ +#define NV_SYSTEM_GPU_ADD_EVENT 0x9001 +#define NV_SYSTEM_GPU_REMOVE_EVENT 0x9002 + +/* + * NVIDIA ACPI sub-event IDs (event types) to be passed into + * to core NVIDIA driver for ACPI events. + */ +#define NV_SYSTEM_ACPI_EVENT_VALUE_DISPLAY_SWITCH_DEFAULT 0 +#define NV_SYSTEM_ACPI_EVENT_VALUE_DOCK_EVENT_UNDOCKED 0 +#define NV_SYSTEM_ACPI_EVENT_VALUE_DOCK_EVENT_DOCKED 1 + +#define NV_ACPI_NVIF_HANDLE_PRESENT 0x01 +#define NV_ACPI_DSM_HANDLE_PRESENT 0x02 +#define NV_ACPI_WMMX_HANDLE_PRESENT 0x04 + +#define NV_EVAL_ACPI_METHOD_NVIF 0x01 +#define NV_EVAL_ACPI_METHOD_WMMX 0x02 + +/* + * Coherent GPU Memory Mode + */ +#define NV_COHERENT_GPU_MEM_MODE_NONE 0 +#define NV_COHERENT_GPU_MEM_MODE_NUMA 1 +#define NV_COHERENT_GPU_MEM_MODE_DRIVER 2 + +typedef enum { + NV_I2C_CMD_READ = 1, + NV_I2C_CMD_WRITE, + NV_I2C_CMD_SMBUS_READ, + NV_I2C_CMD_SMBUS_WRITE, + NV_I2C_CMD_SMBUS_QUICK_WRITE, + NV_I2C_CMD_SMBUS_QUICK_READ, + NV_I2C_CMD_SMBUS_BLOCK_READ, + NV_I2C_CMD_SMBUS_BLOCK_WRITE, + NV_I2C_CMD_BLOCK_READ, + NV_I2C_CMD_BLOCK_WRITE +} nv_i2c_cmd_t; + +// Flags needed by osAllocPagesNode / os_alloc_pages_node +#define NV_ALLOC_PAGES_NODE_NONE 0x0 +#define NV_ALLOC_PAGES_NODE_SKIP_RECLAIM 0x1 + +/* +** where we hide our nv_state_t * ... +*/ +#define NV_SET_NV_STATE(pgpu,p) ((pgpu)->pOsGpuInfo = (p)) +#define NV_GET_NV_STATE(pGpu) \ + (nv_state_t *)((pGpu) ? (pGpu)->pOsGpuInfo : NULL) + +static inline NvBool IS_REG_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length) +{ + return ((offset >= nv->regs->cpu_address) && + ((offset + (length - 1)) >= offset) && + ((offset + (length - 1)) <= (nv->regs->cpu_address + (nv->regs->size - 1)))); +} + +static inline NvBool IS_FB_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length) +{ + return ((nv->fb) && (nv->fb->size != 0) && + (offset >= nv->fb->cpu_address) && + ((offset + (length - 1)) >= offset) && + ((offset + (length - 1)) <= (nv->fb->cpu_address + (nv->fb->size - 1)))); +} + +static inline NvBool IS_UD_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length) +{ + return ((nv->ud.cpu_address != 0) && (nv->ud.size != 0) && + (offset >= nv->ud.cpu_address) && + ((offset + (length - 1)) >= offset) && + ((offset + (length - 1)) <= (nv->ud.cpu_address + (nv->ud.size - 1)))); +} + +static inline NvBool IS_IMEM_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length) +{ + return ((nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address != 0) && + (nv->bars[NV_GPU_BAR_INDEX_IMEM].size != 0) && + (offset >= nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address) && + ((offset + (length - 1)) >= offset) && + ((offset + (length - 1)) <= (nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + + (nv->bars[NV_GPU_BAR_INDEX_IMEM].size - 1)))); +} + +#define NV_RM_MAX_MSIX_LINES 8 + +#define NV_MAX_ISR_DELAY_US 20000 +#define NV_MAX_ISR_DELAY_MS (NV_MAX_ISR_DELAY_US / 1000) + +#define NV_TIMERCMP(a, b, CMP) \ + (((a)->tv_sec == (b)->tv_sec) ? \ + ((a)->tv_usec CMP (b)->tv_usec) : ((a)->tv_sec CMP (b)->tv_sec)) + +#define NV_TIMERADD(a, b, result) \ + { \ + (result)->tv_sec = (a)->tv_sec + (b)->tv_sec; \ + (result)->tv_usec = (a)->tv_usec + (b)->tv_usec; \ + if ((result)->tv_usec >= 1000000) \ + { \ + ++(result)->tv_sec; \ + (result)->tv_usec -= 1000000; \ + } \ + } + +#define NV_TIMERSUB(a, b, result) \ + { \ + (result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \ + (result)->tv_usec = (a)->tv_usec - (b)->tv_usec; \ + if ((result)->tv_usec < 0) \ + { \ + --(result)->tv_sec; \ + (result)->tv_usec += 1000000; \ + } \ + } + +#define NV_TIMEVAL_TO_US(tv) ((NvU64)(tv).tv_sec * 1000000 + (tv).tv_usec) + +#ifndef NV_ALIGN_UP +#define NV_ALIGN_UP(v,g) (((v) + ((g) - 1)) & ~((g) - 1)) +#endif +#ifndef NV_ALIGN_DOWN +#define NV_ALIGN_DOWN(v,g) ((v) & ~((g) - 1)) +#endif + + +/* + * driver internal interfaces + */ + +/* + * --------------------------------------------------------------------------- + * + * Function prototypes for UNIX specific OS interface. + * + * --------------------------------------------------------------------------- + */ + +NvU32 NV_API_CALL nv_get_dev_minor (nv_state_t *); +void* NV_API_CALL nv_alloc_kernel_mapping (nv_state_t *, void *, NvU64, NvU32, NvU64, void **); +void NV_API_CALL nv_free_kernel_mapping (nv_state_t *, void *, void *, void *); +NV_STATUS NV_API_CALL nv_alloc_user_mapping (nv_state_t *, void *, NvU64, NvU32, NvU64, NvU32, NvU64 *, void **); +void NV_API_CALL nv_free_user_mapping (nv_state_t *, void *, NvU64, void *); +NV_STATUS NV_API_CALL nv_add_mapping_context_to_file (nv_state_t *, nv_usermap_access_params_t*, NvU32, void *, NvU64, NvU32); + +NvU64 NV_API_CALL nv_get_kern_phys_address (NvU64); +NvU64 NV_API_CALL nv_get_user_phys_address (NvU64); +nv_state_t* NV_API_CALL nv_get_adapter_state (NvU32, NvU8, NvU8); +nv_state_t* NV_API_CALL nv_get_ctl_state (void); + +void NV_API_CALL nv_set_dma_address_size (nv_state_t *, NvU32 ); + +NV_STATUS NV_API_CALL nv_alias_pages (nv_state_t *, NvU32, NvU64, NvU32, NvU32, NvU64, NvU64 *, NvBool, void **); +NV_STATUS NV_API_CALL nv_alloc_pages (nv_state_t *, NvU32, NvU64, NvBool, NvU32, NvBool, NvBool, NvS32, NvU64 *, void **); +NV_STATUS NV_API_CALL nv_free_pages (nv_state_t *, NvU32, NvBool, NvU32, void *); + +NV_STATUS NV_API_CALL nv_register_user_pages (nv_state_t *, NvU64, NvU64 *, void *, void **, NvBool); +void NV_API_CALL nv_unregister_user_pages (nv_state_t *, NvU64, void **, void **); + +NV_STATUS NV_API_CALL nv_register_peer_io_mem (nv_state_t *, NvU64 *, NvU64, void **); +void NV_API_CALL nv_unregister_peer_io_mem(nv_state_t *, void *); + +struct sg_table; + +NV_STATUS NV_API_CALL nv_register_sgt (nv_state_t *, NvU64 *, NvU64, NvU32, void **, + struct sg_table *, void *, NvBool); +void NV_API_CALL nv_unregister_sgt (nv_state_t *, struct sg_table **, void **, void *); +NV_STATUS NV_API_CALL nv_register_phys_pages (nv_state_t *, NvU64 *, NvU64, NvU32, void **); +void NV_API_CALL nv_unregister_phys_pages (nv_state_t *, void *); + +NV_STATUS NV_API_CALL nv_dma_map_sgt (nv_dma_device_t *, NvU64, NvU64 *, NvU32, NvBool, void **); + +NV_STATUS NV_API_CALL nv_dma_map_alloc (nv_dma_device_t *, NvU64, NvU64 *, NvBool, NvBool, void **); +NV_STATUS NV_API_CALL nv_dma_unmap_alloc (nv_dma_device_t *, NvU64, NvU64 *, void **); + +NV_STATUS NV_API_CALL nv_dma_map_peer (nv_dma_device_t *, nv_dma_device_t *, NvU8, NvU64, NvU64 *); +void NV_API_CALL nv_dma_unmap_peer (nv_dma_device_t *, NvU64, NvU64); + +NV_STATUS NV_API_CALL nv_dma_map_mmio (nv_dma_device_t *, NvU64, NvU64 *); +void NV_API_CALL nv_dma_unmap_mmio (nv_dma_device_t *, NvU64, NvU64); + +void NV_API_CALL nv_dma_cache_invalidate (nv_dma_device_t *, void *); +void* NV_API_CALL nv_dma_get_dev_pagemap (NvU64); +void NV_API_CALL nv_dma_put_dev_pagemap (void *); +NvBool NV_API_CALL nv_grdma_pci_topology_supported(nv_state_t *, nv_dma_device_t *); + +NvS32 NV_API_CALL nv_start_rc_timer (nv_state_t *); +NvS32 NV_API_CALL nv_stop_rc_timer (nv_state_t *); + +void NV_API_CALL nv_post_event (nv_event_t *, NvHandle, NvU32, NvU32, NvU16, NvBool); +NvS32 NV_API_CALL nv_get_event (nv_file_private_t *, nv_event_t *, NvU32 *); + +void* NV_API_CALL nv_i2c_add_adapter (nv_state_t *, NvU32); +void NV_API_CALL nv_i2c_del_adapter (nv_state_t *, void *); + +void NV_API_CALL nv_acpi_methods_init (NvU32 *); +void NV_API_CALL nv_acpi_methods_uninit (void); + +NV_STATUS NV_API_CALL nv_acpi_method (NvU32, NvU32, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *); +NV_STATUS NV_API_CALL nv_acpi_d3cold_dsm_for_upstream_port (nv_state_t *, NvU8 *, NvU32, NvU32, NvU32 *); +NV_STATUS NV_API_CALL nv_acpi_dsm_method (nv_state_t *, NvU8 *, NvU32, NvBool, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *); +NV_STATUS NV_API_CALL nv_acpi_ddc_method (nv_state_t *, void *, NvU32 *, NvBool); +NV_STATUS NV_API_CALL nv_acpi_dod_method (nv_state_t *, NvU32 *, NvU32 *); +NV_STATUS NV_API_CALL nv_acpi_rom_method (nv_state_t *, NvU32 *, NvU32 *); +NV_STATUS NV_API_CALL nv_acpi_get_powersource (NvU32 *); +NvBool NV_API_CALL nv_acpi_is_battery_present(void); + +NV_STATUS NV_API_CALL nv_acpi_mux_method (nv_state_t *, NvU32 *, NvU32, const char *); + +NV_STATUS NV_API_CALL nv_log_error (nv_state_t *, NvU32, const char *, va_list); + +NvBool NV_API_CALL nv_requires_dma_remap (nv_state_t *); + +NvBool NV_API_CALL nv_is_rm_firmware_active(nv_state_t *); +const void*NV_API_CALL nv_get_firmware(nv_state_t *, nv_firmware_type_t, nv_firmware_chip_family_t, const void **, NvU32 *); +void NV_API_CALL nv_put_firmware(const void *); + +nv_file_private_t* NV_API_CALL nv_get_file_private(NvS32, NvBool, void **); +void NV_API_CALL nv_put_file_private(void *); + +NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvS32 *); +NV_STATUS NV_API_CALL nv_get_egm_info(nv_state_t *, NvU64 *, NvU64 *, NvS32 *); + +void NV_API_CALL nv_p2p_free_platform_data(void *data); + +NV_STATUS NV_API_CALL nv_revoke_gpu_mappings (nv_state_t *); +void NV_API_CALL nv_acquire_mmap_lock (nv_state_t *); +void NV_API_CALL nv_release_mmap_lock (nv_state_t *); +NvBool NV_API_CALL nv_get_all_mappings_revoked_locked (nv_state_t *); +void NV_API_CALL nv_set_safe_to_mmap_locked (nv_state_t *, NvBool); + +nv_alloc_mapping_list_node_t** NV_API_CALL nv_acquire_file_va(nv_file_private_t *, NvBool); +void NV_API_CALL nv_release_file_va(nv_file_private_t *, NvBool); + +NV_STATUS NV_API_CALL nv_indicate_idle (nv_state_t *); +NV_STATUS NV_API_CALL nv_indicate_not_idle (nv_state_t *); +void NV_API_CALL nv_idle_holdoff (nv_state_t *); + +NvBool NV_API_CALL nv_dynamic_power_available (nv_state_t *); +void NV_API_CALL nv_audio_dynamic_power (nv_state_t *); + +void NV_API_CALL nv_control_soc_irqs (nv_state_t *, NvBool bEnable); +NV_STATUS NV_API_CALL nv_get_current_irq_priv_data(nv_state_t *, NvU32 *); + +NV_STATUS NV_API_CALL nv_acquire_fabric_mgmt_cap (int, int*); +int NV_API_CALL nv_cap_drv_init(void); +void NV_API_CALL nv_cap_drv_exit(void); +NvBool NV_API_CALL nv_is_gpu_accessible(nv_state_t *); +NvBool NV_API_CALL nv_match_gpu_os_info(nv_state_t *, void *); + +void NV_API_CALL nv_get_updated_emu_seg(NvU32 *start, NvU32 *end); +void NV_API_CALL nv_get_screen_info(nv_state_t *, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU32 *, NvU64 *); +void NV_API_CALL nv_set_gpu_pg_mask(nv_state_t *); + +struct dma_buf; +typedef struct nv_dma_buf nv_dma_buf_t; +struct drm_gem_object; + +NV_STATUS NV_API_CALL nv_dma_import_sgt (nv_dma_device_t *, struct sg_table *, struct drm_gem_object *); +void NV_API_CALL nv_dma_release_sgt(struct sg_table *, struct drm_gem_object *); +NV_STATUS NV_API_CALL nv_dma_import_dma_buf (nv_dma_device_t *, struct dma_buf *, NvBool, NvU32 *, struct sg_table **, nv_dma_buf_t **); +NV_STATUS NV_API_CALL nv_dma_import_from_fd (nv_dma_device_t *, NvS32, NvBool, NvU32 *, struct sg_table **, nv_dma_buf_t **); +void NV_API_CALL nv_dma_release_dma_buf (nv_dma_buf_t *); + +void NV_API_CALL nv_schedule_uvm_isr (nv_state_t *); + +NV_STATUS NV_API_CALL nv_schedule_uvm_drain_p2p (NvU8 *); +void NV_API_CALL nv_schedule_uvm_resume_p2p (NvU8 *); + +NvBool NV_API_CALL nv_platform_supports_s0ix (void); +NvBool NV_API_CALL nv_s2idle_pm_configured (void); + +NvBool NV_API_CALL nv_pci_tegra_register_power_domain (nv_state_t *, NvBool); +NvBool NV_API_CALL nv_pci_tegra_pm_init (nv_state_t *); +void NV_API_CALL nv_pci_tegra_pm_deinit (nv_state_t *); + +NvBool NV_API_CALL nv_is_chassis_notebook (void); +void NV_API_CALL nv_allow_runtime_suspend (nv_state_t *nv); +void NV_API_CALL nv_disallow_runtime_suspend (nv_state_t *nv); + +typedef void (*nvTegraDceClientIpcCallback)(NvU32, NvU32, NvU32, void *, void *); + +NV_STATUS NV_API_CALL nv_get_num_phys_pages (void *, NvU32 *); +NV_STATUS NV_API_CALL nv_get_phys_pages (void *, void *, NvU32 *); +void NV_API_CALL nv_get_disp_smmu_stream_ids (nv_state_t *, NvU32 *, NvU32 *); + +typedef struct TEGRA_IMP_IMPORT_DATA TEGRA_IMP_IMPORT_DATA; +typedef struct nv_i2c_msg_s nv_i2c_msg_t; + +NV_STATUS NV_API_CALL nv_bpmp_send_mrq (nv_state_t *, NvU32, const void *, NvU32, void *, NvU32, NvS32 *, NvS32 *); +NV_STATUS NV_API_CALL nv_i2c_transfer(nv_state_t *, NvU32, NvU8, nv_i2c_msg_t *, int); +void NV_API_CALL nv_i2c_unregister_clients(nv_state_t *); +NV_STATUS NV_API_CALL nv_i2c_bus_status(nv_state_t *, NvU32, NvS32 *, NvS32 *); +NV_STATUS NV_API_CALL nv_imp_get_import_data (TEGRA_IMP_IMPORT_DATA *); +NV_STATUS NV_API_CALL nv_imp_get_uefi_data (nv_state_t *nv, NvU32 *iso_bw_kbps, NvU32 *floor_bw_kbps); +NV_STATUS NV_API_CALL nv_imp_enable_disable_rfl (nv_state_t *nv, NvBool bEnable); +NV_STATUS NV_API_CALL nv_imp_icc_set_bw (nv_state_t *nv, NvU32 avg_bw_kbps, NvU32 floor_bw_kbps); +NV_STATUS NV_API_CALL nv_get_num_dpaux_instances(nv_state_t *nv, NvU32 *num_instances); +NV_STATUS NV_API_CALL nv_get_tegra_brightness_level(nv_state_t *, NvU32 *); +NV_STATUS NV_API_CALL nv_set_tegra_brightness_level(nv_state_t *, NvU32); + +NV_STATUS NV_API_CALL nv_soc_device_reset (nv_state_t *); +NV_STATUS NV_API_CALL nv_soc_pm_powergate (nv_state_t *); +NV_STATUS NV_API_CALL nv_soc_pm_unpowergate (nv_state_t *); +NV_STATUS NV_API_CALL nv_gpio_get_pin_state(nv_state_t *, NvU32, NvU32 *); +void NV_API_CALL nv_gpio_set_pin_state(nv_state_t *, NvU32, NvU32); +NV_STATUS NV_API_CALL nv_gpio_set_pin_direction(nv_state_t *, NvU32, NvU32); +NV_STATUS NV_API_CALL nv_gpio_get_pin_direction(nv_state_t *, NvU32, NvU32 *); +NV_STATUS NV_API_CALL nv_gpio_get_pin_number(nv_state_t *, NvU32, NvU32 *); +NvBool NV_API_CALL nv_gpio_get_pin_interrupt_status(nv_state_t *, NvU32, NvU32); +NV_STATUS NV_API_CALL nv_gpio_set_pin_interrupt(nv_state_t *, NvU32, NvU32); +NvU32 NV_API_CALL nv_tegra_get_rm_interface_type(NvU32); +NV_STATUS NV_API_CALL nv_tegra_dce_register_ipc_client(NvU32, void *, nvTegraDceClientIpcCallback, NvU32 *); +NV_STATUS NV_API_CALL nv_tegra_dce_client_ipc_send_recv(NvU32, void *, NvU32); +NV_STATUS NV_API_CALL nv_tegra_dce_unregister_ipc_client(NvU32); +NV_STATUS NV_API_CALL nv_dsi_parse_panel_props(nv_state_t *, void *); +NvBool NV_API_CALL nv_dsi_is_panel_connected(nv_state_t *); +NV_STATUS NV_API_CALL nv_dsi_panel_enable(nv_state_t *, void *); +NV_STATUS NV_API_CALL nv_dsi_panel_reset(nv_state_t *, void *); +void NV_API_CALL nv_dsi_panel_disable(nv_state_t *, void *); +void NV_API_CALL nv_dsi_panel_cleanup(nv_state_t *, void *); +NV_STATUS NV_API_CALL nv_soc_mipi_cal_reset(nv_state_t *); +NvU32 NV_API_CALL nv_soc_fuse_register_read (NvU32 addr); +NvBool NV_API_CALL nv_get_hdcp_enabled(nv_state_t *nv); +NV_STATUS NV_API_CALL nv_get_valid_window_head_mask(nv_state_t *nv, NvU64 *); +NV_STATUS NV_API_CALL nv_dp_uphy_pll_init(nv_state_t *, NvU32, NvU32); +NV_STATUS NV_API_CALL nv_dp_uphy_pll_deinit(nv_state_t *); +NV_STATUS NV_API_CALL nv_soc_i2c_hsp_semaphore_acquire(NvU32 ownerId, NvBool bAcquire, NvU64 timeout); +typedef void (*nv_soc_tsec_cb_func_t)(void*, void*); +NvU32 NV_API_CALL nv_soc_tsec_send_cmd(void* cmd, nv_soc_tsec_cb_func_t cb_func, void* cb_context); +NvU32 NV_API_CALL nv_soc_tsec_event_register(nv_soc_tsec_cb_func_t cb_func, void* cb_context, NvBool is_init_event); +NvU32 NV_API_CALL nv_soc_tsec_event_unregister(NvBool is_init_event); +void* NV_API_CALL nv_soc_tsec_alloc_mem_desc(NvU32 num_bytes, NvU32 *flcn_addr); +void NV_API_CALL nv_soc_tsec_free_mem_desc(void *mem_desc); +NvBool NV_API_CALL nv_is_clk_enabled (nv_state_t *, TEGRASOC_WHICH_CLK); +NV_STATUS NV_API_CALL nv_set_parent (nv_state_t *, TEGRASOC_WHICH_CLK, TEGRASOC_WHICH_CLK); +NV_STATUS NV_API_CALL nv_get_parent (nv_state_t *, TEGRASOC_WHICH_CLK, TEGRASOC_WHICH_CLK*); +NV_STATUS NV_API_CALL nv_clk_get_handles (nv_state_t *); +void NV_API_CALL nv_clk_clear_handles (nv_state_t *); +NV_STATUS NV_API_CALL nv_enable_clk (nv_state_t *, TEGRASOC_WHICH_CLK); +void NV_API_CALL nv_disable_clk (nv_state_t *, TEGRASOC_WHICH_CLK); +NV_STATUS NV_API_CALL nv_get_curr_freq (nv_state_t *, TEGRASOC_WHICH_CLK, NvU32 *); +NV_STATUS NV_API_CALL nv_get_max_freq (nv_state_t *, TEGRASOC_WHICH_CLK, NvU32 *); +NV_STATUS NV_API_CALL nv_get_min_freq (nv_state_t *, TEGRASOC_WHICH_CLK, NvU32 *); +NV_STATUS NV_API_CALL nv_set_freq (nv_state_t *, TEGRASOC_WHICH_CLK, NvU32); + +void NV_API_CALL nv_pci_cxl_set_caching(nv_state_t *, NvBool); + +/* + * --------------------------------------------------------------------------- + * + * Function prototypes for Resource Manager interface. + * + * --------------------------------------------------------------------------- + */ + +NvBool NV_API_CALL rm_init_rm (nvidia_stack_t *); +void NV_API_CALL rm_shutdown_rm (nvidia_stack_t *); +NvBool NV_API_CALL rm_init_private_state (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_free_private_state (nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_init_adapter (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_disable_adapter (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_shutdown_adapter (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_exclude_adapter (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_acquire_api_lock (nvidia_stack_t *); +NV_STATUS NV_API_CALL rm_release_api_lock (nvidia_stack_t *); +NV_STATUS NV_API_CALL rm_acquire_gpu_lock (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_release_gpu_lock (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_acquire_all_gpus_lock (nvidia_stack_t *); +NV_STATUS NV_API_CALL rm_release_all_gpus_lock (nvidia_stack_t *); +NV_STATUS NV_API_CALL rm_ioctl (nvidia_stack_t *, nv_state_t *, nv_file_private_t *, NvU32, void *, NvU32); +NV_STATUS NV_API_CALL rm_validate_ioctls (NvU32, NvU32); +NvBool NV_API_CALL rm_isr (nvidia_stack_t *, nv_state_t *, NvU32 *); +void NV_API_CALL rm_isr_bh (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_isr_bh_unlocked (nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_is_msix_allowed (nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_wait_for_bar_firewall (nvidia_stack_t *, NvU32 domain, NvU8 bus, NvU8 device, NvU8 function, NvU16 devId, NvU16 subsystemId); +NV_STATUS NV_API_CALL rm_pmu_perfmon_get_load (nvidia_stack_t *, nv_state_t *, NvU32 *, TEGRASOC_DEVFREQ_CLK); +NV_STATUS NV_API_CALL rm_power_management (nvidia_stack_t *, nv_state_t *, nv_pm_action_t); +NV_STATUS NV_API_CALL rm_stop_user_channels (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_restart_user_channels (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_save_low_res_mode (nvidia_stack_t *, nv_state_t *); +char* NV_API_CALL rm_get_gpu_uuid (nvidia_stack_t *, nv_state_t *); +const NvU8* NV_API_CALL rm_get_gpu_uuid_raw (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_set_rm_firmware_requested(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_cleanup_file_private (nvidia_stack_t *, nv_state_t *, nv_file_private_t *); +void NV_API_CALL rm_unbind_lock (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_read_registry_dword (nvidia_stack_t *, nv_state_t *, const char *, NvU32 *); +NV_STATUS NV_API_CALL rm_write_registry_dword (nvidia_stack_t *, nv_state_t *, const char *, NvU32); +NV_STATUS NV_API_CALL rm_write_registry_binary (nvidia_stack_t *, nv_state_t *, const char *, NvU8 *, NvU32); +NV_STATUS NV_API_CALL rm_write_registry_string (nvidia_stack_t *, nv_state_t *, const char *, const char *, NvU32); +void NV_API_CALL rm_parse_option_string (nvidia_stack_t *, const char *); +char* NV_API_CALL rm_remove_spaces (const char *); +char* NV_API_CALL rm_string_token (char **, const char); +void NV_API_CALL rm_vgpu_vfio_set_driver_vm(nvidia_stack_t *, NvBool); +NV_STATUS NV_API_CALL rm_get_adapter_status_external(nvidia_stack_t *, nv_state_t *); + +NV_STATUS NV_API_CALL rm_run_rc_callback (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_execute_work_item (nvidia_stack_t *, void *); +const char* NV_API_CALL rm_get_device_name (NvU16, NvU16, NvU16); + +NV_STATUS NV_API_CALL rm_is_supported_device (nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_is_supported_pci_device(NvU8 pci_class, + NvU8 pci_subclass, + NvU16 vendor, + NvU16 device, + NvU16 subsystem_vendor, + NvU16 subsystem_device, + NvBool print_legacy_warning); + +void NV_API_CALL rm_i2c_remove_adapters (nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_i2c_is_smbus_capable (nvidia_stack_t *, nv_state_t *, void *); +NV_STATUS NV_API_CALL rm_i2c_transfer (nvidia_stack_t *, nv_state_t *, void *, nv_i2c_cmd_t, NvU8, NvU8, NvU32, NvU8 *); + +NV_STATUS NV_API_CALL rm_perform_version_check (nvidia_stack_t *, void *, NvU32); + +void NV_API_CALL rm_power_source_change_event (nvidia_stack_t *, NvU32); + +void NV_API_CALL rm_request_dnotifier_state (nvidia_stack_t *, nv_state_t *); + +void NV_API_CALL rm_disable_gpu_state_persistence (nvidia_stack_t *sp, nv_state_t *); +NV_STATUS NV_API_CALL rm_p2p_init_mapping (nvidia_stack_t *, NvU64, NvU64 *, NvU64 *, NvU64 *, NvU64 *, NvU64, NvU64, NvU64, NvU64, void (*)(void *), void *); +NV_STATUS NV_API_CALL rm_p2p_destroy_mapping (nvidia_stack_t *, NvU64); +NV_STATUS NV_API_CALL rm_p2p_get_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, NvU64, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU8 **, void *, NvBool *); +NV_STATUS NV_API_CALL rm_p2p_get_gpu_info (nvidia_stack_t *, NvU64, NvU64, NvU8 **, void **); +NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent (nvidia_stack_t *, NvU64, NvU64, void **, NvU64 *, NvU32 *, NvBool, void *, void *, void **, NvBool *); +NV_STATUS NV_API_CALL rm_p2p_register_callback (nvidia_stack_t *, NvU64, NvU64, NvU64, void *, void (*)(void *), void *); +NV_STATUS NV_API_CALL rm_p2p_put_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, void *); +NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *, void *); +NV_STATUS NV_API_CALL rm_p2p_dma_map_pages (nvidia_stack_t *, nv_dma_device_t *, NvU8 *, NvU64, NvU32, NvU64 *, void **); +NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, + NvHandle, void *, NvHandle, NvU64, NvU64, NvU8, NvHandle *, void **, + NvBool *, NvU32 *, NvBool *, nv_memory_type_t *); +void NV_API_CALL rm_dma_buf_undup_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle); +NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle (nvidia_stack_t *, nv_state_t *, + NvHandle, NvHandle, MemoryRange, + NvU8, void *, NvBool, MemoryArea *); +void NV_API_CALL rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t *, + NvHandle, NvHandle, NvU8, void *, + NvBool, MemoryArea); +NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(nvidia_stack_t *, + nv_state_t *, NvHandle, NvHandle, + NvU8, NvHandle *, NvHandle *, + NvHandle *, void **, NvBool *, NvBool *); +void NV_API_CALL rm_dma_buf_put_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, void *); + +void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd); +NvBool NV_API_CALL rm_get_device_remove_flag(nvidia_stack_t *sp, NvU32 gpu_id); +NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults(nvidia_stack_t *, nv_state_t *, NvU32 *); +NV_STATUS NV_API_CALL rm_gpu_handle_mmu_faults(nvidia_stack_t *, nv_state_t *, NvU32 *); +NvBool NV_API_CALL rm_is_chipset_io_coherent(nv_stack_t *); +NvBool NV_API_CALL rm_init_event_locks(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_destroy_event_locks(nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_get_gpu_numa_info(nvidia_stack_t *, nv_state_t *, nv_ioctl_numa_info_t *); +NV_STATUS NV_API_CALL rm_gpu_numa_online(nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_gpu_numa_offline(nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_is_device_sequestered(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_check_for_gpu_surprise_removal(nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_set_external_kernel_client_count(nvidia_stack_t *, nv_state_t *, NvBool); +NV_STATUS NV_API_CALL rm_schedule_gpu_wakeup(nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_disable_iomap_wc(void); + +void NV_API_CALL rm_init_tegra_dynamic_power_management(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_init_dynamic_power_management(nvidia_stack_t *, nv_state_t *, NvBool); +void NV_API_CALL rm_cleanup_dynamic_power_management(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_enable_dynamic_power_management(nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_ref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t); +void NV_API_CALL rm_unref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t); +NV_STATUS NV_API_CALL rm_transition_dynamic_power(nvidia_stack_t *, nv_state_t *, NvBool, NvBool *); +void NV_API_CALL rm_get_power_info(nvidia_stack_t *, nv_state_t *, nv_power_info_t *); + +void NV_API_CALL rm_acpi_notify(nvidia_stack_t *, nv_state_t *, NvU32); +void NV_API_CALL rm_acpi_nvpcf_notify(nvidia_stack_t *); + +NvBool NV_API_CALL rm_is_altstack_in_use(void); + +void NV_API_CALL rm_notify_gpu_addition(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_notify_gpu_removal(nvidia_stack_t *, nv_state_t *); + +/* vGPU VFIO specific functions */ +NV_STATUS NV_API_CALL nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *, + NvU32 *, NvU32 *, NvU32); +NV_STATUS NV_API_CALL nv_vgpu_delete(nvidia_stack_t *, const NvU8 *, NvU16); +NV_STATUS NV_API_CALL nv_vgpu_get_type_ids(nvidia_stack_t *, nv_state_t *, NvU32 *, NvU32 *, NvBool, NvU8, NvBool); +NV_STATUS NV_API_CALL nv_vgpu_get_type_info(nvidia_stack_t *, nv_state_t *, NvU32, char *, int, NvU8); +NV_STATUS NV_API_CALL nv_vgpu_get_bar_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU64 *, + NvU64 *, NvU64 *, NvU32 *, NvBool *, NvU8 *); +NV_STATUS NV_API_CALL nv_vgpu_update_sysfs_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU32 *); +NV_STATUS NV_API_CALL nv_vgpu_get_hbm_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU64 *, NvU64 *); +NV_STATUS NV_API_CALL nv_vgpu_process_vf_info(nvidia_stack_t *, nv_state_t *, NvU8, NvU32, NvU8, NvU8, NvU8, NvBool, void *); +NV_STATUS NV_API_CALL nv_gpu_bind_event(nvidia_stack_t *, NvU32, NvBool *); +NV_STATUS NV_API_CALL nv_gpu_unbind_event(nvidia_stack_t *, NvU32, NvBool *); + +nv_soc_irq_type_t NV_API_CALL nv_get_current_irq_type(nv_state_t*); +void NV_API_CALL nv_flush_coherent_cpu_cache_range(nv_state_t *nv, NvU64 cpu_virtual, NvU64 size); + +/* Callbacks should occur roughly every 10ms. */ +#define NV_SNAPSHOT_TIMER_HZ 100 +void NV_API_CALL nv_start_snapshot_timer(void (*snapshot_callback)(void *context)); +void NV_API_CALL nv_flush_snapshot_timer(void); +void NV_API_CALL nv_stop_snapshot_timer(void); + +static inline const NvU8 *nv_get_cached_uuid(nv_state_t *nv) +{ + return nv->nv_uuid_cache.valid ? nv->nv_uuid_cache.uuid : NULL; +} + +/* nano second resolution timer callback structure */ +typedef struct nv_nano_timer nv_nano_timer_t; + +/* nano timer functions */ +void NV_API_CALL nv_create_nano_timer(nv_state_t *, void *pTmrEvent, nv_nano_timer_t **); +void NV_API_CALL nv_start_nano_timer(nv_state_t *nv, nv_nano_timer_t *, NvU64 timens); +NV_STATUS NV_API_CALL rm_run_nano_timer_callback(nvidia_stack_t *, nv_state_t *, void *pTmrEvent); +void NV_API_CALL nv_cancel_nano_timer(nv_state_t *, nv_nano_timer_t *); +void NV_API_CALL nv_destroy_nano_timer(nv_state_t *nv, nv_nano_timer_t *); + +// Host1x specific functions. +NV_STATUS NV_API_CALL nv_get_syncpoint_aperture(NvU32, NvU64 *, NvU64 *, NvU32 *); + +#if defined(NVCPU_X86_64) + +static inline NvU64 nv_rdtsc(void) +{ + NvU64 val; + __asm__ __volatile__ ("rdtsc \t\n" + "shlq $0x20,%%rdx \t\n" + "orq %%rdx,%%rax \t\n" + : "=A" (val)); + return val; +} + +#endif + +static inline NvBool nv_dev_needs_vidmem_preservation(const nv_state_t *nv) +{ + /* + * Tegra iGPUs and SoC display devices don't need vidmem preservation. + */ + return !nv->is_tegra_pci_igpu && !NV_IS_SOC_DISPLAY_DEVICE(nv); +} + +#endif /* NVRM */ + +static inline int nv_count_bits(NvU64 word) +{ + NvU64 bits; + + bits = (word & 0x5555555555555555ULL) + ((word >> 1) & 0x5555555555555555ULL); + bits = (bits & 0x3333333333333333ULL) + ((bits >> 2) & 0x3333333333333333ULL); + bits = (bits & 0x0f0f0f0f0f0f0f0fULL) + ((bits >> 4) & 0x0f0f0f0f0f0f0f0fULL); + bits = (bits & 0x00ff00ff00ff00ffULL) + ((bits >> 8) & 0x00ff00ff00ff00ffULL); + bits = (bits & 0x0000ffff0000ffffULL) + ((bits >> 16) & 0x0000ffff0000ffffULL); + bits = (bits & 0x00000000ffffffffULL) + ((bits >> 32) & 0x00000000ffffffffULL); + + return (int)(bits); +} + +#endif diff --git a/userspace/sdks/nv570/nvalloc/unix/include/nv_escape.h b/userspace/sdks/nv570/nvalloc/unix/include/nv_escape.h new file mode 100644 index 0000000..601d9de --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/nv_escape.h @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NV_ESCAPE_H_INCLUDED +#define NV_ESCAPE_H_INCLUDED + +/* + * NOTE: Any new ioctls added here should also be added to RmValidateIoctl() in escape.c + * or rm_validate_ioctls() in osapi.c + */ +#define NV_ESC_RM_ALLOC_MEMORY 0x27 +#define NV_ESC_RM_ALLOC_OBJECT 0x28 +#define NV_ESC_RM_FREE 0x29 +#define NV_ESC_RM_CONTROL 0x2A +#define NV_ESC_RM_ALLOC 0x2B +#define NV_ESC_RM_DUP_OBJECT 0x34 +#define NV_ESC_RM_SHARE 0x35 +#define NV_ESC_RM_I2C_ACCESS 0x39 +#define NV_ESC_RM_IDLE_CHANNELS 0x41 +#define NV_ESC_RM_VID_HEAP_CONTROL 0x4A +#define NV_ESC_RM_ACCESS_REGISTRY 0x4D +#define NV_ESC_RM_MAP_MEMORY 0x4E +#define NV_ESC_RM_UNMAP_MEMORY 0x4F +#define NV_ESC_RM_GET_EVENT_DATA 0x52 +#define NV_ESC_RM_ALLOC_CONTEXT_DMA2 0x54 +#define NV_ESC_RM_ADD_VBLANK_CALLBACK 0x56 +#define NV_ESC_RM_MAP_MEMORY_DMA 0x57 +#define NV_ESC_RM_UNMAP_MEMORY_DMA 0x58 +#define NV_ESC_RM_BIND_CONTEXT_DMA 0x59 +#define NV_ESC_RM_EXPORT_OBJECT_TO_FD 0x5C +#define NV_ESC_RM_IMPORT_OBJECT_FROM_FD 0x5D +#define NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO 0x5E +#define NV_ESC_RM_LOCKLESS_DIAGNOSTIC 0x5F + +#endif // NV_ESCAPE_H_INCLUDED diff --git a/userspace/sdks/nv570/nvalloc/unix/include/os-interface.h b/userspace/sdks/nv570/nvalloc/unix/include/os-interface.h new file mode 100644 index 0000000..2a80c50 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/os-interface.h @@ -0,0 +1,298 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +/* + * Os interface definitions needed by os-interface.c + */ + +#ifndef OS_INTERFACE_H +#define OS_INTERFACE_H + +/******************* Operating System Interface Routines *******************\ +* * +* Operating system wrapper functions used to abstract the OS. * +* * +\***************************************************************************/ + +#include <nvtypes.h> +#include <nvstatus.h> +#include "nv_stdarg.h" +#include <nv-kernel-interface-api.h> +#include <os/nv_memory_type.h> +#include <os/nv_memory_area.h> +#include <nv-caps.h> + +#include "rs_access.h" + + + +typedef struct +{ + NvU32 os_major_version; + NvU32 os_minor_version; + NvU32 os_build_number; + const char * os_build_version_str; + const char * os_build_date_plus_str; +}os_version_info; + +/* Each OS defines its own version of this opaque type */ +struct os_work_queue; + +/* Each OS defines its own version of this opaque type */ +typedef struct os_wait_queue os_wait_queue; + +/* Flags needed by os_get_current_proccess_flags */ +#define OS_CURRENT_PROCESS_FLAG_NONE 0x0 +#define OS_CURRENT_PROCESS_FLAG_KERNEL_THREAD 0x1 +#define OS_CURRENT_PROCESS_FLAG_EXITING 0x2 + +/* + * --------------------------------------------------------------------------- + * + * Function prototypes for OS interface. + * + * --------------------------------------------------------------------------- + */ + +NV_STATUS NV_API_CALL os_alloc_mem (void **, NvU64); +void NV_API_CALL os_free_mem (void *); +NV_STATUS NV_API_CALL os_get_system_time (NvU32 *, NvU32 *); +NvU64 NV_API_CALL os_get_monotonic_time_ns (void); +NvU64 NV_API_CALL os_get_monotonic_time_ns_hr (void); +NvU64 NV_API_CALL os_get_monotonic_tick_resolution_ns (void); +NV_STATUS NV_API_CALL os_delay (NvU32); +NV_STATUS NV_API_CALL os_delay_us (NvU32); +NvU64 NV_API_CALL os_get_cpu_frequency (void); +NvU32 NV_API_CALL os_get_current_process (void); +void NV_API_CALL os_get_current_process_name (char *, NvU32); +NV_STATUS NV_API_CALL os_get_current_thread (NvU64 *); +char* NV_API_CALL os_string_copy (char *, const char *); +NvU32 NV_API_CALL os_string_length (const char *); +NvU32 NV_API_CALL os_strtoul (const char *, char **, NvU32); +NvS32 NV_API_CALL os_string_compare (const char *, const char *); +NvS32 NV_API_CALL os_snprintf (char *, NvU32, const char *, ...); +NvS32 NV_API_CALL os_vsnprintf (char *, NvU32, const char *, va_list); +void NV_API_CALL os_log_error (const char *, va_list); +void* NV_API_CALL os_mem_copy (void *, const void *, NvU32); +NV_STATUS NV_API_CALL os_memcpy_from_user (void *, const void *, NvU32); +NV_STATUS NV_API_CALL os_memcpy_to_user (void *, const void *, NvU32); +void* NV_API_CALL os_mem_set (void *, NvU8, NvU32); +NvS32 NV_API_CALL os_mem_cmp (const NvU8 *, const NvU8 *, NvU32); +void* NV_API_CALL os_pci_init_handle (NvU32, NvU8, NvU8, NvU8, NvU16 *, NvU16 *); +NV_STATUS NV_API_CALL os_pci_read_byte (void *, NvU32, NvU8 *); +NV_STATUS NV_API_CALL os_pci_read_word (void *, NvU32, NvU16 *); +NV_STATUS NV_API_CALL os_pci_read_dword (void *, NvU32, NvU32 *); +NV_STATUS NV_API_CALL os_pci_write_byte (void *, NvU32, NvU8); +NV_STATUS NV_API_CALL os_pci_write_word (void *, NvU32, NvU16); +NV_STATUS NV_API_CALL os_pci_write_dword (void *, NvU32, NvU32); +NvBool NV_API_CALL os_pci_remove_supported (void); +void NV_API_CALL os_pci_remove (void *); +void* NV_API_CALL os_map_kernel_space (NvU64, NvU64, NvU32); +void NV_API_CALL os_unmap_kernel_space (void *, NvU64); +NV_STATUS NV_API_CALL os_flush_cpu_cache_all (void); +NV_STATUS NV_API_CALL os_flush_user_cache (void); +void NV_API_CALL os_flush_cpu_write_combine_buffer(void); +NvU8 NV_API_CALL os_io_read_byte (NvU32); +NvU16 NV_API_CALL os_io_read_word (NvU32); +NvU32 NV_API_CALL os_io_read_dword (NvU32); +void NV_API_CALL os_io_write_byte (NvU32, NvU8); +void NV_API_CALL os_io_write_word (NvU32, NvU16); +void NV_API_CALL os_io_write_dword (NvU32, NvU32); +NvBool NV_API_CALL os_is_administrator (void); +NvBool NV_API_CALL os_check_access (RsAccessRight accessRight); +void NV_API_CALL os_dbg_init (void); +void NV_API_CALL os_dbg_breakpoint (void); +void NV_API_CALL os_dbg_set_level (NvU32); +NvU32 NV_API_CALL os_get_cpu_count (void); +NvU32 NV_API_CALL os_get_cpu_number (void); +void NV_API_CALL os_disable_console_access (void); +void NV_API_CALL os_enable_console_access (void); +NV_STATUS NV_API_CALL os_registry_init (void); +NvU64 NV_API_CALL os_get_max_user_va (void); +NV_STATUS NV_API_CALL os_schedule (void); +NV_STATUS NV_API_CALL os_alloc_spinlock (void **); +void NV_API_CALL os_free_spinlock (void *); +NvU64 NV_API_CALL os_acquire_spinlock (void *); +void NV_API_CALL os_release_spinlock (void *, NvU64); +NV_STATUS NV_API_CALL os_queue_work_item (struct os_work_queue *, void *); +NV_STATUS NV_API_CALL os_flush_work_queue (struct os_work_queue *, NvBool); +NvBool NV_API_CALL os_is_queue_flush_ongoing (struct os_work_queue *); +NV_STATUS NV_API_CALL os_alloc_mutex (void **); +void NV_API_CALL os_free_mutex (void *); +NV_STATUS NV_API_CALL os_acquire_mutex (void *); +NV_STATUS NV_API_CALL os_cond_acquire_mutex (void *); +void NV_API_CALL os_release_mutex (void *); +void* NV_API_CALL os_alloc_semaphore (NvU32); +void NV_API_CALL os_free_semaphore (void *); +NV_STATUS NV_API_CALL os_acquire_semaphore (void *); +NV_STATUS NV_API_CALL os_cond_acquire_semaphore (void *); +NV_STATUS NV_API_CALL os_release_semaphore (void *); +void* NV_API_CALL os_alloc_rwlock (void); +void NV_API_CALL os_free_rwlock (void *); +NV_STATUS NV_API_CALL os_acquire_rwlock_read (void *); +NV_STATUS NV_API_CALL os_acquire_rwlock_write (void *); +NV_STATUS NV_API_CALL os_cond_acquire_rwlock_read (void *); +NV_STATUS NV_API_CALL os_cond_acquire_rwlock_write (void *); +void NV_API_CALL os_release_rwlock_read (void *); +void NV_API_CALL os_release_rwlock_write (void *); +NvBool NV_API_CALL os_semaphore_may_sleep (void); +NV_STATUS NV_API_CALL os_get_version_info (os_version_info*); +NV_STATUS NV_API_CALL os_get_is_openrm (NvBool *); +NvBool NV_API_CALL os_is_bif_reset_supported (void *); +NvBool NV_API_CALL os_is_isr (void); +void NV_API_CALL os_dump_stack (void); +NvBool NV_API_CALL os_is_efi_enabled (void); +NvBool NV_API_CALL os_is_xen_dom0 (void); +NvBool NV_API_CALL os_is_vgx_hyper (void); +NV_STATUS NV_API_CALL os_inject_vgx_msi (NvU16, NvU64, NvU32); +NvBool NV_API_CALL os_is_grid_supported (void); +NvU32 NV_API_CALL os_get_grid_csp_support (void); +void NV_API_CALL os_bug_check (NvU32, const char *); +NV_STATUS NV_API_CALL os_lock_user_pages (void *, NvU64, void **, NvU32); +NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **); +NV_STATUS NV_API_CALL os_unlock_user_pages (NvU64, void *, NvU32); +NV_STATUS NV_API_CALL os_match_mmap_offset (void *, NvU64, NvU64 *); +NV_STATUS NV_API_CALL os_get_euid (NvU32 *); +NV_STATUS NV_API_CALL os_get_smbios_header (NvU64 *pSmbsAddr); +NV_STATUS NV_API_CALL os_get_acpi_rsdp_from_uefi (NvU32 *); +void NV_API_CALL os_add_record_for_crashLog (void *, NvU32); +void NV_API_CALL os_delete_record_for_crashLog (void *); +NV_STATUS NV_API_CALL os_call_vgpu_vfio (void *, NvU32); +NV_STATUS NV_API_CALL os_device_vm_present (void); +NV_STATUS NV_API_CALL os_numa_memblock_size (NvU64 *); +NV_STATUS NV_API_CALL os_alloc_pages_node (NvS32, NvU32, NvU32, NvU64 *); +NV_STATUS NV_API_CALL os_get_page (NvU64 address); +NV_STATUS NV_API_CALL os_put_page (NvU64 address); +NvU32 NV_API_CALL os_get_page_refcount (NvU64 address); +NvU32 NV_API_CALL os_count_tail_pages (NvU64 address); +void NV_API_CALL os_free_pages_phys (NvU64, NvU32); +NV_STATUS NV_API_CALL os_open_temporary_file (void **); +void NV_API_CALL os_close_file (void *); +NV_STATUS NV_API_CALL os_write_file (void *, NvU8 *, NvU64, NvU64); +NV_STATUS NV_API_CALL os_read_file (void *, NvU8 *, NvU64, NvU64); +NV_STATUS NV_API_CALL os_open_readonly_file (const char *, void **); +NV_STATUS NV_API_CALL os_open_and_read_file (const char *, NvU8 *, NvU64); +NvBool NV_API_CALL os_is_nvswitch_present (void); +NV_STATUS NV_API_CALL os_get_random_bytes (NvU8 *, NvU16); +NvU32 NV_API_CALL os_get_current_process_flags (void); +NV_STATUS NV_API_CALL os_alloc_wait_queue (os_wait_queue **); +void NV_API_CALL os_free_wait_queue (os_wait_queue *); +void NV_API_CALL os_wait_uninterruptible (os_wait_queue *); +void NV_API_CALL os_wait_interruptible (os_wait_queue *); +void NV_API_CALL os_wake_up (os_wait_queue *); +nv_cap_t* NV_API_CALL os_nv_cap_init (const char *); +nv_cap_t* NV_API_CALL os_nv_cap_create_dir_entry (nv_cap_t *, const char *, int); +nv_cap_t* NV_API_CALL os_nv_cap_create_file_entry (nv_cap_t *, const char *, int); +void NV_API_CALL os_nv_cap_destroy_entry (nv_cap_t *); +int NV_API_CALL os_nv_cap_validate_and_dup_fd (const nv_cap_t *, int); +void NV_API_CALL os_nv_cap_close_fd (int); +NvS32 NV_API_CALL os_imex_channel_get (NvU64); +NvS32 NV_API_CALL os_imex_channel_count (void); +NV_STATUS NV_API_CALL os_tegra_igpu_perf_boost (void *, NvBool, NvU32); + +NV_STATUS NV_API_CALL os_get_tegra_platform (NvU32 *); +enum os_pci_req_atomics_type { + OS_INTF_PCIE_REQ_ATOMICS_32BIT, + OS_INTF_PCIE_REQ_ATOMICS_64BIT, + OS_INTF_PCIE_REQ_ATOMICS_128BIT +}; +NV_STATUS NV_API_CALL os_enable_pci_req_atomics (void *, enum os_pci_req_atomics_type); +void NV_API_CALL os_pci_trigger_flr(void *handle); +NV_STATUS NV_API_CALL os_get_numa_node_memory_usage (NvS32, NvU64 *, NvU64 *); +NV_STATUS NV_API_CALL os_numa_add_gpu_memory (void *, NvU64, NvU64, NvU32 *); +NV_STATUS NV_API_CALL os_numa_remove_gpu_memory (void *, NvU64, NvU64, NvU32); +NV_STATUS NV_API_CALL os_offline_page_at_address(NvU64 address); +void* NV_API_CALL os_get_pid_info(void); +void NV_API_CALL os_put_pid_info(void *pid_info); +NV_STATUS NV_API_CALL os_find_ns_pid(void *pid_info, NvU32 *ns_pid); +NvBool NV_API_CALL os_is_init_ns(void); +NV_STATUS NV_API_CALL os_iommu_sva_bind(void *arg, void **handle, NvU32 *pasid); +void NV_API_CALL os_iommu_sva_unbind(void *handle); +NvBool NV_API_CALL os_supports_kernel_suspend_notifiers(void); + +extern NvU64 os_page_size; +extern NvU64 os_max_page_size; +extern NvU64 os_page_mask; +extern NvU8 os_page_shift; +extern NvBool os_cc_enabled; +extern NvBool os_cc_sev_snp_enabled; +extern NvBool os_cc_sme_enabled; +extern NvBool os_cc_snp_vtom_enabled; +extern NvBool os_cc_tdx_enabled; +extern NvBool os_dma_buf_enabled; +extern NvBool os_imex_channel_is_supported; + +/* + * --------------------------------------------------------------------------- + * + * Debug macros. + * + * --------------------------------------------------------------------------- + */ + +#define NV_DBG_INFO 0x0 +#define NV_DBG_SETUP 0x1 +#define NV_DBG_USERERRORS 0x2 +#define NV_DBG_WARNINGS 0x3 +#define NV_DBG_ERRORS 0x4 + + +void NV_API_CALL out_string(const char *str); +int NV_API_CALL nv_printf(NvU32 debuglevel, const char *printf_format, ...); + +#define NV_DEV_PRINTF(debuglevel, nv, format, ... ) \ + nv_printf(debuglevel, "NVRM: GPU " NV_PCI_DEV_FMT ": " format, NV_PCI_DEV_FMT_ARGS(nv), ## __VA_ARGS__) + +#define NV_DEV_PRINTF_STATUS(debuglevel, nv, status, format, ... ) \ + nv_printf(debuglevel, "NVRM: GPU " NV_PCI_DEV_FMT ": " format " (0x%x)\n", NV_PCI_DEV_FMT_ARGS(nv), ## __VA_ARGS__, status) + +/* + * Fields for os_lock_user_pages flags parameter + */ +#define NV_LOCK_USER_PAGES_FLAGS_WRITE 0:0 +#define NV_LOCK_USER_PAGES_FLAGS_WRITE_NO 0x00000000 +#define NV_LOCK_USER_PAGES_FLAGS_WRITE_YES 0x00000001 + +// NV OS Tegra platform type defines +#define NV_OS_TEGRA_PLATFORM_SIM 0 +#define NV_OS_TEGRA_PLATFORM_FPGA 1 +#define NV_OS_TEGRA_PLATFORM_SILICON 2 + +/* + * Plumbing through cgroup dmem api + */ +#define OS_CGROUP_IMPL_NONE 0 +#define OS_CGROUP_IMPL_MISC 1 +#define OS_CGROUP_IMPL_DMEM 2 +NvU32 NV_API_CALL os_cgroup_implementation(void); +void* NV_API_CALL os_dmem_cgroup_register_region(NvU64 size, const char *name); +void NV_API_CALL os_dmem_cgroup_unregister_region(void *region); +NV_STATUS NV_API_CALL os_dmem_cgroup_try_charge(void *region, NvU64 size, void **ret_pool, void **ret_limit_pool); +void NV_API_CALL os_dmem_cgroup_uncharge(void *pool, NvU64 size); + +void* NV_API_CALL os_cgroup_for_pid(int pid, void *pidInfo); +void* NV_API_CALL os_cgroup_get_from_fd(NvU32 fd); +void NV_API_CALL os_cgroup_put(void *cgroup); + +#endif /* OS_INTERFACE_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/os_custom.h b/userspace/sdks/nv570/nvalloc/unix/include/os_custom.h new file mode 100644 index 0000000..1effaf7 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/os_custom.h @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _OS_CUSTOM_H_ +#define _OS_CUSTOM_H_ + +/*! + * @file os_custom.h + * @brief OS module specific definitions for this OS + */ + +#include <os-interface.h> + +// File modes, added for NVIDIA capabilities. +#define OS_RUSR 00400 // read permission, owner +#define OS_WUSR 00200 // write permission, owner +#define OS_XUSR 00100 // execute/search permission, owner +#define OS_RWXU (OS_RUSR | OS_WUSR | OS_XUSR) // read, write, execute/search, owner +#define OS_RGRP 00040 // read permission, group +#define OS_WGRP 00020 // write permission, group +#define OS_XGRP 00010 // execute/search permission, group +#define OS_RWXG (OS_RGRP | OS_WGRP | OS_XGRP) // read, write, execute/search, group +#define OS_ROTH 00004 // read permission, other +#define OS_WOTH 00002 // write permission, other +#define OS_XOTH 00001 // execute/search permission, other +#define OS_RWXO (OS_ROTH | OS_WOTH | OS_XOTH) // read, write, execute/search, other +#define OS_RUGO (OS_RUSR | OS_RGRP | OS_ROTH) +#define OS_WUGO (OS_WUSR | OS_WGRP | OS_WOTH) +#define OS_XUGO (OS_XUSR | OS_XGRP | OS_XOTH) + +// Trigger for collecting GPU state for later extraction. +void RmLogGpuCrash(OBJGPU *); + +// This is callback function in the miniport. +// The argument is a device extension, and must be cast as such to be useful. +typedef void (*MINIPORT_CALLBACK)(void*); + +NV_STATUS osPackageRegistry(OBJGPU *pGpu, PACKED_REGISTRY_TABLE *, NvU32 *); + +#endif // _OS_CUSTOM_H_ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/osapi.h b/userspace/sdks/nv570/nvalloc/unix/include/osapi.h new file mode 100644 index 0000000..6402bdb --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/osapi.h @@ -0,0 +1,203 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _OSAPI_H_ +#define _OSAPI_H_ + +#include "core/system.h" +#include "gpu/gpu.h" + +#include <os-interface.h> // NV_DBG_ERRORS +#include <rmapi/rmapi.h> +#include <core/thread_state.h> + +#if defined(__use_altstack__) +#if defined(QA_BUILD) +//--------------------------------------------------------------------------- +// +// 32 bit debug marker values. +// +//--------------------------------------------------------------------------- + +#define NV_MARKER1 (NvU32)(('M' << 24) | ('R' << 16) | ('V' << 8) | 'N') +#define NV_MARKER2 (NvU32)(('N' << 24) | ('V' << 16) | ('R' << 8) | 'M') + +// +// The two macros below implement a simple alternate stack usage sanity +// check for QA_BUILD RM builds. NV_ALTSTACK_WRITE_MARKERS() fills +// altstacks with NV_MARKER1, which enables NV_ALTSTACK_CHECK_MARKERS() +// to determine the stack usage fairly reliably by looking for the +// first clobbered marker. If more than 7/8 of the alternate stack were +// used, NV_ALTSTACK_CHECK_MARKERS() prints an error and asserts. +// +#define NV_ALTSTACK_WRITE_MARKERS(sp) \ +{ \ + NvU32 i, *stack = (void *)(sp)->stack; \ + for (i = 0; i < ((sp)->size / sizeof(NvU32)); i++) \ + stack[i] = NV_MARKER1; \ +} + +#define NV_ALTSTACK_CHECK_MARKERS(sp) \ +{ \ + NvU32 i, *stack = (void *)(sp)->stack; \ + for (i = 0; i < ((sp)->size / sizeof(NvU32)); i++) \ + { \ + if (stack[i] != NV_MARKER1) \ + break; \ + } \ + if ((i * sizeof(NvU32)) < ((sp)->size / 8)) \ + { \ + nv_printf(NV_DBG_ERRORS, "NVRM: altstack: used %d of %d bytes!\n", \ + ((sp)->size - (i * (NvU32)sizeof(NvU32))), (sp)->size); \ + NV_ASSERT_PRECOMP((i * sizeof(NvU32)) >= ((sp)->size / 8)); \ + } \ +} +#else +#define NV_ALTSTACK_WRITE_MARKERS(sp) +#define NV_ALTSTACK_CHECK_MARKERS(sp) +#endif +#if defined(NVCPU_X86_64) +#define NV_ENTER_RM_RUNTIME(sp,fp) \ +{ \ + NV_ALTSTACK_WRITE_MARKERS(sp); \ + __asm__ __volatile__ ("movq %%rbp,%0" : "=r" (fp)); /* save %rbp */ \ + __asm__ __volatile__ ("movq %0,%%rbp" :: "r" ((sp)->top)); \ +} + +#define NV_EXIT_RM_RUNTIME(sp,fp) \ +{ \ + register void *__rbp __asm__ ("rbp"); \ + if (__rbp != (sp)->top) \ + { \ + nv_printf(NV_DBG_ERRORS, "NVRM: detected corrupted runtime stack!\n"); \ + NV_ASSERT_PRECOMP(__rbp == (sp)->top); \ + } \ + NV_ALTSTACK_CHECK_MARKERS(sp); \ + __asm__ __volatile__ ("movq %0,%%rbp" :: "r" (fp)); /* restore %rbp */ \ +} +#else +#error "gcc \"altstacks\" support is not implemented on this platform!" +#endif +#else +#define NV_ENTER_RM_RUNTIME(sp,fp) { (void)sp; (void)fp; } +#define NV_EXIT_RM_RUNTIME(sp,fp) +#endif + +void RmShutdownRm (void); + +NvBool RmInitPrivateState (nv_state_t *); +void RmFreePrivateState (nv_state_t *); + +NvBool RmInitAdapter (nv_state_t *); +NvBool RmPartiallyInitAdapter (nv_state_t *); +void RmShutdownAdapter (nv_state_t *); +void RmDisableAdapter (nv_state_t *); +void RmPartiallyDisableAdapter(nv_state_t *); +NV_STATUS RmGetAdapterStatus (nv_state_t *, NvU32 *); +NV_STATUS RmExcludeAdapter (nv_state_t *); + +NvBool RmGpuHasIOSpaceEnabled (nv_state_t *); + +void RmFreeUnusedClients (nv_state_t *, nv_file_private_t *); +NV_STATUS RmIoctl (nv_state_t *, nv_file_private_t *, NvU32, void *, NvU32); +NV_STATUS RmValidateIoctl (NvU32, NvU32); + +void RmI2cAddGpuPorts(nv_state_t *); + +NV_STATUS RmInitX86EmuState(OBJGPU *); +void RmFreeX86EmuState(OBJGPU *); +NV_STATUS RmPowerSourceChangeEvent(nv_state_t *, NvU32); + +void RmRequestDNotifierState(nv_state_t *); + +const NvU8 *RmGetGpuUuidRaw(nv_state_t *); + +NV_STATUS nv_vbios_call(OBJGPU *, NvU32 *, NvU32 *); + +int amd_adv_spec_cache_feature(OBJOS *); +int amd_msr_c0011022_incompatible(OBJOS *); + +NV_STATUS rm_get_adapter_status (nv_state_t *, NvU32 *); + +void rm_client_free_os_events (NvHandle); + +NV_STATUS rm_create_mmap_context (NvHandle, NvHandle, NvHandle, NvP64, NvU64, NvU64, NvU32, NvU32); +NV_STATUS rm_update_device_mapping_info (NvHandle, NvHandle, NvHandle, void *, void *); + +NV_STATUS rm_access_registry (NvHandle, NvHandle, NvU32, NvP64, NvU32, NvP64, NvU32, NvP64, NvU32 *, NvU32 *, NvU32 *); + +// registry management +NV_STATUS RmInitRegistry (void); +NV_STATUS RmDestroyRegistry (nv_state_t *); + +NV_STATUS RmWriteRegistryDword (nv_state_t *, const char *, NvU32 ); +NV_STATUS RmReadRegistryDword (nv_state_t *, const char *, NvU32 *); +NV_STATUS RmWriteRegistryString (nv_state_t *, const char *, const char *, NvU32); +NV_STATUS RmReadRegistryBinary (nv_state_t *, const char *, NvU8 *, NvU32 *); +NV_STATUS RmWriteRegistryBinary (nv_state_t *, const char *, NvU8 *, NvU32); +NV_STATUS RmReadRegistryString (nv_state_t *, const char *, NvU8 *, NvU32 *); + +NV_STATUS RmPackageRegistry (nv_state_t *, PACKED_REGISTRY_TABLE *, NvU32 *); + +NvBool RmIsNvifFunctionSupported(NvU32, NvU32); +void RmInitAcpiMethods (OBJOS *, OBJSYS *, OBJGPU *); +void RmUnInitAcpiMethods (OBJSYS *); + +void RmInflateOsToRmPageArray (RmPhysAddr *, NvU64); +void RmDeflateRmToOsPageArray (RmPhysAddr *, NvU64); + +void convertPageArrayBetweenOsAndRm( + RmPhysAddr *pteArray, + NvU64 pageCountFrom, + NvU64 pageSizeFrom, + NvU64 pageCountTo, + NvU64 pageSizeTo, + NvU64 *newPageCountTo); + +#define IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc) \ + (SYS_GET_INSTANCE()->bEnableDynamicGranularityPageArrays && !memdescGetContiguity(pMemDesc, AT_CPU)) + +void RmInitPowerManagement (nv_state_t *); +void RmDestroyPowerManagement (nv_state_t *); + +NV_STATUS RmPowerManagementTegra (OBJGPU *pGpu, nv_pm_action_t pmAction); + +NV_STATUS os_ref_dynamic_power (nv_state_t *, nv_dynamic_power_mode_t); +void os_unref_dynamic_power (nv_state_t *, nv_dynamic_power_mode_t); +void RmHandleDisplayChange (nvidia_stack_t *, nv_state_t *); +void RmUpdateGc6ConsoleRefCount (nv_state_t *, NvBool); + +NvBool rm_get_uefi_console_status (nv_state_t *); +NvU64 rm_get_uefi_console_size (nv_state_t *, NvU64 *); + +void rm_check_s0ix_regkey_and_platform_support(void); + +RM_API *RmUnixRmApiPrologue (nv_state_t *, THREAD_STATE_NODE *, NvU32 module); +void RmUnixRmApiEpilogue (nv_state_t *, THREAD_STATE_NODE *); + +static inline NvBool rm_is_system_notebook(void) +{ + return (nv_is_chassis_notebook() || nv_acpi_is_battery_present()); +} + +#endif // _OSAPI_H_ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/rmobjexportimport.h b/userspace/sdks/nv570/nvalloc/unix/include/rmobjexportimport.h new file mode 100644 index 0000000..a070c4e --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/rmobjexportimport.h @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _RMOBJEXPORTIMPORT_H_ +#define _RMOBJEXPORTIMPORT_H_ + +#include "nvstatus.h" + +typedef NvHandle RmObjExportHandle; + +NV_STATUS RmExportObject(NvHandle hSrcClient, NvHandle hSrcObject, + RmObjExportHandle *pDstObject, NvU32 *pDeviceInstance); + +void RmFreeObjExportHandle(RmObjExportHandle hObject); + +NV_STATUS RmImportObject(NvHandle hDstClient, NvHandle hDstParent, + NvHandle *phDstObject, RmObjExportHandle hSrcObject, + NvU8 *pObjectType); +#endif // _RMOBJEXPORTIMPORT_H_ + diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/debug.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/debug.h new file mode 100644 index 0000000..6a37dfa --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/debug.h @@ -0,0 +1,211 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for debug definitions. +* +****************************************************************************/ + +#ifndef __X86EMU_DEBUG_H +#define __X86EMU_DEBUG_H + +/*---------------------- Macros and type definitions ----------------------*/ + +/* checks to be enabled for "runtime" */ + +#define CHECK_IP_FETCH_F 0x1 +#define CHECK_SP_ACCESS_F 0x2 +#define CHECK_MEM_ACCESS_F 0x4 /*using regular linear pointer */ +#define CHECK_DATA_ACCESS_F 0x8 /*using segment:offset*/ + +#ifdef X86EMU_DEBUG +# define CHECK_IP_FETCH() (M.x86.check & CHECK_IP_FETCH_F) +# define CHECK_SP_ACCESS() (M.x86.check & CHECK_SP_ACCESS_F) +# define CHECK_MEM_ACCESS() (M.x86.check & CHECK_MEM_ACCESS_F) +# define CHECK_DATA_ACCESS() (M.x86.check & CHECK_DATA_ACCESS_F) +#else +# define CHECK_IP_FETCH() 0 +# define CHECK_SP_ACCESS() 0 +# define CHECK_MEM_ACCESS() 0 +# define CHECK_DATA_ACCESS() 0 +#endif + +#ifdef X86EMU_DEBUG +# define DEBUG_INSTRUMENT() (M.x86.debug & DEBUG_INSTRUMENT_F) +# define DEBUG_DECODE() (M.x86.debug & DEBUG_DECODE_F) +# define DEBUG_TRACE() (M.x86.debug & DEBUG_TRACE_F) +# define DEBUG_STEP() (M.x86.debug & DEBUG_STEP_F) +# define DEBUG_DISASSEMBLE() (M.x86.debug & DEBUG_DISASSEMBLE_F) +# define DEBUG_BREAK() (M.x86.debug & DEBUG_BREAK_F) +# define DEBUG_SVC() (M.x86.debug & DEBUG_SVC_F) +# define DEBUG_SAVE_IP_CS() (M.x86.debug & DEBUG_SAVE_IP_CS_F) + +# define DEBUG_FS() (M.x86.debug & DEBUG_FS_F) +# define DEBUG_PROC() (M.x86.debug & DEBUG_PROC_F) +# define DEBUG_SYSINT() (M.x86.debug & DEBUG_SYSINT_F) +# define DEBUG_TRACECALL() (M.x86.debug & DEBUG_TRACECALL_F) +# define DEBUG_TRACECALLREGS() (M.x86.debug & DEBUG_TRACECALL_REGS_F) +# define DEBUG_SYS() (M.x86.debug & DEBUG_SYS_F) +# define DEBUG_MEM_TRACE() (M.x86.debug & DEBUG_MEM_TRACE_F) +# define DEBUG_IO_TRACE() (M.x86.debug & DEBUG_IO_TRACE_F) +# define DEBUG_DECODE_NOPRINT() (M.x86.debug & DEBUG_DECODE_NOPRINT_F) +#else +# define DEBUG_INSTRUMENT() 0 +# define DEBUG_DECODE() 0 +# define DEBUG_TRACE() 0 +# define DEBUG_STEP() 0 +# define DEBUG_DISASSEMBLE() 0 +# define DEBUG_BREAK() 0 +# define DEBUG_SVC() 0 +# define DEBUG_SAVE_IP_CS() 0 +# define DEBUG_FS() 0 +# define DEBUG_PROC() 0 +# define DEBUG_SYSINT() 0 +# define DEBUG_TRACECALL() 0 +# define DEBUG_TRACECALLREGS() 0 +# define DEBUG_SYS() 0 +# define DEBUG_MEM_TRACE() 0 +# define DEBUG_IO_TRACE() 0 +# define DEBUG_DECODE_NOPRINT() 0 +#endif + +#ifdef X86EMU_DEBUG + +# define DECODE_PRINTF(x) if (DEBUG_DECODE()) \ + x86emu_decode_printf(x) +# define DECODE_PRINTF2(x,y) if (DEBUG_DECODE()) \ + x86emu_decode_printf2(x,y) + +/* + * The following allow us to look at the bytes of an instruction. The + * first INCR_INSTRN_LEN, is called everytime bytes are consumed in + * the decoding process. The SAVE_IP_CS is called initially when the + * major opcode of the instruction is accessed. + */ +#define INC_DECODED_INST_LEN(x) \ + if (DEBUG_DECODE()) \ + x86emu_inc_decoded_inst_len(x) + +#define SAVE_IP_CS(x,y) \ + if (DEBUG_DECODE() | DEBUG_TRACECALL() | DEBUG_BREAK() \ + | DEBUG_IO_TRACE() | DEBUG_SAVE_IP_CS()) { \ + M.x86.saved_cs = x; \ + M.x86.saved_ip = y; \ + } +#else +# define INC_DECODED_INST_LEN(x) +# define DECODE_PRINTF(x) +# define DECODE_PRINTF2(x,y) +# define SAVE_IP_CS(x,y) +#endif + +#ifdef X86EMU_DEBUG +#define TRACE_REGS() \ + if (DEBUG_DISASSEMBLE()) { \ + x86emu_just_disassemble(); \ + goto EndOfTheInstructionProcedure; \ + } \ + if (DEBUG_TRACE() || DEBUG_DECODE()) X86EMU_trace_regs() +#else +# define TRACE_REGS() +#endif + +#ifdef X86EMU_DEBUG +# define SINGLE_STEP() if (DEBUG_STEP()) x86emu_single_step() +#else +# define SINGLE_STEP() +#endif + +#define TRACE_AND_STEP() \ + TRACE_REGS(); \ + SINGLE_STEP() + +#ifdef X86EMU_DEBUG +# define START_OF_INSTR() +# define END_OF_INSTR() EndOfTheInstructionProcedure: x86emu_end_instr(); +# define END_OF_INSTR_NO_TRACE() x86emu_end_instr(); +#else +# define START_OF_INSTR() +# define END_OF_INSTR() +# define END_OF_INSTR_NO_TRACE() +#endif + +#ifdef X86EMU_DEBUG +#include <os/os.h> +# define CALL_TRACE(u,v,w,x,s) \ + if (DEBUG_TRACECALLREGS()) \ + x86emu_dump_regs(); \ + if (DEBUG_TRACECALL()) \ + NV_PRINTF(LEVEL_INFO, "%04x:%04x: CALL %s%04x:%04x\n", u , v, s, w, x); +# define RETURN_TRACE(n,u,v) \ + if (DEBUG_TRACECALLREGS()) \ + x86emu_dump_regs(); \ + if (DEBUG_TRACECALL()) \ + NV_PRINTF(LEVEL_INFO, "%04x:%04x: %s\n",u,v,n); +#else +# define CALL_TRACE(u,v,w,x,s) +# define RETURN_TRACE(n,u,v) +#endif + +#ifdef X86EMU_DEBUG +#define DB(x) x +#else +#define DB(x) +#endif + +/*-------------------------- Function Prototypes --------------------------*/ + +#ifdef __cplusplus +extern "C" { /* Use "C" linkage when in C++ mode */ +#endif + +extern void x86emu_inc_decoded_inst_len (int x); +extern void x86emu_decode_printf (const char *x); +extern void x86emu_decode_printf2 (const char *x, int y); +extern void x86emu_just_disassemble (void); +extern void x86emu_single_step (void); +extern void x86emu_end_instr (void); +extern void x86emu_dump_regs (void); +extern void x86emu_dump_xregs (void); +extern void x86emu_print_int_vect (u16 iv); +extern void x86emu_instrument_instruction (void); +extern void x86emu_check_ip_access (void); +extern void x86emu_check_sp_access (void); +extern void x86emu_check_mem_access (u32 p); +extern void x86emu_check_data_access (uint s, uint o); + +#ifdef __cplusplus +} /* End of "C" linkage for C++ */ +#endif + +#endif /* __X86EMU_DEBUG_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/decode.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/decode.h new file mode 100644 index 0000000..52eff18 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/decode.h @@ -0,0 +1,88 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for instruction decoding logic. +* +****************************************************************************/ + +#ifndef __X86EMU_DECODE_H +#define __X86EMU_DECODE_H + +/*---------------------- Macros and type definitions ----------------------*/ + +/* Instruction Decoding Stuff */ + +#define FETCH_DECODE_MODRM(mod,rh,rl) fetch_decode_modrm(&mod,&rh,&rl) +#define DECODE_RM_BYTE_REGISTER(r) decode_rm_byte_register(r) +#define DECODE_RM_WORD_REGISTER(r) decode_rm_word_register(r) +#define DECODE_RM_LONG_REGISTER(r) decode_rm_long_register(r) +#define DECODE_CLEAR_SEGOVR() M.x86.mode &= ~SYSMODE_CLRMASK + +/*-------------------------- Function Prototypes --------------------------*/ + +#ifdef __cplusplus +extern "C" { /* Use "C" linkage when in C++ mode */ +#endif + +void x86emu_intr_raise (u8 type); +void fetch_decode_modrm (int *mod,int *regh,int *regl); +u8 fetch_byte_imm (void); +u16 fetch_word_imm (void); +u32 fetch_long_imm (void); +u8 fetch_data_byte (uint offset); +u8 fetch_data_byte_abs (uint segment, uint offset); +u16 fetch_data_word (uint offset); +u16 fetch_data_word_abs (uint segment, uint offset); +u32 fetch_data_long (uint offset); +u32 fetch_data_long_abs (uint segment, uint offset); +void store_data_byte (uint offset, u8 val); +void store_data_byte_abs (uint segment, uint offset, u8 val); +void store_data_word (uint offset, u16 val); +void store_data_word_abs (uint segment, uint offset, u16 val); +void store_data_long (uint offset, u32 val); +void store_data_long_abs (uint segment, uint offset, u32 val); +u8* decode_rm_byte_register(int reg); +u16* decode_rm_word_register(int reg); +u32* decode_rm_long_register(int reg); +u16* decode_rm_seg_register(int reg); +u32 decode_rm00_address(int rm); +u32 decode_rm01_address(int rm); +u32 decode_rm10_address(int rm); +u32 decode_sib_address(int sib, int mod); + +#ifdef __cplusplus +} /* End of "C" linkage for C++ */ +#endif + +#endif /* __X86EMU_DECODE_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/fpu.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/fpu.h new file mode 100644 index 0000000..7c4f24e --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/fpu.h @@ -0,0 +1,61 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for FPU instruction decoding. +* +****************************************************************************/ + +#ifndef __X86EMU_FPU_H +#define __X86EMU_FPU_H + +#ifdef __cplusplus +extern "C" { /* Use "C" linkage when in C++ mode */ +#endif + +/* these have to be defined, whether 8087 support compiled in or not. */ + +extern void x86emuOp_esc_coprocess_d8 (u8 op1); +extern void x86emuOp_esc_coprocess_d9 (u8 op1); +extern void x86emuOp_esc_coprocess_da (u8 op1); +extern void x86emuOp_esc_coprocess_db (u8 op1); +extern void x86emuOp_esc_coprocess_dc (u8 op1); +extern void x86emuOp_esc_coprocess_dd (u8 op1); +extern void x86emuOp_esc_coprocess_de (u8 op1); +extern void x86emuOp_esc_coprocess_df (u8 op1); + +#ifdef __cplusplus +} /* End of "C" linkage for C++ */ +#endif + +#endif /* __X86EMU_FPU_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/fpu_regs.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/fpu_regs.h new file mode 100644 index 0000000..39ee7fe --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/fpu_regs.h @@ -0,0 +1,119 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for FPU register definitions. +* +****************************************************************************/ + +#ifndef __X86EMU_FPU_REGS_H +#define __X86EMU_FPU_REGS_H + +#ifdef X86_FPU_SUPPORT + +#ifdef PACK +# pragma PACK +#endif + +/* Basic 8087 register can hold any of the following values: */ + +union x86_fpu_reg_u { + s8 tenbytes[10]; + double dval; + float fval; + s16 sval; + s32 lval; + }; + +struct x86_fpu_reg { + union x86_fpu_reg_u reg; + char tag; + }; + +/* + * Since we are not going to worry about the problems of aliasing + * registers, every time a register is modified, its result type is + * set in the tag fields for that register. If some operation + * attempts to access the type in a way inconsistent with its current + * storage format, then we flag the operation. If common, we'll + * attempt the conversion. + */ + +#define X86_FPU_VALID 0x80 +#define X86_FPU_REGTYP(r) ((r) & 0x7F) + +#define X86_FPU_WORD 0x0 +#define X86_FPU_SHORT 0x1 +#define X86_FPU_LONG 0x2 +#define X86_FPU_FLOAT 0x3 +#define X86_FPU_DOUBLE 0x4 +#define X86_FPU_LDBL 0x5 +#define X86_FPU_BSD 0x6 + +#define X86_FPU_STKTOP 0 + +struct x86_fpu_registers { + struct x86_fpu_reg x86_fpu_stack[8]; + int x86_fpu_flags; + int x86_fpu_config; /* rounding modes, etc. */ + short x86_fpu_tos, x86_fpu_bos; + }; + +#ifdef END_PACK +# pragma END_PACK +#endif + +/* + * There are two versions of the following macro. + * + * One version is for opcode D9, for which there are more than 32 + * instructions encoded in the second byte of the opcode. + * + * The other version, deals with all the other 7 i87 opcodes, for + * which there are only 32 strings needed to describe the + * instructions. + */ + +#endif /* X86_FPU_SUPPORT */ + +#ifdef X86EMU_DEBUG +# define DECODE_PRINTINSTR32(t,mod,rh,rl) \ + DECODE_PRINTF(t[(mod<<3)+(rh)]); +# define DECODE_PRINTINSTR256(t,mod,rh,rl) \ + DECODE_PRINTF(t[(mod<<6)+(rh<<3)+(rl)]); +#else +# define DECODE_PRINTINSTR32(t,mod,rh,rl) +# define DECODE_PRINTINSTR256(t,mod,rh,rl) +#endif + +#endif /* __X86EMU_FPU_REGS_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/ops.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/ops.h new file mode 100644 index 0000000..82d2bbe --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/ops.h @@ -0,0 +1,45 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for operand decoding functions. +* +****************************************************************************/ + +#ifndef __X86EMU_OPS_H +#define __X86EMU_OPS_H + +extern void (*x86emu_optab[0x100])(u8 op1); +extern void (*x86emu_optab2[0x100])(u8 op2); + +#endif /* __X86EMU_OPS_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/prim_ops.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/prim_ops.h new file mode 100644 index 0000000..8978004 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/prim_ops.h @@ -0,0 +1,141 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for primitive operation functions. +* +****************************************************************************/ + +#ifndef __X86EMU_PRIM_OPS_H +#define __X86EMU_PRIM_OPS_H + +#ifdef __cplusplus +extern "C" { /* Use "C" linkage when in C++ mode */ +#endif + +u16 aaa_word (u16 d); +u16 aas_word (u16 d); +u16 aad_word (u16 d); +u16 aam_word (u8 d); +u8 adc_byte (u8 d, u8 s); +u16 adc_word (u16 d, u16 s); +u32 adc_long (u32 d, u32 s); +u8 add_byte (u8 d, u8 s); +u16 add_word (u16 d, u16 s); +u32 add_long (u32 d, u32 s); +u8 and_byte (u8 d, u8 s); +u16 and_word (u16 d, u16 s); +u32 and_long (u32 d, u32 s); +u8 cmp_byte (u8 d, u8 s); +u16 cmp_word (u16 d, u16 s); +u32 cmp_long (u32 d, u32 s); +u8 daa_byte (u8 d); +u8 das_byte (u8 d); +u8 dec_byte (u8 d); +u16 dec_word (u16 d); +u32 dec_long (u32 d); +u8 inc_byte (u8 d); +u16 inc_word (u16 d); +u32 inc_long (u32 d); +u8 or_byte (u8 d, u8 s); +u16 or_word (u16 d, u16 s); +u32 or_long (u32 d, u32 s); +u8 neg_byte (u8 s); +u16 neg_word (u16 s); +u32 neg_long (u32 s); +u8 not_byte (u8 s); +u16 not_word (u16 s); +u32 not_long (u32 s); +u8 rcl_byte (u8 d, u8 s); +u16 rcl_word (u16 d, u8 s); +u32 rcl_long (u32 d, u8 s); +u8 rcr_byte (u8 d, u8 s); +u16 rcr_word (u16 d, u8 s); +u32 rcr_long (u32 d, u8 s); +u8 rol_byte (u8 d, u8 s); +u16 rol_word (u16 d, u8 s); +u32 rol_long (u32 d, u8 s); +u8 ror_byte (u8 d, u8 s); +u16 ror_word (u16 d, u8 s); +u32 ror_long (u32 d, u8 s); +u8 shl_byte (u8 d, u8 s); +u16 shl_word (u16 d, u8 s); +u32 shl_long (u32 d, u8 s); +u8 shr_byte (u8 d, u8 s); +u16 shr_word (u16 d, u8 s); +u32 shr_long (u32 d, u8 s); +u8 sar_byte (u8 d, u8 s); +u16 sar_word (u16 d, u8 s); +u32 sar_long (u32 d, u8 s); +u16 shld_word (u16 d, u16 fill, u8 s); +u32 shld_long (u32 d, u32 fill, u8 s); +u16 shrd_word (u16 d, u16 fill, u8 s); +u32 shrd_long (u32 d, u32 fill, u8 s); +u8 sbb_byte (u8 d, u8 s); +u16 sbb_word (u16 d, u16 s); +u32 sbb_long (u32 d, u32 s); +u8 sub_byte (u8 d, u8 s); +u16 sub_word (u16 d, u16 s); +u32 sub_long (u32 d, u32 s); +void test_byte (u8 d, u8 s); +void test_word (u16 d, u16 s); +void test_long (u32 d, u32 s); +u8 xor_byte (u8 d, u8 s); +u16 xor_word (u16 d, u16 s); +u32 xor_long (u32 d, u32 s); +void imul_byte (u8 s); +void imul_word (u16 s); +void imul_long (u32 s); +void imul_long_direct(u32 *res_lo, u32* res_hi,u32 d, u32 s); +void mul_byte (u8 s); +void mul_word (u16 s); +void mul_long (u32 s); +void idiv_byte (u8 s); +void idiv_word (u16 s); +void idiv_long (u32 s); +void div_byte (u8 s); +void div_word (u16 s); +void div_long (u32 s); +void ins (int size); +void outs (int size); +u16 mem_access_word (int addr); +void push_word (u16 w); +void push_long (u32 w); +u16 pop_word (void); +u32 pop_long (void); + +#ifdef __cplusplus +} /* End of "C" linkage for C++ */ +#endif + +#endif /* __X86EMU_PRIM_OPS_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/regs.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/regs.h new file mode 100644 index 0000000..4cd0f8b --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/regs.h @@ -0,0 +1,340 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for x86 register definitions. +* +****************************************************************************/ + +#ifndef __X86EMU_REGS_H +#define __X86EMU_REGS_H + +#include "x86emu/debug.h" + +/*---------------------- Macros and type definitions ----------------------*/ + +#ifdef PACK +# pragma PACK +#endif + +/* + * General EAX, EBX, ECX, EDX type registers. Note that for + * portability, and speed, the issue of byte swapping is not addressed + * in the registers. All registers are stored in the default format + * available on the host machine. The only critical issue is that the + * registers should line up EXACTLY in the same manner as they do in + * the 386. That is: + * + * EAX & 0xff === AL + * EAX & 0xffff == AX + * + * etc. The result is that alot of the calculations can then be + * done using the native instruction set fully. + */ + +#ifdef __BIG_ENDIAN__ + +typedef struct { + u32 e_reg; + } I32_reg_t; + +typedef struct { + u16 filler0, x_reg; + } I16_reg_t; + +typedef struct { + u8 filler0, filler1, h_reg, l_reg; + } I8_reg_t; + +#else /* !__BIG_ENDIAN__ */ + +typedef struct { + u32 e_reg; + } I32_reg_t; + +typedef struct { + u16 x_reg; + } I16_reg_t; + +typedef struct { + u8 l_reg, h_reg; + } I8_reg_t; + +#endif /* __BIG_ENDIAN__ */ + +typedef union { + I32_reg_t I32_reg; + I16_reg_t I16_reg; + I8_reg_t I8_reg; + } i386_general_register; + +struct i386_general_regs { + i386_general_register A, B, C, D; + }; + +typedef struct i386_general_regs Gen_reg_t; + +struct i386_special_regs { + i386_general_register SP, BP, SI, DI, IP; + u32 FLAGS; + }; + +/* + * Segment registers here represent the 16 bit quantities + * CS, DS, ES, SS. + */ + +struct i386_segment_regs { + u16 CS, DS, SS, ES, FS, GS; + }; + +/* 8 bit registers */ +#define R_AH gen.A.I8_reg.h_reg +#define R_AL gen.A.I8_reg.l_reg +#define R_BH gen.B.I8_reg.h_reg +#define R_BL gen.B.I8_reg.l_reg +#define R_CH gen.C.I8_reg.h_reg +#define R_CL gen.C.I8_reg.l_reg +#define R_DH gen.D.I8_reg.h_reg +#define R_DL gen.D.I8_reg.l_reg + +/* 16 bit registers */ +#define R_AX gen.A.I16_reg.x_reg +#define R_BX gen.B.I16_reg.x_reg +#define R_CX gen.C.I16_reg.x_reg +#define R_DX gen.D.I16_reg.x_reg + +/* 32 bit extended registers */ +#define R_EAX gen.A.I32_reg.e_reg +#define R_EBX gen.B.I32_reg.e_reg +#define R_ECX gen.C.I32_reg.e_reg +#define R_EDX gen.D.I32_reg.e_reg + +/* special registers */ +#define R_SP spc.SP.I16_reg.x_reg +#define R_BP spc.BP.I16_reg.x_reg +#define R_SI spc.SI.I16_reg.x_reg +#define R_DI spc.DI.I16_reg.x_reg +#define R_IP spc.IP.I16_reg.x_reg +#define R_FLG spc.FLAGS + +/* special registers */ +#define R_SP spc.SP.I16_reg.x_reg +#define R_BP spc.BP.I16_reg.x_reg +#define R_SI spc.SI.I16_reg.x_reg +#define R_DI spc.DI.I16_reg.x_reg +#define R_IP spc.IP.I16_reg.x_reg +#define R_FLG spc.FLAGS + +/* special registers */ +#define R_ESP spc.SP.I32_reg.e_reg +#define R_EBP spc.BP.I32_reg.e_reg +#define R_ESI spc.SI.I32_reg.e_reg +#define R_EDI spc.DI.I32_reg.e_reg +#define R_EIP spc.IP.I32_reg.e_reg +#define R_EFLG spc.FLAGS + +/* segment registers */ +#define R_CS seg.CS +#define R_DS seg.DS +#define R_SS seg.SS +#define R_ES seg.ES +#define R_FS seg.FS +#define R_GS seg.GS + +/* flag conditions */ +#define FB_CF 0x0001 /* CARRY flag */ +#define FB_PF 0x0004 /* PARITY flag */ +#define FB_AF 0x0010 /* AUX flag */ +#define FB_ZF 0x0040 /* ZERO flag */ +#define FB_SF 0x0080 /* SIGN flag */ +#define FB_TF 0x0100 /* TRAP flag */ +#define FB_IF 0x0200 /* INTERRUPT ENABLE flag */ +#define FB_DF 0x0400 /* DIR flag */ +#define FB_OF 0x0800 /* OVERFLOW flag */ + +/* 80286 and above always have bit#1 set */ +#define F_ALWAYS_ON (0x0002) /* flag bits always on */ + +/* + * Define a mask for only those flag bits we will ever pass back + * (via PUSHF) + */ +#define F_MSK (FB_CF|FB_PF|FB_AF|FB_ZF|FB_SF|FB_TF|FB_IF|FB_DF|FB_OF) + +/* following bits masked in to a 16bit quantity */ + +#define F_CF 0x0001 /* CARRY flag */ +#define F_PF 0x0004 /* PARITY flag */ +#define F_AF 0x0010 /* AUX flag */ +#define F_ZF 0x0040 /* ZERO flag */ +#define F_SF 0x0080 /* SIGN flag */ +#define F_TF 0x0100 /* TRAP flag */ +#define F_IF 0x0200 /* INTERRUPT ENABLE flag */ +#define F_DF 0x0400 /* DIR flag */ +#define F_OF 0x0800 /* OVERFLOW flag */ + +#define TOGGLE_FLAG(flag) (M.x86.R_FLG ^= (flag)) +#define SET_FLAG(flag) (M.x86.R_FLG |= (flag)) +#define CLEAR_FLAG(flag) (M.x86.R_FLG &= ~(flag)) +#define ACCESS_FLAG(flag) (M.x86.R_FLG & (flag)) +#define CLEARALL_FLAG(m) (M.x86.R_FLG = 0) + +#define CONDITIONAL_SET_FLAG(COND,FLAG) \ + if (COND) SET_FLAG(FLAG); else CLEAR_FLAG(FLAG) + +#define F_PF_CALC 0x010000 /* PARITY flag has been calced */ +#define F_ZF_CALC 0x020000 /* ZERO flag has been calced */ +#define F_SF_CALC 0x040000 /* SIGN flag has been calced */ + +#define F_ALL_CALC 0xff0000 /* All have been calced */ + +/* + * Emulator machine state. + * Segment usage control. + */ +#define SYSMODE_SEG_DS_SS 0x00000001 +#define SYSMODE_SEGOVR_CS 0x00000002 +#define SYSMODE_SEGOVR_DS 0x00000004 +#define SYSMODE_SEGOVR_ES 0x00000008 +#define SYSMODE_SEGOVR_FS 0x00000010 +#define SYSMODE_SEGOVR_GS 0x00000020 +#define SYSMODE_SEGOVR_SS 0x00000040 +#define SYSMODE_PREFIX_REPE 0x00000080 +#define SYSMODE_PREFIX_REPNE 0x00000100 +#define SYSMODE_PREFIX_DATA 0x00000200 +#define SYSMODE_PREFIX_ADDR 0x00000400 +#define SYSMODE_INTR_PENDING 0x10000000 +#define SYSMODE_EXTRN_INTR 0x20000000 +#define SYSMODE_HALTED 0x40000000 + +#define SYSMODE_SEGMASK (SYSMODE_SEG_DS_SS | \ + SYSMODE_SEGOVR_CS | \ + SYSMODE_SEGOVR_DS | \ + SYSMODE_SEGOVR_ES | \ + SYSMODE_SEGOVR_FS | \ + SYSMODE_SEGOVR_GS | \ + SYSMODE_SEGOVR_SS) +#define SYSMODE_CLRMASK (SYSMODE_SEG_DS_SS | \ + SYSMODE_SEGOVR_CS | \ + SYSMODE_SEGOVR_DS | \ + SYSMODE_SEGOVR_ES | \ + SYSMODE_SEGOVR_FS | \ + SYSMODE_SEGOVR_GS | \ + SYSMODE_SEGOVR_SS | \ + SYSMODE_PREFIX_DATA | \ + SYSMODE_PREFIX_ADDR) + +#define INTR_SYNCH 0x1 +#define INTR_ASYNCH 0x2 +#define INTR_HALTED 0x4 + +typedef struct { + struct i386_general_regs gen; + struct i386_special_regs spc; + struct i386_segment_regs seg; + /* + * MODE contains information on: + * REPE prefix 2 bits repe,repne + * SEGMENT overrides 5 bits normal,DS,SS,CS,ES + * Delayed flag set 3 bits (zero, signed, parity) + * reserved 6 bits + * interrupt # 8 bits instruction raised interrupt + * BIOS video segregs 4 bits + * Interrupt Pending 1 bits + * Extern interrupt 1 bits + * Halted 1 bits + */ + u32 mode; + volatile int intr; /* mask of pending interrupts */ + int debug; +#ifdef X86EMU_DEBUG + int check; + u16 saved_ip; + u16 saved_cs; + int enc_pos; + int enc_str_pos; + char decode_buf[32]; /* encoded byte stream */ + char decoded_buf[256]; /* disassembled strings */ + char debug_cmd[64]; /* input for x86emu_single_step */ +#endif + u8 intno; + u8 __pad[3]; + } X86EMU_regs; + +/**************************************************************************** +REMARKS: +Structure maintaining the emulator machine state. + +MEMBERS: +mem_base - Base real mode memory for the emulator +mem_size - Size of the real mode memory block for the emulator +private - private data pointer +x86 - X86 registers +****************************************************************************/ +typedef struct { + unsigned long mem_base; + unsigned long mem_size; + void* private; + X86EMU_regs x86; + } X86EMU_sysEnv; + +#ifdef END_PACK +# pragma END_PACK +#endif + +/*----------------------------- Global Variables --------------------------*/ + +#ifdef __cplusplus +extern "C" { /* Use "C" linkage when in C++ mode */ +#endif + +/* Global emulator machine state. + * + * We keep it global to avoid pointer dereferences in the code for speed. + */ + +extern X86EMU_sysEnv _X86EMU_env; +#define M _X86EMU_env + +/*-------------------------- Function Prototypes --------------------------*/ + +/* Function to log information at runtime */ + +void printk(const char *fmt, ...); + +#ifdef __cplusplus +} /* End of "C" linkage for C++ */ +#endif + +#endif /* __X86EMU_REGS_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/types.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/types.h new file mode 100644 index 0000000..53c3e87 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/types.h @@ -0,0 +1,93 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for x86 emulator type definitions. +* +****************************************************************************/ + + +#ifndef __X86EMU_TYPES_H +#define __X86EMU_TYPES_H + +#ifndef NO_SYS_HEADERS +#include <sys/types.h> +#endif + +/* + * The following kludge is an attempt to work around typedef conflicts with + * <sys/types.h>. + */ +#define u8 x86emuu8 +#define u16 x86emuu16 +#define u32 x86emuu32 +#define u64 x86emuu64 +#define s8 x86emus8 +#define s16 x86emus16 +#define s32 x86emus32 +#define s64 x86emus64 +#define uint x86emuuint +#define sint x86emusint + +/*---------------------- Macros and type definitions ----------------------*/ + +/* Currently only for Linux/32bit */ +#undef __HAS_LONG_LONG__ +#if defined(__GNUC__) && !defined(NO_LONG_LONG) +#define __HAS_LONG_LONG__ +#endif + +#define NUM32 int + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned NUM32 u32; +#ifdef __HAS_LONG_LONG__ +typedef unsigned long long u64; +#endif + +typedef char s8; +typedef short s16; +typedef NUM32 s32; +#ifdef __HAS_LONG_LONG__ +typedef long long s64; +#endif + +typedef unsigned int uint; +typedef int sint; + +typedef u16 X86EMU_pioAddr; + +#undef NUM32 + +#endif /* __X86EMU_TYPES_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/x86emu.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/x86emu.h new file mode 100644 index 0000000..935cc70 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/x86emu.h @@ -0,0 +1,205 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for public specific functions. +* Any application linking against us should only +* include this header +* +****************************************************************************/ + +#ifndef __X86EMU_X86EMU_H +#define __X86EMU_X86EMU_H + +// NV +#ifndef NO_SYS_HEADERS +#define NO_SYS_HEADERS +#endif + +#ifdef SCITECH +#include "scitech.h" +#define X86API _ASMAPI +#define X86APIP _ASMAPIP +typedef int X86EMU_pioAddr; +#else +#include "x86emu/types.h" +#define X86API +#define X86APIP * +#endif +#include "x86emu/regs.h" +#include "x86emu/debug.h" + +/*---------------------- Macros and type definitions ----------------------*/ + +#ifdef PACK +# pragma PACK /* Don't pack structs with function pointers! */ +#endif + +/**************************************************************************** +REMARKS: +Data structure containing ponters to programmed I/O functions used by the +emulator. This is used so that the user program can hook all programmed +I/O for the emulator to handled as necessary by the user program. By +default the emulator contains simple functions that do not do access the +hardware in any way. To allow the emualtor access the hardware, you will +need to override the programmed I/O functions using the X86EMU_setupPioFuncs +function. + +HEADER: +x86emu.h + +MEMBERS: +inb - Function to read a byte from an I/O port +inw - Function to read a word from an I/O port +inl - Function to read a dword from an I/O port +outb - Function to write a byte to an I/O port +outw - Function to write a word to an I/O port +outl - Function to write a dword to an I/O port +****************************************************************************/ +typedef struct { + u8 (X86APIP inb)(X86EMU_pioAddr addr); + u16 (X86APIP inw)(X86EMU_pioAddr addr); + u32 (X86APIP inl)(X86EMU_pioAddr addr); + void (X86APIP outb)(X86EMU_pioAddr addr, u8 val); + void (X86APIP outw)(X86EMU_pioAddr addr, u16 val); + void (X86APIP outl)(X86EMU_pioAddr addr, u32 val); + } X86EMU_pioFuncs; + +/**************************************************************************** +REMARKS: +Data structure containing ponters to memory access functions used by the +emulator. This is used so that the user program can hook all memory +access functions as necessary for the emulator. By default the emulator +contains simple functions that only access the internal memory of the +emulator. If you need specialised functions to handle access to different +types of memory (ie: hardware framebuffer accesses and BIOS memory access +etc), you will need to override this using the X86EMU_setupMemFuncs +function. + +HEADER: +x86emu.h + +MEMBERS: +rdb - Function to read a byte from an address +rdw - Function to read a word from an address +rdl - Function to read a dword from an address +wrb - Function to write a byte to an address +wrw - Function to write a word to an address +wrl - Function to write a dword to an address +****************************************************************************/ +typedef struct { + u8 (X86APIP rdb)(u32 addr); + u16 (X86APIP rdw)(u32 addr); + u32 (X86APIP rdl)(u32 addr); + void (X86APIP wrb)(u32 addr, u8 val); + void (X86APIP wrw)(u32 addr, u16 val); + void (X86APIP wrl)(u32 addr, u32 val); + } X86EMU_memFuncs; + +/**************************************************************************** + Here are the default memory read and write + function in case they are needed as fallbacks. +***************************************************************************/ +extern u8 X86API rdb(u32 addr); +extern u16 X86API rdw(u32 addr); +extern u32 X86API rdl(u32 addr); +extern void X86API wrb(u32 addr, u8 val); +extern void X86API wrw(u32 addr, u16 val); +extern void X86API wrl(u32 addr, u32 val); + +#ifdef END_PACK +# pragma END_PACK +#endif + +/*--------------------- type definitions -----------------------------------*/ + +typedef void (X86APIP X86EMU_intrFuncs)(int num); +extern X86EMU_intrFuncs _X86EMU_intrTab[256]; + +/*-------------------------- Function Prototypes --------------------------*/ + +#ifdef __cplusplus +extern "C" { /* Use "C" linkage when in C++ mode */ +#endif + +void X86EMU_setupMemFuncs(X86EMU_memFuncs *funcs); +void X86EMU_setupPioFuncs(X86EMU_pioFuncs *funcs); +void X86EMU_setupIntrFuncs(X86EMU_intrFuncs funcs[]); +void X86EMU_prepareForInt(int num); + +/* decode.c */ + +void X86EMU_exec(void); +void X86EMU_halt_sys(void); + +#ifdef X86EMU_DEBUG +#define HALT_SYS() do { \ + NV_PRINTF(LEVEL_INFO, "halt_sys: file %s, line %d\n", __FILE__, __LINE__); \ + X86EMU_halt_sys(); \ + } while (0) +#else +#define HALT_SYS() X86EMU_halt_sys() +#endif + +/* Debug options */ + +#define DEBUG_DECODE_F 0x000001 /* print decoded instruction */ +#define DEBUG_TRACE_F 0x000002 /* dump regs before/after execution */ +#define DEBUG_STEP_F 0x000004 +#define DEBUG_DISASSEMBLE_F 0x000008 +#define DEBUG_BREAK_F 0x000010 +#define DEBUG_SVC_F 0x000020 +#define DEBUG_SAVE_IP_CS_F 0x000040 +#define DEBUG_FS_F 0x000080 +#define DEBUG_PROC_F 0x000100 +#define DEBUG_SYSINT_F 0x000200 /* bios system interrupts. */ +#define DEBUG_TRACECALL_F 0x000400 +#define DEBUG_INSTRUMENT_F 0x000800 +#define DEBUG_MEM_TRACE_F 0x001000 +#define DEBUG_IO_TRACE_F 0x002000 +#define DEBUG_TRACECALL_REGS_F 0x004000 +#define DEBUG_DECODE_NOPRINT_F 0x008000 +#define DEBUG_EXIT 0x010000 +#define DEBUG_SYS_F (DEBUG_SVC_F|DEBUG_FS_F|DEBUG_PROC_F) + +void X86EMU_trace_regs(void); +void X86EMU_trace_xregs(void); +void X86EMU_dump_memory(u16 seg, u16 off, u32 amt); +int X86EMU_trace_on(void); +int X86EMU_trace_off(void); + +#ifdef __cplusplus +} /* End of "C" linkage for C++ */ +#endif + +#endif /* __X86EMU_X86EMU_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/include/x86emu/x86emui.h b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/x86emui.h new file mode 100644 index 0000000..5788558 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/include/x86emu/x86emui.h @@ -0,0 +1,113 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: Header file for system specific functions. These functions +* are always compiled and linked in the OS depedent libraries, +* and never in a binary portable driver. +* +****************************************************************************/ + + +#ifndef __X86EMU_X86EMUI_H +#define __X86EMU_X86EMUI_H + +/* If we are compiling in C++ mode, we can compile some functions as + * inline to increase performance (however the code size increases quite + * dramatically in this case). + */ + +// NV +#ifndef NO_SYS_HEADERS +#define NO_SYS_HEADERS +#endif +#include <stddef.h> /* NULL */ +#if defined(__cplusplus) && !defined(_NO_INLINE) +#define _INLINE inline +#else +#define _INLINE static +#endif + +/* Get rid of unused parameters in C++ compilation mode */ + +#ifdef __cplusplus +#define X86EMU_UNUSED(v) +#else +#define X86EMU_UNUSED(v) v +#endif + +#include "x86emu.h" +#include "x86emu/regs.h" +#include "x86emu/debug.h" +#include "x86emu/decode.h" +#include "x86emu/ops.h" +#include "x86emu/prim_ops.h" +#include "x86emu/fpu.h" +#include "x86emu/fpu_regs.h" + +#ifndef NO_SYS_HEADERS +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#endif + +// On x86_64 abs() is not recognized. I don't understand why. +#ifndef abs +#define abs(i) __builtin_abs(i) +#endif + +/*--------------------------- Inline Functions ----------------------------*/ + +#ifdef __cplusplus +extern "C" { /* Use "C" linkage when in C++ mode */ +#endif + +extern u8 (X86APIP sys_rdb)(u32 addr); +extern u16 (X86APIP sys_rdw)(u32 addr); +extern u32 (X86APIP sys_rdl)(u32 addr); +extern void (X86APIP sys_wrb)(u32 addr,u8 val); +extern void (X86APIP sys_wrw)(u32 addr,u16 val); +extern void (X86APIP sys_wrl)(u32 addr,u32 val); + +extern u8 (X86APIP sys_inb)(X86EMU_pioAddr addr); +extern u16 (X86APIP sys_inw)(X86EMU_pioAddr addr); +extern u32 (X86APIP sys_inl)(X86EMU_pioAddr addr); +extern void (X86APIP sys_outb)(X86EMU_pioAddr addr,u8 val); +extern void (X86APIP sys_outw)(X86EMU_pioAddr addr,u16 val); +extern void (X86APIP sys_outl)(X86EMU_pioAddr addr,u32 val); + +#ifdef __cplusplus +} /* End of "C" linkage for C++ */ +#endif + +#endif /* __X86EMU_X86EMUI_H */ diff --git a/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_cpuid.c b/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_cpuid.c new file mode 100644 index 0000000..fe6dfa9 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_cpuid.c @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <os/os.h> + +/* inline assembler routines for UNIX platforms */ + +#if defined(NVCPU_X86_64) + +int osNv_cpuid( + int op, + int subop, + NvU32 *eax, + NvU32 *ebx, + NvU32 *ecx, + NvU32 *edx +) +{ + asm volatile (" cpuid \n" + : "=a" (*eax), + "=b" (*ebx), + "=c" (*ecx), + "=d" (*edx) + : "a" (op), + "c" (subop) + : "cc"); + + return 1; +} + +#endif diff --git a/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_rdcr.c b/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_rdcr.c new file mode 100644 index 0000000..6115aa4 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_rdcr.c @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <os/os.h> + +/* inline assembler routines for UNIX platforms */ + +#if defined(NVCPU_X86_64) + +NvU32 osNv_rdcr4(void) +{ + NvU64 val; + asm volatile ("movq %%cr4,%0" : "=r" (val)); + return (NvU32)val; +} + +#endif diff --git a/userspace/sdks/nv570/nvalloc/unix/src/dynamic-power.c b/userspace/sdks/nv570/nvalloc/unix/src/dynamic-power.c new file mode 100644 index 0000000..fa61d94 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/dynamic-power.c @@ -0,0 +1,3185 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <rmconfig.h> + +/* + * NOTE: + * The driver disables dynamic power management by effectively setting + * nv_dynamic_power_t::mode to NV_DYNAMIC_PM_NEVER at two places: + * 1) During module probe, when the _PRx methods are not supported + * by the SBIOS or when the kernel does not support runtime PM. + * 2) During RM Init, when the GPU is not Turing+ or when the system is not + * a laptop or when the VBIOS does not support RTD3/GC6. + * + * Thus, after RMInit, if the nv_dynamic_power_t::mode is still + * NV_DYNAMIC_PM_FINE, then we know for sure that the fine grained power + * control is active and running. In this case, the runtime D3 status is + * shown as "Enabled (fine-grained)". + * + * If the nv_dynamic_power_t::mode is NV_DYNAMIC_PM_NEVER, we have to + * distinguish between four cases: + * 1) The driver disabled dynamic power management due to lack of kernel/SBIOS + * support. + * 2) The driver disabled dynamic power management because the GPU does not + * support it or the system is not a laptop. + * 3) The user has explicitly disabled the feature. + * 4) Configuration is not supported by default. + * + * In order to differentiate between these four cases, + * we use the value of regkey NVreg_DynamicPowerManagement. If this value is + * set to 0, then we know that the user has explicitly disabled the + * feature. In this case we show the runtime D3 status as "Disabled". This + * handles case (3) above. + * + * For case (4), we look at the combination of nv_dynamic_power_t::mode and + * NVreg_DynamicPowerManagement value. If mode is never and regkey value is + * default, then status is shown as "Disabled by default". + * + * For remaining cases, (case (1) and (2)) we show status as "Not Supported". + * + * When the regkey is set to 0 and the driver disables the feature due to lack + * of some support, then precedence will be given to regkey value. Thus, when + * both "Not supported" and "Disabled" are possible values, precedence will be + * given to "Disabled" over "Not Supported". + * + * For coarse-grain power control, we show runtime D3 status as + * "Enabled (coarse-grained)". + * + * The runtime D3 status is shown as "?" when RM has not been initialized. + * This is because we do not have full knowledge regarding driver triggered + * disablement until RM is initialized. + */ + +#include <nvlog_inc.h> +#include <nv.h> +#include <nv-priv.h> +#include <nv-reg.h> +#include <nv_ref.h> + +#include <osapi.h> + +#include <gpu/mem_mgr/mem_mgr.h> +#include <gpu/kern_gpu_power.h> +#include <gpu_mgr/gpu_mgr.h> +#include <core/locks.h> +#include <pex.h> +#include <acpidsmguids.h> +#include "kernel/gpu/intr/intr.h" + +#include <gpu/mem_sys/kern_mem_sys.h> +#include <gpu/subdevice/subdevice.h> +#include <ctrl/ctrl0080/ctrl0080gpu.h> +#include <ctrl/ctrl2080/ctrl2080unix.h> +#include <gpu/timer/objtmr.h> + + +// mask holding dynamic power supported GPUs +static NvU32 dynamicPowerSupportGpuMask = 0; + +// +// Schedule timer based callback, to check for the complete GPU Idleness. +// Windows has idle time from 70msec to 10sec, we opted for present duration +// considering windows limit. Duration is not much aggressive or slow, hence +// less thrashing. +// +#define GC6_PRECONDITION_CHECK_TIME ((NvU64)5 * 1000 * 1000 * 1000) + +// +// iGPU needs a more aggressive timeout value of precondition +// check to quickly engage the Rail-Gating state on an embedded +// system. +// Considering some basic scenarios like GPU monitoring which could +// poll the GPU every 1 second, Rail-Gating cycle should not be triggered +// in between the pollings. So the timeout value is set to 500 ms. +// and the shortest duration to trigger Deferred-Idle would be +// IGPU_RG_PRECONDITION_CHECK_TIME + IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME +// +#define IGPU_RG_PRECONDITION_CHECK_TIME ((NvU64)500 * 1000 * 1000) + +// +// Timeout needed for back to back GC6 cycles. +// Timeout is kept same as the timeout selected for GC6 precondition check. +// There are cases where GPU is in GC6 and then kernel wakes GPU out of GC6 +// as part of say accessing pci tree through lspci and then again ask driver +// to put GPU in GC6 state after access to device info is done. +// Below are the reasons for having timeout limit for back to back GC6 cycle. +// 1> Less chip life cycle. +// 2> P-state is not reached to GC6 supported P-state, in some cycles. +// P-state case returns error to kernel, resulting in corrupted sysfs entry +// and then kernel never calls driver to put device in low power state. +// +#define GC6_CYCLE_IDLE_HOLDOFF_CHECK_TIME GC6_PRECONDITION_CHECK_TIME + +// +// Once GPU is found to be idle, driver will schedule another callback of +// smaller duration. Driver needs to be sure that methods that are present +// in host pipeline are flushed to the respective engines and engines become +// idle upon consumption. +// +#define GC6_BAR1_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME (200 * 1000 * 1000) + +// +// The purpose of this timeout value is similar to +// GC6_BAR1_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME. +// The difference between iGPU Rail-Gating and GC6 is that iGPU RG has a more +// aggressive timeout value of precondition check to quickly engage the +// Rail-Gating state on an embedded system. Meanwhile, the overall duration +// of Deferred-Idle should be larger than 1 second considering some polling +// scenarios which happens every 1 second. +// The value here is extended to 700 ms so the overall Deferred-Idle duration. +// could be keep within within the range: +// - Lower Bound: IGPU_RG_PRECONDITION_CHECK_TIME + IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME +// - Upper Bound: IGPU_RG_PRECONDITION_CHECK_TIME * 2 + IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME +// +#define IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME (700 * 1000 * 1000) + +// +// Cap Maximum FB allocation size for GCOFF. If regkey value is greater +// than this value then it will be capped to this value. +// +#define GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB 1024 + +// +// Safe/Fallback value of PEX reset delay for GCOFF/GPUOFF. +// Used when programming upstream port as per sku specific PEX reset delay from vbios fails +// +#define DEFAULT_GCOFF_PEXRST_DELAY 0 + +static void RmScheduleCallbackForIdlePreConditions(OBJGPU *); +static void RmScheduleCallbackForIdlePreConditionsUnderGpuLock(OBJGPU *); +static void RmScheduleCallbackToIndicateIdle(OBJGPU *); +static NvBool RmCheckForGcxSupportOnCurrentState(OBJGPU *); +static void RmScheduleCallbackToRemoveIdleHoldoff(OBJGPU *); +static void RmQueueIdleSustainedWorkitem(OBJGPU *); + +/*! + * @brief Wrapper that checks lock order for the dynamic power mutex. Locking + * order dictates that the GPU lock must not be taken before taking the dynamic + * power mutex. + * + * The GPU lock must not be held when calling this function. + * + * @param[in] nvp nv_priv_t pointer. + */ +static void acquireDynamicPowerMutex(nv_priv_t *nvp) +{ + /* + * Note that this may be called before nvp->pGpu has been initialized, for + * example at the very beginning of nv_start_device. + */ + NV_ASSERT_CHECKED(!nvp->pGpu || !rmDeviceGpuLockIsOwner(nvp->pGpu->gpuInstance)); + + portSyncMutexAcquire(nvp->dynamic_power.mutex); +} + +/*! + * @brief Wrapper to release the mutex, just for consistency with + * acquireDynamicPowerMutex() above. + * + * @param[in] nvp nv_priv_t pointer. + */ +static void releaseDynamicPowerMutex(nv_priv_t *nvp) +{ + portSyncMutexRelease(nvp->dynamic_power.mutex); +} + +/*! + * @brief: Helper function to get a string for the given state. + * + * @param[in] state Dynamic power state. + * + * @return String for the given state. + */ +const char *nv_dynamic_power_state_string( + nv_dynamic_power_state_t state +) +{ + switch (state) + { +#define DPCASE(_case) \ + case NV_DYNAMIC_POWER_STATE_ ## _case: return # _case; + + DPCASE(IN_USE); + DPCASE(IDLE_INSTANT); + DPCASE(IDLE_SUSTAINED); + DPCASE(IDLE_INDICATED); + DPCASE(UNKNOWN); +#undef DPCASE + } + NV_ASSERT(0); + return "UNEXPECTED"; +} + +/*! + * @brief: Helper function to transition the GPU represented by the 'nv' + * pointer from the power state 'old_state' to the power state 'new_state'. + * + * An atomic compare-and-swap is used; the function returns true if the + * operation was successful. If the function returns false, the state is + * unchanged. + * + * @param[in] nv nv_state_t pointer. + * @param[in] old_state Previous (current) state. + * @param[in] new_state New (requested) state. + * + * @return TRUE if the operation was successful; FALSE otherwise. + */ +NvBool nv_dynamic_power_state_transition( + nv_state_t *nv, + nv_dynamic_power_state_t old_state, + nv_dynamic_power_state_t new_state +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + NV_ASSERT(old_state != new_state); + + ct_assert(sizeof(nv_dynamic_power_state_t) == sizeof(NvS32)); + NvBool ret = portAtomicCompareAndSwapS32((PORT_ATOMIC NvS32*)&nvp->dynamic_power.state, + (NvS32)new_state, (NvS32)old_state); + + if (ret) + { + NV_PRINTF(LEVEL_INFO, "%s: state transition %s -> %s\n", + __FUNCTION__, + nv_dynamic_power_state_string(old_state), + nv_dynamic_power_state_string(new_state)); + } + else + { + NV_PRINTF(LEVEL_INFO, "%s: FAILED state transition %s -> %s\n", + __FUNCTION__, + nv_dynamic_power_state_string(old_state), + nv_dynamic_power_state_string(new_state)); + } + + return ret; + +} + +/*! + * @brief: Check if the GPU hardware appears to be idle. + * + * Assumes the GPU lock is held. + * + * @param[in] pGpu OBJGPU pointer. + * + * @return TRUE if the GPU appears to be currently idle; FALSE otherwise. + */ +static NvBool RmCanEnterGcxUnderGpuLock( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); + + /* + * If GC6 cannot be achieved (either GC6 is unsupported or the upstream port is not configured), + * Check for GCOFF prerequisites. Exclude the check for IGPU which has RG support. + */ + if ((!pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED) || + !nvp->gc6_upstream_port_configured) && + (!pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED))) + { + NvU64 usedFbSize = 0; + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + + /* + * In order to enter GCOFF, the FB memory threshold should be less than gcoff_max_fb_size + * and clients of RM should not hold the refcount to prevent GCOFF. + */ + if (!((memmgrGetUsedRamSize(pGpu, pMemoryManager, &usedFbSize) == NV_OK) && + (usedFbSize <= nvp->dynamic_power.gcoff_max_fb_size) && + (nvp->dynamic_power.clients_gcoff_disallow_refcount == 0))) + { + return NV_FALSE; + } + } + + // Check the instantaneous engine level idleness. + return RmCheckForGcxSupportOnCurrentState(pGpu); +} + +/*! + * @brief: Check the feasibility of GPU engaging in a GCx cycle. + * + * Takes the GPU lock. + * + * @param[in] pGpu OBJGPU pointer. + * + * @return TRUE if the GPU appears to be currently idle; FALSE otherwise. + */ +static NvBool RmCanEnterGcx( + OBJGPU *pGpu +) +{ + NvBool result = NV_FALSE; + NV_STATUS status; + GPU_MASK gpuMask; + + // LOCK: acquire per device lock + status = rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER, + &gpuMask); + if (status == NV_OK) + { + result = RmCanEnterGcxUnderGpuLock(pGpu); + // UNLOCK: release per device lock + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + } + + return result; +} + +/*! + * @brief: Force the GPU to be "not idle", if it had previously been indicated + * idle. This is triggered by GPU usage (i.e., writing through a revoked + * mapping), not tracked through the refcount maintained through + * os_{ref,unref}_dynamic_power. So, if we'd previously indicated the GPU was + * idle, we transition to the IDLE_INSTANT state. + * + * @param[in] gpuInstance GPU instance ID. + * @param[in] pArgs Unused callback closure. + */ +static void RmForceGpuNotIdle( + NvU32 gpuInstance, + void *pArgs +) +{ + OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_dynamic_power_state_t old_state; + NvBool ret; + + acquireDynamicPowerMutex(nvp); + + old_state = nvp->dynamic_power.state; + + switch (old_state) + { + case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED: + nv_indicate_not_idle(nv); + NV_ASSERT(nvp->dynamic_power.deferred_idle_enabled); + RmScheduleCallbackForIdlePreConditions(pGpu); + /* fallthrough */ + case NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED: + ret = nv_dynamic_power_state_transition(nv, old_state, NV_DYNAMIC_POWER_STATE_IDLE_INSTANT); + NV_ASSERT(ret); + break; + case NV_DYNAMIC_POWER_STATE_IDLE_INSTANT: + case NV_DYNAMIC_POWER_STATE_IN_USE: + break; + case NV_DYNAMIC_POWER_STATE_UNKNOWN: + NV_ASSERT(0); + break; + } + + /* + * Now that the GPU is guaranteed to not be powered off, we can resume + * servicing mmap requests as usual. + */ + nv_acquire_mmap_lock(nv); + nv_set_safe_to_mmap_locked(nv, NV_TRUE); + nv_release_mmap_lock(nv); + + releaseDynamicPowerMutex(nvp); +} + +/*! + * @brief: Work item to actually indicate the GPU is idle. This rechecks the + * preconditions one last time and verifies nothing has attempted to write + * through any mappings which were revoked. + * + * @param[in] gpuInstance GPU instance ID. + * @param[in] pArgs Unused callback closure. + */ +static void RmIndicateIdle( + NvU32 gpuInstance, + void *pArgs +) +{ + OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + acquireDynamicPowerMutex(nvp); + + nv_acquire_mmap_lock(nv); + + if (nv_get_all_mappings_revoked_locked(nv) && + nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED && + RmCanEnterGcx(pGpu)) + { + nv_set_safe_to_mmap_locked(nv, NV_FALSE); + nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED, + NV_DYNAMIC_POWER_STATE_IDLE_INDICATED); + nv_indicate_idle(nv); + } + else + { + RmScheduleCallbackForIdlePreConditions(pGpu); + } + + nv_release_mmap_lock(nv); + + releaseDynamicPowerMutex(nvp); +} + +/*! + * @brief: Helper function to trigger the GPU to be forced indicated + * "non-idle", which powers it on if it had been previously powered off, and + * allows mmap processing to continue. + * + * Triggered when a revoked mapping is accessed. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + */ +NV_STATUS NV_API_CALL rm_schedule_gpu_wakeup( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + void *fp; + NV_STATUS ret; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + NV_ENTER_RM_RUNTIME(sp, fp); + + ret = osQueueWorkItem(pGpu, + RmForceGpuNotIdle, + NULL, + (OsQueueWorkItemFlags){0}); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return ret; +} + +/*! + * @brief Function to increment/decrement refcount for GPU driving console. + * + * @params[in] nv nv_state_t pointer. + * @params[in] bIncrease If true, increase refcount else decrease. + */ +void +RmUpdateGc6ConsoleRefCount +( + nv_state_t *nv, + NvBool bIncrease +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + if (nv->console_device == NV_FALSE) + { + return; + } + + acquireDynamicPowerMutex(nvp); + + if (bIncrease) + { + /* + * Here refcount value should atleast be 2 due to below: + * 1> Refcount increased by NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT. + * 2> Refcount increased by the Rm entry points for current thread. + */ + if (nvp->dynamic_power.refcount <= 1) + { + NV_PRINTF(LEVEL_INFO, + "%s: Unexpected dynamic power refcount value\n", __FUNCTION__); + } + else + { + nvp->dynamic_power.refcount++; + } + } + else + { + /* + * Here refcount value should atleast be 3 due to below: + * 1> Refcount set by RM at the probe time for console device. + * 2> Refcount increased by NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT. + * 3> Refcount increased by the Rm entry points for current thread. + */ + if (nvp->dynamic_power.refcount <= 2) + { + NV_PRINTF(LEVEL_INFO, + "%s: Unexpected dynamic power refcount value\n", __FUNCTION__); + } + else + { + nvp->dynamic_power.refcount--; + } + } + + releaseDynamicPowerMutex(nvp); +} + +/*! + * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT RmControl + * request. + * + * @param[in] pSubdevice + * @param[in,out] pParams + * + * @return + * NV_OK Success + * NV_ERR_INVALID_ARGUMENT Invalid pParams->action + */ +NV_STATUS +subdeviceCtrlCmdOsUnixGc6BlockerRefCnt_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS *pParams +) +{ + OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); + + /* TODO: keep track of inc/dec on a per-client basis, clean up after + * clients which are unbalanced. */ + if (pParams->action == NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC) + { + return osRefGpuAccessNeeded(pGpu->pOsGpuInfo); + } + else if (pParams->action == NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC) + { + osUnrefGpuAccessNeeded(pGpu->pOsGpuInfo); + } + else + { + return NV_ERR_INVALID_ARGUMENT; + } + + return NV_OK; +} + +/*! + * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER RmControl + * request. It will perform dynamic power management for GPU + * HDA contoller. + * + * @param[in] pSubdevice + * @param[in,out] pParams + * + * @return + * NV_OK Success + */ +NV_STATUS +subdeviceCtrlCmdOsUnixAudioDynamicPower_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS *pParams +) +{ + OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); + + /* + * Ideally, we need to honor pParams->bEnter. If it is true, then send the + * request for suspending the HDA controller and if is false, then send the + * request for waking up the HDA controller. Currently, for unix based OS, + * we don't have any proper way to do this and it is implemented with + * workaround. This workaround will wake up the HDA controller for few + * seconds and then bring it back to suspended state. We are using this + * workaround for both the cases (bEnter is true or false). It will help + * in keeping HDA controller active for the duration, when GPU is + * processing HDA controller commands (like writing ELD data). + */ + nv_audio_dynamic_power(NV_GET_NV_STATE(pGpu)); + + return NV_OK; +} + +/*! + * @brief: Function to indicate if GC6/GC-OFF is supported + * or not on the SKU. + * + * @param[in] pGpu OBJGPU pointer. + * @param[in] bGcxTypeGc6 If true, returns string indicating GC6 support + * otherwise returns GC-OFF support. + * + * @return String indicating GC6/GC-OFF support status. + */ +static const char* RmGetGpuGcxSupport( + OBJGPU *pGpu, + NvBool bGcxTypeGC6 +) +{ + if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_UNIX_DYNAMIC_POWER_SUPPORTED)) + { + return "Not Supported"; + } + + if (bGcxTypeGC6) + { + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED)) + { + return "Supported"; + } + } + else + { + if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GCOFF_SUPPORTED)) || + (pGpu->getProperty(pGpu, PDB_PROP_GPU_LEGACY_GCOFF_SUPPORTED))) + { + return "Supported"; + } + } + + return "Not Supported"; +} + +/*! + * @brief Function to increment/decrement global Gcoff disallow refcount. + * + * @params[in] pOsGpuInfo OS specific GPU information pointer. + * @params[in] bIsGcoffDisallow If true, increase refcount else decrease. + */ +void osClientGcoffDisallowRefcount( + OS_GPU_INFO *pOsGpuInfo, + NvBool bIsGcoffDisallow +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(pOsGpuInfo); + + if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_NEVER) + { + return; + } + + acquireDynamicPowerMutex(nvp); + + if (bIsGcoffDisallow) + { + nvp->dynamic_power.clients_gcoff_disallow_refcount++; + } + else + { + nvp->dynamic_power.clients_gcoff_disallow_refcount--; + } + + releaseDynamicPowerMutex(nvp); +} + +/*! + * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF RmControl + * request. + * + * @param[in] pSubdevice + * @param[in,out] pParams + * + * @return + * NV_OK Success + * NV_ERR_INVALID_ARGUMENT Invalid pParams->action + * NV_ERR_INVALID_OBJECT_HANDLE Invalid Object handle + * NV_ERR_OBJECT_NOT_FOUND Object not found + */ +NV_STATUS +subdeviceCtrlCmdOsUnixAllowDisallowGcoff_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS *pParams +) +{ + NV_STATUS status; + OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); + + if (pParams->action == NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW) + { + if (pSubdevice->bGcoffDisallowed) + { + pSubdevice->bGcoffDisallowed = NV_FALSE; + osClientGcoffDisallowRefcount(pGpu->pOsGpuInfo, NV_FALSE); + status = NV_OK; + } + else + { + NV_ASSERT(0); + status = NV_ERR_INVALID_ARGUMENT; + } + } + else if (pParams->action == NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW) + { + if (!pSubdevice->bGcoffDisallowed) + { + pSubdevice->bGcoffDisallowed = NV_TRUE; + osClientGcoffDisallowRefcount(pGpu->pOsGpuInfo, NV_TRUE); + status = NV_OK; + } + else + { + NV_ASSERT(0); + status = NV_ERR_INVALID_ARGUMENT; + } + } + else + { + status = NV_ERR_INVALID_ARGUMENT; + } + + return status; +} + +/*! + * @brief Read and Parse the Dynamic Power Management regkey. + * This function modifies the default value of regkey to either enable + * or disable RTD3 as per System form factor and GPU architecture. + * + * @param[in] pNv nv_state_t pointer. + * @param[out] pRegkeyValue Original value of regkey. + * @param[out] pOption Modified value of regkey. + * + * @return NV_STATUS code. + */ + +#define NV_PMC_BOOT_42_CHIP_ID_GA102 0x00000172 + +static NV_STATUS +rmReadAndParseDynamicPowerRegkey +( + nv_state_t *pNv, + NvU32 *pRegkeyValue, + NvU32 *pOption +) +{ + NV_STATUS status; + NvU32 chipId; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + + status = osReadRegistryDword(NULL, NV_REG_DYNAMIC_POWER_MANAGEMENT, pRegkeyValue); + + if (status != NV_OK) + return status; + + // If User has set some value, honor that value + if (*pRegkeyValue != NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT) + { + *pOption = *pRegkeyValue; + return NV_OK; + } + + chipId = decodePmcBoot42ChipId(pNvp->pmc_boot_42); + + // From GA102+, we enable RTD3 only if system is found to be Notebook + if ((chipId >= NV_PMC_BOOT_42_CHIP_ID_GA102) && + (rm_is_system_notebook())) + { + *pOption = NV_REG_DYNAMIC_POWER_MANAGEMENT_FINE; + return NV_OK; + } + + // For Tegra PCI iGPU, support GPU Rail-Gating feature + if (pNv->supports_tegra_igpu_rg) + { + *pOption = NV_REG_DYNAMIC_POWER_MANAGEMENT_FINE; + return NV_OK; + } + + *pOption = NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER; + return NV_OK; +} +#undef NV_PMC_BOOT_42_CHIP_ID_GA102 + +/*! + * @brief Initialize state related to Tegra iGPU dynamic power management. + * Called once per GPU during driver initialization. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + */ +void NV_API_CALL rm_init_tegra_dynamic_power_management( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + void *fp; + + if (!nv->supports_tegra_igpu_rg) + return; + + NV_ENTER_RM_RUNTIME(sp,fp); + + portMemSet(&nvp->dynamic_power, 0, sizeof(nvp->dynamic_power)); + + nv->is_tegra_pci_igpu_rg_enabled = NV_FALSE; + + NV_PRINTF(LEVEL_INFO, + "NVRM: Tegra PCI iGPU detected, Rail-Gating is supported.\n"); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void rm_check_s0ix_regkey_and_platform_support(void) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + NvU32 data; + NvBool status = ((nv_platform_supports_s0ix()) && + ((osReadRegistryDword(NULL, NV_REG_ENABLE_S0IX_POWER_MANAGEMENT, &data) == NV_OK) && (data == 1))); + + pSys->setProperty(pSys, PDB_PROP_SYS_SUPPORTS_S0IX, status); +} + +/*! + * @brief Initialize state related to dynamic power management. + * Called once per GPU during driver initialization. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + */ +void NV_API_CALL rm_init_dynamic_power_management( + nvidia_stack_t *sp, + nv_state_t *nv, + NvBool bPr3AcpiMethodPresent +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 option = NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER; + NvU32 gcOffMaxFbSizeMb; + void *fp; + NvBool bUefiConsole; + NvU32 status; + NvU32 regkeyValue; + NvBool bIsTegraPciIgpuRgSupported = nv->supports_tegra_igpu_rg; + + NV_ENTER_RM_RUNTIME(sp,fp); + + /* + * Program an impossible value so that we show correct status + * during procfs read of runtime D3 status. + */ + nvp->dynamic_power.dynamic_power_regkey = 0xFFFFFFFFu; + + /* + * Dynamic power management will be disabled in either of + * following 3 cases: + * 1. The driver does not support dynamic PM for this + * distro/kernel version. + * 2. _PR3 ACPI method is not supported by upstream port of GPU. + * 3. This feature is not enabled by regkey. + * Also see NOTE + */ + status = rmReadAndParseDynamicPowerRegkey(nv, ®keyValue, &option); + + if (status == NV_OK) + { + nvp->dynamic_power.dynamic_power_regkey = regkeyValue; + } + + nvp->pr3_acpi_method_present = bPr3AcpiMethodPresent; + if (!nv_dynamic_power_available(nv) || (status != NV_OK) || + (!bPr3AcpiMethodPresent && !bIsTegraPciIgpuRgSupported)) + { + NV_PRINTF(LEVEL_NOTICE, + "%s: Disabling dynamic power management either due to lack" + " of system support or due to error (%d) in reading regkey." + "\n", __FUNCTION__, status); + option = NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER; + } + + /* + * Read the OS registry for Maximum FB size during GCOFF based dynamic PM. + * This will be capped to GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB. + */ + if (osReadRegistryDword(NULL, + NV_REG_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD, + &gcOffMaxFbSizeMb) == NV_OK) + { + if (gcOffMaxFbSizeMb > GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB) + gcOffMaxFbSizeMb = GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB; + } + else + { + gcOffMaxFbSizeMb = GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB; + } + + nvp->dynamic_power.gcoff_max_fb_size = + (NvU64)gcOffMaxFbSizeMb * 1024 * 1024; + + nvp->dynamic_power.mutex = portSyncMutexCreate(portMemAllocatorGetGlobalNonPaged()); + if (nvp->dynamic_power.mutex == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to create power mutex\n", __FUNCTION__); + nvp->dynamic_power.mode = NV_DYNAMIC_PM_NEVER; + goto done; + } + + switch (option) + { + case NV_REG_DYNAMIC_POWER_MANAGEMENT_FINE: + nvp->dynamic_power.mode = NV_DYNAMIC_PM_FINE; + break; + case NV_REG_DYNAMIC_POWER_MANAGEMENT_COARSE: + nvp->dynamic_power.mode = NV_DYNAMIC_PM_COARSE; + break; + default: + nv_printf(LEVEL_ERROR, + "NVRM: Unknown DynamicPowerManagement value '%u' specified; " + "disabling dynamic power management.\n", option); + // fallthrough + case NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER: + nvp->dynamic_power.mode = NV_DYNAMIC_PM_NEVER; + bIsTegraPciIgpuRgSupported = NV_FALSE; + break; + } + + // Enable RTD3 infrastructure from OS side. + if ((nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE) && + (nvp->dynamic_power.dynamic_power_regkey == NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT)) + { + // Enable Dynamic Rail-Gating for Tegra PCI iGPU + if (bIsTegraPciIgpuRgSupported) + { + if (nv_pci_tegra_pm_init(nv)) + { + nv_printf(NV_DBG_INFO, "NVRM: Tegra PCI iGPU railgating is enabled\n"); + nv->is_tegra_pci_igpu_rg_enabled = NV_TRUE; + + // Allow runtime suspend for Tegra PCI iGPU device with RG successfully enabled. + nv_allow_runtime_suspend(nv); + } + } + else + { + // Allow runtime suspend for the GPU which is not Tegra PCI iGPU device. + nv_allow_runtime_suspend(nv); + } + } + + // UEFI case: where console is driven by GOP driver. + bUefiConsole = rm_get_uefi_console_status(nv); + + nv->console_device = bUefiConsole || nv->primary_vga; + + // + // Initialize refcount as 1 and state as IN_USE. + // rm_enable_dynamic_power_management() will decrease the refcount + // and change state to IDLE_INDICATED, if all prerequisites are met. + // + nvp->dynamic_power.refcount = 1; + nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_UNKNOWN, + NV_DYNAMIC_POWER_STATE_IN_USE); +done: + NV_EXIT_RM_RUNTIME(sp,fp); +} + +/*! + * @brief Clean up state related to dynamic power management. + * Called during driver removal. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + */ +void NV_API_CALL rm_cleanup_dynamic_power_management( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + void *fp; + NvBool ret; + + NV_ENTER_RM_RUNTIME(sp,fp); + + // Disable RTD3 infrastructure from OS side. + if (((nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE) && + (nvp->dynamic_power.dynamic_power_regkey == NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT)) || + nv->is_tegra_pci_igpu_rg_enabled) + { + nv_disallow_runtime_suspend(nv); + + // Disable Dynamic Rail-Gating for Tegra PCI iGPU + if (nv->is_tegra_pci_igpu_rg_enabled) + { + nv_pci_tegra_pm_deinit(nv); + nv->is_tegra_pci_igpu_rg_enabled = NV_FALSE; + nv_printf(NV_DBG_INFO, "NVRM: Tegra PCI iGPU railgating is disabled\n"); + } + } + + nv_dynamic_power_state_t old_state = nvp->dynamic_power.state; + + switch (old_state) + { + case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED: + nv_indicate_not_idle(nv); + /* fallthrough */ + case NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED: + case NV_DYNAMIC_POWER_STATE_IDLE_INSTANT: + ret = nv_dynamic_power_state_transition(nv, old_state, + NV_DYNAMIC_POWER_STATE_IN_USE); + /* + * Nothing else should be running asynchronous to teardown that could + * change the state so this should always succeed. + */ + NV_ASSERT(ret); + break; + case NV_DYNAMIC_POWER_STATE_IN_USE: + break; + case NV_DYNAMIC_POWER_STATE_UNKNOWN: + NV_ASSERT(0); + break; + } + + if (nvp->dynamic_power.mutex) + { + portSyncMutexDestroy(nvp->dynamic_power.mutex); + nvp->dynamic_power.mutex = NULL; + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +/*! + * @brief Initialize the dynamic power management refcount and enable GPUOFF, + * if all prerequisites are met. + * Called once per GPU during driver initialization. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + */ +void NV_API_CALL rm_enable_dynamic_power_management( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + /* + * If dynamic power management is enabled and device is not driving console, + * then dynamic power management can be engaged (since we're not + * using the GPU yet, so it's okay to power it off). + */ + if ((nvp->dynamic_power.mode != NV_DYNAMIC_PM_NEVER) && !nv->console_device) + { + os_unref_dynamic_power(nv, NV_DYNAMIC_PM_COARSE); + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +/* + * @brief Increment the dynamic power refcount to prevent the GPU from being + * powered off until a corresponding os_unref_dynamic_power() call is made. + * These two calls must always be balanced in pairs. + * + * This call may block until the GPU is powered on. + * + * The caller must not be holding the GPU lock, but may be holding the API + * lock. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] mode Dynamic power mode this refcount bump is used for. + */ +NV_STATUS +os_ref_dynamic_power( + nv_state_t *nv, + nv_dynamic_power_mode_t mode +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NV_STATUS status = NV_OK; + NvS32 ref; + + if (nvp == NULL) + { + NV_ASSERT(nv->removed); + return NV_OK; + } + + if (mode > nvp->dynamic_power.mode) + { + return NV_OK; + } + + acquireDynamicPowerMutex(nvp); + + ref = nvp->dynamic_power.refcount++; + + NV_ASSERT(ref >= 0); + + if (ref > 0) + { + NV_ASSERT(nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IN_USE); + } + else + { + /* + * Just transitioned from 0 -> 1. + * The current dynamic power state could be any of: + * + * - IDLE_INDICATED: call nv_indicate_not_idle() to ensure + * the GPU is at full power and ready to be used. + * + * - IDLE_INSTANT/IDLE_SUSTAINED: transition to IN_USE. + * Note that in these states, callbacks may be operating + * asynchronously, so care has to be taken to retry the write if it + * fails. + */ + + nv_dynamic_power_state_t old_state = nvp->dynamic_power.state; + NvBool ret; + + switch (old_state) + { + default: + case NV_DYNAMIC_POWER_STATE_IN_USE: + NV_PRINTF(LEVEL_ERROR, "Unexpected dynamic power state 0x%x\n", + old_state); + /* fallthrough */ + case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED: + status = nv_indicate_not_idle(nv); + if (status != NV_OK) + { + nvp->dynamic_power.refcount--; + break; + } + if (nvp->dynamic_power.deferred_idle_enabled) + { + RmScheduleCallbackForIdlePreConditions(NV_GET_NV_PRIV_PGPU(nv)); + } + /* fallthrough */ + case NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED: + case NV_DYNAMIC_POWER_STATE_IDLE_INSTANT: + do { + ret = nv_dynamic_power_state_transition(nv, old_state, NV_DYNAMIC_POWER_STATE_IN_USE); + if (!ret) + { + old_state = nvp->dynamic_power.state; + NV_ASSERT(old_state == NV_DYNAMIC_POWER_STATE_IDLE_INSTANT || + old_state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED); + } + } while (!ret); + break; + } + } + + releaseDynamicPowerMutex(nvp); + + return status; +} + +/* + * @brief This function queries the dynamic boost support. + * This function returns -1 in case it failed to query the + * dynamic boost support. It returns 1 if dynamic boost is + * supported and returns 0 if dynamic boost is not supported. + * + * The caller must hold the API lock before calling this + * function. + * + * @param[in] nv nv_state_t pointer. + */ +static int os_get_dynamic_boost_support( + nv_state_t *nv +) +{ + NV_STATUS status = NV_OK; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS *pNvpcfParams; + OBJGPU *pGpu; + int ret; + + status = os_alloc_mem((void**)&pNvpcfParams, sizeof(*pNvpcfParams)); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Failed to allocate memory\n"); + ret = -1; + goto end; + } + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + pNvpcfParams->gpuId = pGpu->gpuId; + pNvpcfParams->subFunc = + NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_SUPPORTED_CASE; + status = pRmApi->Control(pRmApi, + nv->rmapi.hClient, + nv->rmapi.hClient, + NV0000_CTRL_CMD_SYSTEM_NVPCF_GET_POWER_MODE_INFO, + (void*)pNvpcfParams, + sizeof(*pNvpcfParams)); + if (status == NV_OK) + { + ret = 1; + } + else if (status == NV_ERR_NOT_SUPPORTED) + { + ret = 0; + } + else + { + ret = -1; + } + + os_free_mem(pNvpcfParams); +end: + return ret; +} + +/*! + * @brief Wrapper around os_ref_dynamic_power() suitable for use from the + * per-OS layers. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + * @param[in] mode Dynamic power mode this refcount bump is used for. + */ +NV_STATUS NV_API_CALL rm_ref_dynamic_power( + nvidia_stack_t *sp, + nv_state_t *nv, + nv_dynamic_power_mode_t mode +) +{ + NV_STATUS status; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + status = os_ref_dynamic_power(nv, mode); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +/*! + * @brief Decrement the dynamic power refcount to release an earlier + * requirement that the GPU be powered on made by calling + * os_ref_dynamic_power(). + * These two calls must always be balanced in pairs. + * + * This call may block until the GPU is powered on. + * + * The caller must not be holding the GPU lock, but may be holding the API + * lock. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] mode Dynamic power mode this refcount bump is used for. + */ +void +os_unref_dynamic_power( + nv_state_t *nv, + nv_dynamic_power_mode_t mode +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvS32 ref; + + if (nvp == NULL) + { + NV_ASSERT(nv->removed); + return; + } + + if (mode > nvp->dynamic_power.mode) + { + return; + } + + acquireDynamicPowerMutex(nvp); + + ref = --nvp->dynamic_power.refcount; + + NV_ASSERT(ref >= 0); + + if (ref == 0) { + NvBool ret; + + /* + * Just transitioned from 1 -> 0. We should have been in the IN_USE + * state previously; now we transition to an idle state: "instant" idle + * if deferred idle is enabled; otherwise indicate idle immediately. + * + * Nothing should be asynchronously changing the state from IN_USE, + * so the state transitions are not expected to fail. + */ + + NV_ASSERT(nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IN_USE); + + if (nvp->dynamic_power.deferred_idle_enabled) + { + ret = nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IN_USE, + NV_DYNAMIC_POWER_STATE_IDLE_INSTANT); + NV_ASSERT(ret); + } + else + { + nv_indicate_idle(nv); + ret = nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IN_USE, + NV_DYNAMIC_POWER_STATE_IDLE_INDICATED); + NV_ASSERT(ret); + } + } + + releaseDynamicPowerMutex(nvp); +} + +/*! + * @brief Wrapper around os_unref_dynamic_power() suitable for use from the + * per-OS layers. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + * @param[in] mode Dynamic power mode this refcount bump is used for. + */ +void NV_API_CALL rm_unref_dynamic_power( + nvidia_stack_t *sp, + nv_state_t *nv, + nv_dynamic_power_mode_t mode +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + os_unref_dynamic_power(nv, mode); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +/*! + * @brief Wrapper around os_ref_dynamic_power() suitable for use from core RM. + * + * @params[in] pOsGpuInfo OS specific GPU information pointer + */ +NV_STATUS osRefGpuAccessNeeded( + OS_GPU_INFO *pOsGpuInfo +) +{ + return os_ref_dynamic_power(pOsGpuInfo, NV_DYNAMIC_PM_FINE); +} + +/*! + * @brief Wrapper around os_unref_dynamic_power() suitable for use from core RM. + * + * @params[in] pOsGpuInfo OS specific GPU information pointer + */ +void osUnrefGpuAccessNeeded( + OS_GPU_INFO *pOsGpuInfo +) +{ + os_unref_dynamic_power(pOsGpuInfo, NV_DYNAMIC_PM_FINE); +} + +/*! + * @brief Check if GCx is supported on current pstate and if engines are idle. + * + * @param[in] pGpu OBJGPU pointer. + */ +static NvBool RmCheckForGcxSupportOnCurrentState( + OBJGPU *pGpu +) +{ + NV_STATUS status = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS entryPrerequisiteParams; + + portMemSet(&entryPrerequisiteParams, 0, sizeof(entryPrerequisiteParams)); + + status = pRmApi->Control(pRmApi, + nv->rmapi.hClient, + nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_INTERNAL_GCX_ENTRY_PREREQUISITE, + (void*)&entryPrerequisiteParams, + sizeof(entryPrerequisiteParams)); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "NVRM, Failed to get GCx pre-requisite, status=0x%x\n", + status); + return NV_FALSE; + } + + return entryPrerequisiteParams.bIsGC6Satisfied || entryPrerequisiteParams.bIsGCOFFSatisfied; +} + +/*! + * @brief: Work item to actually remove the idle holdoff which was applied + * on GC6 exit. + * This workitem is scheduled after the timeout limit for idle holdoff is + * finished. + * + * Queue with lock flags: + * bLockGpuGroupSubdevice + * + * @param[in] gpuInstance GPU instance ID. + * @param[in] pArgs Unused callback closure. + */ +static void RmRemoveIdleHoldoff( + NvU32 gpuInstance, + void *pArgs +) +{ + OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + if (nvp->dynamic_power.b_idle_holdoff == NV_TRUE) + { + if ((RmCheckForGcxSupportOnCurrentState(pGpu) == NV_TRUE) || + (nvp->dynamic_power.idle_precondition_check_callback_scheduled)) + { + nv_indicate_idle(nv); + nvp->dynamic_power.b_idle_holdoff = NV_FALSE; + } + else + { + RmScheduleCallbackToRemoveIdleHoldoff(pGpu); + } + } +} + +/*! + * @brief Timer callback to schedule a work item to remove idle hold off + * that was applied after GC6 exit. + * + * @param[in] pCallbackData OBJGPU pointer (with void pointer type). + */ +static void timerCallbackToRemoveIdleHoldoff( + void *pCallbackData +) +{ + NV_STATUS status = NV_OK; + OBJGPU *pGpu = reinterpretCast(pCallbackData, OBJGPU *); + + status = osQueueWorkItem(pGpu, + RmRemoveIdleHoldoff, + NULL, + (OsQueueWorkItemFlags){.bLockGpuGroupSubdevice = NV_TRUE}); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "Queuing of remove idle holdoff work item failed with status : 0x%x\n", + status); + } +} + +/*! + * @brief Timer callback to check if all idle conditions remain met and + * if so schedule a work item to indicate the GPU is idle. + * + * @param[in] pCallbackData OBJGPU pointer (with void pointer type). + */ +static void timerCallbackToIndicateIdle( + void *pCallbackData +) +{ + OBJGPU *pGpu = reinterpretCast(pCallbackData, OBJGPU *); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + nv_acquire_mmap_lock(nv); + + if (nv_get_all_mappings_revoked_locked(nv) && + nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED && + RmCanEnterGcxUnderGpuLock(pGpu)) + { + osQueueWorkItem(pGpu, + RmIndicateIdle, + NULL, + (OsQueueWorkItemFlags){0}); + } + else + { + if (nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED) + { + /* + * Something used the GPU between when we last sampled and now. + * Go back to the "instant" state to start waiting from scratch. + */ + nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED, + NV_DYNAMIC_POWER_STATE_IDLE_INSTANT); + } + + RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu); + } + + nv_release_mmap_lock(nv); +} + +/*! + * @brief Timer callback to periodically check if all idle preconditions are + * met. If so, the dynamic power state is advanced from IDLE_INSTANT to + * IDLE_SUSTAINED, or (if already in the IDLE_SUSTAINED state) mappings are + * revoked and timerCallbackToIndicateIdle() is scheduled. + * + * @param[in] pCallbackData OBJGPU pointer (with void pointer type). + */ +static void timerCallbackForIdlePreConditions( + void *pCallbackData +) +{ + OBJGPU *pGpu = reinterpretCast(pCallbackData, OBJGPU *); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + nvp->dynamic_power.idle_precondition_check_callback_scheduled = NV_FALSE; + + /* + * Note: It is not safe to take the dynamic power mutex here. + * Lock ordering dictates that the dynamic power mutex must be taken before + * the GPU lock, and this function is called with the GPU lock held. + * + * Therefore, this function is careful about *only* triggering specific + * transitions: + * NV_DYNAMIC_POWER_STATE_IDLE_INSTANT -> NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED + * NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED -> NV_DYNAMIC_POWER_STATE_IDLE_INSTANT + * + * Other codepaths are resilient to these state transitions occurring + * asynchronously. The state transition itself is done with an atomic + * compare and swap, so there should be no danger of inadvertently + * triggering a transition from a different state due to a + * read-modify-write or "time of check to time of use" (TOCTTOU) race + * condition. + */ + + if (nvp->dynamic_power.state != NV_DYNAMIC_POWER_STATE_IN_USE) + { + if (RmCanEnterGcxUnderGpuLock(pGpu)) + { + nv_dynamic_power_state_t state = nvp->dynamic_power.state; + switch (state) + { + case NV_DYNAMIC_POWER_STATE_UNKNOWN: + NV_ASSERT(0); + /* fallthrough */ + case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED: + NV_PRINTF(LEVEL_ERROR, "unexpected dynamic power state 0x%x\n", + nvp->dynamic_power.state); + /* fallthrough */ + case NV_DYNAMIC_POWER_STATE_IN_USE: + break; + case NV_DYNAMIC_POWER_STATE_IDLE_INSTANT: + nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IDLE_INSTANT, + NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED); + break; + case NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED: + RmQueueIdleSustainedWorkitem(pGpu); + return; + } + + } + else + { + if (nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED) + { + /* + * Something used the GPU between when we last sampled and now. + * Go back to the "instant" state to start waiting from scratch. + */ + nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED, + NV_DYNAMIC_POWER_STATE_IDLE_INSTANT); + } + } + } + RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu); +} + +/*! + * @brief Cancel any in-flight callback to remove idle holdoff. + * + * @param[in] pGpu OBJGPU pointer. + */ +static void RmCancelCallbackToRemoveIdleHoldoff( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + TMR_EVENT_GENERAL_PARAMS cancelParams; + + if (NvP64_VALUE(nvp->dynamic_power.remove_idle_holdoff) != NULL) + { + portMemSet(&cancelParams, 0, sizeof(cancelParams)); + + cancelParams.pEvent = nvp->dynamic_power.remove_idle_holdoff; + + tmrCtrlCmdEventCancel(pGpu, &cancelParams); + } +} + +/*! + * @brief Cancel any in-flight timer callbacks. + * + * @param[in] pGpu OBJGPU pointer. + */ +static void RmCancelDynamicPowerCallbacks( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + TMR_EVENT_GENERAL_PARAMS cancelParams; + + if (NvP64_VALUE(nvp->dynamic_power.idle_precondition_check_event) != NULL) + { + // Cancel precondition check callback. + portMemSet(&cancelParams, 0, sizeof(cancelParams)); + + cancelParams.pEvent = nvp->dynamic_power.idle_precondition_check_event; + + tmrCtrlCmdEventCancel(pGpu, &cancelParams); + } + + if (NvP64_VALUE(nvp->dynamic_power.indicate_idle_event) != NULL) + { + // Cancel callback to indicate idle + portMemSet(&cancelParams, 0, sizeof(cancelParams)); + + cancelParams.pEvent = nvp->dynamic_power.indicate_idle_event; + + tmrCtrlCmdEventCancel(pGpu, &cancelParams); + } + + RmCancelCallbackToRemoveIdleHoldoff(pGpu); + + nvp->dynamic_power.idle_precondition_check_callback_scheduled = NV_FALSE; +} + +/*! + * @brief Free timer events allocated by CreateDynamicPowerCallbacks() + * + * @param[in] pGpu OBJGPU pointer. + */ +static void RmDestroyDynamicPowerCallbacks( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + TMR_EVENT_GENERAL_PARAMS destroyParams; + + if (NvP64_VALUE(nvp->dynamic_power.idle_precondition_check_event) != NULL) + { + // Destroy precondition check callback. + portMemSet(&destroyParams, 0, sizeof(destroyParams)); + + destroyParams.pEvent = nvp->dynamic_power.idle_precondition_check_event; + + tmrCtrlCmdEventDestroy(pGpu, &destroyParams); + nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL; + } + + if (NvP64_VALUE(nvp->dynamic_power.indicate_idle_event) != NULL) + { + // Destroy callback to indicate idle + portMemSet(&destroyParams, 0, sizeof(destroyParams)); + + destroyParams.pEvent = nvp->dynamic_power.indicate_idle_event; + + tmrCtrlCmdEventDestroy(pGpu, &destroyParams); + nvp->dynamic_power.indicate_idle_event = NvP64_NULL; + } + + if (NvP64_VALUE(nvp->dynamic_power.remove_idle_holdoff) != NULL) + { + // Destroy callback to decrease kernel refcount + portMemSet(&destroyParams, 0, sizeof(destroyParams)); + + destroyParams.pEvent = nvp->dynamic_power.remove_idle_holdoff; + + tmrCtrlCmdEventDestroy(pGpu, &destroyParams); + nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL; + } + + nvp->dynamic_power.deferred_idle_enabled = NV_FALSE; +} + +/* + * @brief Adds a GPU to dynamicPowerSupportGpuMask + * + * @param[in] instance + */ +static void AddGpuDynamicPowerSupported +( + NvU32 instance +) +{ + dynamicPowerSupportGpuMask |= NVBIT(instance); +} + +/* + * @brief Removes a GPU from dynamicPowerSupportGpuMask + * + * @param[in] instance + */ +static void RemoveGpuDynamicPowerSupported +( + NvU32 instance +) +{ + dynamicPowerSupportGpuMask &= ~NVBIT(instance); +} + +/* + * @brief queries dynamicPowerSupportGpuMask + * + * @param[in] void + */ +NvU32 osGetDynamicPowerSupportMask +( + void +) +{ + return dynamicPowerSupportGpuMask; +} + +/*! + * @brief Cancel any outstanding callbacks, and free RM resources allocated by + * RmInitDeferredDynamicPowerManagement(). + * + * @param[in] nv nv_state_t pointer. + */ +void RmDestroyDeferredDynamicPowerManagement( + nv_state_t *nv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + // + // Re-instate the dynamic power mode to fine if it was + // disabled due to lack of platform support + // + if (nvp->dynamic_power.b_fine_not_supported == NV_TRUE) + { + nvp->dynamic_power.mode = NV_DYNAMIC_PM_FINE; + nvp->dynamic_power.b_fine_not_supported = NV_FALSE; + } + + // + // Before cancelling and destroying the callbacks, make sure to + // remove the idle holfoff that was applied during gc6 exit. + // + if (nvp->dynamic_power.b_idle_holdoff == NV_TRUE) + { + nv_indicate_idle(nv); + nvp->dynamic_power.b_idle_holdoff = NV_FALSE; + } + + RmCancelDynamicPowerCallbacks(pGpu); + RmDestroyDynamicPowerCallbacks(pGpu); + RemoveGpuDynamicPowerSupported(gpuGetInstance(pGpu)); +} + +/*! + * @brief Allocate timer events to call the functions: + * timerCallbackForIdlePreConditions() + * timerCallbackToIndicateIdle() + * + * The callbacks must be scheduled separately. + * + * @param[in] pGpu OBJGPU pointer. + */ +static NV_STATUS CreateDynamicPowerCallbacks( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + TMR_EVENT_SET_PARAMS createEventParams; + NV_STATUS status; + + // Creating PreCondition check callback. + portMemSet(&createEventParams, 0, sizeof(createEventParams)); + + createEventParams.ppEvent = &(nvp->dynamic_power.idle_precondition_check_event); + createEventParams.pTimeProc = NV_PTR_TO_NvP64(timerCallbackForIdlePreConditions); + createEventParams.pCallbackData = NV_PTR_TO_NvP64(pGpu); + createEventParams.flags = TMR_FLAG_USE_OS_TIMER; + + status = tmrCtrlCmdEventCreate(pGpu, &createEventParams); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "Error creating dynamic power precondition check callback\n"); + nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL; + nvp->dynamic_power.indicate_idle_event = NvP64_NULL; + nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL; + return status; + } + + // Create callback to indicate idle.. + portMemSet(&createEventParams, 0, sizeof(createEventParams)); + + createEventParams.ppEvent = &(nvp->dynamic_power.indicate_idle_event); + createEventParams.pTimeProc = NV_PTR_TO_NvP64(timerCallbackToIndicateIdle); + createEventParams.pCallbackData = NV_PTR_TO_NvP64(pGpu); + createEventParams.flags = TMR_FLAG_USE_OS_TIMER; + + status = tmrCtrlCmdEventCreate(pGpu, &createEventParams); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "Error creating callback to indicate GPU idle\n"); + nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL; + nvp->dynamic_power.indicate_idle_event = NvP64_NULL; + nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL; + return status; + } + + // Create callback to remove idle holdoff + portMemSet(&createEventParams, 0, sizeof(createEventParams)); + + createEventParams.ppEvent = &(nvp->dynamic_power.remove_idle_holdoff); + createEventParams.pTimeProc = NV_PTR_TO_NvP64(timerCallbackToRemoveIdleHoldoff); + createEventParams.pCallbackData = NV_PTR_TO_NvP64(pGpu); + createEventParams.flags = TMR_FLAG_USE_OS_TIMER; + + status = tmrCtrlCmdEventCreate(pGpu, &createEventParams); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "Error creating callback to decrease kernel refcount\n"); + nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL; + nvp->dynamic_power.indicate_idle_event = NvP64_NULL; + nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL; + return status; + } + + return NV_OK; +} + +/*! + * @brief Schedule the timerCallbackForIdlePreConditions() function to be called. + * + * If it is already scheduled, the function is a noop. + * + * The GPU lock must be held when calling this function. + * + * @param[in] pGpu OBJGPU pointer. + */ +static void RmScheduleCallbackForIdlePreConditionsUnderGpuLock( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + TMR_EVENT_SCHEDULE_PARAMS scheduleEventParams; + NV_STATUS status; + + NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); + + if (nvp->dynamic_power.idle_precondition_check_callback_scheduled) + { + return; + } + + if (NvP64_VALUE(nvp->dynamic_power.idle_precondition_check_event) != NULL) + { + portMemSet(&scheduleEventParams, 0, sizeof(scheduleEventParams)); + + scheduleEventParams.pEvent = nvp->dynamic_power.idle_precondition_check_event; + scheduleEventParams.bUseTimeAbs = NV_FALSE; + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED)) + { + /* + * iGPU which supports Rail-Gating may want shorter idleness + * threshold to engage the deferred-idle transition. + * Set the threshold for precondition check to 500 ms. + */ + scheduleEventParams.timeNs = IGPU_RG_PRECONDITION_CHECK_TIME; + } + else + { + scheduleEventParams.timeNs = GC6_PRECONDITION_CHECK_TIME; + } + + status = tmrCtrlCmdEventSchedule(pGpu, &scheduleEventParams); + + if (status == NV_OK) + { + nvp->dynamic_power.idle_precondition_check_callback_scheduled = NV_TRUE; + } + else + { + NV_PRINTF(LEVEL_ERROR, "Error scheduling precondition callback\n"); + } + } +} + +/*! + * @brief Schedule the timerCallbackForIdlePreConditions() function to be called. + * + * If it is already scheduled, the function is a noop. + * + * This function takes the GPU lock. + * + * @param[in] pGpu OBJGPU pointer. + */ +static void RmScheduleCallbackForIdlePreConditions( + OBJGPU *pGpu +) +{ + NV_STATUS status; + GPU_MASK gpuMask; + + // LOCK: acquire per device lock + status = rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER, + &gpuMask); + if (status == NV_OK) + { + RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu); + // UNLOCK: release per device lock + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + } +} + +/*! + * @brief Schedule the timerCallbackToIndicateIdle() function to be called. + * + * The GPU lock must be held when calling this function. + * + * @param[in] pGpu OBJGPU pointer. + */ +static void RmScheduleCallbackToIndicateIdle( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + TMR_EVENT_SCHEDULE_PARAMS scheduleEventParams; + NV_STATUS status; + + NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); + + if (NvP64_VALUE(nvp->dynamic_power.indicate_idle_event) != NULL) + { + portMemSet(&scheduleEventParams, 0, sizeof(scheduleEventParams)); + + scheduleEventParams.pEvent = nvp->dynamic_power.indicate_idle_event; + scheduleEventParams.bUseTimeAbs = NV_FALSE; + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED)) + { + /* + * iGPU which supports Rail-Gating may want shorter idleness + * threshold to engage the deferred-idle transition. + * Set the threshold for idle indicate to 700 ms. + */ + scheduleEventParams.timeNs = IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME; + } + else + { + scheduleEventParams.timeNs = GC6_BAR1_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME; + } + + status = tmrCtrlCmdEventSchedule(pGpu, &scheduleEventParams); + + if (status != NV_OK) + NV_PRINTF(LEVEL_ERROR, "Error scheduling indicate idle callback\n"); + } +} + +/*! + * @brief Schedule the timerCallbackToRemoveIdleHoldoff() function. + * + * The GPU lock must be held when calling this function. + * + * @param[in] pGpu OBJGPU pointer. + */ +static void RmScheduleCallbackToRemoveIdleHoldoff( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + TMR_EVENT_SCHEDULE_PARAMS scheduleEventParams; + NV_STATUS status; + + NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); + + if (NvP64_VALUE(nvp->dynamic_power.remove_idle_holdoff) != NULL) + { + portMemSet(&scheduleEventParams, 0, sizeof(scheduleEventParams)); + + scheduleEventParams.pEvent = nvp->dynamic_power.remove_idle_holdoff; + scheduleEventParams.timeNs = GC6_CYCLE_IDLE_HOLDOFF_CHECK_TIME; + scheduleEventParams.bUseTimeAbs = NV_FALSE; + + status = tmrCtrlCmdEventSchedule(pGpu, &scheduleEventParams); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "Error scheduling kernel refcount decrement callback\n"); + } + else + { + nvp->dynamic_power.b_idle_holdoff = NV_TRUE; + } + } +} + +/*! + * @brief Check if the system supports RTD3-GC6. + * + * @param[in] pGpu OBJGPU pointer. + */ +static NvBool RmCheckRtd3GcxSupport( + nv_state_t *pNv, + NvBool *bGC6Support, + NvBool *bGCOFFSupport +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV_STATUS status; + NV0080_CTRL_GPU_GET_VIRTUALIZATION_MODE_PARAMS virtModeParams = { 0 }; + if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_UNIX_DYNAMIC_POWER_SUPPORTED)) + { + NV_PRINTF(LEVEL_NOTICE, "RTD3 is not supported for this arch\n"); + return NV_FALSE; + } + + *bGC6Support = pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED); + *bGCOFFSupport = pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GCOFF_SUPPORTED) || + pGpu->getProperty(pGpu, PDB_PROP_GPU_LEGACY_GCOFF_SUPPORTED); + + if (!(*bGC6Support) && + !(*bGCOFFSupport) && + !(pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED))) + { + NV_PRINTF(LEVEL_NOTICE, + "Disabling RTD3. [GC6 support=%d GCOFF support=%d RG support=%d]\n", + *bGC6Support, + *bGCOFFSupport, + pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED)); + return NV_FALSE; + } + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDevice, + NV0080_CTRL_CMD_GPU_GET_VIRTUALIZATION_MODE, + &virtModeParams, sizeof(virtModeParams)); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "Failed to get Virtualization mode, status=0x%x\n", + status); + return NV_FALSE; + } + + if ((virtModeParams.virtualizationMode != NV0080_CTRL_GPU_VIRTUALIZATION_MODE_NONE) && + (virtModeParams.virtualizationMode != NV0080_CTRL_GPU_VIRTUALIZATION_MODE_NMOS)) + { + NV_PRINTF(LEVEL_NOTICE, "RTD3 is not supported on VM\n"); + return NV_FALSE; + } + + if (!nvp->pr3_acpi_method_present && + !(pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED))) + { + NV_PRINTF(LEVEL_NOTICE, "RTD3 ACPI support is not available.\n"); + return NV_FALSE; + } + + return NV_TRUE; +} + +/*! + * @brief Allocate resources needed to track deferred power management, and + * schedule the initial periodic callbacks. + * + * @param[in] nv nv_state_t pointer. + */ +static void RmInitDeferredDynamicPowerManagement( + nv_state_t *nv, + NvBool bRtd3Support +) +{ + NV_STATUS status; + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if (!bRtd3Support) + { + nvp->dynamic_power.mode = NV_DYNAMIC_PM_NEVER; + nvp->dynamic_power.b_fine_not_supported = NV_TRUE; + NV_PRINTF(LEVEL_NOTICE, "RTD3 is not supported.\n"); + return; + } + AddGpuDynamicPowerSupported(gpuGetInstance(pGpu)); + nvp->dynamic_power.b_fine_not_supported = NV_FALSE; + status = CreateDynamicPowerCallbacks(pGpu); + + if (status == NV_OK) + { + RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu); + + nvp->dynamic_power.deferred_idle_enabled = NV_TRUE; + // RM's default is GCOFF allow + nvp->dynamic_power.clients_gcoff_disallow_refcount = 0; + } + else + { + NV_PRINTF(LEVEL_ERROR, "Failed to register for dynamic power callbacks\n"); + } + } +} + +/*! + * @brief Check if GCOFF state can be used for Power Management. + * + * @param[in] pGpu OBJGPU pointer. + * @param[in] usedFbSize Used FB Allocation size. + * @param[in] bIsDynamicPM If true, check for dynamic PM; otherwise, check + * for system PM (suspend/resume). + * + * returns NV_TRUE if GCOFF state can be used, otherwise NV_FALSE. + * + */ +static NvBool RmCheckForGcOffPM( + OBJGPU *pGpu, + NvU64 usedFbSize, + NvBool bIsDynamicPM +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU64 gcoff_max_fb_size; + + if (bIsDynamicPM) + { + /* + * GCOFF won't be enaged for Dynamic PM if X server does not allow. + * The dynamic_power.clients_gcoff_disallow_refcount is being accessed + * without acquiring dynamic power mutex since this value can be + * modified only when GPU is in active state. The os_ref_dynamic_power() + * function will be called to hold a reference to the GPU whenever + * RM is processing osClientGcoffDisallowRefcount() calls. This either + * prevents the GPU from going into a low power state or will wait for + * it to be resumed before proceeding. + */ + if (nvp->dynamic_power.clients_gcoff_disallow_refcount != 0) + return NV_FALSE; + + gcoff_max_fb_size = nvp->dynamic_power.gcoff_max_fb_size; + } + else + { + gcoff_max_fb_size = nvp->s0ix_gcoff_max_fb_size; + } + + /* + * GCOFF will be engaged whenever the following necessary preconditions + * are met: + * + * 1. The GCOFF has not been disabled with regkey by setting it to zero. + * 2. Used FB allocation size are within limits. + */ + return (gcoff_max_fb_size > 0) && + (usedFbSize <= gcoff_max_fb_size); +} + +// +// Function to update fixed fbsr modes to support multiple vairants such as +// GCOFF and cuda S3/resume. +// +static void +RmUpdateFixedFbsrModes(OBJGPU *pGpu) +{ + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERING)) + { + pMemoryManager->fixedFbsrModesMask = NVBIT(FBSR_TYPE_DMA); + } + else if (nv->preserve_vidmem_allocations) + { + pMemoryManager->fixedFbsrModesMask = NVBIT(FBSR_TYPE_FILE); + } +} + +static NV_STATUS +RmPowerManagement( + OBJGPU *pGpu, + nv_pm_action_t pmAction +) +{ + // default to NV_OK. there may cases where resman is loaded, but + // no devices are allocated (we're still at the console). in these + // cases, it's fine to let the system do whatever it wants. + NV_STATUS rmStatus = NV_OK; + + if (pGpu) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvBool bcState = gpumgrGetBcEnabledStatus(pGpu); + Intr *pIntr = GPU_GET_INTR(pGpu); + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + + if ((pmAction == NV_PM_ACTION_HIBERNATE) || (pmAction == NV_PM_ACTION_STANDBY)) + { + // + // pFb object store the FBSR mode through which FB state unload has happened, + // so os layer doesn't need to set FBSR mode on resume. + // + RmUpdateFixedFbsrModes(pGpu); + } + + switch (pmAction) + { + case NV_PM_ACTION_HIBERNATE: + nvp->pm_state.InHibernate = NV_TRUE; + nvp->pm_state.IntrEn = intrGetIntrEn(pIntr); + intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); + gpumgrSetBcEnabledStatus(pGpu, NV_FALSE); + + rmStatus = gpuEnterHibernate(pGpu); + gpumgrSetBcEnabledStatus(pGpu, bcState); + + break; + + case NV_PM_ACTION_STANDBY: + nvp->pm_state.InHibernate = NV_FALSE; + nvp->pm_state.IntrEn = intrGetIntrEn(pIntr); + intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); + gpumgrSetBcEnabledStatus(pGpu, NV_FALSE); + + rmStatus = gpuEnterStandby(pGpu); + gpumgrSetBcEnabledStatus(pGpu, bcState); + + break; + + case NV_PM_ACTION_RESUME: + gpumgrSetBcEnabledStatus(pGpu, NV_FALSE); + + if (nvp->pm_state.InHibernate) + { + gpuResumeFromHibernate(pGpu); + } + else + { + gpuResumeFromStandby(pGpu); + } + intrSetIntrEn(pIntr, nvp->pm_state.IntrEn); + gpumgrSetBcEnabledStatus(pGpu, bcState); + NvU32 ac_plugged = 0; + + if (IsMobile(pGpu)) + { + if (nv_acpi_get_powersource(&ac_plugged) == NV_OK) + { + // + // As we have already acquired API and GPU lock here, we are + // directly calling RmPowerSourceChangeEvent. + // + RmPowerSourceChangeEvent(nv, !ac_plugged); + } + RmRequestDNotifierState(nv); + } + break; + + default: + rmStatus = NV_ERR_INVALID_ARGUMENT; + break; + } + + pMemoryManager->fixedFbsrModesMask = 0; + } + + return rmStatus; +} + +/*! + * @brief Call core RM to perform the GCx (GC6/GCOF) entry sequence (before + * powering off the GPU) or exit sequence (after powering the GPU back on). + * + * The GPU lock should be held when calling this function. + * + * @param[in] pGpu OBJGPU pointer. + * @param[in] bEnter If true, perform entry sequence; exit sequence + * otherwise. + * @param[in] bIsDynamicPM If true, it is for dynamic PM; otherwise, it is + * for system PM (suspend/resume). + * + * @return NV_STATUS code indicating if the operation was successful. + */ +NV_STATUS RmGcxPowerManagement( + OBJGPU *pGpu, + NvBool bEnter, + NvBool bIsDynamicPM, + NvBool *bTryAgain +) +{ + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NV_STATUS status = NV_OK; + + if (pGpu->acpiMethodData.jtMethodData.bSBIOSCaps && + !pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_MOBILE)) + { + // + // AML override is expected only on NB platforms to support GCOFF-1.0( + // which is legacy GCOFF) and GC6-3.0 + // GC6 3.0 is possible without AML override but due to changes required + // for GCOFF-1.0 in SBIOS & HW, AML override is needed for GC6-3.0 also. + // + NV_PRINTF(LEVEL_INFO,"AML overrides present in Desktop"); + } + + nv->d0_state_in_suspend = NV_FALSE; + + if (bEnter) + { + NvU64 usedFbSize = 0; + NvBool bCanUseGc6 = NV_FALSE; + NV_STATUS fbsrStatus = NV_ERR_GENERIC; + + // + // If the GPU supports GC6, then check if GC6 can be used for + // the current power management request. + // 1. For dynamic PM, GC6 can be used if it is supported by the GPU. + // 2. For system PM with s2idle, GC6 can be used if it is + // supported by the GPU. + // + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED) && + nvp->gc6_upstream_port_configured) + { + bCanUseGc6 = bIsDynamicPM ? NV_TRUE : nv_s2idle_pm_configured(); + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED)) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERING, NV_TRUE); + status = RmPowerManagement(pGpu, NV_PM_ACTION_STANDBY); + pGpu->setProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERING, NV_FALSE); + if (status == NV_OK) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERED, NV_TRUE); + } + } + // + // If GC6 cannot be used, then no need to compare the used FB size with + // threshold value and select GCOFF irrespective of FB size. + // + else if ((memmgrGetUsedRamSize(pGpu, pMemoryManager, &usedFbSize) == NV_OK) && + (!bCanUseGc6 || RmCheckForGcOffPM(pGpu, usedFbSize, bIsDynamicPM)) && + ((fbsrStatus = fbsrReserveSysMemoryForPowerMgmt(pGpu, pMemoryManager->pFbsr[FBSR_TYPE_DMA], + usedFbSize)) == NV_OK)) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERING, NV_TRUE); + + // + // Set 'bPreserveComptagBackingStoreOnSuspend' so that comptag + // related handling can be done during state unload/load. + // + if (!pMemoryManager->bUseVirtualCopyOnSuspend) + { + pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = NV_TRUE; + } + + status = RmPowerManagement(pGpu, NV_PM_ACTION_STANDBY); + pGpu->setProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERING, NV_FALSE); + + if (status == NV_OK) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED, NV_TRUE); + } + else + { + /* + * Normally the NV_PM_ACTION_RESUME resume should free the + * the reserved memory while doing FB state load. But if the + * failure has happened at very early stage in FB state unload + * while doing NV_PM_ACTION_STANDBY, then reserved system memory + * might not be freed. + */ + fbsrFreeReservedSysMemoryForPowerMgmt(pMemoryManager->pFbsr[FBSR_TYPE_DMA]); + + if (!pMemoryManager->bUseVirtualCopyOnSuspend) + { + pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = nv->preserve_vidmem_allocations; + } + } + } + else if (bCanUseGc6) + { + NV2080_CTRL_GC6_ENTRY_PARAMS entryParams; + portMemSet(&entryParams, 0, sizeof(entryParams)); + + entryParams.flavorId = NV2080_CTRL_GC6_FLAVOR_ID_MSHYBRID; + entryParams.stepMask = NVBIT(NV2080_CTRL_GC6_STEP_ID_GPU_OFF); + entryParams.params.bIsRTD3Transition = NV_TRUE; + + // + // Currently if the GPU is not in the Lowest P-State, then the + // GC6 entry will fail in its internal sanity check. The system + // suspend can be triggered in any P-State, so sanity check + // for the lowest P-State needs to be skipped. In this case, during + // Perf state unload time, the forceful P-State switch will + // happen which will bring the GPU into lowest P-State. + // + entryParams.params.bSkipPstateSanity = !bIsDynamicPM; + + status = gpuGc6Entry(pGpu, &entryParams); + + // + // The GC6 state requires SW involvement before each transition + // from D0 to D3cold. In case of system suspend with pass-through + // mode, the VM suspend should happen before host suspend. During + // VM suspend, the NVIDIA driver running in the VM side will be + // involved in D3cold transition, but in the host side, it will + // go through D3cold->D0->D3cold transition and will make the + // second D3cold transition without NVIDIA driver. To handle this + // use-case, during VM suspend, use the D0 state and the host linux + // kernel will put the device into D3cold state. + // + if ((status == NV_OK) && ((nv->flags & NV_FLAG_PASSTHRU) != 0)) + { + nv->d0_state_in_suspend = NV_TRUE; + } + } + // + // The else condition below will hit in the following cases: + // Case 1. During system suspend transition: For systems that support s2idle but are configured + // for deep sleep, this "else" condition will be hit when the system memory + // is not sufficient. In this case, we should unset bTryAgain to abort the current suspend entry. + // Case 2. During runtime suspend transition: For systems that do not support GC6 but support + // GCOFF, this "else" condition will be hit when system memory is not sufficent, In this case, we + // should set bTryagain so that the kernel can reschedule the callback later. + // + else + { + if (bIsDynamicPM) + { + if (fbsrStatus == NV_ERR_NO_MEMORY) + { + *bTryAgain = NV_TRUE; + } + status = fbsrStatus; + } + else + { + status = NV_ERR_NOT_SUPPORTED; + NV_PRINTF(LEVEL_ERROR, + "System suspend failed with current system suspend configuration. " + "Please change the system suspend configuration to s2idle in /sys/power/mem_sleep.\n"); + } + } + } + else + { + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED)) + { + status = RmPowerManagement(pGpu, NV_PM_ACTION_RESUME); + pGpu->setProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED, NV_FALSE); + if (!pMemoryManager->bUseVirtualCopyOnSuspend) + { + pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = nv->preserve_vidmem_allocations; + } + } + else if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERED)) + { + status = RmPowerManagement(pGpu, NV_PM_ACTION_RESUME); + if (status == NV_OK) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERED, NV_FALSE); + } + } + else + { + NV2080_CTRL_GC6_EXIT_PARAMS exitParams; + portMemSet(&exitParams, 0, sizeof(exitParams)); + + exitParams.flavorId = NV2080_CTRL_GC6_FLAVOR_ID_MSHYBRID; + exitParams.params.bIsRTD3Transition = NV_TRUE; + + status = gpuGc6Exit(pGpu, &exitParams); + } + } + + return status; +} + +NV_STATUS NV_API_CALL rm_power_management( + nvidia_stack_t *sp, + nv_state_t *pNv, + nv_pm_action_t pmAction +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + void *fp; + NvBool bTryAgain = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_DEVICE_INIT); + + NV_ASSERT_OK(os_flush_work_queue(pNv->queue, pmAction != NV_PM_ACTION_RESUME)); + // + // For GPU driving console, disable console access here, to ensure no console + // writes through BAR1 can interfere with physical RM's setup of BAR1 + // + if (pNv->client_managed_console && !pNv->sysmem_mapped_console) + { + os_disable_console_access(); + } + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + if (pGpu != NULL) + { + if ((rmStatus = os_ref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE)) == NV_OK) + { + // LOCK: acquire GPUs lock + if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK) + { + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + rmStatus = RmPowerManagementTegra(pGpu, pmAction); + + // + // RmPowerManagementTegra() is most likely to fail due to + // gpuStateUnload() failures deep in the RM's GPU power + // management paths. However, those paths make no + // attempt to unwind in case of errors. Rather, they + // soldier on and simply report an error at the very end. + // GPU software state meanwhile will indicate the GPU + // has been suspended. + // + // Sadly, in case of an error during suspend/hibernate, + // the only path forward here is to attempt to resume the + // GPU, accepting that the odds of success will vary. + // + if (rmStatus != NV_OK && pmAction != NV_PM_ACTION_RESUME) + { + RmPowerManagementTegra(pGpu, NV_PM_ACTION_RESUME); + } + } + else + { + nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); + + // + // Before going to S3 or S4, remove idle holdoff which was + // applied during gc6 exit. + // + if ((pmAction != NV_PM_ACTION_RESUME) && + (nvp->dynamic_power.b_idle_holdoff == NV_TRUE)) + { + nv_indicate_idle(pNv); + RmCancelCallbackToRemoveIdleHoldoff(pGpu); + nvp->dynamic_power.b_idle_holdoff = NV_FALSE; + } + + // + // Use GCx (GCOFF/GC6) power management if S0ix-based PM is + // enabled and the request is for system suspend/resume. + // Otherwise, use the existing mechanism. + // + if (nvp->s0ix_pm_enabled && + (pmAction == NV_PM_ACTION_STANDBY || + (pmAction == NV_PM_ACTION_RESUME && + !nvp->pm_state.InHibernate))) + { + rmStatus = RmGcxPowerManagement(pGpu, + pmAction == NV_PM_ACTION_STANDBY, + NV_FALSE, &bTryAgain); + + } + else + { + rmStatus = RmPowerManagement(pGpu, pmAction); + } + } + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + os_unref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE); + } + } + // UNLOCK: release API lock + rmapiLockRelease(); + } + + if (pNv->client_managed_console && !pNv->sysmem_mapped_console) + { + os_enable_console_access(); + } + + NV_ASSERT_OK(os_flush_work_queue(pNv->queue, pmAction != NV_PM_ACTION_RESUME)); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +/*! + * @brief Wrapper around RmGcxPowerManagement() and removes the Idle holdoff + * during exit sequence. + * + * The GPU lock should be held when calling this function. + * + * @param[in] pGpu OBJGPU pointer. + * @param[in] bEnter If true, perform entry sequence; exit sequence + * otherwise. + * + * @return NV_STATUS code indicating if the operation was successful. + */ +static NV_STATUS RmTransitionDynamicPower( + OBJGPU *pGpu, + NvBool bEnter, + NvBool *bTryAgain +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NV_STATUS status; + + status = RmGcxPowerManagement(pGpu, bEnter, NV_TRUE, bTryAgain); + + if (!bEnter && status == NV_OK) + { + nv_idle_holdoff(nv); + RmScheduleCallbackToRemoveIdleHoldoff(pGpu); + } + + return status; +} + +/*! + * @brief Wrapper around RmTransitionDynamicPower() which sets up the RM + * runtime (thread state and altstack), and takes the GPU lock for the duration + * of the operation. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + * @param[in] bEnter If true, perform entry sequence; exit sequence + * otherwise. + * + * @return NV_STATUS code indicating if the operation was successful. + */ +NV_STATUS NV_API_CALL rm_transition_dynamic_power( + nvidia_stack_t *sp, + nv_state_t *nv, + NvBool bEnter, + NvBool *bTryAgain +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + NV_STATUS status = NV_OK; + THREAD_STATE_NODE threadState; + void *fp; + + if (NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + return NV_OK; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (!bEnter && + !threadState.bValid && + (threadState.threadId == 0) && + pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED)) + { + /* + * If the thread node is already present in the database, + * reset the thread timeout before the resume operation starts + * for IGPU. + */ + threadStateResetTimeout(pGpu); + } + + if (nv->client_managed_console && !nv->sysmem_mapped_console) + { + os_disable_console_access(); + } + + // LOCK: acquire GPUs lock + status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER); + if (status == NV_OK) + { + rmapiEnterRtd3PmPath(); + + status = RmTransitionDynamicPower(pGpu, bEnter, bTryAgain); + + rmapiLeaveRtd3PmPath(); + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + + if (nv->client_managed_console && !nv->sysmem_mapped_console) + { + os_enable_console_access(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +/*! + * @brief: Notify client about hot-plugged/unplugged displays. + * Called after GPU exits GC6 because of display hot-plug/unplug. + * + * @param[in] nv nv_state_t pointer. + */ +static void RmNotifyClientAboutHotplug( + nv_state_t *nv +) +{ + NV_STATUS status; + GPU_MASK gpuMask; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + Nv2080HotplugNotification hotplugNotificationParams; + + portMemSet(&hotplugNotificationParams, 0, sizeof(hotplugNotificationParams)); + + // LOCK: acquire per device lock + status = rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HOTPLUG, + &gpuMask); + if (status == NV_OK) + { + gpuNotifySubDeviceEvent(pGpu, NV2080_NOTIFIERS_HOTPLUG, + &hotplugNotificationParams, + sizeof(hotplugNotificationParams), 0, 0); + // UNLOCK: release per device lock + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + } +} + +/*! + * @brief If GPU is in GC6, exit GC6 state and + * notify client about display hot-plug/unplug. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + */ +void RmHandleDisplayChange( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if (pGpu && + (IS_GPU_GC6_STATE_ENTERED(pGpu) || + pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED))) + { + if (rm_ref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE) == NV_OK) + { + RmNotifyClientAboutHotplug(nv); + rm_unref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE); + } + else + NV_PRINTF(LEVEL_ERROR, + "Failed to increment dynamic power refcount\n"); + } +} + +/*! + * @brief RmGetDynamicBoostSupport() gets the notebook DB + * support. DB support is static information for vast majority + * of notebooks and hence we fetch it only once (during the first + * call to this function). This way, the GPU remains in RTD3 state + * when procfs read is done. + * + * @param[in] nv nv_state_t pointer. + */ +static const char* RmGetDynamicBoostSupport( + nv_state_t *pNv +) +{ + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + const char* DbStatus = "?"; + + if (pNvp->db_supported == -1) + { + pNvp->db_supported = os_get_dynamic_boost_support(pNv); + } + + if (pNvp->db_supported == 0) + { + DbStatus = "Not Supported"; + } + else if (pNvp->db_supported == 1) + { + DbStatus = "Supported"; + } + return DbStatus; +} + +/*! + * @brief: Function to query Dynamic Power Management + * + * @param[in] nvp nv_priv_t pointer. + * + * @return String indicating Dynamic Power Management status. + */ +static const char* RmGetDynamicPowerManagementStatus( + nv_priv_t *nvp +) +{ + /* + * If the platform/driver does not support Dynamic Power Management, + * we set mode as NV_DYNAMIC_PM_NEVER. Hence, after RmInit if the + * mode is still NV_DYNAMIC_PM_FINE, we are sure that it is + * supported and enabled. Also see NOTE. + */ + if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE) + { + return "Enabled (fine-grained)"; + } + else if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_COARSE) + { + return "Enabled (coarse-grained)"; + } + else if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_NEVER) + { + + if (nvp->dynamic_power.dynamic_power_regkey == + NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER) + return "Disabled"; + else if (nvp->dynamic_power.dynamic_power_regkey == + NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT) + return "Disabled by default"; + } + + return "Not supported"; +} + +static void RmHandleIdleSustained( + NvU32 gpuInstance, + void *pArgs +) +{ + OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + nv_revoke_gpu_mappings(NV_GET_NV_STATE(pGpu)); + RmScheduleCallbackToIndicateIdle(pGpu); + nvp->dynamic_power.b_idle_sustained_workitem_queued = NV_FALSE; +} + +/* + * Queue a workitem for revoking GPU mappings, and scheduling a callback to indicate idle. + */ +static void RmQueueIdleSustainedWorkitem( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NV_STATUS status = NV_OK; + + if (!nvp->dynamic_power.b_idle_sustained_workitem_queued) + { + status = osQueueWorkItem(pGpu, + RmHandleIdleSustained, + NULL, + (OsQueueWorkItemFlags){.bLockGpuGroupSubdevice = NV_TRUE}); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_WARNING, + "Failed to queue RmHandleIdleSustained() workitem.\n"); + RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu); + return; + } + nvp->dynamic_power.b_idle_sustained_workitem_queued = NV_TRUE; + } +} + +/* + * Allocate resources needed for S0ix-based system power management. + */ +static void +RmInitS0ixPowerManagement( + nv_state_t *nv, + NvBool bRtd3Support, + NvBool bGC6Support +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 data; + + // S0ix-based S2Idle, on desktops, is not supported yet. Return early for desktop SKUs + if (!nvp->b_mobile_config_enabled) + { + return; + } + + /* + * The GPU S0ix-based system power management will be enabled + * only if all the following necessary requirements are met: + * + * 1. The GPU has RTD3 support. + * 2. The platform has support for s0ix. + * 3. Feature regkey EnableS0ixPowerManagement is enabled. + */ + if (bRtd3Support && bGC6Support && + nv_platform_supports_s0ix() && + (osReadRegistryDword(NULL, NV_REG_ENABLE_S0IX_POWER_MANAGEMENT, + &data) == NV_OK) && (data == 1)) + { + nvp->s0ix_pm_enabled = NV_TRUE; + + /* + * Read the OS registry for Maximum FB size for S0ix-based PM + * which will be expressed in Megabytes (1048576 bytes) and + * convert it into bytes. + */ + if ((osReadRegistryDword(NULL, + NV_REG_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD, + &data) == NV_OK)) + { + nvp->s0ix_gcoff_max_fb_size = (NvU64)data * 1024 * 1024; + } + } +} + +/*! + * @brief: Helper function to abstract the execution and return value verification + * of PEX DSM methods, necessary for GC6 to work in Desktops. + * + * This function is capable to invoke three _DSM methods, as listed below: + * 0x0: _DSM for "Query supported functions" + * (for more details, refer to section 9.1.1 of the ACPI 6.3 Specification). + * 0xA: _DSM for "Requesting D3cold Aux Power Limit" + * (for more details, refer to section 4.6.10 of PCI Firmware Specification v3.3) + * 0xB: _DSM for "Adding PERST# Assertion Delay" + * (for more details, refer to section 4.6.11 of PCI Firmware Specification v3.3) + * + * @param[in] nv nv_state_t pointer. + * @param[in] dsmSubFunc subfunction to invoke. + * @param[in] data input data for dsmSubFunc. + * + * @return TRUE if dsmSubFunc executed successfully and return value is as expected; + * FALSE otherwise. + */ +static NvBool RmAcpiD3ColdDsm +( + nv_state_t *nv, + NvU32 dsmSubFunc, + NvU32 data +) +{ + NvU32 inData = data; + + /* + * Third argument (i.e. acpiDsmRev) is sent as 4 because PEX_FUNC_AUXPOWERLIMIT and + * PEX_FUNC_PEXRST_DELAY sub-functions were introduced in PCI Firmware Specification 3.2 + * and they are supported only for _DSM revision 4 onwards. But, RM still use _DSM + * revision 2 (see drivers/common/inc/pex.h). Hence, the need to hardcode 4 here. + */ + if (nv_acpi_d3cold_dsm_for_upstream_port(nv, (NvU8 *)&PEX_DSM_GUID, 4, dsmSubFunc, &data) != NV_OK) + { + NV_PRINTF(LEVEL_NOTICE, "%s: PEX _DSM subfunction: 0x%X failed.\n", __FUNCTION__, dsmSubFunc); + return NV_FALSE; + } + + switch (dsmSubFunc) + { + case NV_ACPI_ALL_FUNC_SUPPORT: + return ((data & NVBIT(PEX_FUNC_PEXRST_DELAY)) && + (data & NVBIT(PEX_FUNC_AUXPOWERLIMIT))); + + case PEX_FUNC_PEXRST_DELAY: + return (inData == data); + + case PEX_FUNC_AUXPOWERLIMIT: + return ((data == NV_AUX_POWER_REQUEST_STATUS_GRANTED_WITH_12V_POWER) || + (data == NV_AUX_POWER_REQUEST_STATUS_GRANTED_WITHOUT_12V_POWER)); + + default: + return NV_FALSE; + } + + return NV_FALSE; +} + +static NvBool RmConfigureUpstreamPortForRTD3( + nv_state_t *nv, + NvBool bConfigureForGC6 +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if (nvp->b_mobile_config_enabled) + return NV_TRUE; + + if (!RmAcpiD3ColdDsm(nv, NV_ACPI_ALL_FUNC_SUPPORT, 0)) + { + return NV_FALSE; + } + + if (!RmAcpiD3ColdDsm(nv, PEX_FUNC_PEXRST_DELAY, + bConfigureForGC6 ? pGpu->gc6State.GC6PerstDelay : DEFAULT_GCOFF_PEXRST_DELAY)) + { + return NV_FALSE; + } + + if (!RmAcpiD3ColdDsm(nv, PEX_FUNC_AUXPOWERLIMIT, + bConfigureForGC6 ? pGpu->gc6State.GC6TotalBoardPower : 0)) + { + if (bConfigureForGC6) + RmAcpiD3ColdDsm(nv, PEX_FUNC_PEXRST_DELAY, DEFAULT_GCOFF_PEXRST_DELAY); + + return NV_FALSE; + } + + NV_PRINTF(LEVEL_NOTICE, + "Aux Power and Pex delay settings %s successfully.\n", + bConfigureForGC6? "applied": "cleared"); + + return NV_TRUE; +} + +void RmInitPowerManagement( + nv_state_t *nv +) +{ + // LOCK: acquire GPUs lock + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK) + { + NvBool bGC6Support = NV_FALSE; + NvBool bGCOFFSupport = NV_FALSE; + NvBool bRtd3Support = RmCheckRtd3GcxSupport(nv, &bGC6Support, &bGCOFFSupport); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + RmInitDeferredDynamicPowerManagement(nv, bRtd3Support); + RmInitS0ixPowerManagement(nv, bRtd3Support, bGC6Support); + + if (bRtd3Support && bGC6Support && + (nvp->s0ix_pm_enabled || (nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE))) + { + nvp->gc6_upstream_port_configured = RmConfigureUpstreamPortForRTD3(nv, NV_TRUE); + } + + nv->is_pm_unsupported = pGpu->getProperty(pGpu, PDB_PROP_GPU_POWER_MANAGEMENT_UNSUPPORTED); + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } +} + +void RmDestroyPowerManagement( + nv_state_t *nv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + if (nvp->gc6_upstream_port_configured) + { + RmConfigureUpstreamPortForRTD3(nv, NV_FALSE); + } + + RmDestroyDeferredDynamicPowerManagement(nv); +} + +void NV_API_CALL rm_get_power_info( + nvidia_stack_t *sp, + nv_state_t *pNv, + nv_power_info_t *powerInfo +) +{ + THREAD_STATE_NODE threadState; + void *fp; + GPU_MASK gpuMask; + const char *pVidmemPowerStatus = "?"; + const char *pGc6Supported = "?"; + const char *pGcoffSupported = "?"; + const char *pDynamicPowerStatus = "?"; + const char *pS0ixStatus = "?"; + const char *pDbStatus = "?"; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + // LOCK: acquire per device lock + if ((pGpu != NULL) && + ((rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER, + &gpuMask)) == NV_OK)) + { + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + + pVidmemPowerStatus = pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED) ? + "Off" : "Active"; + pDynamicPowerStatus = RmGetDynamicPowerManagementStatus(pNvp); + pGc6Supported = RmGetGpuGcxSupport(pGpu, NV_TRUE); + pGcoffSupported = RmGetGpuGcxSupport(pGpu, NV_FALSE); + pS0ixStatus = pNvp->s0ix_pm_enabled ? "Enabled" : "Disabled"; + pDbStatus = RmGetDynamicBoostSupport(pNv); + // UNLOCK: release per device lock + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + powerInfo->vidmem_power_status = pVidmemPowerStatus; + powerInfo->dynamic_power_status = pDynamicPowerStatus; + powerInfo->gc6_support = pGc6Supported; + powerInfo->gcoff_support = pGcoffSupported; + powerInfo->s0ix_status = pS0ixStatus; + powerInfo->db_support = pDbStatus; +} + diff --git a/userspace/sdks/nv570/nvalloc/unix/src/escape.c b/userspace/sdks/nv570/nvalloc/unix/src/escape.c new file mode 100644 index 0000000..fae7b8d --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/escape.c @@ -0,0 +1,1017 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + + +//***************************** Module Header ********************************** +// +// This code is linked into the resource manager proper. It receives the +// ioctl from the resource manager's customer, unbundles the args and +// calls the correct resman routines. +// +//****************************************************************************** + +#include <core/prelude.h> +#include <core/locks.h> +#include <nv.h> +#include <nv_escape.h> +#include <osapi.h> +#include <rmapi/exports.h> +#include <nv-unix-nvos-params-wrappers.h> + +#include <nvos.h> +#include <class/cl0000.h> // NV01_ROOT +#include <class/cl0001.h> // NV01_ROOT_NON_PRIV +#include <class/cl0005.h> // NV01_EVENT +#include <class/cl003e.h> // NV01_MEMORY_SYSTEM +#include <class/cl0071.h> // NV01_MEMORY_SYSTEM_OS_DESCRIPTOR + +#include "rmapi/client_resource.h" +#include "nvlog/nvlog.h" +#include <nv-ioctl-lockless-diag.h> + +#include <ctrl/ctrl00fd.h> + +#include <ctrl/ctrl00e0.h> + +#define NV_CTL_DEVICE_ONLY(nv) \ +{ \ + if (((nv)->flags & NV_FLAG_CONTROL) == 0) \ + { \ + rmStatus = NV_ERR_INVALID_ARGUMENT; \ + goto done; \ + } \ +} + +#define NV_ACTUAL_DEVICE_ONLY(nv) \ +{ \ + if (((nv)->flags & NV_FLAG_CONTROL) != 0) \ + { \ + rmStatus = NV_ERR_INVALID_ARGUMENT; \ + goto done; \ + } \ +} + +static NV_STATUS RmGetDeviceFd(NVOS54_PARAMETERS *pApi, NvS32 *pFd, + NvBool *pSkipDeviceRef) +{ + RMAPI_PARAM_COPY paramCopy; + void *pKernelParams; + NvU32 paramSize; + NV_STATUS status; + + *pFd = -1; + *pSkipDeviceRef = NV_TRUE; + + switch(pApi->cmd) + { + case NV00FD_CTRL_CMD_ATTACH_GPU: + paramSize = sizeof(NV00FD_CTRL_ATTACH_GPU_PARAMS); + break; + case NV00E0_CTRL_CMD_EXPORT_MEM: + paramSize = sizeof(NV00E0_CTRL_EXPORT_MEM_PARAMS); + break; + default: + return NV_OK; + } + + RMAPI_PARAM_COPY_INIT(paramCopy, pKernelParams, pApi->params, paramSize, 1); + + status = rmapiParamsAcquire(¶mCopy, NV_TRUE); + if (status != NV_OK) + return status; + + switch(pApi->cmd) + { + case NV00FD_CTRL_CMD_ATTACH_GPU: + { + NV00FD_CTRL_ATTACH_GPU_PARAMS *pAttachGpuParams = pKernelParams; + + *pSkipDeviceRef = NV_FALSE; + *pFd = (NvS32)pAttachGpuParams->devDescriptor; + } + break; + case NV00E0_CTRL_CMD_EXPORT_MEM: + { + NV00E0_CTRL_EXPORT_MEM_PARAMS *pExportMemParams = pKernelParams; + + // If hParent is client, no need to reference device. + *pSkipDeviceRef = (pExportMemParams->hParent == pApi->hClient); + *pFd = (NvS32)pExportMemParams->devDescriptor; + } + break; + default: + NV_ASSERT(0); + break; + } + + NV_ASSERT(rmapiParamsRelease(¶mCopy) == NV_OK); + + return status; +} + +// Only return errors through pApi->status +static void RmCreateOsDescriptor(NVOS32_PARAMETERS *pApi, API_SECURITY_INFO secInfo) +{ + NV_STATUS rmStatus; + NvBool writable; + NvU32 flags = 0; + NvU64 allocSize, pageCount, *pPteArray = NULL; + void *pDescriptor, *pPageArray = NULL; + NVOS32_PARAMETERS allocApi = *pApi; + NVOS32_PARAMETERS *pAllocApi = &allocApi; + + pDescriptor = NvP64_VALUE(pApi->data.AllocOsDesc.descriptor); + if (((NvUPtr)pDescriptor & ~os_page_mask) != 0) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + goto done; + } + + // Check to prevent an NvU64 overflow + if ((pApi->data.AllocOsDesc.limit + 1) == 0) + { + rmStatus = NV_ERR_INVALID_LIMIT; + goto done; + } + + allocSize = (pApi->data.AllocOsDesc.limit + 1); + pageCount = (1 + ((allocSize - 1) / os_page_size)); + + writable = FLD_TEST_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_WRITE, pApi->data.AllocOsDesc.attr2); + + flags = FLD_SET_DRF_NUM(_LOCK_USER_PAGES, _FLAGS, _WRITE, writable, flags); + rmStatus = os_lock_user_pages(pDescriptor, pageCount, &pPageArray, flags); + if (rmStatus == NV_OK) + { + pAllocApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPageArray; + pAllocApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY; + } + else if (rmStatus == NV_ERR_INVALID_ADDRESS) + { + rmStatus = os_lookup_user_io_memory(pDescriptor, pageCount, &pPteArray); + if (rmStatus == NV_OK) + { + pAllocApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPteArray; + pAllocApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY; + } + } + if (rmStatus != NV_OK) + goto done; + + Nv04VidHeapControlWithSecInfo(pAllocApi, secInfo); + + pApi->status = pAllocApi->status; + pApi->data.AllocOsDesc.hMemory = pAllocApi->data.AllocOsDesc.hMemory; + + if ((pAllocApi->status != NV_OK) && + (pAllocApi->data.AllocOsDesc.descriptor != NvP64_NULL)) + { + switch (pAllocApi->data.AllocOsDesc.descriptorType) + { + default: + break; + case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY: + os_unlock_user_pages(pageCount, pPageArray, flags); + break; + } + } + +done: + if (rmStatus != NV_OK) + pApi->status = rmStatus; +} + +// Only return errors through pApi->status +static void RmAllocOsDescriptor(NVOS02_PARAMETERS *pApi, API_SECURITY_INFO secInfo) +{ + NV_STATUS rmStatus = NV_OK; + NvU32 flags, attr, attr2; + NVOS32_PARAMETERS *pVidHeapParams; + + if (!FLD_TEST_DRF(OS02, _FLAGS, _LOCATION, _PCI, pApi->flags) || + !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NO_MAP, pApi->flags)) + { + rmStatus = NV_ERR_INVALID_FLAGS; + goto done; + } + + attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI); + + if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, pApi->flags) || + FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, pApi->flags)) + { + attr = FLD_SET_DRF(OS32, _ATTR, _COHERENCY, _WRITE_BACK, attr); + } + else if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, pApi->flags)) + attr = FLD_SET_DRF(OS32, _ATTR, _COHERENCY, _UNCACHED, attr); + else { + rmStatus = NV_ERR_INVALID_FLAGS; + goto done; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, pApi->flags)) + attr = FLD_SET_DRF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS, attr); + else + attr = FLD_SET_DRF(OS32, _ATTR, _PHYSICALITY, _NONCONTIGUOUS, attr); + + if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, pApi->flags)) + attr2 = DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _YES); + else + attr2 = DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _NO); + + if (FLD_TEST_DRF(OS02, _FLAGS, _MEMORY_PROTECTION, _UNPROTECTED, pApi->flags)) + attr2 = FLD_SET_DRF(OS32, _ATTR2, _MEMORY_PROTECTION, _UNPROTECTED, attr2); + + pVidHeapParams = portMemAllocNonPaged(sizeof(NVOS32_PARAMETERS)); + if (pVidHeapParams == NULL) + { + rmStatus = NV_ERR_NO_MEMORY; + goto done; + } + portMemSet(pVidHeapParams, 0, sizeof(NVOS32_PARAMETERS)); + + pVidHeapParams->hRoot = pApi->hRoot; + pVidHeapParams->hObjectParent = pApi->hObjectParent; + pVidHeapParams->function = NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR; + + flags = (NVOS32_ALLOC_FLAGS_MEMORY_HANDLE_PROVIDED | + NVOS32_ALLOC_FLAGS_MAP_NOT_REQUIRED); + + if (DRF_VAL(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, pApi->flags)) + attr2 = FLD_SET_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_ONLY, attr2); + + // Currently CPU-RO memory implies GPU-RO as well + if (DRF_VAL(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, pApi->flags) || + DRF_VAL(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, pApi->flags)) + attr2 = FLD_SET_DRF(OS32, _ATTR2, _PROTECTION_DEVICE, _READ_ONLY, attr2); + + pVidHeapParams->data.AllocOsDesc.hMemory = pApi->hObjectNew; + pVidHeapParams->data.AllocOsDesc.flags = flags; + pVidHeapParams->data.AllocOsDesc.attr = attr; + pVidHeapParams->data.AllocOsDesc.attr2 = attr2; + pVidHeapParams->data.AllocOsDesc.descriptor = pApi->pMemory; + pVidHeapParams->data.AllocOsDesc.limit = pApi->limit; + pVidHeapParams->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS; + + RmCreateOsDescriptor(pVidHeapParams, secInfo); + + pApi->status = pVidHeapParams->status; + + portMemFree(pVidHeapParams); + +done: + if (rmStatus != NV_OK) + pApi->status = rmStatus; +} + +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hRoot) == NV_OFFSETOF(NVOS64_PARAMETERS, hRoot)); +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hObjectParent) == NV_OFFSETOF(NVOS64_PARAMETERS, hObjectParent)); +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hObjectNew) == NV_OFFSETOF(NVOS64_PARAMETERS, hObjectNew)); +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hClass) == NV_OFFSETOF(NVOS64_PARAMETERS, hClass)); +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, pAllocParms) == NV_OFFSETOF(NVOS64_PARAMETERS, pAllocParms)); + + +NV_STATUS RmValidateIoctl(NvU32 cmd, NvU32 size) +{ + int i = 0; + const unsigned int arg_cmd = (cmd & 0xFF); + static const struct { + unsigned int cmdKey; + unsigned int paramSize; + } RmIoctlsTable[] = { + + #define _RM_ESC_IOCTL_ENTRY(_cmd, _type) \ + { .cmdKey = ((_cmd) & 0xFF), .paramSize = sizeof(_type) } + + _RM_ESC_IOCTL_ENTRY(NV_ESC_STATUS_CODE, nv_ioctl_status_code_t), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_ALLOC_MEMORY, nv_ioctl_nvos02_parameters_with_fd), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_ALLOC_OBJECT, NVOS05_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_FREE, NVOS00_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_CONTROL, NVOS54_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_DUP_OBJECT, NVOS55_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_SHARE, NVOS57_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_I2C_ACCESS, NVOS_I2C_ACCESS_PARAMS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_IDLE_CHANNELS, NVOS30_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_VID_HEAP_CONTROL, NVOS32_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_ACCESS_REGISTRY, NVOS38_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_MAP_MEMORY, nv_ioctl_nvos33_parameters_with_fd), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_UNMAP_MEMORY, NVOS34_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_ALLOC_CONTEXT_DMA2, NVOS39_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_MAP_MEMORY_DMA, NVOS46_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_UNMAP_MEMORY_DMA, NVOS47_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_BIND_CONTEXT_DMA, NVOS49_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO, NVOS56_PARAMETERS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_LOCKLESS_DIAGNOSTIC, NV_LOCKLESS_DIAGNOSTIC_PARAMS), + _RM_ESC_IOCTL_ENTRY(NV_ESC_REGISTER_FD, nv_ioctl_register_fd_t), + }; + + if (arg_cmd == NV_ESC_RM_ALLOC) + { + if (size == sizeof(NVOS64_PARAMETERS) || size == sizeof(NVOS21_PARAMETERS)) + { + return NV_OK; + } + else + { + nv_printf(NV_DBG_ERRORS, "NVRM: invalid %d structure size!\n", arg_cmd); + + return NV_ERR_INVALID_ARGUMENT; + } + } + + for (i = 0; i < NV_ARRAY_ELEMENTS(RmIoctlsTable); ++i) + { + if (arg_cmd == RmIoctlsTable[i].cmdKey) + { + if (size != RmIoctlsTable[i].paramSize) + { + nv_printf(NV_DBG_ERRORS, "NVRM: invalid %d structure size, expected %d, got %d!\n", + RmIoctlsTable[i].cmdKey, RmIoctlsTable[i].paramSize, size); + // If IOCTL is found and size is mismatched, return NV_ERR_INVALID_ARGUMENT + return NV_ERR_INVALID_ARGUMENT; + } + // If IOCTL is found and size is valid, return NV_OK + return NV_OK; + } + } + // If IOCTL is not found, return NV_ERR_INVALID_COMMAND + return NV_ERR_INVALID_COMMAND; +} + +NV_STATUS RmIoctl( + nv_state_t *nv, + nv_file_private_t *nvfp, + NvU32 cmd, + void *data, + NvU32 dataSize +) +{ + // nvfp->ctl_nvfp + ct_assert(sizeof(PORT_ATOMIC nv_file_private_t *) == sizeof(nv_file_private_t *)); + ct_assert(_Alignof(PORT_ATOMIC nv_file_private_t *) == _Alignof(nv_file_private_t *)); + + // nvfp->register_or_refcount + ct_assert(sizeof(PORT_ATOMIC NvU32) == sizeof(NvU32)); + ct_assert(_Alignof(PORT_ATOMIC NvU32) == _Alignof(NvU32)); + + NV_STATUS rmStatus = NV_ERR_GENERIC; + API_SECURITY_INFO secInfo = { }; + + secInfo.privLevel = osIsAdministrator() ? RS_PRIV_LEVEL_USER_ROOT : RS_PRIV_LEVEL_USER; + secInfo.paramLocation = PARAM_LOCATION_USER; + secInfo.pProcessToken = NULL; + secInfo.gpuOsInfo = NULL; + secInfo.clientOSInfo = nvfp->ctl_nvfp; + if (secInfo.clientOSInfo == NULL) + secInfo.clientOSInfo = nvfp; + + switch (cmd) + { + case NV_ESC_RM_ALLOC_MEMORY: + { + nv_ioctl_nvos02_parameters_with_fd *pApi; + NVOS02_PARAMETERS *pParms; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pApi = data; + pParms = &pApi->params; + + NV_ACTUAL_DEVICE_ONLY(nv); + + if (pParms->hClass == NV01_MEMORY_SYSTEM_OS_DESCRIPTOR) + RmAllocOsDescriptor(pParms, secInfo); + else + { + NvU32 flags = pParms->flags; + + Nv01AllocMemoryWithSecInfo(pParms, secInfo); + + // + // If the system memory is going to be mapped immediately, + // create the mmap context for it now. + // + if ((pParms->hClass == NV01_MEMORY_SYSTEM) && + (!FLD_TEST_DRF(OS02, _FLAGS, _ALLOC, _NONE, flags)) && + (!FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NO_MAP, flags)) && + (pParms->status == NV_OK)) + { + if (rm_create_mmap_context(pParms->hRoot, + pParms->hObjectParent, pParms->hObjectNew, + pParms->pMemory, pParms->limit + 1, 0, + NV_MEMORY_DEFAULT, + pApi->fd) != NV_OK) + { + NV_PRINTF(LEVEL_WARNING, + "could not create mmap context for %p\n", + NvP64_VALUE(pParms->pMemory)); + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + } + } + + break; + } + + case NV_ESC_RM_ALLOC_OBJECT: + { + NVOS05_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv01AllocObjectWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_ALLOC: + { + NVOS21_PARAMETERS *pApi = data; + NVOS64_PARAMETERS *pApiAccess = data; + NvBool bAccessApi = (dataSize == sizeof(NVOS64_PARAMETERS)); + + if ((dataSize != sizeof(*pApi)) && + (dataSize != sizeof(*pApiAccess))) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + switch (pApi->hClass) + { + case NV01_ROOT: + case NV01_ROOT_CLIENT: + case NV01_ROOT_NON_PRIV: + { + NV_CTL_DEVICE_ONLY(nv); + + // Force userspace client allocations to be the _CLIENT class. + pApi->hClass = NV01_ROOT_CLIENT; + break; + } + case NV01_EVENT: + case NV01_EVENT_OS_EVENT: + case NV01_EVENT_KERNEL_CALLBACK: + case NV01_EVENT_KERNEL_CALLBACK_EX: + { + break; + } + default: + { + NV_CTL_DEVICE_ONLY(nv); + break; + } + } + + if (!bAccessApi) + { + Nv04AllocWithSecInfo(pApi, secInfo); + } + else + { + Nv04AllocWithAccessSecInfo(pApiAccess, secInfo); + } + + break; + } + + case NV_ESC_RM_FREE: + { + NVOS00_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv01FreeWithSecInfo(pApi, secInfo); + + if (pApi->status == NV_OK && + pApi->hObjectOld == pApi->hRoot) + { + rm_client_free_os_events(pApi->hRoot); + } + + break; + } + + case NV_ESC_RM_VID_HEAP_CONTROL: + { + NVOS32_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + if (pApi->function == NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR) + RmCreateOsDescriptor(pApi, secInfo); + else + Nv04VidHeapControlWithSecInfo(pApi, secInfo); + + break; + } + + case NV_ESC_RM_I2C_ACCESS: + { + NVOS_I2C_ACCESS_PARAMS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_ACTUAL_DEVICE_ONLY(nv); + + Nv04I2CAccessWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_IDLE_CHANNELS: + { + NVOS30_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv04IdleChannelsWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_MAP_MEMORY: + { + nv_ioctl_nvos33_parameters_with_fd *pApi; + NVOS33_PARAMETERS *pParms; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pApi = data; + pParms = &pApi->params; + + NV_CTL_DEVICE_ONLY(nv); + + // Don't allow userspace to override the caching type + pParms->flags = FLD_SET_DRF(OS33, _FLAGS, _CACHING_TYPE, _DEFAULT, pParms->flags); + Nv04MapMemoryWithSecInfo(pParms, secInfo); + + if (pParms->status == NV_OK) + { + pParms->status = rm_create_mmap_context(pParms->hClient, + pParms->hDevice, pParms->hMemory, + pParms->pLinearAddress, pParms->length, + pParms->offset, + DRF_VAL(OS33, _FLAGS, _CACHING_TYPE, pParms->flags), + pApi->fd); + if (pParms->status != NV_OK) + { + NVOS34_PARAMETERS params; + portMemSet(¶ms, 0, sizeof(params)); + params.hClient = pParms->hClient; + params.hDevice = pParms->hDevice; + params.hMemory = pParms->hMemory; + params.pLinearAddress = pParms->pLinearAddress; + params.flags = pParms->flags; + Nv04UnmapMemoryWithSecInfo(¶ms, secInfo); + } + } + break; + } + + case NV_ESC_RM_UNMAP_MEMORY: + { + NVOS34_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv04UnmapMemoryWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_ACCESS_REGISTRY: + { + NVOS38_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + pApi->status = rm_access_registry(pApi->hClient, + pApi->hObject, + pApi->AccessType, + pApi->pDevNode, + pApi->DevNodeLength, + pApi->pParmStr, + pApi->ParmStrLength, + pApi->pBinaryData, + &pApi->BinaryDataLength, + &pApi->Data, + &pApi->Entry); + break; + } + + case NV_ESC_RM_ALLOC_CONTEXT_DMA2: + { + NVOS39_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv04AllocContextDmaWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_BIND_CONTEXT_DMA: + { + NVOS49_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv04BindContextDmaWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_MAP_MEMORY_DMA: + { + NVOS46_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv04MapMemoryDmaWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_UNMAP_MEMORY_DMA: + { + NVOS47_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv04UnmapMemoryDmaWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_DUP_OBJECT: + { + NVOS55_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv04DupObjectWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_SHARE: + { + NVOS57_PARAMETERS *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + Nv04ShareWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_STATUS_CODE: + { + nv_state_t *pNv; + nv_ioctl_status_code_t *pApi = data; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + pNv = nv_get_adapter_state(pApi->domain, pApi->bus, pApi->slot); + if (pNv == NULL) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + rmStatus = rm_get_adapter_status(pNv, &pApi->status); + + if (rmStatus != NV_OK) + goto done; + + break; + } + + case NV_ESC_RM_CONTROL: + { + NVOS54_PARAMETERS *pApi = data; + void *priv = NULL; + nv_file_private_t *dev_nvfp = NULL; + NvS32 fd; + NvBool bSkipDeviceRef; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + rmStatus = RmGetDeviceFd(pApi, &fd, &bSkipDeviceRef); + if (rmStatus != NV_OK) + { + goto done; + } + + if (!bSkipDeviceRef) + { + dev_nvfp = nv_get_file_private(fd, NV_FALSE, &priv); + if (dev_nvfp == NULL) + { + rmStatus = NV_ERR_INVALID_DEVICE; + goto done; + } + + // Check to avoid cyclic dependency with NV_ESC_REGISTER_FD + if (!portAtomicCompareAndSwapU32(&dev_nvfp->register_or_refcount, + NVFP_TYPE_REFCOUNTED, + NVFP_TYPE_NONE)) + { + // Is this already refcounted... + if (dev_nvfp->register_or_refcount != NVFP_TYPE_REFCOUNTED) + { + nv_put_file_private(priv); + rmStatus = NV_ERR_IN_USE; + goto done; + } + } + + secInfo.gpuOsInfo = priv; + } + + Nv04ControlWithSecInfo(pApi, secInfo); + + if ((pApi->status != NV_OK) && (priv != NULL)) + { + // + // No need to reset `register_or_refcount` as it might be set + // for previous successful calls. We let it clear with FD close. + // + nv_put_file_private(priv); + + secInfo.gpuOsInfo = NULL; + } + + break; + } + + case NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO: + { + NVOS56_PARAMETERS *pApi = data; + void *pOldCpuAddress; + void *pNewCpuAddress; + + if (dataSize != sizeof(*pApi)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + pOldCpuAddress = NvP64_VALUE(pApi->pOldCpuAddress); + pNewCpuAddress = NvP64_VALUE(pApi->pNewCpuAddress); + + pApi->status = rm_update_device_mapping_info(pApi->hClient, + pApi->hDevice, + pApi->hMemory, + pOldCpuAddress, + pNewCpuAddress); + break; + } + + case NV_ESC_RM_LOCKLESS_DIAGNOSTIC: + { + NV_LOCKLESS_DIAGNOSTIC_PARAMS *pParams = data; + + if (dataSize != sizeof(*pParams)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + NV_CTL_DEVICE_ONLY(nv); + + if (!osIsAdministrator()) + { + rmStatus = NV_ERR_INSUFFICIENT_PERMISSIONS; + goto done; + } + + switch (pParams->cmd) + { + // Do not use NVOC _DISPATCH here as it dereferences NULL RmClientResource* + case NV0000_CTRL_CMD_NVD_GET_NVLOG_INFO: + rmStatus = cliresCtrlCmdNvdGetNvlogInfo_IMPL(NULL, &pParams->params.getNvlogInfo); + break; + case NV0000_CTRL_CMD_NVD_GET_NVLOG_BUFFER_INFO: + rmStatus = cliresCtrlCmdNvdGetNvlogBufferInfo_IMPL(NULL, &pParams->params.getNvlogBufferInfo); + break; + case NV0000_CTRL_CMD_NVD_GET_NVLOG: + rmStatus = cliresCtrlCmdNvdGetNvlog_IMPL(NULL, &pParams->params.getNvlog); + break; + default: + rmStatus = NV_ERR_NOT_SUPPORTED; + break; + } + + pParams->status = rmStatus; + goto done; + } + + case NV_ESC_REGISTER_FD: + { + nv_ioctl_register_fd_t *params = data; + void *priv = NULL; + nv_file_private_t *ctl_nvfp; + + if (dataSize != sizeof(*params)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus != NV_OK) + goto done; + + // If there is already a ctl fd registered on this nvfp, fail. + if (nvfp->ctl_nvfp != NULL) + { + // UNLOCK: release API lock + rmapiLockRelease(); + rmStatus = NV_ERR_INVALID_STATE; + goto done; + } + + // + // Note that this call is valid for both "actual" devices and ctrl + // devices. In particular, NV_ESC_ALLOC_OS_EVENT can be used with + // both types of devices. + // But, the ctl_fd passed in should always correspond to a control FD. + // + ctl_nvfp = nv_get_file_private(params->ctl_fd, + NV_TRUE, /* require ctl fd */ + &priv); + if (ctl_nvfp == NULL) + { + // UNLOCK: release API lock + rmapiLockRelease(); + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + // Disallow self-referential links, and disallow links to FDs that + // themselves have a link. + if ((ctl_nvfp == nvfp) || (ctl_nvfp->ctl_nvfp != NULL)) + { + nv_put_file_private(priv); + // UNLOCK: release API lock + rmapiLockRelease(); + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + // Check to avoid cyclic dependency with device refcounting + if (!portAtomicCompareAndSwapU32(&nvfp->register_or_refcount, + NVFP_TYPE_REGISTERED, + NVFP_TYPE_NONE)) + { + nv_put_file_private(priv); + // UNLOCK: release API lock + rmapiLockRelease(); + rmStatus = NV_ERR_IN_USE; + goto done; + } + + // + // nvfp->ctl_nvfp is read outside the lock, so set it atomically. + // Note that once set, this can never be removed until the fd + // associated with nvfp is closed. We hold on to 'priv' until the + // fd is closed, too, to ensure that the fd associated with + // ctl_nvfp remains valid. + // + portAtomicSetSize(&nvfp->ctl_nvfp, ctl_nvfp); + nvfp->ctl_nvfp_priv = priv; + + // UNLOCK: release API lock + rmapiLockRelease(); + + // NOTE: nv_put_file_private(priv) is not called here. It MUST be + // called during cleanup of this nvfp. + rmStatus = NV_OK; + break; + } + + default: + { + NV_PRINTF(LEVEL_ERROR, "unknown NVRM ioctl command: 0x%x\n", cmd); + goto done; + } + } + + rmStatus = NV_OK; +done: + + return rmStatus; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/exports-stubs.c b/userspace/sdks/nv570/nvalloc/unix/src/exports-stubs.c new file mode 100644 index 0000000..ff40300 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/exports-stubs.c @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include "os/os.h" +#include <rmconfig.h> +#include <gpu/subdevice/subdevice.h> +#include <ctrl/ctrl0080/ctrl0080unix.h> +#include <ctrl/ctrl2080/ctrl2080unix.h> + diff --git a/userspace/sdks/nv570/nvalloc/unix/src/gcc_helper.c b/userspace/sdks/nv570/nvalloc/unix/src/gcc_helper.c new file mode 100644 index 0000000..3c1037c --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/gcc_helper.c @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nvtypes.h> +#include <os-interface.h> + +void* memset(void* s, int c, NvUPtr n) +{ + return os_mem_set(s, (NvU8)c, (NvU32)n); +} + +void* memcpy(void* dest, const void* src, NvUPtr n) +{ + return os_mem_copy(dest, src, (NvU32)n); +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/os-hypervisor.c b/userspace/sdks/nv570/nvalloc/unix/src/os-hypervisor.c new file mode 100644 index 0000000..26c468f --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/os-hypervisor.c @@ -0,0 +1,994 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2014-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * + * @file os-hypervisor.c + * @brief OS specific Hypervisor interfaces for RM + * + */ + +#include "os/os.h" +#include "nv.h" +#include "nv-priv.h" +#include <nvrm_registry.h> +#include <virtualization/hypervisor/hypervisor.h> +#include "core/thread_state.h" +#include "core/locks.h" +#include "virtualization/kernel_vgpu_mgr.h" +#include "kernel/gpu/mig_mgr/kernel_mig_manager.h" +#include "kernel/gpu/fifo/kernel_fifo.h" +#include "osapi.h" +#include "virtualization/kernel_hostvgpudeviceapi.h" +#include "gpu/timer/objtmr.h" +#include "gpu/bif/kernel_bif.h" +#include "gpu/bus/kern_bus.h" +#include <nv_ref.h> // NV_PMC_BOOT_1_VGPU +#include "nvdevid.h" +#include "ctrl/ctrl0000/ctrl0000vgpu.h" + +#include "g_vgpu_chip_flags.h" // vGPU device names + +#define NV_VFIO_PCI_BAR0_REGION_INDEX 0 +#define NV_VFIO_PCI_BAR1_REGION_INDEX 1 +#define NV_VFIO_PCI_BAR2_REGION_INDEX 2 +#define NV_VFIO_PCI_BAR3_REGION_INDEX 3 + +static NV_STATUS nv_parse_config_params(const char *, const char *, const char, NvU32 *); + + +void hypervisorSetHypervVgpuSupported_IMPL(OBJHYPERVISOR *pHypervisor) +{ + pHypervisor->bIsHypervVgpuSupported = NV_TRUE; +} + +NvBool hypervisorIsVgxHyper_IMPL(void) +{ + return os_is_vgx_hyper(); +} + +NV_STATUS hypervisorInjectInterrupt_IMPL +( + OBJHYPERVISOR *pHypervisor, + VGPU_NS_INTR *pVgpuNsIntr +) +{ + NV_STATUS status = NV_ERR_NOT_SUPPORTED; + + if (osIsVgpuVfioPresent() == NV_OK) + return NV_ERR_NOT_SUPPORTED; + else + { + if (pVgpuNsIntr->guestMSIAddr && pVgpuNsIntr->guestMSIData) + { + status = os_inject_vgx_msi((NvU16)pVgpuNsIntr->guestDomainId, + pVgpuNsIntr->guestMSIAddr, + pVgpuNsIntr->guestMSIData); + } + } + + return status; +} + +HYPERVISOR_TYPE NV_API_CALL nv_get_hypervisor_type(void) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys); + return hypervisorGetHypervisorType(pHypervisor); +} + +#define MAX_STR_LEN 256 +NV_STATUS NV_API_CALL nv_vgpu_get_type_info( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvU32 vgpuTypeId, + char *buffer, + int type_info, + NvU8 devfn +) +{ + THREAD_STATE_NODE threadState; + OBJSYS *pSys = SYS_GET_INSTANCE(); + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + NV_STATUS rmStatus = NV_OK; + VGPU_TYPE *vgpuTypeInfo; + NvU32 pgpuIndex, i, avail_instances = 0; + OBJGPU *pGpu = NULL; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + if ((rmStatus = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex)) == + NV_OK) + { + for (i = 0; i < MAX_VGPU_TYPES_PER_PGPU; i++) + { + vgpuTypeInfo = pKernelVgpuMgr->pgpuInfo[pgpuIndex].vgpuTypes[i]; + if (vgpuTypeInfo == NULL) + break; + + if (vgpuTypeInfo->vgpuTypeId != vgpuTypeId) + continue; + + switch (type_info) + { + case VGPU_TYPE_INSTANCES: + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + if (pGpu == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", + __FUNCTION__); + rmStatus = NV_ERR_INVALID_STATE; + goto exit; + } + + rmStatus = kvgpumgrGetAvailableInstances(&avail_instances, pGpu, + vgpuTypeInfo, + pgpuIndex, devfn); + if (rmStatus != NV_OK) + goto exit; + + os_snprintf(buffer, MAX_STR_LEN, "%d\n", avail_instances); + break; + default: + rmStatus = NV_ERR_INVALID_ARGUMENT; + } + break; + } + } + +exit: + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_get_type_ids( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvU32 *numVgpuTypes, + NvU32 *vgpuTypeIds, + NvBool isVirtfn, + NvU8 devfn, + NvBool getCreatableTypes +) +{ + THREAD_STATE_NODE threadState; + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJGPU *pGpu = NULL; + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + NV_STATUS rmStatus = NV_OK; + NvU32 pgpuIndex, i, avail_instances = 0; + NvU32 numSupportedVgpuTypes = 0; + VGPU_TYPE *vgpuTypeInfo; + void *fp; + + if (!vgpuTypeIds || !numVgpuTypes) + return NV_ERR_INVALID_ARGUMENT; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + if ((rmStatus = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex)) == + NV_OK) + { + if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].sriovEnabled && !isVirtfn) + { + *numVgpuTypes = 0; + } + else + { + numSupportedVgpuTypes = pKernelVgpuMgr->pgpuInfo[pgpuIndex].numVgpuTypes; + *numVgpuTypes = 0; + + for (i = 0; i < numSupportedVgpuTypes; i++) + { + vgpuTypeInfo = pKernelVgpuMgr->pgpuInfo[pgpuIndex].vgpuTypes[i]; + + if (!getCreatableTypes) + { + // Return all available types + vgpuTypeIds[*numVgpuTypes] = vgpuTypeInfo->vgpuTypeId; + (*numVgpuTypes)++; + continue; + } + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + if (pGpu == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", + __FUNCTION__); + goto exit; + } + + rmStatus = kvgpumgrGetAvailableInstances(&avail_instances, pGpu, + vgpuTypeInfo, pgpuIndex, + devfn); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Failed to get available instances for vGPU ID: %d, status: 0x%x\n", + vgpuTypeInfo->vgpuTypeId, rmStatus); + continue; + } + + if (avail_instances == 0) + continue; + + vgpuTypeIds[*numVgpuTypes] = vgpuTypeInfo->vgpuTypeId; + (*numVgpuTypes)++; + } + } + } + +exit: + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_delete( + nvidia_stack_t *sp, + const NvU8 *pVgpuDevName, + NvU16 vgpuId +) +{ + THREAD_STATE_NODE threadState; + void *fp = NULL; + NvU32 rmStatus = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + rmStatus = kvgpumgrDeleteRequestVgpu(pVgpuDevName, vgpuId); + // UNLOCK: release API lock + rmapiLockRelease(); + } + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_process_vf_info( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvU8 cmd, + NvU32 domain, + NvU8 bus, + NvU8 slot, + NvU8 function, + NvBool isMdevAttached, + void *vf_pci_info +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + rmStatus = kvgpumgrProcessVfInfo(pNv->gpu_id, cmd, domain, bus, slot, function, isMdevAttached, (vgpu_vf_pci_info *) vf_pci_info); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_create_request( + nvidia_stack_t *sp, + nv_state_t *pNv, + const NvU8 *pVgpuDevName, + NvU32 vgpuTypeId, + NvU16 *vgpuId, + NvU32 *gpu_instance_id, + NvU32 *placement_id, + NvU32 gpuPciBdf +) +{ + THREAD_STATE_NODE threadState; + void *fp = NULL; + NV_STATUS rmStatus = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + rmStatus = kvgpumgrCreateRequestVgpu(pNv->gpu_id, pVgpuDevName, + vgpuTypeId, vgpuId, gpu_instance_id, placement_id, gpuPciBdf); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +static NV_STATUS is_bar_64bit( + OBJGPU *pGpu, + NvU32 regionIndex, + NvBool *isBar64bit +) +{ + NV_STATUS rmStatus = NV_OK; + + if (pGpu == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__); + rmStatus = NV_ERR_INVALID_STATE; + goto exit; + } + + switch (regionIndex) + { + case NV_VFIO_PCI_BAR0_REGION_INDEX: + *isBar64bit = pGpu->sriovState.b64bitVFBar0; + break; + + case NV_VFIO_PCI_BAR1_REGION_INDEX: + *isBar64bit = pGpu->sriovState.b64bitVFBar1; + break; + + case NV_VFIO_PCI_BAR2_REGION_INDEX: + *isBar64bit = pGpu->sriovState.b64bitVFBar2; + break; + + default: + NV_PRINTF(LEVEL_ERROR, "BAR%d region doesn't exist!\n", regionIndex); + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto exit; + break; + } + + NV_PRINTF(LEVEL_INFO, "BAR%d region is_64bit: %d\n", regionIndex, *isBar64bit);; + +exit: + return rmStatus; +} + +static NV_STATUS +_nv_vgpu_get_bar_size(OBJGPU *pGpu, KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice, + NvU32 regionIndex, NvU64 *size, NvU8 *configParams) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + NV_STATUS status; + KernelBus *pKernelBus; + NvU32 value = 0; + + pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + *size = kbusGetPciBarSize(pKernelBus, regionIndex); + + if (regionIndex == NV_VFIO_PCI_BAR1_REGION_INDEX) + { + VGPU_TYPE *vgpuTypeInfo; + NvU32 pgpuIndex = 0; + NvBool bOverrideBar1Size = NV_FALSE; + + // Read BAR1 length from vgpuTypeInfo + NV_ASSERT_OK_OR_RETURN(kvgpumgrGetVgpuTypeInfo(pKernelHostVgpuDevice->vgpuType, + &vgpuTypeInfo)); + + *size = vgpuTypeInfo->bar1Length << 20; + NV_ASSERT_OK_OR_RETURN(kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pGpu->gpuId, &pgpuIndex)); + + /* + * check for 'override_bar1_size' param in vgpuExtraParams list first, + * if param is missing there then check it in vgpu_params list + */ + status = nv_parse_config_params((const char*)vgpuTypeInfo->vgpuExtraParams, + "override_bar1_size", ';', &value); + if (status == NV_OK && value) + { + bOverrideBar1Size = NV_TRUE; + } + else if (status == NV_ERR_OBJECT_NOT_FOUND) + { + status = nv_parse_config_params((const char *)configParams, + "override_bar1_size", ',', &value); + if (status == NV_OK && value) + bOverrideBar1Size = NV_TRUE; + } + + if (gpuIsVfResizableBAR1Supported(pGpu)) + { + if ((*size > pGpu->sriovState.vfBarSize[1]) || + (!portStringCompare("Compute", (const char *)vgpuTypeInfo->vgpuClass, 7))) + { + *size = pGpu->sriovState.vfBarSize[1]; + } + } + + if (bOverrideBar1Size) + { + NvU64 bar1SizeInBytes, guestBar1; + NvU64 gpuBar1LowerLimit = 256 * 1024 * 1024; // bar1 lower limit for override_bar1_length parameter + bar1SizeInBytes = kbusGetPciBarSize(pKernelBus, NV_VFIO_PCI_BAR1_REGION_INDEX); + + if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].sriovEnabled) + { + *size = pGpu->sriovState.vfBarSize[1]; + } + else if (bar1SizeInBytes > gpuBar1LowerLimit) + { + guestBar1 = bar1SizeInBytes / vgpuTypeInfo->maxInstance; + *size = nvPrevPow2_U64(guestBar1); + } + } + } + else if (regionIndex == NV_VFIO_PCI_BAR2_REGION_INDEX || + regionIndex == NV_VFIO_PCI_BAR3_REGION_INDEX) + { + status = nv_parse_config_params((const char *)configParams, + "address64", ',', &value); + + if ((status != NV_OK) || ((status == NV_OK) && (value != 0))) + { + if (regionIndex == NV_VFIO_PCI_BAR2_REGION_INDEX) + *size = 0; + else if (regionIndex == NV_VFIO_PCI_BAR3_REGION_INDEX) + *size = kbusGetPciBarSize(pKernelBus, NV_VFIO_PCI_BAR2_REGION_INDEX); + } + } + + return NV_OK; +} + +NV_STATUS NV_API_CALL nv_vgpu_get_bar_info +( + nvidia_stack_t *sp, + nv_state_t *pNv, + const NvU8 *pVgpuDevName, + NvU64 *barSizes, + NvU64 *sparseOffsets, + NvU64 *sparseSizes, + NvU32 *sparseCount, + NvBool *isBar064bit, + NvU8 *configParams +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + OBJGPU *pGpu = NULL; + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit); + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + if (pGpu == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__); + rmStatus = NV_ERR_INVALID_STATE; + goto release_lock; + } + + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + nv_vgpu_rm_get_bar_info(pGpu, pVgpuDevName, barSizes, + sparseOffsets, sparseSizes, + sparseCount, isBar064bit, + configParams), + release_lock); +release_lock: + // UNLOCK: release API lock + rmapiLockRelease(); + +exit: + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_update_sysfs_info +( + nvidia_stack_t *sp, + nv_state_t *pNv, + const NvU8 *pVgpuDevName, + NvU32 mode, + NvU32 *sysfs_val +) +{ + THREAD_STATE_NODE threadState; + void *fp = NULL; + OBJSYS *pSys = SYS_GET_INSTANCE(); + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + REQUEST_VGPU_INFO_NODE *pRequestVgpu = NULL; + NV_STATUS rmStatus = NV_OK; + VGPU_SYSFS_OP updateMode = (VGPU_SYSFS_OP)mode; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit); + + // Check the corresponding REQUEST_VGPU_INFO_NODE + for (pRequestVgpu = listHead(&pKernelVgpuMgr->listRequestVgpuHead); + pRequestVgpu != NULL; + pRequestVgpu = listNext(&pKernelVgpuMgr->listRequestVgpuHead, pRequestVgpu)) + { + if (portMemCmp(pRequestVgpu->vgpuDevName, pVgpuDevName, VM_UUID_SIZE) == 0) + break; + } + + if (pRequestVgpu == NULL) + { + rmStatus = NV_ERR_OBJECT_NOT_FOUND; + goto release_lock; + } + + if (updateMode == SET_GPU_INSTANCE_ID) + rmStatus = kvgpumgrSetGpuInstanceId(pRequestVgpu, *sysfs_val); + else if (updateMode == SET_PLACEMENT_ID) + rmStatus = kvgpumgrSetPlacementId(pRequestVgpu, *sysfs_val); + else if (updateMode == GET_PLACEMENT_ID) + *sysfs_val = pRequestVgpu->placementId; + +release_lock: + // UNLOCK: release API lock + rmapiLockRelease(); + +exit: + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_get_hbm_info( + nvidia_stack_t *sp, + nv_state_t *pNv, + const NvU8 *pVgpuDevName, + NvU64 *hbmAddr, + NvU64 *size +) +{ + NV_STATUS rmStatus = NV_OK; + THREAD_STATE_NODE threadState; + OBJGPU *pGpu = NULL; + KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice; + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if ((size == NULL) || (hbmAddr == NULL)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto exit; + } + + // LOCK: acquire API lock + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit); + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + if (pGpu == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__); + rmStatus = NV_ERR_INVALID_STATE; + goto release_lock; + } + + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + kvgpumgrGetHostVgpuDeviceFromVgpuDevName(pNv->gpu_id, + pVgpuDevName, + &pKernelHostVgpuDevice), release_lock); + if (pKernelHostVgpuDevice->numValidHbmRegions > 1) + { + NV_PRINTF(LEVEL_NOTICE, "non contiguous HBM region is not supported\n"); + NV_ASSERT(0); + } + + if (pKernelHostVgpuDevice->hbmRegionList != NULL) + { + *hbmAddr = pKernelHostVgpuDevice->hbmRegionList[0].hbmBaseAddr; + *size = pKernelHostVgpuDevice->hbmRegionList[0].size; + } + else + { + *hbmAddr = 0; + *size = 0; + } + +release_lock: + // UNLOCK: release API lock + rmapiLockRelease(); +exit: + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +static NV_STATUS nv_parse_config_params( + const char *config_params, + const char *key, + const char delim, + NvU32 *config_value +) +{ + char *ptr, *configParams = rm_remove_spaces(config_params); + char *token, *value, *name; + NvU32 data; + NV_STATUS rmStatus = NV_ERR_OBJECT_NOT_FOUND; + + ptr = configParams; + while ((token = rm_string_token(&ptr, delim)) != NULL) + { + if (!(name = rm_string_token(&token, '=')) || !os_string_length(name)) + continue; + + if (!(value = rm_string_token(&token, '=')) || !os_string_length(value)) + continue; + + data = os_strtoul(value, NULL, 0); + + if (os_string_compare(name, key) == 0) + { + rmStatus = NV_OK; + *config_value = data; + } + } + + // Free the memory allocated by rm_remove_spaces() + os_free_mem(configParams); + + return rmStatus; +} + +static NV_STATUS _nv_vgpu_get_sparse_mmap( + OBJGPU *pGpu, + KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice, + NvU64 *offsets, + NvU64 *sizes, + NvU32 *numAreas, + NvU8 *configParams +) +{ + NV_STATUS rmStatus = NV_OK, status; + OBJTMR *pTmr = GPU_GET_TIMER(pGpu);; + KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);; + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + NvU32 value = 0; + + *numAreas = 0; + if (pKernelHostVgpuDevice->gfid != 0) + { + rmStatus = kbifGetVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice, + os_page_size, numAreas, NULL, NULL); + if (rmStatus == NV_OK) + { + if (*numAreas > NVA081_MAX_SPARSE_REGION_COUNT) + { + NV_PRINTF(LEVEL_ERROR, "Not enough space for sparse mmap region info\n"); + return NV_ERR_INSUFFICIENT_RESOURCES; + } + + + rmStatus = kbifGetVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice, os_page_size, + numAreas, offsets, sizes); + if (rmStatus != NV_OK) + return rmStatus; + } + } + else + { + status = nv_parse_config_params((const char *)configParams, + "direct_gpu_timer_access", ',', &value); + if ((status == NV_OK) && (value != 0)) + { + NvU64 offset = 0; + NvU32 size = 0; + + rmStatus = tmrGetTimerBar0MapInfo_HAL(pGpu, pTmr, &offset, &size); + + if (rmStatus == NV_OK) + { + offsets[*numAreas] = offset; + sizes[*numAreas] = size; + (*numAreas)++; + } + } + + value = 0; + { + NvU64 offset = 0; + NvU32 size = 0; + + status = kfifoGetUsermodeMapInfo_HAL(pGpu, pKernelFifo, &offset, &size); + + if (status == NV_OK) + { + offsets[*numAreas] = offset; + sizes[*numAreas] = size; + (*numAreas)++; + } + } + } + + return rmStatus; +} + +NV_STATUS nv_vgpu_rm_get_bar_info +( + OBJGPU *pGpu, + const NvU8 *pVgpuDevName, + NvU64 *barSizes, + NvU64 *sparseOffsets, + NvU64 *sparseSizes, + NvU32 *sparseCount, + NvBool *isBar064bit, + NvU8 *configParams +) +{ + KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice; + NV_STATUS rmStatus; + NvU32 i = 0; + + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + is_bar_64bit(pGpu, NV_VFIO_PCI_BAR0_REGION_INDEX, isBar064bit), + exit); + + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + kvgpumgrGetHostVgpuDeviceFromVgpuDevName(pGpu->gpuId, + pVgpuDevName, + &pKernelHostVgpuDevice), + exit); + + for (i = 0; i < NVA081_MAX_BAR_REGION_COUNT; i++) + { + /* + * For SRIOV, only VF BAR1 is queried via RM, others BARs are directly + * queried via VF config space in vgpu-vfio + */ + if (gpuIsSriovEnabled(pGpu) && (i != NV_VFIO_PCI_BAR1_REGION_INDEX)) + { + barSizes[i] = 0; + continue; + } + + rmStatus = _nv_vgpu_get_bar_size(pGpu, pKernelHostVgpuDevice, i, + &barSizes[i], configParams); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Failed to query BAR size for index %u 0x%x\n", + i, rmStatus); + goto exit; + } + } + + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + _nv_vgpu_get_sparse_mmap(pGpu, pKernelHostVgpuDevice, + sparseOffsets, sparseSizes, + sparseCount, configParams), + exit); + +exit: + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_gpu_unbind_event +( + nvidia_stack_t *sp, + NvU32 gpuId, + NvBool *isEventNotified +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_UNBIND eventData = { 0 }; + + // Send gpu_id so that nvidia-vgpu-mgr daemon knows which GPU is being unbound + eventData.gpuId = gpuId; + CliAddSystemEvent(NV0000_NOTIFIERS_VGPU_UNBIND_EVENT, &eventData, isEventNotified); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_gpu_bind_event( + nvidia_stack_t *sp, + NvU32 gpuId, + NvBool *isEventNotified +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_BIND eventData = { 0 }; + + eventData.gpuId = gpuId; + CliAddSystemEvent(NV0000_NOTIFIERS_VGPU_BIND_EVENT, &eventData, isEventNotified); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS osIsVgpuVfioPresent(void) +{ + vgpu_vfio_info vgpu_info; + + return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VGPU_VFIO_PRESENT); +} + +NV_STATUS osIsVfioPciCorePresent(void) +{ + vgpu_vfio_info vgpu_info; + + return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VFIO_PCI_CORE_PRESENT); +} + +void osWakeRemoveVgpu(NvU32 gpuId, NvU32 returnStatus) +{ + vgpu_vfio_info vgpu_info; + + vgpu_info.return_status = returnStatus; + vgpu_info.domain = GPU_32_BIT_ID_DECODE_DOMAIN(gpuId); + vgpu_info.bus = GPU_32_BIT_ID_DECODE_BUS(gpuId); + vgpu_info.device = GPU_32_BIT_ID_DECODE_DEVICE(gpuId); + + os_call_vgpu_vfio((void *)&vgpu_info, CMD_VFIO_WAKE_REMOVE_GPU); +} + +NvBool osIsGridSupported(OBJGPU *pGpu) +{ + return (os_is_grid_supported() || os_get_grid_csp_support() || hypervisorIsVgxHyper()); +} + +NvU32 osGetGridCspSupport(void) +{ + return os_get_grid_csp_support(); +} + +void initVGXSpecificRegistry(OBJGPU *pGpu) +{ + NvU32 data32; + osWriteRegistryDword(pGpu, NV_REG_STR_RM_POWER_FEATURES, 0x55455555); + osWriteRegistryDword(pGpu, NV_REG_STR_RM_INFOROM_DISABLE_BBX, + NV_REG_STR_RM_INFOROM_DISABLE_BBX_YES); + if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_DUMP_NVLOG, &data32) != NV_OK)) + { + osWriteRegistryDword(pGpu, NV_REG_STR_RM_DUMP_NVLOG, + NV_REG_STR_RM_DUMP_NVLOG_ENABLE); + } + osWriteRegistryDword(pGpu, NV_REG_STR_RM_RC_WATCHDOG, + NV_REG_STR_RM_RC_WATCHDOG_DISABLE); + osWriteRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _READ, _DISABLE) | + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _WRITE, _DISABLE)); +} + + +NV_STATUS rm_is_vgpu_supported_device( + OS_GPU_INFO *pOsGpuInfo, + NvU32 pmc_boot_1, + NvU32 pmc_boot_42 +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys); + NvBool is_sriov_enabled = FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1); + NvU32 i; + + // if not running in vGPU mode (guest VM) return NV_OK + if (!(pHypervisor && pHypervisor->bIsHVMGuest && + (FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _PV, pmc_boot_1) || + is_sriov_enabled))) + { + return NV_OK; + } + + if (!is_sriov_enabled) + { + return NV_OK; + } + + if (os_is_grid_supported() && !gpumgrIsVgxRmFirmwareCapableChip(pmc_boot_42)) + { + return NV_ERR_NOT_SUPPORTED; + } + + if (os_is_grid_supported()) + { + for (i = 0; i < NV_ARRAY_ELEMENTS(sVgpuUsmTypes); i++) + { + if (pOsGpuInfo->pci_info.device_id == sVgpuUsmTypes[i].ulDevID && + pOsGpuInfo->subsystem_id == sVgpuUsmTypes[i].ulSubID) + { + return NV_OK; + } + } + } + + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osIsVgpuDeviceVmPresent(void) +{ + return os_device_vm_present(); +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/os.c b/userspace/sdks/nv570/nvalloc/unix/src/os.c new file mode 100644 index 0000000..b441a66 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/os.c @@ -0,0 +1,6359 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> // NV device driver interface +#include <nv-priv.h> +#include <nv-caps.h> +#include <os/os.h> +#include <nvos.h> +#include <osapi.h> +#include <ctrl/ctrl0000/ctrl0000gpu.h> +#include <ctrl/ctrl0000/ctrl0000unix.h> +#include <class/cl90cd.h> // NV_EVENT_BUFFER_BIND +#include <nvdevid.h> + +#include <nverror.h> +#include <gpu/device/device.h> + +#include "gpu/gpu.h" +#include <gpu_mgr/gpu_mgr.h> +#include <platform/chipset/chipset.h> + +#include "nverror.h" +#include "kernel/gpu/bif/kernel_bif.h" + +#include "gpu/mig_mgr/kernel_mig_manager.h" + +#include "gpu/mem_sys/kern_mem_sys.h" + +#include "mem_mgr/io_vaspace.h" +#include <diagnostics/journal.h> +#include "kernel/diagnostics/xid_context.h" +#include "gpu/mem_mgr/mem_desc.h" +#include "gpu/mem_mgr/mem_mgr.h" +#include "core/thread_state.h" +#include <nvacpitypes.h> +#include <platform/acpi_common.h> +#include <core/locks.h> +#include <ctrl/ctrl2080/ctrl2080gpu.h> +#include "virtualization/hypervisor/hypervisor.h" +#include "rmobjexportimport.h" +#include <nvi2c.h> +#include "rmapi/rs_utils.h" +#include "rmapi/client_resource.h" +#include "os/dce_rm_client_ipc.h" +#include "mem_mgr/mem.h" +#include "gpu/mem_mgr/virt_mem_allocator_common.h" +#include "nvrm_registry.h" + +#include "vgpu/vgpu_util.h" + +#include "platform/chipset/chipset.h" + +#include <acpidsmguids.h> +#include <pex.h> +#include "gps.h" +#include "jt.h" + + + + +extern const char *ppOsBugCheckBugcodeStr[]; + + +ct_assert(NV_RM_PAGE_SIZE == RM_PAGE_SIZE); +ct_assert(NV_RM_PAGE_MASK == RM_PAGE_MASK); +ct_assert(NV_RM_PAGE_SHIFT == RM_PAGE_SHIFT); + +typedef struct +{ + NvU32 euid; + NvU32 pid; +} TOKEN_USER, *PTOKEN_USER; + +struct OS_RM_CAPS +{ + NvU32 count; + + // This should be the last element + nv_cap_t **caps; +}; + +NvBool osIsRaisedIRQL(void) +{ + return (!os_semaphore_may_sleep()); +} + +NvBool osIsISR(void) +{ + return os_is_isr(); +} + +NV_STATUS osGetDriverBlock +( + OS_GPU_INFO *pOsGpuInfo, + OS_DRIVER_BLOCK *pBlock +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NvU64 osGetMonotonicTimeNs(void) +{ + return os_get_monotonic_time_ns(); +} + +NV_STATUS osGetPerformanceCounter(NvU64 *pTimeInNs) +{ + *pTimeInNs = os_get_monotonic_time_ns_hr(); + return NV_OK; +} + +NV_STATUS osGetSystemTime( + NvU32 *pSeconds, + NvU32 *pMicroSeconds +) +{ + return os_get_system_time(pSeconds, pMicroSeconds); +} + +/*! + * @brief Get timestamp for logging. + * + * Everything that logs a time stamp should use this routine for consistency. + * + * The returned value is OS dependent. We want the time stamp to use + * KeQueryPerformanceCounter on Windows so it matches the DirectX timestamps. + * Linux uses microseconds since 1970 (osGetSystemTime), since matching DirectX + * is not a priority. + * + * osGetTimestampFreq returns the frequency required to decode the time stamps. + * + * @returns system dependent timestamp. + */ +NvU64 osGetTimestamp(void) +{ + NvU32 sec = 0; + NvU32 usec = 0; + osGetSystemTime(&sec, &usec); + return (NvU64)sec * 1000000 + usec; +} + +/*! + * @brief Get timestamp frequency. + * + * Timestamps are OS dependent. This call returns the frequency + * required to decode them. + * + * @returns Timestamp frequency. For example, 1000000 for MHz. + */ +NvU64 osGetTimestampFreq(void) +{ + return 1000000; +} + +NV_STATUS osDelay(NvU32 milliseconds) +{ + return os_delay(milliseconds); +} + +NV_STATUS osDelayUs(NvU32 microseconds) +{ + return os_delay_us(microseconds); +} + +NV_STATUS osDelayNs(NvU32 nanoseconds) +{ + NvU32 microseconds = NV_MAX(1, (nanoseconds / 1000)); + return os_delay_us(microseconds); +} + +NvU32 osGetCpuFrequency(void) +{ + /* convert os_get_cpu_frequency()'s return value from Hz to MHz */ + return ((NvU32)(os_get_cpu_frequency() / 1000000ULL)); +} + +void* osPciInitHandle( + NvU32 Domain, + NvU8 Bus, + NvU8 Slot, + NvU8 Function, + NvU16 *pVendor, + NvU16 *pDevice +) +{ + // + // Check if the BDF is for a GPU that's already been attached, for which + // we should already have a handle cached. This won't catch devices that + // have been probed but not yet attached, but that shouldn't be a common + // occurrence. + // + // More importantly, having this check here means we don't need to check + // a global list of devices in the kernel interface layer, which could + // have the implication of taking another lock, causing hairy lock + // ordering issues. + // + if (Function == 0) + { + OBJGPU *pGpu = gpumgrGetGpuFromBusInfo(Domain, Bus, Slot); + if (pGpu != NULL) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + if (pVendor) *pVendor = nv->pci_info.vendor_id; + if (pDevice) *pDevice = nv->pci_info.device_id; + return nv->handle; + } + } + + return os_pci_init_handle(Domain, Bus, Slot, Function, pVendor, pDevice); +} + +NvU8 osPciReadByte( + void *pHandle, + NvU32 Offset +) +{ + NvU8 val; + os_pci_read_byte(pHandle, Offset, &val); + return val; +} + +NvU16 osPciReadWord( + void *pHandle, + NvU32 Offset +) +{ + NvU16 val; + os_pci_read_word(pHandle, Offset, &val); + return val; +} + +NvU32 osPciReadDword( + void *pHandle, + NvU32 Offset +) +{ + NvU32 val; + os_pci_read_dword(pHandle, Offset, &val); + return val; +} + +void osPciWriteByte( + void *pHandle, + NvU32 Offset, + NvU8 Value +) +{ + os_pci_write_byte(pHandle, Offset, Value); +} + +void osPciWriteWord( + void *pHandle, + NvU32 Offset, + NvU16 Value +) +{ + os_pci_write_word(pHandle, Offset, Value); +} + +void osPciWriteDword( + void *pHandle, + NvU32 Offset, + NvU32 Value +) +{ + os_pci_write_dword(pHandle, Offset, Value); +} + +void* osMapKernelSpace( + RmPhysAddr Start, + NvU64 Size, + NvU32 Mode, + NvU32 Protect +) +{ + NvU64 offset; + NvU8 *ptr; + + if (0 == Size) + { + NV_ASSERT(Size != 0); + return NULL; + } + + offset = (Start & ~os_page_mask); + Start &= os_page_mask; + + if (!portSafeAddU64(Size, offset, &Size) || + !portSafeAddU64(Size, ~os_page_mask, &Size)) + { + return NULL; + } + Size &= os_page_mask; + + ptr = os_map_kernel_space(Start, Size, Mode); + if (ptr != NULL) + return (ptr + offset); + + return NULL; +} + +void osUnmapKernelSpace( + void *pAddress, + NvU64 Size +) +{ + NvU64 offset; + NvUPtr ptr = (NvUPtr)pAddress; + + if (0 == Size) + { + NV_ASSERT(Size != 0); + return; + } + + offset = (ptr & ~os_page_mask); + ptr &= os_page_mask; + Size = ((Size + offset + ~os_page_mask) & os_page_mask); + os_unmap_kernel_space((void *)ptr, Size); +} + +static NV_STATUS setNumaPrivData +( + KernelMemorySystem *pKernelMemorySystem, + nv_state_t *nv, + MEMORY_DESCRIPTOR *pMemDesc +) +{ + NV_STATUS rmStatus = NV_OK; + void *pAllocPrivate = NULL; + NvU64 *addrArray = NULL; + NvU64 numPages = pMemDesc->PageCount; + NvU64 numOsPages = numPages; + RmPhysAddr *pteArray = memdescGetPteArray(pMemDesc, AT_CPU); + NvU64 i; + + // + // Since this function will also deal with the memory allocated from PMA, + // we have to perform page size inflate/deflate here. + // + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NvU64 size = pMemDesc->ActualSize; + numOsPages = size / NV_MIN(os_page_size, pMemDesc->pageArrayGranularity); + } + + addrArray = portMemAllocNonPaged(numOsPages * sizeof(NvU64)); + if (addrArray == NULL) + { + return NV_ERR_NO_MEMORY; + } + + if (!IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc) && NV_RM_PAGE_SIZE < os_page_size) + { + numOsPages = NV_RM_PAGES_TO_OS_PAGES(numPages); + } + + if (!memdescGetContiguity(pMemDesc, AT_CPU)) + { + NvU64 copyByteSize = 0; + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + // pMemDesc->pageArraySize is calculated by 4KB, it should NOT smaller than `numOSPages`. + NV_ASSERT_OR_ELSE(pMemDesc->pageArraySize >= numOsPages, rmStatus = NV_ERR_INVALID_STATE; goto errors); + copyByteSize = numOsPages * sizeof(NvU64); + } + else + { + copyByteSize = numPages * sizeof(NvU64); + } + + portMemCopy((void*)addrArray, + copyByteSize, + (void*)pteArray, + copyByteSize); + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NvU64 size = pMemDesc->ActualSize; + // + // Because of PMA case, we need to convert page array + // from RM native granularity to OS granularity. + // + convertPageArrayBetweenOsAndRm( + addrArray, + size / pMemDesc->pageArrayGranularity, + pMemDesc->pageArrayGranularity, + size / os_page_size, + os_page_size, + &numOsPages + ); + } + else if (NV_RM_PAGE_SIZE < os_page_size) + { + RmDeflateRmToOsPageArray(addrArray, numPages); + } + + for (i = 0; i < numOsPages; i++) + { + // Update GPA to system physical address + addrArray[i] += pKernelMemorySystem->coherentCpuFbBase; + } + } + else + { + // + // Original PTE array in contiguous memdesc only holds start address. + // We need to fill the OS page array with adjacent page addresses to + // map contiguously. + // + NvU64 offset = pteArray[0] + pKernelMemorySystem->coherentCpuFbBase; + + for (i = 0; i < numOsPages; i++, offset += os_page_size) + { + addrArray[i] = offset; + } + } + + rmStatus = nv_register_phys_pages(nv, addrArray, numOsPages, NV_MEMORY_CACHED, &pAllocPrivate); + if (rmStatus != NV_OK) + { + goto errors; + } + + memdescSetMemData(pMemDesc, pAllocPrivate, NULL); + +errors: + portMemFree(addrArray); + + return rmStatus; +} + +NV_STATUS osGetNumMemoryPages +( + MEMORY_DESCRIPTOR *pMemDesc, + NvU32 *pNumPages +) +{ + void *pAllocPrivate = NULL; + + pAllocPrivate = memdescGetMemData(pMemDesc); + if (pAllocPrivate == NULL) + { + NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n"); + return NV_ERR_INVALID_STATE; + } + + return nv_get_num_phys_pages(pAllocPrivate, pNumPages); +} + +NV_STATUS osGetMemoryPages +( + MEMORY_DESCRIPTOR *pMemDesc, + void *pPages, + NvU32 *pNumPages +) +{ + void *pAllocPrivate = NULL; + + pAllocPrivate = memdescGetMemData(pMemDesc); + if (pAllocPrivate == NULL) + { + NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n"); + return NV_ERR_INVALID_STATE; + } + + return nv_get_phys_pages(pAllocPrivate, pPages, pNumPages); +} + +NV_STATUS osMapSystemMemory +( + MEMORY_DESCRIPTOR *pMemDesc, + NvU64 Offset, + NvU64 Length, + NvBool Kernel, + NvU32 Protect, + NvP64 *ppAddress, + NvP64 *ppPrivate +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + + NvU64 rootOffset = 0; + pMemDesc = memdescGetRootMemDesc(pMemDesc, &rootOffset); + Offset += rootOffset; + + RmPhysAddr userAddress; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NV_STATUS rmStatus = NV_OK; + void *pAllocPrivate = NULL; + void *pAddress; + void *pPrivate = NULL; + NvU64 pageIndex; + NvU32 pageOffset = (Offset & ~os_page_mask); + + if (pGpu != NULL && + pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) && + memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM) + { + KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); + + rmStatus = setNumaPrivData(pKernelMemorySystem, nv, pMemDesc); + if (rmStatus != NV_OK) + return rmStatus; + + pageOffset = memdescGetPhysAddr(pMemDesc, FORCE_VMMU_TRANSLATION(pMemDesc, AT_GPU), Offset) & ~os_page_mask; + } + + *ppAddress = NvP64_NULL; + *ppPrivate = NvP64_NULL; + + if ((Offset + Length) < Length) + return NV_ERR_INVALID_ARGUMENT; + if ((Offset + Length) > pMemDesc->Size) + return NV_ERR_INVALID_ARGUMENT; + + pageIndex = (Offset >> os_page_shift); + + pAllocPrivate = memdescGetMemData(pMemDesc); + if (!pAllocPrivate) + { + NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n"); + return NV_ERR_INVALID_STATE; + } + + if (Kernel) + { + pAddress = nv_alloc_kernel_mapping(nv, pAllocPrivate, + pageIndex, pageOffset, Length, &pPrivate); + if (pAddress == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "failed to create system memory kernel mapping!\n"); + rmStatus = NV_ERR_GENERIC; + } + else + { + *ppAddress = NV_PTR_TO_NvP64(pAddress); + *ppPrivate = NV_PTR_TO_NvP64(pPrivate); + } + } + else + { + rmStatus = nv_alloc_user_mapping(nv, pAllocPrivate, + pageIndex, pageOffset, Length, Protect, &userAddress, + &pPrivate); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "failed to create system memory user mapping!\n"); + } + else + { + *ppAddress = (NvP64)(userAddress); + *ppPrivate = NV_PTR_TO_NvP64(pPrivate); + } + } + + return rmStatus; +} + +void osUnmapSystemMemory +( + MEMORY_DESCRIPTOR *pMemDesc, + NvBool Kernel, + NvP64 pAddress, + NvP64 pPrivate +) +{ + void *pAllocPrivate; + OBJGPU *pGpu = pMemDesc->pGpu; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + pMemDesc = memdescGetRootMemDesc(pMemDesc, NULL); + pAllocPrivate = memdescGetMemData(pMemDesc); + + if (Kernel) + { + nv_free_kernel_mapping(nv, pAllocPrivate, NvP64_VALUE(pAddress), + NvP64_VALUE(pPrivate)); + } + else + { + nv_free_user_mapping(nv, pAllocPrivate, (NvU64)pAddress, + NvP64_VALUE(pPrivate)); + } + + if (pGpu != NULL && + pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) && + memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM && + pAllocPrivate != NULL) + { + nv_unregister_phys_pages(nv, pAllocPrivate); + memdescSetMemData(pMemDesc, NULL, NULL); + } +} + +void osIoWriteByte( + NvU32 Address, + NvU8 Value +) +{ + os_io_write_byte(Address, Value); +} + +NvU16 osIoReadWord( + NvU32 Address +) +{ + return os_io_read_word(Address); +} + +void osIoWriteWord( + NvU32 Address, + NvU16 Value +) +{ + os_io_write_word(Address, Value); +} + +NvU8 osIoReadByte( + NvU32 Address +) +{ + return os_io_read_byte(Address); +} + +NvBool osIsAdministrator(void) +{ + return os_is_administrator(); +} + +NvBool osCheckAccess(RsAccessRight accessRight) +{ + return os_check_access(accessRight); +} + +NvU32 osGetCurrentProcess(void) +{ + return os_get_current_process(); +} + +void osGetCurrentProcessName(char *ProcName, NvU32 Length) +{ + return os_get_current_process_name(ProcName, Length); +} + +NV_STATUS osGetCurrentThread(OS_THREAD_HANDLE *pThreadId) +{ + NV_STATUS rmStatus; + NvU64 threadId = 0; + + if (pThreadId == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + rmStatus = os_get_current_thread(&threadId); + if (rmStatus == NV_OK) + { + *pThreadId = threadId; + } + else + { + *pThreadId = 0; + } + + return rmStatus; +} + +void* osGetPidInfo(void) +{ + return os_get_pid_info(); +} + +void osPutPidInfo(void *pOsPidInfo) +{ + os_put_pid_info(pOsPidInfo); +} + +NV_STATUS osFindNsPid(void *pOsPidInfo, NvU32 *pNsPid) +{ + return os_find_ns_pid(pOsPidInfo, pNsPid); +} + +NvBool osIsInitNs(void) +{ + return os_is_init_ns(); +} + +NV_STATUS osAttachToProcess(void** ppProcessInfo, NvU32 ProcessId) +{ + // + // This function is used by RmUnmapMemory() to attach to the + // process for which a given device memory mapping was + // created, in order to be able to unmap it. On Linux/UNIX + // platforms, we can't "attach" to a random process, but + // since we don't create/destroy user mappings in the RM, we + // don't need to, either. + // + // Report success to the caller to keep RmUnmapMemory() from + // failing, and memory from being leaked as a result. + // + *ppProcessInfo = NULL; + return NV_OK; +} + +void osDetachFromProcess(void* pProcessInfo) +{ + // stub + return; +} + +NV_STATUS osAcquireRmSema(void *pSema) +{ + return NV_OK; +} + +NV_STATUS osCondAcquireRmSema(void *pSema) +{ + return NV_OK; +} + +NvU32 osReleaseRmSema(void *pSema, OBJGPU *pDpcGpu) +{ + return NV_SEMA_RELEASE_SUCCEED; +} + +void osSpinLoop(void) +{ + +} + +NvU64 osGetMaxUserVa(void) +{ + return os_get_max_user_va(); +} + +NV_STATUS osSchedule(void) +{ + return os_schedule(); +} + +NV_STATUS osQueueWorkItem( + OBJGPU *pGpu, + OSWorkItemFunction pFunction, + void *pParams, + OsQueueWorkItemFlags flags +) +{ + nv_work_item_t *pWi; + nv_state_t *nv; + NV_STATUS status; + + pWi = portMemAllocNonPaged(sizeof(nv_work_item_t)); + + if (NULL == pWi) + { + return NV_ERR_NO_MEMORY; + } + + pWi->flags = (OsQueueWorkItemFlags){0}; + pWi->flags.bDontFreeParams = flags.bDontFreeParams; + pWi->flags.bLockSema = flags.bLockSema; + pWi->flags.apiLock = flags.apiLock; + pWi->flags.bLockGpus = flags.bLockGpus; + pWi->flags.bLockGpuGroupDevice = flags.bLockGpuGroupDevice; + pWi->flags.bLockGpuGroupSubdevice = flags.bLockGpuGroupSubdevice; + pWi->flags.bFullGpuSanity = flags.bFullGpuSanity; + pWi->flags.bDropOnUnloadQueueFlush = flags.bDropOnUnloadQueueFlush; + pWi->flags.bSetExpandedGpuVisibility = flags.bSetExpandedGpuVisibility; + + pWi->flags.bRequiresGpu = NV_TRUE; + pWi->gpuInstance = gpuGetInstance(pGpu); + + pWi->func.pGpuFunction = pFunction; + pWi->pData = pParams; + nv = NV_GET_NV_STATE(pGpu); + + status = os_queue_work_item(nv ? nv->queue : NULL, pWi); + + if (NV_OK != status) + { + portMemFree((void *)pWi); + } + + return status; +} + +NV_STATUS osQueueSystemWorkItem( + OSSystemWorkItemFunction pFunction, + void *pParams +) +{ + nv_work_item_t *pWi; + NV_STATUS status; + + pWi = portMemAllocNonPaged(sizeof(nv_work_item_t)); + + if (NULL == pWi) + { + return NV_ERR_NO_MEMORY; + } + + pWi->flags = (OsQueueWorkItemFlags){0}; + pWi->func.pSystemFunction = pFunction; + pWi->pData = pParams; + + status = os_queue_work_item(NULL, pWi); + + if (NV_OK != status) + { + portMemFree((void *)pWi); + } + + return status; +} + +NV_STATUS osQueueSystemWorkItemWithFlags( + OSSystemWorkItemFunction pFunction, + void *pParams, + OsQueueWorkItemFlags flags +) +{ + nv_work_item_t *pWi; + NV_STATUS status; + + pWi = portMemAllocNonPaged(sizeof(nv_work_item_t)); + + if (NULL == pWi) + { + return NV_ERR_NO_MEMORY; + } + + pWi->flags = (OsQueueWorkItemFlags){0}; + pWi->flags.bDontFreeParams = flags.bDontFreeParams; + pWi->flags.bLockSema = flags.bLockSema; + pWi->flags.apiLock = flags.apiLock; + pWi->flags.bLockGpus = flags.bLockGpus; + pWi->flags.bLockGpuGroupDevice = flags.bLockGpuGroupDevice; + pWi->flags.bLockGpuGroupSubdevice = flags.bLockGpuGroupSubdevice; + pWi->flags.bFullGpuSanity = flags.bFullGpuSanity; + pWi->flags.bDropOnUnloadQueueFlush = flags.bDropOnUnloadQueueFlush; + pWi->func.pSystemFunction = pFunction; + pWi->pData = pParams; + + status = os_queue_work_item(NULL, pWi); + + if (NV_OK != status) + { + portMemFree((void *)pWi); + } + + return status; +} + +void osQueueMMUFaultHandler(OBJGPU *pGpu) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + nv_schedule_uvm_isr(nv); +} + +NvBool osGpuSupportsAts(OBJGPU *pGpu) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + // Checks ATS support from both OS side and RM side. + return nv->ats_support && + pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED); +} + +NV_STATUS osQueueDrainP2PHandler(NvU8 *pUuid) +{ + return nv_schedule_uvm_drain_p2p(pUuid); +} + +void osQueueResumeP2PHandler(NvU8 *pUuid) +{ + nv_schedule_uvm_resume_p2p(pUuid); +} + +static inline nv_dma_device_t* osGetDmaDeviceForMemDesc( + OS_GPU_INFO *pOsGpuInfo, + MEMORY_DESCRIPTOR *pMemDesc +) +{ + return (pOsGpuInfo->niso_dma_dev != NULL) && + memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) ? + pOsGpuInfo->niso_dma_dev : pOsGpuInfo->dma_dev; +} + +// +// Set the DMA address size for the given GPU +// +// This is a global device setting and care would need to be taken if it was to +// be modified outside of GPU initialization. At least on Linux other drivers, +// like UVM, might be requesting its own DMA mappings for the same GPU after +// the GPU has been initialized. +// +void osDmaSetAddressSize( + OS_GPU_INFO *pOsGpuInfo, + NvU32 bits +) +{ + nv_set_dma_address_size(pOsGpuInfo, bits); +} + +static NV_STATUS osGetPagesInfo( + MEMORY_DESCRIPTOR *pMemDesc, + NvU64 *pageSize, + NvU64 *osPageCount, + NvU64 *rmPageCount +) +{ + NvU64 osPageSize = osGetPageSize(); + NvU64 maxPageSize = NV_MAX(osPageSize, RM_PAGE_SIZE); + NvU64 alignedSize = NV_ALIGN_UP(pMemDesc->Size, maxPageSize); + + *osPageCount = alignedSize >> BIT_IDX_32(osPageSize); + *rmPageCount = alignedSize >> RM_PAGE_SHIFT; + *pageSize = memdescGetAdjustedPageSize(pMemDesc); + + // In the non-contig case need to protect against page array overflows. + if (!memdescGetContiguity(pMemDesc, AT_CPU)) + NV_ASSERT_OR_RETURN(*rmPageCount <= pMemDesc->pageArraySize, NV_ERR_INVALID_ARGUMENT); + + if (*osPageCount > NV_U32_MAX || *rmPageCount > NV_U32_MAX) + return NV_ERR_INVALID_LIMIT; + + return NV_OK; +} + +NV_STATUS osAllocPagesInternal( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJGPU *pGpu = pMemDesc->pGpu; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + void *pMemData = NULL; + NV_STATUS status; + NvS32 nodeId = memdescGetNumaNode(pMemDesc); + NV_ADDRESS_SPACE addrSpace = memdescGetAddressSpace(pMemDesc); + NvU64 pageSize; + NvU64 osPageCount; + NvU64 rmPageCount; + NvU32 cpuCacheAttrib = memdescGetCpuCacheAttrib(pMemDesc); + + memdescSetMemData(pMemDesc, NULL, NULL); + +#if (defined(NVCPU_AARCH64) && RMCFG_MODULE_CL) + { + OBJCL *pCl = SYS_GET_CL(pSys); + + if ((pCl != NULL) && + pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT) && + !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_NON_IO_COHERENT)) + { + cpuCacheAttrib = NV_MEMORY_CACHED; + } + } +#endif + + // + // For carveout, the memory is already reserved so we don't have + // to allocate memory. + // + if (memdescIsCarveoutMemory(pMemDesc) || + memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED)) + { + // We only support carveout with contiguous memory. + if (memdescIsCarveoutMemory(pMemDesc) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + status = NV_ERR_NOT_SUPPORTED; + goto done; + } + + status = osGetPagesInfo(pMemDesc, &pageSize, &osPageCount, &rmPageCount); + if (status != NV_OK) + goto done; + + if (!IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc) && + NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + rmPageCount); + } + + status = nv_alias_pages( + NV_GET_NV_STATE(pGpu), + osPageCount, + pageSize, + memdescGetContiguity(pMemDesc, AT_CPU), + cpuCacheAttrib, + memdescGetGuestId(pMemDesc), + memdescGetPteArray(pMemDesc, AT_CPU), + memdescIsCarveoutMemory(pMemDesc), + &pMemData); + } + else + { + NvBool unencrypted = 0; + + if (nv && (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_32BIT_ADDRESSABLE))) + nv->force_dma32_alloc = NV_TRUE; + + // + // If AMD SEV is enabled but APM(Ampere protected) or CC(Confidential + // Compute) mode is not enabled on the GPU, all RM and client + // allocations must be an unprotected sysmem. If APM is enabled and RM + // is allocating sysmem for its internal use, then it has to be + // unprotected as protected sysmem is not accessible to GPU. + // + unencrypted = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY); + + // + // OBJGPU/nv may be NULL if constructing NV01_MEMORY_DEVICELESS. Assume that + // NV01_MEMORY_DEVICELESS won't be used for display. + // + if (pGpu != NULL && + pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY) && + addrSpace == ADDR_SYSMEM) + { + if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) && + !NV_SOC_IS_ISO_IOMMU_PRESENT(nv)) + { + NV_PRINTF(LEVEL_INFO, "Forcing physically contiguous flags for ISO\n"); + memdescSetContiguity(pMemDesc, AT_CPU, NV_TRUE); + } + + if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) && + !NV_SOC_IS_NISO_IOMMU_PRESENT(nv)) + { + NV_PRINTF(LEVEL_INFO, "Forcing physically contiguous flags for NISO\n"); + memdescSetContiguity(pMemDesc, AT_CPU, NV_TRUE); + } + } + + status = osGetPagesInfo(pMemDesc, &pageSize, &osPageCount, &rmPageCount); + if (status != NV_OK) + goto done; + + status = nv_alloc_pages( + NV_GET_NV_STATE(pGpu), + osPageCount, // TODO: This call needs to receive the page count param at the requested page size. + pageSize, + memdescGetContiguity(pMemDesc, AT_CPU), + cpuCacheAttrib, + pSys->getProperty(pSys, + PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS), + unencrypted, + nodeId, + memdescGetPteArray(pMemDesc, AT_CPU), + &pMemData); + + if (nv && nv->force_dma32_alloc) + nv->force_dma32_alloc = NV_FALSE; + } + + if (status != NV_OK) + { + goto done; + } + + // Guest allocated memory is already initialized + if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED)) + { + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, osPageCount * os_page_size, os_page_size)); + } + else + { + NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, rmPageCount * RM_PAGE_SIZE, RM_PAGE_SIZE)); + } + } + + // + // If the OS layer doesn't think in RM page size, we need to inflate the + // PTE array into RM pages. + // + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NV_ASSERT_OR_RETURN(pMemDesc->pageArrayGranularity == os_page_size, NV_ERR_INVALID_ARGUMENT); + } + else if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + memdescSetMemData(pMemDesc, pMemData, NULL); + + if ((pGpu != NULL) && IS_VIRTUAL(pGpu)) + NV_ASSERT_OK_OR_RETURN(vgpuUpdateGuestSysmemPfnBitMap(pGpu, pMemDesc, NV_TRUE)); +done: + return status; +} + +void osFreePagesInternal( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + NV_STATUS rmStatus; + NvU64 osPageCount; + + if ((pGpu != NULL) && IS_VIRTUAL(pGpu)) + NV_ASSERT_OR_RETURN_VOID(vgpuUpdateGuestSysmemPfnBitMap(pGpu, pMemDesc, NV_FALSE) == NV_OK); + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NV_ASSERT(pMemDesc->pageArrayGranularity == os_page_size); + osPageCount = pMemDesc->PageCount; + } + else if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount); + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + else + { + osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount); + } + + rmStatus = nv_free_pages(NV_GET_NV_STATE(pGpu), + osPageCount, + memdescGetContiguity(pMemDesc, AT_CPU), + memdescGetCpuCacheAttrib(pMemDesc), + memdescGetMemData(pMemDesc)); + NV_ASSERT(rmStatus == NV_OK); +} + +NV_STATUS osLockMem( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + // Not supported on this OS. + DBG_BREAKPOINT(); + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osUnlockMem( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + // Not supported on this OS. + DBG_BREAKPOINT(); + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osMapPciMemoryAreaUser +( + OS_GPU_INFO *pOsGpuInfo, + MemoryArea memArea, + NvU32 protect, + NvU32 mode, + NvP64 *pVirtualAddress, + NvP64 *pPriv +) +{ + NV_STATUS status = NV_OK; + NvU64 origStart; + NvU64 origSize; + NvU64 diffStart; + + if (memArea.numRanges == 0) + { + *pVirtualAddress = (NvP64) NULL; + *pPriv = NULL; + return NV_OK; + } + + // Fix alignment insofar as we can (middle blocks can't be force-aligned this way) + origStart = memArea.pRanges[0].start; + memArea.pRanges[0].start = NV_ALIGN_DOWN64(origStart, os_page_size); + diffStart = origStart - memArea.pRanges[0].start; + memArea.pRanges[0].size += diffStart; + origSize = memArea.pRanges[memArea.numRanges - 1llu].size; + memArea.pRanges[memArea.numRanges - 1llu].size = NV_ALIGN_UP64(origSize, os_page_size); + + { + nv_usermap_access_params_t **ppNvuap, tNvuap; + NvU64 totalRangeSize = sizeof(MemoryRange) * memArea.numRanges; + + portMemSet(&tNvuap, 0, sizeof(nv_usermap_access_params_t)); + + tNvuap.memArea = memArea; + // access_size is only for caching, we can use os_page_size for now until linux has been properly plumbed + tNvuap.access_start = memArea.pRanges[0].start; + tNvuap.access_size = os_page_size; + + tNvuap.caching = mode; + tNvuap.contig = NV_TRUE; + + ppNvuap = (nv_usermap_access_params_t **) tlsEntryAcquire(TLS_ENTRY_ID_MAPPING_CONTEXT); + NV_ASSERT_OR_RETURN(ppNvuap != NULL, NV_ERR_INVALID_STATE); + + NV_ASSERT_OK_OR_GOTO(status, + os_alloc_mem((void**) ppNvuap, sizeof(nv_alloc_mapping_context_t)), + free_tls); + + portMemCopy(*ppNvuap, sizeof(nv_usermap_access_params_t), + &tNvuap, sizeof(nv_usermap_access_params_t)); + + NV_ASSERT_OK_OR_GOTO(status, + os_alloc_mem((void**) &((*ppNvuap)->memArea.pRanges), totalRangeSize), + free_nvuap); + + portMemCopy((*ppNvuap)->memArea.pRanges, totalRangeSize, memArea.pRanges, totalRangeSize); + + *pVirtualAddress = (NvP64) (memArea.pRanges[0].start + diffStart); + goto unalign_and_return; + +free_nvuap: + os_free_mem(*ppNvuap); +free_tls: + tlsEntryRelease(TLS_ENTRY_ID_MAPPING_CONTEXT); + } +unalign_and_return: + memArea.pRanges[memArea.numRanges - 1llu].size = origSize; + memArea.pRanges[0].size -= diffStart; + memArea.pRanges[0].start = origStart; + return status; +} + +NV_STATUS osMapPciMemoryUser( + OS_GPU_INFO *pOsGpuInfo, + RmPhysAddr busAddress, + NvU64 length, + NvU32 Protect, + NvP64 *pVirtualAddress, + NvP64 *pPriv, + NvU32 modeFlag +) +{ + MemoryArea memArea; + MemoryRange memRange; + + memArea.numRanges = 1; + memArea.pRanges = &memRange; + + memRange.start = busAddress; + memRange.size = length; + + return osMapPciMemoryAreaUser(pOsGpuInfo, memArea, Protect, modeFlag, pVirtualAddress, pPriv); +} + +void osUnmapPciMemoryUser( + OS_GPU_INFO *pOsGpuInfo, + NvP64 virtualAddress, + NvU64 length, + NvP64 pPriv +) +{ +} + +NV_STATUS osMapPciMemoryKernelOld +( + OBJGPU *pGpu, + RmPhysAddr busAddress, + NvU64 length, + NvU32 Protect, + void **pVirtualAddress, + NvU32 modeFlag +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_kern_mapping_t *mapping; + + if (pVirtualAddress == NULL) + return NV_ERR_GENERIC; + + *pVirtualAddress = os_map_kernel_space(busAddress, length, modeFlag); + if (*pVirtualAddress == NULL) + return NV_ERR_GENERIC; + + mapping = portMemAllocNonPaged(sizeof(nv_kern_mapping_t)); + if (NULL == mapping) + { + os_unmap_kernel_space(*pVirtualAddress, length); + *pVirtualAddress = 0; + return NV_ERR_GENERIC; + } + + mapping->addr = *pVirtualAddress; + mapping->size = length; + mapping->modeFlag = modeFlag; + + mapping->next = nv->kern_mappings; + nv->kern_mappings = mapping; + + return NV_OK; +} + +NV_STATUS osMapPciMemoryKernel64 +( + OBJGPU *pGpu, + RmPhysAddr busAddress, + NvU64 length, + NvU32 Protect, + NvP64 *pVirtualAddress, + NvU32 modeFlag +) +{ + void *tmppVirtualAddress = NvP64_VALUE(pVirtualAddress); + NV_STATUS rc; + + rc = osMapPciMemoryKernelOld(pGpu, + busAddress, + length, + Protect, + &tmppVirtualAddress, + modeFlag); + + *pVirtualAddress = NV_PTR_TO_NvP64(tmppVirtualAddress); + + return rc; +} + +void osUnmapPciMemoryKernelOld +( + OBJGPU *pGpu, + void* virtualAddress +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_kern_mapping_t *mapping, *tmp; + + // this can happen, for example, during a call to RmShutdownAdapter() + // from a failed RmInitAdapter() + if (virtualAddress == NULL) + { + return; + } + + tmp = mapping = nv->kern_mappings; + while (mapping) + { + if (mapping->addr == virtualAddress) + { + if (mapping == nv->kern_mappings) + { + nv->kern_mappings = mapping->next; + } + else + { + tmp->next = mapping->next; + } + + os_unmap_kernel_space(mapping->addr, mapping->size); + + portMemFree(mapping); + return; + } + tmp = mapping; + mapping = mapping->next; + } + + DBG_BREAKPOINT(); +} + +void osUnmapPciMemoryKernel64 +( + OBJGPU *pGpu, + NvP64 virtualAddress +) +{ + osUnmapPciMemoryKernelOld(pGpu, NvP64_VALUE(virtualAddress)); +} + +NV_STATUS osMapGPU( + OBJGPU *pGpu, + RS_PRIV_LEVEL privLevel, + NvU64 offset, + NvU64 length, + NvU32 Protect, + NvP64 *pAddress, + NvP64 *pPriv +) +{ + NV_STATUS rmStatus = NV_OK; + + if (privLevel >= RS_PRIV_LEVEL_KERNEL) + { + if (!portSafeAddU64((NvUPtr)pGpu->deviceMappings[0].gpuNvAddr, offset, (NvU64*)pAddress)) + { + rmStatus = NV_ERR_INVALID_LIMIT; + } + } + else + { + RmPhysAddr busAddress; + if (!portSafeAddU64(pGpu->busInfo.gpuPhysAddr, offset, &busAddress)) + { + rmStatus = NV_ERR_INVALID_LIMIT; + } + else + { + rmStatus = osMapPciMemoryUser(pGpu->pOsGpuInfo, + busAddress, + length, + Protect, + pAddress, + pPriv, + NV_MEMORY_UNCACHED); + } + } + + return rmStatus; +} + +void osUnmapGPU( + OS_GPU_INFO *pOsGpuInfo, + RS_PRIV_LEVEL privLevel, + NvP64 address, + NvU64 length, + NvP64 priv +) +{ + if (privLevel < RS_PRIV_LEVEL_KERNEL) + { + osUnmapPciMemoryUser(pOsGpuInfo, address, length, priv); + } +} + +static NV_STATUS referenceEventForPost( + nv_event_t *event, + NvBool bDoRefCount, + nv_event_t *eventSnapshot, + nv_file_private_t **ppNvfp +) +{ + nv_state_t *nv = nv_get_ctl_state(); + + portSyncSpinlockAcquire(nv->event_spinlock); + + if (event->active == 0) + { + portSyncSpinlockRelease(nv->event_spinlock); + return NV_ERR_INVALID_EVENT; + } + + if (bDoRefCount) + ++event->refcount; + + // Snapshot while event_spinlock still protects the event lifetime. + *eventSnapshot = *event; + *ppNvfp = eventSnapshot->nvfp; + portAtomicIncrementS32(&(*ppNvfp)->event_posting_refcount); + + portSyncSpinlockRelease(nv->event_spinlock); + return NV_OK; +} + +static void dereferenceEventForPost( + nv_event_t *event, + NvBool bDoRefCount, + nv_file_private_t *nvfp +) +{ + if (bDoRefCount) + { + nv_state_t *nv = nv_get_ctl_state(); + + portSyncSpinlockAcquire(nv->event_spinlock); + + NV_ASSERT(event->refcount > 0); + if (--event->refcount == 0 && !event->active) + portMemFree(event); + + portSyncSpinlockRelease(nv->event_spinlock); + } + + portAtomicDecrementS32(&nvfp->event_posting_refcount); +} + +static void postEvent( + nv_event_t *event, + NvU32 hEvent, + NvU32 notifyIndex, + NvU32 info32, + NvU16 info16, + NvBool dataValid, + NvBool bDoRefCount +) +{ + nv_event_t *eventToPost = event; + + nv_file_private_t *nvfp = NULL; + nv_event_t eventSnapshot; + + if (referenceEventForPost(event, bDoRefCount, &eventSnapshot, &nvfp) != NV_OK) + return; + + eventToPost = &eventSnapshot; + + nv_post_event(eventToPost, hEvent, notifyIndex, + info32, info16, dataValid); + + dereferenceEventForPost(bDoRefCount ? event : NULL, bDoRefCount, nvfp); +} + +NvU32 osSetEvent +( + OBJGPU *pGpu, + NvP64 eventID +) +{ + nv_event_t *event = NvP64_VALUE(eventID); + postEvent(event, 0, 0, 0, 0, NV_FALSE, NV_TRUE); + return 1; +} + +NV_STATUS osNotifyEvent( + OBJGPU *pGpu, + PEVENTNOTIFICATION NotifyEvent, + NvU32 Method, + NvU32 Data, + NV_STATUS Status, + NvBool bDoRefCount +) +{ + NV_STATUS rmStatus = NV_OK; + + // notify the event + switch (NotifyEvent->NotifyType) + { + case NV01_EVENT_OS_EVENT: + { + nv_event_t *event = NvP64_VALUE(NotifyEvent->Data); + postEvent(event, + NotifyEvent->hEvent, + NotifyEvent->NotifyIndex, + 0, 0, + NotifyEvent->bEventDataRequired, + bDoRefCount); + break; + } + + // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX. + case NV01_EVENT_KERNEL_CALLBACK: + { + MINIPORT_CALLBACK callBackToMiniport = + (MINIPORT_CALLBACK)NvP64_VALUE(NotifyEvent->Data); + + // perform a direct callback to the miniport + if (callBackToMiniport) + callBackToMiniport(NV_GET_NV_STATE(pGpu)); + break; + } + + case NV01_EVENT_KERNEL_CALLBACK_EX: + { + NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(NotifyEvent->Data); + + // passes two arguments (arg, params) to the kernel callback instead of one (arg). + if (kc && kc->func) + { + kc->func(kc->arg, NULL, NotifyEvent->hEvent, Data, Status); + } + break; + } + + + default: + { + rmStatus = NV_ERR_GENERIC; + break; + } + } + + return rmStatus; + +} // end of osNotifyEvent() + +// Allow CPL Events to be callback or events +NV_STATUS osEventNotification +( + OBJGPU *pGpu, + PEVENTNOTIFICATION pNotifyEvent, + NvU32 notifyIndex, + void * pEventData, + NvU32 eventDataSize +) +{ + return osEventNotificationWithInfo(pGpu, pNotifyEvent, notifyIndex, 0, 0, + pEventData, eventDataSize); +} + +NV_STATUS osEventNotificationWithInfo +( + OBJGPU *pGpu, + PEVENTNOTIFICATION pNotifyEvent, + NvU32 notifyIndex, + NvU32 info32, + NvU16 info16, + void * pEventData, + NvU32 eventDataSize +) +{ + NV_STATUS rmStatus = NV_OK; + NvBool bMatchSubdev = NV_FALSE; + + if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) && + (OS_EVENT_NOTIFICATION_INDEX_MATCH_SUBDEV & notifyIndex)) + { + bMatchSubdev = NV_TRUE; + notifyIndex &= ~OS_EVENT_NOTIFICATION_INDEX_MATCH_SUBDEV; + } + + // walk this object's event list and find any matches for this specific notify + for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next) + { + // notifyIndex must match if request isn't for all + if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) && + (pNotifyEvent->NotifyIndex != notifyIndex)) + { + continue; + } + + if (bMatchSubdev && (pGpu->subdeviceInstance != pNotifyEvent->subdeviceInst)) + continue; + + switch (pNotifyEvent->NotifyType) + { + case NV_EVENT_BUFFER_BIND: + case NV01_EVENT_WIN32_EVENT: + { + nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data); + postEvent(event, + pNotifyEvent->hEvent, + pNotifyEvent->NotifyIndex, + info32, info16, + pNotifyEvent->bEventDataRequired, + NV_TRUE); + break; + } + + // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX. + case NV01_EVENT_KERNEL_CALLBACK: + { + MINIPORT_CALLBACK callBackToMiniport = + (MINIPORT_CALLBACK)NvP64_VALUE(pNotifyEvent->Data); + + // perform a direct callback to the miniport + if (callBackToMiniport) + callBackToMiniport(NV_GET_NV_STATE(pGpu)); + break; + } + + case NV01_EVENT_KERNEL_CALLBACK_EX: + { + NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data); + + if (kc && kc->func) + { + kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK); + } + break; + } + + default: + break; + } + } + + return rmStatus; +} + +// Allow CPL Events to be callback or events +NV_STATUS osObjectEventNotification +( + NvHandle hClient, + NvHandle hObject, + NvU32 hClass, + PEVENTNOTIFICATION pNotifyEvent, + NvU32 notifyIndex, + void *pEventData, + NvU32 eventDataSize +) +{ + NV_STATUS rmStatus = NV_OK; + + NV_PRINTF(LEVEL_INFO, "%s()\n", __FUNCTION__); + // walk this object's event list and find any matches for this specific notify + for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next) + { + // notifyIndex must match if request isn't for all + if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) && + (pNotifyEvent->NotifyIndex != notifyIndex)) + { + continue; + } + + switch (pNotifyEvent->NotifyType) + { + case NV01_EVENT_OS_EVENT: + { + nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data); + postEvent(event, + pNotifyEvent->hEvent, + pNotifyEvent->NotifyIndex, + 0, 0, + pNotifyEvent->bEventDataRequired, + NV_TRUE); + break; + } + + case NV01_EVENT_KERNEL_CALLBACK_EX: + { + NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data); + + if (kc && kc->func) + { + kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK); + } + break; + } + + default: + break; + } + } + + return rmStatus; +} + +NV_STATUS osReferenceObjectCount(void *pEvent) +{ + nv_state_t *nv = nv_get_ctl_state(); + nv_event_t *event = pEvent; + + portSyncSpinlockAcquire(nv->event_spinlock); + // If event->active is false, don't allow any more reference + if (event->active == 0) + { + portSyncSpinlockRelease(nv->event_spinlock); + return NV_ERR_INVALID_EVENT; + } + ++event->refcount; + portSyncSpinlockRelease(nv->event_spinlock); + return NV_OK; +} + +NV_STATUS osDereferenceObjectCount(void *pOSEvent) +{ + nv_state_t *nv = nv_get_ctl_state(); + nv_event_t *event = pOSEvent; + + portSyncSpinlockAcquire(nv->event_spinlock); + NV_ASSERT(event->refcount > 0); + // If event->refcount == 0 but event->active is true, the client + // has not yet freed the OS event. free_os_event will free its + // memory when they do, or else when the client itself is freed. + if (--event->refcount == 0 && !event->active) + portMemFree(event); + portSyncSpinlockRelease(nv->event_spinlock); + + return NV_OK; +} + +NV_STATUS osUserHandleToKernelPtr(NvHandle hClient, NvP64 hEvent, NvP64 *pEvent) +{ + nv_state_t *nv = nv_get_ctl_state(); + NvU32 fd = (NvU64)hEvent; + NV_STATUS result; + + portSyncSpinlockAcquire(nv->event_spinlock); + nv_event_t *e = nv->event_list; + while (e != NULL) + { + if (e->fd == fd && e->hParent == hClient) + break; + e = e->next; + } + + if (e != NULL) + { + ++e->refcount; + *pEvent = NV_PTR_TO_NvP64(e); + result = NV_OK; + } + else + result = NV_ERR_OBJECT_NOT_FOUND; + portSyncSpinlockRelease(nv->event_spinlock); + + return result; +} + +NV_STATUS osFlushCpuCache(void) +{ + return os_flush_cpu_cache_all(); +} + +void osFlushCpuWriteCombineBuffer(void) +{ + os_flush_cpu_write_combine_buffer(); +} + + +// +// Evict GPU memory range from the CPU caches. +// +// On some platforms (e.g. P9+V100), the CPU can coherently cache GPU memory +// and RM takes advantage of that. Most everything is handled transparently, +// but there are two exceptions that require explicitly flushing any CPU cache +// lines of GPU memory. These are: +// +// 1) Flushing memory backing ACR regions before they get locked. +// +// Otherwise the cache could get flushed while the regions are locked causing a +// region violation physical fault. See more details in +// acrFlushRegionsFromGpuCoherentCpuCache_IMPL(). +// +// 2) Flushing all of FB before GPU reset (NVLink going down) +// +// Leaving cache entries on the CPU causes fatal errors when the CPU tries +// flushing them later while the link is down. See more details in +// nvlinkStatePostUnload_IMPL(). +// +void osFlushGpuCoherentCpuCacheRange +( + OS_GPU_INFO *pOsGpuInfo, + NvU64 cpuVirtual, + NvU64 size +) +{ + nv_flush_coherent_cpu_cache_range(pOsGpuInfo, cpuVirtual, size); +} + +void osErrorLogV(OBJGPU *pGpu, XidContext context, const char * pFormat, va_list arglist) +{ + NV_STATUS rmStatus; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((pFormat == NULL) || (*pFormat == '\0')) + { + return; + } + + rmStatus = nv_log_error(nv, context.xid, pFormat, arglist); + NV_ASSERT(rmStatus == NV_OK); +} + +void osErrorLog(OBJGPU *pGpu, NvU32 num, const char* pFormat, ...) +{ + va_list arglist; + va_start(arglist, pFormat); + osErrorLogV(pGpu, (XidContext){.xid = num}, pFormat, arglist); + va_end(arglist); +} + +NvU32 +osPollHotkeyState +( + OBJGPU *pGpu +) +{ + return 0; +} + +void osDevWriteReg008( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress, + NvV8 thisValue +) +{ + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + return; + } + + NV_PRIV_REG_WR08(pMapping->gpuNvAddr, thisAddress, thisValue); +} + +void osDevWriteReg016( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress, + NvV16 thisValue +) +{ + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + return; + } + + NV_PRIV_REG_WR16(pMapping->gpuNvAddr, thisAddress, thisValue); +} + +void osDevWriteReg032( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress, + NvV32 thisValue +) +{ + NvBool vgpuHandled = NV_FALSE; + + vgpuDevWriteReg032(pGpu, thisAddress, thisValue, &vgpuHandled); + if (vgpuHandled) + { + return; + } + + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + return; + } + + NV_PRIV_REG_WR32(pMapping->gpuNvAddr, thisAddress, thisValue); +} + +NvU8 osDevReadReg008( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress +) +{ + NvU8 retval = 0; + + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + } + else + retval = NV_PRIV_REG_RD08(pMapping->gpuNvAddr, thisAddress); + + return retval; +} + +NvU16 osDevReadReg016( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress +) +{ + NvU16 retval = 0; + + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + } + else + retval = NV_PRIV_REG_RD16(pMapping->gpuNvAddr, thisAddress); + + return retval; +} + +NvU32 osDevReadReg032( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress +) +{ + NvU32 retval = 0; + NvBool vgpuHandled = NV_FALSE; + + retval = vgpuDevReadReg032(pGpu, thisAddress, &vgpuHandled); + if (vgpuHandled) + { + return retval; + } + + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + } + else + retval = NV_PRIV_REG_RD32(pMapping->gpuNvAddr, thisAddress); + + return retval; +} + +NV_STATUS osReadRegistryDwordBase( + OBJGPU *pGpu, + const char *regParmStr, + NvU32 *Data +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmReadRegistryDword(nv, regParmStr, Data); +} + +NV_STATUS osWriteRegistryDword( + OBJGPU *pGpu, + const char *regParmStr, + NvU32 Data +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmWriteRegistryDword(nv, regParmStr, Data); +} + +NV_STATUS osReadRegistryBinary( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *Data, + NvU32 *cbLen +) +{ +#if defined(DEBUG) || defined(DEVELOP) + NV_ASSERT_OK_OR_RETURN(osValidateRegistryKeyPrefix(regParmStr)); +#endif + + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmReadRegistryBinary(nv, regParmStr, Data, cbLen); +} + +NV_STATUS osWriteRegistryBinary( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmWriteRegistryBinary(nv, regParmStr, Data, cbLen); +} + +NV_STATUS osWriteRegistryVolatile( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osReadRegistryVolatile +( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osReadRegistryVolatileSize +( + OBJGPU *pGpu, + const char *regParmStr, + NvU32 *pSize +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osReadRegistryStringBase( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *buffer, + NvU32 *pBufferLength +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmReadRegistryString(nv, regParmStr, buffer, pBufferLength); +} + +NV_STATUS osPackageRegistry( + OBJGPU *pGpu, + PACKED_REGISTRY_TABLE *pRegTable, + NvU32 *pSize +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmPackageRegistry(nv, pRegTable, pSize); +} + +NvU32 osGetCpuCount(void) +{ + return os_get_cpu_count(); // Total number of logical CPUs. +} + +NvU32 osGetCurrentProcessorNumber(void) +{ + return os_get_cpu_number(); +} + +ClientGroupID osClientGroupID(NvU64 process, void *pidInfo) +{ + return os_cgroup_for_pid(process, pidInfo); +} + +NvCgroupImpl osCgroupImplementation(void) +{ + switch (os_cgroup_implementation()) + { + case OS_CGROUP_IMPL_DMEM: return CGROUP_IMPL_OS; + case OS_CGROUP_IMPL_MISC: return CGROUP_IMPL_FALLBACK; + default: return CGROUP_IMPL_NONE; + } +} + +void *osCgroupRegisterRegion(OBJGPU *pGpu, NvU64 size) +{ + char name[32]; + nvDbgSnprintf(name, sizeof name, "nvidia/%04x:%02x:%02x/vidmem", + gpuGetDomain(pGpu), + gpuGetBus(pGpu), + gpuGetDevice(pGpu)); + return os_dmem_cgroup_register_region(size, name); +} + +void osCgroupUnregisterRegion(void *region) +{ + os_dmem_cgroup_unregister_region(region); +} + +NV_STATUS osMemacctTryCharge(void *pRegion, NvLength size, void **ppPool) +{ + return os_dmem_cgroup_try_charge(pRegion, size, ppPool, NULL); +} + +void osMemacctReleaseCharge(void *pool, NvLength size) +{ + os_dmem_cgroup_uncharge(pool, size); +} + +void *osCgroupGetFromFd(NvS32 fd) +{ + return os_cgroup_get_from_fd(fd); +} + +void osCgroupPut(void *cgroup) +{ + os_cgroup_put(cgroup); +} + +void osGetTimeoutParams(OBJGPU *pGpu, NvU32 *pTimeoutUs, NvU32 *pScale, NvU32 *pFlags) +{ + NvU32 gpuMode = gpuGetMode(pGpu); + + NV_ASSERT((NV_GPU_MODE_GRAPHICS_MODE == gpuMode) || + (NV_GPU_MODE_COMPUTE_MODE == gpuMode)); + + if (hypervisorIsVgxHyper()) + { + // + // 1.8 seconds is chosen because it is 90% of the overall hard limit of 2.0 + // seconds, imposed by WDDM driver rules. + // Currently primary use case of VGX is Windows, so setting 1.8 as default + // + *pTimeoutUs = 1.8 * 1000000; + } + else + { + switch (gpuMode) + { + default: + case NV_GPU_MODE_GRAPHICS_MODE: + *pTimeoutUs = 4 * 1000000; + break; + + case NV_GPU_MODE_COMPUTE_MODE: + *pTimeoutUs = 30 * 1000000; + break; + } + } + + *pFlags = GPU_TIMEOUT_FLAGS_OSTIMER; + + *pScale = 1; + if (IS_EMULATION(pGpu) || IS_SIMULATION(pGpu)) + { + *pScale = 60; // 1s -> 1m + } + + return; +} + +void osFlushLog(void) +{ + // Not implemented +} + +static NvU32 _osGetTegraPlatform(void) +{ + NV_STATUS status; + NvU32 mode; + + status = os_get_tegra_platform(&mode); + if (status != NV_ERR_NOT_SUPPORTED) + { + return mode; + } + + return NV_OS_TEGRA_PLATFORM_SILICON; +} + +NvU32 osGetSimulationMode(void) +{ + NvU32 mode; + + switch (_osGetTegraPlatform()) + { + case NV_OS_TEGRA_PLATFORM_SIM: + mode = NV_SIM_MODE_CMODEL; + break; + case NV_OS_TEGRA_PLATFORM_FPGA: + mode = NV_SIM_MODE_TEGRA_FPGA; + break; + case NV_OS_TEGRA_PLATFORM_SILICON: + default: + mode = NV_SIM_MODE_HARDWARE; + break; + } + + return mode; +} + +NV_STATUS +cliresCtrlCmdOsUnixFlushUserCache_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS *pAddressSpaceParams +) +{ + Memory *pMemory; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 start, end; + NvBool bInvalidateOnly; + + NV_CHECK_OK_OR_RETURN(LEVEL_SILENT, + memGetByHandle(RES_GET_CLIENT(pRmCliRes), + pAddressSpaceParams->hObject, + &pMemory)); + + pMemDesc = pMemory->pMemDesc; + + if (memdescGetAddressSpace(pMemDesc) != ADDR_SYSMEM) + { + NV_PRINTF(LEVEL_ERROR, "%s: wrong address space %d\n", + __FUNCTION__, memdescGetAddressSpace(pMemDesc)); + return NV_ERR_INVALID_COMMAND; + } + + if (memdescGetCpuCacheAttrib(pMemDesc) != NV_MEMORY_CACHED) + { + NV_PRINTF(LEVEL_ERROR, "%s: wrong caching type %d\n", + __FUNCTION__, memdescGetCpuCacheAttrib(pMemDesc)); + return NV_ERR_INVALID_COMMAND; + } + + start = pAddressSpaceParams->offset; + end = start + pAddressSpaceParams->length; + + switch(pAddressSpaceParams->cacheOps) + { + case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH_INVALIDATE: + case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH: + bInvalidateOnly = NV_FALSE; + break; + + case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_INVALIDATE: + bInvalidateOnly = NV_TRUE; + break; + + default: + NV_PRINTF(LEVEL_ERROR, "%s: cacheOps not specified\n", __FUNCTION__); + return NV_ERR_INVALID_COMMAND; + } + + if ((end - start) > pMemory->Length) + { + NV_PRINTF(LEVEL_ERROR, + "%s: end address 0x%llx exceeded buffer length: 0x%llx\n", + __FUNCTION__, end, pMemory->Length); + return NV_ERR_INVALID_LIMIT; + } + + if (bInvalidateOnly) + { + // + // XXX: this seems fishy - I'm not sure if invalidating by the kernel + // VA only as nv_dma_cache_invalidate() does here is sufficient for + // this control call. + // pAddressSpaceParams->internalOnly is expected to be the RM client + // VA for this control call; if we wanted to invalidate the user VA we + // could do so using that. + // + // For I/O coherent platforms this won't actually do anything. + // On non-I/O-coherent platforms, there's no need to do a second + // invalidation after the full flush. + // + nv_state_t *nv = NV_GET_NV_STATE(pMemDesc->pGpu); + if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) + { + PIOVAMAPPING pIovaMapping = memdescGetIommuMap(pMemDesc, nv->iovaspace_id); + // + // This should only be called for devices that map memory descriptors + // through the nv-dma library, where the memory descriptor data + // contains all the kernel-specific context we need for the + // invalidation. + // + // (These checks match those in osIovaUnmap() leading up to + // nv_dma_unmap_alloc()). + // + if (pIovaMapping == NULL || + pIovaMapping->pOsData == NULL || + memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED) || + memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + nv_dma_cache_invalidate(nv->dma_dev, pIovaMapping->pOsData); + } + else + { + return NV_ERR_INVALID_ARGUMENT; + } + } + else + { + return os_flush_user_cache(); + } + + return NV_OK; +} + +static NV_STATUS +_initializeExportObjectFd +( + nv_file_private_t *nvfp, + NvHandle hClient, + NvHandle hDevice, + NvU16 maxObjects, + NvU8 *metadata +) +{ + NV_STATUS status; + RsResourceRef *pResourceRef; + Device *pDevice; + NvU32 deviceInstance = NV_MAX_DEVICES; + NvU32 gpuInstanceId = NV_U32_MAX; + + if (nvfp->handles != NULL) + { + return NV_ERR_STATE_IN_USE; + } + + if (hDevice != 0) + { + OBJGPU *pGpu; + MIG_INSTANCE_REF ref; + + status = serverutilGetResourceRef(hClient, hDevice, &pResourceRef); + if (status != NV_OK) + { + return status; + } + + pDevice = dynamicCast(pResourceRef->pResource, Device); + if (pDevice == NULL) + { + return NV_ERR_INVALID_PARAMETER; + } + + deviceInstance = pDevice->deviceInst; + pGpu = GPU_RES_GET_GPU(pDevice); + + if (IS_MIG_IN_USE(pGpu)) + { + KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); + status = kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, + pDevice, &ref); + if (status == NV_OK) + { + gpuInstanceId = ref.pKernelMIGGpuInstance->swizzId; + } + } + } + + NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void **)&nvfp->handles, + sizeof(nvfp->handles[0]) * maxObjects)); + + os_mem_set(nvfp->handles, 0, + sizeof(nvfp->handles[0]) * maxObjects); + + nvfp->maxHandles = maxObjects; + nvfp->deviceInstance = deviceInstance; + nvfp->gpuInstanceId = gpuInstanceId; + + if (metadata != NULL) + { + os_mem_copy(nvfp->metadata, metadata, sizeof(nvfp->metadata)); + } + + return NV_OK; +} + +NV_STATUS +cliresCtrlCmdOsUnixExportObjectToFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS *pParams +) +{ + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + RmObjExportHandle hExportHandle = 0; + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + + /* + * This flag is intended to be implemented entirely in the rmapi library in + * userspace, we should never encounter it here. + */ + if (FLD_TEST_DRF(0000_CTRL, _OS_UNIX_EXPORT_OBJECT_TO_FD_FLAGS, + _EMPTY_FD, _TRUE, pParams->flags)) + { + return NV_ERR_INVALID_PARAMETER; + } + + if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM || + pParams->fd == -1) + { + return NV_ERR_INVALID_PARAMETER; + } + + status = RmExportObject(hClient, + pParams->object.data.rmObject.hObject, + &hExportHandle, NULL); + if (status != NV_OK) + { + goto done; + } + NV_ASSERT(hExportHandle != 0); + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + status = _initializeExportObjectFd(nvfp, hClient, + pParams->object.data.rmObject.hDevice, + 1, NULL); + if (status != NV_OK) + { + goto done; + } + + nvfp->handles[0] = hExportHandle; + +done: + + if (status != NV_OK && hExportHandle != 0) + { + RmFreeObjExportHandle(hExportHandle); + } + + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +// This control call has been deprecated. It will be deleted soon. +NV_STATUS +cliresCtrlCmdOsUnixCreateExportObjectFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS *pParams +) +{ + NV_STATUS status; + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + + ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata)); + + if (pParams->maxObjects == 0) + { + return NV_ERR_INVALID_PARAMETER; + } + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice, + pParams->maxObjects, pParams->metadata); + +done: + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixExportObjectsToFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS *pParams +) +{ + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + RmObjExportHandle *pExportHandle; + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + NvU32 i; + NvU32 deviceInstance; + NvU32 result; + NvHandle *exportHandles = NULL; + NvBool bFdSetup = NV_FALSE; + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata)); + + /* Setup export FD if not done */ + if (nvfp->handles == NULL) + { + if (pParams->maxObjects == 0) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice, + pParams->maxObjects, + pParams->metadata); + if (status != NV_OK) + { + goto done; + } + + bFdSetup = NV_TRUE; + } + + if ((nvfp->handles == NULL) || + (pParams->numObjects > + NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS)) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) || + (result > nvfp->maxHandles)) + { + status = NV_ERR_OUT_OF_RANGE; + goto done; + } + + status = os_alloc_mem((void **)&exportHandles, + sizeof(exportHandles[0]) * + pParams->numObjects); + if (status != NV_OK) + { + goto done; + } + + for (i = 0; i < pParams->numObjects; i++) + { + exportHandles[i] = 0; + + if (pParams->objects[i] == 0) + { + continue; + } + + status = RmExportObject(hClient, + pParams->objects[i], + &exportHandles[i], + &deviceInstance); + if (status != NV_OK) + { + goto done; + } + + NV_ASSERT(exportHandles[i] != 0); + + if (deviceInstance != nvfp->deviceInstance) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + } + + for (i = 0; i < pParams->numObjects; i++) + { + pExportHandle = &nvfp->handles[i + pParams->index]; + + // If the handle already exists in this position, free it + if (*pExportHandle != 0) + { + RmFreeObjExportHandle(*pExportHandle); + *pExportHandle = 0; + } + + *pExportHandle = exportHandles[i]; + } + +done: + + if ((status != NV_OK) && (exportHandles != NULL)) + { + for (i = 0; i < pParams->numObjects; i++) + { + if (exportHandles[i] != 0) + { + RmFreeObjExportHandle(exportHandles[i]); + } + } + } + + if (exportHandles != NULL) + { + os_free_mem(exportHandles); + } + + if ((status != NV_OK) && bFdSetup) + { + os_free_mem(nvfp->handles); + nvfp->handles = NULL; + nvfp->maxHandles = 0; + } + + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixImportObjectFromFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS *pParams +) +{ + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + + if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM || + pParams->fd == -1) + { + return NV_ERR_INVALID_PARAMETER; + } + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if ((nvfp->handles == NULL) || (nvfp->handles[0] == 0) || + (nvfp->maxHandles < 1)) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + status = RmImportObject(hClient, + pParams->object.data.rmObject.hParent, + &pParams->object.data.rmObject.hObject, + nvfp->handles[0], NULL); + +done: + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixImportObjectsFromFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS *pParams +) +{ + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + NvU32 i = 0; + RmObjExportHandle hImportHandle; + NvU32 result; + RM_API *pRmApi; + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if ((nvfp->handles == NULL) || + (pParams->numObjects > + NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS)) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) || + (result > nvfp->maxHandles)) + { + status = NV_ERR_OUT_OF_RANGE; + goto done; + } + + for (i = 0; i < pParams->numObjects; i++) + { + hImportHandle = nvfp->handles[i + pParams->index]; + + /* Nothing to import, just continue */ + if (hImportHandle == 0) + { + pParams->objectTypes[i] = \ + NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_NONE; + continue; + } + + status = RmImportObject(hClient, + pParams->hParent, + &pParams->objects[i], + hImportHandle, + &pParams->objectTypes[i]); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "%s: Unable to import handle (%x, %x, %x)\n", + __FUNCTION__, pParams->hParent, pParams->objects[i], hImportHandle); + goto done; + } + } + +done: + + if (status != NV_OK) + { + pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + while (i > 0) + { + i--; + + if (pParams->objects[i] != 0) + { + pRmApi->Free(pRmApi, hClient, pParams->objects[i]); + } + } + } + + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixGetExportObjectInfo_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS *pParams +) +{ + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + + if (pParams->fd < 0) + { + return NV_ERR_INVALID_PARAMETER; + } + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if (nvfp->handles == NULL) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + pParams->maxObjects = nvfp->maxHandles; + pParams->deviceInstance = nvfp->deviceInstance; + pParams->gpuInstanceId = nvfp->gpuInstanceId; + + os_mem_copy(pParams->metadata, nvfp->metadata, sizeof(nvfp->metadata)); + +done: + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixMemacctSetLimits_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_MEMACCT_SET_LIMITS_PARAMS *pParams +) +{ + void *cgroup = osCgroupGetFromFd(pParams->cgroupFd); + if (cgroup == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + NV_STATUS status = memacctSetLimits((ClientGroupID)cgroup, pParams->gpuId, pParams->softLimit, pParams->hardLimit); + osCgroupPut(cgroup); + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixMemacctGetLimits_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_MEMACCT_GET_LIMITS_PARAMS *pParams +) +{ + NvU32 gpuMask = 0; + NvU32 gpuIdx = 0; + OBJGPU *pGpu = NULL; + NV_STATUS status = NV_OK; + + pParams->count = 0; + NV_ASSERT_OK_OR_RETURN(gpumgrGetGpuAttachInfo(NULL, &gpuMask)); + + void *cgroup = osCgroupGetFromFd(pParams->cgroupFd); + if (cgroup == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + NvU32 i = 0; + while ((i < NV0000_CTRL_GPU_MAX_ATTACHED_GPUS) && + ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuIdx)) != NULL)) + { + NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, + memacctGetLimits((ClientGroupID)cgroup, pGpu->gpuId, + &pParams->limits[i].soft, + &pParams->limits[i].hard, + &pParams->limits[i].current), + out); + pParams->limits[i].gpuId = pGpu->gpuId; + i++; + } + pParams->count = i; +out: + osCgroupPut(cgroup); + return status; +} + +/*! + * osAcpiDsm + * + * @brief Handles os specific _DSM method function calls. + * + * Input parameters: + * @param[in] pGpu : OBJGPU pointer + * @param[in] acpiDsmFunction : ACPI DSM function + * @param[in] acpiDsmSubFunction : ACPI DSM subfunction + * @param[in/out] pInOut : in/out buffer, caller should make sure the buffer is large enough. + * @param[in] pSize : when input, size of data that the caller wants to read, in bytes. + * when output, size of valid data in pInOuta in bytes. + */ +NV_STATUS osCallACPI_DSM +( + OBJGPU *pGpu, + ACPI_DSM_FUNCTION acpiDsmFunction, + NvU32 acpiDsmSubFunction, + NvU32 *pInOut, + NvU16 *pSize +) +{ + NV_STATUS status; + NvU8 *pAcpiDsmGuid = NULL; + NvU32 acpiDsmRev; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU16 acpiDsmInArgSize = 4; + NvBool acpiNvpcfDsmFunction = NV_FALSE; + + // do any handling/remapping of guid needed. + status = checkDsmCall(pGpu, + (ACPI_DSM_FUNCTION *) &acpiDsmFunction, + &acpiDsmSubFunction, + pInOut, + pSize); + + // return if subfunction is not supported or we're returning cache data + if (status != NV_WARN_MORE_PROCESSING_REQUIRED) + { + return status; + } + + switch ((NvU32) acpiDsmFunction) + { + case ACPI_DSM_FUNCTION_NBSI: + pAcpiDsmGuid = (NvU8 *) &NBSI_DSM_GUID; + acpiDsmRev = NBSI_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_NVHG: + pAcpiDsmGuid = (NvU8 *) &NVHG_DSM_GUID; + acpiDsmRev = NVHG_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_MXM: + pAcpiDsmGuid = (NvU8 *) &DSM_MXM_GUID; + acpiDsmRev = ACPI_MXM_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_NBCI: + pAcpiDsmGuid = (NvU8 *) &NBCI_DSM_GUID; + acpiDsmRev = NBCI_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_NVOP: + pAcpiDsmGuid = (NvU8 *) &NVOP_DSM_GUID; + acpiDsmRev = NVOP_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_PCFG: + pAcpiDsmGuid = (NvU8 *) &PCFG_DSM_GUID; + acpiDsmRev = PCFG_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_GPS_2X: + pAcpiDsmGuid = (NvU8 *) &GPS_DSM_GUID; + acpiDsmRev = GPS_2X_REVISION_ID; + acpiDsmInArgSize = (*pSize); + break; + case ACPI_DSM_FUNCTION_GPS: + if (IsTU10X(pGpu)) + { + pAcpiDsmGuid = (NvU8 *) &GPS_DSM_GUID; + acpiDsmRev = GPS_REVISION_ID; + acpiDsmInArgSize = (*pSize); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } + break; + case ACPI_DSM_FUNCTION_PEX: + pAcpiDsmGuid = (NvU8 *) &PEX_DSM_GUID; + acpiDsmRev = PEX_REVISION_ID; + if (acpiDsmSubFunction == PEX_FUNC_SETLTRLATENCY) + { + acpiDsmInArgSize = (3 + *pSize); + } + break; + case (ACPI_DSM_FUNCTION_JT): + pAcpiDsmGuid = (NvU8 *) &JT_DSM_GUID; + acpiDsmRev = JT_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_NVPCF: + { + pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID; + acpiDsmRev = NVPCF_ACPI_DSM_REVISION_ID; + acpiDsmInArgSize = (*pSize); + acpiNvpcfDsmFunction = NV_TRUE; + break; + } + case ACPI_DSM_FUNCTION_NVPCF_2X: + pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID; + acpiDsmRev = NVPCF_2X_ACPI_DSM_REVISION_ID; + acpiDsmInArgSize = (*pSize); + acpiNvpcfDsmFunction = NV_TRUE; + break; + + default: + return NV_ERR_NOT_SUPPORTED; + break; + } + + status = nv_acpi_dsm_method(nv, + pAcpiDsmGuid, + acpiDsmRev, + acpiNvpcfDsmFunction, + acpiDsmSubFunction, + pInOut, + acpiDsmInArgSize, + NULL, + pInOut, + pSize); + + if (status == NV_OK) + { + if (acpiDsmSubFunction == NV_ACPI_ALL_FUNC_SUPPORT) + { + // if handling get supported functions list... cache it for later calls + cacheDsmSupportedFunction(pGpu, acpiDsmFunction, acpiDsmSubFunction, pInOut, *pSize); + } + } + else if (nvp->b_mobile_config_enabled) + { + NV_PRINTF(LEVEL_ERROR, + "osCallACPI_DSM: Error during 0x%x DSM subfunction 0x%x! status=0x%x\n", + acpiDsmFunction, acpiDsmSubFunction, status); + } + + return status; +} + +NV_STATUS osCallACPI_DOD +( + OBJGPU *pGpu, + NvU32 *pOut, + NvU32 *pSize +) +{ + NV_STATUS rmStatus; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((pOut == NULL) || (pSize == NULL)) + { + return NV_ERR_INVALID_POINTER; + } + + rmStatus = nv_acpi_dod_method(nv, pOut, pSize); + + return rmStatus; +} + +// +// osAcpiDdc +// +// Handles os specific _DDC method function calls. _DDC is to get EDID from SBIOS. +// +NV_STATUS osCallACPI_DDC +( + OBJGPU *pGpu, + NvU32 ulAcpiId, + NvU8 *pOutData, + NvU32 *pOutSize, + NvBool bReadMultiBlock +) +{ + NV_STATUS rmStatus; + + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((pOutData == NULL) || (pOutSize == NULL)) + { + return NV_ERR_INVALID_POINTER; + } + + portMemSet(pOutData, 0, *pOutSize); + + rmStatus = nv_acpi_ddc_method(nv, pOutData, pOutSize, bReadMultiBlock); + + return rmStatus; +} + +// osCallACPI_NVHG_ROM +// Making ACPI Call into SBIOS with ROM method to get display device's ROM data. +// +NV_STATUS osCallACPI_NVHG_ROM +( + OBJGPU *pGpu, + NvU32 *pInData, + NvU32 *pOutData +) +{ + NV_STATUS rmStatus; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((pOutData == NULL) || (pInData == NULL)) + { + return NV_ERR_INVALID_POINTER; + } + + if (pInData[1] > ROM_METHOD_MAX_RETURN_BUFFER_SIZE) + { + return NV_ERR_INVALID_ARGUMENT; + } + + rmStatus = nv_acpi_rom_method(nv, pInData, pOutData); + + return rmStatus; +} + +void osInitSystemStaticConfig(SYS_STATIC_CONFIG *pConfig) +{ + pConfig->bIsNotebook = rm_is_system_notebook(); + pConfig->bOsCCEnabled = os_cc_enabled; + pConfig->bOsCCSevSnpEnabled = os_cc_sev_snp_enabled; + pConfig->bOsCCSmeEnabled = os_cc_sme_enabled; + pConfig->bOsCCSnpVtomEnabled = os_cc_snp_vtom_enabled; + pConfig->bOsCCTdxEnabled = os_cc_tdx_enabled; +} + +NvU32 osApiLockAcquireConfigureFlags(NvU32 flags) +{ + return flags; +} + +NV_STATUS osGpuLocksQueueRelease(OBJGPU *pGpu, NvU32 dpcGpuLocksRelease) +{ + return NV_SEMA_RELEASE_FAILED; +} + +void osSyncWithRmDestroy(void) +{ +} + +void osSyncWithGpuDestroy(NvBool bEntry) +{ +} + +void osModifyGpuSwStatePersistence +( + OS_GPU_INFO *pOsGpuInfo, + NvBool bEnable +) +{ + if (!osIsLegacyPersistenceModeSupported(pOsGpuInfo)) + return; + + if (bEnable) + { + pOsGpuInfo->flags |= NV_FLAG_PERSISTENT_SW_STATE; + } + else + { + pOsGpuInfo->flags &= ~NV_FLAG_PERSISTENT_SW_STATE; + } +} + +// +//osCallACPI_MXDS +// +//Handles OS specific MXDS function call. +// +NV_STATUS osCallACPI_MXDS +( + OBJGPU *pGpu, + NvU32 acpiId, + NvU32 *pInOut +) +{ + NV_STATUS rmStatus; + + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if (pInOut == NULL) + { + return NV_ERR_INVALID_POINTER; + } + + rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDS"); + + return rmStatus; +} + +// +//osCallACPI_MXDM +// +//Handles OS specific MXDM function call. +// +NV_STATUS osCallACPI_MXDM +( + OBJGPU *pGpu, + NvU32 acpiId, + NvU32 *pInOut +) +{ + NV_STATUS rmStatus; + + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if (pInOut == NULL) + { + return NV_ERR_INVALID_POINTER; + } + + rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDM"); + + return rmStatus; +} + +#include "lib/protobuf/prb.h" +#include "lib/protobuf/prb_util.h" +#include "g_nvdebug_pb.h" + +NV_STATUS osGetVersionDump(void * pVoid) +{ + PRB_ENCODER * pPrbEnc = (PRB_ENCODER *)pVoid; + NV_STATUS rmStatus; + os_version_info * pOsVersionInfo = NULL; + const char NV_UNKNOWN_BUILD_VERSION[] = "Unknown build version"; + const char NV_UNKNOWN_BUILD_DATE[] = "Unknown build date"; + + NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void**)&pOsVersionInfo, + sizeof(os_version_info))); + portMemSet(pOsVersionInfo, 0, sizeof(os_version_info)); + + prbEncAddUInt32(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_FAMILY, + NVDEBUG_OS_UNIX); + + rmStatus = os_get_version_info(pOsVersionInfo); + if (rmStatus != NV_OK) + { + goto cleanup; + } + + prbEncAddUInt32(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_OSMAJORVERSION, + pOsVersionInfo->os_major_version); + + prbEncAddUInt32(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_OSMINORVERSION, + pOsVersionInfo->os_minor_version); + + prbEncAddUInt32(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_OSBLDNUM, + pOsVersionInfo->os_build_number); + + if (NULL == pOsVersionInfo->os_build_version_str) + { + pOsVersionInfo->os_build_version_str = NV_UNKNOWN_BUILD_VERSION; + } + + prbEncAddString(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_BUILDVERSION, + pOsVersionInfo->os_build_version_str); + + if (NULL == pOsVersionInfo->os_build_date_plus_str) + { + pOsVersionInfo->os_build_date_plus_str = NV_UNKNOWN_BUILD_DATE; + } + + prbEncAddString(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_BUILDDATEPLUS, + pOsVersionInfo->os_build_date_plus_str); + +cleanup: + os_free_mem(pOsVersionInfo); + return rmStatus; +} + +NV_STATUS osTegraiGpuPerfBoost(OBJGPU *pGpu, NvBool enable, NvU32 duration) +{ + return os_tegra_igpu_perf_boost(pGpu->pOsGpuInfo, enable, duration); +} + +NV_STATUS osGetVersion(NvU32 *majorVer, NvU32 *minorVer, NvU32 *buildNum, NvU16 *unusedPatchVersion, NvU16 *unusedProductType) +{ + os_version_info osVersionInfo; + NV_STATUS rmStatus; + + portMemSet(&osVersionInfo, 0, sizeof(osVersionInfo)); + + rmStatus = os_get_version_info(&osVersionInfo); + if (rmStatus == NV_OK) + { + if (majorVer) + *majorVer = osVersionInfo.os_major_version; + if (minorVer) + *minorVer = osVersionInfo.os_minor_version; + if (buildNum) + *buildNum = osVersionInfo.os_build_number; + } + + return rmStatus; +} + +NV_STATUS osGetIsOpenRM(NvBool *bOpenRm) +{ + return os_get_is_openrm(bOpenRm); +} + +NvBool osIsBifResetSupported(OS_GPU_INFO *pOsGpuInfo) +{ + return os_is_bif_reset_supported((void *)pOsGpuInfo); +} + +NV_STATUS +osGetCarveoutInfo +( + NvU64 *pAddr, + NvU64 *pSize +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osGetVPRInfo +( + NvU64 *pAddr, + NvU64 *pSize +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osAllocInVPR +( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osGetGenCarveout +( + NvU64 *pAddr, + NvU64 *pSize, + NvU32 id, + NvU64 align +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osI2CClosePorts +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 numPorts +) +{ + nv_i2c_unregister_clients(pOsGpuInfo); + return NV_OK; +} + +static NvU32 +i2cSwPortMapping( + nv_state_t *nv, + NvU32 physicalI2CPort +) +{ + NvU32 linuxI2CSwPort = NV_U32_MAX; + + /** + * For T23X, Linux Tegra I2C controller driver uses logical port(controller) number + * where logical port number of I2C1(Gen1) controller is 0, logical port number for + * I2C2(Gen2) controller is 1 and so on. But RM passes I2C physical port(controller) + * number i.e RM passes "1" for I2C1(Gen1), 2 for I2C2(Gen2), etc. So convert + * physical port number to logical port number(linuxI2CSwPort). + * + * For other chips, the above mentioned logic does not apply and we do not need + * conversion as the physical controller number was updated to 0 and RM passes the + * same. + */ + if ((nv->disp_sw_soc_chip_id == NV_CHIP_ID_T234) + || (nv->disp_sw_soc_chip_id == NV_CHIP_ID_T239) + ) + { + linuxI2CSwPort = physicalI2CPort - 1; + } + else + { + linuxI2CSwPort = physicalI2CPort; + } + + return linuxI2CSwPort; +} + +NV_STATUS +osI2CTransfer +( + OBJGPU *pGpu, + NvU32 Port, + NvU8 Address, + nv_i2c_msg_t *nv_i2c_msgs, + NvU32 count +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + if (NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + Port = i2cSwPortMapping(nv, Port); + return nv_i2c_transfer(nv, Port, Address, + nv_i2c_msgs, count); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +NV_STATUS +osTegraI2CGetBusState +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 port, + NvS32 *scl, + NvS32 *sda +) +{ + if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo)) + { + port = i2cSwPortMapping(pOsGpuInfo, port); + return nv_i2c_bus_status(pOsGpuInfo, port, scl, sda); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +NV_STATUS +osReadI2CBufferDirect +( + OBJGPU *pGpu, + NvU32 Port, + NvU8 Address, + void *pOutputBuffer, + NvU32 OutputSize, + void *pInputBuffer, + NvU32 InputSize +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osWriteI2CBufferDirect +( + OBJGPU *pGpu, + NvU32 Port, + NvU8 Address, + void *pOutputBuffer0, + NvU32 OutputSize0, + void *pOutputBuffer1, + NvU32 OutputSize1 +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osGC6PowerControl +( + OBJGPU *pGpu, + NvU32 cmd, + NvU32 *pOut +) +{ + NV_STATUS status; + NvU32 inOut = cmd; + NvU16 rtnSize = sizeof(inOut); + + if (FLD_TEST_DRF(_JT_FUNC, _POWERCONTROL, _GPU_POWER_CONTROL, _GSS, inOut)) + { + if (!pOut) + { + return NV_ERR_INVALID_ARGUMENT; + } + } + + status = osCallACPI_DSM(pGpu, + ACPI_DSM_FUNCTION_JT, + JT_FUNC_POWERCONTROL, + &inOut, + &rtnSize); + + if ((status != NV_OK) || !pOut) + { + return status; + } + + *pOut = inOut; + + return NV_OK; +} + +NvBool osTestPcieExtendedConfigAccess(void *handle, NvU32 offset) +{ + OBJGPU *pGpu; + KernelBif *pKernelBif; + NvU32 nvXveId = 0; + NvU32 nvXveVccapHdr = 0; + NvU32 pciStart = 0; + NvU32 pcieStart = 0; + + static NvBool retryAllowed = NV_TRUE; + static NvBool configAccess = NV_FALSE; + + // + // Return early for offset within PCI space + // and does not requires extended config space access + // + if (offset < 0x100) + { + return NV_TRUE; + } + + if (!retryAllowed) + { + return configAccess; + } + + pGpu = gpumgrGetSomeGpu(); + if (pGpu == NULL) + { + return configAccess; + } + + retryAllowed = NV_FALSE; + + pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + if (pKernelBif == NULL || gpuGetBusIntfType_HAL(pGpu) != + NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS) + { + return configAccess; + } + + // Now verify PCI and PCIe config start registers. + kbifGetPcieConfigAccessTestRegisters_HAL(pGpu, pKernelBif, &pciStart, &pcieStart); + os_pci_read_dword(handle, pciStart, &nvXveId); + os_pci_read_dword(handle, pcieStart, &nvXveVccapHdr); + + if (NV_OK == kbifVerifyPcieConfigAccessTestRegisters_HAL(pGpu, + pKernelBif, + nvXveId, + nvXveVccapHdr)) + { + configAccess = NV_TRUE; + } + + return configAccess; +} + +static NvBool skipIovaMappingForTegra +( + PIOVAMAPPING pIovaMapping, + nv_state_t *nv +) +{ + // + // TODO: When ISO SMMU is not present, dma mapping of imported ISO memory + // causes crash during __clean_dcache_area_poc. dma mapping of ISO + // memory allocated by RM (via __get_free_pages) still works. + // Skip dma mapping of imported ISO memory to unblock Tegra Display in + // AV+L. Bug 200765629 + // + + NV_ASSERT(nv != NULL); + + if (NV_IS_SOC_DISPLAY_DEVICE(nv) && + !NV_SOC_IS_ISO_IOMMU_PRESENT(nv) && + !memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) && + memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM)) + { + NV_PRINTF(LEVEL_INFO, "%s: Skip memdescMapIommu mapping\n", __FUNCTION__); + return NV_TRUE; + } + + return NV_FALSE; +} + +static NvBool osMemDescRequiresReadOnlyDeviceDmaMap +( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + // + // MEMDESC_FLAGS_USER_READ_ONLY is also used by RM-owned buffers that the + // kernel writes and user/GSP reads. Restrict RO DMA direction to external + // OS descriptor mappings. + // + return memdescGetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM) && + !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE) && + (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY) || + memdescGetFlag(pMemDesc, MEMDESC_FLAGS_DEVICE_READ_ONLY)); +} + +/*! + * @brief Map memory into an IOVA space according to the given mapping info. + * + * @param[in] pIovaMapping IOVA mapping info + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osIovaMap +( + PIOVAMAPPING pIovaMapping +) +{ + OBJGPU *pGpu; + nv_state_t *nv, *peer; + NV_STATUS status; + RmPhysAddr base; + NvBool bIsBar0; + PMEMORY_DESCRIPTOR pRootMemDesc; + NvBool bIsFbOffset = NV_FALSE; + NvBool bIsIndirectPeerMapping = NV_FALSE; + NvBool bIsContig; + NV_ADDRESS_SPACE addressSpace; + NvU32 osPageCount; + + if (pIovaMapping == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId); + if (pGpu == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + pRootMemDesc = memdescGetRootMemDesc(pIovaMapping->pPhysMemDesc, NULL); + addressSpace = memdescGetAddressSpace(pIovaMapping->pPhysMemDesc); + if (gpumgrCheckIndirectPeer(pGpu, pRootMemDesc->pGpu) && + (addressSpace == ADDR_FBMEM)) + { + bIsIndirectPeerMapping = NV_TRUE; + } + + if ((addressSpace != ADDR_SYSMEM) && !bIsIndirectPeerMapping) + { + NV_PRINTF(LEVEL_INFO, + "%s passed memory descriptor in an unsupported address space (%s)\n", + __FUNCTION__, + memdescGetApertureString(memdescGetAddressSpace(pIovaMapping->pPhysMemDesc))); + return NV_ERR_NOT_SUPPORTED; + } + + // + // For guest-allocated memory, we don't actually want to do any remapping, + // since the physical address is already the DMA address to be used by the + // GPU. + // + // For carveout memory, we setup identity mapping, so physical + // address is same as the DMA address. + // + // + if (memdescIsCarveoutMemory(pIovaMapping->pPhysMemDesc) || + memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED)) + { + return NV_OK; + } + + nv = NV_GET_NV_STATE(pGpu); + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc)) + { + osPageCount = pIovaMapping->pPhysMemDesc->PageCount; + NV_ASSERT_OR_RETURN(os_page_size == pIovaMapping->pPhysMemDesc->pageArrayGranularity, NV_ERR_INVALID_PARAMETER); + } + else + { + osPageCount = NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount); + } + + // + // Intercept peer IO type memory. These are contiguous allocations, so no + // need to adjust pages. + // + if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM)) + { + NV_ASSERT(memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU)); + + status = nv_dma_map_mmio(nv->dma_dev, + osPageCount, + &pIovaMapping->iovaArray[0]); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to map peer IO mem (status = 0x%x)\n", + __FUNCTION__, status); + } + + return status; + } + + // + // We need to check against the "root" GPU, e.g., the GPU that owns this + // allocation. If we're trying to map one of its BARs for a peer, we need + // to handle it differently because it wouldn't have gone through our system + // memory page allocation paths, obviously, and wouldn't have alloc private + // data associated with it. + // + peer = NV_GET_NV_STATE(pRootMemDesc->pGpu); + bIsContig = memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU); + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc)) + { + NV_ASSERT_OR_RETURN(pIovaMapping->pPhysMemDesc->pageArrayGranularity == os_page_size, NV_ERR_INVALID_ARGUMENT); + } + else if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig) + { + RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0], + pIovaMapping->pPhysMemDesc->PageCount); + } + + base = memdescGetPhysAddr(pIovaMapping->pPhysMemDesc, AT_CPU, 0); + bIsBar0 = IS_REG_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size); + + bIsFbOffset = IS_FB_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size); + + void *pPriv = memdescGetMemData(pIovaMapping->pPhysMemDesc); + + if (!bIsBar0 && !bIsFbOffset) + { + if (pPriv == NULL) + { + return NV_ERR_INVALID_STATE; + } + } + else if(bIsIndirectPeerMapping) + { + NV_ASSERT(!bIsBar0 && bIsFbOffset); + // + // TODO: Align onlined GPU memory allocation paths with system memory allocation + // That way pMemDesc->pMemData is setup correctly when we try to create mapping + // to onlined memory of indirect peer. After that we can also get rid of some + // extra code in nv_dma_map_alloc. See bug 190324 for details + // + + status = memdescGetNvLinkGpa(pRootMemDesc->pGpu, (bIsContig ? 1 : osPageCount), + &pIovaMapping->iovaArray[0]); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "%s Failed to get SPA\n", __FUNCTION__); + return status; + } + } + + if (skipIovaMappingForTegra(pIovaMapping, nv)) + { + return NV_OK; + } + + if (!bIsBar0 && (!bIsFbOffset || bIsIndirectPeerMapping)) + { + NvBool bReadOnlyDeviceMap = + osMemDescRequiresReadOnlyDeviceDmaMap(pIovaMapping->pPhysMemDesc); + + status = nv_dma_map_alloc( + osGetDmaDeviceForMemDesc(nv, pIovaMapping->pPhysMemDesc), + osPageCount, + &pIovaMapping->iovaArray[0], + bIsContig, bReadOnlyDeviceMap, &pPriv); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to map allocation (status = 0x%x)\n", + __FUNCTION__, status); + return status; + } + + pIovaMapping->pOsData = pPriv; + } + else if (peer != nv) + { + status = nv_dma_map_peer(nv->dma_dev, peer->dma_dev, bIsBar0 ? 0 : 1, + osPageCount, &pIovaMapping->iovaArray[0]); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_INFO, + "%s: failed to map peer (base = 0x%llx, status = 0x%x)\n", + __FUNCTION__, base, status); + return status; + } + + // + // pOsData must be NULL to distinguish a peer DMA mapping from a + // system memory mapping in osIovaUnmap(), so make sure to set it + // accordingly here. + // + pIovaMapping->pOsData = NULL; + } + else + { + NV_PRINTF(LEVEL_INFO, "cannot map a GPU's BAR to itself\n"); + return NV_ERR_NOT_SUPPORTED; + } + + // + // If the OS layer doesn't think in RM page size, we need to inflate the + // PTE array into RM pages. + // + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc)) + { + NV_ASSERT_OR_RETURN(pIovaMapping->pPhysMemDesc->pageArrayGranularity == os_page_size, NV_ERR_INVALID_ARGUMENT); + } + else if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig) + { + RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0], + pIovaMapping->pPhysMemDesc->PageCount); + } + + return NV_OK; +} + +/*! + * @brief Unmap memory from an IOVA space according to the given mapping info. + * + * This mapping info must have been previously mapped by osIovaMap(). + * + * @param[in] pIovaMapping IOVA mapping info + * + */ +void +osIovaUnmap +( + PIOVAMAPPING pIovaMapping +) +{ + OBJGPU *pGpu; + nv_state_t *nv; + void *pPriv; + NV_STATUS status; + NvU64 osPageCount; + + if (pIovaMapping == NULL) + { + return; + } + + pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId); + if (pGpu == NULL) + { + return; + } + + // + // For guest-allocated or carveout memory, we never actually remapped the + // memory, so we shouldn't try to unmap it here. + // + if (memdescIsCarveoutMemory(pIovaMapping->pPhysMemDesc) || + memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED)) + { + return; + } + + nv = NV_GET_NV_STATE(pGpu); + + if (skipIovaMappingForTegra(pIovaMapping, nv)) + { + return; + } + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc)) + { + osPageCount = pIovaMapping->pPhysMemDesc->PageCount; + NV_ASSERT(os_page_size == pIovaMapping->pPhysMemDesc->pageArrayGranularity); + } + else + { + osPageCount = NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount); + } + + if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM)) + { + nv_dma_unmap_mmio(nv->dma_dev, + osPageCount, + pIovaMapping->iovaArray[0]); + + return; + } + + // + // TODO: Formalize the interface with the OS layers so we can use a common + // definition of OS_IOVA_MAPPING_DATA. + // + pPriv = (void *)pIovaMapping->pOsData; + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc)) + { + NV_ASSERT(pIovaMapping->pPhysMemDesc->pageArrayGranularity == os_page_size); + } + else if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0], + pIovaMapping->pPhysMemDesc->PageCount); + } + + if (pPriv != NULL) + { + status = nv_dma_unmap_alloc(nv->dma_dev, + osPageCount, + &pIovaMapping->iovaArray[0], &pPriv); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to unmap allocation (status = 0x%x)\n", + __FUNCTION__, status); + } + } + else + { + nv_dma_unmap_peer(nv->dma_dev, + osPageCount, + pIovaMapping->iovaArray[0]); + } + + // + // If the OS layer doesn't think in RM page size, we need to fluff out the + // PTE array into RM pages. + // + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc)) + { + NV_ASSERT(pIovaMapping->pPhysMemDesc->pageArrayGranularity == os_page_size); + } + else if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU)) + { + RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0], + pIovaMapping->pPhysMemDesc->PageCount); + } + + pIovaMapping->pOsData = NULL; +} + +/*! + * @brief Set the GPU Rail Voltage in Tegra SoC. Currently not supported + * + * @param[in] pGpu GPU object pointer + * @param[in] reqVoltageuV Rail Voltage requested in uV + * @param[out] pSetVoltageuV Rail Voltage set in uV + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osSetGpuRailVoltage +( + OBJGPU *pGpu, + NvU32 reqVoltageuV, + NvU32 *pSetVoltageuV +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Get the GPU Rail Voltage in Tegra SoC. Currently not supported + * + * @param[in] pGpu GPU object pointer + * @param[out] voltageuV Rail Voltage in uV + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osGetGpuRailVoltage +( + OBJGPU *pGpu, + NvU32 *pVoltageuV +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Bring down system in a controlled manner on known error conditions. + * + * @bugCode[in] Error code / reason. + */ +void osBugCheck(NvU32 bugCode) +{ + if (bugCode > OS_BUG_CHECK_BUGCODE_LAST) + { + bugCode = OS_BUG_CHECK_BUGCODE_UNKNOWN; + } + + os_bug_check(bugCode, ppOsBugCheckBugcodeStr[bugCode]); +} + +/*! + * @brief Perform an action at assertion failure. + */ +void osAssertFailed(void) +{ + os_dump_stack(); +} + +/*! + * @brief Get the GPU Chip Info - Speedo and IDDQ values + * + * + * @param[in] pGpu GPU object pointer + * @param[out] pGpuSpeedoHv Pointer to GPU Speedo value at high voltage corner. + * @param[out] pGpuSpeedoLv Pointer to GPU Speedo value at low voltage corner. + * @param[out] pGpuIddq Pointer to GPU Iddq Value + * @param[out] pChipSkuId SKU ID for the chip + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osGetChipInfo +( + OBJGPU *pGpu, + NvU32 *pGpuSpeedoHv, + NvU32 *pGpuSpeedoLv, + NvU32 *pGpuIddq, + NvU32 *pChipSkuId +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/* + * @brief Get the GPU Rail Voltage Info (i.e. Min, Max and StepSize) in Tegra SoC. + * + * @param[in] pGpu GPU object pointer + * @param[out] pMinVoltageuV Minimum Voltage supported on the Rail in Micro Volts + * @param[out] pMaxVoltageuV Maximum Voltage supported on the Rail in Micro Volts + * @param[out] pStepVoltageuV Voltage Step-size supported on the Rail in Micro Volts + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osGetGpuRailVoltageInfo +( + OBJGPU *pGpu, + NvU32 *pMinVoltageuV, + NvU32 *pMaxVoltageuV, + NvU32 *pStepVoltageuV +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Get the current opaque security token. + * + * For Linux the security token is the effective UID of a process and process ID + * + * Note: This function allocates memory for the token. The onus is on the calling + * function to free the memory associated with the token once its done with it. + * + * @return pointer to the security token. + */ +PSECURITY_TOKEN +osGetSecurityToken(void) +{ + NV_STATUS rmStatus; + TOKEN_USER *pTokenUser; + + pTokenUser = portMemAllocNonPaged(sizeof(TOKEN_USER)); + if (pTokenUser == NULL) + { + return NULL; + } + rmStatus = os_get_euid(&pTokenUser->euid); + if (rmStatus != NV_OK) + { + portMemFree(pTokenUser); + return NULL; + } + + pTokenUser->pid = os_get_current_process(); + + return (PSECURITY_TOKEN)pTokenUser; +} + +PUID_TOKEN +osGetCurrentUidToken(void) +{ + NV_STATUS rmStatus; + NvU32 *pUidToken; + + pUidToken = portMemAllocNonPaged(sizeof(NvU32)); + if (pUidToken == NULL) + { + return NULL; + } + + rmStatus = os_get_euid(pUidToken); + if (rmStatus != NV_OK) + { + portMemFree(pUidToken); + return NULL; + } + + return (PUID_TOKEN)pUidToken; +} + +/*! + * @brief Interface function to validate the token for the current client + * + * This function takes two tokens as parameters, validates them and checks + * if either the PID or EUID from client database matches the current PID or EUID. + * + * @param[in] pClientSecurityToken security token cached in the client db + * @param[in] pCurrentSecurityToken security token of the current client + * @return NV_OK if the validation is successful + * NV_ERR_INVALID_CLIENT if the tokens do not match + * NV_ERR_INVALID_POINTER if the tokens are invalid + */ +NV_STATUS +osValidateClientTokens +( + PSECURITY_TOKEN pClientSecurityToken, + PSECURITY_TOKEN pCurrentSecurityToken +) +{ + PTOKEN_USER pClientTokenUser = (PTOKEN_USER)pClientSecurityToken; + PTOKEN_USER pCurrentTokenUser = (PTOKEN_USER)pCurrentSecurityToken; + + if (pClientTokenUser == NULL || pCurrentTokenUser == NULL) + return NV_ERR_INVALID_POINTER; + + if ((pClientTokenUser->euid != pCurrentTokenUser->euid) && + (pClientTokenUser->pid != pCurrentTokenUser->pid)) + { + NV_PRINTF(LEVEL_INFO, + "NVRM: %s: Current security token doesn't match the one in the client database. " + "Current EUID: %d, PID: %d; Client DB EUID: %d, PID: %d\n", + __FUNCTION__, pCurrentTokenUser->euid, pCurrentTokenUser->pid, + pClientTokenUser->euid, pClientTokenUser->pid); + return NV_ERR_INVALID_CLIENT; + } + + return NV_OK; +} + +/*! + * @brief Interface function to compare the tokens for two client + * + * This function takes two tokens as parameters, validates them and checks + * if the EUIDs of each token match. + * + * @param[in] pToken1 Token to compare + * @param[in] pToken2 Token to compare + * @return NV_TRUE if the tokens match + * NV_FALSE if the tokens do not match + */ +NvBool +osUidTokensEqual +( + PUID_TOKEN pUidToken1, + PUID_TOKEN pUidToken2 +) +{ + NvU32 * pTokenUser1 = (NvU32*)pUidToken1; + NvU32 * pTokenUser2 = (NvU32*)pUidToken2; + + NV_ASSERT_OR_RETURN((pTokenUser1 != NULL), NV_FALSE); + NV_ASSERT_OR_RETURN((pTokenUser2 != NULL), NV_FALSE); + + if (*pTokenUser1 != *pTokenUser2) + { + return NV_FALSE; + } + + return NV_TRUE; +} + +NvBool +osRemoveGpuSupported +( + void +) +{ + return os_pci_remove_supported(); +} + +/* + * @brief Get the address ranges assigned to local or peer GPUs on a system that + * supports hardware address translation services (ATS) over NVLink/C2C. + * + * @note + * - All address values are in the System Physical Address (SPA) space + * - Targets can either be "Local" (bIsPeer=False) or for a specified "Peer" + * (bIsPeer=True, peerIndex=#) GPU + * - Target address and mask values have a specified bit width, and represent + * the higher order bits above the target address granularity + * + * @param[in] pGpu GPU object pointer + * @param[out] pAddrSysPhys Pointer to hold SPA + * @param[out] pAddrWidth Address range width value pointer + * @param[out] pMask Mask value pointer + * @param[out] pMaskWidth Mask width value pointer + * @param[in] bIsPeer NV_TRUE if this is a peer, local GPU otherwise + * @param[in] peerIndex Peer index + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED + * + * A return value of NV_ERR_NOT_SUPPORTED for the local GPU would + * indicate that the system does not support ATS over NVLink/C2C + */ +NV_STATUS +osGetAtsTargetAddressRange +( + OBJGPU *pGpu, + NvU64 *pAddrSysPhys, + NvU32 *pAddrWidth, + NvU32 *pMask, + NvU32 *pMaskWidth, + NvBool bIsPeer, + NvU32 peerIndex +) +{ +#if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && defined(NVCPU_AARCH64) + KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + nv_state_t *nv; + const int addrMaskWidth = 0x10; + + if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP)) + return NV_ERR_INVALID_ARGUMENT; + + nv = NV_GET_NV_STATE(pGpu); + + // + // TODO : Bug 1848958 restricts peer device tree parsing. Return early if + // peer values are requested. This should be fixed by passing correct pGpu + // pointer of the peer GPU retrieved using peerIds. + // + if (bIsPeer) + { + const int addrWidth = 0x10; + + *pAddrSysPhys = 0; + *pAddrWidth = addrWidth; + *pMask = 0; + *pMaskWidth = addrMaskWidth; + return NV_OK; + } + else + { + NV_STATUS status = nv_get_device_memory_config(nv, pAddrSysPhys, NULL, NULL, NULL, + pAddrWidth, NULL); + if (status == NV_OK) + { + *pMask = NVBIT(*pAddrWidth) - 1U; + *pMaskWidth = addrMaskWidth; + } + return status; + } + + return NV_OK; +#endif + return NV_ERR_NOT_SUPPORTED; +} + +/* + * @brief Get the physical address in CPU address map and NUMA node id + * of the GPU memory. + * + * @note + * - The physical address is System Physical Address (SPA) in baremetal/host + * and Intermediate Physical Address(IPA) or Guest Physical Address(GPA) + * inside a VM. + * + * @param[in] pGpu GPU object pointer + * @param[out] pAddrPhys Pointer to hold the physical address of FB in + * CPU address map + * @param[out] pSizePhys Pointer to hold the size of FB mapped to CPU + * @param[out] pNodeId NUMA nodeID of respective GPU memory + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED + * + */ +NV_STATUS +osGetFbNumaInfo +( + OBJGPU *pGpu, + NvU64 *pAddrPhys, + NvU64 *pSizePhys, + NvU64 *pAddrRsvdPhys, + NvS32 *pNodeId +) +{ +#if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && defined(NVCPU_AARCH64) + KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + nv_state_t *nv; + + *pNodeId = NV0000_CTRL_NO_NUMA_NODE; + + if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP)) + return NV_ERR_INVALID_ARGUMENT; + + nv = NV_GET_NV_STATE(pGpu); + + NV_STATUS status = nv_get_device_memory_config(nv, NULL, pAddrPhys, pSizePhys, + pAddrRsvdPhys, NULL, pNodeId); + + return status; +#endif + return NV_ERR_NOT_SUPPORTED; +} + + +/* + * @brief Verif only function to get the chiplib overrides for link connection + * state for all C2C links. + * + * If chiplib overrides exist, each link can either be enabled (1) or disabled (0) + * + * @param[in] pGpu GPU object pointer + * @param[in] maxLinks Size of pLinkConnection array + * @param[out] pLinkConnection array of pLinkConnection values to be populated by MODS + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available) + */ +NV_STATUS +osGetForcedC2CConnection +( + OBJGPU *pGpu, + NvU32 maxLinks, + NvU32 *pLinkConnection +) +{ + int i, ret; + NV_STATUS status; + char path[64]; + + NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER); + NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED); + NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT); + + for (i = 0; i < maxLinks; i++) + { + ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|C2C%u", i); + NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1))); + + status = gpuSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]); + if (status == NV_OK) + { + NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__, + path, pLinkConnection[i]); + } + else + { + NV_PRINTF(LEVEL_INFO, "%s: gpuSimEscapeRead for '%s' failed (%u)\n", + __FUNCTION__, path, status); + return NV_ERR_NOT_SUPPORTED; + } + } + return NV_OK; +} + +static NV_STATUS +osGetSmbiosTableInfo +( + const NvU8 *pMappedAddr, + NvU64 *pBaseAddr, + NvU64 *pLength, + NvU64 *pNumSubTypes, + NvU32 *pVersion +) +{ + *pBaseAddr = 0; + *pLength = 0; + *pNumSubTypes = 0; + *pVersion = 0; + + if (portMemCmp(pMappedAddr, "_SM3_", 5) == 0) + { + *pVersion = (pMappedAddr[7] << 8) | pMappedAddr[8]; + portMemCopy(pLength, 4, pMappedAddr + 12, 4); + portMemCopy(pBaseAddr, 8, pMappedAddr + 16, 8); + + *pNumSubTypes = *pLength / 4; + + return NV_OK; + } + + if (portMemCmp(pMappedAddr, "_SM_", 4) == 0) + { + *pVersion = (pMappedAddr[6] << 8) | pMappedAddr[7]; + + pMappedAddr += 16; + + if (portMemCmp(pMappedAddr, "_DMI_", 5) == 0) + { + portMemCopy(pLength, 2, pMappedAddr + 6, 2); + portMemCopy(pBaseAddr, 4, pMappedAddr + 8, 4); + portMemCopy(pNumSubTypes, 2, pMappedAddr + 12, 2); + + if (!*pVersion) + *pVersion = (pMappedAddr[14] & 0xF0) << 4 | + (pMappedAddr[14] & 0x0F); + + return NV_OK; + } + } + + return NV_ERR_INVALID_ADDRESS; +} + + +/* + * @brief Function to export SMBIOS table. Also, maps table in kernel-space. + * + * @param[out] ppBaseVAddr Base virtual address of SMBIOS table. + * @param[out] pLength Size of SMBIOS table. + * @param[out] pNumSubTypes Count of structures (types) embedded in + * the SMBIOS table. + * @param[out] pVersion SMBIOS version + * + * @return NV_OK, NV_ERR_INSUFFICIENT_RESOURCES or NV_ERR_INVALID_ADDRESS + * or errors from OS layer + */ +NV_STATUS +osGetSmbiosTable +( + void **ppBaseVAddr, + NvU64 *pLength, + NvU64 *pNumSubTypes, + NvU32 *pVersion +) +{ + NV_STATUS status = NV_OK; + NvU64 physSmbiosAddr = ~0ull; + void *pMappedAddr = NULL; + NvU64 basePAddr = 0; + + if (!NVCPU_IS_X86_64 && !NVCPU_IS_AARCH64) + { + return NV_ERR_NOT_SUPPORTED; + } + + status = os_get_smbios_header(&physSmbiosAddr); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_INFO, "%s: Failed query SMBIOS table with error: %x \n", + __FUNCTION__, status); + return status; + } + + NV_ASSERT(physSmbiosAddr != ~0ull); + + pMappedAddr = osMapKernelSpace(physSmbiosAddr, + os_page_size, + NV_MEMORY_CACHED, + NV_PROTECT_READ_WRITE); + if (!pMappedAddr) + { + return NV_ERR_INSUFFICIENT_RESOURCES; + } + + status = osGetSmbiosTableInfo(pMappedAddr, + &basePAddr, + pLength, + pNumSubTypes, + pVersion); + + osUnmapKernelSpace(pMappedAddr, os_page_size); + + if (status != NV_OK) + { + return status; + } + + *ppBaseVAddr = osMapKernelSpace(basePAddr, + *pLength, + NV_MEMORY_CACHED, + NV_PROTECT_READ_WRITE); + if (!*ppBaseVAddr) + { + return NV_ERR_INSUFFICIENT_RESOURCES; + } + + return NV_OK; +} + +/* + * @brief Function to free SMBIOS table mappings + * + * @param[in] pBaseVAddr Base virtual address of SMBIOS table. + * @param[in] length Size of SMBIOS table. + * + */ +void +osPutSmbiosTable +( + void *pBaseVAddr, + NvU64 length +) +{ + osUnmapKernelSpace(pBaseVAddr, length); +} + +NV_STATUS +osGetAcpiRsdpFromUefi +( + NvU32 *pRsdpAddr +) +{ + return os_get_acpi_rsdp_from_uefi(pRsdpAddr); +} + +/* + * @brief Returns NV_TRUE if NvSwitch device is present in the system. + */ +NvBool +osIsNvswitchPresent +( + void +) +{ + return os_is_nvswitch_present(); +} + +/* + * @brief Function to add crashlog buffer entry. + * + * @param[in] pBuffer virt_addr of nvlog buffer + * @param[in] length size of nvlog buffer + */ +void +osAddRecordForCrashLog +( + void *pBuffer, + NvU32 length +) +{ + os_add_record_for_crashLog(pBuffer, length); +} + +/* + * @brief Function to delete crashlog buffer entry. + * + * @param[in] pBuffer virt_addr of nvlog buffer + */ +void +osDeleteRecordForCrashLog +( + void *pBuffer +) +{ + os_delete_record_for_crashLog(pBuffer); +} + +/* + * @brief Queries the sysfs interface to get memblock size + * @param[out] memblock_size Pointer to the memblock_size + */ +NV_STATUS +osNumaMemblockSize +( + NvU64 *memblock_size +) +{ + return os_numa_memblock_size(memblock_size); +} + +NvBool +osNumaOnliningEnabled +( + OS_GPU_INFO *pOsGpuInfo +) +{ + NvS32 numaNodeId = NV0000_CTRL_NO_NUMA_NODE; + + // + // Note that this numaNodeId value fetched from Linux layer might not be + // accurate since it is possible to overwrite it with regkey on some configs + // + if (nv_get_device_memory_config(pOsGpuInfo, NULL, NULL, NULL, NULL, NULL, + &numaNodeId) != NV_OK) + { + return NV_FALSE; + } + + return (numaNodeId != NV0000_CTRL_NO_NUMA_NODE); +} + +/* + * @brief Function to call NUMA allocation entry. + * + * @param[in] nid NUMA node id + * @param[in] size Allocation size + * @param[in] flag Allocation flags + * @param[out] pAddress Ptr to the allocated physical address + */ +NV_STATUS +osAllocPagesNode +( + NvS32 nid, + NvLength size, + NvU32 flag, + NvU64 *pAddress +) +{ + NV_STATUS status = NV_OK; + NvU32 localFlag = NV_ALLOC_PAGES_NODE_NONE; + + if (pAddress == NULL || nid < 0 || size > NV_U32_MAX) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // Translate the flags + if (flag & OS_ALLOC_PAGES_NODE_SKIP_RECLAIM) + { + localFlag |= NV_ALLOC_PAGES_NODE_SKIP_RECLAIM; + } + + status = os_alloc_pages_node(nid, (NvU32)size, localFlag, pAddress); + return status; +} + +void +osAllocAcquirePage +( + NvU64 pAddress, + NvU32 pageCount +) +{ + NvU32 i; + + for (i = 0; i < pageCount; i++) + { + os_get_page(pAddress + (i << os_page_shift)); + } +} + +void +osAllocReleasePage +( + NvU64 pAddress, + NvU32 pageCount +) +{ + NvU32 i; + + for (i = 0; i < pageCount; i++) + { + os_put_page(pAddress + (i << os_page_shift)); + } +} + +/* + * @brief Function to return refcount on a page + * @param[in] address The physical address of the page + */ +NvU32 +osGetPageRefcount +( + NvU64 pAddress +) +{ + return os_get_page_refcount(pAddress); +} + +/* + * @brief Function to return the number of tail pages if the address is + * referring to a compound page; For non-compound pages, 1 is returned. + * @param[in] address The physical address of the page + */ +NvU32 +osCountTailPages +( + NvU64 pAddress +) +{ + return os_count_tail_pages(pAddress); +} + +NvU64 +osGetPageSize(void) +{ + return os_page_size; +} + +NvU64 +osGetSupportedSysmemPageSizeMask(void) +{ + // + // We assume that the kernel can support all power-of-two pagesizes + // between os_page_size and os_max_page_size (inclusive). Return a + // bitmask containing all of those. + // + return (((os_max_page_size << 1) - 1) & (~(os_page_size - 1))); +} + +NvU8 +osGetPageShift(void) +{ + return os_page_shift; +} + + + +/* + * @brief Opens a new temporary file for reading and writing + * + * @param[in] ppFile void double pointer + * + * @returns NV_STATUS, NV_OK if success, + NV_ERR_GENERIC, if error + NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osOpenTemporaryFile +( + void **ppFile +) +{ + return os_open_temporary_file(ppFile); +} + +/* + * @brief Closes the specified temporary file + * + * @param[in] pFile Pointer to file + * + * @returns void + */ +void +osCloseFile +( + void *pFile +) +{ + os_close_file(pFile); +} + +/* + * @brief Writes the buffer to the specified file at the given offset + * + * @param[in] pFile Pointer to file (void) + * @param[in] pBuffer Pointer to buffer from which to copy + * @param[in] size Size of the copy + * @parma[in] offset offset in the file + * + * @returns NV_STATUS, NV_OK if success, + NV_ERR_GENERIC, if error + NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osWriteToFile +( + void *pFile, + NvU8 *pBuffer, + NvU64 size, + NvU64 offset +) +{ + return os_write_file(pFile, pBuffer, size, offset); +} + +/* + * @brief Reads from the specified file at the given offset + * + * @param[in] pFile Pointer to file (void *) + * @param[in] pBuffer Pointer to buffer to which the data is copied + * @param[in] size Size of the copy + * @parma[in] offset offset in the file + * + * @returns NV_STATUS, NV_OK if success, + NV_ERR_GENERIC, if error + NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osReadFromFile +( + void *pFile, + NvU8 *pBuffer, + NvU64 size, + NvU64 offset +) +{ + return os_read_file(pFile, pBuffer, size, offset); +} + +/* + * @brief Unregisters caps from the capability framework. + * The function assumes that the caps are allocated and stored in the + * hierarchical order. If they aren't, OS (Linux kernel) would warn and + * leak the caps. + * + * @param[in] pOsRmCaps caps of interest + */ +void +osRmCapUnregister +( + OS_RM_CAPS **ppOsRmCaps +) +{ + OS_RM_CAPS *pOsRmCaps = *ppOsRmCaps; + NvS32 i; + + if (pOsRmCaps == NULL) + { + return; + } + + for (i = pOsRmCaps->count - 1; i >= 0; i--) + { + if (pOsRmCaps->caps[i] != NULL) + { + os_nv_cap_destroy_entry(pOsRmCaps->caps[i]); + } + } + + os_free_mem(pOsRmCaps->caps); + os_free_mem(pOsRmCaps); + + *ppOsRmCaps = NULL; +} + +static NV_STATUS +_allocOsRmCaps +( + OS_RM_CAPS **ppOsRmCaps, + NvU32 count +) +{ + NV_STATUS status; + OS_RM_CAPS *pOsRmCaps; + + *ppOsRmCaps = NULL; + + status = os_alloc_mem((void**)&pOsRmCaps, sizeof(OS_RM_CAPS)); + if (status != NV_OK) + return status; + + pOsRmCaps->count = count; + + status = os_alloc_mem((void**)&pOsRmCaps->caps, sizeof(pOsRmCaps->caps[0]) * count); + if (status != NV_OK) + { + os_free_mem(pOsRmCaps); + return status; + } + + os_mem_set(pOsRmCaps->caps, 0, sizeof(pOsRmCaps->caps[0]) * count); + + *ppOsRmCaps = pOsRmCaps; + return NV_OK; +} + +#define OS_RM_CAP_GPU_DIR 0 +#define OS_RM_CAP_GPU_MIG_DIR 1 +#define OS_RM_CAP_GPU_COUNT 2 + +/* + * @brief Registers OBJGPU with the capability framework. + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[out] ppOsRmCaps GPU OS specific capabilities pointer + */ +NV_STATUS +osRmCapRegisterGpu +( + OS_GPU_INFO *pOsGpuInfo, + OS_RM_CAPS **ppOsRmCaps +) +{ + NvU32 minor = nv_get_dev_minor(pOsGpuInfo); + char name[16]; + NV_STATUS status; + OS_RM_CAPS *pOsRmCaps; + nv_cap_t *parent; + nv_cap_t *cap; + + // Return success on the unsupported platforms. + if (nvidia_caps_root == NULL) + { + return NV_OK; + } + + if (*ppOsRmCaps != NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_GPU_COUNT); + if (status != NV_OK) + return status; + + *ppOsRmCaps = pOsRmCaps; + + os_snprintf(name, sizeof(name), "gpu%u", minor); + name[sizeof(name) - 1] = '\0'; + parent = nvidia_caps_root; + + cap = os_nv_cap_create_dir_entry(parent, name, (OS_RUGO | OS_XUGO)); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup gpu%u directory\n", minor); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_GPU_DIR] = cap; + parent = cap; + + // TODO: Bug 2679591: Add MIG directory only if SMC is enabled. + // For now, always add "mig" directory. + cap = os_nv_cap_create_dir_entry(parent, "mig", (OS_RUGO | OS_XUGO)); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup mig directory\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR] = cap; + + return NV_OK; + +failed: + osRmCapUnregister(ppOsRmCaps); + + return status; +} + +#define OS_RM_CAP_SMC_PART_DIR 0 +#define OS_RM_CAP_SMC_PART_ACCESS_FILE 1 +#define OS_RM_CAP_SMC_PART_COUNT 2 + +/* + * @brief Registers SMC partition (a.k.a. GPU instance) with the capability + * framework + * + * @param[in] pGpuOsRmCaps GPU OS specific capabilities pointer + * @param[out] ppPartitionOsRmCaps OS specific capabilities pointer for SMC partition + * @param[in] swizzId SMC partition swizz ID + */ +NV_STATUS +osRmCapRegisterSmcPartition +( + OS_RM_CAPS *pGpuOsRmCaps, + OS_RM_CAPS **ppPartitionOsRmCaps, + NvU32 swizzId +) +{ + char name[16]; + NV_STATUS status; + nv_cap_t *parent; + nv_cap_t *cap; + OS_RM_CAPS *pOsRmCaps; + + // Return success as there is nothing to do. + if (pGpuOsRmCaps == NULL) + { + return NV_OK; + } + + if (*ppPartitionOsRmCaps != NULL || swizzId >= NV_U32_MAX) + { + return NV_ERR_INVALID_ARGUMENT; + } + + parent = pGpuOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR]; + if (parent == NULL) + { + return NV_ERR_INVALID_STATE; + } + + status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_PART_COUNT); + if (status != NV_OK) + return status; + + *ppPartitionOsRmCaps = pOsRmCaps; + + os_snprintf(name, sizeof(name), "gi%u", swizzId); + name[sizeof(name) - 1] = '\0'; + + cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup gi%u directory\n", + swizzId); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR] = cap; + parent = cap; + + cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n", + swizzId); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_SMC_PART_ACCESS_FILE] = cap; + + return NV_OK; + +failed: + osRmCapUnregister(ppPartitionOsRmCaps); + + return status; +} + +#define OS_RM_CAP_SMC_EXEC_PART_DIR 0 +#define OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE 1 +#define OS_RM_CAP_SMC_EXEC_PART_COUNT 2 + +/* + * @brief Registers SMC execution partition (a.k.a. compute instance) with the + * capability framework + * + * @param[in] pPartitionOsRmCaps OS specific capabilities pointer for SMC partition + * @param[out] ppExecPartitionOsRmCaps OS specific capabilities pointer for SMC execution partition + * @param[in] execPartitionId SMC execution partition ID + */ +NV_STATUS +osRmCapRegisterSmcExecutionPartition +( + OS_RM_CAPS *pPartitionOsRmCaps, + OS_RM_CAPS **ppExecPartitionOsRmCaps, + NvU32 execPartitionId +) +{ + char name[16]; + NV_STATUS status; + nv_cap_t *parent; + nv_cap_t *cap; + OS_RM_CAPS *pOsRmCaps; + + // Return success as there is nothing to do. + if (pPartitionOsRmCaps == NULL) + { + return NV_OK; + } + + if ((*ppExecPartitionOsRmCaps != NULL) || (execPartitionId >= NV_U32_MAX)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + parent = pPartitionOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR]; + if (parent == NULL) + { + return NV_ERR_INVALID_STATE; + } + + status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_EXEC_PART_COUNT); + if (status != NV_OK) + { + return status; + } + + *ppExecPartitionOsRmCaps = pOsRmCaps; + + os_snprintf(name, sizeof(name), "ci%u", execPartitionId); + name[sizeof(name) - 1] = '\0'; + + cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup ci%u directory\n", + execPartitionId); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_DIR] = cap; + parent = cap; + + cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n", + execPartitionId); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE] = cap; + + return NV_OK; + +failed: + osRmCapUnregister(ppExecPartitionOsRmCaps); + + return status; +} + +/* + * @brief Release the acquired capability + * + * @param[in] dupedCapDescriptor descriptor to be released + */ +void +osRmCapRelease +( + NvU64 dupedCapDescriptor +) +{ + if (dupedCapDescriptor == NV_U64_MAX) + { + return; + } + + os_nv_cap_close_fd((int)dupedCapDescriptor); +} + +#define OS_RM_CAP_SYS_MIG_DIR 0 +#define OS_RM_CAP_SYS_SMC_CONFIG_FILE 1 +#define OS_RM_CAP_SYS_SMC_MONITOR_FILE 2 +#define OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE 3 +#define OS_RM_CAP_SYS_PROFILER_DEVICE_FILE 4 +#define OS_RM_CAP_SYS_PROFILER_CONTEXT_FILE 5 +#define OS_RM_CAP_SYS_TRACE_DEVICE_FILE 6 +#define OS_RM_CAP_SYS_COUNT 7 + +NV_STATUS +osRmCapRegisterSys +( + OS_RM_CAPS **ppOsRmCaps +) +{ + nv_cap_t **ppCaps; + nv_cap_t *parent; + nv_cap_t *cap; + NV_STATUS status; + OS_RM_CAPS *pOsRmCaps; + + if (nvidia_caps_root == NULL) + return NV_ERR_NOT_SUPPORTED; + + status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SYS_COUNT); + if (status != NV_OK) + return status; + + *ppOsRmCaps = pOsRmCaps; + + ppCaps = pOsRmCaps->caps; + + parent = os_nv_cap_create_dir_entry(nvidia_caps_root, "mig", OS_RUGO | OS_XUGO); + if (parent == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create mig directory\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_MIG_DIR] = parent; + + cap = os_nv_cap_create_file_entry(parent, "config", OS_RUSR); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create mig config file\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_SMC_CONFIG_FILE] = cap; + + cap = os_nv_cap_create_file_entry(parent, "monitor", OS_RUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create mig monitor file\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_SMC_MONITOR_FILE] = cap; + + cap = os_nv_cap_create_file_entry(nvidia_caps_root, "fabric-imex-mgmt", OS_RUSR); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create imex file\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE] = cap; + + cap = os_nv_cap_create_file_entry(nvidia_caps_root, "profiler-device", OS_RUSR); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create profiler-device file\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_PROFILER_DEVICE_FILE] = cap; + + cap = os_nv_cap_create_file_entry(nvidia_caps_root, "profiler-context", OS_RUSR); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create profiler-context file\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_PROFILER_CONTEXT_FILE] = cap; + + cap = os_nv_cap_create_file_entry(nvidia_caps_root, "trace-device", OS_RUSR); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create trace-device file\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_TRACE_DEVICE_FILE] = cap; + + return NV_OK; + +failed: + osRmCapUnregister(ppOsRmCaps); + return status; +} + +/* + * @brief Acquire the requested capability + * + * @param[in] pOsRmCaps opaque pointer to the caps. + * @param[in] rmCap the capability to be acquired. + * @param[in] capDescriptor descriptor to be used for validation + * @param[out] dupedCapDescriptor returns duplicated descriptor if validation + * is successful + * + * Note: On Linux, duplicating fd is helpful to let administrators know about + * the capability users. See https://linux.die.net/man/8/lsof usage. + */ +NV_STATUS +osRmCapAcquire +( + OS_RM_CAPS *pOsRmCaps, + NvU32 rmCap, + NvU64 capDescriptor, + NvU64 *dupedCapDescriptor +) +{ + nv_cap_t *cap; + int fd = (int)capDescriptor; + int duped_fd; + NvU32 index; + NV_STATUS status; + + *dupedCapDescriptor = NV_U64_MAX; + + switch (rmCap) + { + case NV_RM_CAP_SMC_PARTITION_ACCESS: + { + index = OS_RM_CAP_SMC_PART_ACCESS_FILE; + break; + } + case NV_RM_CAP_EXT_FABRIC_MGMT: + { + status = nv_acquire_fabric_mgmt_cap(fd, &duped_fd); + if (status != NV_OK) + { + return status; + } + + goto done; + } + case NV_RM_CAP_SMC_EXEC_PARTITION_ACCESS: + { + index = OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE; + break; + } + case NV_RM_CAP_SYS_SMC_CONFIG: + { + index = OS_RM_CAP_SYS_SMC_CONFIG_FILE; + break; + } + case NV_RM_CAP_SYS_SMC_MONITOR: + { + index = OS_RM_CAP_SYS_SMC_MONITOR_FILE; + break; + } + case NV_RM_CAP_SYS_FABRIC_IMEX_MGMT: + { + index = OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE; + break; + } + case NV_RM_CAP_SYS_PROFILER_DEVICE: + { + index = OS_RM_CAP_SYS_PROFILER_DEVICE_FILE; + break; + } + case NV_RM_CAP_SYS_PROFILER_CONTEXT: + { + index = OS_RM_CAP_SYS_PROFILER_CONTEXT_FILE; + break; + } + case NV_RM_CAP_SYS_TRACE_DEVICE: + { + index = OS_RM_CAP_SYS_TRACE_DEVICE_FILE; + break; + } + default: + { + return NV_ERR_INVALID_ARGUMENT; + } + } + + if (pOsRmCaps == NULL) + { + return NV_ERR_NOT_SUPPORTED; + } + + if (index >= pOsRmCaps->count) + { + return NV_ERR_INVALID_ARGUMENT; + } + + cap = pOsRmCaps->caps[index]; + + duped_fd = os_nv_cap_validate_and_dup_fd(cap, fd); + if (duped_fd < 0) + { + return NV_ERR_INSUFFICIENT_PERMISSIONS; + } + +done: + *dupedCapDescriptor = duped_fd; + + return NV_OK; +} + +/* + * @brief Initializes capability descriptor + * + * @param[out] pCapDescriptor descriptor to be used + * + */ +void +osRmCapInitDescriptor +( + NvU64 *pCapDescriptor +) +{ + *pCapDescriptor = NV_U64_MAX; +} + +/* + * @brief Checks if IMEX channel support is present. + */ +NvBool +osImexChannelIsSupported(void) +{ + return os_imex_channel_is_supported; +} + +/* + * @brief Returns IMEX channel count. + */ +NvS32 +osImexChannelCount +( + void +) +{ + return os_imex_channel_count(); +} + +/* + * @brief Returns IMEX channel number. + * + * @param[in] descriptor OS specific descriptor to query channel number. + * + */ +NvS32 +osImexChannelGet(NvU64 descriptor) +{ + return os_imex_channel_get(descriptor); +} + +/* + * @brief Generates random bytes which can be used as a universally unique + * identifier. + * + * This function may sleep (interruptible). + * + * @param[out] pBytes Array of random bytes + * @param[in] numBytes Size of the array + */ +NV_STATUS +osGetRandomBytes +( + NvU8 *pBytes, + NvU16 numBytes +) +{ + return os_get_random_bytes(pBytes, numBytes); +} + +/* + * @brief Get current process flags.. + */ +NvU32 +osGetCurrentProcessFlags +( + void +) +{ + return os_get_current_process_flags(); +} + +/* + * @brief Allocate wait queue + * + * @param[out] ppWq Wait queue + */ +NV_STATUS +osAllocWaitQueue +( + OS_WAIT_QUEUE **ppWq +) +{ + return os_alloc_wait_queue(ppWq); +} + +/* + * @brief Free wait queue + * + * @param[in] pWq Wait queue + */ +void +osFreeWaitQueue +( + OS_WAIT_QUEUE *pWq +) +{ + os_free_wait_queue(pWq); +} + +/* + * @brief Put thread to uninterruptible sleep + * + * @param[in] pWq Wait queue + */ +void +osWaitUninterruptible +( + OS_WAIT_QUEUE *pWq +) +{ + os_wait_uninterruptible(pWq); +} + +/* + * @brief Put thread to interruptible sleep + * + * @param[in] pWq Wait queue + */ +void +osWaitInterruptible +( + OS_WAIT_QUEUE *pWq +) +{ + os_wait_interruptible(pWq); +} + +/* + * @brief Wake up thread from uninterruptible sleep + * + * @param[in] pWq Wait queue + */ +void +osWakeUp +( + OS_WAIT_QUEUE *pWq +) +{ + os_wake_up(pWq); +} + +NV_STATUS +osReadPFPciConfigInVF +( + NvU32 addr, + NvU32 *data +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Callback function to notify RM when unix layer receives an event + * + * This function is basically a wrapper to call the Core RM layer and is + * being called from DCE KMD when an event is received from DCE RM. + * + * @param[in] handle handle allocated for corresponding IPC type with DCE + * @param[in] interfaceType RM IPC interface type + * @param[in] length length of the message passed from DCE + * @param[in] data any specific data if present + * @param[in] usrCtx any specific user context if present + * + * @returns void + */ +static void +osTegraDceClientIpcCallback +( + NvU32 handle, + NvU32 interfaceType, + NvU32 length, + void *data, + void *usrCtx +) +{ + THREAD_STATE_NODE threadState; + NvU32 rmInterfaceType = nv_tegra_get_rm_interface_type(interfaceType); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (rmLocksAcquireAll(RM_LOCK_MODULES_KERNEL_RM_EVENTS) == NV_OK) + { + dceclientHandleAsyncRpcCallback(handle, rmInterfaceType, length, data, usrCtx); + + rmLocksReleaseAll(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); +} + +/*! + * @brief Performs IPC Client registration with DCE + * + * This function is basically a wrapper to call the unix/linux layer. + * + * @param[in] interfaceType RM IPC interface type + * @param[in] usrCtx any specific user context if present + * @param[out] clientId unique ID registered with DCE for IPC + * + * @returns NV_OK if successful, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * other errors as may be returned by subfunctions. + */ +NV_STATUS +osTegraDceRegisterIpcClient +( + NvU32 interfaceType, + void *usrCtx, + NvU32 *clientId +) +{ + if (interfaceType == DCE_CLIENT_RM_IPC_TYPE_SYNC) + return nv_tegra_dce_register_ipc_client(interfaceType, usrCtx, NULL, clientId); + else if (interfaceType == DCE_CLIENT_RM_IPC_TYPE_EVENT) + return nv_tegra_dce_register_ipc_client(interfaceType, usrCtx, osTegraDceClientIpcCallback, clientId); + else + return NV_ERR_INVALID_ARGUMENT; +} + +/*! + * @brief Performs IPC Client destroy with DCE + * + * This function is basically a wrapper to call the unix/linux layer. + * + * @param[in] clientId unique ID registered with DCE for IPC + * + * @returns NV_OK if successful, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * other errors as may be returned by subfunctions. + */ +NV_STATUS +osTegraDceUnregisterIpcClient +( + NvU32 clientId +) +{ + return nv_tegra_dce_unregister_ipc_client(clientId); +} + +/*! + * @brief Performs IPC Send/Receive to/from DCE + * + * This function is basically a wrapper to call the unix/linux layer. + * + * @param[in] clientId unique ID registered with DCE KMD for corresponding IPC type + * @param[in] msg structure to hold dce ipc message info + * @param[in] msgLength length of the message + * + * @returns NV_OK if successful, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * other errors as may be returned by subfunctions. + */ +NV_STATUS +osTegraDceClientIpcSendRecv +( + NvU32 clientId, + void *msg, + NvU32 msgLength +) +{ + return nv_tegra_dce_client_ipc_send_recv(clientId, msg, msgLength); +} + +/*! + * @brief Sends an MRQ (message-request) to BPMP + * + * The request, response, and ret parameters of this function correspond to the + * components of the tegra_bpmp_message struct, which BPMP uses to receive + * MRQs. + * + * @param[in] pGpu OBJGPU pointer + * @param[in] mrq MRQ_xxx ID specifying what is requested + * @param[in] pRequestData Pointer to request input data + * @param[in] requestDataSize Size of structure pointed to by pRequestData + * @param[out] pResponseData Pointer to response output data + * @param[in] responseDataSize Size of structure pointed to by pResponseData + * @param[out] ret MRQ return code (from "ret" element of + * tegra_bpmp_message struct) + * @param[out] apiRet Return code from tegra_bpmp_transfer call + * + * @returns NV_OK if successful, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, + * NV_ERR_INVALID_POINTER if the tegra_bpmp struct pointer could not + * be obtained from nv, or + * NV_ERR_GENERIC if the tegra_bpmp_transfer call failed (see apiRet + * for Linux error code). + */ +NV_STATUS +osTegraSocBpmpSendMrq +( + OBJGPU *pGpu, + NvU32 mrq, + const void *pRequestData, + NvU32 requestDataSize, + void *pResponseData, + NvU32 responseDataSize, + NvS32 *pRet, + NvS32 *pApiRet +) +{ + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_SOC_SDM)) + { + return NV_ERR_NOT_SUPPORTED; + } + else if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + return nv_bpmp_send_mrq(pGpu->pOsGpuInfo, + mrq, + pRequestData, + requestDataSize, + pResponseData, + responseDataSize, + pRet, + pApiRet); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/*! + * @brief Returns IMP-relevant data collected from other modules + * + * This function is basically a wrapper to call the unix/linux layer. + * + * @param[out] pTegraImpImportData Structure to receive the data + * + * @returns NV_OK if successful, + * NV_ERR_BUFFER_TOO_SMALL if the array in TEGRA_IMP_IMPORT_DATA is + * too small, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * other errors as may be returned by subfunctions. + */ +NV_STATUS +osTegraSocGetImpImportData +( + OBJGPU *pGpu, + TEGRA_IMP_IMPORT_DATA *pTegraImpImportData +) +{ + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + return nv_imp_get_import_data(pTegraImpImportData); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/*! + * @brief Returns IMP-relevant data collected from UEFI + * + * This function is basically a wrapper to call the unix/linux layer. + * + * @param[in] pOsGpuInfo Per GPU Linux state + * @param[out] pIsoBwKbps ISO BW set by UEFI + * @param[out] pFloorBwKbps DRAM Floor BW set by UEFI + * + * @returns NV_OK if successful, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * other errors as may be returned by subfunctions. + */ +NV_STATUS +osTegraSocGetImpUefiData +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *pIsoBwKbps, + NvU32 *pFloorBwKbps +) +{ + return nv_imp_get_uefi_data(pOsGpuInfo, pIsoBwKbps, pFloorBwKbps); +} + +/*! + * @brief Tells BPMP whether or not RFL is valid + * + * Display HW generates an ok_to_switch signal which asserts when mempool + * occupancy is high enough to be able to turn off memory long enough to + * execute a dramclk frequency switch without underflowing display output. + * ok_to_switch drives the RFL ("request for latency") signal in the memory + * unit, and the switch sequencer waits for this signal to go active before + * starting a dramclk switch. However, if the signal is not valid (e.g., if + * display HW or SW has not been initialized yet), the switch sequencer ignores + * the signal. This API tells BPMP whether or not the signal is valid. + * + * @param[in] pOsGpuInfo Per GPU Linux state + * @param[in] bEnable True if RFL will be valid; false if invalid + * + * @returns NV_OK if successful, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * NV_ERR_GENERIC if some other kind of error occurred. + */ +NV_STATUS +osTegraSocEnableDisableRfl +( + OS_GPU_INFO *pOsGpuInfo, + NvBool bEnable +) +{ + if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo)) + { + return nv_imp_enable_disable_rfl(pOsGpuInfo, bEnable); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/*! + * @brief Returns max rates for display clocks passed by UEFI + * + * @param[in] pOsGpuInfo Per GPU Linux state + * @param[out] pMaxDispClkRateDisppll disp clock maxrate with disppll as parent + * @param[out] pMaxDispClkRateSppllClkouta disp clock maxrate with sppllclkouta as parent + * @param[out] pMaxHubClkRateSppllClkoutb hub clock maxrate with sppllclkoutb as parent + * + * @returns NV_OK if successful, + * NV_ERR_NOT_SUPPORTED if the functionality is not available + */ +NV_STATUS +osTegraSocGetDispClockRates +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *pMaxDispClkRateDisppll, + NvU32 *pMaxDispClkRateSppllClkouta, + NvU32 *pMaxHubClkRateSppllClkoutb +) +{ + nv_state_t *nv = pOsGpuInfo; + + if (NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + *pMaxDispClkRateDisppll = nv->clocks.max_dispclk_rate_using_disppllkhz; + *pMaxDispClkRateSppllClkouta = nv->clocks.max_dispclk_rate_using_sppll0clkoutakhz; + *pMaxHubClkRateSppllClkoutb = nv->clocks.max_hubclk_rate_using_sppll0clkoutbkhz; + + return NV_OK; + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/*! + * @brief Allocates a specified amount of ISO memory bandwidth for display + * + * floorBandwidthKBPS is the minimum required (i.e., floor) dramclk frequency + * multiplied by the width of the pipe over which the display data will travel. + * (It is understood that the bandwidth calculated by multiplying the clock + * frequency by the pipe width will not be realistically achievable, due to + * overhead in the memory subsystem. The infrastructure will not actually use + * the bandwidth value, except to reverse the calculation to get the required + * dramclk frequency.) + * + * This function is basically a wrapper to call the unix/linux layer. + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[in] averageBandwidthKBPS Amount of ISO memory bandwidth requested + * @param[in] floorBandwidhtKBPS Min required dramclk freq * pipe width + * + * @returns NV_OK if successful, + * NV_ERR_INSUFFICIENT_RESOURCES if one of the bandwidth values is too + * high, and bandwidth cannot be allocated, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * NV_ERR_GENERIC if some other kind of error occurred. + */ +NV_STATUS +osTegraAllocateDisplayBandwidth +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 averageBandwidthKBPS, + NvU32 floorBandwidthKBPS +) +{ + if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo)) + { + return nv_imp_icc_set_bw(pOsGpuInfo, + averageBandwidthKBPS, + floorBandwidthKBPS); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/*! + * @brief Creates or sets up platform specific nano second resolution timer + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[in] pTmrEvent Pointer to timer event information + * @param[in/out] pTimer pointer to hold high resolution timer object + */ +NV_STATUS +osCreateNanoTimer +( + OS_GPU_INFO *pOsGpuInfo, + void *pTmrEvent, + void **pTimer +) +{ + nv_create_nano_timer(pOsGpuInfo, pTmrEvent, (nv_nano_timer_t **)pTimer); + return NV_OK; +} + +/*! + * @brief Starts platform specific nano second resolution timer + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[in] pTimer pointer to high resolution timer object + * @param[in] timeNs Relative time in nano seconds + */ +NV_STATUS +osStartNanoTimer +( + OS_GPU_INFO *pOsGpuInfo, + void *pTimer, + NvU64 timeNs +) +{ + nv_start_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer, timeNs); + return NV_OK; +} + +/*! + * @brief Cancels platform specific nano second resolution timer + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[in] pTimer pointer to timer object + */ +NV_STATUS +osCancelNanoTimer +( + OS_GPU_INFO *pOsGpuInfo, + void *pTimer +) +{ + nv_cancel_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer); + return NV_OK; +} + +/*! + * @brief Destroys & cancels platform specific nano second resolution timer + * + * + * @param[in] pGpu Device of interest + * @param[in] pTimer pointer to timer object + */ +NV_STATUS +osDestroyNanoTimer +( + OS_GPU_INFO *pOsGpuInfo, + void *pTimer +) +{ + nv_destroy_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer); + return NV_OK; +} + + +NvBool osIsNonPreemptableDebuggerSessionActive(OBJGPU *pGpu) +{ + NvU32 data = 0; + if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_NON_PREEMTABLE_DEBUGGER_SESSION_ACTIVE, &data) == NV_OK) && (data == NV_REG_STR_RM_NON_PREEMTABLE_DEBUGGER_SESSION_ACTIVE_ENABLE)) + { + return NV_TRUE; + } + + return NV_FALSE; +} + +/*! + * @brief Get number of dpAux instances. + * It is wrapper function to call unix/linux layer. + * + * @param[in] pGpu GPU object pointer + * @param[out] pNumIntances Number of dpAux instances. + * + * @returns NV_STATUS, NV_OK if success, + * NV_ERR_GENERIC, if error + * NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ + +NV_STATUS +osGetTegraNumDpAuxInstances +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *pNumIntances +) +{ + if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo)) + { + return nv_get_num_dpaux_instances(pOsGpuInfo, pNumIntances); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/* + * @brief Return the priv Data of current IRQ. + * It is wrapper function to call unix/linux layer. + * + * @param[in] pGpu Device of interest + * @param[out] pPrivData privData of current IRQ + * + * @returns NV_STATUS, NV_OK if success, + * NV_ERR_GENERIC, if error + * NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osGetCurrentIrqPrivData +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *pPrivData +) +{ + if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo)) + { + return nv_get_current_irq_priv_data(pOsGpuInfo, pPrivData); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/*! + * @brief Get the brightness level + * It is wrapper function to call unix/linux layer. + * + * @param[in] pGpu GPU object pointer + * @param[out] brightness Pointer to brightness level + * + * @returns NV_STATUS, NV_OK if success, + * NV_ERR_GENERIC, if error + * NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osGetTegraBrightnessLevel +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *brightness +) +{ + if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo)) + { + return nv_get_tegra_brightness_level(pOsGpuInfo, brightness); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/*! + * @brief Set the brightness level + * It is wrapper function to call unix/linux layer. + * + * @param[in] pGpu GPU object pointer + * @param[out] brightness brightness level + * + * @returns NV_STATUS, NV_OK if success, + * NV_ERR_GENERIC, if error + * NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osSetTegraBrightnessLevel +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 brightness +) +{ + if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo)) + { + return nv_set_tegra_brightness_level(pOsGpuInfo, brightness); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +/* @brief Gets syncpoint aperture information + * + * @param[in] OS_GPU_INFO OS specific GPU information pointer + * @param[in] syncpointId + * @param[out] *physAddr + * @param[out] *limit + * @param[out] *offset + */ +NV_STATUS +osGetSyncpointAperture +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 syncpointId, + NvU64 *physAddr, + NvU64 *limit, + NvU32 *offset +) +{ + return nv_get_syncpoint_aperture(syncpointId, physAddr, limit, offset); +} + +/*! + * @brief Enable PCIe AtomicOp Requester Enable and return + * the completer side capabilities that the requester can send. + * + * @param[in] pOsGpuInfo OS_GPU_INFO OS specific GPU information pointer + * @param[out] pMask mask of supported atomic size, including one or more of: + * OS_PCIE_CAP_MASK_REQ_ATOMICS_32 + * OS_PCIE_CAP_MASK_REQ_ATOMICS_64 + * OS_PCIE_CAP_MASK_REQ_ATOMICS_128 + * + * @returns NV_STATUS, NV_OK if success + * NV_ERR_NOT_SUPPORTED if platform doesn't support this + * feature. + * NV_ERR_GENERIC for any other error + */ + +NV_STATUS +osConfigurePcieReqAtomics +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *pMask +) +{ + if (pMask) + { + *pMask = 0U; + if (pOsGpuInfo) + { + if (os_enable_pci_req_atomics(pOsGpuInfo->handle, + OS_INTF_PCIE_REQ_ATOMICS_32BIT) == NV_OK) + *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_32; + if (os_enable_pci_req_atomics(pOsGpuInfo->handle, + OS_INTF_PCIE_REQ_ATOMICS_64BIT) == NV_OK) + *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_64; + if (os_enable_pci_req_atomics(pOsGpuInfo->handle, + OS_INTF_PCIE_REQ_ATOMICS_128BIT) == NV_OK) + *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_128; + + if (*pMask != 0) + return NV_OK; + } + } + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Check GPU is accessible or not + * + * @param[in] pGpu GPU object pointer + * + * @returns NVBool, Returns TRUE if the GPU is accessible, + * FALSE, if error + */ +NvBool +osIsGpuAccessible +( + OBJGPU *pGpu +) +{ + return nv_is_gpu_accessible(NV_GET_NV_STATE(pGpu)); +} + +/*! + * @brief Check whether GPU has received a shutdown notification from the OS + */ +NvBool +osIsGpuShutdown +( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return nv ? nv->is_shutdown : NV_TRUE; +} + +/*! + * @brief Check GPU OS info matches + * + * @param[in] pGpu GPU object pointer + * + * @returns NVBool, Returns TRUE if matched. + */ +NvBool +osMatchGpuOsInfo +( + OBJGPU *pGpu, + void *pOsInfo +) +{ + return nv_match_gpu_os_info(NV_GET_NV_STATE(pGpu), pOsInfo); +} + +/*! + * @brief Release GPU OS info. + * + * @param[in] pOsInfo GPU OS info pointer + * + * @returns void + */ +void +osReleaseGpuOsInfo +( + void *pOsInfo +) +{ + nv_put_file_private(pOsInfo); +} + +/*! + * @brief Get free, total memory of a NUMA node by NUMA node ID from kernel. + * + * @param[in] numaId NUMA node ID. + * @param[out] free_memory_bytes free memory in bytes. + * @param[out] total_memory_bytes total memory in bytes. + * + */ +void +osGetNumaMemoryUsage +( + NvS32 numaId, + NvU64 *free_memory_bytes, + NvU64 *total_memory_bytes +) +{ + NV_STATUS status = os_get_numa_node_memory_usage(numaId, + free_memory_bytes, + total_memory_bytes); + NV_ASSERT(status == NV_OK); +} + +/*! + * @brief Add GPU memory as a NUMA node. + * + * @param[in/out] pOsGpuInfo OS specific GPU information pointer + * @param[in] offset start offset of the partition within FB + * @param[in] size size of the partition + * @param[out] pNumaNodeId OS NUMA node id for the added memory. + * + * @returns NV_OK if all is okay. Otherwise an error-specific value. + * + */ +NV_STATUS +osNumaAddGpuMemory +( + OS_GPU_INFO *pOsGpuInfo, + NvU64 offset, + NvU64 size, + NvU32 *pNumaNodeId +) +{ + nv_state_t *nv = pOsGpuInfo; + + return os_numa_add_gpu_memory(nv->handle, offset, size, pNumaNodeId); +} + +/*! + * @brief Remove a particular SMC partition's GPU memory from OS kernel. + * + * Remove GPU memory from the OS kernel that is earlier added as a NUMA node + * to the kernel in platforms where GPU is coherently connected to the CPU. + * + * @param[in/out] pOsGpuInfo OS_GPU_INFO OS specific GPU information pointer + * @param[in] offset start offset of the partition within FB + * @param[in] size size of the partition + * @param[in] numaNodeId OS NUMA node id of the memory to be removed. + * + */ +void +osNumaRemoveGpuMemory +( + OS_GPU_INFO *pOsGpuInfo, + NvU64 offset, + NvU64 size, + NvU32 numaNodeId +) +{ + nv_state_t *nv = pOsGpuInfo; + + NV_STATUS status = os_numa_remove_gpu_memory(nv->handle, offset, size, numaNodeId); + NV_ASSERT(status == NV_OK); + + return; +} + +NvBool +osDmabufIsSupported(void) +{ + return os_dma_buf_enabled; +} + +NV_STATUS +osGetEgmInfo +( + OBJGPU *pGpu, + NvU64 *pPhysAddr, + NvU64 *pSize, + NvS32 *pNodeId +) +{ + return nv_get_egm_info(NV_GET_NV_STATE(pGpu), pPhysAddr, pSize, pNodeId); +} + +/*! + * @brief Offline (i.e., blacklist) the page containing a given address from OS kernel. + * + * @param[in] address Address (SPA) of the page to be offlined + * + */ +NV_STATUS +osOfflinePageAtAddress +( + NvU64 address +) +{ + return os_offline_page_at_address(address); +} + +void osAllocatedRmClient(void *pOsInfo) +{ + nv_file_private_t* nvfp = (nv_file_private_t*)pOsInfo; + + if (nvfp != NULL) + nvfp->bCleanupRmapi = NV_TRUE; +} + +/*! + * @brief Update variable to indicate console managed by drm driver. + * + * @param[in] OBJGPU GPU object pointer + * + * @returns void + */ +void +osDisableConsoleManagement +( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + nv->client_managed_console = NV_TRUE; +} + +/* + * @brief Return NV_TRUE if GPU is configured as CXL device. + * + * @param[in] pGpu GPU object pointer + */ +NvBool +osGpuIsCxlDevice +( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + return nv->is_cxl_dev; +} + +/* + * @brief Disabel CXL cache and perform writeback and invalidate for CXL device. + * + * @param[in] pGpu GPU object pointer + * @param[in] bEnableCache Enable/Disable flag + */ +void +osCxlSetCaching +( + OBJGPU *pGpu, + NvBool bEnableCache +) +{ + + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + nv_pci_cxl_set_caching(nv, bEnableCache); +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osapi.c b/userspace/sdks/nv570/nvalloc/unix/src/osapi.c new file mode 100644 index 0000000..a3d614d --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/osapi.c @@ -0,0 +1,6284 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + + + +#include <nv_ref.h> +#include <nv.h> +#include <nv_escape.h> +#include <nv-priv.h> +#include <os/os.h> +#include <osapi.h> +#include <class/cl0000.h> +#include <rmosxfac.h> // Declares RmInitRm(). +#include "gpu/gpu.h" +#include "gps.h" +#include <platform/chipset/chipset.h> + +#include <gpu/timer/objtmr.h> +#include <gpu/subdevice/subdevice.h> +#include <mem_mgr/mem.h> +#include "kernel/gpu/mem_mgr/mem_mgr.h" + +#include <gpu/mem_sys/kern_mem_sys.h> + +#include <diagnostics/journal.h> +#include <nvrm_registry.h> + +#include <nvUnixVersion.h> +#include <gpu_mgr/gpu_mgr.h> +#include <core/thread_state.h> +#include <platform/acpi_common.h> +#include <core/locks.h> +#include "platform/sli/sli.h" + +#include <mem_mgr/p2p.h> + +#include "rmapi/exports.h" +#include "rmapi/rmapi_utils.h" +#include "rmapi/rs_utils.h" +#include "rmapi/resource_fwd_decls.h" +#include <nv-kernel-rmapi-ops.h> +#include <rmobjexportimport.h> +#include "nv-reg.h" +#include "nv-firmware-registry.h" +#include "core/hal_mgr.h" +#include "gpu/device/device.h" + +#include "resserv/rs_server.h" +#include "resserv/rs_client.h" +#include "resserv/rs_resource.h" +#include "gpu/gpu_uuid.h" + +#include "platform/chipset/pci_pbi.h" + +#include "ctrl/ctrl0000/ctrl0000system.h" +#include "ctrl/ctrl0073/ctrl0073dp.h" +#include "ctrl/ctrl0073/ctrl0073system.h" +#include "ctrl/ctrl0073/ctrl0073specific.h" +#include "ctrl/ctrl2080/ctrl2080bios.h" +#include "ctrl/ctrl2080/ctrl2080fb.h" +#include "ctrl/ctrl2080/ctrl2080perf.h" +#include "ctrl/ctrl2080/ctrl2080gpu.h" +#include "ctrl/ctrl402c.h" + +#include "g_nv_name_released.h" // released chip entries from nvChipAutoFlags.h + +#include <virtualization/hypervisor/hypervisor.h> + +#include "gpu/bus/kern_bus.h" + +// +// If timer callback comes when PM resume is in progress, then it can't be +// serviced. The timer needs to be rescheduled in this case. This time controls +// the duration of rescheduling. +// +#define TIMER_RESCHED_TIME_DURING_PM_RESUME_NS (100 * 1000 * 1000) + +static void RmInflatePageArray( + RmPhysAddr *pteArray, + NvU64 pageCountFrom, + NvU64 pageSizeFrom, + NvU64 pageCountTo, + NvU64 pageSizeTo); +static void RmDeflatePageArray( + RmPhysAddr *pteArray, + NvU64 pageCountFrom, + NvU64 pageSizeFrom, + NvU64 pageCountTo, + NvU64 pageSizeTo); + +static void RmInflatePageArray( + RmPhysAddr *pteArray, + NvU64 pageCountFrom, + NvU64 pageSizeFrom, + NvU64 pageCountTo, + NvU64 pageSizeTo) +{ + NV_ASSERT(pageCountFrom <= pageCountTo); + NV_ASSERT(pageSizeFrom >= pageSizeTo); + NvUPtr dstPageIdx, dstPageOffset; + NvU64 i; + // convert the idx from pageCount_to to pageCount_from. + NvU64 dstPageShift = NV_SRC_TO_DST_PAGE_SHIFT(pageSizeTo, pageSizeFrom); + + // + // We can do the translation in place by moving backwards, since there + // will always be more RM pages than OS pages + // + for (i = pageCountTo - 1; i != NV_U64_MAX; i--) + { + dstPageIdx = i >> dstPageShift; + dstPageOffset = (i & ((1 << dstPageShift) - 1)) * + pageSizeTo; + pteArray[i] = pteArray[dstPageIdx] + dstPageOffset; + } +} + +static void RmDeflatePageArray( + RmPhysAddr *pteArray, + NvU64 pageCountFrom, + NvU64 pageSizeFrom, + NvU64 pageCountTo, + NvU64 pageSizeTo) +{ + NV_ASSERT(pageCountFrom >= pageCountTo); + NV_ASSERT(pageSizeFrom <= pageSizeTo); + NvU64 i; + + for (i = 0; i < pageCountTo; i++) + { + pteArray[i] = pteArray[(i << NV_SRC_TO_DST_PAGE_SHIFT(pageSizeFrom, pageSizeTo))]; + } + + // Zero out the rest of the addresses, which are now invalid + portMemSet(pteArray + i, 0, sizeof(*pteArray) * (pageCountFrom - i)); +} + +void convertPageArrayBetweenOsAndRm( + RmPhysAddr *pteArray, + NvU64 pageCountFrom, + NvU64 pageSizeFrom, + NvU64 pageCountTo, + NvU64 pageSizeTo, + NvU64 *newPageCountTo) +{ + if (pageSizeFrom < pageSizeTo) + { + RmDeflatePageArray(pteArray, + pageCountFrom, + pageSizeFrom, + pageCountTo, + pageSizeTo + ); + if (newPageCountTo) + { + *newPageCountTo = pageCountTo; + } + } + else if (pageSizeFrom > pageSizeTo) + { + RmInflatePageArray(pteArray, + pageCountFrom, + pageSizeFrom, + pageCountTo, + pageSizeTo + ); + } +} + +// +// Helper function which can be called before doing any RM control +// This function: +// +// a. Performs threadStateInit(). +// b. Acquires API lock. +// c. Checks if RMAPI client handle is valid (i.e. RM is initialized) and +// returns early if RMAPI client handle is invalid. +// d. Increments the dynamic power refcount. If GPU is in RTD3 suspended +// state, then it will wake-up the GPU. +// e. Returns the RMAPI interface handle. +// +// This function should be called only when caller doesn't have acquired API +// lock. Caller needs to use RmUnixRmApiEpilogue() after RM control, if +// RmUnixRmApiPrologue() is successful. +// +RM_API *RmUnixRmApiPrologue(nv_state_t *pNv, THREAD_STATE_NODE *pThreadNode, NvU32 module) +{ + threadStateInit(pThreadNode, THREAD_STATE_FLAGS_NONE); + + if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, module)) == NV_OK) + { + if ((pNv->rmapi.hClient != 0) && + (os_ref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE) == NV_OK)) + { + return rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + } + + rmapiLockRelease(); + } + + threadStateFree(pThreadNode, THREAD_STATE_FLAGS_NONE); + + return NULL; +} + +// +// Helper function which can be called after doing RM control, if +// caller has used RmUnixRmApiPrologue() helper function. This function: +// +// a. Decrements the dynamic power refcount. +// b. Release API lock. +// c. Performs threadStateFree(). +// +void RmUnixRmApiEpilogue(nv_state_t *pNv, THREAD_STATE_NODE *pThreadNode) +{ + os_unref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE); + rmapiLockRelease(); + threadStateFree(pThreadNode, THREAD_STATE_FLAGS_NONE); +} + +NvBool RmGpuHasIOSpaceEnabled(nv_state_t * nv) +{ + NvU16 val; + NvBool has_io; + os_pci_read_word(nv->handle, NV_CONFIG_PCI_NV_1, &val); + has_io = FLD_TEST_DRF(_CONFIG, _PCI_NV_1, _IO_SPACE, _ENABLED, val); + return has_io; +} + +// This is a stub function for unix +void osHandleDeferredRecovery( + OBJGPU *pGpu +) +{ + +} + +// This is a stub function for unix +NvBool osIsSwPreInitOnly +( + OS_GPU_INFO *pOsGpuInfo +) +{ + return NV_FALSE; +} + +const NvU8 * RmGetGpuUuidRaw( + nv_state_t *pNv +) +{ + NV_STATUS rmStatus; + OBJGPU *pGpu = NULL; + NvU32 gidFlags; + NvBool isApiLockTaken = NV_FALSE; + + if (pNv->nv_uuid_cache.valid) + return pNv->nv_uuid_cache.uuid; + + if (!rmapiLockIsOwner()) + { + rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU); + if (rmStatus != NV_OK) + { + return NULL; + } + + isApiLockTaken = NV_TRUE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + // + // PBI is not present in simulation and the loop inside + // pciPbiReadUuid takes up considerable amount of time in + // simulation environment during RM load. + // + if (pGpu && IS_SIMULATION(pGpu)) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + } + else + { + if (!pNv->nv_uuid_cache.pci_uuid_read_attempted) + { + rmStatus = pciPbiReadUuid(pNv->handle, pNv->nv_uuid_cache.uuid); + pNv->nv_uuid_cache.pci_uuid_read_attempted = NV_TRUE; + pNv->nv_uuid_cache.pci_uuid_status = rmStatus; + } + else + { + rmStatus = pNv->nv_uuid_cache.pci_uuid_status; + } + } + + if (rmStatus == NV_OK) + { + rmStatus = gpumgrSetUuid(pNv->gpu_id, pNv->nv_uuid_cache.uuid); + if (rmStatus != NV_OK) + { + goto err; + } + + pNv->nv_uuid_cache.valid = NV_TRUE; + goto done; + } + else if (rmStatus == NV_ERR_NOT_SUPPORTED) + { + nv_printf(NV_DBG_INFO, + "NVRM: PBI is not supported for GPU " NV_PCI_DEV_FMT "\n", + NV_PCI_DEV_FMT_ARGS(pNv)); + } + + gidFlags = DRF_DEF(2080_GPU_CMD,_GPU_GET_GID_FLAGS,_TYPE,_SHA1) + | DRF_DEF(2080_GPU_CMD,_GPU_GET_GID_FLAGS,_FORMAT,_BINARY); + + if (!pGpu) + goto err; + + rmStatus = gpuGetGidInfo(pGpu, NULL, NULL, gidFlags); + if (rmStatus != NV_OK) + goto err; + + if (!pGpu->gpuUuid.isInitialized) + goto err; + + // copy the uuid from the OBJGPU uuid cache + os_mem_copy(pNv->nv_uuid_cache.uuid, pGpu->gpuUuid.uuid, GPU_UUID_LEN); + pNv->nv_uuid_cache.valid = NV_TRUE; + +done: + if (isApiLockTaken) + { + rmapiLockRelease(); + } + + return pNv->nv_uuid_cache.uuid; + +err: + if (isApiLockTaken) + { + rmapiLockRelease(); + } + + return NULL; +} + +static NV_STATUS RmGpuUuidRawToString( + const NvU8 *pGidData, + char *pBuf, + NvU32 bufLen +) +{ + NvU8 *pGidString; + NvU32 GidStrlen; + NV_STATUS rmStatus; + NvU32 gidFlags; + + gidFlags = DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _ASCII) | + DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1); + + rmStatus = transformGidToUserFriendlyString(pGidData, RM_SHA1_GID_SIZE, + &pGidString, &GidStrlen, + gidFlags, + RM_UUID_PREFIX_GPU); + if (rmStatus != NV_OK) + return rmStatus; + + if (bufLen >= GidStrlen) + portMemCopy(pBuf, bufLen, pGidString, GidStrlen); + else + rmStatus = NV_ERR_BUFFER_TOO_SMALL; + + portMemFree((void *)pGidString); + + return rmStatus; +} + +// This function should be called with the API and GPU locks already acquired. +void +RmLogGpuCrash(OBJGPU *pGpu) +{ + NvBool bGpuIsLost, bGpuIsConnected; + + // + // Re-evaluate whether or not the GPU is accessible. This could be called + // from a recovery context where the OS has re-enabled MMIO for the device. + // This happens during EEH processing on IBM Power + Linux, and marking + // the device as connected again will allow rcdbAddRmGpuDump() to collect + // more GPU state. + // + bGpuIsLost = pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_LOST); + bGpuIsConnected = pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED); + if (!bGpuIsConnected || bGpuIsLost) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 pmcBoot0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0); + if (pmcBoot0 == nvp->pmc_boot_0) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED, NV_TRUE); + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_LOST, NV_FALSE); + } + } + + { + // + // Log the engine data to the Journal object, to be pulled out later. This + // will return NV_WARN_MORE_PROCESSING_REQUIRED if the dump needed to be + // deferred to a passive IRQL. We still log the crash dump as being created + // in that case since it (should) be created shortly thereafter, and + // there's currently not a good way to print the below notification + // publicly from the core RM when the DPC completes. + // + NV_STATUS status = rcdbAddRmGpuDump(pGpu); + if (status != NV_OK && status != NV_WARN_MORE_PROCESSING_REQUIRED) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to save GPU crash data\n", __FUNCTION__); + } + else + { + nv_printf(NV_DBG_ERRORS, + "NVRM: A GPU crash dump has been created. If possible, please run\n" + "NVRM: nvidia-bug-report.sh as root to collect this data before\n" + "NVRM: the NVIDIA kernel module is unloaded.\n"); + if (!IS_GSP_CLIENT(pGpu) && hypervisorIsVgxHyper()) + { + nv_printf(NV_DBG_ERRORS, "NVRM: Dumping nvlogs buffers\n"); + nvlogDumpToKernelLog(NV_FALSE); + } + } + } + + // Restore the disconnected properties, if they were reset + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED, bGpuIsConnected); + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_LOST, bGpuIsLost); + + // Restore persistence mode to the way it was prior to the crash + osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, + pGpu->getProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE)); +} + +static void free_os_event_under_lock(nv_event_t *event) +{ + portAtomicSetS32(&event->active, 0); + + // If refcount > 0, event will be freed by osDereferenceObjectCount + // when the last associated RM event is freed. + if (event->refcount == 0) + portMemFree(event); +} + +static void free_os_events( + nv_file_private_t *nvfp, + NvHandle client +) +{ + nv_state_t *nv = nv_get_ctl_state(); + nv_event_t **pprev; + + portSyncSpinlockAcquire(nv->event_spinlock); + + pprev = &nv->event_list; + while (*pprev != NULL) + { + nv_event_t *cur = *pprev; + // + // XXX We must be called from either rm_client_free_os_events() or + // RmFreeUnusedClients() for this to work. + // + if ((cur->hParent == client) || (cur->nvfp == nvfp)) + { + *pprev = cur->next; + free_os_event_under_lock(cur); + } + else + { + pprev = &cur->next; + } + } + + portSyncSpinlockRelease(nv->event_spinlock); +} + +static NV_STATUS get_os_event_data( + nv_file_private_t *nvfp, + NvP64 pEvent, + NvU32 *MoreEvents +) +{ + nv_event_t nv_event; + NvUnixEvent *nv_unix_event; + NV_STATUS status; + + status = os_alloc_mem((void**)&nv_unix_event, sizeof(NvUnixEvent)); + if (status != NV_OK) + return status; + + status = nv_get_event(nvfp, &nv_event, MoreEvents); + if (status != NV_OK) + { + status = NV_ERR_OPERATING_SYSTEM; + goto done; + } + + os_mem_set(nv_unix_event, 0, sizeof(NvUnixEvent)); + nv_unix_event->hObject = nv_event.hObject; + nv_unix_event->NotifyIndex = nv_event.index; + nv_unix_event->info32 = nv_event.info32; + nv_unix_event->info16 = nv_event.info16; + + status = os_memcpy_to_user(NvP64_VALUE(pEvent), nv_unix_event, sizeof(NvUnixEvent)); +done: + os_free_mem(nv_unix_event); + return status; +} + +void rm_client_free_os_events( + NvHandle client +) +{ + free_os_events(NULL, client); +} + +void RmFreeUnusedClients( + nv_state_t *nv, + nv_file_private_t *nvfp +) +{ + NvU32 *pClientList; + NvU32 numClients, i; + NV_STATUS status; + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + // + // The 'nvfp' pointer uniquely identifies an open instance in kernel space + // and the kernel interface layer guarantees that we are not called before + // the associated nvfp descriptor is closed. We can thus safely free + // abandoned clients with matching 'nvfp' pointers. + // + status = rmapiGetClientHandlesFromOSInfo(nvfp, &pClientList, &numClients); + if (status != NV_OK) + { + numClients = 0; + } + + for (i = 0; i < numClients; ++i) + { + NV_PRINTF(LEVEL_INFO, "freeing abandoned client 0x%x\n", + pClientList[i]); + + } + + if (numClients != 0) + { + pRmApi->DisableClients(pRmApi, pClientList, numClients); + + portMemFree(pClientList); + } + + // Clean up any remaining events using this nvfp. + free_os_events(nvfp, 0); +} + +static void RmUnbindLock( + nv_state_t *nv +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if ((pGpu == NULL) || (gpuGetUserClientCount(pGpu) == 0)) + { + nv->flags |= NV_FLAG_UNBIND_LOCK; + } +} + +static NV_STATUS allocate_os_event( + NvHandle hParent, + nv_file_private_t *nvfp, + NvU32 fd +) +{ + nv_state_t *nv = nv_get_ctl_state(); + NvU32 status = NV_OK; + nv_event_t *event; + + nv_event_t *new_event = portMemAllocNonPaged(sizeof(nv_event_t)); + if (new_event == NULL) + { + status = NV_ERR_NO_MEMORY; + goto done; + } + + new_event->hParent = hParent; + new_event->nvfp = nvfp; + new_event->fd = fd; + portAtomicSetS32(&new_event->active, 1); + new_event->refcount = 0; + + portSyncSpinlockAcquire(nv->event_spinlock); + for (event = nv->event_list; event; event = event->next) + { + // Only one event may be associated with a given fd. + if (event->hParent == hParent && event->fd == fd) + { + status = NV_ERR_INVALID_ARGUMENT; + portSyncSpinlockRelease(nv->event_spinlock); + goto done; + } + } + + new_event->next = nv->event_list; + nv->event_list = new_event; + nvfp->bCleanupRmapi = NV_TRUE; + portSyncSpinlockRelease(nv->event_spinlock); + +done: + if (status == NV_OK) + { + NV_PRINTF(LEVEL_INFO, "allocated OS event:\n"); + NV_PRINTF(LEVEL_INFO, " hParent: 0x%x\n", hParent); + NV_PRINTF(LEVEL_INFO, " fd: %d\n", fd); + } + else + { + NV_PRINTF(LEVEL_INFO, "failed to allocate OS event: 0x%08x\n", status); + status = NV_ERR_INSUFFICIENT_RESOURCES; + portMemFree(new_event); + } + + return status; +} + +static NV_STATUS free_os_event( + NvHandle hParent, + NvU32 fd +) +{ + nv_state_t *nv = nv_get_ctl_state(); + nv_event_t *event, *tmp; + NV_STATUS result; + + portSyncSpinlockAcquire(nv->event_spinlock); + tmp = event = nv->event_list; + while (event) + { + if ((event->fd == fd) && (event->hParent == hParent)) + { + if (event == nv->event_list) + nv->event_list = event->next; + else + tmp->next = event->next; + break; + } + tmp = event; + event = event->next; + } + + if (event != NULL) + { + free_os_event_under_lock(event); + result = NV_OK; + } + else + result = NV_ERR_INVALID_EVENT; + portSyncSpinlockRelease(nv->event_spinlock); + + if (result == NV_OK) + { + NV_PRINTF(LEVEL_INFO, "freed OS event:\n"); + NV_PRINTF(LEVEL_INFO, " hParent: 0x%x\n", hParent); + NV_PRINTF(LEVEL_INFO, " fd: %d\n", fd); + } + else + { + NV_PRINTF(LEVEL_ERROR, "failed to find OS event:\n"); + NV_PRINTF(LEVEL_ERROR, " hParent: 0x%x\n", hParent); + NV_PRINTF(LEVEL_ERROR, " fd: %d\n", fd); + } + + return result; +} + +static void RmExecuteWorkItem( + void *pWorkItem +) +{ + nv_work_item_t *pWi = (nv_work_item_t *)pWorkItem; + NvU32 gpuMask = 0; + OsQueueWorkItemFlags releaseLocks = {0}; + + if (!(pWi->flags.bRequiresGpu) && + (pWi->flags.bLockGpus || + pWi->flags.bLockGpuGroupDevice || + pWi->flags.bLockGpuGroupSubdevice || + pWi->flags.bFullGpuSanity)) + { + // Requesting one of the GPU locks without providing a GPU instance + NV_ASSERT(0); + goto done; + } + + // Get locks requested by workitem + if (NV_OK != workItemLocksAcquire(pWi->gpuInstance, pWi->flags, + &releaseLocks, &gpuMask)) + { + goto done; + } + + // Some work items may not require a valid GPU instance + if (pWi->flags.bRequiresGpu) + { + // Make sure that pGpu is present + OBJGPU *pGpu = gpumgrGetGpu(pWi->gpuInstance); + if (pGpu != NULL) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + NV_ASSERT_OR_GOTO(nv != NULL, done); + + NvU32 prevGpuInst = gpumgrSetCurrentGpuInstance(pGpu->gpuInstance); + + if (!(pWi->flags.bDropOnUnloadQueueFlush && os_is_queue_flush_ongoing(nv->queue))) + { + pWi->func.pGpuFunction(pWi->gpuInstance, pWi->pData); + } + + gpumgrSetCurrentGpuInstance(prevGpuInst); + } + else + { + NV_PRINTF(LEVEL_ERROR, "Invalid GPU instance for workitem\n"); + } + } + else + { + if (!(pWi->flags.bDropOnUnloadQueueFlush && os_is_queue_flush_ongoing(NULL))) + { + pWi->func.pSystemFunction(pWi->pData); + } + } + +done: + // Release any locks taken + workItemLocksRelease(releaseLocks, gpuMask); + + if ((pWi->pData != NULL) && !(pWi->flags.bDontFreeParams)) + { + portMemFree(pWi->pData); + } + + portMemFree((void *)pWi); +} + +static NV_STATUS RmAccessRegistry( + NvHandle hClient, + NvHandle hObject, + NvU32 AccessType, + NvP64 clientDevNodeAddress, + NvU32 DevNodeLength, + NvP64 clientParmStrAddress, + NvU32 ParmStrLength, + NvP64 clientBinaryDataAddress, + NvU32 *pBinaryDataLength, + NvU32 *Data, + NvU32 *Entry +) +{ + NvU32 gpuMask = 0, gpuInstance = 0; + OBJGPU *pGpu; + NvBool isDevice = NV_FALSE; + NV_STATUS RmStatus = NV_ERR_OPERATING_SYSTEM; + CLIENT_ENTRY *pClientEntry; + Device *pDevice; + Subdevice *pSubdevice; + + RMAPI_PARAM_COPY devNodeParamCopy; + NvU8 *tmpDevNode = NULL; + NvU32 copyOutDevNodeLength = 0; + + RMAPI_PARAM_COPY parmStrParamCopy; + char *tmpParmStr = NULL; + NvU32 copyOutParmStrLength = 0; + + RMAPI_PARAM_COPY binaryDataParamCopy; + NvU8 *tmpBinaryData = NULL; + NvU32 BinaryDataLength = 0; + NvU32 copyOutBinaryDataLength = 0; + + if (NV_OK != serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, + &pClientEntry)) + { + return NV_ERR_INVALID_CLIENT; + } + + if (hClient == hObject) + { + pGpu = NULL; + } + else + { + RsClient *pClient = pClientEntry->pClient; + + RmStatus = deviceGetByHandle(pClient, hObject, &pDevice); + if (RmStatus != NV_OK) + { + RmStatus = subdeviceGetByHandle(pClient, hObject, &pSubdevice); + if (RmStatus != NV_OK) + goto done; + + RmStatus = rmGpuGroupLockAcquire(pSubdevice->subDeviceInst, + GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU, + &gpuMask); + if (RmStatus != NV_OK) + { + gpuMask = 0; + goto done; + } + + GPU_RES_SET_THREAD_BC_STATE(pSubdevice); + pGpu = GPU_RES_GET_GPU(pSubdevice); + } + else + { + RmStatus = rmGpuGroupLockAcquire(pDevice->deviceInst, + GPU_LOCK_GRP_DEVICE, + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU, + &gpuMask); + if (RmStatus != NV_OK) + { + gpuMask = 0; + goto done; + } + + GPU_RES_SET_THREAD_BC_STATE(pDevice); + pGpu = GPU_RES_GET_GPU(pDevice); + isDevice = NV_TRUE; + } + } + + if (pBinaryDataLength) + { + BinaryDataLength = *pBinaryDataLength; + } + + // a passed-in devNode + if (DevNodeLength) + { + // the passed-in DevNodeLength does not account for '\0' + DevNodeLength++; + + if (DevNodeLength > NVOS38_MAX_REGISTRY_STRING_LENGTH) + { + RmStatus = NV_ERR_INVALID_STRING_LENGTH; + goto done; + } + + // get access to client's DevNode + RMAPI_PARAM_COPY_INIT(devNodeParamCopy, tmpDevNode, clientDevNodeAddress, DevNodeLength, 1); + devNodeParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_ZERO_BUFFER; + RmStatus = rmapiParamsAcquire(&devNodeParamCopy, NV_TRUE); + if (RmStatus != NV_OK) + { + RmStatus = NV_ERR_OPERATING_SYSTEM; + goto done; + } + } + + // a passed-in parmStr + if (ParmStrLength) + { + // the passed-in ParmStrLength does not account for '\0' + ParmStrLength++; + + if ((ParmStrLength == 0) || (ParmStrLength > NVOS38_MAX_REGISTRY_STRING_LENGTH)) + { + RmStatus = NV_ERR_INVALID_STRING_LENGTH; + goto done; + } + // get access to client's parmStr + RMAPI_PARAM_COPY_INIT(parmStrParamCopy, tmpParmStr, clientParmStrAddress, ParmStrLength, 1); + RmStatus = rmapiParamsAcquire(&parmStrParamCopy, NV_TRUE); + if (RmStatus != NV_OK) + { + RmStatus = NV_ERR_OPERATING_SYSTEM; + goto done; + } + if (tmpParmStr[ParmStrLength - 1] != '\0') + { + RmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + } + + if ((AccessType == NVOS38_ACCESS_TYPE_READ_BINARY) || + (AccessType == NVOS38_ACCESS_TYPE_WRITE_BINARY)) + { + if ((BinaryDataLength > NVOS38_MAX_REGISTRY_BINARY_LENGTH) || + (BinaryDataLength == 0)) + { + RmStatus = NV_ERR_INVALID_STRING_LENGTH; + goto done; + } + + // get access to client's binaryData + RMAPI_PARAM_COPY_INIT(binaryDataParamCopy, tmpBinaryData, clientBinaryDataAddress, BinaryDataLength, 1); + if (AccessType == NVOS38_ACCESS_TYPE_READ_BINARY) + binaryDataParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYIN; + RmStatus = rmapiParamsAcquire(&binaryDataParamCopy, NV_TRUE); + if (RmStatus != NV_OK) + { + RmStatus = NV_ERR_OPERATING_SYSTEM; + goto done; + } + } + + switch (AccessType) + { + case NVOS38_ACCESS_TYPE_READ_DWORD: + RmStatus = osReadRegistryDword(pGpu, + tmpParmStr, Data); + break; + + case NVOS38_ACCESS_TYPE_WRITE_DWORD: + if (!osIsAdministrator()) + { + RmStatus = NV_ERR_INSUFFICIENT_PERMISSIONS; + break; + } + if (isDevice) + { + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + RmStatus = osWriteRegistryDword(pGpu, + tmpParmStr, *Data); + + if (RmStatus != NV_OK) + goto done; + } + break; + } + + RmStatus = osWriteRegistryDword(pGpu, + tmpParmStr, *Data); + break; + + case NVOS38_ACCESS_TYPE_READ_BINARY: + RmStatus = osReadRegistryBinary(pGpu, + tmpParmStr, tmpBinaryData, &BinaryDataLength); + + if (RmStatus != NV_OK) + { + goto done; + } + + if (BinaryDataLength) + copyOutBinaryDataLength = BinaryDataLength; + + break; + + case NVOS38_ACCESS_TYPE_WRITE_BINARY: + if (!osIsAdministrator()) + { + RmStatus = NV_ERR_INSUFFICIENT_PERMISSIONS; + break; + } + if (isDevice) + { + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + RmStatus = osWriteRegistryBinary(pGpu, + tmpParmStr, tmpBinaryData, + BinaryDataLength); + + if (RmStatus != NV_OK) + goto done; + } + break; + } + + RmStatus = osWriteRegistryBinary(pGpu, + tmpParmStr, tmpBinaryData, + BinaryDataLength); + break; + + default: + RmStatus = NV_ERR_INVALID_ACCESS_TYPE; + } + + done: + if (gpuMask != 0) + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + + if (tmpDevNode != NULL) + { + // skip copyout on error + if ((RmStatus != NV_OK) || (copyOutDevNodeLength == 0)) + devNodeParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT; + devNodeParamCopy.paramsSize = copyOutDevNodeLength; + if (NV_OK != rmapiParamsRelease(&devNodeParamCopy)) + if (RmStatus == NV_OK) + RmStatus = NV_ERR_OPERATING_SYSTEM; + } + if (tmpParmStr != NULL) + { + // skip copyout on error + if ((RmStatus != NV_OK) || (copyOutParmStrLength == 0)) + parmStrParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT; + parmStrParamCopy.paramsSize = copyOutParmStrLength; + if (NV_OK != rmapiParamsRelease(&parmStrParamCopy)) + if (RmStatus == NV_OK) + RmStatus = NV_ERR_OPERATING_SYSTEM; + } + if (tmpBinaryData != NULL) + { + // skip copyout on error + if ((RmStatus != NV_OK) || (copyOutBinaryDataLength == 0)) + binaryDataParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT; + binaryDataParamCopy.paramsSize = copyOutBinaryDataLength; + if (NV_OK != rmapiParamsRelease(&binaryDataParamCopy)) + if (RmStatus == NV_OK) + RmStatus = NV_ERR_OPERATING_SYSTEM; + *pBinaryDataLength = copyOutBinaryDataLength; + } + + serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClientEntry); + return RmStatus; +} + +static NV_STATUS RmUpdateDeviceMappingInfo( + NvHandle hClient, + NvHandle hDevice, + NvHandle hMappable, + void *pOldCpuAddress, + void *pNewCpuAddress +) +{ + NV_STATUS status; + CLIENT_ENTRY *pClientEntry; + RsClient *pClient; + RsResourceRef *pMappableRef; + RsCpuMapping *pCpuMapping; + Device *pDevice; + Subdevice *pSubdevice; + NvU32 gpuMask = 0; + + status = serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClientEntry); + if (status != NV_OK) + return status; + + pClient = pClientEntry->pClient; + + status = deviceGetByHandle(pClient, hDevice, &pDevice); + if (status != NV_OK) + { + status = subdeviceGetByHandle(pClient, hDevice, &pSubdevice); + if (status != NV_OK) + goto done; + + status = rmGpuGroupLockAcquire(pSubdevice->subDeviceInst, + GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU, + &gpuMask); + if (status != NV_OK) + goto done; + + GPU_RES_SET_THREAD_BC_STATE(pSubdevice); + } + else + { + status = rmGpuGroupLockAcquire(pDevice->deviceInst, + GPU_LOCK_GRP_DEVICE, + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU, + &gpuMask); + if (status != NV_OK) + goto done; + + GPU_RES_SET_THREAD_BC_STATE(pDevice); + } + + status = clientGetResourceRef(pClient, hMappable, &pMappableRef); + if (status != NV_OK) + goto done; + + if ((objDynamicCastById(pMappableRef->pResource, classId(Memory)) == NULL) && + (objDynamicCastById(pMappableRef->pResource, classId(KernelCcuApi)) == NULL) && + (objDynamicCastById(pMappableRef->pResource, classId(KernelChannel)) == NULL)) + { + status = NV_ERR_INVALID_OBJECT_HANDLE; + goto done; + } + + status = refFindCpuMappingWithFilter(pMappableRef, + NV_PTR_TO_NvP64(pOldCpuAddress), + serverutilMappingFilterCurrentUserProc, + &pCpuMapping); + if (status != NV_OK) + goto done; + + pCpuMapping->pLinearAddress = NV_PTR_TO_NvP64(pNewCpuAddress); + +done: + + if (gpuMask != 0) + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + + serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClientEntry); + return status; +} + +static void RmSanitizeLogString( + char *pString, + NvU32 stringLength +) +{ + NvU32 i; + + if (pString == NULL) + return; + + for (i = 0; i < stringLength; i++) + { + NvU8 ch = (NvU8)pString[i]; + + if (ch == '\0') + return; + + if ((ch < 0x20) || (ch == 0x7f)) + { + pString[i] = '?'; + } + } +} + +static NV_STATUS RmPerformVersionCheck( + void *pData, + NvU32 dataSize +) +{ + nv_ioctl_rm_api_version_t *pParams; + char clientCh, rmCh; + const char *rmStr = NV_VERSION_STRING; + NvBool relaxed = NV_FALSE; + NvU32 i; + NvU32 procId; + char procName[32]; + + // + // rmStr (i.e., NV_VERSION_STRING) must be null-terminated and fit within + // NV_RM_API_VERSION_STRING_LENGTH, so that: + // + // (1) If the versions don't match, we can return rmStr in + // pParams->versionString. + // (2) The below loop is guaranteed to not overrun rmStr. + // + ct_assert(sizeof(NV_VERSION_STRING) <= NV_RM_API_VERSION_STRING_LENGTH); + + if (dataSize != sizeof(nv_ioctl_rm_api_version_t)) + return NV_ERR_INVALID_ARGUMENT; + + pParams = pData; + + // + // write the reply value, so that the client knows we recognized + // the request + // + pParams->reply = NV_RM_API_VERSION_REPLY_RECOGNIZED; + + // + // the client is just querying the version, not verifying against expected. + // + if (pParams->cmd == NV_RM_API_VERSION_CMD_QUERY) + { + os_string_copy(pParams->versionString, rmStr); + return NV_OK; + } + + // + // the client requested relaxed version checking; we will only + // compare the strings until the first decimal point. + // + if (pParams->cmd == NV_RM_API_VERSION_CMD_RELAXED) + { + relaxed = NV_TRUE; + } + + for (i = 0; i < NV_RM_API_VERSION_STRING_LENGTH; i++) + { + clientCh = pParams->versionString[i]; + rmCh = rmStr[i]; + + // + // fail if the current character is not the same + // + if (clientCh != rmCh) + { + break; + } + + // + // if relaxed matching was requested, succeed when we find the + // first decimal point + // + if ((relaxed) && (clientCh == '.')) + { + return NV_OK; + } + + // + // we found the end of the strings: succeed + // + if (clientCh == '\0') + { + return NV_OK; + } + } + + // + // the version strings did not match: print an error message and + // copy the RM's version string into pParams->versionString, so + // that the client can report the mismatch; explicitly NULL + // terminate the client's string, since we cannot trust it + // + pParams->versionString[NV_RM_API_VERSION_STRING_LENGTH - 1] = '\0'; + + procId = os_get_current_process(); + os_get_current_process_name(procName, sizeof(procName)); + + RmSanitizeLogString(pParams->versionString, NV_RM_API_VERSION_STRING_LENGTH); + RmSanitizeLogString(procName, sizeof(procName)); + + nv_printf(NV_DBG_ERRORS, + "NVRM: API mismatch: the client '%s' (pid %u)\n" + "NVRM: has the version %s, but this kernel module has\n" + "NVRM: the version %s. Please make sure that this\n" + "NVRM: kernel module and all NVIDIA driver components\n" + "NVRM: have the same version.\n", + procName, procId, + pParams->versionString, NV_VERSION_STRING); + + os_string_copy(pParams->versionString, rmStr); + + return NV_ERR_GENERIC; +} + +NV_STATUS RmPowerSourceChangeEvent( + nv_state_t *pNv, + NvU32 event_val +) +{ + NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS params = {0}; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + + params.powerStateInfo.powerState = event_val ? NV2080_CTRL_PERF_POWER_SOURCE_BATTERY : + NV2080_CTRL_PERF_POWER_SOURCE_AC; + + return pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_PERF_SET_POWERSTATE, + ¶ms, sizeof(params)); +} + +/*! + * @brief Function to request latest D-Notifier status from SBIOS. + * + * Handle certain scenarios (like a fresh boot or suspend/resume + * of the system) when RM is not available to receive the Dx notifiers. + * This function gets the latest D-Notifier status from SBIOS + * when RM is ready to receive and handle those events. + * Use GPS_FUNC_REQUESTDXSTATE subfunction to invoke current Dx state. + * + * @param[in] pNv nv_state_t pointer. + */ +void RmRequestDNotifierState( + nv_state_t *pNv +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + NvU32 supportedFuncs = 0; + NvU16 dsmDataSize = sizeof(supportedFuncs); + NV_STATUS status = NV_OK; + + status = osCallACPI_DSM(pGpu, ACPI_DSM_FUNCTION_GPS_2X, + GPS_FUNC_REQUESTDXSTATE, &supportedFuncs, + &dsmDataSize); + if (status != NV_OK) + { + // + // Call for 'GPS_FUNC_REQUESTDXSTATE' subfunction may fail if the + // SBIOS/EC does not have the corresponding implementation. + // + NV_PRINTF(LEVEL_INFO, + "%s: Failed to request Dx event update, status 0x%x\n", + __FUNCTION__, status); + } +} + +/*! + * @brief Deal with D-notifier events to apply a performance + * level based on the requested auxiliary power-state. + * Read confluence page "D-Notifiers on Linux" for more details. + * + * @param[in] pNv nv_state_t pointer. + * @param[in] event_type NvU32 Event type. + */ +static void RmHandleDNotifierEvent( + nv_state_t *pNv, + NvU32 event_type +) +{ + NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + + switch (event_type) + { + case ACPI_NOTIFY_POWER_LEVEL_D1: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P0; + break; + case ACPI_NOTIFY_POWER_LEVEL_D2: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P1; + break; + case ACPI_NOTIFY_POWER_LEVEL_D3: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P2; + break; + case ACPI_NOTIFY_POWER_LEVEL_D4: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P3; + break; + case ACPI_NOTIFY_POWER_LEVEL_D5: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P4; + break; + default: + return; + } + + pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_ACPI); + if (pRmApi == NULL) + { + return; + } + + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_PERF_SET_AUX_POWER_STATE, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(pNv, &threadState); + + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: Failed to handle ACPI D-Notifier event, status=0x%x\n", + __FUNCTION__, rmStatus); + } +} + +static NvU32 RmDmabufMmapGetCpuCacheType( + OBJGPU *pGpu, + MEMORY_DESCRIPTOR *pMemDesc, + NvU8 mappingType +) +{ + if (pGpu->pGpuArch->bGpuArchIsZeroFb) + { + return memdescGetCpuCacheAttrib(pMemDesc); + } + +#if defined(NVCPU_X86_64) + return NV_MEMORY_WRITECOMBINED; +#elif defined(NVCPU_AARCH64) + if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) && + (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT)) + { + return NV_MEMORY_CACHED; + } + else + { + return NV_MEMORY_UNCACHED; + } +#else + // Default to uncached for other platforms for now + return NV_MEMORY_UNCACHED; +#endif +} + +static NV_STATUS +RmDmabufVerifyMemHandle( + OBJGPU *pGpu, + NvHandle hSrcClient, + NvHandle hMemory, + NvU64 offset, + NvU64 size, + void *pGpuInstanceInfo, + MEMORY_DESCRIPTOR **ppMemDesc +) +{ + NV_STATUS status; + RsClient *pClient = NULL; + RsResourceRef *pSrcMemoryRef = NULL; + Memory *pSrcMemory = NULL; + MEMORY_DESCRIPTOR *pMemDesc = NULL; + + NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hSrcClient, &pClient)); + + status = clientGetResourceRef(pClient, hMemory, &pSrcMemoryRef); + if (status != NV_OK) + { + return status; + } + + pSrcMemory = dynamicCast(pSrcMemoryRef->pResource, Memory); + if (pSrcMemory == NULL) + { + return NV_ERR_INVALID_OBJECT; + } + + pMemDesc = pSrcMemory->pMemDesc; + if (pMemDesc == NULL) + { + return NV_ERR_INVALID_STATE; + } + + // + // We skip the partitionable heap check when the source memory is in + // sysmem as there is no valid heap (pHeap will be NULL). + // + if (pGpuInstanceInfo != NULL && memdescGetAddressSpace(pMemDesc) != ADDR_SYSMEM) + { + KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance; + pKernelMIGGpuInstance = (KERNEL_MIG_GPU_INSTANCE *) pGpuInstanceInfo; + + if ((pKernelMIGGpuInstance->pMemoryPartitionHeap != pSrcMemory->pHeap)) + { + return NV_ERR_INVALID_OBJECT_PARENT; + } + } + + // Check if hMemory belongs to the same pGpu + if ((pMemDesc->pGpu != pGpu) || + (pSrcMemory->pGpu != pGpu)) + { + return NV_ERR_INVALID_OBJECT_PARENT; + } + + // Offset and size must be aligned to OS page-size + if (!NV_IS_ALIGNED64(offset, os_page_size) || + !NV_IS_ALIGNED64(size, os_page_size)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // Only supported for vidmem and sysmem(only for 0FB) handles + if ((memdescGetAddressSpace(pMemDesc) != ADDR_FBMEM) && + (!pGpu->pGpuArch->bGpuArchIsZeroFb)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if ((size == 0) || + (size > memdescGetSize(pMemDesc)) || + (offset > (memdescGetSize(pMemDesc) - size))) + { + return NV_ERR_INVALID_ARGUMENT; + } + + *ppMemDesc = pMemDesc; + + return NV_OK; +} + +static NV_STATUS +RmDmabufGetClientAndDevice( + OBJGPU *pGpu, + NvHandle hClient, + NvHandle hMemory, + NvU8 mappingType, + NvHandle *phClient, + NvHandle *phDevice, + NvHandle *phSubdevice, + void **ppGpuInstanceInfo +) +{ + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + + // No dma-buf support for SLI-enabled GPU + if (IsSLIEnabled(pGpu)) + { + return NV_ERR_NOT_SUPPORTED; + } + + // + // MAPPING_TYPE_FORCE_PCIE is to be used only on coherent systems with a + // direct PCIe connection between the exporter and importer. + // MIG is not a supported use-case for dma-buf on these systems. + // + if (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE) + { + KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + + if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) || + pKernelBus->bBar1Disabled || + IS_MIG_ENABLED(pGpu)) + { + return NV_ERR_NOT_SUPPORTED; + } + } + + if (IS_MIG_ENABLED(pGpu)) + { + MIG_INSTANCE_REF ref; + RsClient *pClient; + Memory *pMemory; + KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); + + NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, + serverGetClientUnderLock(&g_resServ, hClient, &pClient)); + + NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, + memGetByHandle(pClient, hMemory, &pMemory)); + + NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, + kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, + pMemory->pDevice, &ref)); + + NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, + kmigmgrIncRefCount(ref.pKernelMIGGpuInstance->pShare)); + + *phClient = ref.pKernelMIGGpuInstance->instanceHandles.hClient; + *phDevice = ref.pKernelMIGGpuInstance->instanceHandles.hDevice; + *phSubdevice = ref.pKernelMIGGpuInstance->instanceHandles.hSubdevice; + *ppGpuInstanceInfo = (void *) ref.pKernelMIGGpuInstance; + + return NV_OK; + } + + *phClient = pMemoryManager->hClient; + *phDevice = pMemoryManager->hDevice; + *phSubdevice = pMemoryManager->hSubdevice; + *ppGpuInstanceInfo = NULL; + + return NV_OK; +} + +static void +RmDmabufPutClientAndDevice( + OBJGPU *pGpu, + NvHandle hClient, + NvHandle hDevice, + NvHandle hSubdevice, + void *pGpuInstanceInfo +) +{ + KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance; + + if (pGpuInstanceInfo == NULL) + { + return; + } + + pKernelMIGGpuInstance = (KERNEL_MIG_GPU_INSTANCE *) pGpuInstanceInfo; + + NV_ASSERT_OK(kmigmgrDecRefCount(pKernelMIGGpuInstance->pShare)); +} + +static NV_STATUS RmP2PDmaMapPages( + nv_state_t *pNv, + nv_dma_device_t *peer, + NvU64 pageSize, + NvU32 pageCount, + NvU64 *pDmaAddresses, + void **ppPriv +) +{ + NV_STATUS status; + NvU64 *pOsDmaAddresses = NULL; + NvU32 osPagesPerP2PPage, osPageCount, count; + NvBool bDmaMapped = NV_FALSE; + NvU32 i, j, index; + void *pgmap = NULL; + NvBool bPageMapRef = NV_FALSE; + + NV_ASSERT_OR_RETURN((pageSize >= os_page_size), NV_ERR_INVALID_ARGUMENT); + + if (pNv->coherent_gpu_mem_mode == NV_COHERENT_GPU_MEM_MODE_DRIVER) + { + // Take a refcount on the dev_pagemap created for GPU memory by UVM + pgmap = nv_dma_get_dev_pagemap(pDmaAddresses[0]); + if (pgmap == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "Failed to get a reference on dev_pagemap in CDMM(driver) mode\n"); + return NV_ERR_NOT_SUPPORTED; + } + + bPageMapRef = NV_TRUE; + } + + if (pageSize == os_page_size) + { + status = nv_dma_map_alloc(peer, pageCount, pDmaAddresses, NV_FALSE, NV_FALSE, ppPriv); + + goto put_pgmap; + } + + // + // If P2P page_size returned from nvidia_p2p_get_pages is bigger than + // OS page size, the page array need to be inflated to correctly do the DMA mapping. + // For example, if page_size = 64K and OS page size = 4K, + // there are 16 OS pages to this 64K P2P page. + // + + osPagesPerP2PPage = pageSize / os_page_size; + osPageCount = osPagesPerP2PPage * pageCount; + + status = os_alloc_mem((void **)&pOsDmaAddresses, + (osPageCount * sizeof(NvU64))); + if (status != NV_OK) + { + goto failed; + } + + index = 0; + for (i = 0; i < pageCount; i++) + { + pOsDmaAddresses[index] = pDmaAddresses[i]; + index++; + + for (j = 1; j < osPagesPerP2PPage; j++) + { + pOsDmaAddresses[index] = pOsDmaAddresses[index - 1] + os_page_size; + index++; + } + } + + status = nv_dma_map_alloc(peer, osPageCount, pOsDmaAddresses, NV_FALSE, NV_FALSE, ppPriv); + if (status != NV_OK) + { + goto failed; + } + bDmaMapped = NV_TRUE; + + // + // The driver breaks down the size into submaps and dma-maps those individually. + // This may result in non-contiguous IOVA segments allocated to each submap. + // We check for OS page contiguity within the bigger pageSize since we need to + // return IOVA chunks at pageSize granularity. + // + count = 0; + for (i = 1; i < osPageCount; i++) + { + count++; + if (count == osPagesPerP2PPage) + { + count = 0; + continue; + } + + if (pOsDmaAddresses[i] != (pOsDmaAddresses[i - 1] + os_page_size)) + { + status = NV_ERR_INVALID_STATE; + goto failed; + } + } + + // + // Fill pDmaAddresses at pageSize granularity + // from expanded pOsDmaAddresses. + // + index = 0; + for (i = 0; i < pageCount; i++) + { + pDmaAddresses[i] = pOsDmaAddresses[index]; + index += osPagesPerP2PPage; + } + + os_free_mem(pOsDmaAddresses); + + if (bPageMapRef) + { + nv_dma_put_dev_pagemap(pgmap); + } + + return NV_OK; + +failed: + if (bDmaMapped) + { + NV_ASSERT_OK(nv_dma_unmap_alloc(peer, osPageCount, + pOsDmaAddresses, ppPriv)); + } + + if (pOsDmaAddresses != NULL) + { + os_free_mem(pOsDmaAddresses); + } + +put_pgmap: + if (bPageMapRef) + { + nv_dma_put_dev_pagemap(pgmap); + } + + return status; +} + +/* + * --------------------------------------------------------------------------- + * + * The routines below are part of the interface between the kernel interface + * layer and the kernel-agnostic portions of the resource manager. + * + * --------------------------------------------------------------------------- + */ + +NvBool NV_API_CALL rm_init_private_state( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NvBool retval; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + retval = RmInitPrivateState(pNv); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return retval; +} + +void NV_API_CALL rm_free_private_state( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + RmFreePrivateState(pNv); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NvBool NV_API_CALL rm_init_adapter( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + NvBool retval = NV_FALSE; + void *fp; + NvBool bEnabled; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_DEVICE_INIT); + + // + // If GPU is driving any frame buffer console(vesafb, efifb etc) + // mark the console as client driven + // + pNv->client_managed_console = rm_get_uefi_console_status(pNv); + + // + // For GPU driving console, disable console access here, to ensure no console + // writes through BAR1 can interfere with physical RM's setup of BAR1 + // + if (pNv->client_managed_console && !pNv->sysmem_mapped_console) + { + os_disable_console_access(); + } + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK) + { + if (!((gpumgrQueryGpuDrainState(pNv->gpu_id, &bEnabled, NULL) == NV_OK) + && bEnabled)) + { + if (pNv->flags & NV_FLAG_PERSISTENT_SW_STATE) + { + retval = RmPartiallyInitAdapter(pNv); + } + else + { + retval = RmInitAdapter(pNv); + } + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + if (pNv->client_managed_console && !pNv->sysmem_mapped_console) + { + os_enable_console_access(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return retval; +} + +void NV_API_CALL rm_disable_adapter( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_ASSERT_OK(os_flush_work_queue(pNv->queue, NV_TRUE)); + + if (pNv->client_managed_console && !pNv->sysmem_mapped_console) + { + os_disable_console_access(); + } + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + if (pNv->flags & NV_FLAG_PERSISTENT_SW_STATE) + { + RmPartiallyDisableAdapter(pNv); + } + else + { + RmDisableAdapter(pNv); + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + if (pNv->client_managed_console && !pNv->sysmem_mapped_console) + { + os_enable_console_access(); + } + + NV_ASSERT_OK(os_flush_work_queue(pNv->queue, NV_TRUE)); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_shutdown_adapter( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (pNv->client_managed_console && !pNv->sysmem_mapped_console) + { + os_disable_console_access(); + } + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + RmShutdownAdapter(pNv); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + if (pNv->client_managed_console && !pNv->sysmem_mapped_console) + { + os_enable_console_access(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_exclude_adapter( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NV_STATUS rmStatus; + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + rmStatus = RmExcludeAdapter(pNv); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_acquire_api_lock( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_release_api_lock( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // UNLOCK: release API lock + rmapiLockRelease(); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_acquire_gpu_lock( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire GPU lock + rmStatus = rmDeviceGpuLocksAcquire(NV_GET_NV_PRIV_PGPU(nv), + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_OSAPI); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_release_gpu_lock( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // UNLOCK: release GPU lock + rmDeviceGpuLocksRelease(NV_GET_NV_PRIV_PGPU(nv), GPUS_LOCK_FLAGS_NONE, NULL); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_acquire_all_gpus_lock( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire all GPUs lock + rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_release_all_gpus_lock( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // UNLOCK: release all GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +/*! + * @brief Handle ACPI_NOTIFY_GPS_STATUS_CHANGE event. + * + * This function is called for GPS when SBIOS trigger + * gps STATUS_CHANGE event, which calls rm control call + * NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL to init the GPS + * data from SBIOS. + */ +static void RmHandleGPSStatusChange +( + nv_state_t *pNv +) +{ + NV_STATUS rmStatus; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS gpsControl = { 0 }; + + pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_ACPI); + if (pRmApi == NULL) + { + return; + } + + gpsControl.command = NV0000_CTRL_CMD_SYSTEM_GPS_DATA_INIT_USING_SBIOS_AND_ACK; + gpsControl.locale = NV0000_CTRL_CMD_SYSTEM_GPS_LOCALE_SYSTEM; + + // + // NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL is not a subdevice command. + // But, Inside GPS module, the first GPU, which has GPS + // enabled will be grabbed with GPU manager. The RMAPI_API_LOCK_INTERNAL + // will internally grab the GPU locks and wake-up all the GPUs. Ideally + // this control call should be GPU specific command, if internally + // it uses the GPU. + // + rmStatus = pRmApi->Control(pRmApi, + pNv->rmapi.hClient, + pNv->rmapi.hClient, + NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL, + (void *)&gpsControl, + sizeof(NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS)); + + RmUnixRmApiEpilogue(pNv, &threadState); + + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: Failed to handle ACPI GPS status change event, status=0x%x\n", + __FUNCTION__, rmStatus); + } +} + +/*! + * @brief Function to handle device specific ACPI events. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + * @param[in] event_type NvU32 Event type. + */ +void NV_API_CALL rm_acpi_notify( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 event_type +) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + + switch (event_type) + { + case ACPI_VIDEO_NOTIFY_PROBE: + { + THREAD_STATE_NODE threadState; + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + RmHandleDisplayChange(sp, nv); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + break; + } + + case ACPI_NOTIFY_GPS_STATUS_CHANGE: + RmHandleGPSStatusChange(nv); + break; + + case ACPI_NOTIFY_POWER_LEVEL_D1: /* fallthrough */ + case ACPI_NOTIFY_POWER_LEVEL_D2: /* fallthrough */ + case ACPI_NOTIFY_POWER_LEVEL_D3: /* fallthrough */ + case ACPI_NOTIFY_POWER_LEVEL_D4: /* fallthrough */ + case ACPI_NOTIFY_POWER_LEVEL_D5: + RmHandleDNotifierEvent(nv, event_type); + break; + + default: + NV_PRINTF(LEVEL_INFO, "No support for 0x%x event\n", event_type); + NV_ASSERT(0); + break; + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +static void nv_align_mmap_offset_length( + nv_usermap_access_params_t *nvuap) +{ + NvU64 page_size = os_page_size; + NvU64 end = nvuap->size + (nvuap->addr & (page_size - 1)); + + nvuap->memArea.numRanges = 1; + nvuap->memArea.pRanges[0].start = NV_ALIGN_DOWN(nvuap->addr, page_size); + nvuap->memArea.pRanges[0].size = NV_ALIGN_UP(end, page_size); + nvuap->offset = NV_ALIGN_DOWN(nvuap->offset, page_size); +} + +static inline NV_STATUS RmGetArrayMinMax( + NvU64 *array, + NvU64 count, + NvU64 *min, + NvU64 *max +) +{ + NvU64 i; + + if (array == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + *min = array[0]; + *max = array[0]; + + if (count == 1) + return NV_OK; + + for (i = 1; i < count; i++) + { + if (array[i] > *max) + *max = array[i]; + + if (array[i] < *min) + *min = array[i]; + } + + return NV_OK; +} + +static NV_STATUS RmGetAllocPrivate(RmClient *, NvU32, NvU64, NvU64, NvU32 *, void **, + NvU64 *); + +static NV_STATUS RmValidateMmapRequest( + nv_state_t *pNv, + NvU64 offset, + NvU64 length, + NvU32 *pProtection +) +{ + NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS params = { 0 }; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV_STATUS status; + + if (osIsAdministrator()) + { + *pProtection = NV_PROTECT_READ_WRITE; + return NV_OK; + } + + params.addressStart = offset; + params.addressLength = length; + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_GPU_VALIDATE_MEM_MAP_REQUEST, + ¶ms, sizeof(params)); + + if (status == NV_OK) + { + *pProtection = params.protection; + } + + return status; +} + +static NV_STATUS RmSetUserMapAccessRange( + nv_usermap_access_params_t *nvuap +) +{ + NV_STATUS status = NV_OK; + + if (nvuap->contig) + { + nvuap->access_start = nvuap->memArea.pRanges[0].start; + nvuap->access_size = nvuap->memArea.pRanges[0].size; + } + else + { + NvU64 highest_address_mapped; + NvU64 lowest_address_mapped; + + status = RmGetArrayMinMax(nvuap->page_array, nvuap->num_pages, + &lowest_address_mapped, + &highest_address_mapped); + if (status != NV_OK) + { + return status; + } + + nvuap->access_start = lowest_address_mapped; + nvuap->access_size = (highest_address_mapped + os_page_size) - lowest_address_mapped; + } + + return status; +} + +static NV_STATUS RmGetMmapPteArray( + KernelMemorySystem *pKernelMemorySystem, + RsClient *pClient, + NvHandle hMemory, + nv_usermap_access_params_t *nvuap +) +{ + NV_STATUS status = NV_OK; + NvU64 pages = 0; + Memory *pMemory = NULL; + PMEMORY_DESCRIPTOR pMemDesc = NULL; + RsResourceRef *pResourceRef; + NvU64 i; + NvU64 *pteArray; + NvU64 index; + + if (nvuap == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // + // If we're mapping a memory handle, we can get the data from the + // descriptor (required if the allocation being mapped is discontiguous). + // + if (clientGetResourceRef(pClient, hMemory, &pResourceRef) == NV_OK) + { + pMemory = dynamicCast(pResourceRef->pResource, Memory); + if (pMemory != NULL) + { + pMemDesc = pMemory->pMemDesc; + nvuap->contig = memdescGetContiguity(pMemDesc, AT_CPU); + } + } + + // + // In the discontiguous case, the page array needs to be large enough to hold + // the 4K-page-sized addresses that we will then deflate to OS page addresses. + // For the contiguous case, we can create the OS-page-sized addresses + // directly in the array. + // + if (nvuap->contig) + { + pages = nvuap->memArea.pRanges[0].size / os_page_size; + } + else + { + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + pages = nvuap->memArea.pRanges[0].size / NV_MIN(os_page_size, pMemDesc->pageArrayGranularity); + } + else + { + pages = nvuap->memArea.pRanges[0].size / NV_RM_PAGE_SIZE; + } + } + + NV_ASSERT_OR_RETURN(pages != 0, NV_ERR_INVALID_ARGUMENT); + + status = os_alloc_mem((void **)&nvuap->page_array, (pages * sizeof(NvU64))); + if (status != NV_OK) + { + return status; + } + + if (!nvuap->contig) + { + pteArray = memdescGetPteArray(pMemDesc, AT_CPU); + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + index = nvuap->offset / NV_MIN(os_page_size, pMemDesc->pageArrayGranularity); + } + else + { + index = nvuap->offset / NV_RM_PAGE_SIZE; + } + + // + // We're guaranteed to have a MEMORY_DESCRIPTOR in the discontiguous + // case. Copy over the addresses now. + // + NvU64 copyByteSize = NV_MIN(pages, pMemDesc->pageArraySize) * sizeof(NvU64); + portMemCopy((void *)nvuap->page_array, + copyByteSize, (void *)&pteArray[index], + copyByteSize); + + if (!IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + if (NV_RM_PAGE_SIZE < os_page_size) + { + RmDeflateRmToOsPageArray(nvuap->page_array, pages); + pages = NV_RM_PAGES_TO_OS_PAGES(pages); + } + } + else + { + // + // Because of PMA case, we need to convert page array + // from RM native granularity to OS granularity. + // + convertPageArrayBetweenOsAndRm( + nvuap->page_array, + nvuap->memArea.pRanges[0].size / pMemDesc->pageArrayGranularity, + pMemDesc->pageArrayGranularity, + nvuap->memArea.pRanges[0].size / os_page_size, + os_page_size, + &pages + ); + } + + // + // Convert the GPU physical addresses to system physical addresses, + // if applicable. + // + for (i = 0; i < pages; i++) + { + nvuap->page_array[i] += pKernelMemorySystem->coherentCpuFbBase; + } + } + else + { + // Offset is accounted in nvuap->memArea.pRanges[0].start. + for (nvuap->page_array[0] = nvuap->memArea.pRanges[0].start, i = 1; + i < pages; i++) + { + nvuap->page_array[i] = nvuap->page_array[i-1] + os_page_size; + } + } + + nvuap->num_pages = pages; + + return status; +} + +/* Must be called with the API lock and the GPU locks */ +static NV_STATUS RmCreateMmapContextLocked( + RmClient *pRmClient, + NvHandle hDevice, + NvHandle hMemory, + NvP64 address, + NvU64 size, + NvU64 offset, + NvU32 cachingType, + NvU32 fd +) +{ + NV_STATUS status = NV_OK; + void *pAllocPriv = NULL; + OBJGPU *pGpu = NULL; + RsClient *pClient = staticCast(pRmClient, RsClient); + KernelMemorySystem *pKernelMemorySystem = NULL; + NvBool bCoherentAtsCpuOffset = NV_FALSE; + NvBool bHostCoherentFbOffset = NV_FALSE; + nv_state_t *pNv = NULL; + NvU64 addr = (NvU64)address; + NvU32 prot = 0; + NvU64 pageIndex = 0; + nv_usermap_access_params_t *nvuap = NULL; + NvBool bClientMap = (pClient->hClient == hDevice); + + if (!bClientMap) + { + if (CliSetGpuContext(pClient->hClient, hDevice, &pGpu, NULL) != NV_OK) + { + Subdevice *pSubdevice; + + status = subdeviceGetByHandle(pClient, hDevice, &pSubdevice); + if (status != NV_OK) + return status; + + pGpu = GPU_RES_GET_GPU(pSubdevice); + + GPU_RES_SET_THREAD_BC_STATE(pSubdevice); + } + } + nvuap = tlsEntryGet(TLS_ENTRY_ID_MAPPING_CONTEXT); + if (nvuap != NULL) + { + NV_ASSERT(tlsEntryRelease(TLS_ENTRY_ID_MAPPING_CONTEXT) == 0); + prot = NV_PROTECT_READ_WRITE; + // + // TODO: Add this to the mapping context so we don't need to do gymnastics + // to figure out what the correct pNv is + // + pNv = NV_GET_NV_STATE(pGpu); + goto add_ctx_to_file; + } + + NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void**)&nvuap, sizeof(nv_usermap_access_params_t))); + portMemSet(nvuap, 0, sizeof(nv_usermap_access_params_t)); + + NV_ASSERT_OK_OR_GOTO(status, os_alloc_mem((void**)&(nvuap->memArea.pRanges), sizeof(MemoryRange)), free_nvuap); + + nvuap->addr = addr; + nvuap->size = size; + nvuap->offset = offset; + nvuap->caching = cachingType; + + // + // Assume the allocation is contiguous until RmGetMmapPteArray + // determines otherwise. + // + nvuap->contig = NV_TRUE; + nv_align_mmap_offset_length(nvuap); + + if (pGpu != NULL) + { + pNv = NV_GET_NV_STATE(pGpu); + pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); + bCoherentAtsCpuOffset = IS_COHERENT_CPU_ATS_OFFSET(pGpu, pKernelMemorySystem, addr, size); + bHostCoherentFbOffset = !bCoherentAtsCpuOffset && + IS_COHERENT_FB_OFFSET(pGpu, pKernelMemorySystem, addr, size); + } + // + // If no device is given, or the address isn't in the given device's BARs, + // validate this as a system memory mapping and associate it with the + // control device. + // + if ((pNv == NULL) || + (!IS_REG_OFFSET(pNv, addr, size) && + !IS_FB_OFFSET(pNv, addr, size) && + !(bCoherentAtsCpuOffset || bHostCoherentFbOffset) && + !IS_IMEM_OFFSET(pNv, addr, size))) + { + pNv = nv_get_ctl_state(); + + // + // Validate the mapping request by looking up the underlying sysmem + // allocation. + // + status = RmGetAllocPrivate(pRmClient, hMemory, addr, size, &prot, &pAllocPriv, + &pageIndex); + + if (status != NV_OK) + { + goto done; + } + } + else + { + // + // Validate the mapping request for ATS and get pteArray + // + if (bCoherentAtsCpuOffset) + { + status = RmGetMmapPteArray(pKernelMemorySystem, pClient, hMemory, nvuap); + if (status != NV_OK) + { + goto done; + } + } + else if (bHostCoherentFbOffset) + { + status = RmGetMmapPteArray(pKernelMemorySystem, pClient, hMemory, nvuap); + if (status != NV_OK) + { + goto done; + } + + // + // nvuap->page_array(allocated in RmGetMmapPteArray) is not assigned + // to nvamc->page_array if onlining status is false(which is the case with + // bHostCoherentFbOffset) and so doesn't get freed if not done here. + // The call to RmGetMmapPteArray is for getting the contig and num + // pages of the allocation. + // + os_free_mem(nvuap->page_array); + nvuap->page_array = NULL; + } + + NV_ASSERT_OK_OR_GOTO(status, RmSetUserMapAccessRange(nvuap), done); + + // Validate the mapping request for BAR's. + status = RmValidateMmapRequest(pNv, nvuap->access_start, + nvuap->access_size, &prot); + if (status != NV_OK) + { + goto done; + } + } + +add_ctx_to_file: + status = nv_add_mapping_context_to_file(pNv, nvuap, prot, pAllocPriv, + pageIndex, fd); + +done: + os_free_mem(nvuap->memArea.pRanges); +free_nvuap: + os_free_mem(nvuap); + return status; +} + +// TODO: Bug 1802250: [uvm8] Use an alt stack in all functions in unix/src/osapi.c +NV_STATUS rm_create_mmap_context( + NvHandle hClient, + NvHandle hDevice, + NvHandle hMemory, + NvP64 address, + NvU64 size, + NvU64 offset, + NvU32 cachingType, + NvU32 fd +) +{ + NV_STATUS rmStatus = NV_OK; + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) == NV_OK) + { + CLIENT_ENTRY *pClientEntry; + RmClient *pRmClient; + + if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClientEntry, + &pRmClient)) + { + // UNLOCK: release API lock + rmapiLockRelease(); + return NV_ERR_INVALID_CLIENT; + } + + if (pRmClient->ProcID != osGetCurrentProcess()) + { + rmStatus = NV_ERR_INVALID_CLIENT; + } + // LOCK: acquire GPUs lock + else if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) == NV_OK) + { + rmStatus = RmCreateMmapContextLocked(pRmClient, hDevice, + hMemory, address, size, offset, + cachingType, fd); + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + + serverutilReleaseClient(LOCK_ACCESS_READ, pClientEntry); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return rmStatus; +} + +static NV_STATUS RmGetAllocPrivate( + RmClient *pRmClient, + NvU32 hMemory, + NvU64 offset, + NvU64 length, + NvU32 *pProtection, + void **ppPrivate, + NvU64 *pPageIndex +) +{ + NV_STATUS rmStatus; + PMEMORY_DESCRIPTOR pMemDesc; + NvU32 pageOffset; + NvU64 pageCount; + NvU64 endingOffset; + RsResourceRef *pResourceRef; + RmResource *pRmResource; + void *pMemData; + NvBool bPeerIoMem; + NvBool bReadOnlyMem; + *pProtection = NV_PROTECT_READ_WRITE; + *ppPrivate = NULL; + + pageOffset = (offset & ~os_page_mask); + offset &= os_page_mask; + + NV_ASSERT_OR_RETURN(rmapiLockIsOwner(), NV_ERR_INVALID_LOCK_STATE); + + rmStatus = clientGetResourceRef(staticCast(pRmClient, RsClient), + hMemory, &pResourceRef); + if (rmStatus != NV_OK) + goto done; + + pRmResource = dynamicCast(pResourceRef->pResource, RmResource); + if (!pRmResource) + { + rmStatus = NV_ERR_INVALID_OBJECT; + goto done; + } + + rmStatus = rmresGetMemoryMappingDescriptor(pRmResource, &pMemDesc); + if (rmStatus != NV_OK) + goto done; + + bReadOnlyMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY); + bPeerIoMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM); + + if (!(pMemDesc->Allocated || bPeerIoMem || + memdescIsCarveoutMemory(pMemDesc) || + (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOW_EXT_SYSMEM_USER_CPU_MAPPING)))) + { + NV_PRINTF(LEVEL_ERROR, "Mmap is not allowed\n"); + rmStatus = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + switch (memdescGetAddressSpace(pMemDesc)) + { + case ADDR_SYSMEM: + break; + default: + rmStatus = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + pMemData = memdescGetMemData(pMemDesc); + if (pMemData == NULL) + { + rmStatus = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + rmStatus = os_match_mmap_offset(pMemData, offset, pPageIndex); + if (rmStatus != NV_OK) + goto done; + + if (!portSafeAddU64(pageOffset, length, &endingOffset)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pageCount = (endingOffset / os_page_size); + + if (!portSafeAddU64(*pPageIndex + ((endingOffset % os_page_size) ? 1 : 0), + pageCount, &pageCount)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NV_ASSERT_OR_ELSE(pageCount <= pMemDesc->PageCount, rmStatus = NV_ERR_INVALID_ARGUMENT; goto done); + } + else if (pageCount > NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + if (bReadOnlyMem) + *pProtection = NV_PROTECT_READABLE; + + *ppPrivate = pMemData; + +done: + return rmStatus; +} + +NV_STATUS rm_get_adapter_status( + nv_state_t *pNv, + NvU32 *pStatus +) +{ + NV_STATUS rmStatus = NV_ERR_OPERATING_SYSTEM; + + // LOCK: acquire API lock + if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI) == NV_OK) + { + rmStatus = RmGetAdapterStatus(pNv, pStatus); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_get_adapter_status_external( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (rm_get_adapter_status(pNv, &rmStatus) != NV_OK) + { + rmStatus = NV_ERR_OPERATING_SYSTEM; + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NvBool NV_API_CALL rm_init_rm( + nvidia_stack_t *sp +) +{ + NvBool retval; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + retval = RmInitRm(); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return retval; +} + +void NV_API_CALL rm_shutdown_rm( + nvidia_stack_t *sp +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + RmShutdownRm(); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NvBool NV_API_CALL rm_init_event_locks( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + NvBool ret; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pNv->event_spinlock = portSyncSpinlockCreate(portMemAllocatorGetGlobalNonPaged()); + ret = (pNv->event_spinlock != NULL); + + NV_EXIT_RM_RUNTIME(sp,fp); + return ret; +} + +void NV_API_CALL rm_destroy_event_locks( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (pNv && pNv->event_spinlock) + portSyncSpinlockDestroy(pNv->event_spinlock); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_stop_user_channels( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_ERR_INVALID_STATE; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_FIFO); + if (pRmApi != NULL) + { + params.bDisable = NV_TRUE; + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(pNv, &threadState); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_restart_user_channels( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_ERR_INVALID_STATE; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_FIFO); + if (pRmApi != NULL) + { + params.bDisable = NV_FALSE; + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(pNv, &threadState); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +// +// Use this call to check if the chipset is io coherent +// +NvBool NV_API_CALL rm_is_chipset_io_coherent( + nvidia_stack_t *sp) +{ + void *fp; + OBJSYS *pSys; + OBJCL *pCl; + NvBool bIoCoherent = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pSys = SYS_GET_INSTANCE(); + pCl = SYS_GET_CL(pSys); + if (pCl == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "%s: no CL object found, setting io coherent by default\n", + __FUNCTION__); + goto done; + } + + bIoCoherent = pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT); + +done: + NV_EXIT_RM_RUNTIME(sp,fp); + + return bIoCoherent; +} + +NV_STATUS NV_API_CALL rm_validate_ioctls( + NvU32 command, + NvU32 dataSize +) +{ + size_t arg_size = dataSize; + int arg_cmd = (command & 0xFF); + int i; + NV_STATUS status; + static const struct { + unsigned int cmdKey; + size_t paramSize; + } rm_ioctls_table[] = { + #define _RM_IOCTL_ENTRY(_cmd, _type) \ + { .cmdKey = ((_cmd) & 0xFF), .paramSize = sizeof(_type),} + /*rm_ioctls */ + _RM_IOCTL_ENTRY(NV_ESC_ALLOC_OS_EVENT, nv_ioctl_alloc_os_event_t), + _RM_IOCTL_ENTRY(NV_ESC_FREE_OS_EVENT, nv_ioctl_free_os_event_t), + _RM_IOCTL_ENTRY(NV_ESC_RM_GET_EVENT_DATA, NVOS41_PARAMETERS), + }; + + status = RmValidateIoctl(command, dataSize); + if (status != NV_ERR_INVALID_COMMAND) + { + return status; + } + + for (i = 0; i < NV_ARRAY_ELEMENTS(rm_ioctls_table); ++i) + { + if (arg_cmd == rm_ioctls_table[i].cmdKey) + { + if (arg_size == rm_ioctls_table[i].paramSize) + { + return NV_OK; + } + nv_printf(NV_DBG_ERRORS, "NVRM: invalid %d structure size, expected %d, got %d!\n", + rm_ioctls_table[i].cmdKey, rm_ioctls_table[i].paramSize, arg_size); + + return NV_ERR_INVALID_ARGUMENT; + } + } + // Unknown IOCTL case. Not found in any rm and RM Tables. + return NV_ERR_INVALID_COMMAND; +} + +NV_STATUS NV_API_CALL rm_ioctl( + nvidia_stack_t *sp, + nv_state_t *pNv, + nv_file_private_t *nvfp, + NvU32 Command, + void *pData, + NvU32 dataSize +) +{ + NV_STATUS rmStatus = NV_OK; + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + // + // Some ioctls are handled entirely inside the OS layer and don't need to + // suffer the overhead of calling into RM core. + // + switch (Command) + { + case NV_ESC_ALLOC_OS_EVENT: + { + nv_ioctl_alloc_os_event_t *pApi = pData; + + if (dataSize != sizeof(nv_ioctl_alloc_os_event_t)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + break; + } + + pApi->Status = allocate_os_event(pApi->hClient, nvfp, pApi->fd); + break; + } + case NV_ESC_FREE_OS_EVENT: + { + nv_ioctl_free_os_event_t *pApi = pData; + + if (dataSize != sizeof(nv_ioctl_free_os_event_t)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + break; + } + + pApi->Status = free_os_event(pApi->hClient, pApi->fd); + break; + } + case NV_ESC_RM_GET_EVENT_DATA: + { + NVOS41_PARAMETERS *pApi = pData; + + if (dataSize != sizeof(NVOS41_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + break; + } + + pApi->status = get_os_event_data(nvfp, + pApi->pEvent, + &pApi->MoreEvents); + break; + } + default: + { + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + rmStatus = RmIoctl(pNv, nvfp, Command, pData, dataSize); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + break; + } + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +static void _deferredClientListFreeCallback(void *unused) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + NV_STATUS status = serverFreeDisabledClients(&g_resServ, 0, pSys->clientListDeferredFreeLimit); + // + // Possible return values: + // NV_WARN_MORE_PROCESSING_REQUIRED - Iteration limit reached, need to call again + // NV_ERR_IN_USE - Already running on another thread, try again later + // In both cases, schedule a worker to clean up anything that remains + // + if (status != NV_OK) + { + status = osQueueSystemWorkItem(_deferredClientListFreeCallback, unused); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_NOTICE, "Failed to schedule deferred free callback. Freeing immediately.\n"); + serverFreeDisabledClients(&g_resServ, 0, 0); + } + } +} + +void NV_API_CALL rm_cleanup_file_private( + nvidia_stack_t *sp, + nv_state_t *pNv, + nv_file_private_t *nvfp +) +{ + THREAD_STATE_NODE threadState; + void *fp; + RM_API *pRmApi; + RM_API_CONTEXT rmApiContext = {0}; + NvU32 i; + OBJSYS *pSys = SYS_GET_INSTANCE(); + + NV_ENTER_RM_RUNTIME(sp,fp); + + // + // Skip cleaning up this fd if: + // - no RMAPI clients and events were ever allocated on this fd + // - no RMAPI object handles were exported on this fd + // Access nvfp->handles without locking as fd cleanup is synchronised by the kernel + // + if (!nvfp->bCleanupRmapi && nvfp->handles == NULL) + goto done; + + pRmApi = rmapiGetInterface(RMAPI_EXTERNAL); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + threadStateSetTimeoutOverride(&threadState, 10 * 1000); + + if (rmapiPrologue(pRmApi, &rmApiContext) != NV_OK) { + NV_EXIT_RM_RUNTIME(sp,fp); + return; + } + + // LOCK: acquire API lock. Low priority so cleanup doesn't block active threads + if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_LOW_PRIORITY, RM_LOCK_MODULES_OSAPI) == NV_OK) + { + // Unref any object which was exported on this file. + if (nvfp->handles != NULL) + { + for (i = 0; i < nvfp->maxHandles; i++) + { + if (nvfp->handles[i] == 0) + { + continue; + } + + RmFreeObjExportHandle(nvfp->handles[i]); + nvfp->handles[i] = 0; + } + + os_free_mem(nvfp->handles); + nvfp->handles = NULL; + nvfp->maxHandles = 0; + } + + // Disable any RM clients associated with this file. + RmFreeUnusedClients(pNv, nvfp); + + // Unless configured otherwise, immediately free all disabled clients + if (!pSys->bUseDeferredClientListFree) + serverFreeDisabledClients(&g_resServ, RM_LOCK_STATES_API_LOCK_ACQUIRED, 0); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + // Start the deferred free callback if necessary + if (pSys->bUseDeferredClientListFree) + _deferredClientListFreeCallback(NULL); + + rmapiEpilogue(pRmApi, &rmApiContext); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + +done: + if (nvfp->ctl_nvfp != NULL) + { + nv_put_file_private(nvfp->ctl_nvfp_priv); + nvfp->ctl_nvfp = NULL; + nvfp->ctl_nvfp_priv = NULL; + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_unbind_lock( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK) + { + RmUnbindLock(pNv); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_read_registry_dword( + nvidia_stack_t *sp, + nv_state_t *nv, + const char *regParmStr, + NvU32 *Data +) +{ + OBJGPU *pGpu = NULL; + NV_STATUS RmStatus; + void *fp; + NvBool isApiLockTaken = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + // + // We can be called from different contexts: + // + // 1) early initialization without device state. + // 2) from outside the RM API (without the lock held) + // + // In context 1)the API lock is not needed and + // in context 2), it needs to be acquired. + // + if (nv != NULL) + { + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) != NV_OK) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return RmStatus; + } + + isApiLockTaken = NV_TRUE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + // Skipping the NULL check as osReadRegistryDword takes care of it. + RmStatus = osReadRegistryDword(pGpu, regParmStr, Data); + + if (isApiLockTaken == NV_TRUE) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return RmStatus; +} + +NV_STATUS NV_API_CALL rm_write_registry_dword( + nvidia_stack_t *sp, + nv_state_t *nv, + const char *regParmStr, + NvU32 Data +) +{ + NV_STATUS RmStatus; + void *fp; + NvBool isApiLockTaken = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (nv != NULL) + { + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return RmStatus; + } + + isApiLockTaken = NV_TRUE; + } + + RmStatus = RmWriteRegistryDword(nv, regParmStr, Data); + + if (isApiLockTaken == NV_TRUE) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return RmStatus; +} + +NV_STATUS NV_API_CALL rm_write_registry_binary( + nvidia_stack_t *sp, + nv_state_t *nv, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + NV_STATUS RmStatus; + void *fp; + NvBool isApiLockTaken = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (nv != NULL) + { + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return RmStatus; + } + + isApiLockTaken = NV_TRUE; + } + + RmStatus = RmWriteRegistryBinary(nv, regParmStr, Data, cbLen); + + if (isApiLockTaken == NV_TRUE) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return RmStatus; +} + +NV_STATUS NV_API_CALL rm_write_registry_string( + nvidia_stack_t *sp, + nv_state_t *nv, + const char *regParmStr, + const char *string, + NvU32 stringLength +) +{ + NV_STATUS rmStatus; + void *fp; + NvBool isApiLockTaken = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (nv != NULL) + { + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; + } + + isApiLockTaken = NV_TRUE; + } + + rmStatus = RmWriteRegistryString(nv, regParmStr, string, (stringLength + 1)); + + if (isApiLockTaken == NV_TRUE) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +static NvBool NV_API_CALL rm_is_space(const char ch) +{ + // + // return true if it is a: + // ' ' : (space - decimal 32.) + // '\t' : (TAB - decimal 9) + // 'LF' : (Line feed, new line - decimal 10) + // 'VT' : (Vertical TAB - decimal 11) + // 'FF' : (Form feed, new page - decimal 12) + // '\r' : (carriage return - decimal 13) + // + return ((ch == ' ') || ((ch >= '\t') && (ch <= '\r'))); +} + +char* NV_API_CALL rm_remove_spaces(const char *in) +{ + unsigned int len = os_string_length(in) + 1; + const char *in_ptr; + char *out, *out_ptr; + + if (os_alloc_mem((void **)&out, len) != NV_OK) + return NULL; + + in_ptr = in; + out_ptr = out; + + while (*in_ptr != '\0') + { + if (!rm_is_space(*in_ptr)) + *out_ptr++ = *in_ptr; + in_ptr++; + } + *out_ptr = '\0'; + + return out; +} + +char* NV_API_CALL rm_string_token(char **strp, const char delim) +{ + char *s, *token; + + if ((strp == NULL) || (*strp == NULL)) + return NULL; + + s = token = *strp; + *strp = NULL; + + for (; *s != '\0'; s++) { + if (*s == delim) { + *s = '\0'; + *strp = ++s; + break; + } + } + + return token; +} + +// Parse string passed in NVRM as module parameter. +void NV_API_CALL rm_parse_option_string(nvidia_stack_t *sp, const char *nvRegistryDwords) +{ + unsigned int i; + nv_parm_t *entry; + char *option_string = NULL; + char *ptr, *token; + char *name, *value; + NvU32 data; + + if (nvRegistryDwords != NULL) + { + if ((option_string = rm_remove_spaces(nvRegistryDwords)) == NULL) + { + return; + } + + ptr = option_string; + + while ((token = rm_string_token(&ptr, ';')) != NULL) + { + if (!(name = rm_string_token(&token, '=')) || !os_string_length(name)) + { + continue; + } + + if (!(value = rm_string_token(&token, '=')) || !os_string_length(value)) + { + continue; + } + + if (rm_string_token(&token, '=') != NULL) + { + continue; + } + + data = os_strtoul(value, NULL, 0); + + for (i = 0; (entry = &nv_parms[i])->name != NULL; i++) + { + if (os_string_compare(entry->name, name) == 0) + break; + } + + if (!entry->name) + rm_write_registry_dword(sp, NULL, name, data); + else + *entry->data = data; + } + + // Free the memory allocated by rm_remove_spaces() + os_free_mem(option_string); + } +} + +NV_STATUS NV_API_CALL rm_run_rc_callback( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + OBJGPU *pGpu; + void *fp; + + /* make sure our timer's not still running when it shouldn't be */ + if (nv == NULL) + return NV_ERR_GENERIC; + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL) + return NV_ERR_GENERIC; + + if (nv->rc_timer_enabled == 0) + return NV_ERR_GENERIC; + + if (!FULL_GPU_SANITY_CHECK(pGpu)) + { + return NV_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + osRun1HzCallbacksNow(pGpu); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +static void _tmrEventServiceTimerWorkItem +( + NvU32 gpuInstance, + void *pArgs +) +{ + OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); + OBJTMR *pTmr = GPU_GET_TIMER(pGpu); + NV_STATUS status = NV_OK; + + status = tmrEventServiceTimer(pGpu, pTmr, (TMR_EVENT *)pArgs); + + if (status != NV_OK) + NV_PRINTF(LEVEL_ERROR, "Timer event failed from OS timer callback workitem with status :0x%x\n", status); +} + +static NV_STATUS RmRunNanoTimerCallback( + OBJGPU *pGpu, + void *pTmrEvent +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS status = NV_OK; + + threadStateInitISRAndDeferredIntHandler(&threadState, pGpu, + THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER); + + // + // OS timers come in ISR context, hence schedule workitem for timer event service. + // GPU timer events are also handled in same mammer allowing us to have same functionality + // for callback functions. + // + status = osQueueWorkItem(pGpu, + _tmrEventServiceTimerWorkItem, + pTmrEvent, + (OsQueueWorkItemFlags){ + .bLockGpuGroupDevice = NV_TRUE, + .bDontFreeParams = NV_TRUE}); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR,"Queuing workitem for timer event failed with status :0x%x\n", status); + } + + // Out of conflicting thread + threadStateFreeISRAndDeferredIntHandler(&threadState, + pGpu, THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER); + + return status; +} + +NV_STATUS NV_API_CALL rm_run_nano_timer_callback +( + nvidia_stack_t *sp, + nv_state_t *nv, + void *pTmrEvent +) +{ + NV_STATUS status; + OBJGPU *pGpu = NULL; + void *fp; + + if (nv == NULL) + return NV_ERR_GENERIC; + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL) + return NV_ERR_GENERIC; + + if (!FULL_GPU_SANITY_FOR_PM_RESUME(pGpu)) + { + return NV_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + status = RmRunNanoTimerCallback(pGpu, pTmrEvent); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +void NV_API_CALL rm_execute_work_item( + nvidia_stack_t *sp, + void *pNvWorkItem +) +{ + void *fp; + THREAD_STATE_NODE threadState; + + NV_ENTER_RM_RUNTIME(sp, fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + RmExecuteWorkItem(pNvWorkItem); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp, fp); +} + +const char* NV_API_CALL rm_get_device_name( + NvU16 device, + NvU16 subsystem_vendor, + NvU16 subsystem_device +) +{ + unsigned int i; + const char *tmpName = NULL; + + for (i = 0; i < NV_ARRAY_ELEMENTS(sChipsReleased); i++) + { + // if the device ID doesn't match, go to the next entry + if (device != sChipsReleased[i].devID) + { + continue; + } + + // if the entry has 0 for the subsystem IDs, then the device + // ID match is sufficient, but continue scanning through + // sChipsReleased[] in case there is a subsystem ID match later + // in the table + if (sChipsReleased[i].subSystemVendorID == 0 && + sChipsReleased[i].subSystemID == 0) + { + tmpName = sChipsReleased[i].name; + continue; + } + + if (subsystem_vendor == sChipsReleased[i].subSystemVendorID && + subsystem_device == sChipsReleased[i].subSystemID) + { + tmpName = sChipsReleased[i].name; + break; + } + } + + return (tmpName != NULL) ? tmpName : "Unknown"; +} + +NV_STATUS rm_access_registry( + NvHandle hClient, + NvHandle hObject, + NvU32 AccessType, + NvP64 clientDevNodeAddress, + NvU32 DevNodeLength, + NvP64 clientParmStrAddress, + NvU32 ParmStrLength, + NvP64 clientBinaryDataAddress, + NvU32 *pBinaryDataLength, + NvU32 *Data, + NvU32 *Entry +) +{ + NV_STATUS RmStatus; + NvBool bReadOnly = (AccessType == NVOS38_ACCESS_TYPE_READ_DWORD) || + (AccessType == NVOS38_ACCESS_TYPE_READ_BINARY); + + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(bReadOnly ? RMAPI_LOCK_FLAGS_READ : RMAPI_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_OSAPI)) == NV_OK) + { + RmStatus = RmAccessRegistry(hClient, + hObject, + AccessType, + clientDevNodeAddress, + DevNodeLength, + clientParmStrAddress, + ParmStrLength, + clientBinaryDataAddress, + pBinaryDataLength, + Data, + Entry); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return RmStatus; +} + +NV_STATUS rm_update_device_mapping_info( + NvHandle hClient, + NvHandle hDevice, + NvHandle hMemory, + void *pOldCpuAddress, + void *pNewCpuAddress +) +{ + NV_STATUS RmStatus; + + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU)) == NV_OK) + { + RmStatus = RmUpdateDeviceMappingInfo(hClient, + hDevice, + hMemory, + pOldCpuAddress, + pNewCpuAddress); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return RmStatus; +} + +static NvBool NV_API_CALL rm_is_legacy_device( + NvU16 device_id, + NvU16 subsystem_vendor, + NvU16 subsystem_device, + NvBool print_warning +) +{ + return NV_FALSE; +} + +static NvBool NV_API_CALL rm_is_legacy_arch( + NvU32 pmc_boot_0, + NvU32 pmc_boot_42 +) +{ + NvBool legacy = NV_FALSE; + + return legacy; +} + +NV_STATUS NV_API_CALL rm_is_supported_device( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + OBJSYS *pSys; + OBJHALMGR *pHalMgr; + GPUHWREG *reg_mapping; + NvU32 myHalPublicID; + void *fp; + NvU32 pmc_boot_0; + NvU32 pmc_boot_42; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pSys = SYS_GET_INSTANCE(); + pHalMgr = SYS_GET_HALMGR(pSys); + + reg_mapping = osMapKernelSpace(pNv->regs->cpu_address, + os_page_size, + NV_MEMORY_UNCACHED, + NV_PROTECT_READABLE); + + if (reg_mapping == NULL) + { + nv_printf(NV_DBG_ERRORS, "NVRM: failed to map registers!\n"); + rmStatus = NV_ERR_OPERATING_SYSTEM; + goto threadfree; + } + NvU32 pmc_boot_1 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_1); + pmc_boot_0 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_0); + pmc_boot_42 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_42); + + osUnmapKernelSpace(reg_mapping, os_page_size); + + if ((pmc_boot_0 == 0xFFFFFFFF) && (pmc_boot_42 == 0xFFFFFFFF)) + { + nv_printf(NV_DBG_ERRORS, + "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x\n" + "NVRM: (PCI ID: %04x:%04x) installed in this system has\n" + "NVRM: fallen off the bus and is not responding to commands.\n", + pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot, + pNv->pci_info.function, pNv->pci_info.vendor_id, + pNv->pci_info.device_id); + rmStatus = NV_ERR_GPU_IS_LOST; + goto threadfree; + } + + /* + * For legacy architectures, rm_is_legacy_arch() prints "legacy" message. + * We do not want to print "unsupported" message for legacy architectures + * to avoid confusion. Also, the probe should not continue for legacy + * architectures. Hence, we set rmStatus to NV_ERR_NOT_SUPPORTED and + * goto threadfree. + */ + if (rm_is_legacy_arch(pmc_boot_0, pmc_boot_42)) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + goto threadfree; + } + + rmStatus = halmgrGetHalForGpu(pHalMgr, pmc_boot_0, pmc_boot_42, &myHalPublicID); + + if (rmStatus != NV_OK) + { + NvBool bIsFirmwareCapable; + + bIsFirmwareCapable = gpumgrIsDeviceRmFirmwareCapable(pmc_boot_42, + FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1), + NV_IS_SOC_DISPLAY_DEVICE(pNv), + NULL, + NV_FALSE /* bIsTccOrMcdm */); + if (!bIsFirmwareCapable) + { + if (hypervisorIsVgxHyper()) + { + nv_printf(NV_DBG_ERRORS, + "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n" + "NVRM: installed in this vGPU host system is not supported by\n" + "NVRM: open nvidia.ko.\n" + "NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n" + "NVRM: Firmware' sections in the NVIDIA Virtual GPU (vGPU)\n" + "NVRM: Software documentation, available at docs.nvidia.com.\n", + pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot, + pNv->pci_info.function, pNv->pci_info.vendor_id, + pNv->pci_info.device_id); + } + else + { + nv_printf(NV_DBG_ERRORS, + "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n" + "NVRM: installed in this system is not supported by open\n" + "NVRM: nvidia.ko because it does not include the required GPU\n" + "NVRM: System Processor (GSP).\n" + "NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n" + "NVRM: Firmware' sections in the driver README, available on\n" + "NVRM: the Linux graphics driver download page at\n" + "NVRM: www.nvidia.com.\n", + pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot, + pNv->pci_info.function, pNv->pci_info.vendor_id, + pNv->pci_info.device_id); + } + goto threadfree; + } + goto print_unsupported; + } + + rmStatus = rm_is_vgpu_supported_device(pNv, pmc_boot_1, pmc_boot_42); + + if (rmStatus != NV_OK) + { + nv_printf(NV_DBG_ERRORS, + "NVRM: The NVIDIA vGPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n" + "NVRM: installed in this system is not supported by open\n" + "NVRM: nvidia.ko.\n" + "NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n" + "NVRM: Firmware' sections in the NVIDIA Virtual GPU (vGPU)\n" + "NVRM: Software documentation, available at docs.nvidia.com.\n", + pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot, + pNv->pci_info.function, pNv->pci_info.vendor_id, + pNv->pci_info.device_id); + goto threadfree; + } + goto threadfree; + +print_unsupported: + nv_printf(NV_DBG_ERRORS, + "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n" + "NVRM: installed in this system is not supported by the\n" + "NVRM: NVIDIA %s driver release.\n" + "NVRM: Please see 'Appendix A - Supported NVIDIA GPU Products'\n" + "NVRM: in this release's README, available on the operating system\n" + "NVRM: specific graphics driver download page at www.nvidia.com.\n", + pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot, + pNv->pci_info.function, pNv->pci_info.vendor_id, + pNv->pci_info.device_id, NV_VERSION_STRING); + +threadfree: + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NvBool NV_API_CALL rm_is_supported_pci_device( + NvU8 pci_class, + NvU8 pci_subclass, + NvU16 vendor, + NvU16 device, + NvU16 subsystem_vendor, + NvU16 subsystem_device, + NvBool print_legacy_warning +) +{ + const NvU16 nv_pci_vendor_id = 0x10DE; + const NvU16 nv_pci_id_riva_tnt = 0x0020; + const NvU8 nv_pci_class_display = 0x03; + const NvU8 nv_pci_subclass_display_vga = 0x00; + const NvU8 nv_pci_subclass_display_3d = 0x02; + + if (pci_class != nv_pci_class_display) + { + return NV_FALSE; + } + + if ((pci_subclass != nv_pci_subclass_display_vga) && + (pci_subclass != nv_pci_subclass_display_3d)) + { + return NV_FALSE; + } + + if (vendor != nv_pci_vendor_id) + { + return NV_FALSE; + } + + if (device < nv_pci_id_riva_tnt) + { + return NV_FALSE; + } + + if (rm_is_legacy_device( + device, + subsystem_vendor, + subsystem_device, + print_legacy_warning)) + { + return NV_FALSE; + } + + return NV_TRUE; +} + +/* + * Performs the I2C transfers which are related with DP AUX channel + */ +static NV_STATUS RmDpAuxI2CTransfer +( + nv_state_t *pNv, + NvU32 displayId, + NvU8 addr, + NvU32 len, + NvU8 *pData, + NvBool bWrite +) +{ + NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_CTRL_PARAMS *pParams; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV_STATUS status; + + if (len > NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE) + { + NV_PRINTF(LEVEL_ERROR, + "%s: requested I2C transfer length %u is greater than maximum supported length %u\n", + __FUNCTION__, len, NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE); + return NV_ERR_NOT_SUPPORTED; + } + + pParams = portMemAllocNonPaged(sizeof(*pParams)); + if (pParams == NULL) + { + return NV_ERR_NO_MEMORY; + } + + portMemSet(pParams, 0, sizeof(*pParams)); + + pParams->subDeviceInstance = 0; + pParams->displayId = displayId; + pParams->addr = addr; + pParams->size = len; + pParams->bWrite = bWrite; + + if (bWrite) + { + portMemCopy(pParams->data, NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE, + pData, len); + } + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp, + NV0073_CTRL_CMD_DP_AUXCH_I2C_TRANSFER_CTRL, + pParams, sizeof(*pParams)); + + if ((status == NV_OK) && !bWrite) + { + portMemCopy(pData, len, pParams->data, pParams->size); + } + + portMemFree(pParams); + + return status; +} + +/* + * Performs the I2C transfers which are not related with DP AUX channel + */ +static NV_STATUS RmNonDPAuxI2CTransfer +( + nv_state_t *pNv, + NvU8 portId, + nv_i2c_cmd_t type, + NvU8 addr, + NvU8 command, + NvU32 len, + NvU8 *pData +) +{ + NV402C_CTRL_I2C_TRANSACTION_PARAMS *params; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV_STATUS rmStatus = NV_OK; + + params = portMemAllocNonPaged(sizeof(*params)); + if (params == NULL) + { + return NV_ERR_NO_MEMORY; + } + + portMemSet(params, 0, sizeof(*params)); + + params->portId = portId; + // precondition our address (our stack requires this) + params->deviceAddress = addr << 1; + + switch (type) + { + case NV_I2C_CMD_WRITE: + params->transData.i2cBlockData.bWrite = NV_TRUE; + /* fall through*/ + + case NV_I2C_CMD_READ: + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_I2C_BLOCK_RW; + params->transData.i2cBlockData.messageLength = len; + params->transData.i2cBlockData.pMessage = pData; + break; + + case NV_I2C_CMD_SMBUS_WRITE: + if (len == 2) + { + params->transData.smbusWordData.bWrite = NV_TRUE; + } + else + { + params->transData.smbusByteData.bWrite = NV_TRUE; + } + /* fall through*/ + + case NV_I2C_CMD_SMBUS_READ: + if (len == 2) + { + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_WORD_RW; + params->transData.smbusWordData.message = pData[0] | ((NvU16)pData[1] << 8); + params->transData.smbusWordData.registerAddress = command; + } + else + { + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BYTE_RW; + params->transData.smbusByteData.message = pData[0]; + params->transData.smbusByteData.registerAddress = command; + } + break; + + case NV_I2C_CMD_SMBUS_BLOCK_WRITE: + if (pData[0] >= len) { + portMemFree(params); + return NV_ERR_INVALID_ARGUMENT; + } + params->transData.smbusBlockData.bWrite = NV_TRUE; + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BLOCK_RW; + params->transData.smbusBlockData.registerAddress = command; + params->transData.smbusBlockData.messageLength = pData[0]; + params->transData.smbusBlockData.pMessage = pData + 1; + break; + + case NV_I2C_CMD_SMBUS_BLOCK_READ: + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BLOCK_RW; + params->transData.smbusBlockData.registerAddress = command; + params->transData.smbusBlockData.messageLength = len; + params->transData.smbusBlockData.pMessage = pData; + break; + + case NV_I2C_CMD_SMBUS_QUICK_WRITE: + params->transData.smbusQuickData.bWrite = NV_TRUE; + /* fall through*/ + + case NV_I2C_CMD_SMBUS_QUICK_READ: + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_QUICK_RW; + break; + + case NV_I2C_CMD_BLOCK_WRITE: + params->transData.i2cBufferData.bWrite = NV_TRUE; + /* fall through */ + + case NV_I2C_CMD_BLOCK_READ: + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_I2C_BUFFER_RW; + params->transData.i2cBufferData.registerAddress = command; + params->transData.i2cBufferData.messageLength = len; + params->transData.i2cBufferData.pMessage = pData; + break; + + default: + portMemFree(params); + return NV_ERR_INVALID_ARGUMENT; + } + + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hI2C, + NV402C_CTRL_CMD_I2C_TRANSACTION, + params, sizeof(*params)); + + // + // For NV_I2C_CMD_SMBUS_READ, copy the read data to original + // data buffer. + // + if (rmStatus == NV_OK && type == NV_I2C_CMD_SMBUS_READ) + { + if (len == 2) + { + pData[0] = (params->transData.smbusWordData.message & 0xff); + pData[1] = params->transData.smbusWordData.message >> 8; + } + else + { + pData[0] = params->transData.smbusByteData.message; + } + } + + portMemFree(params); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_i2c_transfer( + nvidia_stack_t *sp, + nv_state_t *pNv, + void *pI2cAdapter, + nv_i2c_cmd_t type, + NvU8 addr, + NvU8 command, + NvU32 len, + NvU8 *pData +) +{ + THREAD_STATE_NODE threadState; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + NV_STATUS rmStatus = NV_OK; + OBJGPU *pGpu = NULL; + NvBool unlockApi = NV_FALSE; + NvBool unlockGpu = NV_FALSE; + NvU32 x; + void *fp; + NvU32 numDispId = 0; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (pNvp->flags & NV_INIT_FLAG_PUBLIC_I2C) + { + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK) + goto finish; + + unlockApi = NV_TRUE; + + // LOCK: acquire GPUs lock + if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK) + goto finish; + + unlockGpu = NV_TRUE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + if (!pGpu) + { + rmStatus = NV_ERR_GENERIC; + goto finish; + } + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + if (pNvp->i2c_adapters[x].pOsAdapter == pI2cAdapter) + { + break; + } + } + + if (x == MAX_I2C_ADAPTERS) + { + rmStatus = NV_ERR_GENERIC; + goto finish; + } + + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + NvU32 displayId = pNvp->i2c_adapters[x].displayId[numDispId]; + + if (displayId == INVALID_DISP_ID) + { + continue; + } + + // Handle i2c-over-DpAux adapters separately from regular i2c adapters + if (displayId == 0) + { + rmStatus = RmNonDPAuxI2CTransfer(pNv, pNvp->i2c_adapters[x].port, + type, addr, command, len, pData); + } + else + { + if ((type != NV_I2C_CMD_READ) && (type != NV_I2C_CMD_WRITE)) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + goto semafinish; + } + + rmStatus = RmDpAuxI2CTransfer(pNv, displayId, addr, len, pData, + type == NV_I2C_CMD_WRITE); + } +semafinish: + if (rmStatus == NV_OK) + { + break; + } + } + +finish: + if (unlockGpu) + { + // UNLOCK: release GPU lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + + if (unlockApi) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +static void rm_i2c_add_adapter( + nv_state_t *pNv, + NvU32 port, + NvU32 displayId +) +{ + NvU32 y, free; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + NvU32 numDispId = 0; + + for (y = 0, free = MAX_I2C_ADAPTERS; y < MAX_I2C_ADAPTERS; y++) + { + if (pNvp->i2c_adapters[y].pOsAdapter == NULL) + { + // Only find the first free entry, and ignore the rest + if (free == MAX_I2C_ADAPTERS) + { + free = y; + } + } + else if (pNvp->i2c_adapters[y].port == port) + { + break; + } + } + + if (y < MAX_I2C_ADAPTERS) + { + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + if (pNvp->i2c_adapters[y].displayId[numDispId] == INVALID_DISP_ID) + { + pNvp->i2c_adapters[y].displayId[numDispId] = displayId; + break; + } + else + { + NV_PRINTF(LEVEL_INFO, + "%s: adapter already exists (port=0x%x, displayId=0x%x)\n", + __FUNCTION__, port, + pNvp->i2c_adapters[y].displayId[numDispId]); + } + } + + if (numDispId == MAX_DISP_ID_PER_ADAPTER) + { + NV_PRINTF(LEVEL_ERROR, + "%s: no more free display Id entries in adapter\n", + __FUNCTION__); + } + + return; + } + + if (free == MAX_I2C_ADAPTERS) + { + NV_PRINTF(LEVEL_ERROR, "%s: no more free adapter entries exist\n", + __FUNCTION__); + return; + } + + pNvp->i2c_adapters[free].pOsAdapter = nv_i2c_add_adapter(pNv, port); + pNvp->i2c_adapters[free].port = port; + // When port is added, numDispId will be 0. + pNvp->i2c_adapters[free].displayId[numDispId] = displayId; +} + +void RmI2cAddGpuPorts(nv_state_t * pNv) +{ + NvU32 x = 0; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + NvU32 displayMask; + NV_STATUS status; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS systemGetSupportedParams = { 0 }; + + if (IS_VIRTUAL(pGpu) || NV_IS_SOC_DISPLAY_DEVICE(pNv)) + return; + + // Make displayId as Invalid. + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + NvU32 numDispId; + + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + pNvp->i2c_adapters[x].displayId[numDispId] = INVALID_DISP_ID; + } + } + + // First, set up the regular i2c adapters - one per i2c port + if (pNv->rmapi.hI2C != 0) + { + NV402C_CTRL_I2C_GET_PORT_INFO_PARAMS i2cPortInfoParams = { 0 }; + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hI2C, + NV402C_CTRL_CMD_I2C_GET_PORT_INFO, + &i2cPortInfoParams, sizeof(i2cPortInfoParams)); + + if (status == NV_OK) + { + for (x = 0; x < NV_ARRAY_ELEMENTS(i2cPortInfoParams.info); x++) + { + // + // Check if this port is implemented and RM I2C framework has + // validated this port. Only limited amount of ports can + // be added to the OS framework. + // + if (FLD_TEST_DRF(402C_CTRL, _I2C_GET_PORT_INFO, _IMPLEMENTED, + _YES, i2cPortInfoParams.info[x]) && + FLD_TEST_DRF(402C_CTRL, _I2C_GET_PORT_INFO, _VALID, + _YES, i2cPortInfoParams.info[x])) + { + rm_i2c_add_adapter(pNv, x, 0); + } + } + } + } + + // + // Now set up the i2c-over-DpAux adapters - one per DP OD + // + // 1. Perform NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS RM control which + // will return the mask for all the display ID's. + // 2. Loop for all the display ID's and do + // NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO RM control call. For each + // output resource, check for the following requirements: + // a. It must be DisplayPort. + // b. It must be internal to the GPU (ie, not on the board) + // c. It must be directly connected to the physical connector (ie, no DP + // 1.2 multistream ODs). + // 3. Perform NV0073_CTRL_CMD_SPECIFIC_GET_I2C_PORTID RM control for + // getting the I2C port data. + // + // With these restrictions, we should only end up with at most one OD + // per DP connector. + // + + if (pNv->rmapi.hDisp == 0) + { + return; + } + + systemGetSupportedParams.subDeviceInstance = 0; + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp, + NV0073_CTRL_CMD_SYSTEM_GET_SUPPORTED, + &systemGetSupportedParams, sizeof(systemGetSupportedParams)); + + if (status != NV_OK) + { + return; + } + + for (displayMask = systemGetSupportedParams.displayMask; + displayMask != 0; + displayMask &= ~LOWESTBIT(displayMask)) + { + NV0073_CTRL_SPECIFIC_OR_GET_INFO_PARAMS orInfoParams = { 0 }; + NvU32 displayId = LOWESTBIT(displayMask); + + orInfoParams.subDeviceInstance = 0; + orInfoParams.displayId = displayId; + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp, + NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO, + &orInfoParams, sizeof(orInfoParams)); + + if ((status == NV_OK) && + (orInfoParams.type == NV0073_CTRL_SPECIFIC_OR_TYPE_SOR) && + ((orInfoParams.protocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DP_A) || + (orInfoParams.protocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DP_B)) && + (orInfoParams.location == NV0073_CTRL_SPECIFIC_OR_LOCATION_CHIP) && + (!orInfoParams.bIsDispDynamic)) + { + NV0073_CTRL_SPECIFIC_GET_I2C_PORTID_PARAMS i2cPortIdParams = { 0 }; + + i2cPortIdParams.subDeviceInstance = 0; + i2cPortIdParams.displayId = displayId; + + status = pRmApi->Control(pRmApi, + pNv->rmapi.hClient, + pNv->rmapi.hDisp, + NV0073_CTRL_CMD_SPECIFIC_GET_I2C_PORTID, + &i2cPortIdParams, + sizeof(i2cPortIdParams)); + + if ((status == NV_OK) && + (i2cPortIdParams.ddcPortId != NV0073_CTRL_SPECIFIC_I2C_PORT_NONE)) + { + rm_i2c_add_adapter(pNv, i2cPortIdParams.ddcPortId - 1, displayId); + } + } + } +} + +void NV_API_CALL rm_i2c_remove_adapters( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + // + // Cycle through all adapter entries, and first remove the adapter + // from the list from the kernel, then remove the i2c adapter + // list once that is completed. This should only be used from exit + // module time. Otherwise it could fail to remove some of the + // kernel adapters and subsequent transfer requests would result + // in crashes. + // + NvU32 x = 0; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + NvU32 numDispId; + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + if (pNvp->i2c_adapters[x].pOsAdapter != NULL) + { + nv_i2c_del_adapter(pNv, pNvp->i2c_adapters[x].pOsAdapter); + + pNvp->i2c_adapters[x].pOsAdapter = NULL; + pNvp->i2c_adapters[x].port = 0; + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + pNvp->i2c_adapters[x].displayId[numDispId] = INVALID_DISP_ID; + } + } + } +} + +NvBool NV_API_CALL rm_i2c_is_smbus_capable( + nvidia_stack_t *sp, + nv_state_t *pNv, + void *pI2cAdapter +) +{ + THREAD_STATE_NODE threadState; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + NV_STATUS rmStatus = NV_OK; + OBJGPU *pGpu = NULL; + NvBool unlock = NV_FALSE; + NvU32 x; + NvBool ret = NV_FALSE; + void *fp; + NvU32 numDispId = 0; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (pNvp->flags & NV_INIT_FLAG_PUBLIC_I2C) + { + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK) + goto semafinish; + + unlock = NV_TRUE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + if (!pGpu) + { + goto semafinish; + } + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + if (pNvp->i2c_adapters[x].pOsAdapter == pI2cAdapter) + { + break; + } + } + + if (x == MAX_I2C_ADAPTERS) + { + goto semafinish; + } + + // we do not support smbus functions on i2c-over-DPAUX + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + if (pNvp->i2c_adapters[x].displayId[numDispId] == 0x0) + { + ret = NV_TRUE; + } + } + +semafinish: + if (unlock) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return ret; +} + +NV_STATUS NV_API_CALL rm_perform_version_check( + nvidia_stack_t *sp, + void *pData, + NvU32 dataSize +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + rmStatus = RmPerformVersionCheck(pData, dataSize); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +// +// Handles the Power Source Change event(AC/DC) for Notebooks. +// Notebooks from Maxwell have only one Gpu, so this functions grabs first Gpu +// from GpuMgr and call subdevice RmControl. +// +void NV_API_CALL rm_power_source_change_event( + nvidia_stack_t *sp, + NvU32 event_val +) +{ + THREAD_STATE_NODE threadState; + void *fp; + nv_state_t *nv; + static NvBool first_event_seen = NV_FALSE; + static NvU32 last_event_val = 0; + NV_STATUS rmStatus = NV_OK; + + // Some systems generate spurious power source changed ACPI events. + // Before entering the PM runtime and waking the GPU, see if the power + // state has actually changed. If not, return without waking the GPU. + if ((first_event_seen != NV_FALSE) && (last_event_val == event_val)) + { + return; + } + else + { + first_event_seen = NV_TRUE; + last_event_val = event_val; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_EVENT)) == NV_OK) + { + OBJGPU *pGpu = gpumgrGetGpu(0); + if (pGpu != NULL) + { + nv = NV_GET_NV_STATE(pGpu); + if ((rmStatus = os_ref_dynamic_power(nv, NV_DYNAMIC_PM_FINE)) == + NV_OK) + { + // LOCK: acquire GPU lock + if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_EVENT)) == + NV_OK) + { + rmStatus = RmPowerSourceChangeEvent(nv, event_val); + + // UNLOCK: release GPU lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + os_unref_dynamic_power(nv, NV_DYNAMIC_PM_FINE); + } + // UNLOCK: release API lock + rmapiLockRelease(); + } + } + + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: Failed to handle Power Source change event, status=0x%x\n", + __FUNCTION__, rmStatus); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_request_dnotifier_state( + nv_stack_t *sp, + nv_state_t *pNv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); + + if (nvp->b_mobile_config_enabled) + { + THREAD_STATE_NODE threadState; + void *fp; + GPU_MASK gpuMask; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_ACPI)) == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + // LOCK: acquire per device lock + if ((pGpu != NULL) && + ((rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_ACPI, + &gpuMask)) == NV_OK)) + { + RmRequestDNotifierState(pNv); + + // UNLOCK: release per device lock + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + } +} + +NV_STATUS NV_API_CALL rm_p2p_dma_map_pages( + nvidia_stack_t *sp, + nv_dma_device_t *peer, + NvU8 *pGpuUuid, + NvU64 pageSize, + NvU32 pageCount, + NvU64 *pDmaAddresses, + void **ppPriv +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + NvU32 i, j; + OBJGPU *pGpu; + nv_state_t *pNv; + + if (ppPriv == NULL) + return NV_ERR_INVALID_ARGUMENT; + + *ppPriv = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P); + if (rmStatus != NV_OK) + goto done; + + pGpu = gpumgrGetGpuFromUuid(pGpuUuid, + DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1) | + DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _BINARY)); + if (pGpu == NULL) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto unlock; + } + + pNv = NV_GET_NV_STATE(pGpu); + + if (pNv->coherent) + { + rmStatus = RmP2PDmaMapPages(pNv, peer, pageSize, pageCount, + pDmaAddresses, ppPriv); + goto unlock; + } + + // Check if the topology is supported for third party p2p + if (!nv_grdma_pci_topology_supported(pNv, peer)) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + goto unlock; + } + + for (i = 0; i < pageCount; i++) + { + // Map BAR1 PFNs for non-coherent platforms + rmStatus = nv_dma_map_peer(peer, pNv->dma_dev, 0x1, + pageSize / os_page_size, + &pDmaAddresses[i]); + if (rmStatus != NV_OK) + { + for (j = 0; j < i; j++) + { + nv_dma_unmap_peer(peer, pageSize / os_page_size, + pDmaAddresses[j]); + } + goto unlock; + } + } + +unlock: + rmapiLockRelease(); + +done: + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_get_gpu_info( + nvidia_stack_t *sp, + NvU64 gpuVirtualAddress, + NvU64 length, + NvU8 **ppGpuUuid, + void **ppGpuInfo +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P); + if (rmStatus == NV_OK) + { + OBJGPU *pGpu; + rmStatus = RmP2PGetGpuByAddress(gpuVirtualAddress, + length, + &pGpu); + if (rmStatus == NV_OK) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + const NvU8 *pGid; + + pGid = RmGetGpuUuidRaw(nv); + if (pGid == NULL) + { + rmStatus = NV_ERR_GPU_UUID_NOT_FOUND; + } + else + { + rmStatus = os_alloc_mem((void **)ppGpuUuid, GPU_UUID_LEN); + if (rmStatus == NV_OK) + { + os_mem_copy(*ppGpuUuid, pGid, GPU_UUID_LEN); + } + } + + *ppGpuInfo = (void *) pGpu; + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent( + nvidia_stack_t *sp, + NvU64 gpuVirtualAddress, + NvU64 length, + void **p2pObject, + NvU64 *pPhysicalAddresses, + NvU32 *pEntries, + NvBool bForcePcie, + void *pPlatformData, + void *pGpuInfo, + void **ppMigInfo, + NvBool *pMemCpuCacheable +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + rmStatus = RmP2PGetPagesPersistent(gpuVirtualAddress, + length, + p2pObject, + pPhysicalAddresses, + pEntries, + pMemCpuCacheable, + bForcePcie, + pPlatformData, + pGpuInfo, + ppMigInfo); + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_get_pages( + nvidia_stack_t *sp, + NvU64 p2pToken, + NvU32 vaSpaceToken, + NvU64 gpuVirtualAddress, + NvU64 length, + NvU64 *pPhysicalAddresses, + NvU32 *pWreqMbH, + NvU32 *pRreqMbH, + NvU32 *pEntries, + NvU8 **ppGpuUuid, + void *pPlatformData, + NvBool *pMemCpuCacheable +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + OBJGPU *pGpu; + rmStatus = RmP2PGetPagesWithoutCallbackRegistration(p2pToken, + vaSpaceToken, + gpuVirtualAddress, + length, + pPhysicalAddresses, + pWreqMbH, + pRreqMbH, + pEntries, + pMemCpuCacheable, + &pGpu, + pPlatformData); + if (rmStatus == NV_OK) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + const NvU8 *pGid; + + pGid = RmGetGpuUuidRaw(nv); + if (pGid == NULL) + { + NV_ASSERT_OK(RmP2PPutPages(p2pToken, vaSpaceToken, + gpuVirtualAddress, + pPlatformData)); + rmStatus = NV_ERR_GENERIC; + } + else + { + rmStatus = os_alloc_mem((void **)ppGpuUuid, GPU_UUID_LEN); + if (rmStatus == NV_OK) + os_mem_copy(*ppGpuUuid, pGid, GPU_UUID_LEN); + } + + *pMemCpuCacheable = nv->coherent; + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_register_callback( + nvidia_stack_t *sp, + NvU64 p2pToken, + NvU64 gpuVirtualAddress, + NvU64 length, + void *pPlatformData, + void (*pFreeCallback)(void *pData), + void *pData +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + rmStatus = RmP2PRegisterCallback(p2pToken, gpuVirtualAddress, length, + pPlatformData, pFreeCallback, pData); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent( + nvidia_stack_t *sp, + void *p2pObject, + void *pKey, + void *pMigInfo +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + rmStatus = RmP2PPutPagesPersistent(p2pObject, pKey, pMigInfo); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_put_pages( + nvidia_stack_t *sp, + NvU64 p2pToken, + NvU32 vaSpaceToken, + NvU64 gpuVirtualAddress, + void *pKey +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + rmStatus = RmP2PPutPages(p2pToken, + vaSpaceToken, + gpuVirtualAddress, + pKey); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +char* NV_API_CALL rm_get_gpu_uuid( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + NV_STATUS rmStatus; + const NvU8 *pGid; + char *pGidString; + + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // Allocate space for the ASCII string + rmStatus = os_alloc_mem((void **)&pGidString, GPU_UUID_ASCII_LEN); + if (rmStatus != NV_OK) + { + pGidString = NULL; + goto done; + } + + // Get the raw UUID; note the pGid is cached, so we do not need to free it + pGid = RmGetGpuUuidRaw(nv); + + if (pGid != NULL) + { + // Convert the raw UUID to ASCII + rmStatus = RmGpuUuidRawToString(pGid, pGidString, GPU_UUID_ASCII_LEN); + if (rmStatus != NV_OK) + { + os_free_mem(pGidString); + pGidString = NULL; + } + } + else + { + const char *pTmpString = "GPU-???????\?-???\?-???\?-???\?-????????????"; + + portStringCopy(pGidString, GPU_UUID_ASCII_LEN, pTmpString, + portStringLength(pTmpString) + 1); + } + +done: + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return pGidString; +} + +// +// This function will return the UUID in the binary format +// +const NvU8 * NV_API_CALL rm_get_gpu_uuid_raw( + nvidia_stack_t *sp, + nv_state_t *nv) +{ + THREAD_STATE_NODE threadState; + void *fp; + const NvU8 *pGid; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGid = RmGetGpuUuidRaw(nv); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return pGid; +} + +static void rm_set_firmware_logs( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + void *fp; + NvU32 enableFirmwareLogsRegVal = NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG; + + NV_ENTER_RM_RUNTIME(sp,fp); + + (void) RmReadRegistryDword(nv, NV_REG_ENABLE_GPU_FIRMWARE_LOGS, + &enableFirmwareLogsRegVal); + nv->enable_firmware_logs = gpumgrGetRmFirmwareLogsEnabled(enableFirmwareLogsRegVal); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +static NvBool rm_get_is_gsp_capable_vgpu( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + void *fp; + THREAD_STATE_NODE threadState; + NvBool isVgpu = NV_FALSE; + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys); + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if ((pHypervisor != NULL) && pHypervisor->bIsHVMGuest) + { + GPUHWREG *reg_mapping; + reg_mapping = osMapKernelSpace(nv->regs->cpu_address, + os_page_size, + NV_MEMORY_UNCACHED, + NV_PROTECT_READABLE); + + if (reg_mapping != NULL) + { + NvU32 pmc_boot_1 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_1); + NvU32 pmc_boot_42 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_42); + + osUnmapKernelSpace(reg_mapping, os_page_size); + + if (FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1) && + gpumgrIsVgxRmFirmwareCapableChip(pmc_boot_42)) + { + isVgpu = NV_TRUE; + } + } + } + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return isVgpu; +} + +void NV_API_CALL rm_set_rm_firmware_requested( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + nv->request_firmware = NV_TRUE; + nv->allow_fallback_to_monolithic_rm = NV_FALSE; + if (rm_get_is_gsp_capable_vgpu(sp, nv)) + { + nv->request_firmware = NV_FALSE; + } + + // Check if we want firmware logs + if (nv->request_firmware) + rm_set_firmware_logs(sp, nv); +} + +// +// disable GPU SW state persistence +// + +void NV_API_CALL rm_disable_gpu_state_persistence(nvidia_stack_t *sp, nv_state_t *nv) +{ + THREAD_STATE_NODE threadState; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu->setProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE, NV_FALSE); + osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, NV_FALSE); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd) +{ + nvidia_kernel_rmapi_ops_t *ops = ops_cmd; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + switch (ops->op) + { + case NV01_FREE: + Nv01FreeKernel(&ops->params.free); + break; + + case NV01_ALLOC_MEMORY: + Nv01AllocMemoryKernel(&ops->params.allocMemory64); + break; + + case NV04_ALLOC: + Nv04AllocWithAccessKernel(&ops->params.alloc); + break; + + case NV04_VID_HEAP_CONTROL: + Nv04VidHeapControlKernel(ops->params.pVidHeapControl); + break; + + case NV04_MAP_MEMORY: + { + // + // We need to free NVUAP for MEM_SPACE_USER mappings, + // since we're not going through the regular path + // + nv_usermap_access_params_t *pNvuap = NULL; + Nv04MapMemoryKernel(&ops->params.mapMemory); + pNvuap = (nv_usermap_access_params_t *) tlsEntryGet(TLS_ENTRY_ID_MAPPING_CONTEXT); + if (pNvuap != NULL) + { + NV_ASSERT(tlsEntryRelease(TLS_ENTRY_ID_MAPPING_CONTEXT) == 0); + os_free_mem(pNvuap->memArea.pRanges); + os_free_mem(pNvuap); + } + break; + } + + case NV04_UNMAP_MEMORY: + Nv04UnmapMemoryKernel(&ops->params.unmapMemory); + break; + + case NV04_ALLOC_CONTEXT_DMA: + Nv04AllocContextDmaKernel(&ops->params.allocContextDma2); + break; + + case NV04_MAP_MEMORY_DMA: + Nv04MapMemoryDmaKernel(&ops->params.mapMemoryDma); + break; + + case NV04_UNMAP_MEMORY_DMA: + Nv04UnmapMemoryDmaKernel(&ops->params.unmapMemoryDma); + break; + + case NV04_BIND_CONTEXT_DMA: + Nv04BindContextDmaKernel(&ops->params.bindContextDma); + break; + + case NV04_CONTROL: + Nv04ControlKernel(&ops->params.control); + break; + + case NV04_DUP_OBJECT: + Nv04DupObjectKernel(&ops->params.dupObject); + break; + + case NV04_SHARE: + Nv04ShareKernel(&ops->params.share); + break; + + case NV04_ADD_VBLANK_CALLBACK: + Nv04AddVblankCallbackKernel(&ops->params.addVblankCallback); + break; + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +// +// ACPI method (NVIF/_DSM/WMMX/MXM*/etc.) initialization +// +void RmInitAcpiMethods(OBJOS *pOS, OBJSYS *pSys, OBJGPU *pGpu) +{ + NvU32 handlesPresent; + + if (pSys->getProperty(pSys, PDB_PROP_SYS_NVIF_INIT_DONE)) + return; + + nv_acpi_methods_init(&handlesPresent); + + acpiDsmInit(pGpu); +} + +// +// ACPI method (NVIF/_DSM/WMMX/MXM*/etc.) teardown +// +void RmUnInitAcpiMethods(OBJSYS *pSys) +{ + pSys->setProperty(pSys, PDB_PROP_SYS_NVIF_INIT_DONE, NV_FALSE); + + nv_acpi_methods_uninit(); +} + +// +// Converts an array of OS page address to an array of RM page addresses.This +// assumes that: +// (1) The pteArray is at least pageCount entries large, +// (2) The pageCount is given in RM pages, and +// (3) The OS page entries start at index 0. +// +void RmInflateOsToRmPageArray(RmPhysAddr *pteArray, NvU64 pageCount) +{ + RmInflatePageArray( + pteArray, + NV_RM_PAGES_TO_OS_PAGES(pageCount), + os_page_size, + pageCount, + NV_RM_PAGE_SIZE + ); +} + +void RmDeflateRmToOsPageArray(RmPhysAddr *pteArray, NvU64 pageCount) +{ + RmDeflatePageArray( + pteArray, + pageCount, + NV_RM_PAGE_SIZE, + NV_RM_PAGES_TO_OS_PAGES(pageCount), + os_page_size + ); +} + +NvBool NV_API_CALL +rm_get_device_remove_flag +( + nvidia_stack_t * sp, + NvU32 gpu_id +) +{ + THREAD_STATE_NODE threadState; + void *fp; + NvBool bRemove; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (gpumgrQueryGpuDrainState(gpu_id, NULL, &bRemove) != NV_OK) + { + bRemove = NV_FALSE; + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return bRemove; +} + +// +// This API updates only the following fields in nv_ioctl_numa_info_t: +// - nid +// - numa_mem_addr +// - numa_mem_size +// - offline_addresses +// +// Rest of the fields should be updated by caller. +NV_STATUS NV_API_CALL rm_get_gpu_numa_info( + nvidia_stack_t *sp, + nv_state_t *nv, + nv_ioctl_numa_info_t *numa_info +) +{ + NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS *pParams; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + void *fp; + NV_STATUS status = NV_OK; + + ct_assert(NV_ARRAY_ELEMENTS(numa_info->offline_addresses.addresses) >= + NV_ARRAY_ELEMENTS(pParams->numaOfflineAddresses)); + + if ((numa_info == NULL) || + (numa_info->offline_addresses.numEntries > + NV_ARRAY_ELEMENTS(pParams->numaOfflineAddresses))) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if (NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + numa_info->nid = NV0000_CTRL_NO_NUMA_NODE; + numa_info->numa_mem_addr = 0; + numa_info->numa_mem_size = 0; + numa_info->offline_addresses.numEntries = 0; + return NV_OK; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + pParams = portMemAllocNonPaged(sizeof(*pParams)); + if (pParams == NULL) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_ERR_NO_MEMORY; + } + + portMemSet(pParams, 0, sizeof(*pParams)); + pParams->numaOfflineAddressesCount = + numa_info->offline_addresses.numEntries; + + pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM); + if (pRmApi == NULL) + { + status = NV_ERR_INVALID_STATE; + goto finish; + } + + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FB_GET_NUMA_INFO, + pParams, sizeof(*pParams)); + + RmUnixRmApiEpilogue(nv, &threadState); + + if (status == NV_OK) + { + NvU32 i; + + numa_info->nid = pParams->numaNodeId; + numa_info->numa_mem_addr = pParams->numaMemAddr; + numa_info->numa_mem_size = pParams->numaMemSize; + numa_info->offline_addresses.numEntries = + pParams->numaOfflineAddressesCount; + + for (i = 0; i < pParams->numaOfflineAddressesCount; i++) + { + numa_info->offline_addresses.addresses[i] = + pParams->numaOfflineAddresses[i]; + } + } + +finish: + portMemFree(pParams); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +NV_STATUS NV_API_CALL rm_gpu_numa_online( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + void *fp; + NV_STATUS status = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM); + if (pRmApi == NULL) + { + status = NV_ERR_INVALID_STATE; + goto finish; + } + + params.bOnline = NV_TRUE; + + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(nv, &threadState); + +finish: + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + + +NV_STATUS NV_API_CALL rm_gpu_numa_offline( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + void *fp; + NV_STATUS status = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM); + if (pRmApi == NULL) + { + status = NV_ERR_INVALID_STATE; + goto finish; + } + + params.bOnline = NV_FALSE; + + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, + nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(nv, &threadState); + +finish: + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +// +// A device is considered "sequestered" if it has drain state enabled for it. +// The kernel interface layer can use this to check the drain state of a device +// in paths outside of initialization, e.g., when clients attempt to reference +// count the device. +// +NvBool NV_API_CALL rm_is_device_sequestered( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + NvBool bDrain = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // + // If gpumgrQueryGpuDrainState succeeds, bDrain will be set as needed. + // If gpumgrQueryGpuDrainState fails, bDrain will stay false; we assume + // that if core RM can't tell us the drain state, it must not be + // attached and the "sequestered" question is not relevant. + // + (void) gpumgrQueryGpuDrainState(pNv->gpu_id, &bDrain, NULL); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return bDrain; +} + +void NV_API_CALL rm_check_for_gpu_surprise_removal( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + NV_STATUS rmStatus; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock. + if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU)) == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if ((rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GPU)) == NV_OK) + { + osHandleGpuLost(pGpu, NV_TRUE); + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + + // UNLOCK: release api lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_set_external_kernel_client_count( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvBool bIncr +) +{ + THREAD_STATE_NODE threadState; + void *fp; + OBJGPU *pGpu; + NV_STATUS rmStatus = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + if (pGpu != NULL) + { + rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU); + if (rmStatus == NV_OK) + { + rmStatus = gpuSetExternalKernelClientCount(pGpu, bIncr); + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NvBool rm_get_uefi_console_status( + nv_state_t *nv +) +{ + NvU32 fbWidth, fbHeight, fbDepth, fbPitch; + NvU64 fbSize; + NvU64 fbBaseAddress = 0; + NvBool bConsoleDevice = NV_FALSE; + + // + // nv_get_screen_info() will return dimensions and an address for + // any fbdev driver (e.g., efifb, vesafb, etc). + // + nv_get_screen_info(nv, &fbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch, &fbSize); + + bConsoleDevice = (fbSize != 0); + + return bConsoleDevice; +} + +NvU64 rm_get_uefi_console_size( + nv_state_t *nv, + NvU64 *pFbBaseAddress +) +{ + NvU32 fbWidth, fbHeight, fbDepth, fbPitch; + NvU64 fbSize; + + fbSize = fbWidth = fbHeight = fbDepth = fbPitch = 0; + + // + // nv_get_screen_info() will return dimensions and an address for + // any fbdev driver (e.g., efifb, vesafb, etc). + // + nv_get_screen_info(nv, pFbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch, &fbSize); + + return fbSize; +} + +NvBool NV_API_CALL rm_disable_iomap_wc(void) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = pSys->pCl; + return pCl->getProperty(pCl, PDB_PROP_CL_DISABLE_IOMAP_WC) == NV_TRUE; +} + +// +// Verifies the handle, offset and size and dups hMemory. +// Must be called with API lock and GPU lock held. +// +NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hSrcClient, + NvHandle hDstClient, + NvHandle hDevice, + NvHandle hSubdevice, + void *pGpuInstanceInfo, + NvHandle hMemory, + NvU64 offset, + NvU64 size, + NvU8 mappingType, + NvHandle *phMemoryDuped, + void **ppMemInfo, + NvBool *pbCanMmap, + NvU32 *pCacheType, + NvBool *pbReadOnlyMem, + nv_memory_type_t *pMemoryType +) +{ + MEMORY_DESCRIPTOR *pMemDesc; + THREAD_STATE_NODE *pThreadState; + NV_STATUS rmStatus = NV_ERR_NO_MEMORY; + OBJGPU *pGpu; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + pThreadState = threadStateAlloc(THREAD_STATE_FLAGS_NONE); + NV_ASSERT_OR_GOTO(pThreadState != NULL, Done); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + NV_ASSERT(rmapiLockIsOwner()); + + NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu))); + + rmStatus = RmDmabufVerifyMemHandle(pGpu, hSrcClient, hMemory, + offset, size, pGpuInstanceInfo, + &pMemDesc); + if (rmStatus == NV_OK) + { + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NvHandle hMemoryDuped = 0; + + rmStatus = pRmApi->DupObject(pRmApi, + hDstClient, + hDevice, + &hMemoryDuped, + hSrcClient, + hMemory, + 0); + if (rmStatus == NV_OK) + { + *phMemoryDuped = hMemoryDuped; + } + else if (rmStatus == NV_ERR_INVALID_OBJECT_PARENT) + { + hMemoryDuped = 0; + + // If duping under Device fails, try duping under Subdevice + rmStatus = pRmApi->DupObject(pRmApi, + hDstClient, + hSubdevice, + &hMemoryDuped, + hSrcClient, + hMemory, + 0); + if (rmStatus == NV_OK) + { + *phMemoryDuped = hMemoryDuped; + } + } + *ppMemInfo = (void *) pMemDesc; + + *pCacheType = RmDmabufMmapGetCpuCacheType(pGpu, pMemDesc, mappingType); + *pbReadOnlyMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY); + + if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM)) + { + *pMemoryType = NV_MEMORY_TYPE_DEVICE_MMIO; + } + else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM) + { + *pMemoryType = NV_MEMORY_TYPE_SYSTEM; + } + else + { + // TODO: Assume NV_MEMORY_TYPE_FRAMEBUFFER is for now. + // Add Proper handling for NV_MEMORY_TYPE_REGISTERS. + *pMemoryType = NV_MEMORY_TYPE_FRAMEBUFFER; + } + + // mmap is allowed on 0FB chips(iGPU) for sysmem and on dGPU for vidmem only + *pbCanMmap = (pGpu->pGpuArch->bGpuArchIsZeroFb || + (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)); + } + +Done: + if (pThreadState != NULL) + threadStateFree(pThreadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +// +// Frees dup'd hMemory. +// Must be called with API lock and GPU lock held. +// +void NV_API_CALL rm_dma_buf_undup_mem_handle( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hMemory +) +{ + THREAD_STATE_NODE threadState; + RM_API *pRmApi; + OBJGPU *pGpu; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + NV_ASSERT(rmapiLockIsOwner()); + + NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu))); + + pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + + pRmApi->Free(pRmApi, hClient, hMemory); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +// +// Maps a handle to system physical addresses: +// C2C for coherent platforms with DEFAULT mapping type +// BAR1(static & dynamic) for non-coherent platforms and for +// coherent platforms with mapping type FORCE_PCIE +// Must be called with API lock and GPU lock held for dynamic BAR1. +// +NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hMemory, + MemoryRange memRange, + NvU8 mappingType, + void *pMemInfo, + NvBool bStaticPhysAddrs, + MemoryArea *pMemArea +) +{ + THREAD_STATE_NODE *pThreadState; + NV_STATUS rmStatus = NV_ERR_INVALID_ARGUMENT; + OBJGPU *pGpu; + MEMORY_DESCRIPTOR *pMemDesc; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + pThreadState = threadStateAlloc(THREAD_STATE_FLAGS_NONE); + if (pThreadState == NULL) + { + NV_ASSERT(0); + rmStatus = NV_ERR_NO_MEMORY; + goto Done; + } + + NV_ASSERT_OR_GOTO(((pMemArea != NULL) && + (pMemInfo != NULL)) && + (memRange.size != 0), Done); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + pMemDesc = (MEMORY_DESCRIPTOR *) pMemInfo; + + if (((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) || + (pGpu->pGpuArch->bGpuArchIsZeroFb)) && + (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT)) + { + KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); + NvBool contiguity = memdescCheckContiguity(pMemDesc, AT_CPU); + + // On localized allocations over C2C/nvlink mappings, RDMA is not supported + if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_AS_LOCALIZED)) + { + NV_PRINTF(LEVEL_ERROR, "RDMA is not supported for localized memory" + " over coherent mappings\n"); + rmStatus = NV_ERR_NOT_SUPPORTED; + goto Done; + } + + if (contiguity) + { + NvU64 physAddr = memdescGetPhysAddr(pMemDesc, AT_CPU, memRange.start); + + NV_ASSERT_OK_OR_GOTO(rmStatus, os_alloc_mem((void **) &pMemArea->pRanges, + sizeof(MemoryRange)), Done); + + pMemArea->pRanges[0].start = pKernelMemorySystem->coherentCpuFbBase + physAddr; + pMemArea->pRanges[0].size = memRange.size; + pMemArea->numRanges = 1; + } + else + { + NvU64 idx = 0; + NvU64 memdescPageSize = memdescGetPageSize(pMemDesc, AT_CPU); + NvU64 origEnd = memRange.start + memRange.size; + NvU64 realStart = NV_ALIGN_DOWN64(memRange.start, memdescPageSize); + NvU64 realEnd = NV_ALIGN_UP64(memRange.start + memRange.size, memdescPageSize); + NvU64 realSize = realEnd - realStart; + NvU64 pageCount = realSize / memdescPageSize; + + NV_ASSERT_OK_OR_GOTO(rmStatus, os_alloc_mem((void **) &pMemArea->pRanges, + pageCount * sizeof(MemoryRange)), Done); + + for(idx = 0; idx < pageCount; idx++) + { + NvU64 physAddr = memdescGetPhysAddr(pMemDesc, AT_CPU, + realStart + (idx * memdescPageSize)); + pMemArea->pRanges[idx].start = pKernelMemorySystem->coherentCpuFbBase + physAddr; + pMemArea->pRanges[idx].size = memdescPageSize; + } + + pMemArea->pRanges[0].start += (memRange.start - realStart); + pMemArea->pRanges[0].size -= (memRange.start - realStart); + pMemArea->pRanges[pageCount - 1llu].size -= (realEnd - origEnd); + + pMemArea->numRanges = pageCount; + } + } + else + { + Device *pDevice = NULL; + RsClient *pClient; + NvU64 idx; + NvU64 barOffset; + KernelBus *pKernelBus; + NvBool bForcePcie; + + pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + + if (!bStaticPhysAddrs) + { + // Dynamic BAR1 requires RMAPI and GPU lock to be held + NV_ASSERT(rmapiLockIsOwner()); + NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu))); + + NV_ASSERT_OK_OR_GOTO(rmStatus, + serverGetClientUnderLock(&g_resServ, hClient, &pClient), + Done); + + NV_ASSERT_OK_OR_GOTO(rmStatus, + deviceGetByGpu(pClient, pGpu, NV_TRUE, &pDevice), + Done); + } + + NV_CHECK_OR_GOTO(LEVEL_ERROR, + memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM, + Done); + + // + // We don't take locks for static BAR1. + // Thus, we operate directly on the memdesc passed to + // bypass the client lookup which requires client locks. + // + NV_ASSERT_OK_OR_GOTO(rmStatus, + kbusMapFbAperture_HAL(pGpu, pKernelBus, pMemDesc, memRange, pMemArea, + (BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_ALLOW_DISCONTIG), + pDevice), + Done); + + bForcePcie = (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE); + + NV_ASSERT_OK_OR_GOTO(rmStatus, + kbusGetGpuFbPhysAddressForRdma(pGpu, pKernelBus, + bForcePcie, &barOffset), + Done); + + for (idx = 0; idx < pMemArea->numRanges; idx++) + { + pMemArea->pRanges[idx].start += barOffset; + } + } + +Done: + if (pThreadState != NULL) + threadStateFree(pThreadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +// +// Unmaps a handle that was mapped to system physical addresses. +// Must be called with API lock and GPU lock held for dynamic BAR1. +// +void NV_API_CALL rm_dma_buf_unmap_mem_handle( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hMemory, + NvU8 mappingType, + void *pMemInfo, + NvBool bStaticPhysAddrs, + MemoryArea memArea +) +{ + THREAD_STATE_NODE threadState; + OBJGPU *pGpu; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) || + pGpu->pGpuArch->bGpuArchIsZeroFb) && + (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT)) + { + os_free_mem(memArea.pRanges); + } + else + { + KernelBus *pKernelBus; + NvU64 idx; + NvU64 barOffset; + MEMORY_DESCRIPTOR *pMemDesc = (MEMORY_DESCRIPTOR *) pMemInfo; + NvBool bForcePcie = (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE); + + pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + + NV_ASSERT_OK(kbusGetGpuFbPhysAddressForRdma(pGpu, pKernelBus, + bForcePcie, &barOffset)); + + for (idx = 0; idx < memArea.numRanges; idx++) + { + memArea.pRanges[idx].start -= barOffset; + } + + if (!bStaticPhysAddrs) + { + // Dynamic BAR1 requires RMAPI and GPU lock to be held + NV_ASSERT(rmapiLockIsOwner()); + NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu))); + } + + // + // We don't take locks for static BAR1. + // Thus, we operate directly on the memdesc passed to + // bypass the client lookup which requires client locks. + // + NV_ASSERT_OK(kbusUnmapFbAperture_HAL(pGpu, pKernelBus, pMemDesc, memArea, + BUS_MAP_FB_FLAGS_MAP_UNICAST)); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hMemory, + NvU8 mappingType, + NvHandle *phClient, + NvHandle *phDevice, + NvHandle *phSubdevice, + void **ppGpuInstanceInfo, + NvBool *pbStaticPhysAddrs, + NvBool *pbAcquireReleaseAllGpuLockOnDup +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus == NV_OK) + { + rmStatus = RmDmabufGetClientAndDevice(pGpu, hClient, hMemory, mappingType, + phClient, phDevice, + phSubdevice, ppGpuInstanceInfo); + if (rmStatus == NV_OK) + { + *pbStaticPhysAddrs = ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) || + pGpu->pGpuArch->bGpuArchIsZeroFb) && + (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT)) || + kbusIsStaticBar1Enabled(pGpu, GPU_GET_KERNEL_BUS(pGpu)); + } + + *pbAcquireReleaseAllGpuLockOnDup = pGpu->pGpuArch->bGpuArchIsZeroFb; + + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +void NV_API_CALL rm_dma_buf_put_client_and_device( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hDevice, + NvHandle hSubdevice, + void *pGpuInstanceInfo +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus == NV_OK) + { + RmDmabufPutClientAndDevice(pGpu, hClient, hDevice, hSubdevice, + pGpuInstanceInfo); + + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + NV_ASSERT_OK(rmStatus); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +// +// Fetches GSP ucode data for usage during RM Init +// NOTE: Used only on VMWware +// + +void NV_API_CALL rm_vgpu_vfio_set_driver_vm( + nvidia_stack_t *sp, + NvBool is_driver_vm +) +{ + OBJSYS *pSys; + OBJHYPERVISOR *pHypervisor; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pSys = SYS_GET_INSTANCE(); + pHypervisor = SYS_GET_HYPERVISOR(pSys); + + pHypervisor->setProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED, is_driver_vm); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NvBool NV_API_CALL rm_is_altstack_in_use(void) +{ +#if defined(__use_altstack__) + return NV_TRUE; +#else + return NV_FALSE; +#endif +} + +void NV_API_CALL rm_acpi_nvpcf_notify( + nvidia_stack_t *sp +) +{ + void *fp; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_EVENT)) == NV_OK) + { + OBJGPU *pGpu = gpumgrGetGpu(0); + if (pGpu != NULL) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + if ((rmStatus = os_ref_dynamic_power(nv, NV_DYNAMIC_PM_FINE)) == + NV_OK) + { + gpuNotifySubDeviceEvent(pGpu, NV2080_NOTIFIERS_NVPCF_EVENTS, + NULL, 0, 0, 0); + } + os_unref_dynamic_power(nv, NV_DYNAMIC_PM_FINE); + } + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +static void rm_notify_gpu_addition_removal_helper( + nv_state_t *nv, + NvBool bBind) +{ + THREAD_STATE_NODE threadState; + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK) + { + NV0000_CTRL_SYSTEM_EVENT_DATA_GPU_BIND_UNBIND eventData = { 0 }; + + eventData.gpuId = nv->gpu_id; + eventData.bBind = bBind; + + CliAddSystemEvent(NV0000_NOTIFIERS_GPU_BIND_UNBIND_EVENT, &eventData, NULL); + rmapiLockRelease(); + } + else + { + NV_PRINTF(LEVEL_ERROR, "Fail to acquire rmApi lock. Skip notification."); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); +} + +void NV_API_CALL rm_notify_gpu_addition( + nvidia_stack_t *sp, + nv_state_t *nv) +{ + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + rm_notify_gpu_addition_removal_helper(nv, NV_TRUE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_notify_gpu_removal( + nvidia_stack_t *sp, + nv_state_t *nv) +{ + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + rm_notify_gpu_addition_removal_helper(nv, NV_FALSE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NvBool NV_API_CALL rm_wait_for_bar_firewall( + nvidia_stack_t * sp, + NvU32 domain, + NvU8 bus, + NvU8 device, + NvU8 function, + NvU16 devId, + NvU16 subsystemId +) +{ + NvBool ret; + void *fp = NULL; + + // no state set up yet for threadstate or RM locks + NV_ENTER_RM_RUNTIME(sp,fp); + ret = gpumgrWaitForBarFirewall(domain, bus, device, function, devId, subsystemId); + NV_EXIT_RM_RUNTIME(sp,fp); + + return ret; +} + +static NvU32 devfreq_clk_to_domain( + TEGRASOC_DEVFREQ_CLK devfreqClk +) +{ + NvU32 clkDomain = NV2080_CTRL_CLK_DOMAIN_TEGRA_UNDEFINED; + switch (devfreqClk) + { + case TEGRASOC_DEVFREQ_CLK_GPC: + clkDomain = NV2080_CTRL_CLK_DOMAIN_TEGRA_GPCCLK; + break; + + case TEGRASOC_DEVFREQ_CLK_NVD: + clkDomain = NV2080_CTRL_CLK_DOMAIN_TEGRA_NVDCLK; + break; + + default: + break; + } + + return clkDomain; +} + +NV_STATUS NV_API_CALL rm_pmu_perfmon_get_load( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 *load, + TEGRASOC_DEVFREQ_CLK devfreqClk +) +{ + NV2080_CTRL_PERF_GET_TEGRA_PERFMON_SAMPLE_PARAMS params = { 0 }; + NvU32 clkDomain = devfreq_clk_to_domain(devfreqClk); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + RM_API *pRmApi; + NV_STATUS status; + void *fp; + + if (clkDomain == NV2080_CTRL_CLK_DOMAIN_TEGRA_UNDEFINED) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if (nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_INDICATED) + { + *load = 0; + return NV_OK; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK) + { + pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + if (pRmApi == NULL) + { + status = NV_ERR_INVALID_STATE; + rmapiLockRelease(); + goto rm_pmu_perfmon_get_load_exit; + } + + params.clkDomain = clkDomain; + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_PERF_GET_TEGRA_PERFMON_SAMPLE, + ¶ms, sizeof(params)); + if (status == NV_OK) + { + *load = params.clkPercentBusy; + } + + rmapiLockRelease(); + } + else + { + status = NV_ERR_INVALID_STATE; + } + +rm_pmu_perfmon_get_load_exit: + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osinit.c b/userspace/sdks/nv570/nvalloc/unix/src/osinit.c new file mode 100644 index 0000000..b3fd343 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/osinit.c @@ -0,0 +1,2768 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/************************************************************************************************************** +* +* Description: +* UNIX-general, device-independent initialization code for +* the resource manager. +* +* +**************************************************************************************************************/ + +#include <nv_ref.h> +#include <nv.h> // NV device driver interface +#include <nv-reg.h> +#include <nv-priv.h> +#include <nvos.h> +#include <nvrm_registry.h> +#include <platform/sli/sli.h> +#include <nverror.h> +#include <osapi.h> +#include <core/system.h> +#include <os/os.h> +#include "gpu/gpu.h" +#include <gpu/timer/objtmr.h> +#include "gpu/bus/kern_bus.h" +#include "nverror.h" +#include <nv-nb-regs.h> +#include <gpu/bif/kernel_bif.h> +#include <gpu/mem_mgr/mem_mgr.h> +#include <gpu/mem_sys/kern_mem_sys.h> +#include "kernel/gpu/intr/intr.h" +#include <mem_mgr/io_vaspace.h> +#include <gpu/disp/kern_disp.h> +#include <platform/chipset/chipset.h> +#include <kernel/gpu/rc/kernel_rc.h> +#include <kernel/gpu/fifo/kernel_fifo.h> +#include <nv-firmware-chip-family-select.h> +#include <kernel/gpu/gsp/kernel_gsp.h> +#include "liblogdecode.h" + +#include <mem_mgr/virt_mem_mgr.h> +#include <virtualization/kernel_vgpu_mgr.h> + +#include <rmosxfac.h> +#include <gpu_mgr/gpu_mgr.h> +#include <core/thread_state.h> +#include <core/locks.h> +#include <rmapi/client.h> +#include <ctrl/ctrl2080/ctrl2080gpu.h> +#include <class/cl00f2.h> + +#include <platform/chipset/pci_pbi.h> + +#include "platform/nbsi/nbsi_read.h" +#include "gpu_mgr/gpu_db.h" +#include <class/cl0080.h> +#include <class/cl0073.h> +#include <class/cl2080.h> +#include <class/cl402c.h> +#include <ctrl/ctrl2080/ctrl2080gsp.h> + +#include <gpu/dce_client/dce_client.h> +// RMCONFIG: need definition of REGISTER_ALL_HALS() +#include "g_hal_register.h" + +typedef enum +{ + RM_INIT_OK, + + /* general os errors */ + RM_INIT_REG_SETUP_FAILED = 0x10, + RM_INIT_SYS_ENVIRONMENT_FAILED, + + /* gpu errors */ + RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED = 0x20, + RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED, + RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED, + RM_INIT_GPU_PRE_INIT_FAILED, + RM_INIT_GPU_STATE_INIT_FAILED, + RM_INIT_GPU_LOAD_FAILED, + RM_INIT_GPU_DMA_CONFIGURATION_FAILED, + RM_INIT_GPU_GPUMGR_EXPANDED_VISIBILITY_FAILED, + + /* vbios errors */ + RM_INIT_VBIOS_FAILED = 0x30, + RM_INIT_VBIOS_POST_FAILED, + RM_INIT_VBIOS_X86EMU_FAILED, + + /* scalability errors */ + RM_INIT_SCALABILITY_FAILED = 0x40, + + /* general core rm errors */ + RM_INIT_WATCHDOG_FAILED, + RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED, + RM_GPUDB_REGISTER_FAILED, + + RM_INIT_ALLOC_RMAPI_FAILED, + RM_INIT_GPUINFO_WITH_RMAPI_FAILED, + + /* rm firmware errors */ + RM_INIT_FIRMWARE_POLICY_FAILED = 0x60, + RM_INIT_FIRMWARE_FETCH_FAILED, + RM_INIT_FIRMWARE_INIT_FAILED, + + RM_INIT_MAX_FAILURES +} rm_init_status; + +typedef rm_init_status RM_INIT_STATUS; + +typedef struct { + RM_INIT_STATUS initStatus; + NV_STATUS rmStatus; + NvU32 line; +} UNIX_STATUS; + +#define INIT_UNIX_STATUS { RM_INIT_OK, NV_OK, 0 } +#define RM_INIT_SUCCESS(init) ((init) == RM_INIT_OK) + +#define RM_SET_ERROR(status, err) { (status).initStatus = (err); \ + (status).line = __LINE__; } + + +static inline NvU64 nv_encode_pci_info(nv_pci_info_t *pci_info) +{ + return gpuEncodeDomainBusDevice(pci_info->domain, pci_info->bus, pci_info->slot); +} + +static inline NvU32 nv_generate_id_from_pci_info(nv_pci_info_t *pci_info) +{ + return gpuGenerate32BitId(pci_info->domain, pci_info->bus, pci_info->slot); +} + +static void nv_set_probed_gpu_flags(nv_state_t *nv) +{ + NvU32 flags = 0; + + if (NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + flags |= DRF_DEF(0000, _CTRL_GPU_PROBED_ID_FLAGS, _SOC_DISPLAY, _TRUE); + } + + gpumgrSetProbedFlags(nv->gpu_id, flags); +} + +static inline void nv_os_map_kernel_space(nv_state_t *nv, nv_aperture_t *aperture) +{ + NV_ASSERT(aperture->map == NULL); + + // let's start off assuming a standard device and map the registers + // normally. It is unfortunate to hard-code the register size here, but we don't + // want to fail trying to map all of a multi-devices' register space + aperture->map = osMapKernelSpace(aperture->cpu_address, + aperture->size, + NV_MEMORY_UNCACHED, + NV_PROTECT_READ_WRITE); + aperture->map_u = (nv_phwreg_t)aperture->map; +} + +// local prototypes +static NV_STATUS initCoreLogic(OBJGPU *); +static void initVendorSpecificRegistry(OBJGPU *, NvU16); +static NV_STATUS teardownCoreLogic(OBJOS *, OBJGPU *); +static void initUnixSpecificRegistry(OBJGPU *); + +NV_STATUS osRmInitRm(void) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + NV_STATUS status; + + if (os_is_efi_enabled()) + { + pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_TRUE); + } + + // have to init this before the debug subsystem, which will + // try to check the value of ResmanDebugLevel + RmInitRegistry(); + + NvU32 data = 0; + if ((osReadRegistryDword(NULL, + NV_REG_ENABLE_PCIE_GEN3, &data) == NV_OK) && + (data != 0)) + { + data = DRF_DEF(_REG_STR, _RM_PCIE, _LINK_SPEED_ALLOW_GEN3, _ENABLE); + + osWriteRegistryDword(NULL, + NV_REG_STR_RM_PCIE_LINK_SPEED, data); + } + + if (!(osReadRegistryDword(NULL, + NV_REG_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, + &data) == NV_OK) || data) + { + pSys->setProperty(pSys, + PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, + NV_TRUE); + } + + if ((osReadRegistryDword(NULL, + NV_REG_ENABLE_STREAM_MEMOPS, + &data) == NV_OK) && (data != 0)) + { + data = DRF_DEF(_REG_STR, _RM_STREAM_MEMOPS, _ENABLE, _YES); + osWriteRegistryDword(NULL, NV_REG_STR_RM_STREAM_MEMOPS, data); + } + + if ((osReadRegistryDword(NULL, + NV_REG_STR_RM_FORCE_BAR_PATH, + &data) == NV_OK) && (data == 1)) + { + // + // Exposing the GPU memory as NUMA node memory requires coherent CPU + // mappings over NVLink. If those mappings are disabled, also disable + // NUMA-onlining of the device memory. + // + osWriteRegistryDword(NULL, + NV_REG_ENABLE_USER_NUMA_MANAGEMENT, 0); + } + + if ((osReadRegistryDword(NULL, + NV_REG_NVLINK_DISABLE, &data) == NV_OK) && + (data == 1)) + { + osWriteRegistryDword(NULL, NV_REG_STR_RM_NVLINK_CONTROL, 0x1); + } + + if ((osReadRegistryDword(NULL, + NV_REG_ENABLE_PCIE_RELAXED_ORDERING_MODE, &data) == NV_OK) && + (data == 1)) + { + osWriteRegistryDword(NULL, + NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING, + NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING_ENABLE); + } + + // init the debug subsystem if necessary + os_dbg_init(); + nvDbgInitRmMsg(NULL); + + // Force nvlog reinit since module params are now available + NVLOG_UPDATE(); + + // Register all supported hals + status = REGISTER_ALL_HALS(); + if (status != NV_OK) + { + RmDestroyRegistry(NULL); + return status; + } + + rm_check_s0ix_regkey_and_platform_support(); + + // Setup any ThreadState defaults + threadStateInitSetupFlags(THREAD_STATE_SETUP_FLAGS_ENABLED | + THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED | + THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED | + THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED); + + return NV_OK; +} + +void RmShutdownRm(void) +{ + NV_PRINTF(LEVEL_INFO, "shutdown rm\n"); + + RmDestroyRegistry(NULL); + + // Free objects created with RmInitRm, including the system object + RmDestroyRm(); +} + +// +// osAttachGpu +// +// This routine is used as a callback by the gpumgrAttachGpu +// interface to allow os-dependent code to set up any state +// before engine construction begins. +// +NV_STATUS osAttachGpu( + OBJGPU *pGpu, + void *pOsGpuInfo +) +{ + nv_state_t *nv = (nv_state_t *)pOsGpuInfo; + nv_priv_t *nvp; + + nvp = NV_GET_NV_PRIV(nv); + + nvp->pGpu = pGpu; + + NV_SET_NV_STATE(pGpu, (void *)nv); + + initUnixSpecificRegistry(pGpu); + + // Assign default values to Registry keys for VGX + if (os_is_vgx_hyper()) + { + initVGXSpecificRegistry(pGpu); + } + + return NV_OK; +} + +NV_STATUS osDpcAttachGpu( + OBJGPU *pGpu, + void *pOsGpuInfo +) +{ + return NV_OK; // Nothing to do for unix +} + +void osDpcDetachGpu( + OBJGPU *pGpu +) +{ + return; // Nothing to do for unix +} + +NV_STATUS +osHandleGpuLost +( + OBJGPU *pGpu, + NvBool bEmitXid +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 pmc_boot_0; + + // Determine if we've already run the handler + if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED)) + { + return NV_OK; + } + + pmc_boot_0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0); + if (pmc_boot_0 != nvp->pmc_boot_0) + { + // + // This doesn't support PEX Reset and Recovery yet. + // This will help to prevent accessing registers of a GPU + // which has fallen off the bus. + // + if (bEmitXid) + { + nvErrorLog_va((void *)pGpu, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, + "GPU has fallen off the bus."); + } + + gpuNotifySubDeviceEvent(pGpu, NV2080_NOTIFIERS_GPU_UNAVAILABLE, NULL, + 0, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, 0); + + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU has fallen off the bus.\n"); + + if (pGpu->boardInfo != NULL && pGpu->boardInfo->serialNumber[0] != '\0') + { + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, + "GPU serial number is %s.\n", + pGpu->boardInfo->serialNumber); + } + + gpuSetDisconnectedProperties(pGpu); + + if (IS_GSP_CLIENT(pGpu)) + { + KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); + + // Notify all channels of the error so that UVM can fail gracefully + if (pKernelRc != NULL) + { + krcRcAndNotifyAllChannels(pGpu, pKernelRc, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, NV_FALSE); + } + } + + // Initiate a crash dump immediately. + RmLogGpuCrash(pGpu); + + // Set SURPRISE_REMOVAL flag for eGPU to help in device removal. + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_EXTERNAL_GPU)) + { + nv->flags |= NV_FLAG_IN_SURPRISE_REMOVAL; + } + + DBG_BREAKPOINT(); + } + + return NV_OK; +} + +/*! + * @brief Traverse bus topology till Gpu's root port. + * If any of the intermediate bridge has TB3 supported vendorId and hotplug + * capability(not necessarily same bridge), mark the Gpu as External Gpu. + * + * @params[in] pGpu OBJGPU pointer + * @params[in] pCl OBJCL pointer + * +* @return NV_OK +* Identified to be eGPU +* @return others + * Not an eGPU / error on identfying + * + */ +NvBool +RmCheckForExternalGpu +( + OBJGPU *pGpu, + OBJCL *pCl +) +{ + NvU8 bus; + NvU32 domain; + void *handleUp; + NvU8 busUp, devUp, funcUp; + NvU16 vendorIdUp, deviceIdUp; + NvU32 portCaps, pciCaps, slotCaps; + NvU32 PCIECapPtr; + RM_API *pRmApi; + NV_STATUS status, rmStatus; + NvBool bTb3Bridge = NV_FALSE, bSlotHotPlugSupport = NV_FALSE; + NvBool iseGPUBridge = NV_FALSE; + + pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); + domain = gpuGetDomain(pGpu); + bus = gpuGetBus(pGpu); + do + { + // Find the upstream bridge + handleUp = clFindP2PBrdg(pCl, domain, bus, &busUp, &devUp, &funcUp, &vendorIdUp, &deviceIdUp); + if (!handleUp) + { + return iseGPUBridge; + } + + if (vendorIdUp == PCI_VENDOR_ID_INTEL) + { + // Check for the supported TB3(ThunderBolt 3) bridges. + NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS params = { 0 }; + + // LOCK: acquire GPUs lock + rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_INIT); + if (rmStatus != NV_OK) + { + return iseGPUBridge; + } + params.pciDeviceId = deviceIdUp; + status = pRmApi->Control(pRmApi, + pGpu->hInternalClient, + pGpu->hInternalSubdevice, + NV2080_CTRL_CMD_INTERNAL_GET_EGPU_BRIDGE_INFO, + ¶ms, + sizeof(params)); + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "Error 0x%08x on eGPU Approval for Bridge ID: 0x%08x\n", status, deviceIdUp); + DBG_BREAKPOINT(); + return iseGPUBridge; + } + else + { + // Check for the approved eGPU BUS TB3 + if (params.iseGPUBridge && + params.approvedBusType == NV2080_CTRL_INTERNAL_EGPU_BUS_TYPE_TB3) + { + bTb3Bridge = NV_TRUE; + } + } + } + + if (NV_OK != clSetPortPcieCapOffset(pCl, handleUp, &PCIECapPtr)) + { + // PCIE bridge but no cap pointer. + break; + } + + // Get the PCIE capabilities. + pciCaps = osPciReadDword(handleUp, CL_PCIE_CAP - CL_PCIE_BEGIN + PCIECapPtr); + if (CL_PCIE_CAP_SLOT & pciCaps) + { + // Get the slot capabilities. + slotCaps = osPciReadDword(handleUp, CL_PCIE_SLOT_CAP - CL_PCIE_BEGIN + PCIECapPtr); + + if ((CL_PCIE_SLOT_CAP_HOTPLUG_CAPABLE & slotCaps) && + (CL_PCIE_SLOT_CAP_HOTPLUG_SURPRISE & slotCaps)) + { + bSlotHotPlugSupport = NV_TRUE; + } + } + + if (bTb3Bridge && bSlotHotPlugSupport) + { + iseGPUBridge = NV_TRUE; + break; + } + + bus = busUp; + + // Get port caps to check if PCIE bridge is the root port + portCaps = osPciReadDword(handleUp, CL_PCIE_CAP - CL_PCIE_BEGIN + PCIECapPtr); + + } while (!CL_IS_ROOT_PORT(portCaps)); + return iseGPUBridge; +} + +static NV_STATUS +RmGetFirmwareVersion +( + nv_state_t *pNv, + RM_API *pRmApi +) +{ + NV2080_CTRL_GSP_GET_FEATURES_PARAMS gsp_params; + NV_STATUS rmStatus = NV_OK; + pNv->cached_gpu_info.firmware_version[0] = '\0'; + + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_GSP_GET_FEATURES, &gsp_params, + sizeof(gsp_params)); + + if (rmStatus != NV_OK) + { + if (RMCFG_FEATURE_GSP_CLIENT_RM && pNv->request_firmware) + { + const char *pTmpString = "N/A"; + portStringCopy(pNv->cached_gpu_info.firmware_version, + sizeof(pNv->cached_gpu_info.firmware_version), + pTmpString, portStringLength(pTmpString) + 1); + } + NV_PRINTF(LEVEL_INFO, + "%s: Failed to query gpu firmware version, status=0x%x\n", + __FUNCTION__, rmStatus); + } + else + { + portMemCopy(pNv->cached_gpu_info.firmware_version, + sizeof(pNv->cached_gpu_info.firmware_version), + gsp_params.firmwareVersion, sizeof(gsp_params.firmwareVersion)); + } + + return NV_OK; +} + +static NV_STATUS +RmGetVbiosVersion +( + nv_state_t *pNv, + RM_API *pRmApi +) +{ + NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS *vbios_params; + const size_t vbiosStringLen = 15; // "xx.xx.xx.xx.xx" + NV_STATUS rmStatus = NV_OK; + + vbios_params = portMemAllocNonPaged(sizeof(*vbios_params)); + if (vbios_params == NULL) + { + return NV_ERR_NO_MEMORY;; + } + + portMemSet(vbios_params, 0, sizeof(*vbios_params)); + + vbios_params->biosInfoList[0].index = NV2080_CTRL_BIOS_INFO_INDEX_REVISION; + vbios_params->biosInfoList[1].index = NV2080_CTRL_BIOS_INFO_INDEX_OEM_REVISION; + vbios_params->biosInfoListSize = 2; + + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, NV2080_CTRL_CMD_BIOS_GET_INFO_V2, + vbios_params, sizeof(*vbios_params)); + + if (rmStatus == NV_OK) + { + const NvU32 biosRevision = vbios_params->biosInfoList[0].data; + const NvU32 biosOEMRevision = vbios_params->biosInfoList[1].data; + + os_snprintf(pNv->cached_gpu_info.vbios_version, vbiosStringLen, + "%02x.%02x.%02x.%02x.%02x", + (biosRevision & 0xff000000) >> 24, + (biosRevision & 0x00ff0000) >> 16, + (biosRevision & 0x0000ff00) >> 8, + (biosRevision & 0x000000ff) >> 0, + biosOEMRevision); + } + else + { + NV_PRINTF(LEVEL_INFO, + "%s: Failed to query vbios version, status=0x%x\n", + __FUNCTION__, rmStatus); + } + portMemFree(vbios_params); + + return NV_OK; +} + +/* + * Initialize the required GPU information by doing RMAPI control calls + * and store the same in the UNIX specific data structures. + */ +static NV_STATUS +RmInitGpuInfoWithRmApi +( + OBJGPU *pGpu +) +{ + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NV2080_CTRL_GPU_GET_INFO_V2_PARAMS *pGpuInfoParams = { 0 }; + NV_STATUS status; + const NvU8 *uuid; + + // LOCK: acquire GPUs lock + status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); + if (status != NV_OK) + { + return status; + } + + pGpuInfoParams = portMemAllocNonPaged(sizeof(*pGpuInfoParams)); + if (pGpuInfoParams == NULL) + { + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + return NV_ERR_NO_MEMORY; + } + + + portMemSet(pGpuInfoParams, 0, sizeof(*pGpuInfoParams)); + + pGpuInfoParams->gpuInfoListSize = 3; + pGpuInfoParams->gpuInfoList[0].index = NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED; + pGpuInfoParams->gpuInfoList[1].index = NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY; + pGpuInfoParams->gpuInfoList[2].index = NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE; + + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, + nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_GPU_GET_INFO_V2, + pGpuInfoParams, sizeof(*pGpuInfoParams)); + + if (status == NV_OK) + { + nvp->b_mobile_config_enabled = + (pGpuInfoParams->gpuInfoList[0].data == + NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_YES); + nv->dma_buf_supported = + (pGpuInfoParams->gpuInfoList[1].data == + NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES); + } + + nv->coherent = + (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) || + pGpu->pGpuArch->bGpuArchIsZeroFb); + + // + // If coherent GPU memory mode is NONE, then GPU memory has struct page + // on coherent platforms and no struct page on non-coherent ones. + // If the mode is enabled and _NUMA, then GPU memory has struct page. + // If the mode is enabled and _DRIVER, then GPU memory doesn't have struct page. + // Tegra iGPU also falls under struct page category, although + // COHERENT_GPU_MEMORY_MODE doesn't apply there. + // + if (pGpuInfoParams->gpuInfoList[2].data == + NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NONE) + { + nv->mem_has_struct_page = nv->coherent; + } + else + { + // If mode is not _NONE, we're already on a PDB_PROP_GPU_COHERENT_CPU_MAPPING platform. + nv->mem_has_struct_page = pGpu->pGpuArch->bGpuArchIsZeroFb || + (pGpuInfoParams->gpuInfoList[2].data == + NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NUMA); + } + + switch (pGpuInfoParams->gpuInfoList[2].data) + { + case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NONE: + nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_NONE; + break; + case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NUMA: + nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_NUMA; + break; + case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_DRIVER: + nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_DRIVER; + break; + } + + portMemFree(pGpuInfoParams); + + if (status != NV_OK) + { + goto finish; + } + + uuid = RmGetGpuUuidRaw(nv); + if (uuid == NULL) + { + status = NV_ERR_GPU_UUID_NOT_FOUND; + goto finish; + } + + status = RmGetFirmwareVersion(nv, pRmApi); + if (status != NV_OK) + { + goto finish; + } + + status = RmGetVbiosVersion(nv, pRmApi); + +finish: + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + return status; +} + +static void RmSetSocDispDeviceMappings( + GPUATTACHARG *gpuAttachArg, + nv_state_t *nv +) +{ + gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvAddr = (GPUHWREG*) nv->regs->map; + gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvPAddr = nv->regs->cpu_address; + gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvLength = (NvU32) nv->regs->size; +} + +static void RmSetSocDpauxDeviceMappings( + GPUATTACHARG *gpuAttachArg, + nv_state_t *nv +) +{ +} + +static void RmSetSocHdacodecDeviceMappings( + GPUATTACHARG *gpuAttachArg, + nv_state_t *nv +) +{ +} + +static void RmSetSocMipiCalDeviceMappings( + GPUATTACHARG *gpuAttachArg, + nv_state_t *nv +) +{ +} + +static void +osInitNvMapping( + nv_state_t *nv, + NvU32 *pDeviceReference, + UNIX_STATUS *status +) +{ + OBJGPU *pGpu; + OBJSYS *pSys = SYS_GET_INSTANCE(); + GPUATTACHARG *gpuAttachArg; + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 deviceInstance; + NvU32 data = 0; + NvU32 dispIsoStreamId; + NvU32 dispNisoStreamId; + + NV_PRINTF(LEVEL_INFO, "osInitNvMapping:\n"); + + // allocate the next available gpu device number + status->rmStatus = gpumgrAllocGpuInstance(pDeviceReference); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot get valid gpu instance\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); + return; + } + + // RM_BASIC_LOCK_MODEL: allocate GPU lock + status->rmStatus = rmGpuLockAlloc(*pDeviceReference); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** cannot allocate GPU lock\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); + return; + } + + // attach default single-entry broadcast device for this gpu + status->rmStatus = gpumgrCreateDevice(&deviceInstance, NVBIT(*pDeviceReference), NULL); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot attach bc gpu\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED); + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(*pDeviceReference); + return; + } + + // init attach state + gpuAttachArg = portMemAllocNonPaged(sizeof(GPUATTACHARG)); + if (gpuAttachArg == NULL) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot allocate gpuAttachArg\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(*pDeviceReference); + return; + } + + portMemSet(gpuAttachArg, 0, sizeof(GPUATTACHARG)); + + if (NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + gpuAttachArg->socDeviceArgs.specified = NV_TRUE; + + RmSetSocDispDeviceMappings(gpuAttachArg, nv); + + RmSetSocDpauxDeviceMappings(gpuAttachArg, nv); + + RmSetSocHdacodecDeviceMappings(gpuAttachArg, nv); + + RmSetSocMipiCalDeviceMappings(gpuAttachArg, nv); + + gpuAttachArg->socDeviceArgs.socChipId0 = nv->disp_sw_soc_chip_id; + + gpuAttachArg->socDeviceArgs.iovaspaceId = nv->iovaspace_id; + } + else + { + gpuAttachArg->fbPhysAddr = nv->fb->cpu_address; + gpuAttachArg->fbBaseAddr = (GPUHWREG*) 0; // not mapped + gpuAttachArg->devPhysAddr = nv->regs->cpu_address; + gpuAttachArg->regBaseAddr = (GPUHWREG*) nv->regs->map; + gpuAttachArg->intLine = 0; // don't know yet + gpuAttachArg->instPhysAddr = nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address; + gpuAttachArg->instBaseAddr = (GPUHWREG*) 0; // not mapped + + gpuAttachArg->regLength = nv->regs->size; + gpuAttachArg->fbLength = nv->fb->size; + gpuAttachArg->instLength = nv->bars[NV_GPU_BAR_INDEX_IMEM].size; + + gpuAttachArg->iovaspaceId = nv->iovaspace_id; + gpuAttachArg->cpuNumaNodeId = nv->cpu_numa_node_id; + + if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) + { + // Default - PCIe GPUs are connected via NISO IOMMU + nv->iommus.iso_iommu_present = NV_FALSE; + nv->iommus.niso_iommu_present = NV_TRUE; + } + } + + // + // we need this to check if we are running on virtual GPU + // in gpuBindHal function later. + // + if (!NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + gpuAttachArg->nvDomainBusDeviceFunc = nv_encode_pci_info(&nv->pci_info); + } + + gpuAttachArg->bRequestFwClientRm = nv->request_fw_client_rm; + + gpuAttachArg->pOsAttachArg = (void *)nv; + + // use gpu manager to attach gpu + status->rmStatus = gpumgrAttachGpu(*pDeviceReference, gpuAttachArg); + portMemFree(gpuAttachArg); + if (status->rmStatus != NV_OK) + { + gpumgrDestroyDevice(deviceInstance); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED); + NV_PRINTF(LEVEL_ERROR, "*** Cannot attach gpu\n"); + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(*pDeviceReference); + return; + } + nvp->flags |= NV_INIT_FLAG_GPUMGR_ATTACH; + + pGpu = gpumgrGetGpu(*pDeviceReference); + + sysInitRegistryOverrides(pSys); + + sysApplyLockingPolicy(pSys); + + pGpu->busInfo.IntLine = nv->interrupt_line; + + if (nv->fb != NULL) + { + pGpu->registerAccess.gpuFbAddr = (GPUHWREG*) nv->fb->map; + pGpu->busInfo.gpuPhysFbAddr = nv->fb->cpu_address; + } + + // set default parent gpu + gpumgrSetParentGPU(pGpu, pGpu); + + NV_PRINTF(LEVEL_INFO, "device instance : %d\n", *pDeviceReference); + NV_PRINTF(LEVEL_INFO, "NV regs using linear address : 0x%p\n", + pGpu->deviceMappings[SOC_DEV_MAPPING_DISP].gpuNvAddr); + NV_PRINTF(LEVEL_INFO, + "NV fb using linear address : 0x%p\n", pGpu->registerAccess.gpuFbAddr); + + if (!os_is_vgx_hyper()) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE); + } + else + { + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE); + } + } + + if ((osReadRegistryDword(NULL, + NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS, + &data) == NV_OK) && data) + { + if (data == NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_ENABLED) + { + nv->preserve_vidmem_allocations = NV_TRUE; + } + else if (data == NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_AUTO) + { + /* If the kernel supports suspend notifiers, enable video memory allocations */ + nv->preserve_vidmem_allocations = os_supports_kernel_suspend_notifiers(); + } + + if (nv->preserve_vidmem_allocations && nv_dev_needs_vidmem_preservation(nv)) + { + KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + + memmgrSetPmaForcePersistence(pMemoryManager, NV_TRUE); + + // Monolithic uses Fifolite channel which doesn't support Virtual mode + if (!pMemoryManager->bUseVirtualCopyOnSuspend) + { + pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = NV_TRUE; + } + } + } + + // Check if SMMU can be enabled on PushBuffer Aperture + nv_get_disp_smmu_stream_ids(nv, &dispIsoStreamId, &dispNisoStreamId); + if (dispNisoStreamId != NV_U32_MAX) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_FALSE); + } + else + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_TRUE); + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TRIGGER_PCIE_FLR)) + { + nv->flags |= NV_FLAG_TRIGGER_FLR; + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_SOC_SDM)) + { + nv->flags |= NV_FLAG_HAS_CONSOLE_IN_SYSMEM_CARVEOUT; + } +} + +static NV_STATUS +osInitScalability( + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + // We need PCI-E chipset information very early. + if (pCl != NULL) + { + return clInitPcie(pGpu, pCl); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +static NV_STATUS +osTeardownScalability( + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + return clTeardownPcie(pGpu, pCl); +} + +static NV_STATUS +RmInitNvHal( + nv_state_t *nv, + NvU32 deviceReference, + UNIX_STATUS *status +) +{ + OBJGPU *pGpu = gpumgrGetGpu(deviceReference); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + PORT_UNREFERENCED_VARIABLE(pGpu); + + nvp->flags |= NV_INIT_FLAG_HAL; + + nvp->flags |= NV_INIT_FLAG_HAL_COMPONENTS; + + return NV_OK; +} + +static void +RmDeterminePrimaryDevice(OBJGPU *pGpu) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + // Skip updating nv->primary_vga while RM is recovering after GPU reset + if (nv->flags & NV_FLAG_IN_RECOVERY) + { + return; + } + + // + // In case of Passthru, GPU will always be secondary + // + if (IS_PASSTHRU(pGpu)) + { + return; + } + + // + // In case of VIRTUAL GPU, there is no display, hence it will be secondary + // + if (IS_VIRTUAL(pGpu)) + { + return; + } + + // + // If GPU is driving any frame buffer console(vesafb, efifb etc) + // mark the console as client driven and GPU as Primary. + // + nv->client_managed_console = rm_get_uefi_console_status(nv); + + pGpu->setProperty(pGpu, PDB_PROP_GPU_PRIMARY_DEVICE, + (nv->client_managed_console || !!nv->primary_vga)); +} + +static void +RmSetConsolePreservationParams(OBJGPU *pGpu) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NvU64 fbBaseAddress = 0; + NvU64 fbConsoleSize = 0; + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + // + // PDB_PROP_GPU_PRIMARY_DEVICE should be NV_FALSE for vGPU configuration so + // return early + // + if (os_is_vgx_hyper() || IS_VIRTUAL(pGpu)) + return; + + if (!gpuFuseSupportsDisplay_HAL(pGpu)) + { + return; + } + + // + // If this is the zero FB SOC GPU with a console in system carveout, then + // the console memory is not mapped onto the BAR. The kernel can directly + // access the console memory, while the GPU accesses system carveout memory + // through the SMMU. + // The carveout is a reserved region of system memory that doesn't get used + // in standard memory allocation. + // + // In this scenario, you don't need to reserve console memory or BAR mapping. + // + if (NV_HAS_CONSOLE_IN_SYSMEM_CARVEOUT(nv)) + { + return; + } + + // + // Check the OS layer for any video memory used by a console + // driver that should be reserved. + // + // If the console driver is using some amount of memory, its + // mapping is presumably either pointing at the start of BAR1 + // (this is the mapping the EFI GOP driver provides for + // efifb), or the upper 16MB of BAR2 (this is the mapping the + // VBIOS provides for, e.g., vesafb). + // + // In the BAR2 case, there is nothing more to do: the upper + // 16MB of BAR2 will remain in physical mode + // (see the documentation for NV_PBUS_BAR2_BLOCK in dev_bus.ref), + // and the console driver can continue to access it. + // + // In the BAR1 case, RM will put all of BAR1 into virtual + // mode, so we need to create a BAR1 mapping for the console + // driver to use. + // + fbConsoleSize = rm_get_uefi_console_size(nv, &fbBaseAddress); + + if ((fbConsoleSize > 0) && (fbBaseAddress != 0) && (pKernelBus != NULL)) + { + pKernelBus->bPreserveBar1ConsoleEnabled = + (fbBaseAddress == nv->fb->cpu_address); + } + + // + // Case for legacy console, where OS will not give console size + // and primary_vga will be set for the device. + // Hence, reserving default size for VGA text modes. + // + if ((fbConsoleSize == 0) && nv->primary_vga) + { + fbConsoleSize = 0x40000; + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + pMemoryManager->Ram.ReservedConsoleDispMemSize = fbConsoleSize; + pMemoryManager->Ram.ReservedConsoleDispMemBase = fbBaseAddress; + } + else + { + pMemoryManager->Ram.ReservedConsoleDispMemSize = NV_ALIGN_UP(fbConsoleSize, 0x10000); + } +} + +static NV_STATUS +RmInitDeviceDma( + nv_state_t *nv +) +{ + if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) + { + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJVMM *pVmm = SYS_GET_VMM(pSys); + OBJVASPACE *pIOVAS; + NV_STATUS status = vmmCreateVaspace(pVmm, IO_VASPACE_A, + nv->iovaspace_id, 0, 0ULL, ~0ULL, + 0ULL, 0ULL, + NULL, VASPACE_FLAGS_ENABLE_VMM, + &pIOVAS); + if (status != NV_OK) + { + return status; + } + } + + return NV_OK; +} + +static void +RmTeardownDeviceDma( + nv_state_t *nv +) +{ + if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) + { + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJVMM *pVmm = SYS_GET_VMM(pSys); + OBJVASPACE *pIOVAS; + + if (NV_OK == vmmGetVaspaceFromId(pVmm, nv->iovaspace_id, IO_VASPACE_A, &pIOVAS)) + { + vmmDestroyVaspace(pVmm, pIOVAS); + } + } +} + +static void +RmEnableDeviceClks(nv_state_t *nv) +{ + NvU32 freqKHz = 0; + NvU32 i; + NV_STATUS ret; + + for (i = TEGRASOC_WHICH_CLK_GPU_FIRST; i <= TEGRASOC_WHICH_CLK_GPU_LAST; i++) + { + ret = nv_get_max_freq(nv, i, &freqKHz); + if (ret != NV_OK) + { + NV_PRINTF(LEVEL_INFO, "NVRM: Max Freq fetch failed for Clk:%d\n", i); + continue; + } + + ret = nv_enable_clk(nv, i); + if (ret != NV_OK) + { + NV_PRINTF(LEVEL_INFO, "NVRM: Clk prepare enable failed for Clk:%d\n", i); + continue; + } + + ret = nv_set_freq(nv, i, freqKHz); + if (ret != NV_OK) + { + NV_PRINTF(LEVEL_INFO, "NVRM: Set Freq failed for Clk:%d\n", i); + } + else + { + NV_PRINTF(LEVEL_INFO, "NVRM: Set Freq:%d for Clk:%d\n", freqKHz, i); + } + } +} + +static void +RmDisableDeviceClks(nv_state_t *nv) +{ + NvU32 i; + + for (i = TEGRASOC_WHICH_CLK_GPU_FIRST; i <= TEGRASOC_WHICH_CLK_GPU_LAST; i++) + { + nv_disable_clk(nv, i); + } +} + +static void +RmInitNvDevice( + NvU32 deviceReference, + UNIX_STATUS *status +) +{ + // set the device context + OBJGPU *pGpu = gpumgrGetGpu(deviceReference); + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + NV_PRINTF(LEVEL_INFO, "RmInitNvDevice:\n"); + + NV_PRINTF(LEVEL_INFO, + "device instance : 0x%08x\n", deviceReference); + + // initialize all engines -- calls back osInitMapping() + status->rmStatus = gpumgrStatePreInitGpu(pGpu); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot pre-initialize the device\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_PRE_INIT_FAILED); + return; + } + + // Configure eGPU setting + if (RmCheckForExternalGpu(pGpu, pCl)) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_EXTERNAL_GPU, NV_TRUE); + nv->is_external_gpu = NV_TRUE; + } + status->rmStatus = gpumgrStateInitGpu(pGpu); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize the device\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_STATE_INIT_FAILED); + return; + } + nvp->flags |= NV_INIT_FLAG_GPU_STATE; + + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + // + // Make sure bifIsMSIEnabled() gets a chance to look up the OS PCI + // handle for this GPU. This must happen after GPU state + // initialization. + // + if (pKernelBif != NULL) + { + kbifCheckAndRearmMSI(pGpu, pKernelBif); + } + + // Set RM's interrupt enable state to zero here so that interrupts + // won't be enabled during loading + Intr *pIntr = GPU_GET_INTR(pGpu); + if (pIntr != NULL) + { + intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); + } + + status->rmStatus = gpumgrStateLoadGpu(pGpu, GPU_STATE_DEFAULT); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "*** Cannot load state into the device\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_LOAD_FAILED); + return; + } + nvp->flags |= NV_INIT_FLAG_GPU_STATE_LOAD; + + // Setup GPU scalability + (void) RmInitScalability(pGpu); + + return; +} + +NV_STATUS osInitMapping( + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + NV_PRINTF(LEVEL_INFO, "osInitMapping:\n"); + + // Some PCI BIOSs leave the ROM mapped. This causes problems if it overlays system RAM. Just disable it. + GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_12, 0); + + // make sure our PCI latency timer is sufficient (max it out) + GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_3, + DRF_DEF(_CONFIG, _PCI_NV_3, _LATENCY_TIMER, _248_CLOCKS)); + + if (pCl != NULL) + { + initCoreLogic(pGpu); + } + nvp->flags |= NV_INIT_FLAG_CORE_LOGIC; + + return NV_OK; + +} // end of osInitMapping() + +static NV_STATUS +initCoreLogic( + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + return clInit(pGpu, pCl); +} + +static NV_STATUS +teardownCoreLogic( + OBJOS *pOS, + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + return clTeardown(pGpu, pCl); +} + +static void RmTeardownDpauxRegisters( + nv_state_t *nv +) +{ +} + +static void RmTeardownHdacodecRegisters( + nv_state_t *nv +) +{ +} + +static void RmTeardownMipiCalRegisters( + nv_state_t *nv +) +{ +} + +static NV_STATUS +RmTeardownRegisters( + nv_state_t *nv +) +{ + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Tearing down registers\n"); + + if (nv->regs && nv->regs->map) + { + osUnmapKernelSpace(nv->regs->map, nv->regs->size); + nv->regs->map = 0; + nv->regs->map_u = NULL; + } + + RmTeardownDpauxRegisters(nv); + + RmTeardownHdacodecRegisters(nv); + + RmTeardownMipiCalRegisters(nv); + + return NV_OK; +} + +static NV_STATUS +RmSetupDpauxRegisters( + nv_state_t *nv, + UNIX_STATUS *status +) +{ + + return NV_OK; +} + +static NV_STATUS +RmSetupHdacodecRegisters( + nv_state_t *nv, + UNIX_STATUS *status +) +{ + + return NV_OK; +} + +static NV_STATUS +RmSetupMipiCalRegisters( + nv_state_t *nv, + UNIX_STATUS *status +) +{ + + return NV_OK; +} + +static void +RmSetupRegisters( + nv_state_t *nv, + UNIX_STATUS *status +) +{ + NV_STATUS ret; + + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmSetupRegisters for 0x%x:0x%x\n", + nv->pci_info.vendor_id, nv->pci_info.device_id); + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "pci config info:\n"); + NV_DEV_PRINTF(NV_DBG_SETUP, nv, " registers look like: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx, + nv->regs->cpu_address, nv->regs->size); + + if (nv->fb != NULL) + { + NV_DEV_PRINTF(NV_DBG_SETUP, nv, " fb looks like: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx "\n", + nv->fb->cpu_address, nv->fb->size); + } + + nv_os_map_kernel_space(nv, nv->regs); + + if (nv->regs->map == NULL) + { + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "Failed to map regs registers!!\n"); + RM_SET_ERROR(*status, RM_INIT_REG_SETUP_FAILED); + status->rmStatus = NV_ERR_OPERATING_SYSTEM; + return; + } + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Successfully mapped framebuffer and registers\n"); + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "final mappings:\n"); + NV_DEV_PRINTF(NV_DBG_SETUP, nv, " regs: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx " 0x%p\n", + nv->regs->cpu_address, nv->regs->size, nv->regs->map); + + ret = RmSetupDpauxRegisters(nv, status); + if (ret != NV_OK) + goto err_unmap_disp_regs; + + ret = RmSetupHdacodecRegisters(nv, status); + if (ret != NV_OK) + { + RmTeardownDpauxRegisters(nv); + goto err_unmap_disp_regs; + } + + ret = RmSetupMipiCalRegisters(nv, status); + if (ret != NV_OK) + { + RmTeardownHdacodecRegisters(nv); + RmTeardownDpauxRegisters(nv); + goto err_unmap_disp_regs; + } + + return; + +err_unmap_disp_regs: + if (nv->regs && nv->regs->map) + { + osUnmapKernelSpace(nv->regs->map, nv->regs->size); + nv->regs->map = 0; + } + + return; +} + +NvBool RmInitPrivateState( + nv_state_t *pNv +) +{ + nv_priv_t *nvp; + NvU32 gpuId; + NvU32 socChipId0 = 0; + NvU32 pmc_boot_0 = 0; + NvU32 pmc_boot_1 = 0; + NvU32 pmc_boot_42 = 0; + NvU32 dmaAddrWidth = 0; + NvU64 gpuDomainBusDevice = 0; + + NV_SET_NV_PRIV(pNv, NULL); + + if (NV_IS_SOC_DISPLAY_DEVICE(pNv)) + socChipId0 = pNv->disp_sw_soc_chip_id; + else + { + pNv->regs->map_u = os_map_kernel_space(pNv->regs->cpu_address, + os_page_size, + NV_MEMORY_UNCACHED); + if (pNv->regs->map_u == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "failed to map GPU registers (DISABLE_INTERRUPTS).\n"); + return NV_FALSE; + } + + pmc_boot_0 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_0); + pmc_boot_1 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_1); + pmc_boot_42 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_42); + + os_unmap_kernel_space(pNv->regs->map_u, os_page_size); + pNv->regs->map_u = NULL; + } + + if (os_alloc_mem((void **)&nvp, sizeof(*nvp)) != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "failed to allocate private device state.\n"); + return NV_FALSE; + } + + if (!NV_IS_SOC_DISPLAY_DEVICE(pNv)) + { + gpuId = nv_generate_id_from_pci_info(&pNv->pci_info); + gpuDomainBusDevice = nv_encode_pci_info(&pNv->pci_info); + } + else + { + gpuId = NV0000_CTRL_GPU_INVALID_ID - 1; + gpuDomainBusDevice = 0; + } + + if (gpumgrRegisterGpuId(gpuId, gpuDomainBusDevice) != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "failed to register GPU with GPU manager.\n"); + os_free_mem(nvp); + return NV_FALSE; + } + + pNv->gpu_id = gpuId; + + nv_set_probed_gpu_flags(pNv); + + pNv->iovaspace_id = nv_requires_dma_remap(pNv) ? gpuId : + NV_IOVA_DOMAIN_NONE; + pNv->cpu_numa_node_id = NV0000_CTRL_NO_NUMA_NODE; + + // Get the GpuArch instance for this architecture to determine the DMA address width. + GpuArch *pGpuArch = gpumgrGetGpuArch(pmc_boot_42, socChipId0, TEGRA_CHIP_TYPE_DEFAULT); + if (pGpuArch == NULL) + { + NV_PRINTF(LEVEL_ERROR, "failed to get GpuArch for 0x%x/0x%x.\n", + pmc_boot_42, socChipId0); + gpumgrUnregisterGpuId(gpuId); + os_free_mem(nvp); + return NV_FALSE; + } + + dmaAddrWidth = gpuarchGetDmaAddrWidth(pGpuArch); + if (dmaAddrWidth == 0) + { + dmaAddrWidth = gpuarchGetSystemPhysAddrWidth(pGpuArch); + } + nv_set_dma_address_size(pNv, dmaAddrWidth); + + pNv->is_tegra_pci_igpu = !NV_IS_SOC_DISPLAY_DEVICE(pNv) && pGpuArch->bGpuArchIsZeroFb; + // Only certain Tegra PCI iGPUs support Rail-Gating + pNv->supports_tegra_igpu_rg = pNv->is_tegra_pci_igpu && pGpuArch->bGpuarchSupportsIgpuRg; + + pNv->sysmem_mapped_console = pGpuArch->bGpuArchIsZeroFb; + + kvgpumgrAttachGpu(pNv->gpu_id); + + os_mem_set(nvp, 0, sizeof(*nvp)); + nvp->status = NV_ERR_INVALID_STATE; + nvp->pmc_boot_0 = pmc_boot_0; + nvp->pmc_boot_1 = pmc_boot_1; + nvp->pmc_boot_42 = pmc_boot_42; + nvp->db_supported = -1; + NV_SET_NV_PRIV(pNv, nvp); + + return NV_TRUE; +} + +void RmClearPrivateState( + nv_state_t *pNv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); + NvU32 status; + void *pVbiosCopy = NULL; + void *pRegistryCopy = NULL; + NvU32 vbiosSize; + nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS]; + nv_dynamic_power_t dynamicPowerCopy; + NvU32 x = 0; + NvU32 pmc_boot_0, pmc_boot_1, pmc_boot_42; + NvBool pr3_acpi_method_present = 0; + int db_supported; + + // + // Do not clear private state after GPU resets, it is used while + // recovering the GPU. Only clear the pGpu pointer, which is + // restored during next initialization cycle. + // + if (pNv->flags & NV_FLAG_IN_RECOVERY) + { + nvp->pGpu = NULL; + } + + status = nvp->status; + pVbiosCopy = nvp->pVbiosCopy; + vbiosSize = nvp->vbiosSize; + pRegistryCopy = nvp->pRegistry; + dynamicPowerCopy = nvp->dynamic_power; + pmc_boot_0 = nvp->pmc_boot_0; + pmc_boot_1 = nvp->pmc_boot_1; + pmc_boot_42 = nvp->pmc_boot_42; + pr3_acpi_method_present = nvp->pr3_acpi_method_present; + db_supported = nvp->db_supported; + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + i2c_adapters[x] = nvp->i2c_adapters[x]; + } + + portMemSet(nvp, 0, sizeof(nv_priv_t)); + + nvp->status = status; + nvp->pVbiosCopy = pVbiosCopy; + nvp->vbiosSize = vbiosSize; + nvp->pRegistry = pRegistryCopy; + nvp->dynamic_power = dynamicPowerCopy; + nvp->pmc_boot_0 = pmc_boot_0; + nvp->pmc_boot_1 = pmc_boot_1; + nvp->pmc_boot_42 = pmc_boot_42; + nvp->pr3_acpi_method_present = pr3_acpi_method_present; + nvp->db_supported = db_supported; + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + nvp->i2c_adapters[x] = i2c_adapters[x]; + } + + nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C; +} + +void RmFreePrivateState( + nv_state_t *pNv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); + + gpumgrUnregisterGpuId(pNv->gpu_id); + + kvgpumgrDetachGpu(pNv->gpu_id); + + RmDestroyRegistry(pNv); + + if (nvp != NULL) + { + portMemFree(nvp->pVbiosCopy); + os_free_mem(nvp); + } + + NV_SET_NV_PRIV(pNv, NULL); +} + +NvBool RmPartiallyInitAdapter( + nv_state_t *nv +) +{ + NV_PRINTF(LEVEL_INFO, "%s: %04x:%02x:%02x.0\n", __FUNCTION__, + nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot); + + nv_start_rc_timer(nv); + + return NV_TRUE; +} + +static NV_STATUS +RmInitX86Emu( + OBJGPU *pGpu +) +{ + NV_STATUS status = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + PORT_UNREFERENCED_VARIABLE(nv); + +#if NVCPU_IS_X86_64 + status = RmInitX86EmuState(pGpu); +#else + // We don't expect a "primary VGA" adapter on non-amd64 platforms + NV_ASSERT(!NV_PRIMARY_VGA(nv)); +#endif + + return status; +} + +static NV_STATUS RmRegisterGpudb( + OBJGPU *pGpu +) +{ + NV_STATUS rmStatus; + const NvU8 *pGid; + nv_state_t *pNv = NV_GET_NV_STATE(pGpu); + + pGid = RmGetGpuUuidRaw(pNv); + if (pGid == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to get UUID\n"); + return NV_ERR_OPERATING_SYSTEM; + } + + rmStatus = gpudbRegisterGpu(pGid, &pGpu->gpuClData.upstreamPort.addr, + pGpu->busInfo.nvDomainBusDeviceFunc); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Failed to register GPU with GPU data base\n"); + } + + return rmStatus; +} + +static void RmUnixFreeRmApi( + nv_state_t *nv +) +{ + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + + if (nv->rmapi.hClient != 0) + { + pRmApi->Free(pRmApi, nv->rmapi.hClient, nv->rmapi.hClient); + } + + portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi)); + + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } +} + +static NvBool RmUnixAllocRmApi( + nv_state_t *nv, + NvU32 deviceId +) +{ + NV_STATUS status = NV_OK; + NV0080_ALLOC_PARAMETERS deviceParams = { 0 }; + NV2080_ALLOC_PARAMETERS subDeviceParams = { 0 }; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + + portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi)); + + // LOCK: acquire GPUs lock + status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); + if (status != NV_OK) + { + return status; + } + + if (pRmApi->AllocWithHandle( + pRmApi, + NV01_NULL_OBJECT, + NV01_NULL_OBJECT, + NV01_NULL_OBJECT, + NV01_ROOT, + &nv->rmapi.hClient, + sizeof(nv->rmapi.hClient)) != NV_OK) + { + goto fail; + } + + deviceParams.deviceId = deviceId; + + if (pRmApi->Alloc( + pRmApi, + nv->rmapi.hClient, + nv->rmapi.hClient, + &nv->rmapi.hDevice, + NV01_DEVICE_0, + &deviceParams, + sizeof(deviceParams)) != NV_OK) + { + goto fail; + } + + subDeviceParams.subDeviceId = 0; + + if (pRmApi->Alloc( + pRmApi, + nv->rmapi.hClient, + nv->rmapi.hDevice, + &nv->rmapi.hSubDevice, + NV20_SUBDEVICE_0, + &subDeviceParams, + sizeof(subDeviceParams)) != NV_OK) + { + goto fail; + } + + // + // The NV40_I2C allocation expected to fail, if it is disabled + // with RM config. + // + if (pRmApi->Alloc( + pRmApi, + nv->rmapi.hClient, + nv->rmapi.hSubDevice, + &nv->rmapi.hI2C, + NV40_I2C, + NULL, + 0) != NV_OK) + { + nv->rmapi.hI2C = 0; + } + + // + // The NV04_DISPLAY_COMMON allocation expected to fail for displayless + // system. nv->rmapi.hDisp value needs to be checked before doing display + // related control calls. + // + if (pRmApi->Alloc( + pRmApi, + nv->rmapi.hClient, + nv->rmapi.hDevice, + &nv->rmapi.hDisp, + NV04_DISPLAY_COMMON, + NULL, + 0) != NV_OK) + { + nv->rmapi.hDisp = 0; + } + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + return NV_TRUE; + +fail: + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + RmUnixFreeRmApi(nv); + return NV_FALSE; +} + +static NV_STATUS RmFetchGspRmImages +( + nv_state_t *nv, + GSP_FIRMWARE *pGspFw, + const void **gspFwHandle, + const void **gspFwLogHandle, + const void **gspUcodesHandle +) +{ + nv_firmware_chip_family_t chipFamily; + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 gpuArch = (decodePmcBoot42Architecture(nvp->pmc_boot_42) << + GPU_ARCH_SHIFT); + NvU32 gpuImpl = DRF_VAL(_PMC, _BOOT_42, _IMPLEMENTATION, nvp->pmc_boot_42); + + chipFamily = nv_firmware_get_chip_family(gpuArch, gpuImpl); + + portMemSet(pGspFw, 0, sizeof(*pGspFw)); + + *gspFwHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP, + chipFamily, + &pGspFw->pBuf, + &pGspFw->size); + + NvBool bFirmwareLoadSuccessful = *gspFwHandle != NULL; + + // + // GR-3428 development: Temporarily gated by regkey so this code can be + // decoupled from the build infra changes to package up ucodes.bin. + // + // Because loading the bindata ends up calling request_firmware() on Linux, + // and that function will always print an error to dmesg if the requested + // file is not found, we use this to skip trying to load it. + // + // Check will be removed once ucodes.bin becomes part of the driver package. + // + NvU32 data = 0; + if ((osReadRegistryDword(NULL, "GR3428ReadUcodesBin", &data) == NV_OK) && data) + { + *gspUcodesHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_UCODES, + chipFamily, + &pGspFw->pUcodesBin, + &pGspFw->ucodesBinSize); + bFirmwareLoadSuccessful &= *gspUcodesHandle != NULL; + } + + if (!bFirmwareLoadSuccessful && !nv->allow_fallback_to_monolithic_rm) + { + NV_PRINTF(LEVEL_ERROR, "No firmware image found\n"); + return NV_ERR_NOT_SUPPORTED; + } + else if (bFirmwareLoadSuccessful) + { +#if LIBOS_LOG_DECODE_ENABLE + if (nv->enable_firmware_logs) + { + *gspFwLogHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP_LOG, + chipFamily, + &pGspFw->pLogElf, + &pGspFw->logElfSize); + if (*gspFwLogHandle == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to load gsp_log_*.bin, no GSP-RM logs will be printed (non-fatal)\n"); + } + } +#endif + nv->request_fw_client_rm = NV_TRUE; + } + else + { + nv->request_fw_client_rm = NV_FALSE; + } + + return NV_OK; +} + +static void _checkP2pChipsetSupport( + nv_state_t *nv +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + // Check for PCIe P2P chipset support + if (!pCl->bPciePeerWriteCapable || + !pCl->bPciePeerReadCapable) + { + nv->flags |= NV_FLAG_PCI_P2P_UNSUPPORTED_CHIPSET; + } +} + +NvBool RmInitAdapter( + nv_state_t *nv +) +{ + NvU32 devicereference = 0; + UNIX_STATUS status = INIT_UNIX_STATUS; + nv_priv_t *nvp; + NvBool retVal = NV_FALSE; + OBJSYS *pSys; + OBJGPU *pGpu = NULL; + OBJOS *pOS; + KernelDisplay *pKernelDisplay; + const void *gspFwHandle = NULL; + const void *gspFwLogHandle = NULL; + const void *gspUcodesHandle = NULL; + + GSP_FIRMWARE gspFw = {0}; + PORT_UNREFERENCED_VARIABLE(gspFw); + + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter\n"); + + nv->flags &= ~NV_FLAG_PASSTHRU; + nv->flags &= ~NV_FLAG_PCI_P2P_UNSUPPORTED_CHIPSET; + + RmSetupRegisters(nv, &status); + if (! RM_INIT_SUCCESS(status.initStatus) ) + goto failed; + + nvp = NV_GET_NV_PRIV(nv); + nvp->status = NV_ERR_OPERATING_SYSTEM; + + status.rmStatus = RmInitDeviceDma(nv); + if (status.rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Cannot configure the device for DMA\n"); + RM_SET_ERROR(status, RM_INIT_GPU_DMA_CONFIGURATION_FAILED); + goto shutdown; + } + + nvp->flags |= NV_INIT_FLAG_DMA; + + pSys = SYS_GET_INSTANCE(); + + // + // Get firmware from the OS, if requested, and decide if RM will run as a + // firmware client. + // + if (nv->request_firmware) + { + if (!NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + status.rmStatus = RmFetchGspRmImages(nv, &gspFw, &gspFwHandle, &gspFwLogHandle, &gspUcodesHandle); + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_INIT_FIRMWARE_FETCH_FAILED); + goto shutdown; + } + } + else + { + nv->request_fw_client_rm = NV_TRUE; + } + } + + // + // Initialization path requires expanded GPU visibility in GPUMGR + // in order to access the GPU undergoing initialization. + // + status.rmStatus = gpumgrThreadEnableExpandedGpuVisibility(); + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_INIT_GPU_GPUMGR_EXPANDED_VISIBILITY_FAILED); + goto shutdown; + } + + + // initialize the RM device register mapping + osInitNvMapping(nv, &devicereference, &status); + if (! RM_INIT_SUCCESS(status.initStatus) ) + { + switch (status.rmStatus) + { + case NV_ERR_NOT_SUPPORTED: + nvp->status = NV_ERR_NOT_SUPPORTED; + break; + } + NV_PRINTF(LEVEL_ERROR, + "osInitNvMapping failed, bailing out of RmInitAdapter\n"); + goto shutdown; + } + + // + // now we can have a pdev for the first time... + // + pGpu = gpumgrGetGpu(devicereference); + + gpumgrSetCurrentGpuInstance(pGpu->gpuInstance); + + pOS = SYS_GET_OS(pSys); + + status.rmStatus = osInitScalability(pGpu); + if (status.rmStatus == NV_OK) + { + nvp->flags |= NV_INIT_FLAG_SCALABILITY; + } + else if (status.rmStatus != NV_ERR_NOT_SUPPORTED) + { + RM_SET_ERROR(status, RM_INIT_SCALABILITY_FAILED); + goto shutdown; + } + + // Boot GSP-RM proxy through COT command either via FSP or SEC2 + if (!IS_GSP_CLIENT(pGpu) && !IS_VIRTUAL(pGpu)) + { + status.rmStatus = gpuBootGspRmProxy(pGpu); + if (status.rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "GSP-RM proxy boot command failed.\n"); + RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); + goto shutdown; + } + } + + RmDeterminePrimaryDevice(pGpu); + + RmSetConsolePreservationParams(pGpu); + + RmInitAcpiMethods(pOS, pSys, pGpu); + + // This needs to run before GSP-RM is booted, or else it will timeout + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_CLKS_IN_TEGRA_SOC)) + { + NV_PRINTF(LEVEL_INFO, "Enable Clocks to Max\n"); + RmEnableDeviceClks(nv); + } + + // + // If GSP fw RM support is enabled then start the GSP microcode + // (including the task running the full instance of the RM) and + // exchange the necessary initial RPC messages before continuing + // with GPU initialization here. + // + if (IS_GSP_CLIENT(pGpu)) + { + status.rmStatus = kgspInitRm(pGpu, GPU_GET_KERNEL_GSP(pGpu), &gspFw); + if (status.rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Cannot initialize GSP firmware RM\n"); + RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); + goto shutdown; + } + } + else if (IS_DCE_CLIENT(pGpu)) + { + status.rmStatus = dceclientDceRmInit(pGpu, GPU_GET_DCECLIENTRM(pGpu), NV_TRUE); + if (status.rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Cannot initialize DCE firmware RM\n"); + RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); + goto shutdown; + } + } + else if (nv->request_fw_client_rm) + { + // We were expecting to enable GSP-RM but something went wrong. + if (!nv->allow_fallback_to_monolithic_rm) + { + RM_SET_ERROR(status, RM_INIT_FIRMWARE_POLICY_FAILED); + goto shutdown; + } + else + { + NV_PRINTF(LEVEL_NOTICE, "Falling back to monolithic RM\n"); + } + } + + pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + if (pKernelDisplay != NULL) + { + kdispSetWarPurgeSatellitesOnCoreFree(pKernelDisplay, NV_TRUE); + } + + if (IS_PASSTHRU(pGpu)) + nv->flags |= NV_FLAG_PASSTHRU; + + RmInitNvHal(nv, devicereference, &status); + if (!RM_INIT_SUCCESS(status.initStatus)) + { + NV_PRINTF(LEVEL_ERROR, + "RmInitNvHal() failed, bailing out of RmInitAdapter!\n"); + goto shutdown; + } + + status.rmStatus = RmInitX86Emu(pGpu); + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_INIT_VBIOS_X86EMU_FAILED); + NV_PRINTF(LEVEL_ERROR, + "RmInitX86Emu failed, bailing out of RmInitAdapter\n"); + goto shutdown; + } + + + initVendorSpecificRegistry(pGpu, nv->pci_info.device_id); + + if (!IS_VIRTUAL(pGpu) && !NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + initNbsiTable(pGpu); + } + + // finally, initialize the device + RmInitNvDevice(devicereference, &status); + if (! RM_INIT_SUCCESS(status.initStatus) ) + { + NV_PRINTF(LEVEL_ERROR, + "RmInitNvDevice failed, bailing out of RmInitAdapter\n"); + switch (status.rmStatus) + { + case NV_ERR_INSUFFICIENT_POWER: + nvp->status = NV_ERR_INSUFFICIENT_POWER; + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, + "GPU does not have the necessary power cables connected.\n"); + break; + } + goto shutdown; + } + + // + // Expanded GPU visibility in GPUMGR is no longer needed once the + // GPU is initialized. + // + gpumgrThreadDisableExpandedGpuVisibility(); + + // LOCK: acquire GPUs lock + status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_INIT); + if (status.rmStatus != NV_OK) + { + goto shutdown; + } + + status.rmStatus = osVerifySystemEnvironment(pGpu); + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_INIT_SYS_ENVIRONMENT_FAILED); + switch (status.rmStatus) + { + case NV_ERR_IRQ_NOT_FIRING: + nvp->status = NV_ERR_IRQ_NOT_FIRING; + break; + } + NV_PRINTF(LEVEL_ERROR, "osVerifySystemEnvironment failed, bailing!\n"); + goto shutdown; + } + + Intr *pIntr = GPU_GET_INTR(pGpu); + if (pIntr != NULL) + { + intrSetIntrEn(pIntr, INTERRUPT_TYPE_HARDWARE); + } + + // LOCK: acquire GPUs lock + status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_INIT); + if (status.rmStatus != NV_OK) + { + goto shutdown; + } + + KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); + // initialize the watchdog (disabled by default) + status.rmStatus = pKernelRc != NULL ? krcWatchdogInit_HAL(pGpu, pKernelRc, NULL) : + NV_ERR_NOT_SUPPORTED; + if (status.rmStatus == NV_OK) + { + krcWatchdogDisable(pKernelRc, NULL); + nvp->flags |= NV_INIT_FLAG_FIFO_WATCHDOG; + } + else if (status.rmStatus == NV_ERR_NOT_SUPPORTED) + { + NV_PRINTF(LEVEL_INFO, + "krcWatchdogInit returned _NOT_SUPPORTED. For Kepler GPUs in PGOB mode, this is normal\n"); + } + else + { + RM_SET_ERROR(status, RM_INIT_WATCHDOG_FAILED); + NV_PRINTF(LEVEL_ERROR, + "krcWatchdogInit failed, bailing out of RmInitAdapter\n"); + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + goto shutdown; + } + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + nv_start_rc_timer(nv); + + nvp->status = NV_OK; + + if (!RmUnixAllocRmApi(nv, devicereference)) { + RM_SET_ERROR(status, RM_INIT_ALLOC_RMAPI_FAILED); + status.rmStatus = NV_ERR_GENERIC; + goto shutdown; + } + + if (!NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + status.rmStatus = RmInitGpuInfoWithRmApi(pGpu); + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_INIT_GPUINFO_WITH_RMAPI_FAILED); + goto shutdown; + } + } + + // i2c only on master device?? + RmI2cAddGpuPorts(nv); + nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C; + + // This fifo hal call will fail on pre-fermi gpus. In that case, userd + // info will remain 0 and skipped by fb mmap code + KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); + NvU64 udaddr = 0; + NvU32 udsize = 0; + if (pKernelFifo != NULL) + { + status.rmStatus = kfifoGetUserdBar1MapInfo_HAL(pGpu, pKernelFifo, &udaddr, &udsize); + if (status.rmStatus == NV_OK) + { + nv->ud.cpu_address = (nv->fb->cpu_address + udaddr); + nv->ud.size = udsize; + } + else if (status.rmStatus != NV_ERR_NOT_SUPPORTED) + { + RM_SET_ERROR(status, RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED); + NV_PRINTF(LEVEL_ERROR, + "kfifoGetUserdBar1MapInfo failed, bailing out of RmInitAdapter\n"); + goto shutdown; + } + } + + nv->flags &= ~NV_FLAG_IN_RECOVERY; + + pSys->setProperty(pSys, PDB_PROP_SYS_SYSTEM_EVENTS_SUPPORTED, NV_TRUE); + + RmInitPowerManagement(nv); + + if (!NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + status.rmStatus = RmRegisterGpudb(pGpu); + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_GPUDB_REGISTER_FAILED); + goto shutdown; + } + } + + if (nvp->b_mobile_config_enabled) + { + NvU32 ac_plugged = 0; + if (nv_acpi_get_powersource(&ac_plugged) == NV_OK) + { + // LOCK: acquire GPU lock + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_NONE) == NV_OK) + { + // + // As we have already acquired the API Lock here, we are calling + // RmPowerSourceChangeEvent directly instead of rm_power_source_change_event. + // + RmPowerSourceChangeEvent(nv, !ac_plugged); + + // UNLOCK: release GPU lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + } + } + + _checkP2pChipsetSupport(nv); + + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter succeeded!\n"); + + retVal = NV_TRUE; + goto done; + + shutdown: + nv->flags &= ~NV_FLAG_IN_RECOVERY; + + gpumgrThreadDisableExpandedGpuVisibility(); + + // call ShutdownAdapter to undo anything we've done above + RmShutdownAdapter(nv); + + failed: + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "RmInitAdapter failed! (0x%x:0x%x:%d)\n", + status.initStatus, status.rmStatus, status.line); + +done: + nv_put_firmware(gspFwHandle); + nv_put_firmware(gspFwLogHandle); + nv_put_firmware(gspUcodesHandle); + + gpumgrSetCurrentGpuInstance(NV_U32_MAX); + + return retVal; +} + +void RmShutdownAdapter( + nv_state_t *nv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + NV_STATUS rmStatus; + + if ((pGpu != NULL) && (nvp->flags & NV_INIT_FLAG_GPUMGR_ATTACH)) + { + NvU32 gpuInstance = gpuGetInstance(pGpu); + NvU32 deviceInstance = gpuGetDeviceInstance(pGpu); + OBJSYS *pSys = SYS_GET_INSTANCE(); + + RmUnixFreeRmApi(nv); + + nv->ud.cpu_address = 0; + nv->ud.size = 0; + + // + // LOCK: lock all clients in case of eGPU hot unplug, which + // will not wait for all existing RM clients to stop using the GPU. + // + if (!nv->is_external_gpu || serverLockAllClients(&g_resServ) == NV_OK) + { + // LOCK: acquire GPUs lock + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + // + // Shutdown path requires expanded GPU visibility in GPUMGR in order + // to access the GPU undergoing shutdown which may not be fully + // initialized, and to continue accessing the GPU undergoing shutdown + // after state destroy. + // + NV_ASSERT_OK(gpumgrThreadEnableExpandedGpuVisibility()); + + RmDestroyPowerManagement(nv); + + freeNbsiTable(pGpu); + + OBJCL *pCl = SYS_GET_CL(pSys); + OBJOS *pOS = GPU_GET_OS(pGpu); + if (pCl != NULL) + { + if (nvp->flags & NV_INIT_FLAG_CORE_LOGIC) + { + teardownCoreLogic(pOS, pGpu); + } + else if (nvp->flags & NV_INIT_FLAG_SCALABILITY) + { + osTeardownScalability(pGpu); + } + } + + if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG) + { + krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu), NULL); + nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG; + } + + rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance)); + rmapiDelPendingDevices(NVBIT(gpuInstance)); + + if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) + { + rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT); + NV_ASSERT(rmStatus == NV_OK); + } + + if (nvp->flags & NV_INIT_FLAG_GPU_STATE) + { + rmStatus = gpuStateDestroy(pGpu); + NV_ASSERT(rmStatus == NV_OK); + } + + if (IS_DCE_CLIENT(pGpu)) + { + rmStatus = dceclientDceRmInit(pGpu, GPU_GET_DCECLIENTRM(pGpu), NV_FALSE); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "DCE firmware RM Shutdown failure\n"); + } + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_CLKS_IN_TEGRA_SOC)) + { + NV_PRINTF(LEVEL_INFO, "Disable Clocks\n"); + RmDisableDeviceClks(nv); + } + + //if (nvp->flags & NV_INIT_FLAG_HAL) + // destroyHal(pDev); + +#if NVCPU_IS_X86_64 + RmFreeX86EmuState(pGpu); +#endif + + gpumgrDetachGpu(gpuInstance); + gpumgrDestroyDevice(deviceInstance); + + // + // Expanded GPU visibility in GPUMGR is no longer needed once the + // GPU is removed from GPUMGR. + // + gpumgrThreadDisableExpandedGpuVisibility(); + + if (nvp->flags & NV_INIT_FLAG_DMA) + { + RmTeardownDeviceDma(nv); + } + + RmClearPrivateState(nv); + + RmUnInitAcpiMethods(pSys); + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(deviceInstance); + } + + // UNLOCK: unlock all clients for eGPU hot unplug path + if (nv->is_external_gpu) + serverUnlockAllClients(&g_resServ); + } + } + else + { + RmClearPrivateState(nv); + } + + RmTeardownRegisters(nv); +} + +void RmPartiallyDisableAdapter( + nv_state_t *nv +) +{ + NV_PRINTF(LEVEL_INFO, "%s: RM is in SW Persistence mode\n", __FUNCTION__); + + nv_stop_rc_timer(nv); +} + +void RmDisableAdapter( + nv_state_t *nv +) +{ + NV_STATUS rmStatus; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + NvU32 gpuMask; + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + if (pGpu == NULL) + { + return; + } + + gpumgrSetCurrentGpuInstance(pGpu->gpuInstance); + + // + // Normally, we re-enable interrupts when we release the lock + // after RmDisableAdapter(), before disabling the bottom-half ISR. + // If an interrupt came in in that time window, but the bottom half + // wasn't scheduled to run in time, the semaphore acquired by the + // top-half would never be released afterwards (since the bottom-half + // would never run). This call to intrSetIntrEn() ensures that this won't + // happen, by making sure interrupts remain disabled on the GPU in question after + // RmDisableAdapter(). + // + Intr *pIntr = GPU_GET_INTR(pGpu); + if (pIntr != NULL) + { + intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SECONDARY_BUS_RESET) || + pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_FULLCHIP_RESET)) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_TIMEOUT_RECOVERY, NV_TRUE); + nv->flags |= NV_FLAG_IN_RECOVERY; + } + + // + // LOCK: lock all clients in case of eGPU hot unplug, which + // will not wait for all existing RM clients to stop using the GPU. + // + if (!nv->is_external_gpu || serverLockAllClients(&g_resServ) == NV_OK) + { + + // LOCK: acquire GPUs lock + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + // + // Free the client allocated resources. + // + // Note this doesn't free RM internal resource allocations. Those are + // freed during gpuStateUnload. + // + gpuMask = gpumgrGetGpuMask(pGpu); + + rmapiSetDelPendingClientResourcesFromGpuMask(gpuMask); + rmapiDelPendingDevices(gpuMask); + + nv_stop_rc_timer(nv); + + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + OBJOS *pOS = SYS_GET_OS(pSys); + if (pCl != NULL) + { + teardownCoreLogic(pOS, pGpu); + nvp->flags &= ~NV_INIT_FLAG_CORE_LOGIC; + } + + if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG) + { + krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu), NULL); + nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG; + } + + if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) + { + rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT); + NV_ASSERT(rmStatus == NV_OK); + nvp->flags &= ~NV_INIT_FLAG_GPU_STATE_LOAD; + } + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + + // UNLOCK: unlock all clients for eGPU hot unplug path + if (nv->is_external_gpu) + serverUnlockAllClients(&g_resServ); + } + + gpumgrSetCurrentGpuInstance(NV_U32_MAX); +} + +NV_STATUS RmGetAdapterStatus( + nv_state_t *pNv, + NvU32 *pStatus +) +{ + // + // This status is determined in RmInitAdapter(); the glue layer + // requests it when the adapter failed to initialize to learn + // more about the error condition. This is currently limited to + // osVerifySystemEnvironment() failures. + // + nv_priv_t *nvp; + + nvp = NV_GET_NV_PRIV(pNv); + if (nvp == NULL) + { + return NV_ERR_INVALID_STATE; + } + + *pStatus = nvp->status; + return NV_OK; +} + +static void initVendorSpecificRegistry( + OBJGPU *pGpu, + NvU16 device_id +) +{ + NV_STATUS rmStatus; + NvU32 i; + NvU32 subsystem_id; + NvU32 subsystem_vendor_id; + NvU32 subsystem_device_id; + NvU32 vendor_id = 0; + + if (!pGpu) + return; + + if (pGpu->bIsSOC) + return; + + rmStatus = GPU_BUS_CFG_RD32(pGpu, + NV_CONFIG_PCI_NV_11, &subsystem_id); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: Cannot read NV_CONFIG_PCI_NV_11\n", __FUNCTION__); + return; + } + + subsystem_vendor_id = (subsystem_id & 0xffff); + subsystem_device_id = (subsystem_id >> 16); + + for (i = 0; (nb_id_table[i].subsystem_vendor_id) != 0; i++) + { + if ((nb_id_table[i].subsystem_vendor_id == subsystem_vendor_id) && + (nb_id_table[i].subsystem_device_id == subsystem_device_id) && + (nb_id_table[i].gpu_device_id == device_id)) + { + vendor_id = subsystem_vendor_id; + break; + } + } + + if (vendor_id != 0) + { + for (i = 0; nb_reg_table[i].vendor_id != 0; i++) + { + if (nb_reg_table[i].vendor_id == vendor_id) + { + osWriteRegistryDword(pGpu, nb_reg_table[i].name, + nb_reg_table[i].data); + } + } + } +} + +static void initUnixSpecificRegistry( + OBJGPU *pGpu +) +{ + // By default, enable GPU reset on Unix + osWriteRegistryDword(pGpu, "RMSecBusResetEnable", 1); + osWriteRegistryDword(pGpu, "RMForcePcieConfigSave", 1); + +} + +void +osRemoveGpu( + NvU32 domain, + NvU8 bus, + NvU8 device +) +{ + void *handle; + + handle = os_pci_init_handle(domain, bus, device, 0, NULL, NULL); + if (handle != NULL) + { + os_pci_remove(handle); + } +} + +/* + * Check to see if this board supports GPU exclusion. + */ +static NvBool RmIsExcludingAllowed( + nv_state_t *pNv +) +{ + NvU32 feature = 0; + NvU32 brand; + + // DGX-2/HGX-2 systems pre-date the PBI call + if (pNv->pci_info.device_id == 0x1db8) + return NV_TRUE; + + if (pciPbiGetFeature(pNv->handle, &feature) != NV_OK) + return NV_FALSE; + + brand = DRF_VAL(_PBI, _EXECUTE_ROUTINE, _GET_FEATURE_EXCLUSION, feature); + + return (brand == NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION_ALLOWED); +} + +NV_STATUS RmExcludeAdapter( + nv_state_t *nv +) +{ + NV_STATUS rmStatus; + + if (!RmIsExcludingAllowed(nv)) + return NV_ERR_NOT_SUPPORTED; + + rmStatus = gpumgrExcludeGpuId(nv->gpu_id); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "failed to exclude GPU: 0x%x\n", rmStatus); + return rmStatus; + } + + /* + * GPU exclusion only happens during initialization, and therefore there + * is no need to stop activity on the GPU + */ + + return rmStatus; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osmemdesc.c b/userspace/sdks/nv570/nvalloc/unix/src/osmemdesc.c new file mode 100644 index 0000000..84730cb --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/osmemdesc.c @@ -0,0 +1,1325 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2012-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/******************* OS Memory Descriptor APIS *****************************\ +* * +* This contains routines to create and destroy OS memory descriptor * +* * +****************************************************************************/ + +#include <nv.h> // NV device driver interface +#include <os/os.h> +#include <osapi.h> +#include <nvos.h> +#include <rmapi/client.h> +#include <rmapi/rs_utils.h> +#include <gpu_mgr/gpu_mgr.h> + +#include "gpu/bif/kernel_bif.h" + +static NV_STATUS osCreateOsDescriptorFromPageArray(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void **); +static void osDestroyOsDescriptorPageArray(PMEMORY_DESCRIPTOR); + +static NV_STATUS osCreateOsDescriptorFromIoMemory(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static void osDestroyOsDescriptorFromIoMemory(PMEMORY_DESCRIPTOR); + +static NV_STATUS osCreateOsDescriptorFromPhysAddr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static void osDestroyOsDescriptorFromPhysAddr(PMEMORY_DESCRIPTOR); + +static NV_STATUS osCreateOsDescriptorFromFileHandle(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static NV_STATUS osCreateOsDescriptorFromDmaBufPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static void osDestroyOsDescriptorFromDmaBuf(PMEMORY_DESCRIPTOR); +static NV_STATUS osCreateOsDescriptorFromSgtPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static void osDestroyOsDescriptorFromSgt(PMEMORY_DESCRIPTOR); + +static NV_STATUS osCheckGpuBarsOverlapAddrRange(NvRangeU64 addrRange); + +NV_STATUS +osCreateMemFromOsDescriptor +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + NvU32 descriptorType, + RS_PRIV_LEVEL privilegeLevel +) +{ + RmClient* pClient; + NV_STATUS rmStatus; + void *pPrivate; + + pClient = serverutilGetClientUnderLock(hClient); + if ((pDescriptor == NvP64_NULL) || + (*pLimit == 0) || + (pClient == NULL)) + { + return NV_ERR_INVALID_PARAM_STRUCT; + } + + // + // For the sake of simplicity, unmatched RM and OS page + // sizes are not currently supported in this path, except for + // aarch64. + // + // Also, the nvmap handle is sent which can be any random number so + // the virtual address alignment sanity check can't be done here. + // + if (!NVCPU_IS_AARCH64 && + (NV_RM_PAGE_SIZE != os_page_size)) + { + return NV_ERR_NOT_SUPPORTED; + } + + // + // The two checks below use cached privilege because they + // concern the privilege level of the client, and not the + // privilege level of the calling context which may be + // overridden to KERNEL at some internal callsites. + // + + // + // The RM cannot obtain a table of physical addresses + // for a kernel virtual address range on all of + // the supported UNIX platforms. Since this path is + // not really compelling for kernel allocations on any + // of those platforms, it is not supported. + // For UVM, they could have pre-allocated sysmem to register + // with RM so we put in an exception for that case. + // + if ((rmclientGetCachedPrivilege(pClient) >= RS_PRIV_LEVEL_KERNEL) && + (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR) && + (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE) && + (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) && + (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR)) + { + return NV_ERR_NOT_SUPPORTED; + } + + // + // NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR and + // NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR can only be utilized by kernel space + // rm-clients. + // + if ((rmclientGetCachedPrivilege(pClient) < RS_PRIV_LEVEL_KERNEL) && + ((descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) || + (descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR))) + { + return NV_ERR_NOT_SUPPORTED; + } + + switch (descriptorType) + { + case NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS: + rmStatus = NV_ERR_NOT_SUPPORTED; + break; + case NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR: + if (privilegeLevel < RS_PRIV_LEVEL_KERNEL) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + break; + } + rmStatus = osCreateOsDescriptorFromPhysAddr(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY: + rmStatus = osCreateOsDescriptorFromIoMemory(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY: + rmStatus = osCreateOsDescriptorFromPageArray(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE: + rmStatus = osCreateOsDescriptorFromFileHandle(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR: + rmStatus = osCreateOsDescriptorFromDmaBufPtr(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR: + rmStatus = osCreateOsDescriptorFromSgtPtr(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + default: + rmStatus = NV_ERR_INVALID_ARGUMENT; + break; + } + + return rmStatus; +} + +static NV_STATUS +osCreateMemdescFromPages +( + OBJGPU *pGpu, + NvU64 size, + NvU32 flags, + NvU32 cacheType, + MEMORY_DESCRIPTOR **ppMemDesc, + void *pImportPriv, + void **ppPrivate +) +{ + NV_STATUS rmStatus = NV_OK; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 memdescFlags = MEMDESC_FLAGS_NONE; + NvU32 gpuCachedFlags; + NvBool bUnprotected = NV_FALSE; + NvU64 osPageCount; + + // + // Align size up to os page size. This is important in + // order to support submemdesc mappings at native page size. + // Once dynamic tracking is enabled this will not be needed + // as all submemory will get tracked at its native page size. + // + size = NV_ALIGN_UP64(size, os_page_size); + + osPageCount = size >> os_page_shift; + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _MEMORY_PROTECTION, _UNPROTECTED, flags)) + { + memdescFlags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY; + bUnprotected = NV_TRUE; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_USER_READ_ONLY; + memdescFlags |= MEMDESC_FLAGS_DEVICE_READ_ONLY; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_DEVICE_READ_ONLY; + } + + rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0, + NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM, + cacheType, memdescFlags); + if (rmStatus != NV_OK) + { + return rmStatus; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags)) + gpuCachedFlags = NV_MEMORY_CACHED; + else + gpuCachedFlags = NV_MEMORY_UNCACHED; + + pMemDesc = *ppMemDesc; + rmStatus = nv_register_user_pages(NV_GET_NV_STATE(pGpu), + osPageCount, + memdescGetPteArray(pMemDesc, AT_CPU), pImportPriv, + ppPrivate, bUnprotected); + if (rmStatus != NV_OK) + { + memdescDestroy(pMemDesc); + return rmStatus; + } + + memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE); + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NV_ASSERT_OR_GOTO((os_page_size & (pMemDesc->pageArrayGranularity - 1ULL)) == 0, cleanup); + NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, os_page_size), cleanup); + } + else + { + NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, NV_RM_PAGE_SIZE), cleanup); + } + + if (!NV_IS_ALIGNED64(memdescGetPhysAddr(pMemDesc, AT_CPU, 0), os_page_size)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + NV_ASSERT_OR_GOTO(0, cleanup); + } + + + // + // If the OS layer doesn't think in RM page size, we need to inflate the + // PTE array into RM pages. + // + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion. + NV_ASSERT_OR_ELSE(os_page_size == pMemDesc->pageArrayGranularity, rmStatus = NV_ERR_INVALID_ARGUMENT; goto cleanup); + } + else if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU), pMemDesc->PageCount); + } + + // + // memdescMapIommu() requires the OS-private data to be set on the memory + // descriptor, but we don't want to wire up the teardown callback just yet: + // that callback needs to unpin the pages, but that will already be done + // as part of failure handling further up the stack if memdescMapIommu() + // fails. So we only set up the priv-data cleanup callback once we're sure + // this call will succeed. + // + memdescSetMemData(pMemDesc, *ppPrivate, NULL); + + rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); +cleanup: + if (rmStatus != NV_OK) + { + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion. + NV_ASSERT(os_page_size == pMemDesc->pageArrayGranularity); + } + else if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + nv_unregister_user_pages(NV_GET_NV_STATE(pGpu), + osPageCount, + NULL /* import_priv */, ppPrivate); + memdescDestroy(pMemDesc); + return rmStatus; + } + + return NV_OK; +} + +static NV_STATUS +osCreateOsDescriptorFromPageArray +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus; + + *ppPrivate = NvP64_VALUE(pDescriptor); + + // + // Since the only type of memory permitted in this path + // is anonymous user memory, certain restrictions + // apply for the allocation flags: + // + // 1) anonymous memory is write-back cacheable, hence + // the _COHERENCY flag must match. + // + // 2) the RM has no control over the location of the + // associated pages in memory and thus cannot + // honor requests for contiguous memory. + // + // These restrictions are enforced here to avoid subtle + // bugs later on. + // + if ((!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, flags) && + !FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, flags)) || + FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags)) + { + return NV_ERR_INVALID_FLAGS; + } + + rmStatus = osCreateMemdescFromPages(pGpu, (*pLimit + 1), flags, + NV_MEMORY_CACHED, ppMemDesc, + NULL /* pImportPriv */, ppPrivate); + if (rmStatus != NV_OK) + { + return rmStatus; + } + + // All is well - wire up the cleanup callback now + memdescSetMemData(*ppMemDesc, memdescGetMemData(*ppMemDesc), + osDestroyOsDescriptorPageArray); + + return NV_OK; +} + +/*! + * @brief Checks if the given address range overlaps with the BARs for any of + * the GPUs. + */ +static NV_STATUS +osCheckGpuBarsOverlapAddrRange +( + NvRangeU64 addrRange +) +{ + NvRangeU64 gpuPhysAddrRange; + NvRangeU64 gpuPhysFbAddrRange; + NvRangeU64 gpuPhysInstAddrRange; + NvU32 gpuInstance; + OBJGPU *pGpu; + NvU32 gpuMask; + NV_STATUS rmStatus; + + rmStatus = gpumgrGetGpuAttachInfo(NULL, &gpuMask); + NV_ASSERT_OR_RETURN(rmStatus == NV_OK, rmStatus); + + gpuInstance = 0; + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + NV_INIT_RANGE(gpuPhysAddrRange, pGpu->busInfo.gpuPhysAddr, + pGpu->busInfo.gpuPhysAddr + pGpu->deviceMappings[0].gpuNvLength -1); + + if (NV_IS_OVERLAPPING_RANGE(gpuPhysAddrRange, addrRange)) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): phys range 0x%016llx-0x%016llx overlaps with GPU BAR0\n", + __FUNCTION__, addrRange.min, addrRange.max); + return NV_ERR_INVALID_ADDRESS; + } + + if ((pGpu->pGpuArch->bGpuArchIsZeroFb) || + (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))) + { + // No local FB or BAR2 for ZERO_FB and TEGRA_SOC_NVDISPLAY. + continue; + } + + NV_INIT_RANGE(gpuPhysFbAddrRange, gpumgrGetGpuPhysFbAddr(pGpu), + gpumgrGetGpuPhysFbAddr(pGpu) + pGpu->fbLength -1); + + NV_INIT_RANGE(gpuPhysInstAddrRange, pGpu->busInfo.gpuPhysInstAddr, + pGpu->busInfo.gpuPhysInstAddr + pGpu->instLength -1); + + if (NV_IS_OVERLAPPING_RANGE(gpuPhysFbAddrRange, addrRange) || + NV_IS_OVERLAPPING_RANGE(gpuPhysInstAddrRange, addrRange)) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): phys range 0x%016llx-0x%016llx overlaps with FB or GPU BAR2\n", + __FUNCTION__, addrRange.min, addrRange.max); + return NV_ERR_INVALID_ADDRESS; + } + } + + return NV_OK; +} + +static NvU64 +_doWarBug4040336 +( + OBJGPU *pGpu, + NvU64 addr +) +{ + if (gpuIsWarBug4040336Enabled(pGpu)) + { + if ((addr & 0xffffffff00000000ULL) == 0x7fff00000000ULL) + { + addr = addr & 0xffffffffULL; + } + } + return addr; +} + +static NV_STATUS +osCreateOsDescriptorFromIoMemory +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus; + NvU32 gpuCachedFlags; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 *pPteArray; + NvRangeU64 physAddrRange; + NvU64 *base = 0; + NvBool bAllowMmap; + NvU64 size; + + // + // Unlike the page array path, this one deals exclusively + // with I/O memory, which is expected to be contiguous + // physically, and which may only be accessed with uncached + // transactions. + // + if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags) || + !FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags)) + { + return NV_ERR_INVALID_FLAGS; + } + + // + // _PEER_MAP_OVERRIDE flag is controlled by the RM and not the client. + // + // RM will set the _PEER_MAP_OVERRIDE_REQUIRED flag itself for IO memory + // memory imported with RmVidHeapControl. + // + if (FLD_TEST_DRF(OS02, _FLAGS, _PEER_MAP_OVERRIDE, _REQUIRED, flags)) + { + // + // Don't allow MMIO mappings for unprivileged users + // This is a temporary WAR for bug 1630288 "[PeerSync] threat related + // to GPU" + // + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + if (!pKernelBif->peerMappingOverride && !osIsAdministrator()) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): permission denied, allowPeermapping=%d\n", + __FUNCTION__, pKernelBif->peerMappingOverride); + return NV_ERR_INSUFFICIENT_PERMISSIONS; + } + } + + bAllowMmap = !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NEVER_MAP, flags); + + base = (void *)(NvUPtr)pDescriptor; + + // + // There is an architectural deadlock scenario involved when full-duplex P2P + // enabled over BAR1. See #3 in the description of bug 1571948 which explains + // the classic deadlock. So, make sure to error out usermode's memory + // registration if a memory range falls within any of the available GPU's + // BAR window. + // + physAddrRange.min = *base; + physAddrRange.max = *base + *pLimit; + + rmStatus = osCheckGpuBarsOverlapAddrRange(physAddrRange); + if (rmStatus != NV_OK) + { + return rmStatus; + } + + // + // BF3's PCIe MMIO bus address at 0x800000000000(CPU PA 0x7fff00000000) is + // too high for Ampere to address. As a result, BF3's bus address is + // moved to < 4GB. Now, the CPU PA and the bus address are no longer 1:1 + // and needs to be adjusted. + // + *base = _doWarBug4040336(pGpu, *base); + + size = NV_ALIGN_UP64(*pLimit + 1, os_page_size); + + rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0, + NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM, + NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): error %d while attempting to create the MMIO mapping\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + pMemDesc = *ppMemDesc; + + if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags)) + gpuCachedFlags = NV_MEMORY_CACHED; + else + gpuCachedFlags = NV_MEMORY_UNCACHED; + + memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags); + memdescSetMemData(pMemDesc, NULL, NULL); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM, NV_TRUE); + + pPteArray = memdescGetPteArray(pMemDesc, AT_CPU); + pPteArray[0] = *base; + + *ppPrivate = NULL; + + if (!NV_IS_ALIGNED64(pPteArray[0], os_page_size)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + NV_ASSERT_OR_GOTO(0, cleanup); + } + NvU64 osPageCount; + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, os_page_size), cleanup); + osPageCount = pMemDesc->PageCount; + } + else + { + NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, NV_RM_PAGE_SIZE), cleanup); + osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount); + } + + if (bAllowMmap) + { + rmStatus = nv_register_peer_io_mem(NV_GET_NV_STATE(pGpu), pPteArray, + osPageCount, + ppPrivate); + if (rmStatus != NV_OK) + { + memdescDestroy(pMemDesc); + return rmStatus; + } + } + + memdescSetMemData(pMemDesc, *ppPrivate, NULL); + + // + // memdescMapIommu() requires the OS-private data to be set on the memory + // descriptor, but we don't want to wire up the teardown callback just yet: + // that callback needs to unpin the pages, but that will already be done + // as part of failure handling further up the stack if memdescMapIommu() + // fails. So we only set up the priv-data cleanup callback once we're sure + // this call will succeed. + // + rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); +cleanup: + if (rmStatus != NV_OK) + { + if (*ppPrivate != NULL) + { + nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), *ppPrivate); + } + memdescDestroy(pMemDesc); + return rmStatus; + } + + // All is well - wire up the cleanup callback now + memdescSetMemData(pMemDesc, memdescGetMemData(pMemDesc), + osDestroyOsDescriptorFromIoMemory); + + return NV_OK; +} + +static NV_STATUS +osCreateOsDescriptorFromPhysAddr +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 *pPteArray; + NvU64 base = 0; + NvU32 cache_type = NV_MEMORY_CACHED; + NvU64 memdescFlags = MEMDESC_FLAGS_NONE; + NvU64 *pPhys_addrs; + NvU64 num_os_pages; + NvU32 idx; + NvU64 size; + + // Currently only work with contiguous sysmem allocations + if (!FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags)) + { + return NV_ERR_INVALID_FLAGS; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_TYPE_SYNCPOINT, _APERTURE, flags)) + { + // Syncpoint memory is uncached, DMA mapping needs to skip CPU sync. + cache_type = NV_MEMORY_UNCACHED; + + // + // Syncpoint memory is NISO. Don't attempt to IOMMU map if the NISO + // IOMMU isn't enabled. + // + if (!NV_SOC_IS_NISO_IOMMU_PRESENT(nv)) + { + memdescFlags |= MEMDESC_FLAGS_SKIP_IOMMU_MAPPING; + } + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO; + } + + base = (NvU64)pDescriptor; + size = NV_ALIGN_UP64((*pLimit + 1), os_page_size); + + NV_ASSERT_OR_RETURN(NV_IS_ALIGNED64(base, os_page_size), NV_ERR_INVALID_ARGUMENT); + + rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0, + NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM, + cache_type, memdescFlags); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): error %d while creating memdesc for kernel memory\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + pMemDesc = *ppMemDesc; + + memdescSetMemData(pMemDesc, NULL, NULL); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE); + + pPteArray = memdescGetPteArray(pMemDesc, AT_CPU); + pPteArray[0] = base; + + num_os_pages = size >> os_page_shift; + pPhys_addrs = portMemAllocNonPaged(sizeof(NvU64) * num_os_pages); + if (pPhys_addrs == NULL) + goto cleanup_memdesc; + + for (idx = 0; idx < num_os_pages; idx++) + { + pPhys_addrs[idx] = base + (idx * os_page_size); + } + + *ppPrivate = NULL; + rmStatus = nv_register_phys_pages(nv, pPhys_addrs, num_os_pages, + memdescGetCpuCacheAttrib(pMemDesc), + ppPrivate); + if (rmStatus != NV_OK) + goto cleanup_memdesc; + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, size, os_page_size)); + } + else + { + NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, size, NV_RM_PAGE_SIZE)); + } + + // If IOMMU skip flag wasn't set earlier, create IOVA mapping. + if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_SKIP_IOMMU_MAPPING)) + { + // + // memdescMapIommu() requires the OS-private data to be set on the memory + // descriptor, but we don't want to wire up the teardown callback just yet: + // that callback needs to unpin the pages, but that will already be done + // as part of failure handling further up the stack if memdescMapIommu() + // fails. So we only set up the priv-data cleanup callback once we're sure + // this call will succeed. + // + memdescSetMemData(pMemDesc, *ppPrivate, NULL); + + rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); + if (rmStatus != NV_OK) + goto cleanup_pages; + } + + // All is well - wire up the cleanup callback now + memdescSetMemData(pMemDesc, *ppPrivate, + osDestroyOsDescriptorFromPhysAddr); + + portMemFree(pPhys_addrs); + + return NV_OK; + +cleanup_pages: + if (*ppPrivate != NULL) + { + nv_unregister_phys_pages(NV_GET_NV_STATE(pGpu), *ppPrivate); + } + +cleanup_memdesc: + memdescDestroy(pMemDesc); + + portMemFree(pPhys_addrs); + + return rmStatus; +} + +static NV_STATUS +_createMemdescFromDmaBufSgtHelper +( + OBJGPU *pGpu, + NvU32 flags, + void *pImportPriv, + struct sg_table *pImportSgt, + NvU32 size, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate, + MEM_DATA_RELEASE_CALL_BACK *pMemDataReleaseCallback +) +{ + NV_STATUS rmStatus = NV_OK; + NvU32 cacheType = NV_MEMORY_UNCACHED; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 memdescFlags = MEMDESC_FLAGS_NONE; + NvU32 gpuCachedFlags; + NvBool isPeerMmio = NV_FALSE; + NvU64 osPageCount; + + // + // Align size up to os page size. This is important in + // order to support submemdesc mappings at native page size. + // Once dynamic tracking is enabled this will not be needed + // as all submemor + // + size = NV_ALIGN_UP64(size, os_page_size); + + osPageCount = size >> os_page_shift; + + NV_ASSERT((pMemDataReleaseCallback == osDestroyOsDescriptorFromDmaBuf) || + (pMemDataReleaseCallback == osDestroyOsDescriptorFromSgt)); + + if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_COMBINE, flags)) + { + cacheType = NV_MEMORY_WRITECOMBINED; + } + else if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags)) + { + cacheType = NV_MEMORY_CACHED; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags)) + { + gpuCachedFlags = NV_MEMORY_CACHED; + } + else + { + gpuCachedFlags = NV_MEMORY_UNCACHED; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_TYPE_SYNCPOINT, _APERTURE, flags)) + { + // Syncpoint memory is uncached. + if ((cacheType != NV_MEMORY_UNCACHED) || + (gpuCachedFlags != NV_MEMORY_UNCACHED)) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): Error: Syncpoint memory region should be uncached!!!\n", + __FUNCTION__); + return NV_ERR_INVALID_FLAGS; + } + + isPeerMmio = NV_TRUE; + + NV_PRINTF(LEVEL_INFO, + "%s(): Syncpoint type sgt!\n", __FUNCTION__); + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_USER_READ_ONLY; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_DEVICE_READ_ONLY; + } + + rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0, + NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM, + cacheType, memdescFlags); + if (rmStatus != NV_OK) + { + return rmStatus; + } + + pMemDesc = *ppMemDesc; + + memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags); + memdescSetMemData(pMemDesc, NULL, NULL); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE); + + *ppPrivate = NULL; + rmStatus = nv_register_sgt(NV_GET_NV_STATE(pGpu), + memdescGetPteArray(pMemDesc, AT_CPU), + osPageCount, + memdescGetCpuCacheAttrib(pMemDesc), + ppPrivate, + pImportSgt, + pImportPriv, + isPeerMmio); + if (rmStatus != NV_OK) + { + memdescDestroy(pMemDesc); + return rmStatus; + } + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, os_page_size), cleanup); + } + else + { + NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, NV_RM_PAGE_SIZE), cleanup); + } + + if (!NV_IS_ALIGNED64(memdescGetPhysAddr(pMemDesc, AT_CPU, 0), os_page_size)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + NV_ASSERT_OR_GOTO(0, cleanup); + } + + // + // If the OS layer doesn't think in RM page size, we need to inflate the + // PTE array into RM pages. + // + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion. + NV_ASSERT_OR_ELSE(os_page_size == pMemDesc->pageArrayGranularity, rmStatus = NV_ERR_INVALID_ARGUMENT; goto cleanup); + } + else if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU), pMemDesc->PageCount); + } + + memdescSetMemData(*ppMemDesc, *ppPrivate, NULL); + + // + // memdescMapIommu() requires the OS-private data to be set on the memory + // descriptor, but we don't want to wire up the teardown callback just yet: + // that callback does teardown that will already be done as part of failure + // handling further up the stack if memdescMapIommu() fails. So we only + // setup the priv-data cleanup callback once we're sure this call will + // succeed. + // + rmStatus = memdescMapIommu(*ppMemDesc, pGpu->busInfo.iovaspaceId); +cleanup: + if (rmStatus != NV_OK) + { + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion. + NV_ASSERT(os_page_size == pMemDesc->pageArrayGranularity); + } + else if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + if (*ppPrivate != NULL) + { + nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt, + (void **) &pImportPriv, *ppPrivate); + } + memdescDestroy(pMemDesc); + return rmStatus; + } + + // All is well - wire up the cleanup callback now + memdescSetMemData(*ppMemDesc, *ppPrivate, pMemDataReleaseCallback); + + return rmStatus; +} + +static NV_STATUS +_createMemdescFromDmaBuf +( + OBJGPU *pGpu, + NvU32 flags, + nv_dma_buf_t *pImportPriv, + struct sg_table *pImportSgt, + NvU32 size, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = + _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPriv, pImportSgt, + size, ppMemDesc, ppPrivate, + osDestroyOsDescriptorFromDmaBuf); + if (rmStatus != NV_OK) + { + nv_dma_release_dma_buf(pImportPriv); + } + + return rmStatus; +} + +static NV_STATUS +_createMemdescFromSgt +( + OBJGPU *pGpu, + NvU32 flags, + struct drm_gem_object *pImportPrivGem, + struct sg_table *pImportSgt, + NvU32 size, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = + _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPrivGem, + pImportSgt, size, ppMemDesc, + ppPrivate, + osDestroyOsDescriptorFromSgt); + if (rmStatus != NV_OK) + { + nv_dma_release_sgt(pImportSgt, pImportPrivGem); + } + + return rmStatus; +} + +static nv_dma_device_t *GetDmaDeviceForImport +( + nv_state_t *nv, + NvU32 flags +) +{ + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags) && + (nv->niso_dma_dev != NULL)) + { + return nv->niso_dma_dev; + } + else + { + return nv->dma_dev; + } +} + +static NV_STATUS +osCreateOsDescriptorFromFileHandle +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_dma_device_t *dma_dev = NULL; + NvU32 size = 0; + nv_dma_buf_t *pImportPriv = NULL; + struct sg_table *pImportSgt = NULL; + NvBool bRoDeviceMap = NV_FALSE; + NvS32 fd; + + fd = (NvS32)((NvU64)pDescriptor); + if ((NvU64)fd != (NvU64)pDescriptor) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): fd must fit within a signed 32-bit integer!\n", + __FUNCTION__); + return NV_ERR_INVALID_ARGUMENT; + } + + dma_dev = GetDmaDeviceForImport(nv, flags); + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, _YES, flags)) + { + bRoDeviceMap = NV_TRUE; + NV_PRINTF(LEVEL_INFO, + "%s(): RO DMA Mapping - flags [%x]!\n", + __FUNCTION__, flags); + } + + rmStatus = nv_dma_import_from_fd(dma_dev, fd, bRoDeviceMap, &size, + &pImportSgt, &pImportPriv); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): Error (%d) while trying to import fd!\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv, + pImportSgt, + size, ppMemDesc, ppPrivate); +} + +static NV_STATUS +osCreateOsDescriptorFromSgtPtr +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS *params = + (NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS*)((NvUPtr) pDescriptor); + + struct sg_table *sgt = params->sgt; + struct drm_gem_object *gem = params->gem; + + rmStatus = nv_dma_import_sgt(nv->dma_dev, sgt, gem); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): Error (%d) while trying to import sgt!\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + return _createMemdescFromSgt(pGpu, flags, gem, sgt, + (*pLimit + 1), ppMemDesc, ppPrivate); +} + +static NV_STATUS +osCreateOsDescriptorFromDmaBufPtr +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_dma_device_t *dma_dev = NULL; + NvU32 size = 0; + nv_dma_buf_t *pImportPriv = NULL; + struct sg_table *pImportSgt = NULL; + void *dmaBuf = (void*)((NvUPtr)pDescriptor); + NvBool bRoDeviceMap = NV_FALSE; + + dma_dev = GetDmaDeviceForImport(nv, flags); + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, _YES, flags)) + { + bRoDeviceMap = NV_TRUE; + NV_PRINTF(LEVEL_INFO, + "%s(): RO DMA Mapping - flags [%x]!\n", + __FUNCTION__, flags); + } + + rmStatus = nv_dma_import_dma_buf(dma_dev, dmaBuf, bRoDeviceMap, &size, + &pImportSgt, &pImportPriv); + if (rmStatus != NV_OK) + { + NV_PRINTF_COND(rmStatus == NV_ERR_NOT_SUPPORTED, LEVEL_INFO, LEVEL_ERROR, + "Error (%d) while trying to import dma_buf!\n", rmStatus); + return rmStatus; + } + + return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv, + pImportSgt, + size, ppMemDesc, ppPrivate); +} + +static void +osDestroyOsDescriptorFromPhysAddr +( + PMEMORY_DESCRIPTOR pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + void *pPrivate; + + pPrivate = memdescGetMemData(pMemDesc); + NV_ASSERT(pPrivate != NULL); + + if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_SKIP_IOMMU_MAPPING)) + memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); + + nv_unregister_phys_pages(NV_GET_NV_STATE(pGpu), pPrivate); +} + +static void +osDestroyOsDescriptorFromIoMemory +( + PMEMORY_DESCRIPTOR pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + void *pPrivate = memdescGetMemData(pMemDesc); + + if (pPrivate == NULL) + { + return; + } + + nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), pPrivate); +} + +static void +osDestroyOsDescriptorPageArray +( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + NvU64 osPageCount; + NV_STATUS status; + void *pPrivate; + + pPrivate = memdescGetMemData(pMemDesc); + + NV_ASSERT(pPrivate != NULL); + + // + // TODO: Bug 1811006: Notably skip any IOMMU mapping management as the + // pMemDesc->pGpu might have been torn down already and the pGpu passed in + // doesn't necessarily have IOMMU mappings. For now just allow memdescDestroy() + // to clean up whatever is there (this may not work correctly either if any + // of the IOMMU mappings have outlasted their VASPACEs). This should + // be cleaned up once the fix for bug 1811006 is known. + // + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion. + NV_ASSERT_OR_RETURN_VOID(os_page_size == pMemDesc->pageArrayGranularity); + osPageCount = pMemDesc->PageCount; + } + else + { + osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount); + if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + } + + nv_unregister_user_pages(NV_GET_NV_STATE(pGpu), osPageCount, + NULL /* import_priv */, &pPrivate); + + if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_FOREIGN_PAGE)) + { + os_free_mem(pPrivate); + } + else + { + // + // We use MEMDESC_FLAGS_USER_READ_ONLY because this reflects the + // NVOS02_FLAGS_ALLOC_USER_READ_ONLY flag value passed into + // os_lock_user_pages(). That flag also results in + // MEMDESC_FLAGS_DEVICE_READ_ONLY being set. + // + NvBool writable = !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY); + NvU32 flags = DRF_NUM(_LOCK_USER_PAGES, _FLAGS, _WRITE, writable); + status = os_unlock_user_pages(osPageCount, pPrivate, flags); + NV_ASSERT(status == NV_OK); + } +} + +static void +osDestroyOsDescriptorFromDmaBuf +( + PMEMORY_DESCRIPTOR pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + void *pPrivate = memdescGetMemData(pMemDesc); + + struct sg_table *pImportSgt; + void *pImportPriv; + + /* + * Unmap IOMMU now or we will get a kernel crash when it is unmapped after + * pImportSgt is freed. + */ + memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion. + NV_ASSERT_OR_RETURN_VOID(os_page_size == pMemDesc->pageArrayGranularity); + } + else if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt, + &pImportPriv, pPrivate); + + /* + * pImportSgt doesn't need to be passed to nv_dma_release_dma_buf() because + * the DMA-BUF associated with pImportPriv already has a reference to the + * SGT. + */ + + nv_dma_release_dma_buf(pImportPriv); +} + +static void +osDestroyOsDescriptorFromSgt +( + PMEMORY_DESCRIPTOR pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + void *pPrivate = memdescGetMemData(pMemDesc); + + struct sg_table *pImportSgt; + struct drm_gem_object *pImportPrivGem; + + NV_ASSERT(pPrivate != NULL); + + /* + * Unmap IOMMU now or we will get a kernel crash when it is unmapped after + * pImportSgt is freed. + */ + memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); + + if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) + { + // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion. + NV_ASSERT_OR_RETURN_VOID(os_page_size == pMemDesc->pageArrayGranularity); + } + else if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt, + (void **) &pImportPrivGem, pPrivate); + + nv_dma_release_sgt(pImportSgt, pImportPrivGem); +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osnvlink.c b/userspace/sdks/nv570/nvalloc/unix/src/osnvlink.c new file mode 100644 index 0000000..af65f9e --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/osnvlink.c @@ -0,0 +1,783 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2015-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include "kernel/gpu/nvlink/kernel_nvlink.h" +#include "osapi.h" + +#if defined(INCLUDE_NVLINK_LIB) +#include "nvlink.h" +// +// The functions in this file are a workaround for a significant design flaw +// where RM callbacks for the nvlink library are built with the altstack +// enabled, but the nvlink library is not built with altstack support. Whenever +// the library calls a callback, the stack switching needs to be accounted for +// or else we will observe corruption of data structures in the nvlink library +// as data is pushed onto what the callback thinks is the stack. See bug +// 1710300. +// +// This bug has also exposed other problems, such as the complete lack of +// locking awareness by these callbacks (e.g., assumption that the RMAPI and +// GPU locks are always held on entry, which is not a legitimate assumption). +// For now, we ignore that just to unblock testing. +// +extern NvlStatus knvlinkCoreAddLinkCallback(struct nvlink_link *); +extern NvlStatus knvlinkCoreRemoveLinkCallback(struct nvlink_link *); +extern NvlStatus knvlinkCoreLockLinkCallback(struct nvlink_link *); +extern void knvlinkCoreUnlockLinkCallback(struct nvlink_link *); +extern NvlStatus knvlinkCoreQueueLinkChangeCallback(struct nvlink_link_change *); +extern NvlStatus knvlinkCoreSetDlLinkModeCallback(struct nvlink_link *, NvU64, NvU32); +extern NvlStatus knvlinkCoreGetDlLinkModeCallback(struct nvlink_link *, NvU64 *); +extern NvlStatus knvlinkCoreSetTlLinkModeCallback(struct nvlink_link *, NvU64, NvU32); +extern NvlStatus knvlinkCoreGetTlLinkModeCallback(struct nvlink_link *, NvU64 *); +extern NvlStatus knvlinkCoreGetTxSublinkModeCallback(struct nvlink_link *, NvU64 *, NvU32 *); +extern NvlStatus knvlinkCoreSetTxSublinkModeCallback(struct nvlink_link *, NvU64, NvU32); +extern NvlStatus knvlinkCoreGetRxSublinkModeCallback(struct nvlink_link *, NvU64 *, NvU32 *); +extern NvlStatus knvlinkCoreSetRxSublinkModeCallback(struct nvlink_link *, NvU64, NvU32); +extern NvlStatus knvlinkCoreReadDiscoveryTokenCallback(struct nvlink_link *, NvU64 *); +extern NvlStatus knvlinkCoreWriteDiscoveryTokenCallback(struct nvlink_link *, NvU64); +extern void knvlinkCoreTrainingCompleteCallback(struct nvlink_link *); +extern void knvlinkCoreGetUphyLoadCallback(struct nvlink_link *, NvBool*); +extern NvlStatus knvlinkCoreGetCciLinkModeCallback(struct nvlink_link *, NvU64 *); + +/*! + * @brief Helper to allocate an alternate stack from within core RM. + * + * This needs to be an NV_API_CALL (built to use the original stack instead + * of the altstack) since it is called before we switch to using the altstack. + */ +static NV_STATUS NV_API_CALL osNvlinkAllocAltStack(nvidia_stack_t **pSp) +{ + NV_STATUS status = NV_OK; + nvidia_stack_t *sp = NULL; +#if defined(NVCPU_X86_64) && defined(__use_altstack__) + status = os_alloc_mem((void **)&sp, sizeof(nvidia_stack_t)); + if (status == NV_OK) + { + sp->size = sizeof(sp->stack); + sp->top = sp->stack + sp->size; + } +#endif + *pSp = sp; + return status; +} + +/*! + * @brief Helper to free an alternate stack from within core RM. + * + * This needs to be an NV_API_CALL (built to use the original stack instead + * of the altstack) since it is called after we've switched back to using the + * original stack. + */ +static void NV_API_CALL osNvlinkFreeAltStack(nvidia_stack_t *sp) +{ +#if defined(NVCPU_X86_64) && defined(__use_altstack__) + os_free_mem(sp); +#endif +} + +static NV_STATUS NV_API_CALL osNvlinkGetAltStack(nvidia_stack_t **sp) +{ + return osNvlinkAllocAltStack(sp); +} +static void NV_API_CALL osNvlinkPutAltStack(nvidia_stack_t *sp) +{ + osNvlinkFreeAltStack(sp); +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_add_link +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + status = knvlinkCoreAddLinkCallback(link); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_remove_link +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + status = knvlinkCoreRemoveLinkCallback(link); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_lock_link +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreLockLinkCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static void NV_API_CALL rm_nvlink_ops_unlock_link +( + struct nvlink_link *link +) +{ + void *fp; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + knvlinkCoreUnlockLinkCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_queue_link_change +( + struct nvlink_link_change *link_change +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreQueueLinkChangeCallback(link_change); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_dl_link_mode +( + struct nvlink_link *link, + NvU64 mode, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetDlLinkModeCallback(link, mode, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_dl_link_mode +( + struct nvlink_link *link, + NvU64 *mode +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetDlLinkModeCallback(link, mode); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_tl_link_mode +( + struct nvlink_link *link, + NvU64 mode, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetTlLinkModeCallback(link, mode, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_tl_link_mode +( + struct nvlink_link *link, + NvU64 *mode +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetTlLinkModeCallback(link, mode); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_tx_mode +( + struct nvlink_link *link, + NvU64 mode, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetTxSublinkModeCallback(link, mode, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_tx_mode +( + struct nvlink_link *link, + NvU64 *mode, + NvU32 *subMode +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetTxSublinkModeCallback(link, mode, subMode); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_rx_mode +( + struct nvlink_link *link, + NvU64 mode, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetRxSublinkModeCallback(link, mode, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_rx_mode +( + struct nvlink_link *link, + NvU64 *mode, + NvU32 *subMode +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetRxSublinkModeCallback(link, mode, subMode); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_rx_detect +( + struct nvlink_link *link, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetRxSublinkDetectCallback(link, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_rx_detect +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetRxSublinkDetectCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static void NV_API_CALL rm_nvlink_get_uphy_load +( + struct nvlink_link *link, + NvBool *bUnlocked +) +{ + void *fp; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + knvlinkCoreGetUphyLoadCallback(link, bUnlocked); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_read_link_discovery_token +( + struct nvlink_link *link, + NvU64 *token +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreReadDiscoveryTokenCallback(link, token); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_write_link_discovery_token +( + struct nvlink_link *link, + NvU64 token +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreWriteDiscoveryTokenCallback(link, token); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + + return status; +} + +static void NV_API_CALL rm_nvlink_ops_training_complete +( + struct nvlink_link *link +) +{ + void *fp; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t *sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + knvlinkCoreTrainingCompleteCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_ali_training +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + nvidia_stack_t * sp; + + if (osNvlinkGetAltStack(&sp) != NV_OK) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreAliTrainingCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkPutAltStack(sp); + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_cci_link_mode +( + struct nvlink_link *link, + NvU64 *mode +) +{ + return NVL_SUCCESS; +} + +#endif /* defined(INCLUDE_NVLINK_LIB) */ + +const struct nvlink_link_handlers* osGetNvlinkLinkCallbacks(void) +{ +#if defined(INCLUDE_NVLINK_LIB) + static const struct nvlink_link_handlers rm_nvlink_link_ops = + { + .add = rm_nvlink_ops_add_link, + .remove = rm_nvlink_ops_remove_link, + .lock = rm_nvlink_ops_lock_link, + .unlock = rm_nvlink_ops_unlock_link, + .queue_link_change = rm_nvlink_ops_queue_link_change, + .set_dl_link_mode = rm_nvlink_ops_set_dl_link_mode, + .get_dl_link_mode = rm_nvlink_ops_get_dl_link_mode, + .set_tl_link_mode = rm_nvlink_ops_set_tl_link_mode, + .get_tl_link_mode = rm_nvlink_ops_get_tl_link_mode, + .set_tx_mode = rm_nvlink_ops_set_link_tx_mode, + .get_tx_mode = rm_nvlink_ops_get_link_tx_mode, + .set_rx_mode = rm_nvlink_ops_set_link_rx_mode, + .get_rx_mode = rm_nvlink_ops_get_link_rx_mode, + .set_rx_detect = rm_nvlink_ops_set_link_rx_detect, + .get_rx_detect = rm_nvlink_ops_get_link_rx_detect, + .write_discovery_token = rm_nvlink_ops_write_link_discovery_token, + .read_discovery_token = rm_nvlink_ops_read_link_discovery_token, + .training_complete = rm_nvlink_ops_training_complete, + .get_uphy_load = rm_nvlink_get_uphy_load, + .get_cci_link_mode = rm_nvlink_ops_get_cci_link_mode, + .ali_training = rm_nvlink_ops_ali_training, + }; + + return &rm_nvlink_link_ops; +#else + return NULL; +#endif +} + +/* + * @brief Verif only function to get the chiplib overrides for link connection + * state for all NVLINKs. + * + * If chiplib overrides exist, each link can either be enabled (1) or disabled (0) + * + * @param[in] pGpu GPU object pointer + * @param[in] maxLinks Size of pLinkConnection array + * @param[out] pLinkConnection array of pLinkConnection values to be populated by MODS + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available) + */ +NV_STATUS +osGetForcedNVLinkConnection +( + OBJGPU *pGpu, + NvU32 maxLinks, + NvU32 *pLinkConnection +) +{ + int i, ret; + NV_STATUS status; + char path[64]; + + NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER); + NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED); + NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT); + + for (i = 0; i < maxLinks; i++) + { + ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|NVLink%u", i); + NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1))); + + status = gpuSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]); + if (status == NV_OK) + { + NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__, + path, pLinkConnection[i]); + } + else + { + NV_PRINTF(LEVEL_INFO, "%s: gpuSimEscapeRead for '%s' failed (%u)\n", + __FUNCTION__, path, status); + return NV_ERR_NOT_SUPPORTED; + } + } + return NV_OK; +} + +/* + * @brief Get Platform suggested NVLink linerate + * + * NVLink will use this function to get the platform suggested linerate + * if available in FRU or device tree. + * + * @param[in] pGpu GPU object pointer + * @param[out] NvU32 * Suggested datarate + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED (platform linerate data not available) + */ +NV_STATUS +osGetPlatformNvlinkLinerate +( + OBJGPU *pGpu, + NvU32 *lineRate +) +{ + // TODO: Remove this function + return NV_ERR_NOT_SUPPORTED; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osunix.c b/userspace/sdks/nv570/nvalloc/unix/src/osunix.c new file mode 100644 index 0000000..635bf86 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/osunix.c @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +/***************************** HW State Routines ***************************\ +* * +* Fills in os specific function pointers for the Unix OS object. * +* * +\***************************************************************************/ + +#include <os/os.h> +#include <core/system.h> + +void +osInitObjOS(OBJOS *pOS) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + + pSys->setProperty(pSys, PDB_PROP_SYS_ONDEMAND_VBLANK_CONTROL_ENABLE_DEFAULT, NV_TRUE); + pOS->setProperty(pOS, PDB_PROP_OS_CACHED_MEMORY_MAPPINGS_FOR_ACPI_TABLE, NV_TRUE); + pOS->setProperty(pOS, PDB_PROP_OS_LIMIT_GPU_RESET, NV_TRUE); +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/power-management-tegra.c b/userspace/sdks/nv570/nvalloc/unix/src/power-management-tegra.c new file mode 100644 index 0000000..38c91d0 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/power-management-tegra.c @@ -0,0 +1,100 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> + +#include <nv-priv.h> +#include <osapi.h> +#include <core/locks.h> + +NV_STATUS +RmPowerManagementTegra( + OBJGPU *pGpu, + nv_pm_action_t pmAction +) +{ + // + // Default to NV_OK. there may cases where resman is loaded, but + // no devices are allocated (we're still at the console). in these + // cases, it's fine to let the system do whatever it wants. + // + NV_STATUS rmStatus = NV_OK; + + if (pGpu) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + switch (pmAction) + { + case NV_PM_ACTION_HIBERNATE: + nvp->pm_state.InHibernate = NV_TRUE; + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH, NV_TRUE); + rmStatus = gpuStateUnload(pGpu, GPU_STATE_FLAGS_PM_HIBERNATE | + GPU_STATE_FLAGS_PRESERVING | + GPU_STATE_FLAGS_PM_TRANSITION); + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_HIBERNATE, NV_TRUE); + break; + + case NV_PM_ACTION_STANDBY: + nvp->pm_state.InHibernate = NV_FALSE; + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH, NV_TRUE); + rmStatus = gpuStateUnload(pGpu, GPU_STATE_FLAGS_PM_SUSPEND | + GPU_STATE_FLAGS_PRESERVING | + GPU_STATE_FLAGS_PM_TRANSITION);; + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_STANDBY, NV_TRUE); + pGpu->bInD3Cold = NV_TRUE; + break; + + case NV_PM_ACTION_RESUME: + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_RESUME_CODEPATH, NV_TRUE); + + if (nvp->pm_state.InHibernate) + { + rmStatus = gpuStateLoad(pGpu, GPU_STATE_FLAGS_PM_HIBERNATE | + GPU_STATE_FLAGS_PRESERVING | + GPU_STATE_FLAGS_PM_TRANSITION);; + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_HIBERNATE, NV_FALSE); + } + else + { + rmStatus = gpuStateLoad(pGpu, GPU_STATE_FLAGS_PM_SUSPEND | + GPU_STATE_FLAGS_PRESERVING | + GPU_STATE_FLAGS_PM_TRANSITION);; + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_STANDBY, NV_FALSE); + pGpu->bInD3Cold = NV_FALSE; + } + + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH, NV_FALSE); + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_RESUME_CODEPATH, NV_FALSE); + break; + + default: + rmStatus = NV_ERR_INVALID_ARGUMENT; + break; + } + } + + return rmStatus; +} + diff --git a/userspace/sdks/nv570/nvalloc/unix/src/registry.c b/userspace/sdks/nv570/nvalloc/unix/src/registry.c new file mode 100644 index 0000000..70bbcc5 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/registry.c @@ -0,0 +1,712 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include <nv-priv.h> +#include <nvos.h> + +#if defined(DEBUG_REGISTRY) +#define DBG_REG_PRINTF(a, ...) \ + NV_PRINTF(LEVEL_INFO, a, ##__VA_ARGS__) +#else +#define DBG_REG_PRINTF(a, ...) +#endif + +static NvS32 stringCaseCompare( + const char *string1, + const char *string2 +) +{ + NvU8 c1, c2; + + do + { + c1 = *string1, c2 = *string2; + if (c1 >= 'A' && c1 <= 'Z') + c1 += ('a' - 'A'); + if (c2 >= 'A' && c2 <= 'Z') + c2 += ('a' - 'A'); + string1++, string2++; + } + while ((c1 == c2) && (c1 != '\0')); + + return (c1 - c2); +} + +static nv_reg_entry_t *the_registry = NULL; + +static nv_reg_entry_t* regCreateNewRegistryKey( + nv_state_t *nv, + const char *regParmStr +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_reg_entry_t *new_reg = NULL; + char *new_ParmStr = NULL; + NvU32 parm_size; + + if (regParmStr == NULL) + { + DBG_BREAKPOINT(); + return NULL; + } + + new_reg = portMemAllocNonPaged(sizeof(nv_reg_entry_t)); + if (NULL == new_reg) + { + NV_PRINTF(LEVEL_ERROR, "failed to grow registry\n"); + return NULL; + } + + portMemSet(new_reg, 0, sizeof(nv_reg_entry_t)); + + if (regParmStr != NULL) + { + parm_size = (portStringLength(regParmStr) + 1); + new_ParmStr = portMemAllocNonPaged(parm_size); + if (NULL == new_ParmStr) + { + NV_PRINTF(LEVEL_ERROR, "failed to allocate registry param string\n"); + portMemFree(new_reg); + return NULL; + } + + NV_ASSERT(parm_size <= NVOS38_MAX_REGISTRY_STRING_LENGTH); + + if (portMemCopy(new_ParmStr, parm_size, regParmStr, parm_size) == NULL) + { + NV_PRINTF(LEVEL_ERROR, "failed to copy registry param string\n"); + portMemFree(new_ParmStr); + portMemFree(new_reg); + return NULL; + } + } + + new_reg->regParmStr = new_ParmStr; + new_reg->type = NV_REGISTRY_ENTRY_TYPE_UNKNOWN; + + if (nvp != NULL) + { + new_reg->next = nvp->pRegistry; + nvp->pRegistry = new_reg; + DBG_REG_PRINTF("local registry now at 0x%p\n", nvp->pRegistry); + } + else + { + new_reg->next = the_registry; + the_registry = new_reg; + DBG_REG_PRINTF("global registry now at 0x%p\n", the_registry); + } + + return new_reg; +} + +static NV_STATUS regFreeEntry(nv_reg_entry_t *tmp) +{ + portMemFree(tmp->regParmStr); + tmp->regParmStr = NULL; + { + portMemFree(tmp->pdata); + tmp->pdata = NULL; + tmp->len = 0; + } + portMemFree(tmp); + + return NV_OK; +} + +static nv_reg_entry_t* regFindRegistryEntry( + nv_state_t *nv, + const char *regParmStr, + NvU32 type, + NvBool *bGlobalEntry +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_reg_entry_t *tmp; + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + if (nvp != NULL) + { + tmp = nvp->pRegistry; + DBG_REG_PRINTF(" local registry at 0x%p\n", tmp); + + while ((tmp != NULL) && (tmp->regParmStr != NULL)) + { + DBG_REG_PRINTF(" Testing against %s\n", + tmp->regParmStr); + if ((stringCaseCompare(tmp->regParmStr, regParmStr) == 0) && + (type == tmp->type)) + { + DBG_REG_PRINTF(" found a match!\n"); + if (bGlobalEntry) + *bGlobalEntry = NV_FALSE; + return tmp; + } + tmp = tmp->next; + } + } + + tmp = the_registry; + DBG_REG_PRINTF(" global registry at 0x%p\n", tmp); + + while ((tmp != NULL) && (tmp->regParmStr != NULL)) + { + DBG_REG_PRINTF(" Testing against %s\n", + tmp->regParmStr); + if ((stringCaseCompare(tmp->regParmStr, regParmStr) == 0) && + (type == tmp->type)) + { + DBG_REG_PRINTF(" found a match!\n"); + if (bGlobalEntry) + *bGlobalEntry = NV_TRUE; + return tmp; + } + tmp = tmp->next; + } + + DBG_REG_PRINTF(" no match\n"); + return NULL; +} + +NV_STATUS RmWriteRegistryDword( + nv_state_t *nv, + const char *regParmStr, + NvU32 Data +) +{ + nv_reg_entry_t *tmp; + NvBool bGlobalEntry; + + if (regParmStr == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s -> 0x%x\n", __FUNCTION__, regParmStr, Data); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_DWORD, &bGlobalEntry); + + // If we found an entry and we were looking for a global entry and + // found a global, or we were looking for a per-GPU entry and found a + // per-GPU entry + if (tmp != NULL && + ((nv == NULL && bGlobalEntry) || + (nv != NULL && !bGlobalEntry))) + { + tmp->data = Data; + + if (stringCaseCompare(regParmStr, "ResmanDebugLevel") == 0) + { + os_dbg_set_level(Data); + } + + return NV_OK; + } + + tmp = regCreateNewRegistryKey(nv, regParmStr); + if (tmp == NULL) + return NV_ERR_GENERIC; + + tmp->type = NV_REGISTRY_ENTRY_TYPE_DWORD; + tmp->data = Data; + + return NV_OK; +} + +NV_STATUS RmReadRegistryDword( + nv_state_t *nv, + const char *regParmStr, + NvU32 *Data +) +{ + nv_reg_entry_t *tmp; + + if ((regParmStr == NULL) || (Data == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_DWORD, NULL); + if (tmp == NULL) + { + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_BINARY, NULL); + if ((tmp != NULL) && (tmp->len >= sizeof(NvU32))) + { + *Data = *(NvU32 *)tmp->pdata; + } + else + { + DBG_REG_PRINTF(" not found\n"); + return NV_ERR_GENERIC; + } + } + else + { + *Data = tmp->data; + } + + DBG_REG_PRINTF(" found in the_registry: 0x%x\n", *Data); + + return NV_OK; +} + +NV_STATUS RmReadRegistryBinary( + nv_state_t *nv, + const char *regParmStr, + NvU8 *Data, + NvU32 *cbLen +) +{ + nv_reg_entry_t *tmp; + NV_STATUS status; + + if ((regParmStr == NULL) || (Data == NULL) || (cbLen == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_BINARY, NULL); + if (tmp == NULL) + { + DBG_REG_PRINTF(" not found\n"); + return NV_ERR_GENERIC; + } + + DBG_REG_PRINTF(" found\n"); + + if (*cbLen >= tmp->len) + { + portMemCopy((NvU8 *)Data, *cbLen, (NvU8 *)tmp->pdata, tmp->len); + *cbLen = tmp->len; + status = NV_OK; + } + else + { + NV_PRINTF(LEVEL_ERROR, + "buffer (length: %u) is too small (data length: %u)\n", + *cbLen, tmp->len); + status = NV_ERR_GENERIC; + } + + return status; +} + +NV_STATUS RmWriteRegistryBinary( + nv_state_t *nv, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + nv_reg_entry_t *tmp; + NvBool bGlobalEntry; + + if ((regParmStr == NULL) || (Data == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_BINARY, &bGlobalEntry); + + // If we found an entry and we were looking for a global entry and + // found a global, or we were looking for a per-GPU entry and found a + // per-GPU entry + if (tmp != NULL && + ((nv == NULL && bGlobalEntry) || + (nv != NULL && !bGlobalEntry))) + { + if (tmp->pdata != NULL) + { + portMemFree(tmp->pdata); + tmp->pdata = NULL; + tmp->len = 0; + } + } + else + { + tmp = regCreateNewRegistryKey(nv, regParmStr); + if (tmp == NULL) + { + NV_PRINTF(LEVEL_ERROR, "failed to create binary registry entry\n"); + return NV_ERR_GENERIC; + } + } + + tmp->pdata = portMemAllocNonPaged(cbLen); + if (NULL == tmp->pdata) + { + NV_PRINTF(LEVEL_ERROR, "failed to write binary registry entry\n"); + return NV_ERR_GENERIC; + } + + tmp->type = NV_REGISTRY_ENTRY_TYPE_BINARY; + tmp->len = cbLen; + portMemCopy((NvU8 *)tmp->pdata, tmp->len, (NvU8 *)Data, cbLen); + + return NV_OK; +} + +NV_STATUS RmWriteRegistryString( + nv_state_t *nv, + const char *regParmStr, + const char *buffer, + NvU32 bufferLength +) +{ + nv_reg_entry_t *tmp; + NvBool bGlobalEntry; + + if ((regParmStr == NULL) || (buffer == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_STRING, &bGlobalEntry); + + // If we found an entry and we were looking for a global entry and + // found a global, or we were looking for a per-GPU entry and found a + // per-GPU entry + if (tmp != NULL && + ((nv == NULL && bGlobalEntry) || + (nv != NULL && !bGlobalEntry))) + { + if (tmp->pdata != NULL) + { + portMemFree(tmp->pdata); + tmp->len = 0; + tmp->pdata = NULL; + } + } + else + { + tmp = regCreateNewRegistryKey(nv, regParmStr); + if (tmp == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "failed to allocate a string registry entry!\n"); + return NV_ERR_INSUFFICIENT_RESOURCES; + } + } + + tmp->pdata = portMemAllocNonPaged(bufferLength); + if (tmp->pdata == NULL) + { + NV_PRINTF(LEVEL_ERROR, "failed to write a string registry entry!\n"); + return NV_ERR_NO_MEMORY; + } + + tmp->type = NV_REGISTRY_ENTRY_TYPE_STRING; + tmp->len = bufferLength; + portMemCopy((void *)tmp->pdata, tmp->len, buffer, (bufferLength - 1)); + tmp->pdata[bufferLength-1] = '\0'; + + return NV_OK; +} + +NV_STATUS RmReadRegistryString( + nv_state_t *nv, + const char *regParmStr, + NvU8 *buffer, + NvU32 *pBufferLength +) +{ + NvU32 bufferLength; + nv_reg_entry_t *tmp; + + if ((regParmStr == NULL) || (buffer == NULL) || (pBufferLength == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + bufferLength = *pBufferLength; + *pBufferLength = 0; + *buffer = '\0'; + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_STRING, NULL); + if (tmp == NULL) + { + return NV_ERR_GENERIC; + } + + if (bufferLength >= tmp->len) + { + portMemCopy((void *)buffer, bufferLength, (void *)tmp->pdata, tmp->len); + *pBufferLength = tmp->len; + } + else + { + NV_PRINTF(LEVEL_ERROR, + "buffer (length: %u) is too small (data length: %u)\n", + bufferLength, tmp->len); + return NV_ERR_BUFFER_TOO_SMALL; + } + + return NV_OK; +} + + +NV_STATUS RmInitRegistry(void) +{ + NV_STATUS rmStatus; + + rmStatus = os_registry_init(); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "failed to initialize the OS registry!\n"); + } + + return rmStatus; +} + +NV_STATUS RmDestroyRegistry(nv_state_t *nv) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_reg_entry_t *tmp; + + if (nvp != NULL) + { + tmp = nvp->pRegistry; + nvp->pRegistry = NULL; + } + else + { + tmp = the_registry; + the_registry = NULL; + } + + while (tmp != NULL) + { + nv_reg_entry_t *entry = tmp; + tmp = tmp->next; + regFreeEntry(entry); + } + + return NV_OK; +} + +static void regCountEntriesAndSize( + NvU32 *pNumEntries, // Pointer to number of entries + NvU32 *pSize, // Pointer to total size + nv_reg_entry_t *pRegEntry // Pointer local or global registry +) +{ + // + // Note that *pNumEntries and *pSize are not initialized here. This is so + // we can accumulate totals of both global and local registries. + // + NvU32 numEntries = *pNumEntries; + NvU32 size = *pSize; + + while ((pRegEntry != NULL) && (pRegEntry->regParmStr != NULL)) + { + size += portStringLength(pRegEntry->regParmStr) + 1 + pRegEntry->len; + numEntries++; + pRegEntry = pRegEntry->next; + } + + *pNumEntries = numEntries; + *pSize = size; +} + +static NV_STATUS regCopyEntriesToPackedBuffer( + PACKED_REGISTRY_TABLE *pRegTable, // Pointer to packed record + nv_reg_entry_t *pRegEntry, // Pointer local or global registry + NvU32 *pEntryIndex, // Pointer to next index + NvU32 *pDataOffset // Pointer to offset of next data byte. +) +{ + NvU8 *pByte = (NvU8 *)pRegTable; // Byte version of record pointer. + NV_STATUS nvStatus = NV_OK; + NvU32 entryIndex = *pEntryIndex; + NvU32 dataOffset = *pDataOffset; + + // Walk the records and copy the data. + while ((pRegEntry != NULL) && (pRegEntry->regParmStr != NULL)) + { + PACKED_REGISTRY_ENTRY *pEntry = &pRegTable->entries[entryIndex]; + NvU32 slen = portStringLength(pRegEntry->regParmStr) + 1; + + // Sanity check the data offset and index against counted totals. + if ((dataOffset + slen + pRegEntry->len > pRegTable->size) || + (entryIndex >= pRegTable->numEntries)) + { + // Something has changed since we counted them? + NV_PRINTF(LEVEL_ERROR, "Registry entry record is full\n"); + nvStatus = NV_ERR_INVALID_STATE; + break; + } + + // Copy registry entry name to data blob. + pEntry->nameOffset = dataOffset; + portMemCopy(&pByte[dataOffset], slen, pRegEntry->regParmStr, slen); + dataOffset += slen; + + switch (pRegEntry->type) + { + case NV_REGISTRY_ENTRY_TYPE_DWORD: + pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_DWORD; + pEntry->length = sizeof(NvU32); + pEntry->data = pRegEntry->data; + break; + + case NV_REGISTRY_ENTRY_TYPE_BINARY: + case NV_REGISTRY_ENTRY_TYPE_STRING: + pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_STRING; + if (pRegEntry->type == NV_REGISTRY_ENTRY_TYPE_BINARY) + pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_BINARY; + + if (pRegEntry->len == 0) + { + NV_PRINTF(LEVEL_WARNING, + "Registry Key not sent to GSP-RM because it has 0 DataLength\n"); + break; + } + pEntry->length = pRegEntry->len; + pEntry->data = dataOffset; + portMemCopy(&pByte[dataOffset], pEntry->length, + pRegEntry->pdata, pRegEntry->len); + dataOffset += pRegEntry->len; + break; + + default: + // We should never get here. + pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_UNKNOWN; + pEntry->length = 0; + pEntry->data = 0; + DBG_BREAKPOINT(); + break; + } + + pRegEntry = pRegEntry->next; + entryIndex++; + } + + *pEntryIndex = entryIndex; + *pDataOffset = dataOffset; + return nvStatus; +} + +// Package registry entries +NV_STATUS RmPackageRegistry( + nv_state_t *nv, + PACKED_REGISTRY_TABLE *pRegTable, + NvU32 *pSize +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_reg_entry_t *pLocalRegistry = NULL; + NV_STATUS nvStatus = NV_OK; + NvU32 totalSize; + NvU32 numEntries; + + if (pSize == NULL) + return NV_ERR_INVALID_ARGUMENT; + + // Use the local (per-device) registry if we have one. + if (nvp != NULL) + pLocalRegistry = nvp->pRegistry; + + numEntries = 0; + totalSize = NV_OFFSETOF(PACKED_REGISTRY_TABLE, entries); + + // Count the number of global entries and total size. + regCountEntriesAndSize(&numEntries, &totalSize, the_registry); + + // Count the number of local entries and total size. + regCountEntriesAndSize(&numEntries, &totalSize, pLocalRegistry); + + // Add table record size into total size. + totalSize += sizeof(PACKED_REGISTRY_ENTRY) * numEntries; + + // + // If this function is called to only compute total size of registry table, + // then we are done here. + // + if (pRegTable == NULL) + { + *pSize = totalSize; + return NV_OK; + } + + // Return warning if there are no registry entries. + if (numEntries == 0) + return NV_WARN_NOTHING_TO_DO; + + if (totalSize > *pSize) + { + NV_PRINTF(LEVEL_ERROR, "Registry entries overflow RPC record\n"); + return NV_ERR_BUFFER_TOO_SMALL; + } + + // Fill in our new structure with the first pass (counting) values. + pRegTable->size = totalSize; + *pSize = totalSize; + pRegTable->numEntries = numEntries; + + // Offset of first byte after the registry entry table. + totalSize = NV_OFFSETOF(PACKED_REGISTRY_TABLE, entries) + + (sizeof(PACKED_REGISTRY_ENTRY) * numEntries); + + // Starting index in the registry entry table. + numEntries = 0; + + // Walk the global registry and copy the data. + nvStatus = regCopyEntriesToPackedBuffer(pRegTable, + the_registry, &numEntries, &totalSize); + + // Walk the local registry and copy the data. + if (nvStatus == NV_OK) + { + nvStatus = regCopyEntriesToPackedBuffer(pRegTable, + pLocalRegistry, &numEntries, &totalSize); + } + + // Sanity check second pass against first pass. + if ((numEntries != pRegTable->numEntries) || (totalSize != pRegTable->size)) + { + NV_PRINTF(LEVEL_ERROR, "First/second pass mismatch\n"); + nvStatus = NV_ERR_INVALID_STATE; + } + + return nvStatus; +} + diff --git a/userspace/sdks/nv570/nvalloc/unix/src/rm-gpu-ops.c b/userspace/sdks/nv570/nvalloc/unix/src/rm-gpu-ops.c new file mode 100644 index 0000000..688b4c5 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/rm-gpu-ops.c @@ -0,0 +1,1079 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include <os/os.h> +#include <osapi.h> +#include <core/thread_state.h> +#include "rmapi/nv_gpu_ops.h" +#include "gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h" + +NV_STATUS NV_API_CALL rm_gpu_ops_create_session( + nvidia_stack_t *sp, + struct gpuSession **session) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCreateSession(session); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_destroy_session ( + nvidia_stack_t *sp, gpuSessionHandle session) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDestroySession(session); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_device_create ( + nvidia_stack_t *sp, + nvgpuSessionHandle_t session, + const gpuInfo *pGpuInfo, + const NvProcessorUuid *gpuUuid, + nvgpuDeviceHandle_t *device, + NvBool bCreateSmcPartition) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDeviceCreate(session, pGpuInfo, gpuUuid, device, bCreateSmcPartition); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_device_destroy ( + nvidia_stack_t *sp, + gpuDeviceHandle device) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDeviceDestroy(device); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_address_space_create ( + nvidia_stack_t *sp, + gpuDeviceHandle device, + NvU64 vaBase, + NvU64 vaSize, + NvU32 enableAts, + gpuAddressSpaceHandle *vaSpace, + gpuAddressSpaceInfo *vaSpaceInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsAddressSpaceCreate(device, vaBase, vaSize, enableAts, + vaSpace, vaSpaceInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_dup_address_space( + nvidia_stack_t *sp, + gpuDeviceHandle device, + NvHandle hUserClient, + NvHandle hUserVASpace, + gpuAddressSpaceHandle *dupedVaspace, + gpuAddressSpaceInfo *vaSpaceInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDupAddressSpace(device, hUserClient, hUserVASpace, + dupedVaspace, vaSpaceInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_address_space_destroy(nvidia_stack_t *sp, + gpuAddressSpaceHandle vaspace) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsAddressSpaceDestroy(vaspace); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_alloc_fb( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, + NvLength size, NvU64 *gpuOffset, gpuAllocInfo *allocInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsMemoryAllocFb(vaspace, size, gpuOffset, allocInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_p2p_caps(nvidia_stack_t *sp, + gpuDeviceHandle device1, + gpuDeviceHandle device2, + getP2PCapsParams *pP2pCapsParams) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetP2PCaps(device1, device2, pP2pCapsParams); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_alloc_sys( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, + NvLength size, NvU64 *gpuOffset, gpuAllocInfo *allocInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsMemoryAllocSys(vaspace, size, gpuOffset, allocInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_register_callbacks( + nvidia_stack_t *sp, + void *pPma, + pmaEvictPagesCb_t evictPages, + pmaEvictRangeCb_t evictRange, + void *callbackData) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + rmStatus = pmaRegisterEvictionCb(pPma, evictPages, evictRange, callbackData); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +void NV_API_CALL rm_gpu_ops_pma_unregister_callbacks( + nvidia_stack_t *sp, + void *pPma) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pmaUnregisterEvictionCb(pPma); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_pma_object( + nvidia_stack_t *sp, + gpuDeviceHandle device, + void **pPma, + const nvgpuPmaStatistics_t *pPmaPubStats) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetPmaObject(device, pPma, + (const UvmPmaStatistics **)pPmaPubStats); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_alloc_pages( + nvidia_stack_t *sp, void *pPma, + NvLength pageCount, NvU64 pageSize, + nvgpuPmaAllocationOptions_t pPmaAllocOptions, + NvU64 *pPages) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPmaAllocPages(pPma, pageCount, pageSize, + pPmaAllocOptions, pPages); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_pin_pages( + nvidia_stack_t *sp, void *pPma, + NvU64 *pPages, NvLength pageCount, NvU64 pageSize, NvU32 flags) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPmaPinPages(pPma, pPages, pageCount, pageSize, flags); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_map( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, + NvU64 gpuOffset, NvLength length, void **cpuPtr, NvU64 pageSize) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsMemoryCpuMap(vaspace, gpuOffset, length, cpuPtr, + pageSize); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_ummap( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, void* cpuPtr) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsMemoryCpuUnMap(vaspace, cpuPtr); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_tsg_allocate(nvidia_stack_t *sp, + gpuAddressSpaceHandle vaspace, + const gpuTsgAllocParams *allocParams, + gpuTsgHandle *tsg) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsTsgAllocate(vaspace, allocParams, tsg); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_tsg_destroy(nvidia_stack_t * sp, + nvgpuTsgHandle_t tsg) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsTsgDestroy(tsg); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_channel_allocate(nvidia_stack_t *sp, + const gpuTsgHandle tsg, + const gpuChannelAllocParams *allocParams, + gpuChannelHandle *channel, + gpuChannelInfo *channelInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsChannelAllocate(tsg, allocParams, channel, + channelInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_channel_destroy(nvidia_stack_t * sp, + nvgpuChannelHandle_t channel) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsChannelDestroy(channel); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_free_pages(nvidia_stack_t *sp, + void *pPma, NvU64 *pPages, NvLength pageCount, NvU64 pageSize, NvU32 flags) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsPmaFreePages(pPma, pPages, pageCount, pageSize, flags); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_free( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, NvU64 gpuOffset) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsMemoryFree(vaspace, gpuOffset); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_query_caps(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuCaps * caps) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsQueryCaps(device, caps); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_query_ces_caps(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuCesCaps *caps) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsQueryCesCaps(device, caps); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_gpu_info(nvidia_stack_t *sp, + const NvProcessorUuid *pUuid, + const gpuClientInfo *pGpuClientInfo, + gpuInfo *pGpuInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetGpuInfo(pUuid, pGpuClientInfo, pGpuInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_service_device_interrupts_rm(nvidia_stack_t *sp, + gpuDeviceHandle device) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsServiceDeviceInterruptsRM(device); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_set_page_directory (nvidia_stack_t *sp, + gpuAddressSpaceHandle vaSpace, + NvU64 physAddress, unsigned numEntries, + NvBool bVidMemAperture, NvU32 pasid, + NvU64 *dmaAddress) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsSetPageDirectory(vaSpace, physAddress, numEntries, + bVidMemAperture, pasid, dmaAddress); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_unset_page_directory (nvidia_stack_t *sp, + gpuAddressSpaceHandle vaSpace) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsUnsetPageDirectory(vaSpace); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_dup_allocation(nvidia_stack_t *sp, + gpuAddressSpaceHandle srcVaSpace, + NvU64 srcAddress, + gpuAddressSpaceHandle dstVaSpace, + NvU64 dstVaAlignment, + NvU64 *dstAddress) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDupAllocation(srcVaSpace, srcAddress, dstVaSpace, dstVaAlignment, dstAddress); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_dup_memory (nvidia_stack_t *sp, + gpuDeviceHandle device, + NvHandle hClient, + NvHandle hPhysMemory, + NvHandle *hDupMemory, + nvgpuMemoryInfo_t gpuMemoryInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDupMemory(device, hClient, hPhysMemory, hDupMemory, gpuMemoryInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_free_duped_handle (nvidia_stack_t *sp, + gpuDeviceHandle device, + NvHandle hPhysHandle) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsFreeDupedHandle(device, hPhysHandle); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_fb_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuFbInfo * fbInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetFbInfo(device, fbInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_ecc_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuEccInfo * eccInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetEccInfo(device, eccInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +// +// Please see the comments for nvUvmInterfaceOwnPageFaultIntr(), in +// nv_uvm_interface.h, for the recommended way to use this routine. +// +// How it works: +// +// The rmGpuLocksAcquire call generally saves the current GPU interrupt +// state, then disables interrupt generation for one (or all) GPUs. +// Likewise, the rmGpuLocksRelease call restores (re-enables) those +// interrupts to their previous state. However, the rmGpuLocksRelease +// call does NOT restore interrupts that RM does not own. +// +// This is rather hard to find in the code, so: very approximately, the +// following sequence happens: rmGpuLocksRelease, osEnableInterrupts, +// intrRestoreNonStall_HAL, intrEncodeIntrEn_HAL, and that last one skips +// over any interrupts that RM does not own. +// +// This means that things are a bit asymmetric, because this routine +// actually changes that ownership in between the rmGpuLocksAcquire and +// rmGpuLocksRelease calls. So: +// +// -- If you call this routine with bOwnInterrupts == NV_TRUE (UVM is +// taking ownership from the RM), then rmGpuLocksAcquire disables all +// GPU interrupts. Then the ownership is taken away from RM, so the +// rmGpuLocksRelease call leaves the replayable page fault interrupts +// disabled. It is then up to UVM (the caller) to enable replayable +// page fault interrupts when it is ready. +// +// -- If you call this routine with bOwnInterrupts == NV_FALSE (UVM is +// returning ownership to the RM), then rmGpuLocksAcquire disables +// all GPU interrupts that RM owns. Then the ownership is returned to +// RM, so the rmGpuLocksRelease call re-enables replayable page fault +// interrupts. So, that implies that you need to disable replayable page +// fault interrupts before calling this routine, in order to hand +// over a GPU to RM that is not generating interrupts, until RM is +// ready to handle the interrupts. +// +NV_STATUS NV_API_CALL rm_gpu_ops_own_page_fault_intr(nvidia_stack_t *sp, + struct gpuDevice *device, + NvBool bOwnInterrupts) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsOwnPageFaultIntr(device, bOwnInterrupts); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_init_fault_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuFaultInfo *pFaultInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsInitFaultInfo(device, pFaultInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_destroy_fault_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuFaultInfo *pFaultInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDestroyFaultInfo(device, pFaultInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +// Functions +// +// - rm_gpu_ops_has_pending_non_replayable_faults +// - rm_gpu_ops_get_non_replayable_faults +// +// Cannot take the GPU/RM lock because it is called during fault servicing. +// This could produce deadlocks if the UVM bottom half gets stuck behind a +// stalling interrupt that cannot be serviced if UVM is holding the lock. +// +// However, these functions can be safely called with no locks because it is +// just accessing the given client shadow fault buffer, which is implemented +// using a lock-free queue. There is a different client shadow fault buffer +// per GPU: RM top-half producer, UVM top/bottom-half consumer. + +NV_STATUS NV_API_CALL rm_gpu_ops_has_pending_non_replayable_faults(nvidia_stack_t *sp, + gpuFaultInfo *pFaultInfo, + NvBool *hasPendingFaults) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsHasPendingNonReplayableFaults(pFaultInfo, hasPendingFaults); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_non_replayable_faults(nvidia_stack_t *sp, + gpuFaultInfo *pFaultInfo, + void *faultBuffer, + NvU32 *numFaults) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetNonReplayableFaults(pFaultInfo, faultBuffer, numFaults); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_flush_replayable_fault_buffer(nvidia_stack_t *sp, + gpuFaultInfo *pFaultInfo, + NvBool bCopyAndFlush) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsFlushReplayableFaultBuffer(pFaultInfo, bCopyAndFlush); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_toggle_prefetch_faults(nvidia_stack_t *sp, + gpuFaultInfo *pFaultInfo, + NvBool bEnable) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsTogglePrefetchFaults(pFaultInfo, bEnable); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_access_bits_buffer_alloc(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessBitsBufferAlloc *accessBitsInfo) +{ + NV_STATUS rmStatus; + void* fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsAccessBitsBufAlloc(device, accessBitsInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_access_bits_buffer_free(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessBitsBufferAlloc *accessBitsInfo) +{ + NV_STATUS rmStatus; + void* fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsAccessBitsBufFree(device, accessBitsInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_access_bits_dump(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessBitsBufferAlloc *accessBitsInfo, + UVM_ACCESS_BITS_DUMP_MODE mode) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsAccessBitsDump(device, accessBitsInfo, mode); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_init_access_cntr_info(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessCntrInfo *accessCntrInfo, + NvU32 accessCntrIndex) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsInitAccessCntrInfo(device, accessCntrInfo, accessCntrIndex); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_destroy_access_cntr_info(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessCntrInfo *accessCntrInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDestroyAccessCntrInfo(device, accessCntrInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_enable_access_cntr(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessCntrInfo *accessCntrInfo, + gpuAccessCntrConfig *accessCntrConfig) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsEnableAccessCntr(device, accessCntrInfo, accessCntrConfig); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_disable_access_cntr(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessCntrInfo *accessCntrInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDisableAccessCntr(device, accessCntrInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_nvlink_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuNvlinkInfo *nvlinkInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetNvlinkInfo(device, nvlinkInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_p2p_object_create(nvidia_stack_t *sp, + gpuDeviceHandle device1, + gpuDeviceHandle device2, + NvHandle *hP2pObject) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsP2pObjectCreate(device1, device2, hP2pObject); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_p2p_object_destroy(nvidia_stack_t *sp, + nvgpuSessionHandle_t session, + NvHandle hP2pObject) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + nvGpuOpsP2pObjectDestroy(session, hP2pObject); + NV_EXIT_RM_RUNTIME(sp, fp); +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_get_external_alloc_ptes(nvidia_stack_t* sp, + nvgpuAddressSpaceHandle_t vaSpace, + NvHandle hDupedMemory, + NvU64 offset, + NvU64 size, + nvgpuExternalMappingInfo_t gpuExternalMappingInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsGetExternalAllocPtes(vaSpace, hDupedMemory, offset, size, + gpuExternalMappingInfo); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_get_external_alloc_phys_addrs(nvidia_stack_t* sp, + nvgpuAddressSpaceHandle_t vaSpace, + NvHandle hDupedMemory, + NvU64 offset, + NvU64 size, + nvgpuExternalPhysAddrInfo_t gpuExternalPhysAddrsInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsGetExternalAllocPhysAddrs(vaSpace, hDupedMemory, offset, size, + gpuExternalPhysAddrsInfo); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_retain_channel(nvidia_stack_t* sp, + nvgpuAddressSpaceHandle_t vaSpace, + NvHandle hClient, + NvHandle hChannel, + void **retainedChannel, + nvgpuChannelInstanceInfo_t channelInstanceInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsRetainChannel(vaSpace, hClient, hChannel, + (gpuRetainedChannel **)retainedChannel, + channelInstanceInfo); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_bind_channel_resources(nvidia_stack_t* sp, + void *retainedChannel, + nvgpuChannelResourceBindParams_t channelResourceBindParams) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsBindChannelResources(retainedChannel, + channelResourceBindParams); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_release_channel(nvidia_stack_t *sp, void *retainedChannel) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + nvGpuOpsReleaseChannel(retainedChannel); + NV_EXIT_RM_RUNTIME(sp, fp); +} + +void NV_API_CALL +rm_gpu_ops_stop_channel(nvidia_stack_t * sp, + void *retainedChannel, + NvBool bImmediate) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsStopChannel(retainedChannel, bImmediate); + NV_EXIT_RM_RUNTIME(sp, fp); +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_get_channel_resource_ptes(nvidia_stack_t* sp, + nvgpuAddressSpaceHandle_t vaSpace, + NvP64 resourceDescriptor, + NvU64 offset, + NvU64 size, + nvgpuExternalMappingInfo_t gpuExternalMappingInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsGetChannelResourcePtes(vaSpace, resourceDescriptor, + offset, size, + gpuExternalMappingInfo); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_report_non_replayable_fault(nvidia_stack_t *sp, + nvgpuDeviceHandle_t device, + const void *pFaultPacket) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsReportNonReplayableFault(device, pFaultPacket); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_paging_channel_allocate(nvidia_stack_t *sp, + gpuDeviceHandle device, + const gpuPagingChannelAllocParams *allocParams, + gpuPagingChannelHandle *channel, + gpuPagingChannelInfo *channelInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPagingChannelAllocate(device, allocParams, channel, + channelInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_paging_channel_destroy(nvidia_stack_t *sp, + gpuPagingChannelHandle channel) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsPagingChannelDestroy(channel); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_paging_channels_map(nvidia_stack_t *sp, + gpuAddressSpaceHandle srcVaSpace, + NvU64 srcAddress, + gpuDeviceHandle device, + NvU64 *dstAddress) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPagingChannelsMap(srcVaSpace, srcAddress, device, dstAddress); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_paging_channels_unmap(nvidia_stack_t *sp, + gpuAddressSpaceHandle srcVaSpace, + NvU64 srcAddress, + gpuDeviceHandle device) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsPagingChannelsUnmap(srcVaSpace, srcAddress, device); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_paging_channel_push_stream(nvidia_stack_t *sp, + gpuPagingChannelHandle channel, + char *methodStream, + NvU32 methodStreamSize) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPagingChannelPushStream(channel, methodStream, methodStreamSize); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_report_fatal_error(nvidia_stack_t *sp, + NV_STATUS error) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsReportFatalError(error); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_init(nvidia_stack_t *sp, + struct ccslContext_t **ctx, + gpuChannelHandle channel) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCcslContextInit(ctx, channel); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_clear(nvidia_stack_t *sp, + struct ccslContext_t *ctx) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCcslContextClear(ctx); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_rotate_key(nvidia_stack_t *sp, + UvmCslContext *contextList[], + NvU32 contextListCount) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCcslRotateKey(contextList, contextListCount); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_rotate_iv(nvidia_stack_t *sp, + struct ccslContext_t *ctx, + NvU8 direction) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCcslRotateIv(ctx, direction); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_encrypt_with_iv(nvidia_stack_t *sp, + struct ccslContext_t *ctx, + NvU32 bufferSize, + NvU8 const *inputBuffer, + NvU8 *encryptIv, + NvU8 *outputBuffer, + NvU8 *authTagData) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCcslEncryptWithIv(ctx, bufferSize, inputBuffer, encryptIv, outputBuffer, authTagData); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_encrypt(nvidia_stack_t *sp, + struct ccslContext_t *ctx, + NvU32 bufferSize, + NvU8 const *inputBuffer, + NvU8 *outputBuffer, + NvU8 *authTagData) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCcslEncrypt(ctx, bufferSize, inputBuffer, outputBuffer, authTagData); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_decrypt(nvidia_stack_t *sp, + struct ccslContext_t *ctx, + NvU32 bufferSize, + NvU8 const *inputBuffer, + NvU8 const *decryptIv, + NvU32 keyRotationId, + NvU8 *outputBuffer, + NvU8 const *addAuthData, + NvU32 addAuthDataSize, + NvU8 const *authTagData) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCcslDecrypt(ctx, bufferSize, inputBuffer, decryptIv, keyRotationId, outputBuffer, + addAuthData, addAuthDataSize, authTagData); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_sign(nvidia_stack_t *sp, + struct ccslContext_t *ctx, + NvU32 bufferSize, + NvU8 const *inputBuffer, + NvU8 *authTagData) + +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCcslSign(ctx, bufferSize, inputBuffer, authTagData); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_query_message_pool(nvidia_stack_t *sp, + struct ccslContext_t *ctx, + NvU8 direction, + NvU64 *messageNum) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsQueryMessagePool(ctx, direction, messageNum); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_increment_iv(nvidia_stack_t *sp, + struct ccslContext_t *ctx, + NvU8 direction, + NvU64 increment, + NvU8 *iv) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsIncrementIv(ctx, direction, increment, iv); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_log_encryption(nvidia_stack_t *sp, + struct ccslContext_t *ctx, + NvU8 direction, + NvU32 bufferSize) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsLogEncryption(ctx, direction, bufferSize); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/rmobjexportimport.c b/userspace/sdks/nv570/nvalloc/unix/src/rmobjexportimport.c new file mode 100644 index 0000000..558f5fd --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/rmobjexportimport.c @@ -0,0 +1,650 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @file + * + * @brief Provides RmExportObject, RmImportObject, RmFreeObjExportHandle and + * RmGetExportObjectInfo interfaces : + * + * These interfaces allow rm clients to export their objects into + * a unique RmObjExportHandle which another rm client could + * import, even if the source rm client gets destroyed. + * + * RM's device instance may get destroyed asynchronously, in which + * case exported objects residing on that device instance also get + * destroyed. This means it is not possible to import it back, but the + * RmObjExportHandle into which the object had been exported still + * remains valid but no other object could get it. + * + * There are not init/fini routines, it is the responsibility of the + * rest of RM's eco-system to make sure that all RmObjExportHandles get + * freed during driver unload. + * + * The api lock is expected to be held before calling into + * rmobjexportimport.c; do not hold gpu or any other lock. + */ + +#include "rmobjexportimport.h" +#include "nvlimits.h" +#include "gpu/device/device.h" +#include "gpu/mig_mgr/kernel_mig_manager.h" +#include "gpu/mig_mgr/gpu_instance_subscription.h" + +#include "containers/map.h" +#include "rmapi/rmapi.h" +#include "rmapi/rs_utils.h" + +#include "class/cl0080.h" +#include "class/cl2080.h" +#include <ctrl/ctrl0000/ctrl0000unix.h> +#include <ctrl/ctrl0000/ctrl0000client.h> + +// +// A reference to an RmObjExportHandle +// generated by function RmGenerateObjExportHandle(). +// +MAKE_MAP(RmObjExportHandleMap, NvU8); + +// +// Memory allocator +// +PORT_MEM_ALLOCATOR *pMemAllocator; + +// +// Map RmObjExportHandle -> RmObjExportHandleRef +// +RmObjExportHandleMap objExportHandleMap; + +// +// Rm client to use to dup an object exported to RmObjExportHandle. The minimal +// requirement for duping is to have a device object allocated. This rm client +// is simply like any other external rm client and has no any special handling. +// +// We keep this rm client just like any other external rm client: if +// gpu(s)/device gets powered-down/uninitialized, rm objects allocated by +// external rm clients and located on that gpu(s)/device gets freed (the +// os-layer does that). In that way, code in this file doesn't need to worry +// about freeing exported objects located on that gpu(s)/device. +// +NvHandle hObjExportRmClient; + +// +// Tracker for device and subdevice handles. For now only one subdevice +// (instance 0) is supported per device. +// +typedef struct +{ + NvHandle hRmDevice; + NvHandle hRmSubDevice; + NvHandle hGpuInstSub; +} RmObjExportDevice; +MAKE_MAP(RmObjExportDeviceMap, RmObjExportDevice); + +RmObjExportDeviceMap objExportDeviceMap; + +// +// Usage reference counter for static object in this file like rm client used to +// dup an exported object, memory allocator, map etc. +// +NvU64 objExportImportRefCount; + +// +// Static functions for internal use to code in this file. +// +static NV_STATUS RmRefObjExportImport (void); +static void RmUnrefObjExportImport (void); + +static RmObjExportHandle RmGenerateObjExportHandle (void); +static NV_STATUS RmUnrefObjExportHandle (RmObjExportHandle hObject); + +// +// Free the RmObjExportHandle. +// +static NV_STATUS RmUnrefObjExportHandle(RmObjExportHandle hObject) +{ + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + void *pHandleRef = mapFind(&objExportHandleMap, hObject); + if (pHandleRef == NULL) + { + return NV_ERR_OBJECT_NOT_FOUND; + } + + if (pRmApi->Free(pRmApi, + hObjExportRmClient, + (NvHandle)mapKey(&objExportHandleMap, pHandleRef)) != NV_OK) + { + NV_PRINTF(LEVEL_WARNING, + "Exported object trying to free was zombie in %s\n", + __FUNCTION__); + } + + mapRemove(&objExportHandleMap, pHandleRef); + + return NV_OK; +} + +// +// Generate unique RmObjExportHandle. +// +static RmObjExportHandle RmGenerateObjExportHandle(void) +{ + // + // The object export handle belongs to range of 0 to + // (MAX_OBJ_EXPORT_HANDLES - 1). + // + // Handle 0 is considered as invalid object handle, this function generates + // handle from range of 1 to (MAX_OBJ_EXPORT_HANDLES - 1). + // + #define MAX_OBJ_EXPORT_HANDLES 0x80000 + + static NvHandle hObjExportHandleNext = 1; + + RmObjExportHandle hStartHandle = hObjExportHandleNext; + RmObjExportHandle hObject = 0; + + do + { + void *pHandleRef; + + hObject = hObjExportHandleNext++; + /* Reset hObjExportHandleNext to next valid handle */ + if (hObjExportHandleNext == MAX_OBJ_EXPORT_HANDLES) { + hObjExportHandleNext = 1; + } + + pHandleRef = mapFind(&objExportHandleMap, hObject); + + if (hObject != hObjExportRmClient && pHandleRef == NULL) + { + break; + } + else + { + hObject = 0; + } + + } while(hObjExportHandleNext != hStartHandle); + + if (hObject != 0) + { + void *pHandleRef = mapInsertNew(&objExportHandleMap, hObject); + + if (pHandleRef == NULL) + { + hObject = 0; + } + } + + return hObject; +} + +// +// Validate that the given hObject is not one of our internally used handles. +// +// Note that mapFind(&objExportHandleMap, hObject) could still fail; that is the +// caller's responsibility. +// +static NvBool RmValidateHandleAgainstInternalHandles(RmObjExportHandle hObject) +{ + RmObjExportDeviceMapIter iter; + + // + // No external RmObjExportHandle could be valid if hObjExportRmClient has + // not been allocated yet, or if it is equal to any of the handles used + // internally by code in this file. + // + if (objExportImportRefCount == 0 || hObjExportRmClient == 0 || + hObject == hObjExportRmClient) + { + return NV_FALSE; + } + + iter = mapIterAll(&objExportDeviceMap); + while (mapIterNext(&iter)) + { + RmObjExportDevice *pRmObjExportDevice = iter.pValue; + + if (pRmObjExportDevice->hRmDevice != 0 && + (hObject == pRmObjExportDevice->hRmDevice || + hObject == pRmObjExportDevice->hRmSubDevice || + hObject == pRmObjExportDevice->hGpuInstSub)) + { + return NV_FALSE; + } + } + + return NV_TRUE; +} + +// +// Increment reference count of static objects internally +// used by code in this file. +// +static NV_STATUS RmRefObjExportImport(void) +{ + NV_STATUS rmStatus = NV_OK; + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + if ((objExportImportRefCount++) != 0) + { + NV_ASSERT(hObjExportRmClient != 0); + NV_ASSERT(pMemAllocator != NULL); + return NV_OK; + } + + rmStatus = pRmApi->AllocWithHandle(pRmApi, + NV01_NULL_OBJECT, + NV01_NULL_OBJECT, + NV01_NULL_OBJECT, + NV01_ROOT, + &hObjExportRmClient, + sizeof(hObjExportRmClient)); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Unable to alloc root in %s\n", __FUNCTION__); + goto failed; + } + + pMemAllocator = portMemAllocatorCreateNonPaged(); + + if (pMemAllocator == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to alloc memory allocator in %s\n", + __FUNCTION__); + goto failed; + } + + mapInit(&objExportHandleMap, pMemAllocator); + mapInit(&objExportDeviceMap, pMemAllocator); + + return NV_OK; + +failed: + + RmUnrefObjExportImport(); + + return rmStatus; +} + +// +// Decrement reference count of static objects internally used by code in this +// file, and free them if reference count reaches to zero. +// +static void RmUnrefObjExportImport(void) +{ + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + if ((--objExportImportRefCount) != 0) + { + return; + } + + if (pMemAllocator != NULL) + { + mapDestroy(&objExportDeviceMap); + mapDestroy(&objExportHandleMap); + + portMemAllocatorRelease(pMemAllocator); + pMemAllocator = NULL; + } + + if (hObjExportRmClient != 0) + { + NV_STATUS rmStatus = pRmApi->Free(pRmApi, + hObjExportRmClient, + hObjExportRmClient); + + NV_ASSERT(rmStatus == NV_OK); + hObjExportRmClient = 0; + } +} + +NV_STATUS RmExportObject(NvHandle hSrcClient, NvHandle hSrcObject, + RmObjExportHandle *pDstObject, NvU32 *pDeviceInstance) +{ + RmObjExportHandle hDstObject; + NvU32 deviceInstance = NV_MAX_DEVICES; + NvBool bClientAsDstParent = NV_FALSE; + NV_STATUS status; + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + MIG_INSTANCE_REF ref = kmigmgrMakeNoMIGReference(); + NvU64 deviceMapIdx = 0; + RmObjExportDevice *pObjExportDevice = NULL; + RsResourceRef *pSrcResourceRef; + RsResourceRef *pDeviceRef; + + if (pDstObject == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // + // Find the device instance on which the rm object exists. + // + NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, + serverutilGetResourceRef(hSrcClient, hSrcObject, &pSrcResourceRef)); + + status = refFindAncestorOfType(pSrcResourceRef, classId(Device), &pDeviceRef); + if (status == NV_OK) + { + Device *pDevice = dynamicCast(pDeviceRef->pResource, Device); + OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); + NvU64 swizzId = KMIGMGR_MAX_GPU_SWIZZID; + + if (IS_MIG_IN_USE(pGpu)) + { + KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); + + status = kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, + pDevice, &ref); + if (status == NV_OK) + { + swizzId = ref.pKernelMIGGpuInstance->swizzId; + } + } + + deviceMapIdx = pDevice->deviceInst | (swizzId << 32); + deviceInstance = pDevice->deviceInst; + pObjExportDevice = mapFind(&objExportDeviceMap, deviceMapIdx); + } + else + { + bClientAsDstParent = NV_TRUE; + } + + status = RmRefObjExportImport(); + + if (status != NV_OK) + { + return status; + } + + if (!bClientAsDstParent && + ((pObjExportDevice == NULL) || + serverutilValidateNewResourceHandle(hObjExportRmClient, + pObjExportDevice->hRmDevice))) + { + // + // Device object has not been created or it got destroyed in the + // teardown path of device instance destruction; allocate a fresh device + // object. + // + NV0080_ALLOC_PARAMETERS params; + NV2080_ALLOC_PARAMETERS subdevParams; + + if (pObjExportDevice == NULL) + { + pObjExportDevice = mapInsertNew(&objExportDeviceMap, deviceMapIdx); + + pObjExportDevice->hRmDevice = RmGenerateObjExportHandle(); + pObjExportDevice->hRmSubDevice = RmGenerateObjExportHandle(); + pObjExportDevice->hGpuInstSub = NV01_NULL_OBJECT; + + if (pObjExportDevice->hRmDevice == 0 || + pObjExportDevice->hRmSubDevice == 0) + { + NV_PRINTF(LEVEL_ERROR, "Failed to allocate object handles in %s\n", + __FUNCTION__); + + mapRemove(&objExportDeviceMap, pObjExportDevice); + + status = NV_ERR_NO_MEMORY; + goto done; + } + } + + portMemSet(¶ms, 0, sizeof(NV0080_ALLOC_PARAMETERS)); + + params.deviceId = deviceInstance; + params.hClientShare = hObjExportRmClient; + + status = pRmApi->AllocWithHandle(pRmApi, + hObjExportRmClient, + hObjExportRmClient, + pObjExportDevice->hRmDevice, + NV01_DEVICE_0, + ¶ms, + sizeof(params)); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Unable to alloc device in %s\n", + __FUNCTION__); + goto done; + } + + portMemSet(&subdevParams, 0, sizeof(NV2080_ALLOC_PARAMETERS)); + + subdevParams.subDeviceId = 0; + + status = pRmApi->AllocWithHandle(pRmApi, + hObjExportRmClient, + pObjExportDevice->hRmDevice, + pObjExportDevice->hRmSubDevice, + NV20_SUBDEVICE_0, + &subdevParams, + sizeof(subdevParams)); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Unable to alloc subdevice in %s\n", + __FUNCTION__); + + (void) pRmApi->Free(pRmApi, hObjExportRmClient, + pObjExportDevice->hRmDevice); + goto done; + } + + // Allocate MIG subscription, if needed + if ((ref.pKernelMIGGpuInstance != NULL) && + (ref.pKernelMIGGpuInstance->swizzId < KMIGMGR_MAX_GPU_SWIZZID)) + { + NVC637_ALLOCATION_PARAMETERS giSubAllocParams; + + portMemSet(&giSubAllocParams, 0, sizeof(giSubAllocParams)); + giSubAllocParams.swizzId = ref.pKernelMIGGpuInstance->swizzId; + + pObjExportDevice->hGpuInstSub = RmGenerateObjExportHandle(); + + status = pRmApi->AllocWithHandle(pRmApi, + hObjExportRmClient, + pObjExportDevice->hRmSubDevice, + pObjExportDevice->hGpuInstSub, + AMPERE_SMC_PARTITION_REF, + &giSubAllocParams, + sizeof(giSubAllocParams)); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Unable to alloc gpu instance subscription in %s\n", + __FUNCTION__); + + (void) pRmApi->Free(pRmApi, hObjExportRmClient, + pObjExportDevice->hRmDevice); + goto done; + } + } + } + + hDstObject = RmGenerateObjExportHandle(); + + if (hDstObject == 0) + { + NV_PRINTF(LEVEL_ERROR, "Failed to allocate object handle in %s\n", + __FUNCTION__); + status = NV_ERR_NO_MEMORY; + goto done; + } + + // If duping under device handle fails, try subdevice handle. + status = pRmApi->DupObject(pRmApi, + hObjExportRmClient, + bClientAsDstParent ? hObjExportRmClient : + pObjExportDevice->hRmDevice, + &hDstObject, + hSrcClient, + hSrcObject, + 0 /* flags */); + if (status != NV_OK) + { + if (!bClientAsDstParent && (status == NV_ERR_INVALID_OBJECT_PARENT)) + { + NV_PRINTF(LEVEL_INFO, + "pRmApi->DupObject(Dev, failed due to invalid parent in %s." + " Now attempting DupObject with Subdev handle.\n", + __FUNCTION__); + + status = pRmApi->DupObject(pRmApi, + hObjExportRmClient, + pObjExportDevice->hRmSubDevice, + &hDstObject, + hSrcClient, + hSrcObject, + 0 /* flags */); + if (status != NV_OK) + { + RmUnrefObjExportHandle(hDstObject); + + NV_PRINTF(LEVEL_ERROR, + "pRmApi->DupObject(Subdev, failed with error code 0x%x in %s\n", + status, __FUNCTION__); + goto done; + } + } + else + { + RmUnrefObjExportHandle(hDstObject); + + NV_PRINTF(LEVEL_ERROR, + "pRmApi->DupObject(Dev, failed with error code 0x%x in %s\n", + status, __FUNCTION__); + goto done; + } + } + + if (pDeviceInstance != NULL) + { + *pDeviceInstance = deviceInstance; + } + + *pDstObject = hDstObject; + +done: + if (status != NV_OK) + { + RmUnrefObjExportImport(); + } + + return status; +} + +void RmFreeObjExportHandle(RmObjExportHandle hObject) +{ + if (!RmValidateHandleAgainstInternalHandles(hObject)) + { + NV_PRINTF(LEVEL_ERROR, "Invalid handle to exported object in %s\n", + __FUNCTION__); + return; + } + + RmUnrefObjExportHandle(hObject); + + RmUnrefObjExportImport(); +} + +NV_STATUS RmImportObject(NvHandle hDstClient, NvHandle hDstParent, + NvHandle *phDstObject, RmObjExportHandle hSrcObject, + NvU8 *pObjectType) +{ + NV_STATUS status; + NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS params; + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + if (!RmValidateHandleAgainstInternalHandles(hSrcObject)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if (mapFind(&objExportHandleMap, hSrcObject) == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if (pObjectType != NULL) + { + params.hObject = hSrcObject; + // + // rmApiGetEffectiveAddrSpace expect mapping flags to be set as DIRECT for + // GPU cacheable Sysmem for Pre-Ampere chips. We are not doing any mapping + // here, so passing this as workaround to get the expected address space. + // + params.mapFlags = FLD_SET_DRF(OS33, _FLAGS, _MAPPING, _DIRECT, 0); + params.addrSpaceType = \ + NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_INVALID; + + status = pRmApi->Control(pRmApi, hObjExportRmClient, hObjExportRmClient, + NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE, + ¶ms, sizeof(params)); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "GET_ADDR_SPACE_TYPE failed with error code 0x%x in %s\n", + status, __FUNCTION__); + return status; + } + + switch (params.addrSpaceType) + { + case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_SYSMEM: + *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_SYSMEM; + break; + case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_VIDMEM: + *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_VIDMEM; + break; + case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC: + *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC; + break; +#if defined(NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC) && \ + defined(NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC) + case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC: + *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC; + break; +#endif + default: + NV_ASSERT_OK_OR_RETURN(NV_ERR_INVALID_ARGUMENT); + } + } + + status = pRmApi->DupObject(pRmApi, hDstClient, hDstParent, phDstObject, + hObjExportRmClient, hSrcObject, + 0 /* flags */); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "pRmApi->DupObject(pRmApi, failed with error code 0x%x in %s\n", + status, __FUNCTION__); + return status; + } + + return NV_OK; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/subdevice_ctrl_os_unix.c b/userspace/sdks/nv570/nvalloc/unix/src/subdevice_ctrl_os_unix.c new file mode 100644 index 0000000..fdbdb6b --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/subdevice_ctrl_os_unix.c @@ -0,0 +1,97 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> // NV device driver interface +#include <nv-priv.h> +#include <os/os.h> +#include <nvos.h> +#include <osapi.h> +#include "gpu/gpu.h" +#include "gpu/gpu_resource.h" +#include "gpu/subdevice/subdevice.h" +#include <diagnostics/journal.h> +#include "gpu/mem_mgr/mem_desc.h" +#include "mem_mgr/mem.h" +#include <nvpcie.h> +#include <core/locks.h> +#include "rmapi/rs_utils.h" +#include "rmapi/client_resource.h" +#include <class/cl0000.h> +#include <class/cl90cd.h> +#include <class/cl0005.h> // NV01_EVENT +#include <class/cl003e.h> // NV01_MEMORY_SYSTEM +#include <class/cl844c.h> // G84_PERFBUFFER +#include <ctrl/ctrl0000/ctrl0000gpu.h> +#include <ctrl/ctrl0000/ctrl0000unix.h> +#include <ctrl/ctrl2080/ctrl2080gpu.h> +#include <ctrl/ctrl2080/ctrl2080unix.h> + +/*! + * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS + * RmControl request. It will check if the GPU video memory will be + * persistent during system suspend/resume cycle. + * + * @param[in] pSubdevice + * @param[in,out] pParams + * + * @return + * NV_OK Success + */ +NV_STATUS +subdeviceCtrlCmdOsUnixVidmemPersistenceStatus_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS *pParams +) +{ + OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + pParams->bVidmemPersistent = (nv->preserve_vidmem_allocations || nvp->s0ix_pm_enabled); + + return NV_OK; +} + +/*! + * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS + * RmControl request. It sets restore TGP flag which is used + * to restore TGP limits when client is killed. + * + * @param[in] pSubdevice + * @param[in] pParams + * + * @return + * NV_OK Success + */ +NV_STATUS +subdeviceCtrlCmdOsUnixUpdateTgpStatus_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS *pParams +) +{ + pSubdevice->bUpdateTGP = pParams->bUpdateTGP; + + return NV_OK; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/unix_console.c b/userspace/sdks/nv570/nvalloc/unix/src/unix_console.c new file mode 100644 index 0000000..28b597b --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/unix_console.c @@ -0,0 +1,412 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <core/locks.h> +#include <ctrl/ctrl0080/ctrl0080unix.h> +#include <gpu/device/device.h> +#include <gpu/gpu.h> +#include <nv-priv.h> +#include <nv.h> +#include <osapi.h> +#include <gpu/mem_mgr/mem_mgr.h> +#include <platform/sli/sli.h> + +#include <vgpu/rpc.h> +#include "vgpu/vgpu_events.h" + +static NV_STATUS +unixCallVideoBIOS +( + OBJGPU *pGpu, + NvU32 *eax, + NvU32 *ebx +) +{ + NV_STATUS status = NV_ERR_NOT_SUPPORTED; + + if (NVCPU_IS_X86_64) + { + NvU32 eax_in = *eax; + NvU32 ebx_in = *ebx; + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI)) + { + return NV_ERR_NOT_SUPPORTED; + } + + NV_PRINTF(LEVEL_INFO, "unixCallVideoBIOS: 0x%x 0x%x, vga_satus = %d\n", *eax, *ebx, NV_PRIMARY_VGA(NV_GET_NV_STATE(pGpu))); + + status = nv_vbios_call(pGpu, eax, ebx); + + // this was originally changed for nt in changelist 644223 + if (*eax != 0x4f) + { + NV_PRINTF(LEVEL_ERROR, + "int10h(%04x, %04x) vesa call failed! (%04x, %04x)\n", + eax_in, ebx_in, *eax, *ebx); + status = NV_ERR_GENERIC; + } + } + + return status; +} + +static void +RmSaveDisplayState +( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + NvBool use_vbios = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv); + NvU32 eax, ebx; + NV_STATUS status; + NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS preUnixConsoleParams = {0}; + NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS postUnixConsoleParams = {0}; + + if (IS_VIRTUAL(pGpu) || (pKernelDisplay == NULL) || nv->client_managed_console) + { + return; + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI)) + { + NV_PRINTF(LEVEL_INFO, "RM fallback doesn't support saving of efifb console\n"); + return; + } + + preUnixConsoleParams.bSave = NV_TRUE; + preUnixConsoleParams.bUseVbios = use_vbios; + + NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE, + &preUnixConsoleParams, sizeof(preUnixConsoleParams)), return); + + if (preUnixConsoleParams.bReturnEarly) + { + return; + } + + if (use_vbios) + { + // + // Attempt to identify the currently set VESA mode; assume + // vanilla VGA text if the VBIOS call fails. + // + eax = 0x4f03; + ebx = 0; + if (NV_OK == unixCallVideoBIOS(pGpu, &eax, &ebx)) + { + nvp->vga.vesaMode = (ebx & 0x3fff); + } + else + { + nvp->vga.vesaMode = 3; + } + } + + postUnixConsoleParams.bSave = NV_TRUE; + postUnixConsoleParams.bUseVbios = use_vbios; + + NV_CHECK_OK(status, LEVEL_ERROR, + pRmApi->Control(pRmApi, nv->rmapi.hClient,nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE, + &postUnixConsoleParams, sizeof(postUnixConsoleParams))); +} + +static void RmRestoreDisplayState +( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NvBool use_vbios = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv);; + KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + NV_STATUS status; + NvU32 eax, ebx; + NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS preUnixConsoleParams = {0}; + NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS postUnixConsoleParams = {0}; + + // + // Since vGPU does all real hardware management in the host, + // there is nothing to do at this point in the guest OS. + // + if (IS_VIRTUAL(pGpu) || (pKernelDisplay == NULL) || nv->client_managed_console) + { + return; + } + + // + // Fix up DCB index VBIOS scratch registers. + // The strategies employed are: + // + // SBIOS/VBIOS: + // Clear the DCB index, and set the previous DCB index to the original + // value. This allows the VBIOS (during the int10h mode-set) to + // determine which display to enable, and to set the head-enabled bit + // as needed (see bugs #264873 and #944398). + // + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI)) + { + NV_PRINTF(LEVEL_INFO, "RM fallback doesn't support efifb console restore\n"); + return; + } + + preUnixConsoleParams.bUseVbios = use_vbios; + preUnixConsoleParams.bSave = NV_FALSE; + + NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, pRmApi->Control(pRmApi, nv->rmapi.hClient, + nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE, + &preUnixConsoleParams, sizeof(preUnixConsoleParams)), return); + + if (preUnixConsoleParams.bReturnEarly) + { + return; + } + + if (use_vbios) + { + eax = 0x4f02; + ebx = nvp->vga.vesaMode; + + if (NV_OK == unixCallVideoBIOS(pGpu, &eax, &ebx)) + { + postUnixConsoleParams.bVbiosCallSuccessful = NV_TRUE; + } + } + + postUnixConsoleParams.bSave = NV_FALSE; + postUnixConsoleParams.bUseVbios = use_vbios; + + NV_CHECK_OK(status, LEVEL_ERROR, + pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE, + &postUnixConsoleParams, sizeof(postUnixConsoleParams))); +} + +static void +RmChangeResMode +( + OBJGPU *pGpu, + NvBool hires +) +{ + if (hires) + { + SLI_LOOP_START(SLI_LOOP_FLAGS_NONE) + + RmSaveDisplayState(pGpu); + + SLI_LOOP_END + } + else + { + SLI_LOOP_START(SLI_LOOP_FLAGS_NONE) + + RmRestoreDisplayState(pGpu); + // + // vGPU: + // + // Since vGPU does all real hardware management in the host, if we + // are in guest OS (where IS_VIRTUAL(pGpu) is true), do an RPC to + // the host to trigger switch from HIRES to (LORES)VGA. + // + if (IS_VIRTUAL(pGpu)) + { + NV_STATUS status = NV_OK; + NV_RM_RPC_SWITCH_TO_VGA(pGpu, status); + } + + SLI_LOOP_END + } +} + +NV_STATUS NV_API_CALL +rm_save_low_res_mode +( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + RmSaveDisplayState(pGpu); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +NV_STATUS +deviceCtrlCmdOsUnixVTSwitch_IMPL +( + Device *pDevice, + NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS *pParams +) +{ + OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NvBool hires; + NvBool bChangeResMode = NV_TRUE; + + switch (pParams->cmd) + { + case NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_SAVE_VT_STATE: + hires = NV_TRUE; + break; + + case NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_RESTORE_VT_STATE: + hires = NV_FALSE; + break; + + case NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_CONSOLE_RESTORED: + bChangeResMode = NV_FALSE; + break; + + default: + return NV_ERR_INVALID_ARGUMENT; + } + + RmUpdateGc6ConsoleRefCount(nv, + pParams->cmd != NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_SAVE_VT_STATE); + + if (!bChangeResMode) + { + return NV_OK; + } + + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FB) == NV_OK) + { + RmChangeResMode(pGpu, hires); + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + else + { + NV_PRINTF(LEVEL_INFO,"%s: Failed to acquire GPU lock", __FUNCTION__); + } + return NV_OK; +} + +NV_STATUS deviceCtrlCmdOsUnixVTGetFBInfo_IMPL +( + Device *pDevice, + NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS *pParams +) +{ + OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); + NvBool bContinue = NV_TRUE; + + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FB) == NV_OK) + { + // See if the console is on one of the subdevices of this device. + portMemSet(pParams, 0, sizeof(*pParams)); + + SLI_LOOP_START(SLI_LOOP_FLAGS_NONE) + + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((memmgrGetReservedConsoleMemDesc(pGpu, pMemoryManager) != NULL) && bContinue) + { + NvU64 baseAddr, size; + NvU32 width, height, depth, pitch; + + // There should only be one. + NV_ASSERT(pParams->width == 0); + + pParams->subDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu); + + // Console is either mapped to BAR1 or BAR2 + 16 MB + nv_get_screen_info(nv, &baseAddr, &width, &height, &depth, + &pitch, &size); + + pParams->baseAddress = baseAddr; + pParams->size = size; + pParams->width = (NvU16)width; + pParams->height = (NvU16)height; + pParams->depth = (NvU16)depth; + pParams->pitch = (NvU16)pitch; + + if (baseAddr != 0) + { + bContinue = NV_FALSE; + } + } + + SLI_LOOP_END + + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + else + { + NV_PRINTF(LEVEL_INFO,"%s: Failed to acquire GPU lock", __FUNCTION__); + } + + return NV_OK; +} + + +void +dispdeviceFillVgaSavedDisplayState +( + OBJGPU *pGpu, + NvU64 vgaAddr, + NvU8 vgaMemType, + NvBool vgaValid, + NvU64 workspaceAddr, + NvU8 workspaceMemType, + NvBool workspaceValid, + NvBool baseValid, + NvBool workspaceBaseValid +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvBool use_vbios = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv); + + if (use_vbios) + { + nvp->vga.base.addr = vgaAddr; + nvp->vga.base.memTarget = vgaMemType; + nvp->vga.base.valid = vgaValid; + nvp->vga.baseValid = baseValid; + + nvp->vga.workspaceBase.addr = workspaceAddr; + nvp->vga.workspaceBase.memTarget = workspaceMemType; + nvp->vga.workspaceBase.valid = workspaceValid; + nvp->vga.workspaceBaseValid = workspaceBaseValid; + } +} + diff --git a/userspace/sdks/nv570/nvalloc/unix/src/unix_intr.c b/userspace/sdks/nv570/nvalloc/unix/src/unix_intr.c new file mode 100644 index 0000000..aef36ab --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/unix_intr.c @@ -0,0 +1,1050 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include <nv-priv.h> +#include <osapi.h> +#include <core/thread_state.h> +#include <core/locks.h> +#include <gpu_mgr/gpu_mgr.h> +#include <gpu/gpu.h> +#include "kernel/gpu/intr/intr.h" +#include "gpu/bif/kernel_bif.h" +#include "gpu/mmu/kern_gmmu.h" +#include "gpu/disp/kern_disp.h" +#include "gpu/disp/head/kernel_head.h" +#include <nv_sriov_defines.h> + +#include "platform/sli/sli.h" + +// +// Service Interrupts that are not Masked when using +// INTR_USE_INTR_MASK_FOR_LOCKING +// Note that everything done here is without holding the GPUs Lock. +// Returning NV_OK means an interrupt has been serviced +// +static NV_STATUS _osIsrIntrMask_GpusUnlocked(OBJGPU *pGpu, THREAD_STATE_NODE *pThreadState) +{ + NV_STATUS rmStatus = NV_ERR_NO_INTR_PENDING; + Intr *pIntr = GPU_GET_INTR(pGpu); + MC_ENGINE_BITVECTOR intrPending, intrEngMask; + NvU32 vblankIntrServicedHeadMask = 0; + KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + NvBool bFailedLockAcquire = NV_FALSE; + + if (gpuIsGpuFullPower(pGpu) == NV_FALSE) + { + return rmStatus; + } + + intrGetPendingLowLatencyHwDisplayIntr_HAL(pGpu, pIntr, &intrPending, pThreadState); + + // Only disp interrupt remains in INTR_MASK + if ((pKernelDisplay != NULL) && + (( pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP_LOW)) || + (!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP)))) + { + if (kdispAcquireLowLatencyLockConditional(&pKernelDisplay->lowLatencyLock)) + { + + if (pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_AGGRESSIVE_VBLANK_HANDLING)) + { + NV_ASSERT_OK( + kdispOptimizePerFrameOsCallbacks(pGpu, pKernelDisplay, NV_TRUE, + pThreadState, + &vblankIntrServicedHeadMask, + &intrPending)); + + // + // Need to setup deferred vblank handling for once the + // GPUs Lock is released since we cleared vblank intrs. + // + if (vblankIntrServicedHeadMask) + { + vblankIntrServicedHeadMask |= kdispGetDeferredVblankHeadMask(pKernelDisplay); + kdispSetDeferredVblankHeadMask(pKernelDisplay, vblankIntrServicedHeadMask); + } + } + else + { + kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0, + (VBLANK_STATE_PROCESS_LOW_LATENCY | + VBLANK_STATE_PROCESS_CALLED_FROM_ISR), + pThreadState, + &vblankIntrServicedHeadMask, + &intrPending); + } + + kdispReleaseLowLatencyLock(&pKernelDisplay->lowLatencyLock); + } + else + { + // + // If we lose this, then another thread is already servicing the low latency interrupts. + // Claim the interrupt, but go through the below disable code so that we don't immediately + // cause another interrupt indefinitely which could starve the other thread that already + // has the lock + // + rmStatus = NV_OK; + bFailedLockAcquire = NV_TRUE; + } + + // + // Bug 4512542: The check for INTR_MASK is currently redundant since it's + // already checked outside this function, but we would like to unconditionally + // call this function if we can service the vblank first. + // + if (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING)) + { + // + // If an unmasked intr is still pending after the above, that means a + // intr other than the vblank, so we'll MSK this off + // for now and handle once the GPUs Lock is released. + // + intrGetPendingLowLatencyHwDisplayIntr_HAL(pGpu, pIntr, &intrPending, pThreadState); + + // + // Only disp interrupt remains in INTR_MASK + // Don't re-mask it if we have an entirely separate interrupt line, unless we have lost the race above. + // + if (( pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP_LOW) && bFailedLockAcquire) || + (!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP))) + { + NvU32 intrMaskFlags; + NvU64 oldIrql; + + oldIrql = rmIntrMaskLockAcquire(pGpu); + + // Skip the IntrMask update if told to by another thread. + intrMaskFlags = intrGetIntrMaskFlags(pIntr); + if ((intrMaskFlags & INTR_MASK_FLAGS_ISR_SKIP_MASK_UPDATE) == 0) + { + if (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING)) + { + intrGetIntrMask_HAL(pGpu, pIntr, &intrEngMask, pThreadState); + + // Only disp interrupt remains in INTR_MASK + bitVectorClr(&intrEngMask, MC_ENGINE_IDX_DISP); + + if (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_TOP_EN_FOR_VBLANK_HANDLING)) + { + intrSetDisplayInterruptEnable_HAL(pGpu, pIntr, NV_FALSE, pThreadState); + } + else + { + intrSetIntrMask_HAL(pGpu, pIntr, &intrEngMask, pThreadState); + } + } + } + + rmIntrMaskLockRelease(pGpu, oldIrql); + } + } + + if (vblankIntrServicedHeadMask) + { + rmStatus = NV_OK; + } + } + + return rmStatus; +} + +static NvBool osInterruptPending( + OBJGPU *pGpu, + NvBool *serviced, + THREAD_STATE_NODE *pThreadState +) +{ + KernelDisplay *pKernelDisplay; + NvBool pending, sema_release; + THREAD_STATE_NODE threadState; + Intr *pIntr = NULL; + MC_ENGINE_BITVECTOR intr0Pending; + MC_ENGINE_BITVECTOR intr1Pending; + + *serviced = NV_FALSE; + pending = NV_FALSE; + sema_release = NV_TRUE; + NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer + PORT_MEM_ALLOCATOR *pIsrAllocator; + + // + // GPU interrupt servicing ("top half") + // + // Top-level processing of GPU interrupts is performed using the + // steps below; although the code is straight forward, there + // are a few points to be aware of: + // + // 1) The GPUs lock is acquired for two reasons: to allow + // looping over GPUs atomically in SLI and to sanity + // check the PCI configuration space of any initialized + // GPUs. If the acquisition fails, the early return + // is acceptable since GPU interrupts are disabled while + // the lock is held; note that returning success + // in this case could interfere with the processing + // of third-party device interrupts if the IRQ is shared. + // Due to the above, some interrupts may be reported as + // unhandled if invocations of the ISR registered with + // the kernel are not serialized. This is bad, but + // ignored by currently supported kernels, provided most + // interrupts are handled. + // + // 2) Since acquisition of the lock disables interrupts + // on all initialized GPUs, NV_PMC_INTR_EN_0 can not be + // relied up on to determine whether interrupts are + // expected from a given GPU. The code below is therefore + // forced to rely on software state. NV_PMC_INTR_EN_0 + // is read only as a sanity check to guard against + // invalid GPU state (lack of PCI memory access, etc.). + // + // 3) High priority interrupts (VBLANK, etc.), are serviced in + // this function, service of all other interrupts is + // deferred until a bottom half. If a bottom half needs + // to be scheduled, release of the GPUs lock is + // likewise deferred until completion of the bottom half. + // + // 4) To reduce the risk of starvation, an effort is made to + // consolidate processing of interrupts pending on + // all GPUs sharing a given IRQ. + // + // 5) Care is taken to ensure that the consolidated interrupt + // processing is performed in the context of a GPU + // that has interrupts pending. Else if additional ISR + // processing via a bottom-half is required, this + // bottom-half ISR might race against the GPU's shut-down + // path. + // + + pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); + tlsIsrInit(pIsrAllocator); + + // + // Also check if we need to acquire the GPU lock at all and get critical interrupts + // This should not violate (1) from above since we are not servicing the GPUs in SLI, + // only checking their state. + // + // To do so, two steps are required: + // Step 1: Check if we can service nonstall interrupts outside the GPUs lock. Otherwise + // we have to acquire the GPUs lock to service the nonstall interrupts anyway, and we + // can't get around acquiring the GPUs lock. + // + // Step 2 is inline below + // + NvBool bIsAnyStallIntrPending = NV_TRUE; + NvBool bIsAnyBottomHalfStallPending = NV_FALSE; + NvU32 isDispPendingPerGpu = 0, isDispLowLatencyPendingPerGpu = 0; + NvU32 isTmrPendingPerGpu = 0; + + { + threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + bIsAnyStallIntrPending = NV_FALSE; + + { + pIntr = GPU_GET_INTR(pGpu); + KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + bIsAnyStallIntrPending = NV_TRUE; + } + else + if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))) + { + NvBool bCtxswLog = NV_FALSE; + intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState); + intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog); + if (!bitVectorTestAllCleared(&intr1Pending)) + { + intrServiceNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState); + *serviced = NV_TRUE; + } + + // + // Step 2: Service nonstall interrupts and check if there are any stall interrupts + // pending on any GPUs. If there are, then we will take the GPU lock. + // + // Bug 4223192: calling intrGetPendingStall_HAL is rather expensive, + // so save off the critical interrupts to be handled in the top half. + // + + // vblank read leaf 4 + read the rest + + NvU32 intrStaticReg; + intrGetPendingDispLowLatencyAndStaticReg_HAL(pGpu, pIntr, &intrStaticReg, &intr0Pending, &threadState); + intrGetRemainingPendingStall_HAL(pGpu, pIntr, intrStaticReg, &intr0Pending, &threadState); + + if (!bitVectorTestAllCleared(&intr0Pending)) + { + if ((pKernelDisplay != NULL) && pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE)) + { + if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP_LOW)) + { + isDispLowLatencyPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu)); + + // We will attempt to handle this separately from the other stall interrupts + bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP_LOW); + } + } + else + { + if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP)) + { + isDispPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu)); + + // We will attempt to handle this separately from the other stall interrupts + bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP); + } + } + + if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && + !IS_VIRTUAL(pGpu) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_TMR)) + { + isTmrPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu)); + + // We will clear this in the top half + bitVectorClr(&intr0Pending, MC_ENGINE_IDX_TMR); + } + + if (!bitVectorTestAllCleared(&intr0Pending)) + { + // + // If any other stall interrupts are pending, + // they will be handled in the bottom half. + // + bIsAnyBottomHalfStallPending = NV_TRUE; + } + + bIsAnyStallIntrPending = NV_TRUE; + } + } + } + threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + } + + // LOCK: try to acquire GPUs lock + if (bIsAnyStallIntrPending && + (rmDeviceGpuLocksAcquire(pGpu, GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_ISR) == NV_OK)) + { + threadStateInitISRAndDeferredIntHandler(&threadState, + pGpu, THREAD_STATE_FLAGS_IS_ISR); + + { + pIntr = GPU_GET_INTR(pGpu); + pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_soc_irq_type_t irq_type; + + irq_type = nv_get_current_irq_type(nv); + + if (irq_type == NV_SOC_IRQ_DISPLAY_TYPE) + { + NvU32 dispIntr0Pending = 0; + + if (kdispAcquireLowLatencyLockConditional(&pKernelDisplay->lowLatencyLock)) + { + kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0, + (VBLANK_STATE_PROCESS_LOW_LATENCY | + VBLANK_STATE_PROCESS_CALLED_FROM_ISR), + &threadState, + NULL, + NULL); + + kdispReleaseLowLatencyLock(&pKernelDisplay->lowLatencyLock); + + *serviced = NV_TRUE; + } + + kdispGetPendingInterrupts_HAL(pGpu, pKernelDisplay, &dispIntr0Pending); + + if (dispIntr0Pending != 0) + { + pending = NV_TRUE; + sema_release = NV_FALSE; + } + } + } + else if ((pIntr != NULL) && INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)) + { + // If interrupt enable is garbage the GPU is probably in a bad state + if (!(intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX)) + { + // + // If lockless interrupt handling was enabled, we have already cached which critical + // interrupts are pending. We only set the appropriate bits in the intr0Pending bitvector + // for the subsequent code to service them in the same manner as the lockless nonstall + // interrupt handling disabled case. But we also clear them from intr0Pending and + // for disp, check if they're still pending afterwards. We already checked whether any + // other bottom half stall interrupts are pending in bIsAnyBottomHalfStallPending above. + // + // After all this, the combination of bIsAnyBottomHalfStallPending and intr0Pending + // contains whether any stall interrupts are still pending, so check both to determine if + // we need a bottom half. + // + bitVectorClrAll(&intr0Pending); + + if ((pKernelDisplay != NULL) && pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE)) + { + if ((isDispLowLatencyPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0) + { + bitVectorSet(&intr0Pending, MC_ENGINE_IDX_DISP_LOW); + } + } + else + { + if ((isDispPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0) + { + bitVectorSet(&intr0Pending, MC_ENGINE_IDX_DISP); + } + + } + + if ((isTmrPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0) + { + bitVectorSet(&intr0Pending, MC_ENGINE_IDX_TMR); + } + } + + if ((pKernelDisplay != NULL) && + (( pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP_LOW)) || + (!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP)))) + { + MC_ENGINE_BITVECTOR intrDispPending; + NvBool bIsLowLatencyIntrPending; + + // If this is true, then we're here because it is true + bIsLowLatencyIntrPending = pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE); + + if (kdispAcquireLowLatencyLockConditional(&pKernelDisplay->lowLatencyLock)) + { + if (pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_AGGRESSIVE_VBLANK_HANDLING)) + { + // Need to queue a bottom half to service the interrupts since unix otherwise won't + kdispHandleAggressiveVblank(pGpu, pKernelDisplay, pThreadState, &intr0Pending); + } + else + { + kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0, + (VBLANK_STATE_PROCESS_LOW_LATENCY | + VBLANK_STATE_PROCESS_CALLED_FROM_ISR), + &threadState, + NULL, + &intr0Pending); + + } + + *serviced = NV_TRUE; + + kdispReleaseLowLatencyLock(&pKernelDisplay->lowLatencyLock); + } + + if (bIsLowLatencyIntrPending) + { + bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP_LOW); + } + else + { + bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP); + } + + intrGetPendingLowLatencyHwDisplayIntr_HAL(pGpu, pIntr, &intrDispPending, &threadState); + + // + // If deferred vblank is used (depends on PDB_PROP_KDISP_AGGRESSIVE_VBLANK_HANDLING) + // and pending, we need to queue a bottom half to service the rest of the interrupts, + // so signal that an interrupt is still pending + // + if (kdispGetDeferredVblankHeadMask(pKernelDisplay) != 0) + { + bitVectorSet(&intrDispPending, MC_ENGINE_IDX_DISP); + } + + bitVectorOr(&intr0Pending, &intr0Pending, &intrDispPending); + } + + if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && + !IS_VIRTUAL(pGpu) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_TMR)) + { + // We have to clear the top level interrupt bit here since otherwise + // the bottom half will attempt to service the interrupt on the CPU + // side before GSP receives the notification and services it + intrClearLeafVector_HAL(pGpu, pIntr, intrGetVectorFromEngineId(pGpu, pIntr, MC_ENGINE_IDX_TMR, NV_FALSE), &threadState); + bitVectorClr(&intr0Pending, MC_ENGINE_IDX_TMR); + + NV_ASSERT_OK(intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_TMR_HANDLE)); + } + + { + bitVectorClrAll(&intr1Pending); + } + + if (bIsAnyBottomHalfStallPending || + !bitVectorTestAllCleared(&intr0Pending) || + !bitVectorTestAllCleared(&intr1Pending)) + { + pending = NV_TRUE; + sema_release = NV_FALSE; + } + } + } + + if (sema_release) + { + NV_ASSERT(!pending); + + // UNLOCK: release GPUs lock + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + else + { + rmDeviceGpuLockSetOwner(pGpu, GPUS_LOCK_OWNER_PENDING_DPC_REFRESH); + } + + threadStateFreeISRAndDeferredIntHandler(&threadState, + pGpu, THREAD_STATE_FLAGS_IS_ISR); + } + else if (bIsAnyStallIntrPending) + { + threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + + { + pIntr = GPU_GET_INTR(pGpu); + + if ((pIntr != NULL) && (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING))) + { + if (_osIsrIntrMask_GpusUnlocked(pGpu, &threadState) == NV_OK) + { + *serviced = NV_TRUE; + } + } + } + + threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + } + + tlsIsrDestroy(pIsrAllocator); + portMemAllocatorRelease(pIsrAllocator); + + return pending; +} + +NV_STATUS osIsr( + OBJGPU *pGpu +) +{ + NV_STATUS status = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvBool pending = NV_FALSE; + NvBool serviced = NV_FALSE; + Intr *pIntr; + + if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) + { + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */); + } + else + { + pIntr = GPU_GET_INTR(pGpu); + if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)) + { + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */); + kbifCheckAndRearmMSI(pGpu, pKernelBif); + } + } + } + + if (!pending && (IS_VIRTUAL(pGpu) || !serviced)) + status = NV_ERR_NO_INTR_PENDING; + else if (pending) + status = NV_WARN_MORE_PROCESSING_REQUIRED; + + return status; +} + +/* + * Helper function to determine when the RM SEMA/GPUS LOCK should toggle + * interrupts. Based on the state of the GPU - we must add cases here as we + * discover them. + * + * Noteworthy special cases: + * + * - Suspend/resume: the GPU could still be suspended and not accessible + * on the bus, while passive-level threads need to grab the GPUs + * lock, or other GPUs are being resumed and triggering interrupts. + * + * @param[in] pGpu OBJGPU pointer + * + * @return NV_TRUE if the RM SEMA/GPUS LOCK should toggle interrupts, NV_FALSE + * otherwise. + */ +NvBool osLockShouldToggleInterrupts(OBJGPU *pGpu) +{ + if (IS_DCE_CLIENT(pGpu)) + { + return NV_TRUE; + } + + return (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH) && + gpuIsStateLoaded(pGpu)); +} + +void osEnableInterrupts(OBJGPU *pGpu) +{ + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + // enable irq through os call + nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_TRUE); + return; + } + else + { + Intr *pIntr = GPU_GET_INTR(pGpu); + NvU32 intrEn; + + if (!pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING)) + NV_ASSERT(intrGetIntrEnFromHw_HAL(pGpu, pIntr, NULL) == INTERRUPT_TYPE_DISABLED); + + intrEn = intrGetIntrEn(pIntr); + intrSetIntrEnInHw_HAL(pGpu, pIntr, intrEn, NULL); + intrSetStall_HAL(pGpu, pIntr, intrEn, NULL); + + intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL); + } +} + +void osDisableInterrupts( + OBJGPU *pGpu, + NvBool bIsr +) +{ + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + // disable irq through os call + nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_FALSE); + return; + } + else + { + Intr *pIntr = GPU_GET_INTR(pGpu); + NvU32 new_intr_en_0 = INTERRUPT_TYPE_DISABLED; + + intrSetIntrEnInHw_HAL(pGpu, pIntr, new_intr_en_0, NULL); + intrSetStall_HAL(pGpu, pIntr, new_intr_en_0, NULL); + + intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL); + } +} + +static void RmIsrBottomHalf( + nv_state_t *pNv +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + OS_THREAD_HANDLE threadId; + Intr *pIntr = NULL; + NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer + PORT_MEM_ALLOCATOR *pIsrAllocator; + + pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); + tlsIsrInit(pIsrAllocator); + + // + // The owning thread changes as the ISR acquires the GPUs lock, + // but the bottom half releases it. Refresh the ThreadId owner to be + // correct here for the bottom half context. + // + osGetCurrentThread(&threadId); + rmDeviceGpuLockSetOwner(pGpu, threadId); + + { + threadStateInitISRAndDeferredIntHandler(pGpu->pDpcThreadState, pGpu, + THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER); + } + + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + gpumgrSetCurrentGpuInstance(pGpu->gpuInstance); + + pIntr = GPU_GET_INTR(pGpu); + + // + // Call disp service incase of SOC Display, + // TODO : with multi interrupt handling based on irq aux interrupts are serviced by dpAuxService + // See JIRA task TDS-4253. + // + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + if (nv_get_current_irq_type(nv) == NV_SOC_IRQ_DISPLAY_TYPE) + { + if (IS_DCE_CLIENT(pGpu)) + { + kdispAcquireLowLatencyLock(&pKernelDisplay->lowLatencyLock); + + kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0, + VBLANK_STATE_PROCESS_LOW_LATENCY, + pGpu->pDpcThreadState, + NULL, + NULL); + + kdispReleaseLowLatencyLock(&pKernelDisplay->lowLatencyLock); + } + else + { + } + } + } + else if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))) + { + intrServiceStall_HAL(pGpu, pIntr); + } + + gpumgrSetCurrentGpuInstance(NV_U32_MAX); + } + + // UNLOCK: release GPUs lock + rmDeviceGpuLocksReleaseAndThreadStateFreeDeferredIntHandlerOptimized(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + + tlsIsrDestroy(pIsrAllocator); + portMemAllocatorRelease(pIsrAllocator); +} + +static void RmIsrBottomHalfUnlocked( + nv_state_t *pNv +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + Intr *pIntr; + THREAD_STATE_NODE threadState; + + // In the GSP client scenario, the fatal fault interrupt is not shared + // by UVM and CPU-RM. Instead, it is handled entirely by GSP-RM. We + // therefore do not expect this function to be called. But if it is, bail + // without attempting to service interrupts. + if (IS_GSP_CLIENT(pGpu)) + { + return; + } + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // Grab GPU lock here as this kthread-item was enqueued without grabbing GPU lock + if (rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DPC) == NV_OK) + { + if (FULL_GPU_SANITY_CHECK(pGpu)) + { + pIntr = GPU_GET_INTR(pGpu); + + if (intrGetIntrEn(pIntr) != INTERRUPT_TYPE_DISABLED) + { + MC_ENGINE_BITVECTOR intrPending; + intrGetPendingStall_HAL(pGpu, pIntr, &intrPending, &threadState); + intrServiceNonStallBottomHalf(pGpu, pIntr, &intrPending, &threadState); + } + } + + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); +} + +NvBool NV_API_CALL rm_isr( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 *NeedBottomHalf +) +{ + NV_STATUS status; + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + OBJGPU *pGpu; + NvBool retval; + void *fp; + + if ((nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) == 0) + { + return NV_FALSE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL) + { + return NV_FALSE; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + // call actual isr function here + status = isrWrapper(pGpu->testIntr, pGpu); + + switch (status) + { + case NV_OK: + *NeedBottomHalf = NV_FALSE; + retval = NV_TRUE; + break; + case NV_WARN_MORE_PROCESSING_REQUIRED: + *NeedBottomHalf = NV_TRUE; + retval = NV_TRUE; + break; + case NV_ERR_NO_INTR_PENDING: + default: + *NeedBottomHalf = NV_FALSE; + retval = NV_FALSE; + break; + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return retval; +} + +void NV_API_CALL rm_isr_bh( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + RmIsrBottomHalf(pNv); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_isr_bh_unlocked( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + RmIsrBottomHalfUnlocked(pNv); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 *faultsCopied +) +{ + NV_STATUS status = NV_OK; + OBJGPU *pGpu; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL || faultsCopied == NULL) + { + status = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + if (IS_GSP_CLIENT(pGpu)) + { + // Non-replayable faults are copied to the client shadow buffer by GSP-RM. + status = NV_OK; + goto done; + } + else + { + KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); + PORT_MEM_ALLOCATOR *pIsrAllocator; + THREAD_STATE_NODE threadState; + NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer + + if (pKernelGmmu == NULL) + { + status = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + // If MMU fault buffer is not enabled, return early + if (!gpuIsVoltaHubIntrSupported(pGpu)) + goto done; + + pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); + tlsIsrInit(pIsrAllocator); + threadStateInitISRAndDeferredIntHandler(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR); + + // Copies all valid packets in RM's and client's shadow buffer + status = kgmmuCopyMmuFaults_HAL(pGpu, pKernelGmmu, &threadState, faultsCopied, + NON_REPLAYABLE_FAULT_BUFFER, NV_FALSE); + + threadStateFreeISRAndDeferredIntHandler(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR); + tlsIsrDestroy(pIsrAllocator); + portMemAllocatorRelease(pIsrAllocator); + + } + +done: + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +// +// Use this call when MMU faults needs to be copied +// outisde of RM lock. +// +static NV_STATUS _rm_gpu_copy_mmu_faults_unlocked( + OBJGPU *pGpu, + NvU32 *pFaultsCopied, + THREAD_STATE_NODE *pThreadState +) +{ + KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); + + if (pKernelGmmu == NULL) + { + return NV_ERR_OBJECT_NOT_FOUND; + } + + // If MMU fault buffer is not enabled, return early + if (!gpuIsVoltaHubIntrSupported(pGpu)) + return NV_OK; + + // Copies all valid packets in RM's and client's shadow buffer + return kgmmuCopyMmuFaults_HAL(pGpu, pKernelGmmu, pThreadState, pFaultsCopied, + NON_REPLAYABLE_FAULT_BUFFER, NV_FALSE); + + return NV_OK; +} + +// +// Wrapper to handle calls to copy mmu faults +// +NV_STATUS rm_gpu_handle_mmu_faults( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 *faultsCopied +) +{ + NvU32 status = NV_OK; + OBJGPU *pGpu; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + *faultsCopied = 0; + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if (pGpu == NULL) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_ERR_OBJECT_NOT_FOUND; + } + + { + KernelGmmu *pKernelGmmu; + PORT_MEM_ALLOCATOR *pIsrAllocator; + THREAD_STATE_NODE threadState; + NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer + + pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); + tlsIsrInit(pIsrAllocator); + threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + + pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); + + if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && !IS_VIRTUAL(pGpu)) + { + Intr *pIntr = GPU_GET_INTR(pGpu); + + if (kgmmuIsNonReplayableFaultPending_HAL(pGpu, pKernelGmmu, &threadState)) + { + // We have to clear the top level interrupt bit here since otherwise + // the bottom half will attempt to service the interrupt on the CPU + // side before GSP receives the notification and services it + kgmmuClearNonReplayableFaultIntr_HAL(pGpu, pKernelGmmu, &threadState); + status = intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_NON_REPLAYABLE_FAULT_HANDLE); + + } + } + else if (IS_VIRTUAL_WITH_SRIOV(pGpu)) + { + if (kgmmuIsNonReplayableFaultPending_HAL(pGpu, pKernelGmmu, &threadState)) + { + status = _rm_gpu_copy_mmu_faults_unlocked(pGpu, faultsCopied, &threadState); + } + } + else + { + if (IS_GSP_CLIENT(pGpu)) + { + // Non-replayable faults are copied to the client shadow buffer by GSP-RM. + status = NV_OK; + } + else + { + status = _rm_gpu_copy_mmu_faults_unlocked(pGpu, faultsCopied, &threadState); + } + } + + threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + tlsIsrDestroy(pIsrAllocator); + portMemAllocatorRelease(pIsrAllocator); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + return status; +} + +NvBool NV_API_CALL rm_is_msix_allowed( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + nv_priv_t *pNvp = NV_GET_NV_PRIV(nv); + THREAD_STATE_NODE threadState; + void *fp; + NvBool ret = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK) + { + ret = gpumgrIsDeviceMsixAllowed(nv->regs->cpu_address, + pNvp->pmc_boot_1, pNvp->pmc_boot_42); + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return ret; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/src/vbioscall.c b/userspace/sdks/nv570/nvalloc/unix/src/vbioscall.c new file mode 100644 index 0000000..8eeb85e --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/src/vbioscall.c @@ -0,0 +1,457 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +/****************************************************************************** +* +* Description: +* This file provides the glue layer between the RM's OS object +* unixCallVideoBIOS() method and the x86 real mode emulator +* +******************************************************************************/ + + +#include <core/core.h> +#include <nv.h> +#include <nv-priv.h> +#include <osapi.h> + +#include <os/os.h> + +#include <x86emu/x86emu.h> + +#define IO_LOG(port, val) + +#define NV_ROMLEY_VGA_PHYS_ADDR 0x9d000 +#define NV_VGA_PHYS_ADDR 0xa0000 +#define NV_VIDEO_ROM_PHYS_ADDR 0xc0000 +#define NV_EXPANSION_VIDEO_ROM_PHYS_ADDR 0xc8000 +#define NV_SYSTEM_ROM_PHYS_ADDR 0xf0000 + +#define X86EMU_LOWMEM 0x600 // Interrupt vectors, BIOS data size +#define X86EMU_STACK_ADDR 0x10000 +#define X86EMU_STACK_SIZE 0x1000 +#define X86EMU_BUF_ADDR 0x20000 // EDID buffer for VESA int10 0x4f15 +#define X86EMU_BUF_SIZE 128 + +#define OP_HLT 0xf4 // HLT instruction - causes the emulator to return + +// This is where IRET from int10 will take us +#define X86EMU_IRET_SEG (X86EMU_STACK_ADDR >> 4) +#define X86EMU_IRET_OFF 0 + +#define X86EMU_DFLT_FLAGS (F_IF) + +static NvBool x86emuReady; +static void *x86emuStack; + +static struct x86emu_mem_seg { + NvU32 start; + NvU32 end; + void *vaddr; +} x86emu_mem_map[] = { + {NV_ROMLEY_VGA_PHYS_ADDR, NV_VGA_PHYS_ADDR - 1}, + {NV_VGA_PHYS_ADDR, NV_VIDEO_ROM_PHYS_ADDR - 1}, // vga + {NV_VIDEO_ROM_PHYS_ADDR, NV_EXPANSION_VIDEO_ROM_PHYS_ADDR -1}, // vbios, sbios + {NV_EXPANSION_VIDEO_ROM_PHYS_ADDR, NV_SYSTEM_ROM_PHYS_ADDR -1}, // vbios, sbios + {NV_SYSTEM_ROM_PHYS_ADDR, 0xFFFFF}, // Motherboard BIOS + {X86EMU_STACK_ADDR, X86EMU_STACK_ADDR + X86EMU_STACK_SIZE - 1}, // emulator stack + {0, X86EMU_LOWMEM - 1}, // Interrupt vectors, BIOS data + {X86EMU_BUF_ADDR, X86EMU_BUF_ADDR + X86EMU_BUF_SIZE - 1}, +}; + +enum { + X86EMU_SEG_ROMLEY_VGA = 0, + X86EMU_SEG_VGA, + X86EMU_SEG_ROM, + X86EMU_SEG_ROM_EXPANSION, + X86EMU_SEG_ROM_MOTHERBOARD_BIOS, + X86EMU_SEG_EMULATOR_STACK, + X86EMU_SEG_BIOS_DATA, + X86EMU_SEG_EDID_BUFFER, + X86EMU_NUM_SEGS +}; + +ct_assert(X86EMU_NUM_SEGS == NV_ARRAY_ELEMENTS(x86emu_mem_map)); + +static NvU8 +x_inb(NvU16 port) +{ + NvU8 val; + + val = os_io_read_byte(port); + IO_LOG(port, val); + return val; +} + +static void +x_outb(NvU16 port, NvU8 val) +{ + IO_LOG(port, val); + os_io_write_byte(port, val); +} + +static NvU16 +x_inw(NvU16 port) +{ + NvU16 val; + + val = os_io_read_word(port); + IO_LOG(port, val); + return val; +} + +static void +x_outw(NvU16 port, NvU16 val) +{ + IO_LOG(port, val); + os_io_write_word(port, val); +} + +static u32 +x_inl(X86EMU_pioAddr port) +{ + NvU32 val; + + val = os_io_read_dword(port); + IO_LOG(port, val); + return val; +} + +static void +x_outl(X86EMU_pioAddr port, u32 val) +{ + IO_LOG(port, val); + os_io_write_dword(port, val); +} + +static void * +Mem_addr_xlat(NvU32 addr, NvU32 sz) +{ + int i; + struct x86emu_mem_seg *pseg; + + for (i = 0; i < X86EMU_NUM_SEGS; ++i) + { + pseg = x86emu_mem_map + i; + if (pseg->vaddr != 0 && addr >= pseg->start && addr + sz - 1 <= pseg->end) + return (void *)((char *)pseg->vaddr + addr - pseg->start); + } + + X86EMU_halt_sys(); + + return 0; +} + +static NvU8 +Mem_rb(u32 addr) +{ + NvU8 *va = Mem_addr_xlat(addr, 1); + + return va != 0 ? *va : 0; +} + +static NvU16 +Mem_rw(u32 addr) +{ + NvU16 *va = Mem_addr_xlat(addr, 2); + + return va != 0 ? *va : 0; +} + +static u32 +Mem_rl(u32 addr) +{ + NvU32 *va = Mem_addr_xlat(addr, 4); + + return va != 0 ? *va : 0; +} + +static void +Mem_wb(u32 addr, NvU8 val) +{ + NvU8 *va = Mem_addr_xlat(addr, 1); + + if (va != 0) + *va = val; +} + +static void +Mem_ww(u32 addr, NvU16 val) +{ + NvU16 *va = Mem_addr_xlat(addr, 2); + + if (va != 0) + *va = val; +} + +static void +Mem_wl(u32 addr, u32 val) +{ + NvU32 *va = Mem_addr_xlat(addr, 4); + + if (va != 0) + *va = val; +} + +static NvU16 +get_int_seg(int i) +{ + return Mem_rw(i * 4 + 2); +} + + +static NvU16 +get_int_off(int i) +{ + return Mem_rw(i * 4); +} + + +static void +pushw(NvU16 i) +{ + M.x86.R_ESP -= 2; + Mem_ww((M.x86.R_SS << 4) + M.x86.R_ESP, i); +} + +static void +x86emu_do_int(int num) +{ +#if 0 + Int10Current->num = num; + + if (!int_handler(Int10Current)) { + X86EMU_halt_sys(); + } +#else + if ((num == 0x15) && (M.x86.R_AX == 0x5f80)) + { + // + // Handle the MXM_SYS_INFO_CALLBACK_NUM int 15h SBIOS + // callback: disclaim support by returning a value + // other than 005fh (MXM_SYS_INFO_CALLBACK_FUNC_SUPPORTED) + // to the caller. + // + M.x86.R_AX = 0; + } + else + { + NV_PRINTF(LEVEL_ERROR, "x86emu: int $%d (eax = %08x)\n", + num, M.x86.R_EAX); + DBG_BREAKPOINT(); + X86EMU_halt_sys(); + } +#endif +} + +NV_STATUS +RmInitX86EmuState(OBJGPU *pGpu) { + int i; + struct x86emu_mem_seg *pseg; + X86EMU_intrFuncs *intFuncs; + X86EMU_pioFuncs pioFuncs = { + (&x_inb), + (&x_inw), + (&x_inl), + (&x_outb), + (&x_outw), + (&x_outl) + }; + + X86EMU_memFuncs memFuncs = { + (&Mem_rb), + (&Mem_rw), + (&Mem_rl), + (&Mem_wb), + (&Mem_ww), + (&Mem_wl) + }; + + if (!NV_PRIMARY_VGA(NV_GET_NV_STATE(pGpu))) // not the primary GPU + return NV_OK; + + NV_ASSERT(!x86emuReady); + + x86emuStack = portMemAllocNonPaged( + X86EMU_STACK_SIZE + sizeof(X86EMU_intrFuncs*) * 256); + if (x86emuStack == NULL) + return NV_ERR_NO_MEMORY; + + // Interrupt dispatch table + intFuncs = (void *)((NvU8 *)x86emuStack + X86EMU_STACK_SIZE); + + // Fill virtual addresses in the memory map + for (i = 0; i < X86EMU_NUM_SEGS; ++i) + { + pseg = x86emu_mem_map + i; + switch (i) + { + case X86EMU_SEG_ROM: + case X86EMU_SEG_ROM_EXPANSION: + case X86EMU_SEG_ROM_MOTHERBOARD_BIOS: + nv_get_updated_emu_seg(&pseg->start, &pseg->end); + /* fallthrough */ + case X86EMU_SEG_VGA: + case X86EMU_SEG_BIOS_DATA: + pseg->vaddr = os_map_kernel_space(pseg->start, + pseg->end - pseg->start + 1, + NV_MEMORY_CACHED); + if (pseg->vaddr == 0) + { + NV_PRINTF(LEVEL_ERROR, "x86emu can't map phys addr 0x%05x\n", + pseg->start); + return NV_ERR_GENERIC; + } + break; + case X86EMU_SEG_EMULATOR_STACK: + pseg->vaddr = x86emuStack; + break; + default: + pseg->vaddr = 0; + break; + } + } + X86EMU_setupMemFuncs(&memFuncs); + + M.mem_base = 0; + M.mem_size = 1024*1024; + X86EMU_setupPioFuncs(&pioFuncs); + + for (i=0;i<256;i++) + intFuncs[i] = x86emu_do_int; + X86EMU_setupIntrFuncs(intFuncs); + + x86emuReady = NV_TRUE; + + return NV_OK; +} + +void +RmFreeX86EmuState(OBJGPU *pGpu) +{ + int i; + struct x86emu_mem_seg *pseg; + + if (! x86emuReady || !NV_PRIMARY_VGA(NV_GET_NV_STATE(pGpu))) // not the primary GPU + return; + + portMemFree(x86emuStack); + x86emuStack = 0; + + for (i = 0; i < X86EMU_NUM_SEGS; ++i) + { + pseg = x86emu_mem_map + i; + switch (i) + { + case X86EMU_SEG_ROMLEY_VGA: + case X86EMU_SEG_VGA: + case X86EMU_SEG_ROM: + case X86EMU_SEG_ROM_EXPANSION: + case X86EMU_SEG_ROM_MOTHERBOARD_BIOS: + case X86EMU_SEG_BIOS_DATA: + if (pseg->vaddr != 0) + os_unmap_kernel_space(pseg->vaddr, pseg->end - pseg->start + 1); + } + pseg->vaddr = 0; + } + + x86emuReady = NV_FALSE; +} + +NV_STATUS +nv_vbios_call( + OBJGPU *pGpu, + NvU32 *eax, + NvU32 *ebx) +{ + NvU16 seg; + NvU16 off; + struct x86emu_mem_seg *pseg; + + if (!NV_PRIMARY_VGA(NV_GET_NV_STATE(pGpu))) + return NV_ERR_GENERIC; + + seg = get_int_seg(0x10); + off = get_int_off(0x10); + + pseg = &x86emu_mem_map[X86EMU_SEG_VGA]; + if (seg < (pseg->start >> 4)) + { + pseg = &x86emu_mem_map[X86EMU_SEG_ROMLEY_VGA]; + if (seg < (pseg->start >> 4)) + { + NV_PRINTF(LEVEL_ERROR, + "cannot call the VBIOS. INT10 vector not in ROM: %04x:%04x\n", + seg, off); + return NV_ERR_GENERIC; + } + else if (pseg->vaddr == NULL) + { + pseg->vaddr = os_map_kernel_space(pseg->start, + (pseg->end - pseg->start + 1), + NV_MEMORY_CACHED); + if (pseg->vaddr == NULL) + { + NV_PRINTF(LEVEL_ERROR, "x86emu can't map phys addr 0x%05x\n", + pseg->start); + return NV_ERR_GENERIC; + } + } + } + + // Reset the CPU + portMemSet(&M, 0, sizeof(M)); + + M.x86.R_SS = X86EMU_STACK_ADDR >> 4; + M.x86.R_ESP = X86EMU_STACK_SIZE; + M.x86.R_CS = seg; + M.x86.R_EIP = off; + M.x86.R_EFLG = X86EMU_DFLT_FLAGS; + M.x86.R_EAX = *eax; + M.x86.R_EBX = *ebx; + M.x86.R_ECX = 0; + M.x86.R_EDX = 0; + M.x86.R_ES = X86EMU_BUF_ADDR >> 4; + + X86EMU_trace_on(); + + // Map ES:EDI to buffer. Used by vesa intr 4f15 - read EDID + pseg = &x86emu_mem_map[X86EMU_SEG_EDID_BUFFER]; + pseg->vaddr = NULL; + + // Prepare the code for IRET to jump to + Mem_wb((X86EMU_IRET_SEG << 4) + X86EMU_IRET_OFF, OP_HLT); + + pushw(X86EMU_DFLT_FLAGS); + pushw(X86EMU_IRET_SEG); + pushw(X86EMU_IRET_OFF); + + // Run the emulator + X86EMU_exec(); + + *eax = M.x86.R_EAX; + *ebx = M.x86.R_EBX; + + return NV_OK; +} diff --git a/userspace/sdks/nv570/nvalloc/unix/x86emu/debug.c b/userspace/sdks/nv570/nvalloc/unix/x86emu/debug.c new file mode 100644 index 0000000..ccf8be7 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/x86emu/debug.c @@ -0,0 +1,547 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: This file contains the code to handle debugging of the +* emulator. +* +****************************************************************************/ + +#include "x86emu/x86emui.h" +#ifndef NO_SYS_HEADERS +#include <stdarg.h> +#include <stdlib.h> +#endif + +/*----------------------------- Implementation ----------------------------*/ + +#ifdef X86EMU_DEBUG + +static void print_encoded_bytes (u16 s, u16 o); +static void print_decoded_instruction (void); +static int parse_line (char *s, int *ps, int *n); + +/* should look something like debug's output. */ +void X86EMU_trace_regs (void) +{ + if (DEBUG_TRACE()) { + x86emu_dump_regs(); + } + if (DEBUG_DECODE() && ! DEBUG_DECODE_NOPRINT()) { + NV_PRINTF(LEVEL_INFO, "%04x:%04x ", M.x86.saved_cs, + M.x86.saved_ip); + print_encoded_bytes( M.x86.saved_cs, M.x86.saved_ip); + print_decoded_instruction(); + } +} + +void X86EMU_trace_xregs (void) +{ + if (DEBUG_TRACE()) { + x86emu_dump_xregs(); + } +} + +void x86emu_just_disassemble (void) +{ + /* + * This routine called if the flag DEBUG_DISASSEMBLE is set kind + * of a hack! + */ + NV_PRINTF(LEVEL_INFO, "%04x:%04x ", M.x86.saved_cs, + M.x86.saved_ip); + print_encoded_bytes( M.x86.saved_cs, M.x86.saved_ip); + print_decoded_instruction(); +} + +static void disassemble_forward (u16 seg, u16 off, int n) +{ + X86EMU_sysEnv tregs; + int i; + u8 op1; + /* + * hack, hack, hack. What we do is use the exact machinery set up + * for execution, except that now there is an additional state + * flag associated with the "execution", and we are using a copy + * of the register struct. All the major opcodes, once fully + * decoded, have the following two steps: TRACE_REGS(r,m); + * SINGLE_STEP(r,m); which disappear if DEBUG is not defined to + * the preprocessor. The TRACE_REGS macro expands to: + * + * if (debug&DEBUG_DISASSEMBLE) + * {just_disassemble(); goto EndOfInstruction;} + * if (debug&DEBUG_TRACE) trace_regs(r,m); + * + * ...... and at the last line of the routine. + * + * EndOfInstruction: end_instr(); + * + * Up to the point where TRACE_REG is expanded, NO modifications + * are done to any register EXCEPT the IP register, for fetch and + * decoding purposes. + * + * This was done for an entirely different reason, but makes a + * nice way to get the system to help debug codes. + */ + tregs = M; + tregs.x86.R_IP = off; + tregs.x86.R_CS = seg; + + /* reset the decoding buffers */ + tregs.x86.enc_str_pos = 0; + tregs.x86.enc_pos = 0; + + /* turn on the "disassemble only, no execute" flag */ + tregs.x86.debug |= DEBUG_DISASSEMBLE_F; + + /* DUMP NEXT n instructions to screen in straight_line fashion */ + /* + * This looks like the regular instruction fetch stream, except + * that when this occurs, each fetched opcode, upon seeing the + * DEBUG_DISASSEMBLE flag set, exits immediately after decoding + * the instruction. XXX --- CHECK THAT MEM IS NOT AFFECTED!!! + * Note the use of a copy of the register structure... + */ + for (i=0; i<n; i++) { + op1 = (*sys_rdb)(((u32)M.x86.R_CS<<4) + (M.x86.R_IP++)); + (x86emu_optab[op1])(op1); + } + /* end major hack mode. */ +} + +void x86emu_check_ip_access (void) +{ + /* NULL as of now */ +} + +void x86emu_check_sp_access (void) +{ +} + +void x86emu_check_mem_access (u32 dummy) +{ + /* check bounds, etc */ +} + +void x86emu_check_data_access (uint dummy1, uint dummy2) +{ + /* check bounds, etc */ +} + +void x86emu_inc_decoded_inst_len (int x) +{ + M.x86.enc_pos += x; +} + +void x86emu_decode_printf (const char *x) +{ + NvU32 max_size = sizeof(M.x86.decoded_buf) - M.x86.enc_str_pos; + + nvDbgSnprintf(M.x86.decoded_buf + M.x86.enc_str_pos, max_size, "%s", x); + M.x86.enc_str_pos += portStringLength(x); +} + +void x86emu_decode_printf2 (const char *x, int y) +{ + char temp[100]; + nvDbgSnprintf(temp, sizeof(temp), x, y); + x86emu_decode_printf(temp); +} + +void x86emu_end_instr (void) +{ + M.x86.enc_str_pos = 0; + M.x86.enc_pos = 0; +} + +static void print_encoded_bytes (u16 s, u16 o) +{ + int i; + char buf1[64]; + for (i=0; i< M.x86.enc_pos; i++) { + nvDbgSnprintf(buf1 + 2 * i, sizeof(buf1) - (2 * i), + "%02x", fetch_data_byte_abs(s, o + i)); + } + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "%-20s", buf1); +} + +static void print_decoded_instruction (void) +{ + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "%s", M.x86.decoded_buf); +} + +void x86emu_print_int_vect (u16 iv) +{ + u16 seg,off; + + if (iv > 256) return; + seg = fetch_data_word_abs(0,iv*4); + off = fetch_data_word_abs(0,iv*4+2); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "%04x:%04x ", seg, off); +} + +void X86EMU_dump_memory (u16 seg, u16 off, u32 amt) +{ + u32 start = off & 0xfffffff0; + u32 end = (off+16) & 0xfffffff0; + u32 i; + + while (end <= off + amt) { + NV_PRINTF(LEVEL_INFO, "%04x:%04x ", seg, start); + for (i=start; i< off; i++) + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " "); + for ( ; i< end; i++) + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "%02x ", fetch_data_byte_abs(seg, i)); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "\n"); + start = end; + end = start + 16; + } +} + +void x86emu_single_step (void) +{ + int ps[10]; + int ntok; + int cmd; + int done; + int segment; + int offset; + static int breakpoint; + static int noDecode = 1; + + if (DEBUG_BREAK()) { + if (M.x86.saved_ip != breakpoint) { + return; + } else { + M.x86.debug &= ~DEBUG_DECODE_NOPRINT_F; + M.x86.debug |= DEBUG_TRACE_F; + M.x86.debug &= ~DEBUG_BREAK_F; + print_decoded_instruction (); + X86EMU_trace_regs(); + } + } + done=0; + offset = M.x86.saved_ip; + while (!done) { + NV_PRINTF(LEVEL_INFO, "-"); + + /* + * Set _X86EMU_env.x86.debug_cmd from a kernel debugger to a command to + * control the emulator debugger + */ + if (M.x86.debug_cmd[0] == '\0') + { + DBG_BREAKPOINT(); + } + + cmd = parse_line(M.x86.debug_cmd, ps, &ntok); + switch(cmd) { + case 'u': + disassemble_forward(M.x86.saved_cs,(u16)offset,10); + break; + case 'd': + if (ntok == 2) { + segment = M.x86.saved_cs; + offset = ps[1]; + X86EMU_dump_memory(segment,(u16)offset,16); + offset += 16; + } else if (ntok == 3) { + segment = ps[1]; + offset = ps[2]; + X86EMU_dump_memory(segment,(u16)offset,16); + offset += 16; + } else { + segment = M.x86.saved_cs; + X86EMU_dump_memory(segment,(u16)offset,16); + offset += 16; + } + break; + case 'c': + M.x86.debug ^= DEBUG_TRACECALL_F; + break; + case 's': + M.x86.debug ^= DEBUG_SVC_F | DEBUG_SYS_F | DEBUG_SYSINT_F; + break; + case 'r': + X86EMU_trace_regs(); + break; + case 'x': + X86EMU_trace_xregs(); + break; + case 'g': + if (ntok == 2) { + breakpoint = ps[1]; + if (noDecode) { + M.x86.debug |= DEBUG_DECODE_NOPRINT_F; + } else { + M.x86.debug &= ~DEBUG_DECODE_NOPRINT_F; + } + M.x86.debug &= ~DEBUG_TRACE_F; + M.x86.debug |= DEBUG_BREAK_F; + done = 1; + } + break; + case 'q': + M.x86.debug |= DEBUG_EXIT; + return; + case 'P': + noDecode = (noDecode)?0:1; + NV_PRINTF(LEVEL_INFO, "Toggled decoding to %s\n", + (noDecode) ? "FALSE" : "TRUE"); + break; + case 'a': + X86EMU_trace_off(); + /* fall thru */ + case 't': + case 0: + done = 1; + break; + } + portMemSet(&M.x86.debug_cmd, 0, sizeof(M.x86.debug_cmd)); + } +} + +int X86EMU_trace_on(void) +{ + return M.x86.debug |= DEBUG_STEP_F | DEBUG_DECODE_F | DEBUG_TRACE_F; +} + +int X86EMU_trace_off(void) +{ + return M.x86.debug &= ~(DEBUG_STEP_F | DEBUG_DECODE_F | DEBUG_TRACE_F); +} + +static void parse_hex (char *s, int *pval) +{ + unsigned int val = 0; + unsigned char c; + + if (s[0] == '0' && s[1] == 'x') + s += 2; + + while ((c = *(s++))) + { + if (c >= '0' && c <= '9') + { + val <<= 4; + val |= c - '0'; + continue; + } + else if (c >= 'a' && c <= 'f') + { + val <<= 4; + val |= c - 'a'; + continue; + } + else if (c >= 'A' && c <= 'F') + { + val <<= 4; + val |= c - 'A'; + continue; + } + break; + } + + *pval = val; +} + +static int parse_line (char *s, int *ps, int *n) +{ + int cmd; + + *n = 0; + while(*s == ' ' || *s == '\t') s++; + ps[*n] = *s; + switch (*s) { + case '\n': + *n += 1; + return 0; + default: + cmd = *s; + *n += 1; + } + + while (1) + { + while (*s != ' ' && *s != '\t' && *s != '\n' && *s != '\0') + s++; + + if (*s == '\n' || *s == '\0') + return cmd; + + while(*s == ' ' || *s == '\t') + s++; + + parse_hex(s, &ps[*n]); + *n += 1; + } +} + +void x86emu_dump_regs (void) +{ + NV_PRINTF(LEVEL_INFO, "\tAX=%04x ", M.x86.R_AX); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "BX=%04x ", M.x86.R_BX); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "CX=%04x ", M.x86.R_CX); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "DX=%04x ", M.x86.R_DX); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "SP=%04x ", M.x86.R_SP); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "BP=%04x ", M.x86.R_BP); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "SI=%04x ", M.x86.R_SI); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "DI=%04x\n", M.x86.R_DI); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "\tDS=%04x ", M.x86.R_DS); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "ES=%04x ", M.x86.R_ES); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "SS=%04x ", M.x86.R_SS); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "CS=%04x ", M.x86.R_CS); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "IP=%04x ", M.x86.R_IP); + if (ACCESS_FLAG(F_OF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "OV "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NV "); + } + if (ACCESS_FLAG(F_DF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "DN "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "UP "); + } + if (ACCESS_FLAG(F_IF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "EI "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "DI "); + } + if (ACCESS_FLAG(F_SF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NG "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "PL "); + } + if (ACCESS_FLAG(F_ZF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "ZR "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NZ "); + } + if (ACCESS_FLAG(F_AF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "AC "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NA "); + } + if (ACCESS_FLAG(F_PF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "PE "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "PO "); + } + if (ACCESS_FLAG(F_CF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "CY "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NC "); + } + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "\n"); +} + +void x86emu_dump_xregs (void) +{ + NV_PRINTF(LEVEL_INFO, "\tEAX=%08x ", M.x86.R_EAX); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "EBX=%08x ", M.x86.R_EBX); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "ECX=%08x ", M.x86.R_ECX); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "EDX=%08x \n", M.x86.R_EDX); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "\tESP=%08x ", M.x86.R_ESP); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "EBP=%08x ", M.x86.R_EBP); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "ESI=%08x ", M.x86.R_ESI); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "EDI=%08x\n", M.x86.R_EDI); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "\tDS=%04x ", M.x86.R_DS); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "ES=%04x ", M.x86.R_ES); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "SS=%04x ", M.x86.R_SS); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "CS=%04x ", M.x86.R_CS); + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "EIP=%08x\n\t", M.x86.R_EIP); + if (ACCESS_FLAG(F_OF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "OV "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NV "); + } + if (ACCESS_FLAG(F_DF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "DN "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "UP "); + } + if (ACCESS_FLAG(F_IF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "EI "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "DI "); + } + if (ACCESS_FLAG(F_SF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NG "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "PL "); + } + if (ACCESS_FLAG(F_ZF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "ZR "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NZ "); + } + if (ACCESS_FLAG(F_AF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "AC "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NA "); + } + if (ACCESS_FLAG(F_PF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "PE "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "PO "); + } + if (ACCESS_FLAG(F_CF)) { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "CY "); + } else { + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "NC "); + } + NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "\n"); +} + +#else + +void x86emu_dump_regs (void) +{ +} + +void x86emu_dump_xregs (void) +{ +} + +int X86EMU_trace_on(void) +{ + return 0; +} + +int X86EMU_trace_off(void) +{ + return 0; +} + +#endif /* X86EMU_DEBUG */ + diff --git a/userspace/sdks/nv570/nvalloc/unix/x86emu/decode.c b/userspace/sdks/nv570/nvalloc/unix/x86emu/decode.c new file mode 100644 index 0000000..a683ce4 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/x86emu/decode.c @@ -0,0 +1,1094 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: This file includes subroutines which are related to +* instruction decoding and accessess of immediate data via IP. etc. +* +****************************************************************************/ + + +#include "x86emu/x86emui.h" + +/*----------------------------- Implementation ----------------------------*/ + +/**************************************************************************** +REMARKS: +Handles any pending asychronous interrupts. +****************************************************************************/ +static void x86emu_intr_handle(void) +{ + u8 intno; + + if (M.x86.intr & INTR_SYNCH) { + intno = M.x86.intno; + if (_X86EMU_intrTab[intno]) { + (*_X86EMU_intrTab[intno])(intno); + } else { + push_word((u16)M.x86.R_FLG); + CLEAR_FLAG(F_IF); + CLEAR_FLAG(F_TF); + push_word(M.x86.R_CS); + M.x86.R_CS = mem_access_word(intno * 4 + 2); + push_word(M.x86.R_IP); + M.x86.R_IP = mem_access_word(intno * 4); + M.x86.intr = 0; + } + } +} + +/**************************************************************************** +PARAMETERS: +intrnum - Interrupt number to raise + +REMARKS: +Raise the specified interrupt to be handled before the execution of the +next instruction. +****************************************************************************/ +void x86emu_intr_raise( + u8 intrnum) +{ + M.x86.intno = intrnum; + M.x86.intr |= INTR_SYNCH; +} + +/**************************************************************************** +REMARKS: +Main execution loop for the emulator. We return from here when the system +halts, which is normally caused by a stack fault when we return from the +original real mode call. +****************************************************************************/ +void X86EMU_exec(void) +{ + u8 op1; + + M.x86.intr = 0; + DB(x86emu_end_instr();) + + for (;;) { +DB( if (CHECK_IP_FETCH()) + x86emu_check_ip_access();) + /* If debugging, save the IP and CS values. */ + SAVE_IP_CS(M.x86.R_CS, M.x86.R_IP); + INC_DECODED_INST_LEN(1); + if (M.x86.intr) { + if (M.x86.intr & INTR_HALTED) { +DB( if (M.x86.R_SP != 0) { + NV_PRINTF(LEVEL_INFO, "halted\n"); + X86EMU_trace_regs(); + } + else { + if (M.x86.debug) + NV_PRINTF(LEVEL_INFO, + "Service completed successfully\n"); + }) + return; + } + if (((M.x86.intr & INTR_SYNCH) && (M.x86.intno == 0 || M.x86.intno == 2)) || + !ACCESS_FLAG(F_IF)) { + x86emu_intr_handle(); + } + } + op1 = (*sys_rdb)(((u32)M.x86.R_CS << 4) + (M.x86.R_IP++)); + (*x86emu_optab[op1])(op1); + if (M.x86.debug & DEBUG_EXIT) { + M.x86.debug &= ~DEBUG_EXIT; + return; + } + } +} + +/**************************************************************************** +REMARKS: +Halts the system by setting the halted system flag. +****************************************************************************/ +void X86EMU_halt_sys(void) +{ + M.x86.intr |= INTR_HALTED; +} + +/**************************************************************************** +PARAMETERS: +mod - Mod value from decoded byte +regh - Reg h value from decoded byte +regl - Reg l value from decoded byte + +REMARKS: +Raise the specified interrupt to be handled before the execution of the +next instruction. + +NOTE: Do not inline this function, as (*sys_rdb) is already inline! +****************************************************************************/ +void fetch_decode_modrm( + int *mod, + int *regh, + int *regl) +{ + int fetched; + +DB( if (CHECK_IP_FETCH()) + x86emu_check_ip_access();) + fetched = (*sys_rdb)(((u32)M.x86.R_CS << 4) + (M.x86.R_IP++)); + INC_DECODED_INST_LEN(1); + *mod = (fetched >> 6) & 0x03; + *regh = (fetched >> 3) & 0x07; + *regl = (fetched >> 0) & 0x07; +} + +/**************************************************************************** +RETURNS: +Immediate byte value read from instruction queue + +REMARKS: +This function returns the immediate byte from the instruction queue, and +moves the instruction pointer to the next value. + +NOTE: Do not inline this function, as (*sys_rdb) is already inline! +****************************************************************************/ +u8 fetch_byte_imm(void) +{ + u8 fetched; + +DB( if (CHECK_IP_FETCH()) + x86emu_check_ip_access();) + fetched = (*sys_rdb)(((u32)M.x86.R_CS << 4) + (M.x86.R_IP++)); + INC_DECODED_INST_LEN(1); + return fetched; +} + +/**************************************************************************** +RETURNS: +Immediate word value read from instruction queue + +REMARKS: +This function returns the immediate byte from the instruction queue, and +moves the instruction pointer to the next value. + +NOTE: Do not inline this function, as (*sys_rdw) is already inline! +****************************************************************************/ +u16 fetch_word_imm(void) +{ + u16 fetched; + +DB( if (CHECK_IP_FETCH()) + x86emu_check_ip_access();) + fetched = (*sys_rdw)(((u32)M.x86.R_CS << 4) + (M.x86.R_IP)); + M.x86.R_IP += 2; + INC_DECODED_INST_LEN(2); + return fetched; +} + +/**************************************************************************** +RETURNS: +Immediate lone value read from instruction queue + +REMARKS: +This function returns the immediate byte from the instruction queue, and +moves the instruction pointer to the next value. + +NOTE: Do not inline this function, as (*sys_rdw) is already inline! +****************************************************************************/ +u32 fetch_long_imm(void) +{ + u32 fetched; + +DB( if (CHECK_IP_FETCH()) + x86emu_check_ip_access();) + fetched = (*sys_rdl)(((u32)M.x86.R_CS << 4) + (M.x86.R_IP)); + M.x86.R_IP += 4; + INC_DECODED_INST_LEN(4); + return fetched; +} + +/**************************************************************************** +RETURNS: +Value of the default data segment + +REMARKS: +Inline function that returns the default data segment for the current +instruction. + +On the x86 processor, the default segment is not always DS if there is +no segment override. Address modes such as -3[BP] or 10[BP+SI] all refer to +addresses relative to SS (ie: on the stack). So, at the minimum, all +decodings of addressing modes would have to set/clear a bit describing +whether the access is relative to DS or SS. That is the function of the +cpu-state-varible M.x86.mode. There are several potential states: + + repe prefix seen (handled elsewhere) + repne prefix seen (ditto) + + cs segment override + ds segment override + es segment override + fs segment override + gs segment override + ss segment override + + ds/ss select (in absense of override) + +Each of the above 7 items are handled with a bit in the mode field. +****************************************************************************/ +_INLINE u32 get_data_segment(void) +{ +#define GET_SEGMENT(segment) + switch (M.x86.mode & SYSMODE_SEGMASK) { + case 0: /* default case: use ds register */ + case SYSMODE_SEGOVR_DS: + case SYSMODE_SEGOVR_DS | SYSMODE_SEG_DS_SS: + return M.x86.R_DS; + case SYSMODE_SEG_DS_SS: /* non-overridden, use ss register */ + return M.x86.R_SS; + case SYSMODE_SEGOVR_CS: + case SYSMODE_SEGOVR_CS | SYSMODE_SEG_DS_SS: + return M.x86.R_CS; + case SYSMODE_SEGOVR_ES: + case SYSMODE_SEGOVR_ES | SYSMODE_SEG_DS_SS: + return M.x86.R_ES; + case SYSMODE_SEGOVR_FS: + case SYSMODE_SEGOVR_FS | SYSMODE_SEG_DS_SS: + return M.x86.R_FS; + case SYSMODE_SEGOVR_GS: + case SYSMODE_SEGOVR_GS | SYSMODE_SEG_DS_SS: + return M.x86.R_GS; + case SYSMODE_SEGOVR_SS: + case SYSMODE_SEGOVR_SS | SYSMODE_SEG_DS_SS: + return M.x86.R_SS; + default: +#ifdef X86EMU_DEBUG + NV_PRINTF(LEVEL_INFO, + "error: should not happen: multiple overrides.\n"); +#endif + HALT_SYS(); + return 0; + } +} + +/**************************************************************************** +PARAMETERS: +offset - Offset to load data from + +RETURNS: +Byte value read from the absolute memory location. + +NOTE: Do not inline this function as (*sys_rdX) is already inline! +****************************************************************************/ +u8 fetch_data_byte( + uint offset) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access((u16)get_data_segment(), offset); +#endif + return (*sys_rdb)((get_data_segment() << 4) + offset); +} + +/**************************************************************************** +PARAMETERS: +offset - Offset to load data from + +RETURNS: +Word value read from the absolute memory location. + +NOTE: Do not inline this function as (*sys_rdX) is already inline! +****************************************************************************/ +u16 fetch_data_word( + uint offset) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access((u16)get_data_segment(), offset); +#endif + return (*sys_rdw)((get_data_segment() << 4) + offset); +} + +/**************************************************************************** +PARAMETERS: +offset - Offset to load data from + +RETURNS: +Long value read from the absolute memory location. + +NOTE: Do not inline this function as (*sys_rdX) is already inline! +****************************************************************************/ +u32 fetch_data_long( + uint offset) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access((u16)get_data_segment(), offset); +#endif + return (*sys_rdl)((get_data_segment() << 4) + offset); +} + +/**************************************************************************** +PARAMETERS: +segment - Segment to load data from +offset - Offset to load data from + +RETURNS: +Byte value read from the absolute memory location. + +NOTE: Do not inline this function as (*sys_rdX) is already inline! +****************************************************************************/ +u8 fetch_data_byte_abs( + uint segment, + uint offset) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access(segment, offset); +#endif + return (*sys_rdb)(((u32)segment << 4) + offset); +} + +/**************************************************************************** +PARAMETERS: +segment - Segment to load data from +offset - Offset to load data from + +RETURNS: +Word value read from the absolute memory location. + +NOTE: Do not inline this function as (*sys_rdX) is already inline! +****************************************************************************/ +u16 fetch_data_word_abs( + uint segment, + uint offset) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access(segment, offset); +#endif + return (*sys_rdw)(((u32)segment << 4) + offset); +} + +/**************************************************************************** +PARAMETERS: +segment - Segment to load data from +offset - Offset to load data from + +RETURNS: +Long value read from the absolute memory location. + +NOTE: Do not inline this function as (*sys_rdX) is already inline! +****************************************************************************/ +u32 fetch_data_long_abs( + uint segment, + uint offset) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access(segment, offset); +#endif + return (*sys_rdl)(((u32)segment << 4) + offset); +} + +/**************************************************************************** +PARAMETERS: +offset - Offset to store data at +val - Value to store + +REMARKS: +Writes a word value to an segmented memory location. The segment used is +the current 'default' segment, which may have been overridden. + +NOTE: Do not inline this function as (*sys_wrX) is already inline! +****************************************************************************/ +void store_data_byte( + uint offset, + u8 val) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access((u16)get_data_segment(), offset); +#endif + (*sys_wrb)((get_data_segment() << 4) + offset, val); +} + +/**************************************************************************** +PARAMETERS: +offset - Offset to store data at +val - Value to store + +REMARKS: +Writes a word value to an segmented memory location. The segment used is +the current 'default' segment, which may have been overridden. + +NOTE: Do not inline this function as (*sys_wrX) is already inline! +****************************************************************************/ +void store_data_word( + uint offset, + u16 val) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access((u16)get_data_segment(), offset); +#endif + (*sys_wrw)((get_data_segment() << 4) + offset, val); +} + +/**************************************************************************** +PARAMETERS: +offset - Offset to store data at +val - Value to store + +REMARKS: +Writes a long value to an segmented memory location. The segment used is +the current 'default' segment, which may have been overridden. + +NOTE: Do not inline this function as (*sys_wrX) is already inline! +****************************************************************************/ +void store_data_long( + uint offset, + u32 val) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access((u16)get_data_segment(), offset); +#endif + (*sys_wrl)((get_data_segment() << 4) + offset, val); +} + +/**************************************************************************** +PARAMETERS: +segment - Segment to store data at +offset - Offset to store data at +val - Value to store + +REMARKS: +Writes a byte value to an absolute memory location. + +NOTE: Do not inline this function as (*sys_wrX) is already inline! +****************************************************************************/ +void store_data_byte_abs( + uint segment, + uint offset, + u8 val) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access(segment, offset); +#endif + (*sys_wrb)(((u32)segment << 4) + offset, val); +} + +/**************************************************************************** +PARAMETERS: +segment - Segment to store data at +offset - Offset to store data at +val - Value to store + +REMARKS: +Writes a word value to an absolute memory location. + +NOTE: Do not inline this function as (*sys_wrX) is already inline! +****************************************************************************/ +void store_data_word_abs( + uint segment, + uint offset, + u16 val) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access(segment, offset); +#endif + (*sys_wrw)(((u32)segment << 4) + offset, val); +} + +/**************************************************************************** +PARAMETERS: +segment - Segment to store data at +offset - Offset to store data at +val - Value to store + +REMARKS: +Writes a long value to an absolute memory location. + +NOTE: Do not inline this function as (*sys_wrX) is already inline! +****************************************************************************/ +void store_data_long_abs( + uint segment, + uint offset, + u32 val) +{ +#ifdef X86EMU_DEBUG + if (CHECK_DATA_ACCESS()) + x86emu_check_data_access(segment, offset); +#endif + (*sys_wrl)(((u32)segment << 4) + offset, val); +} + +/**************************************************************************** +PARAMETERS: +reg - Register to decode + +RETURNS: +Pointer to the appropriate register + +REMARKS: +Return a pointer to the register given by the R/RM field of the +modrm byte, for byte operands. Also enables the decoding of instructions. +****************************************************************************/ +u8* decode_rm_byte_register( + int reg) +{ + switch (reg) { + case 0: + DECODE_PRINTF("AL"); + return &M.x86.R_AL; + case 1: + DECODE_PRINTF("CL"); + return &M.x86.R_CL; + case 2: + DECODE_PRINTF("DL"); + return &M.x86.R_DL; + case 3: + DECODE_PRINTF("BL"); + return &M.x86.R_BL; + case 4: + DECODE_PRINTF("AH"); + return &M.x86.R_AH; + case 5: + DECODE_PRINTF("CH"); + return &M.x86.R_CH; + case 6: + DECODE_PRINTF("DH"); + return &M.x86.R_DH; + case 7: + DECODE_PRINTF("BH"); + return &M.x86.R_BH; + } + HALT_SYS(); + return NULL; /* NOT REACHED OR REACHED ON ERROR */ +} + +/**************************************************************************** +PARAMETERS: +reg - Register to decode + +RETURNS: +Pointer to the appropriate register + +REMARKS: +Return a pointer to the register given by the R/RM field of the +modrm byte, for word operands. Also enables the decoding of instructions. +****************************************************************************/ +u16* decode_rm_word_register( + int reg) +{ + switch (reg) { + case 0: + DECODE_PRINTF("AX"); + return &M.x86.R_AX; + case 1: + DECODE_PRINTF("CX"); + return &M.x86.R_CX; + case 2: + DECODE_PRINTF("DX"); + return &M.x86.R_DX; + case 3: + DECODE_PRINTF("BX"); + return &M.x86.R_BX; + case 4: + DECODE_PRINTF("SP"); + return &M.x86.R_SP; + case 5: + DECODE_PRINTF("BP"); + return &M.x86.R_BP; + case 6: + DECODE_PRINTF("SI"); + return &M.x86.R_SI; + case 7: + DECODE_PRINTF("DI"); + return &M.x86.R_DI; + } + HALT_SYS(); + return NULL; /* NOTREACHED OR REACHED ON ERROR */ +} + +/**************************************************************************** +PARAMETERS: +reg - Register to decode + +RETURNS: +Pointer to the appropriate register + +REMARKS: +Return a pointer to the register given by the R/RM field of the +modrm byte, for dword operands. Also enables the decoding of instructions. +****************************************************************************/ +u32* decode_rm_long_register( + int reg) +{ + switch (reg) { + case 0: + DECODE_PRINTF("EAX"); + return &M.x86.R_EAX; + case 1: + DECODE_PRINTF("ECX"); + return &M.x86.R_ECX; + case 2: + DECODE_PRINTF("EDX"); + return &M.x86.R_EDX; + case 3: + DECODE_PRINTF("EBX"); + return &M.x86.R_EBX; + case 4: + DECODE_PRINTF("ESP"); + return &M.x86.R_ESP; + case 5: + DECODE_PRINTF("EBP"); + return &M.x86.R_EBP; + case 6: + DECODE_PRINTF("ESI"); + return &M.x86.R_ESI; + case 7: + DECODE_PRINTF("EDI"); + return &M.x86.R_EDI; + } + HALT_SYS(); + return NULL; /* NOTREACHED OR REACHED ON ERROR */ +} + +/**************************************************************************** +PARAMETERS: +reg - Register to decode + +RETURNS: +Pointer to the appropriate register + +REMARKS: +Return a pointer to the register given by the R/RM field of the +modrm byte, for word operands, modified from above for the weirdo +special case of segreg operands. Also enables the decoding of instructions. +****************************************************************************/ +u16* decode_rm_seg_register( + int reg) +{ + switch (reg) { + case 0: + DECODE_PRINTF("ES"); + return &M.x86.R_ES; + case 1: + DECODE_PRINTF("CS"); + return &M.x86.R_CS; + case 2: + DECODE_PRINTF("SS"); + return &M.x86.R_SS; + case 3: + DECODE_PRINTF("DS"); + return &M.x86.R_DS; + case 4: + DECODE_PRINTF("FS"); + return &M.x86.R_FS; + case 5: + DECODE_PRINTF("GS"); + return &M.x86.R_GS; + case 6: + case 7: + DECODE_PRINTF("ILLEGAL SEGREG"); + break; + } + HALT_SYS(); + return NULL; /* NOT REACHED OR REACHED ON ERROR */ +} + +/* + * + * return offset from the SIB Byte + */ +u32 decode_sib_address(int sib, int mod) +{ + u32 base = 0, i = 0, scale = 1; + + switch(sib & 0x07) { + case 0: + DECODE_PRINTF("[EAX]"); + base = M.x86.R_EAX; + break; + case 1: + DECODE_PRINTF("[ECX]"); + base = M.x86.R_ECX; + break; + case 2: + DECODE_PRINTF("[EDX]"); + base = M.x86.R_EDX; + break; + case 3: + DECODE_PRINTF("[EBX]"); + base = M.x86.R_EBX; + break; + case 4: + DECODE_PRINTF("[ESP]"); + base = M.x86.R_ESP; + M.x86.mode |= SYSMODE_SEG_DS_SS; + break; + case 5: + if (mod == 0) { + base = fetch_long_imm(); + DECODE_PRINTF2("%08x", base); + } else { + DECODE_PRINTF("[EBP]"); + base = M.x86.R_ESP; + M.x86.mode |= SYSMODE_SEG_DS_SS; + } + break; + case 6: + DECODE_PRINTF("[ESI]"); + base = M.x86.R_ESI; + break; + case 7: + DECODE_PRINTF("[EDI]"); + base = M.x86.R_EDI; + break; + } + switch ((sib >> 3) & 0x07) { + case 0: + DECODE_PRINTF("[EAX"); + i = M.x86.R_EAX; + break; + case 1: + DECODE_PRINTF("[ECX"); + i = M.x86.R_ECX; + break; + case 2: + DECODE_PRINTF("[EDX"); + i = M.x86.R_EDX; + break; + case 3: + DECODE_PRINTF("[EBX"); + i = M.x86.R_EBX; + break; + case 4: + i = 0; + break; + case 5: + DECODE_PRINTF("[EBP"); + i = M.x86.R_EBP; + break; + case 6: + DECODE_PRINTF("[ESI"); + i = M.x86.R_ESI; + break; + case 7: + DECODE_PRINTF("[EDI"); + i = M.x86.R_EDI; + break; + } + scale = 1 << ((sib >> 6) & 0x03); + if (((sib >> 3) & 0x07) != 4) { + if (scale == 1) { + DECODE_PRINTF("]"); + } else { + DECODE_PRINTF2("*%d]", scale); + } + } + return base + (i * scale); +} + +/**************************************************************************** +PARAMETERS: +rm - RM value to decode + +RETURNS: +Offset in memory for the address decoding + +REMARKS: +Return the offset given by mod=00 addressing. Also enables the +decoding of instructions. + +NOTE: The code which specifies the corresponding segment (ds vs ss) + below in the case of [BP+..]. The assumption here is that at the + point that this subroutine is called, the bit corresponding to + SYSMODE_SEG_DS_SS will be zero. After every instruction + except the segment override instructions, this bit (as well + as any bits indicating segment overrides) will be clear. So + if a SS access is needed, set this bit. Otherwise, DS access + occurs (unless any of the segment override bits are set). +****************************************************************************/ +u32 decode_rm00_address( + int rm) +{ + u32 offset; + int sib; + + if (M.x86.mode & SYSMODE_PREFIX_ADDR) { + /* 32-bit addressing */ + switch (rm) { + case 0: + DECODE_PRINTF("[EAX]"); + return M.x86.R_EAX; + case 1: + DECODE_PRINTF("[ECX]"); + return M.x86.R_ECX; + case 2: + DECODE_PRINTF("[EDX]"); + return M.x86.R_EDX; + case 3: + DECODE_PRINTF("[EBX]"); + return M.x86.R_EBX; + case 4: + sib = fetch_byte_imm(); + return decode_sib_address(sib, 0); + case 5: + offset = fetch_long_imm(); + DECODE_PRINTF2("[%08x]", offset); + return offset; + case 6: + DECODE_PRINTF("[ESI]"); + return M.x86.R_ESI; + case 7: + DECODE_PRINTF("[EDI]"); + return M.x86.R_EDI; + } + HALT_SYS(); + } else { + /* 16-bit addressing */ + switch (rm) { + case 0: + DECODE_PRINTF("[BX+SI]"); + return (M.x86.R_BX + M.x86.R_SI) & 0xffff; + case 1: + DECODE_PRINTF("[BX+DI]"); + return (M.x86.R_BX + M.x86.R_DI) & 0xffff; + case 2: + DECODE_PRINTF("[BP+SI]"); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return (M.x86.R_BP + M.x86.R_SI) & 0xffff; + case 3: + DECODE_PRINTF("[BP+DI]"); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return (M.x86.R_BP + M.x86.R_DI) & 0xffff; + case 4: + DECODE_PRINTF("[SI]"); + return M.x86.R_SI; + case 5: + DECODE_PRINTF("[DI]"); + return M.x86.R_DI; + case 6: + offset = fetch_word_imm(); + DECODE_PRINTF2("[%04x]", offset); + return offset; + case 7: + DECODE_PRINTF("[BX]"); + return M.x86.R_BX; + } + HALT_SYS(); + } + return 0; +} + +/**************************************************************************** +PARAMETERS: +rm - RM value to decode + +RETURNS: +Offset in memory for the address decoding + +REMARKS: +Return the offset given by mod=01 addressing. Also enables the +decoding of instructions. +****************************************************************************/ +u32 decode_rm01_address( + int rm) +{ + int displacement = 0; + int sib; + + /* Fetch disp8 if no SIB byte */ + if (!((M.x86.mode & SYSMODE_PREFIX_ADDR) && (rm == 4))) + displacement = (s8)fetch_byte_imm(); + + if (M.x86.mode & SYSMODE_PREFIX_ADDR) { + /* 32-bit addressing */ + switch (rm) { + case 0: + DECODE_PRINTF2("%d[EAX]", displacement); + return M.x86.R_EAX + displacement; + case 1: + DECODE_PRINTF2("%d[ECX]", displacement); + return M.x86.R_ECX + displacement; + case 2: + DECODE_PRINTF2("%d[EDX]", displacement); + return M.x86.R_EDX + displacement; + case 3: + DECODE_PRINTF2("%d[EBX]", displacement); + return M.x86.R_EBX + displacement; + case 4: + sib = fetch_byte_imm(); + displacement = (s8)fetch_byte_imm(); + DECODE_PRINTF2("%d", displacement); + return decode_sib_address(sib, 1) + displacement; + case 5: + DECODE_PRINTF2("%d[EBP]", displacement); + return M.x86.R_EBP + displacement; + case 6: + DECODE_PRINTF2("%d[ESI]", displacement); + return M.x86.R_ESI + displacement; + case 7: + DECODE_PRINTF2("%d[EDI]", displacement); + return M.x86.R_EDI + displacement; + } + HALT_SYS(); + } else { + /* 16-bit addressing */ + switch (rm) { + case 0: + DECODE_PRINTF2("%d[BX+SI]", displacement); + return (M.x86.R_BX + M.x86.R_SI + displacement) & 0xffff; + case 1: + DECODE_PRINTF2("%d[BX+DI]", displacement); + return (M.x86.R_BX + M.x86.R_DI + displacement) & 0xffff; + case 2: + DECODE_PRINTF2("%d[BP+SI]", displacement); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return (M.x86.R_BP + M.x86.R_SI + displacement) & 0xffff; + case 3: + DECODE_PRINTF2("%d[BP+DI]", displacement); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return (M.x86.R_BP + M.x86.R_DI + displacement) & 0xffff; + case 4: + DECODE_PRINTF2("%d[SI]", displacement); + return (M.x86.R_SI + displacement) & 0xffff; + case 5: + DECODE_PRINTF2("%d[DI]", displacement); + return (M.x86.R_DI + displacement) & 0xffff; + case 6: + DECODE_PRINTF2("%d[BP]", displacement); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return (M.x86.R_BP + displacement) & 0xffff; + case 7: + DECODE_PRINTF2("%d[BX]", displacement); + return (M.x86.R_BX + displacement) & 0xffff; + } + HALT_SYS(); + } + return 0; /* SHOULD NOT HAPPEN */ +} + +/**************************************************************************** +PARAMETERS: +rm - RM value to decode + +RETURNS: +Offset in memory for the address decoding + +REMARKS: +Return the offset given by mod=10 addressing. Also enables the +decoding of instructions. +****************************************************************************/ +u32 decode_rm10_address( + int rm) +{ + u32 displacement = 0; + int sib; + + /* Fetch disp16 if 16-bit addr mode */ + if (!(M.x86.mode & SYSMODE_PREFIX_ADDR)) + displacement = (u16)fetch_word_imm(); + else { + /* Fetch disp32 if no SIB byte */ + if (rm != 4) + displacement = (u32)fetch_long_imm(); + } + + if (M.x86.mode & SYSMODE_PREFIX_ADDR) { + /* 32-bit addressing */ + switch (rm) { + case 0: + DECODE_PRINTF2("%08x[EAX]", displacement); + return M.x86.R_EAX + displacement; + case 1: + DECODE_PRINTF2("%08x[ECX]", displacement); + return M.x86.R_ECX + displacement; + case 2: + DECODE_PRINTF2("%08x[EDX]", displacement); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return M.x86.R_EDX + displacement; + case 3: + DECODE_PRINTF2("%08x[EBX]", displacement); + return M.x86.R_EBX + displacement; + case 4: + sib = fetch_byte_imm(); + displacement = (u32)fetch_long_imm(); + DECODE_PRINTF2("%08x", displacement); + return decode_sib_address(sib, 2) + displacement; + break; + case 5: + DECODE_PRINTF2("%08x[EBP]", displacement); + return M.x86.R_EBP + displacement; + case 6: + DECODE_PRINTF2("%08x[ESI]", displacement); + return M.x86.R_ESI + displacement; + case 7: + DECODE_PRINTF2("%08x[EDI]", displacement); + return M.x86.R_EDI + displacement; + } + HALT_SYS(); + } else { + /* 16-bit addressing */ + switch (rm) { + case 0: + DECODE_PRINTF2("%04x[BX+SI]", displacement); + return (M.x86.R_BX + M.x86.R_SI + displacement) & 0xffff; + case 1: + DECODE_PRINTF2("%04x[BX+DI]", displacement); + return (M.x86.R_BX + M.x86.R_DI + displacement) & 0xffff; + case 2: + DECODE_PRINTF2("%04x[BP+SI]", displacement); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return (M.x86.R_BP + M.x86.R_SI + displacement) & 0xffff; + case 3: + DECODE_PRINTF2("%04x[BP+DI]", displacement); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return (M.x86.R_BP + M.x86.R_DI + displacement) & 0xffff; + case 4: + DECODE_PRINTF2("%04x[SI]", displacement); + return (M.x86.R_SI + displacement) & 0xffff; + case 5: + DECODE_PRINTF2("%04x[DI]", displacement); + return (M.x86.R_DI + displacement) & 0xffff; + case 6: + DECODE_PRINTF2("%04x[BP]", displacement); + M.x86.mode |= SYSMODE_SEG_DS_SS; + return (M.x86.R_BP + displacement) & 0xffff; + case 7: + DECODE_PRINTF2("%04x[BX]", displacement); + return (M.x86.R_BX + displacement) & 0xffff; + } + HALT_SYS(); + } + return 0; + /*NOTREACHED */ +} diff --git a/userspace/sdks/nv570/nvalloc/unix/x86emu/fpu.c b/userspace/sdks/nv570/nvalloc/unix/x86emu/fpu.c new file mode 100644 index 0000000..0a21c99 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/x86emu/fpu.c @@ -0,0 +1,965 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: This file contains the code to implement the decoding and +* emulation of the FPU instructions. +* +****************************************************************************/ + +#include "x86emu/x86emui.h" + +/*----------------------------- Implementation ----------------------------*/ + +/* opcode=0xd8 */ +void x86emuOp_esc_coprocess_d8(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("ESC D8\n"); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR_NO_TRACE(); +} + +#ifdef X86EMU_DEBUG + +static char *x86emu_fpu_op_d9_tab[] = { + "FLD\tDWORD PTR ", "ESC_D9\t", "FST\tDWORD PTR ", "FSTP\tDWORD PTR ", + "FLDENV\t", "FLDCW\t", "FSTENV\t", "FSTCW\t", + + "FLD\tDWORD PTR ", "ESC_D9\t", "FST\tDWORD PTR ", "FSTP\tDWORD PTR ", + "FLDENV\t", "FLDCW\t", "FSTENV\t", "FSTCW\t", + + "FLD\tDWORD PTR ", "ESC_D9\t", "FST\tDWORD PTR ", "FSTP\tDWORD PTR ", + "FLDENV\t", "FLDCW\t", "FSTENV\t", "FSTCW\t", +}; + +static char *x86emu_fpu_op_d9_tab1[] = { + "FLD\t", "FLD\t", "FLD\t", "FLD\t", + "FLD\t", "FLD\t", "FLD\t", "FLD\t", + + "FXCH\t", "FXCH\t", "FXCH\t", "FXCH\t", + "FXCH\t", "FXCH\t", "FXCH\t", "FXCH\t", + + "FNOP", "ESC_D9", "ESC_D9", "ESC_D9", + "ESC_D9", "ESC_D9", "ESC_D9", "ESC_D9", + + "FSTP\t", "FSTP\t", "FSTP\t", "FSTP\t", + "FSTP\t", "FSTP\t", "FSTP\t", "FSTP\t", + + "FCHS", "FABS", "ESC_D9", "ESC_D9", + "FTST", "FXAM", "ESC_D9", "ESC_D9", + + "FLD1", "FLDL2T", "FLDL2E", "FLDPI", + "FLDLG2", "FLDLN2", "FLDZ", "ESC_D9", + + "F2XM1", "FYL2X", "FPTAN", "FPATAN", + "FXTRACT", "ESC_D9", "FDECSTP", "FINCSTP", + + "FPREM", "FYL2XP1", "FSQRT", "ESC_D9", + "FRNDINT", "FSCALE", "ESC_D9", "ESC_D9", +}; + +#endif /* X86EMU_DEBUG */ + +/* opcode=0xd9 */ +void x86emuOp_esc_coprocess_d9(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset = 0; + u8 stkelem = 0; + + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (mod != 3) { + DECODE_PRINTINSTR32(x86emu_fpu_op_d9_tab, mod, rh, rl); + } else { + DECODE_PRINTF(x86emu_fpu_op_d9_tab1[(rh << 3) + rl]); + } +#endif + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + break; + case 3: /* register to register */ + stkelem = (u8)rl; + if (rh < 4) { + DECODE_PRINTF2("ST(%d)\n", stkelem); + } else { + DECODE_PRINTF("\n"); + } + break; + } +#ifdef X86EMU_FPU_PRESENT + /* execute */ + switch (mod) { + case 3: + switch (rh) { + case 0: + x86emu_fpu_R_fld(X86EMU_FPU_STKTOP, stkelem); + break; + case 1: + x86emu_fpu_R_fxch(X86EMU_FPU_STKTOP, stkelem); + break; + case 2: + switch (rl) { + case 0: + x86emu_fpu_R_nop(); + break; + default: + x86emu_fpu_illegal(); + break; + } + case 3: + x86emu_fpu_R_fstp(X86EMU_FPU_STKTOP, stkelem); + break; + case 4: + switch (rl) { + case 0: + x86emu_fpu_R_fchs(X86EMU_FPU_STKTOP); + break; + case 1: + x86emu_fpu_R_fabs(X86EMU_FPU_STKTOP); + break; + case 4: + x86emu_fpu_R_ftst(X86EMU_FPU_STKTOP); + break; + case 5: + x86emu_fpu_R_fxam(X86EMU_FPU_STKTOP); + break; + default: + /* 2,3,6,7 */ + x86emu_fpu_illegal(); + break; + } + break; + + case 5: + switch (rl) { + case 0: + x86emu_fpu_R_fld1(X86EMU_FPU_STKTOP); + break; + case 1: + x86emu_fpu_R_fldl2t(X86EMU_FPU_STKTOP); + break; + case 2: + x86emu_fpu_R_fldl2e(X86EMU_FPU_STKTOP); + break; + case 3: + x86emu_fpu_R_fldpi(X86EMU_FPU_STKTOP); + break; + case 4: + x86emu_fpu_R_fldlg2(X86EMU_FPU_STKTOP); + break; + case 5: + x86emu_fpu_R_fldln2(X86EMU_FPU_STKTOP); + break; + case 6: + x86emu_fpu_R_fldz(X86EMU_FPU_STKTOP); + break; + default: + /* 7 */ + x86emu_fpu_illegal(); + break; + } + break; + + case 6: + switch (rl) { + case 0: + x86emu_fpu_R_f2xm1(X86EMU_FPU_STKTOP); + break; + case 1: + x86emu_fpu_R_fyl2x(X86EMU_FPU_STKTOP); + break; + case 2: + x86emu_fpu_R_fptan(X86EMU_FPU_STKTOP); + break; + case 3: + x86emu_fpu_R_fpatan(X86EMU_FPU_STKTOP); + break; + case 4: + x86emu_fpu_R_fxtract(X86EMU_FPU_STKTOP); + break; + case 5: + x86emu_fpu_illegal(); + break; + case 6: + x86emu_fpu_R_decstp(); + break; + case 7: + x86emu_fpu_R_incstp(); + break; + } + break; + + case 7: + switch (rl) { + case 0: + x86emu_fpu_R_fprem(X86EMU_FPU_STKTOP); + break; + case 1: + x86emu_fpu_R_fyl2xp1(X86EMU_FPU_STKTOP); + break; + case 2: + x86emu_fpu_R_fsqrt(X86EMU_FPU_STKTOP); + break; + case 3: + x86emu_fpu_illegal(); + break; + case 4: + x86emu_fpu_R_frndint(X86EMU_FPU_STKTOP); + break; + case 5: + x86emu_fpu_R_fscale(X86EMU_FPU_STKTOP); + break; + case 6: + case 7: + default: + x86emu_fpu_illegal(); + break; + } + break; + + default: + switch (rh) { + case 0: + x86emu_fpu_M_fld(X86EMU_FPU_FLOAT, destoffset); + break; + case 1: + x86emu_fpu_illegal(); + break; + case 2: + x86emu_fpu_M_fst(X86EMU_FPU_FLOAT, destoffset); + break; + case 3: + x86emu_fpu_M_fstp(X86EMU_FPU_FLOAT, destoffset); + break; + case 4: + x86emu_fpu_M_fldenv(X86EMU_FPU_WORD, destoffset); + break; + case 5: + x86emu_fpu_M_fldcw(X86EMU_FPU_WORD, destoffset); + break; + case 6: + x86emu_fpu_M_fstenv(X86EMU_FPU_WORD, destoffset); + break; + case 7: + x86emu_fpu_M_fstcw(X86EMU_FPU_WORD, destoffset); + break; + } + } + } +#else + (void)destoffset; + (void)stkelem; +#endif /* X86EMU_FPU_PRESENT */ + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR_NO_TRACE(); +} + +#ifdef X86EMU_DEBUG + +char *x86emu_fpu_op_da_tab[] = { + "FIADD\tDWORD PTR ", "FIMUL\tDWORD PTR ", "FICOM\tDWORD PTR ", + "FICOMP\tDWORD PTR ", + "FISUB\tDWORD PTR ", "FISUBR\tDWORD PTR ", "FIDIV\tDWORD PTR ", + "FIDIVR\tDWORD PTR ", + + "FIADD\tDWORD PTR ", "FIMUL\tDWORD PTR ", "FICOM\tDWORD PTR ", + "FICOMP\tDWORD PTR ", + "FISUB\tDWORD PTR ", "FISUBR\tDWORD PTR ", "FIDIV\tDWORD PTR ", + "FIDIVR\tDWORD PTR ", + + "FIADD\tDWORD PTR ", "FIMUL\tDWORD PTR ", "FICOM\tDWORD PTR ", + "FICOMP\tDWORD PTR ", + "FISUB\tDWORD PTR ", "FISUBR\tDWORD PTR ", "FIDIV\tDWORD PTR ", + "FIDIVR\tDWORD PTR ", + + "ESC_DA ", "ESC_DA ", "ESC_DA ", "ESC_DA ", + "ESC_DA ", "ESC_DA ", "ESC_DA ", "ESC_DA ", +}; + +#endif /* X86EMU_DEBUG */ + +/* opcode=0xda */ +void x86emuOp_esc_coprocess_da(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset = 0; + u8 stkelem = 0; + + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); + DECODE_PRINTINSTR32(x86emu_fpu_op_da_tab, mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + break; + case 3: /* register to register */ + stkelem = (u8)rl; + DECODE_PRINTF2("\tST(%d),ST\n", stkelem); + break; + } +#ifdef X86EMU_FPU_PRESENT + switch (mod) { + case 3: + x86emu_fpu_illegal(); + break; + default: + switch (rh) { + case 0: + x86emu_fpu_M_iadd(X86EMU_FPU_SHORT, destoffset); + break; + case 1: + x86emu_fpu_M_imul(X86EMU_FPU_SHORT, destoffset); + break; + case 2: + x86emu_fpu_M_icom(X86EMU_FPU_SHORT, destoffset); + break; + case 3: + x86emu_fpu_M_icomp(X86EMU_FPU_SHORT, destoffset); + break; + case 4: + x86emu_fpu_M_isub(X86EMU_FPU_SHORT, destoffset); + break; + case 5: + x86emu_fpu_M_isubr(X86EMU_FPU_SHORT, destoffset); + break; + case 6: + x86emu_fpu_M_idiv(X86EMU_FPU_SHORT, destoffset); + break; + case 7: + x86emu_fpu_M_idivr(X86EMU_FPU_SHORT, destoffset); + break; + } + } +#else + (void)destoffset; + (void)stkelem; +#endif + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR_NO_TRACE(); +} + +#ifdef X86EMU_DEBUG + +char *x86emu_fpu_op_db_tab[] = { + "FILD\tDWORD PTR ", "ESC_DB\t19", "FIST\tDWORD PTR ", "FISTP\tDWORD PTR ", + "ESC_DB\t1C", "FLD\tTBYTE PTR ", "ESC_DB\t1E", "FSTP\tTBYTE PTR ", + + "FILD\tDWORD PTR ", "ESC_DB\t19", "FIST\tDWORD PTR ", "FISTP\tDWORD PTR ", + "ESC_DB\t1C", "FLD\tTBYTE PTR ", "ESC_DB\t1E", "FSTP\tTBYTE PTR ", + + "FILD\tDWORD PTR ", "ESC_DB\t19", "FIST\tDWORD PTR ", "FISTP\tDWORD PTR ", + "ESC_DB\t1C", "FLD\tTBYTE PTR ", "ESC_DB\t1E", "FSTP\tTBYTE PTR ", +}; + +#endif /* X86EMU_DEBUG */ + +/* opcode=0xdb */ +void x86emuOp_esc_coprocess_db(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset = 0; + + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (mod != 3) { + DECODE_PRINTINSTR32(x86emu_fpu_op_db_tab, mod, rh, rl); + } else if (rh == 4) { /* === 11 10 0 nnn */ + switch (rl) { + case 0: + DECODE_PRINTF("FENI\n"); + break; + case 1: + DECODE_PRINTF("FDISI\n"); + break; + case 2: + DECODE_PRINTF("FCLEX\n"); + break; + case 3: + DECODE_PRINTF("FINIT\n"); + break; + } + } else { + DECODE_PRINTF2("ESC_DB %0x\n", (mod << 6) + (rh << 3) + (rl)); + } +#endif /* X86EMU_DEBUG */ + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + break; + case 1: + destoffset = decode_rm01_address(rl); + break; + case 2: + destoffset = decode_rm10_address(rl); + break; + case 3: /* register to register */ + break; + } +#ifdef X86EMU_FPU_PRESENT + /* execute */ + switch (mod) { + case 3: + switch (rh) { + case 4: + switch (rl) { + case 0: + x86emu_fpu_R_feni(); + break; + case 1: + x86emu_fpu_R_fdisi(); + break; + case 2: + x86emu_fpu_R_fclex(); + break; + case 3: + x86emu_fpu_R_finit(); + break; + default: + x86emu_fpu_illegal(); + break; + } + break; + default: + x86emu_fpu_illegal(); + break; + } + break; + default: + switch (rh) { + case 0: + x86emu_fpu_M_fild(X86EMU_FPU_SHORT, destoffset); + break; + case 1: + x86emu_fpu_illegal(); + break; + case 2: + x86emu_fpu_M_fist(X86EMU_FPU_SHORT, destoffset); + break; + case 3: + x86emu_fpu_M_fistp(X86EMU_FPU_SHORT, destoffset); + break; + case 4: + x86emu_fpu_illegal(); + break; + case 5: + x86emu_fpu_M_fld(X86EMU_FPU_LDBL, destoffset); + break; + case 6: + x86emu_fpu_illegal(); + break; + case 7: + x86emu_fpu_M_fstp(X86EMU_FPU_LDBL, destoffset); + break; + } + } +#else + (void)destoffset; +#endif + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR_NO_TRACE(); +} + +#ifdef X86EMU_DEBUG +char *x86emu_fpu_op_dc_tab[] = { + "FADD\tQWORD PTR ", "FMUL\tQWORD PTR ", "FCOM\tQWORD PTR ", + "FCOMP\tQWORD PTR ", + "FSUB\tQWORD PTR ", "FSUBR\tQWORD PTR ", "FDIV\tQWORD PTR ", + "FDIVR\tQWORD PTR ", + + "FADD\tQWORD PTR ", "FMUL\tQWORD PTR ", "FCOM\tQWORD PTR ", + "FCOMP\tQWORD PTR ", + "FSUB\tQWORD PTR ", "FSUBR\tQWORD PTR ", "FDIV\tQWORD PTR ", + "FDIVR\tQWORD PTR ", + + "FADD\tQWORD PTR ", "FMUL\tQWORD PTR ", "FCOM\tQWORD PTR ", + "FCOMP\tQWORD PTR ", + "FSUB\tQWORD PTR ", "FSUBR\tQWORD PTR ", "FDIV\tQWORD PTR ", + "FDIVR\tQWORD PTR ", + + "FADD\t", "FMUL\t", "FCOM\t", "FCOMP\t", + "FSUBR\t", "FSUB\t", "FDIVR\t", "FDIV\t", +}; +#endif /* X86EMU_DEBUG */ + +/* opcode=0xdc */ +void x86emuOp_esc_coprocess_dc(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset = 0; + u8 stkelem = 0; + + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); + DECODE_PRINTINSTR32(x86emu_fpu_op_dc_tab, mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + break; + case 3: /* register to register */ + stkelem = (u8)rl; + DECODE_PRINTF2("\tST(%d),ST\n", stkelem); + break; + } +#ifdef X86EMU_FPU_PRESENT + /* execute */ + switch (mod) { + case 3: + switch (rh) { + case 0: + x86emu_fpu_R_fadd(stkelem, X86EMU_FPU_STKTOP); + break; + case 1: + x86emu_fpu_R_fmul(stkelem, X86EMU_FPU_STKTOP); + break; + case 2: + x86emu_fpu_R_fcom(stkelem, X86EMU_FPU_STKTOP); + break; + case 3: + x86emu_fpu_R_fcomp(stkelem, X86EMU_FPU_STKTOP); + break; + case 4: + x86emu_fpu_R_fsubr(stkelem, X86EMU_FPU_STKTOP); + break; + case 5: + x86emu_fpu_R_fsub(stkelem, X86EMU_FPU_STKTOP); + break; + case 6: + x86emu_fpu_R_fdivr(stkelem, X86EMU_FPU_STKTOP); + break; + case 7: + x86emu_fpu_R_fdiv(stkelem, X86EMU_FPU_STKTOP); + break; + } + break; + default: + switch (rh) { + case 0: + x86emu_fpu_M_fadd(X86EMU_FPU_DOUBLE, destoffset); + break; + case 1: + x86emu_fpu_M_fmul(X86EMU_FPU_DOUBLE, destoffset); + break; + case 2: + x86emu_fpu_M_fcom(X86EMU_FPU_DOUBLE, destoffset); + break; + case 3: + x86emu_fpu_M_fcomp(X86EMU_FPU_DOUBLE, destoffset); + break; + case 4: + x86emu_fpu_M_fsub(X86EMU_FPU_DOUBLE, destoffset); + break; + case 5: + x86emu_fpu_M_fsubr(X86EMU_FPU_DOUBLE, destoffset); + break; + case 6: + x86emu_fpu_M_fdiv(X86EMU_FPU_DOUBLE, destoffset); + break; + case 7: + x86emu_fpu_M_fdivr(X86EMU_FPU_DOUBLE, destoffset); + break; + } + } +#else + (void)destoffset; + (void)stkelem; +#endif + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR_NO_TRACE(); +} + +#ifdef X86EMU_DEBUG + +static char *x86emu_fpu_op_dd_tab[] = { + "FLD\tQWORD PTR ", "ESC_DD\t29,", "FST\tQWORD PTR ", "FSTP\tQWORD PTR ", + "FRSTOR\t", "ESC_DD\t2D,", "FSAVE\t", "FSTSW\t", + + "FLD\tQWORD PTR ", "ESC_DD\t29,", "FST\tQWORD PTR ", "FSTP\tQWORD PTR ", + "FRSTOR\t", "ESC_DD\t2D,", "FSAVE\t", "FSTSW\t", + + "FLD\tQWORD PTR ", "ESC_DD\t29,", "FST\tQWORD PTR ", "FSTP\tQWORD PTR ", + "FRSTOR\t", "ESC_DD\t2D,", "FSAVE\t", "FSTSW\t", + + "FFREE\t", "FXCH\t", "FST\t", "FSTP\t", + "ESC_DD\t2C,", "ESC_DD\t2D,", "ESC_DD\t2E,", "ESC_DD\t2F,", +}; + +#endif /* X86EMU_DEBUG */ + +/* opcode=0xdd */ +void x86emuOp_esc_coprocess_dd(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset = 0; + u8 stkelem = 0; + + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); + DECODE_PRINTINSTR32(x86emu_fpu_op_dd_tab, mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + break; + case 3: /* register to register */ + stkelem = (u8)rl; + DECODE_PRINTF2("\tST(%d),ST\n", stkelem); + break; + } +#ifdef X86EMU_FPU_PRESENT + switch (mod) { + case 3: + switch (rh) { + case 0: + x86emu_fpu_R_ffree(stkelem); + break; + case 1: + x86emu_fpu_R_fxch(stkelem); + break; + case 2: + x86emu_fpu_R_fst(stkelem); /* register version */ + break; + case 3: + x86emu_fpu_R_fstp(stkelem); /* register version */ + break; + default: + x86emu_fpu_illegal(); + break; + } + break; + default: + switch (rh) { + case 0: + x86emu_fpu_M_fld(X86EMU_FPU_DOUBLE, destoffset); + break; + case 1: + x86emu_fpu_illegal(); + break; + case 2: + x86emu_fpu_M_fst(X86EMU_FPU_DOUBLE, destoffset); + break; + case 3: + x86emu_fpu_M_fstp(X86EMU_FPU_DOUBLE, destoffset); + break; + case 4: + x86emu_fpu_M_frstor(X86EMU_FPU_WORD, destoffset); + break; + case 5: + x86emu_fpu_illegal(); + break; + case 6: + x86emu_fpu_M_fsave(X86EMU_FPU_WORD, destoffset); + break; + case 7: + x86emu_fpu_M_fstsw(X86EMU_FPU_WORD, destoffset); + break; + } + } +#else + (void)destoffset; + (void)stkelem; +#endif + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR_NO_TRACE(); +} + +#ifdef X86EMU_DEBUG + +static char *x86emu_fpu_op_de_tab[] = +{ + "FIADD\tWORD PTR ", "FIMUL\tWORD PTR ", "FICOM\tWORD PTR ", + "FICOMP\tWORD PTR ", + "FISUB\tWORD PTR ", "FISUBR\tWORD PTR ", "FIDIV\tWORD PTR ", + "FIDIVR\tWORD PTR ", + + "FIADD\tWORD PTR ", "FIMUL\tWORD PTR ", "FICOM\tWORD PTR ", + "FICOMP\tWORD PTR ", + "FISUB\tWORD PTR ", "FISUBR\tWORD PTR ", "FIDIV\tWORD PTR ", + "FIDIVR\tWORD PTR ", + + "FIADD\tWORD PTR ", "FIMUL\tWORD PTR ", "FICOM\tWORD PTR ", + "FICOMP\tWORD PTR ", + "FISUB\tWORD PTR ", "FISUBR\tWORD PTR ", "FIDIV\tWORD PTR ", + "FIDIVR\tWORD PTR ", + + "FADDP\t", "FMULP\t", "FCOMP\t", "FCOMPP\t", + "FSUBRP\t", "FSUBP\t", "FDIVRP\t", "FDIVP\t", +}; + +#endif /* X86EMU_DEBUG */ + +/* opcode=0xde */ +void x86emuOp_esc_coprocess_de(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset = 0; + u8 stkelem = 0; + + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); + DECODE_PRINTINSTR32(x86emu_fpu_op_de_tab, mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + break; + case 3: /* register to register */ + stkelem = (u8)rl; + DECODE_PRINTF2("\tST(%d),ST\n", stkelem); + break; + } +#ifdef X86EMU_FPU_PRESENT + switch (mod) { + case 3: + switch (rh) { + case 0: + x86emu_fpu_R_faddp(stkelem, X86EMU_FPU_STKTOP); + break; + case 1: + x86emu_fpu_R_fmulp(stkelem, X86EMU_FPU_STKTOP); + break; + case 2: + x86emu_fpu_R_fcomp(stkelem, X86EMU_FPU_STKTOP); + break; + case 3: + if (stkelem == 1) + x86emu_fpu_R_fcompp(stkelem, X86EMU_FPU_STKTOP); + else + x86emu_fpu_illegal(); + break; + case 4: + x86emu_fpu_R_fsubrp(stkelem, X86EMU_FPU_STKTOP); + break; + case 5: + x86emu_fpu_R_fsubp(stkelem, X86EMU_FPU_STKTOP); + break; + case 6: + x86emu_fpu_R_fdivrp(stkelem, X86EMU_FPU_STKTOP); + break; + case 7: + x86emu_fpu_R_fdivp(stkelem, X86EMU_FPU_STKTOP); + break; + } + break; + default: + switch (rh) { + case 0: + x86emu_fpu_M_fiadd(X86EMU_FPU_WORD, destoffset); + break; + case 1: + x86emu_fpu_M_fimul(X86EMU_FPU_WORD, destoffset); + break; + case 2: + x86emu_fpu_M_ficom(X86EMU_FPU_WORD, destoffset); + break; + case 3: + x86emu_fpu_M_ficomp(X86EMU_FPU_WORD, destoffset); + break; + case 4: + x86emu_fpu_M_fisub(X86EMU_FPU_WORD, destoffset); + break; + case 5: + x86emu_fpu_M_fisubr(X86EMU_FPU_WORD, destoffset); + break; + case 6: + x86emu_fpu_M_fidiv(X86EMU_FPU_WORD, destoffset); + break; + case 7: + x86emu_fpu_M_fidivr(X86EMU_FPU_WORD, destoffset); + break; + } + } +#else + (void)destoffset; + (void)stkelem; +#endif + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR_NO_TRACE(); +} + +#ifdef X86EMU_DEBUG + +static char *x86emu_fpu_op_df_tab[] = { + /* mod == 00 */ + "FILD\tWORD PTR ", "ESC_DF\t39\n", "FIST\tWORD PTR ", "FISTP\tWORD PTR ", + "FBLD\tTBYTE PTR ", "FILD\tQWORD PTR ", "FBSTP\tTBYTE PTR ", + "FISTP\tQWORD PTR ", + + /* mod == 01 */ + "FILD\tWORD PTR ", "ESC_DF\t39 ", "FIST\tWORD PTR ", "FISTP\tWORD PTR ", + "FBLD\tTBYTE PTR ", "FILD\tQWORD PTR ", "FBSTP\tTBYTE PTR ", + "FISTP\tQWORD PTR ", + + /* mod == 10 */ + "FILD\tWORD PTR ", "ESC_DF\t39 ", "FIST\tWORD PTR ", "FISTP\tWORD PTR ", + "FBLD\tTBYTE PTR ", "FILD\tQWORD PTR ", "FBSTP\tTBYTE PTR ", + "FISTP\tQWORD PTR ", + + /* mod == 11 */ + "FFREE\t", "FXCH\t", "FST\t", "FSTP\t", + "ESC_DF\t3C,", "ESC_DF\t3D,", "ESC_DF\t3E,", "ESC_DF\t3F," +}; + +#endif /* X86EMU_DEBUG */ + +/* opcode=0xdf */ +void x86emuOp_esc_coprocess_df(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset = 0; + u8 stkelem = 0; + + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); + DECODE_PRINTINSTR32(x86emu_fpu_op_df_tab, mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + break; + case 3: /* register to register */ + stkelem = (u8)rl; + DECODE_PRINTF2("\tST(%d)\n", stkelem); + break; + } +#ifdef X86EMU_FPU_PRESENT + switch (mod) { + case 3: + switch (rh) { + case 0: + x86emu_fpu_R_ffree(stkelem); + break; + case 1: + x86emu_fpu_R_fxch(stkelem); + break; + case 2: + x86emu_fpu_R_fst(stkelem); /* register version */ + break; + case 3: + x86emu_fpu_R_fstp(stkelem); /* register version */ + break; + default: + x86emu_fpu_illegal(); + break; + } + break; + default: + switch (rh) { + case 0: + x86emu_fpu_M_fild(X86EMU_FPU_WORD, destoffset); + break; + case 1: + x86emu_fpu_illegal(); + break; + case 2: + x86emu_fpu_M_fist(X86EMU_FPU_WORD, destoffset); + break; + case 3: + x86emu_fpu_M_fistp(X86EMU_FPU_WORD, destoffset); + break; + case 4: + x86emu_fpu_M_fbld(X86EMU_FPU_BSD, destoffset); + break; + case 5: + x86emu_fpu_M_fild(X86EMU_FPU_LONG, destoffset); + break; + case 6: + x86emu_fpu_M_fbstp(X86EMU_FPU_BSD, destoffset); + break; + case 7: + x86emu_fpu_M_fistp(X86EMU_FPU_LONG, destoffset); + break; + } + } +#else + (void)destoffset; + (void)stkelem; +#endif + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR_NO_TRACE(); +} diff --git a/userspace/sdks/nv570/nvalloc/unix/x86emu/ops.c b/userspace/sdks/nv570/nvalloc/unix/x86emu/ops.c new file mode 100644 index 0000000..52b97b4 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/x86emu/ops.c @@ -0,0 +1,11761 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: This file includes subroutines to implement the decoding +* and emulation of all the x86 processor instructions. +* +* There are approximately 250 subroutines in here, which correspond +* to the 256 byte-"opcodes" found on the 8086. The table which +* dispatches this is found in the files optab.[ch]. +* +* Each opcode proc has a comment preceeding it which gives it's table +* address. Several opcodes are missing (undefined) in the table. +* +* Each proc includes information for decoding (DECODE_PRINTF and +* DECODE_PRINTF2), debugging (TRACE_REGS, SINGLE_STEP), and misc +* functions (START_OF_INSTR, END_OF_INSTR). +* +* Many of the procedures are *VERY* similar in coding. This has +* allowed for a very large amount of code to be generated in a fairly +* short amount of time (i.e. cut, paste, and modify). The result is +* that much of the code below could have been folded into subroutines +* for a large reduction in size of this file. The downside would be +* that there would be a penalty in execution speed. The file could +* also have been *MUCH* larger by inlining certain functions which +* were called. This could have resulted even faster execution. The +* prime directive I used to decide whether to inline the code or to +* modularize it, was basically: 1) no unnecessary subroutine calls, +* 2) no routines more than about 200 lines in size, and 3) modularize +* any code that I might not get right the first time. The fetch_* +* subroutines fall into the latter category. The The decode_* fall +* into the second category. The coding of the "switch(mod){ .... }" +* in many of the subroutines below falls into the first category. +* Especially, the coding of {add,and,or,sub,...}_{byte,word} +* subroutines are an especially glaring case of the third guideline. +* Since so much of the code is cloned from other modules (compare +* opcode #00 to opcode #01), making the basic operations subroutine +* calls is especially important; otherwise mistakes in coding an +* "add" would represent a nightmare in maintenance. +* +****************************************************************************/ + +#include "x86emu/x86emui.h" + +/*----------------------------- Implementation ----------------------------*/ + +/**************************************************************************** +PARAMETERS: +op1 - Instruction op code + +REMARKS: +Handles illegal opcodes. +****************************************************************************/ +static void x86emuOp_illegal_op( + u8 op1) +{ + START_OF_INSTR(); + if (M.x86.R_SP != 0) { + DECODE_PRINTF("ILLEGAL X86 OPCODE\n"); + TRACE_REGS(); + DB( NV_PRINTF(LEVEL_INFO, "%04x:%04x: %02X ILLEGAL X86 OPCODE!\n", + M.x86.R_CS, M.x86.R_IP - 1, op1);) + HALT_SYS(); + } + else { + /* If we get here, it means the stack pointer is back to zero + * so we are just returning from an emulator service call + * so therte is no need to display an error message. We trap + * the emulator with an 0xF1 opcode to finish the service + * call. + */ + X86EMU_halt_sys(); + } + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x00 +****************************************************************************/ +static void x86emuOp_add_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + u8 *destreg, *srcreg; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("ADD\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x01 +****************************************************************************/ +static void x86emuOp_add_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("ADD\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = add_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x02 +****************************************************************************/ +static void x86emuOp_add_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("ADD\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_byte(*destreg, srcval); + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_byte(*destreg, srcval); + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_byte(*destreg, srcval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x03 +****************************************************************************/ +static void x86emuOp_add_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("ADD\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_word(*destreg, srcval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_word(*destreg, srcval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_word(*destreg, srcval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = add_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x04 +****************************************************************************/ +static void x86emuOp_add_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("ADD\tAL,"); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + M.x86.R_AL = add_byte(M.x86.R_AL, srcval); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x05 +****************************************************************************/ +static void x86emuOp_add_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("ADD\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("ADD\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = add_long(M.x86.R_EAX, srcval); + } else { + M.x86.R_AX = add_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x06 +****************************************************************************/ +static void x86emuOp_push_ES(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("PUSH\tES\n"); + TRACE_AND_STEP(); + push_word(M.x86.R_ES); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x07 +****************************************************************************/ +static void x86emuOp_pop_ES(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("POP\tES\n"); + TRACE_AND_STEP(); + M.x86.R_ES = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x08 +****************************************************************************/ +static void x86emuOp_or_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("OR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x09 +****************************************************************************/ +static void x86emuOp_or_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("OR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = or_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0a +****************************************************************************/ +static void x86emuOp_or_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("OR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_byte(*destreg, srcval); + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_byte(*destreg, srcval); + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_byte(*destreg, srcval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0b +****************************************************************************/ +static void x86emuOp_or_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("OR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_word(*destreg, srcval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_word(*destreg, srcval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_word(*destreg, srcval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = or_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0c +****************************************************************************/ +static void x86emuOp_or_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("OR\tAL,"); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + M.x86.R_AL = or_byte(M.x86.R_AL, srcval); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0d +****************************************************************************/ +static void x86emuOp_or_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("OR\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("OR\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = or_long(M.x86.R_EAX, srcval); + } else { + M.x86.R_AX = or_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0e +****************************************************************************/ +static void x86emuOp_push_CS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("PUSH\tCS\n"); + TRACE_AND_STEP(); + push_word(M.x86.R_CS); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f. Escape for two-byte opcode (286 or better) +****************************************************************************/ +static void x86emuOp_two_byte(u8 X86EMU_UNUSED(op1)) +{ + u8 op2 = (*sys_rdb)(((u32)M.x86.R_CS << 4) + (M.x86.R_IP++)); + INC_DECODED_INST_LEN(1); + (*x86emu_optab2[op2])(op2); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x10 +****************************************************************************/ +static void x86emuOp_adc_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("ADC\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x11 +****************************************************************************/ +static void x86emuOp_adc_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("ADC\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = adc_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x12 +****************************************************************************/ +static void x86emuOp_adc_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("ADC\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_byte(*destreg, srcval); + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_byte(*destreg, srcval); + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_byte(*destreg, srcval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x13 +****************************************************************************/ +static void x86emuOp_adc_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("ADC\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_word(*destreg, srcval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_word(*destreg, srcval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_word(*destreg, srcval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = adc_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x14 +****************************************************************************/ +static void x86emuOp_adc_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("ADC\tAL,"); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + M.x86.R_AL = adc_byte(M.x86.R_AL, srcval); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x15 +****************************************************************************/ +static void x86emuOp_adc_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("ADC\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("ADC\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = adc_long(M.x86.R_EAX, srcval); + } else { + M.x86.R_AX = adc_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x16 +****************************************************************************/ +static void x86emuOp_push_SS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("PUSH\tSS\n"); + TRACE_AND_STEP(); + push_word(M.x86.R_SS); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x17 +****************************************************************************/ +static void x86emuOp_pop_SS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("POP\tSS\n"); + TRACE_AND_STEP(); + M.x86.R_SS = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x18 +****************************************************************************/ +static void x86emuOp_sbb_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("SBB\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x19 +****************************************************************************/ +static void x86emuOp_sbb_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("SBB\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sbb_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x1a +****************************************************************************/ +static void x86emuOp_sbb_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("SBB\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_byte(*destreg, srcval); + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_byte(*destreg, srcval); + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_byte(*destreg, srcval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x1b +****************************************************************************/ +static void x86emuOp_sbb_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("SBB\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_word(*destreg, srcval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_word(*destreg, srcval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_word(*destreg, srcval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sbb_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x1c +****************************************************************************/ +static void x86emuOp_sbb_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("SBB\tAL,"); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + M.x86.R_AL = sbb_byte(M.x86.R_AL, srcval); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x1d +****************************************************************************/ +static void x86emuOp_sbb_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("SBB\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("SBB\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = sbb_long(M.x86.R_EAX, srcval); + } else { + M.x86.R_AX = sbb_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x1e +****************************************************************************/ +static void x86emuOp_push_DS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("PUSH\tDS\n"); + TRACE_AND_STEP(); + push_word(M.x86.R_DS); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x1f +****************************************************************************/ +static void x86emuOp_pop_DS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("POP\tDS\n"); + TRACE_AND_STEP(); + M.x86.R_DS = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x20 +****************************************************************************/ +static void x86emuOp_and_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("AND\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x21 +****************************************************************************/ +static void x86emuOp_and_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("AND\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = and_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x22 +****************************************************************************/ +static void x86emuOp_and_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("AND\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_byte(*destreg, srcval); + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_byte(*destreg, srcval); + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_byte(*destreg, srcval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x23 +****************************************************************************/ +static void x86emuOp_and_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("AND\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_word(*destreg, srcval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_long(*destreg, srcval); + break; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_word(*destreg, srcval); + break; + } + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_word(*destreg, srcval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = and_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x24 +****************************************************************************/ +static void x86emuOp_and_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("AND\tAL,"); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + M.x86.R_AL = and_byte(M.x86.R_AL, srcval); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x25 +****************************************************************************/ +static void x86emuOp_and_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("AND\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("AND\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = and_long(M.x86.R_EAX, srcval); + } else { + M.x86.R_AX = and_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x26 +****************************************************************************/ +static void x86emuOp_segovr_ES(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("ES:\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_SEGOVR_ES; + /* + * note the lack of DECODE_CLEAR_SEGOVR(r) since, here is one of 4 + * opcode subroutines we do not want to do this. + */ + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x27 +****************************************************************************/ +static void x86emuOp_daa(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("DAA\n"); + TRACE_AND_STEP(); + M.x86.R_AL = daa_byte(M.x86.R_AL); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x28 +****************************************************************************/ +static void x86emuOp_sub_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("SUB\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x29 +****************************************************************************/ +static void x86emuOp_sub_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("SUB\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = sub_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x2a +****************************************************************************/ +static void x86emuOp_sub_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("SUB\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_byte(*destreg, srcval); + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_byte(*destreg, srcval); + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_byte(*destreg, srcval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x2b +****************************************************************************/ +static void x86emuOp_sub_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("SUB\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_word(*destreg, srcval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_word(*destreg, srcval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_word(*destreg, srcval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = sub_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x2c +****************************************************************************/ +static void x86emuOp_sub_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("SUB\tAL,"); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + M.x86.R_AL = sub_byte(M.x86.R_AL, srcval); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x2d +****************************************************************************/ +static void x86emuOp_sub_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("SUB\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("SUB\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = sub_long(M.x86.R_EAX, srcval); + } else { + M.x86.R_AX = sub_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x2e +****************************************************************************/ +static void x86emuOp_segovr_CS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("CS:\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_SEGOVR_CS; + /* note no DECODE_CLEAR_SEGOVR here. */ + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x2f +****************************************************************************/ +static void x86emuOp_das(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("DAS\n"); + TRACE_AND_STEP(); + M.x86.R_AL = das_byte(M.x86.R_AL); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x30 +****************************************************************************/ +static void x86emuOp_xor_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("XOR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_byte(destval, *srcreg); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x31 +****************************************************************************/ +static void x86emuOp_xor_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("XOR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_long(destval, *srcreg); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = xor_word(destval, *srcreg); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x32 +****************************************************************************/ +static void x86emuOp_xor_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("XOR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_byte(*destreg, srcval); + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_byte(*destreg, srcval); + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_byte(*destreg, srcval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x33 +****************************************************************************/ +static void x86emuOp_xor_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("XOR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_word(*destreg, srcval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_word(*destreg, srcval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_word(*destreg, srcval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = xor_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x34 +****************************************************************************/ +static void x86emuOp_xor_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("XOR\tAL,"); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + M.x86.R_AL = xor_byte(M.x86.R_AL, srcval); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x35 +****************************************************************************/ +static void x86emuOp_xor_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("XOR\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("XOR\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = xor_long(M.x86.R_EAX, srcval); + } else { + M.x86.R_AX = xor_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x36 +****************************************************************************/ +static void x86emuOp_segovr_SS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("SS:\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_SEGOVR_SS; + /* no DECODE_CLEAR_SEGOVR ! */ + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x37 +****************************************************************************/ +static void x86emuOp_aaa(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("AAA\n"); + TRACE_AND_STEP(); + M.x86.R_AX = aaa_word(M.x86.R_AX); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x38 +****************************************************************************/ +static void x86emuOp_cmp_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + u8 *destreg, *srcreg; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("CMP\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_byte(destval, *srcreg); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_byte(destval, *srcreg); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_byte(destval, *srcreg); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x39 +****************************************************************************/ +static void x86emuOp_cmp_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("CMP\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_long(destval, *srcreg); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_word(destval, *srcreg); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_long(destval, *srcreg); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_word(destval, *srcreg); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_long(destval, *srcreg); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_word(destval, *srcreg); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x3a +****************************************************************************/ +static void x86emuOp_cmp_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("CMP\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_byte(*destreg, srcval); + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_byte(*destreg, srcval); + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_byte(*destreg, srcval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x3b +****************************************************************************/ +static void x86emuOp_cmp_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("CMP\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_word(*destreg, srcval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_word(*destreg, srcval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_word(*destreg, srcval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + cmp_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x3c +****************************************************************************/ +static void x86emuOp_cmp_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("CMP\tAL,"); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + cmp_byte(M.x86.R_AL, srcval); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x3d +****************************************************************************/ +static void x86emuOp_cmp_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("CMP\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("CMP\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + cmp_long(M.x86.R_EAX, srcval); + } else { + cmp_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x3e +****************************************************************************/ +static void x86emuOp_segovr_DS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("DS:\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_SEGOVR_DS; + /* NO DECODE_CLEAR_SEGOVR! */ + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x3f +****************************************************************************/ +static void x86emuOp_aas(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("AAS\n"); + TRACE_AND_STEP(); + M.x86.R_AX = aas_word(M.x86.R_AX); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x40 +****************************************************************************/ +static void x86emuOp_inc_AX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tEAX\n"); + } else { + DECODE_PRINTF("INC\tAX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = inc_long(M.x86.R_EAX); + } else { + M.x86.R_AX = inc_word(M.x86.R_AX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x41 +****************************************************************************/ +static void x86emuOp_inc_CX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tECX\n"); + } else { + DECODE_PRINTF("INC\tCX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ECX = inc_long(M.x86.R_ECX); + } else { + M.x86.R_CX = inc_word(M.x86.R_CX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x42 +****************************************************************************/ +static void x86emuOp_inc_DX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tEDX\n"); + } else { + DECODE_PRINTF("INC\tDX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDX = inc_long(M.x86.R_EDX); + } else { + M.x86.R_DX = inc_word(M.x86.R_DX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x43 +****************************************************************************/ +static void x86emuOp_inc_BX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tEBX\n"); + } else { + DECODE_PRINTF("INC\tBX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EBX = inc_long(M.x86.R_EBX); + } else { + M.x86.R_BX = inc_word(M.x86.R_BX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x44 +****************************************************************************/ +static void x86emuOp_inc_SP(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tESP\n"); + } else { + DECODE_PRINTF("INC\tSP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ESP = inc_long(M.x86.R_ESP); + } else { + M.x86.R_SP = inc_word(M.x86.R_SP); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x45 +****************************************************************************/ +static void x86emuOp_inc_BP(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tEBP\n"); + } else { + DECODE_PRINTF("INC\tBP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EBP = inc_long(M.x86.R_EBP); + } else { + M.x86.R_BP = inc_word(M.x86.R_BP); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x46 +****************************************************************************/ +static void x86emuOp_inc_SI(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tESI\n"); + } else { + DECODE_PRINTF("INC\tSI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ESI = inc_long(M.x86.R_ESI); + } else { + M.x86.R_SI = inc_word(M.x86.R_SI); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x47 +****************************************************************************/ +static void x86emuOp_inc_DI(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tEDI\n"); + } else { + DECODE_PRINTF("INC\tDI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDI = inc_long(M.x86.R_EDI); + } else { + M.x86.R_DI = inc_word(M.x86.R_DI); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x48 +****************************************************************************/ +static void x86emuOp_dec_AX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tEAX\n"); + } else { + DECODE_PRINTF("DEC\tAX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = dec_long(M.x86.R_EAX); + } else { + M.x86.R_AX = dec_word(M.x86.R_AX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x49 +****************************************************************************/ +static void x86emuOp_dec_CX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tECX\n"); + } else { + DECODE_PRINTF("DEC\tCX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ECX = dec_long(M.x86.R_ECX); + } else { + M.x86.R_CX = dec_word(M.x86.R_CX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x4a +****************************************************************************/ +static void x86emuOp_dec_DX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tEDX\n"); + } else { + DECODE_PRINTF("DEC\tDX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDX = dec_long(M.x86.R_EDX); + } else { + M.x86.R_DX = dec_word(M.x86.R_DX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x4b +****************************************************************************/ +static void x86emuOp_dec_BX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tEBX\n"); + } else { + DECODE_PRINTF("DEC\tBX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EBX = dec_long(M.x86.R_EBX); + } else { + M.x86.R_BX = dec_word(M.x86.R_BX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x4c +****************************************************************************/ +static void x86emuOp_dec_SP(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tESP\n"); + } else { + DECODE_PRINTF("DEC\tSP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ESP = dec_long(M.x86.R_ESP); + } else { + M.x86.R_SP = dec_word(M.x86.R_SP); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x4d +****************************************************************************/ +static void x86emuOp_dec_BP(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tEBP\n"); + } else { + DECODE_PRINTF("DEC\tBP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EBP = dec_long(M.x86.R_EBP); + } else { + M.x86.R_BP = dec_word(M.x86.R_BP); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x4e +****************************************************************************/ +static void x86emuOp_dec_SI(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tESI\n"); + } else { + DECODE_PRINTF("DEC\tSI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ESI = dec_long(M.x86.R_ESI); + } else { + M.x86.R_SI = dec_word(M.x86.R_SI); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x4f +****************************************************************************/ +static void x86emuOp_dec_DI(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tEDI\n"); + } else { + DECODE_PRINTF("DEC\tDI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDI = dec_long(M.x86.R_EDI); + } else { + M.x86.R_DI = dec_word(M.x86.R_DI); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x50 +****************************************************************************/ +static void x86emuOp_push_AX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSH\tEAX\n"); + } else { + DECODE_PRINTF("PUSH\tAX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_EAX); + } else { + push_word(M.x86.R_AX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x51 +****************************************************************************/ +static void x86emuOp_push_CX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSH\tECX\n"); + } else { + DECODE_PRINTF("PUSH\tCX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_ECX); + } else { + push_word(M.x86.R_CX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x52 +****************************************************************************/ +static void x86emuOp_push_DX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSH\tEDX\n"); + } else { + DECODE_PRINTF("PUSH\tDX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_EDX); + } else { + push_word(M.x86.R_DX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x53 +****************************************************************************/ +static void x86emuOp_push_BX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSH\tEBX\n"); + } else { + DECODE_PRINTF("PUSH\tBX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_EBX); + } else { + push_word(M.x86.R_BX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x54 +****************************************************************************/ +static void x86emuOp_push_SP(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSH\tESP\n"); + } else { + DECODE_PRINTF("PUSH\tSP\n"); + } + TRACE_AND_STEP(); + /* Always push (E)SP, since we are emulating an i386 and above + * processor. This is necessary as some BIOS'es use this to check + * what type of processor is in the system. + */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_ESP); + } else { + push_word((u16)(M.x86.R_SP)); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x55 +****************************************************************************/ +static void x86emuOp_push_BP(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSH\tEBP\n"); + } else { + DECODE_PRINTF("PUSH\tBP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_EBP); + } else { + push_word(M.x86.R_BP); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x56 +****************************************************************************/ +static void x86emuOp_push_SI(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSH\tESI\n"); + } else { + DECODE_PRINTF("PUSH\tSI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_ESI); + } else { + push_word(M.x86.R_SI); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x57 +****************************************************************************/ +static void x86emuOp_push_DI(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSH\tEDI\n"); + } else { + DECODE_PRINTF("PUSH\tDI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_EDI); + } else { + push_word(M.x86.R_DI); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x58 +****************************************************************************/ +static void x86emuOp_pop_AX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POP\tEAX\n"); + } else { + DECODE_PRINTF("POP\tAX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = pop_long(); + } else { + M.x86.R_AX = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x59 +****************************************************************************/ +static void x86emuOp_pop_CX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POP\tECX\n"); + } else { + DECODE_PRINTF("POP\tCX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ECX = pop_long(); + } else { + M.x86.R_CX = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x5a +****************************************************************************/ +static void x86emuOp_pop_DX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POP\tEDX\n"); + } else { + DECODE_PRINTF("POP\tDX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDX = pop_long(); + } else { + M.x86.R_DX = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x5b +****************************************************************************/ +static void x86emuOp_pop_BX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POP\tEBX\n"); + } else { + DECODE_PRINTF("POP\tBX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EBX = pop_long(); + } else { + M.x86.R_BX = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x5c +****************************************************************************/ +static void x86emuOp_pop_SP(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POP\tESP\n"); + } else { + DECODE_PRINTF("POP\tSP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ESP = pop_long(); + } else { + M.x86.R_SP = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x5d +****************************************************************************/ +static void x86emuOp_pop_BP(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POP\tEBP\n"); + } else { + DECODE_PRINTF("POP\tBP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EBP = pop_long(); + } else { + M.x86.R_BP = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x5e +****************************************************************************/ +static void x86emuOp_pop_SI(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POP\tESI\n"); + } else { + DECODE_PRINTF("POP\tSI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ESI = pop_long(); + } else { + M.x86.R_SI = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x5f +****************************************************************************/ +static void x86emuOp_pop_DI(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POP\tEDI\n"); + } else { + DECODE_PRINTF("POP\tDI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDI = pop_long(); + } else { + M.x86.R_DI = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x60 +****************************************************************************/ +static void x86emuOp_push_all(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSHAD\n"); + } else { + DECODE_PRINTF("PUSHA\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 old_sp = M.x86.R_ESP; + + push_long(M.x86.R_EAX); + push_long(M.x86.R_ECX); + push_long(M.x86.R_EDX); + push_long(M.x86.R_EBX); + push_long(old_sp); + push_long(M.x86.R_EBP); + push_long(M.x86.R_ESI); + push_long(M.x86.R_EDI); + } else { + u16 old_sp = M.x86.R_SP; + + push_word(M.x86.R_AX); + push_word(M.x86.R_CX); + push_word(M.x86.R_DX); + push_word(M.x86.R_BX); + push_word(old_sp); + push_word(M.x86.R_BP); + push_word(M.x86.R_SI); + push_word(M.x86.R_DI); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x61 +****************************************************************************/ +static void x86emuOp_pop_all(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POPAD\n"); + } else { + DECODE_PRINTF("POPA\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDI = pop_long(); + M.x86.R_ESI = pop_long(); + M.x86.R_EBP = pop_long(); + M.x86.R_ESP += 4; /* skip ESP */ + M.x86.R_EBX = pop_long(); + M.x86.R_EDX = pop_long(); + M.x86.R_ECX = pop_long(); + M.x86.R_EAX = pop_long(); + } else { + M.x86.R_DI = pop_word(); + M.x86.R_SI = pop_word(); + M.x86.R_BP = pop_word(); + M.x86.R_SP += 2; /* skip SP */ + M.x86.R_BX = pop_word(); + M.x86.R_DX = pop_word(); + M.x86.R_CX = pop_word(); + M.x86.R_AX = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/*opcode 0x62 ILLEGAL OP, calls x86emuOp_illegal_op() */ +/*opcode 0x63 ILLEGAL OP, calls x86emuOp_illegal_op() */ + +/**************************************************************************** +REMARKS: +Handles opcode 0x64 +****************************************************************************/ +static void x86emuOp_segovr_FS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("FS:\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_SEGOVR_FS; + /* + * note the lack of DECODE_CLEAR_SEGOVR(r) since, here is one of 4 + * opcode subroutines we do not want to do this. + */ + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x65 +****************************************************************************/ +static void x86emuOp_segovr_GS(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("GS:\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_SEGOVR_GS; + /* + * note the lack of DECODE_CLEAR_SEGOVR(r) since, here is one of 4 + * opcode subroutines we do not want to do this. + */ + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x66 - prefix for 32-bit register +****************************************************************************/ +static void x86emuOp_prefix_data(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("DATA:\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_PREFIX_DATA; + /* note no DECODE_CLEAR_SEGOVR here. */ + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x67 - prefix for 32-bit address +****************************************************************************/ +static void x86emuOp_prefix_addr(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("ADDR:\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_PREFIX_ADDR; + /* note no DECODE_CLEAR_SEGOVR here. */ + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x68 +****************************************************************************/ +static void x86emuOp_push_word_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 imm; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + imm = fetch_long_imm(); + } else { + imm = fetch_word_imm(); + } + DECODE_PRINTF2("PUSH\t%x\n", imm); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(imm); + } else { + push_word((u16)imm); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x69 +****************************************************************************/ +static void x86emuOp_imul_word_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("IMUL\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + s32 imm; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + imm = fetch_long_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)srcval,(s32)imm); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + s16 imm; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + imm = fetch_word_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + res = (s16)srcval * (s16)imm; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + s32 imm; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + imm = fetch_long_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)srcval,(s32)imm); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + s16 imm; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + imm = fetch_word_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + res = (s16)srcval * (s16)imm; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + s32 imm; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + imm = fetch_long_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)srcval,(s32)imm); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + s16 imm; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + imm = fetch_word_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + res = (s16)srcval * (s16)imm; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + u32 res_lo,res_hi; + s32 imm; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + imm = fetch_long_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)*srcreg,(s32)imm); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg,*srcreg; + u32 res; + s16 imm; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + imm = fetch_word_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + res = (s16)*srcreg * (s16)imm; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x6a +****************************************************************************/ +static void x86emuOp_push_byte_IMM(u8 X86EMU_UNUSED(op1)) +{ + s16 imm; + + START_OF_INSTR(); + imm = (s8)fetch_byte_imm(); + DECODE_PRINTF2("PUSH\t%d\n", imm); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long((s32)imm); + } else { + push_word(imm); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x6b +****************************************************************************/ +static void x86emuOp_imul_byte_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + s8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("IMUL\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)srcval,(s32)imm); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + res = (s16)srcval * (s16)imm; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)srcval,(s32)imm); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + res = (s16)srcval * (s16)imm; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)srcval,(s32)imm); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + res = (s16)srcval * (s16)imm; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + u32 res_lo,res_hi; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)*srcreg,(s32)imm); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg,*srcreg; + u32 res; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%d\n", (s32)imm); + res = (s16)*srcreg * (s16)imm; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x6c +****************************************************************************/ +static void x86emuOp_ins_byte(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("INSB\n"); + ins(1); + TRACE_AND_STEP(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x6d +****************************************************************************/ +static void x86emuOp_ins_word(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INSD\n"); + ins(4); + } else { + DECODE_PRINTF("INSW\n"); + ins(2); + } + TRACE_AND_STEP(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x6e +****************************************************************************/ +static void x86emuOp_outs_byte(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("OUTSB\n"); + outs(1); + TRACE_AND_STEP(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x6f +****************************************************************************/ +static void x86emuOp_outs_word(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("OUTSD\n"); + outs(4); + } else { + DECODE_PRINTF("OUTSW\n"); + outs(2); + } + TRACE_AND_STEP(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x70 +****************************************************************************/ +static void x86emuOp_jump_near_O(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if overflow flag is set */ + START_OF_INSTR(); + DECODE_PRINTF("JO\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_OF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x71 +****************************************************************************/ +static void x86emuOp_jump_near_NO(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if overflow is not set */ + START_OF_INSTR(); + DECODE_PRINTF("JNO\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (!ACCESS_FLAG(F_OF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x72 +****************************************************************************/ +static void x86emuOp_jump_near_B(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if carry flag is set. */ + START_OF_INSTR(); + DECODE_PRINTF("JB\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_CF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x73 +****************************************************************************/ +static void x86emuOp_jump_near_NB(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if carry flag is clear. */ + START_OF_INSTR(); + DECODE_PRINTF("JNB\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (!ACCESS_FLAG(F_CF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x74 +****************************************************************************/ +static void x86emuOp_jump_near_Z(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if zero flag is set. */ + START_OF_INSTR(); + DECODE_PRINTF("JZ\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_ZF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x75 +****************************************************************************/ +static void x86emuOp_jump_near_NZ(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if zero flag is clear. */ + START_OF_INSTR(); + DECODE_PRINTF("JNZ\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (!ACCESS_FLAG(F_ZF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x76 +****************************************************************************/ +static void x86emuOp_jump_near_BE(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if carry flag is set or if the zero + flag is set. */ + START_OF_INSTR(); + DECODE_PRINTF("JBE\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_CF) || ACCESS_FLAG(F_ZF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x77 +****************************************************************************/ +static void x86emuOp_jump_near_NBE(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if carry flag is clear and if the zero + flag is clear */ + START_OF_INSTR(); + DECODE_PRINTF("JNBE\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (!(ACCESS_FLAG(F_CF) || ACCESS_FLAG(F_ZF))) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x78 +****************************************************************************/ +static void x86emuOp_jump_near_S(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if sign flag is set */ + START_OF_INSTR(); + DECODE_PRINTF("JS\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_SF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x79 +****************************************************************************/ +static void x86emuOp_jump_near_NS(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if sign flag is clear */ + START_OF_INSTR(); + DECODE_PRINTF("JNS\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (!ACCESS_FLAG(F_SF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x7a +****************************************************************************/ +static void x86emuOp_jump_near_P(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if parity flag is set (even parity) */ + START_OF_INSTR(); + DECODE_PRINTF("JP\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_PF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x7b +****************************************************************************/ +static void x86emuOp_jump_near_NP(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + + /* jump to byte offset if parity flag is clear (odd parity) */ + START_OF_INSTR(); + DECODE_PRINTF("JNP\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (!ACCESS_FLAG(F_PF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x7c +****************************************************************************/ +static void x86emuOp_jump_near_L(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + int sf, of; + + /* jump to byte offset if sign flag not equal to overflow flag. */ + START_OF_INSTR(); + DECODE_PRINTF("JL\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + sf = ACCESS_FLAG(F_SF) != 0; + of = ACCESS_FLAG(F_OF) != 0; + if (sf ^ of) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x7d +****************************************************************************/ +static void x86emuOp_jump_near_NL(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + int sf, of; + + /* jump to byte offset if sign flag not equal to overflow flag. */ + START_OF_INSTR(); + DECODE_PRINTF("JNL\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + sf = ACCESS_FLAG(F_SF) != 0; + of = ACCESS_FLAG(F_OF) != 0; + /* note: inverse of above, but using == instead of xor. */ + if (sf == of) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x7e +****************************************************************************/ +static void x86emuOp_jump_near_LE(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + int sf, of; + + /* jump to byte offset if sign flag not equal to overflow flag + or the zero flag is set */ + START_OF_INSTR(); + DECODE_PRINTF("JLE\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + sf = ACCESS_FLAG(F_SF) != 0; + of = ACCESS_FLAG(F_OF) != 0; + if ((sf ^ of) || ACCESS_FLAG(F_ZF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x7f +****************************************************************************/ +static void x86emuOp_jump_near_NLE(u8 X86EMU_UNUSED(op1)) +{ + s8 offset; + u16 target; + int sf, of; + + /* jump to byte offset if sign flag equal to overflow flag. + and the zero flag is clear */ + START_OF_INSTR(); + DECODE_PRINTF("JNLE\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + (s16)offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + sf = ACCESS_FLAG(F_SF) != 0; + of = ACCESS_FLAG(F_OF) != 0; + if ((sf == of) && !ACCESS_FLAG(F_ZF)) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +static u8 (*opc80_byte_operation[])(u8 d, u8 s) = +{ + add_byte, /* 00 */ + or_byte, /* 01 */ + adc_byte, /* 02 */ + sbb_byte, /* 03 */ + and_byte, /* 04 */ + sub_byte, /* 05 */ + xor_byte, /* 06 */ + cmp_byte, /* 07 */ +}; + +/**************************************************************************** +REMARKS: +Handles opcode 0x80 +****************************************************************************/ +static void x86emuOp_opc80_byte_RM_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg; + uint destoffset; + u8 imm; + u8 destval; + + /* + * Weirdo special case instruction format. Part of the opcode + * held below in "RH". Doubly nested case would result, except + * that the decoded instruction + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + + switch (rh) { + case 0: + DECODE_PRINTF("ADD\t"); + break; + case 1: + DECODE_PRINTF("OR\t"); + break; + case 2: + DECODE_PRINTF("ADC\t"); + break; + case 3: + DECODE_PRINTF("SBB\t"); + break; + case 4: + DECODE_PRINTF("AND\t"); + break; + case 5: + DECODE_PRINTF("SUB\t"); + break; + case 6: + DECODE_PRINTF("XOR\t"); + break; + case 7: + DECODE_PRINTF("CMP\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + switch (mod) { + case 0: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc80_byte_operation[rh]) (destval, imm); + if (rh != 7) + store_data_byte(destoffset, destval); + break; + case 1: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc80_byte_operation[rh]) (destval, imm); + if (rh != 7) + store_data_byte(destoffset, destval); + break; + case 2: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc80_byte_operation[rh]) (destval, imm); + if (rh != 7) + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc80_byte_operation[rh]) (*destreg, imm); + if (rh != 7) + *destreg = destval; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +static u16 (*opc81_word_operation[])(u16 d, u16 s) = +{ + add_word, /*00 */ + or_word, /*01 */ + adc_word, /*02 */ + sbb_word, /*03 */ + and_word, /*04 */ + sub_word, /*05 */ + xor_word, /*06 */ + cmp_word, /*07 */ +}; + +static u32 (*opc81_long_operation[])(u32 d, u32 s) = +{ + add_long, /*00 */ + or_long, /*01 */ + adc_long, /*02 */ + sbb_long, /*03 */ + and_long, /*04 */ + sub_long, /*05 */ + xor_long, /*06 */ + cmp_long, /*07 */ +}; + +/**************************************************************************** +REMARKS: +Handles opcode 0x81 +****************************************************************************/ +static void x86emuOp_opc81_word_RM_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + /* + * Weirdo special case instruction format. Part of the opcode + * held below in "RH". Doubly nested case would result, except + * that the decoded instruction + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + + switch (rh) { + case 0: + DECODE_PRINTF("ADD\t"); + break; + case 1: + DECODE_PRINTF("OR\t"); + break; + case 2: + DECODE_PRINTF("ADC\t"); + break; + case 3: + DECODE_PRINTF("SBB\t"); + break; + case 4: + DECODE_PRINTF("AND\t"); + break; + case 5: + DECODE_PRINTF("SUB\t"); + break; + case 6: + DECODE_PRINTF("XOR\t"); + break; + case 7: + DECODE_PRINTF("CMP\t"); + break; + } + } +#endif + /* + * Know operation, decode the mod byte to find the addressing + * mode. + */ + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + imm = fetch_long_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc81_long_operation[rh]) (destval, imm); + if (rh != 7) + store_data_long(destoffset, destval); + } else { + u16 destval,imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + imm = fetch_word_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc81_word_operation[rh]) (destval, imm); + if (rh != 7) + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + imm = fetch_long_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc81_long_operation[rh]) (destval, imm); + if (rh != 7) + store_data_long(destoffset, destval); + } else { + u16 destval,imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + imm = fetch_word_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc81_word_operation[rh]) (destval, imm); + if (rh != 7) + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + imm = fetch_long_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc81_long_operation[rh]) (destval, imm); + if (rh != 7) + store_data_long(destoffset, destval); + } else { + u16 destval,imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + imm = fetch_word_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc81_word_operation[rh]) (destval, imm); + if (rh != 7) + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 destval,imm; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + imm = fetch_long_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc81_long_operation[rh]) (*destreg, imm); + if (rh != 7) + *destreg = destval; + } else { + u16 *destreg; + u16 destval,imm; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + imm = fetch_word_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc81_word_operation[rh]) (*destreg, imm); + if (rh != 7) + *destreg = destval; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +static u8 (*opc82_byte_operation[])(u8 s, u8 d) = +{ + add_byte, /*00 */ + or_byte, /*01 *//*YYY UNUSED ???? */ + adc_byte, /*02 */ + sbb_byte, /*03 */ + and_byte, /*04 *//*YYY UNUSED ???? */ + sub_byte, /*05 */ + xor_byte, /*06 *//*YYY UNUSED ???? */ + cmp_byte, /*07 */ +}; + +/**************************************************************************** +REMARKS: +Handles opcode 0x82 +****************************************************************************/ +static void x86emuOp_opc82_byte_RM_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg; + uint destoffset; + u8 imm; + u8 destval; + + /* + * Weirdo special case instruction format. Part of the opcode + * held below in "RH". Doubly nested case would result, except + * that the decoded instruction Similar to opcode 81, except that + * the immediate byte is sign extended to a word length. + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + switch (rh) { + case 0: + DECODE_PRINTF("ADD\t"); + break; + case 1: + DECODE_PRINTF("OR\t"); + break; + case 2: + DECODE_PRINTF("ADC\t"); + break; + case 3: + DECODE_PRINTF("SBB\t"); + break; + case 4: + DECODE_PRINTF("AND\t"); + break; + case 5: + DECODE_PRINTF("SUB\t"); + break; + case 6: + DECODE_PRINTF("XOR\t"); + break; + case 7: + DECODE_PRINTF("CMP\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + switch (mod) { + case 0: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm00_address(rl); + destval = fetch_data_byte(destoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc82_byte_operation[rh]) (destval, imm); + if (rh != 7) + store_data_byte(destoffset, destval); + break; + case 1: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm01_address(rl); + destval = fetch_data_byte(destoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc82_byte_operation[rh]) (destval, imm); + if (rh != 7) + store_data_byte(destoffset, destval); + break; + case 2: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm10_address(rl); + destval = fetch_data_byte(destoffset); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc82_byte_operation[rh]) (destval, imm); + if (rh != 7) + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc82_byte_operation[rh]) (*destreg, imm); + if (rh != 7) + *destreg = destval; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +static u16 (*opc83_word_operation[])(u16 s, u16 d) = +{ + add_word, /*00 */ + or_word, /*01 *//*YYY UNUSED ???? */ + adc_word, /*02 */ + sbb_word, /*03 */ + and_word, /*04 *//*YYY UNUSED ???? */ + sub_word, /*05 */ + xor_word, /*06 *//*YYY UNUSED ???? */ + cmp_word, /*07 */ +}; + +static u32 (*opc83_long_operation[])(u32 s, u32 d) = +{ + add_long, /*00 */ + or_long, /*01 *//*YYY UNUSED ???? */ + adc_long, /*02 */ + sbb_long, /*03 */ + and_long, /*04 *//*YYY UNUSED ???? */ + sub_long, /*05 */ + xor_long, /*06 *//*YYY UNUSED ???? */ + cmp_long, /*07 */ +}; + +/**************************************************************************** +REMARKS: +Handles opcode 0x83 +****************************************************************************/ +static void x86emuOp_opc83_word_RM_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + /* + * Weirdo special case instruction format. Part of the opcode + * held below in "RH". Doubly nested case would result, except + * that the decoded instruction Similar to opcode 81, except that + * the immediate byte is sign extended to a word length. + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + switch (rh) { + case 0: + DECODE_PRINTF("ADD\t"); + break; + case 1: + DECODE_PRINTF("OR\t"); + break; + case 2: + DECODE_PRINTF("ADC\t"); + break; + case 3: + DECODE_PRINTF("SBB\t"); + break; + case 4: + DECODE_PRINTF("AND\t"); + break; + case 5: + DECODE_PRINTF("SUB\t"); + break; + case 6: + DECODE_PRINTF("XOR\t"); + break; + case 7: + DECODE_PRINTF("CMP\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm00_address(rl); + destval = fetch_data_long(destoffset); + imm = (s8) fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc83_long_operation[rh]) (destval, imm); + if (rh != 7) + store_data_long(destoffset, destval); + } else { + u16 destval,imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm00_address(rl); + destval = fetch_data_word(destoffset); + imm = (s8) fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc83_word_operation[rh]) (destval, imm); + if (rh != 7) + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm01_address(rl); + destval = fetch_data_long(destoffset); + imm = (s8) fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc83_long_operation[rh]) (destval, imm); + if (rh != 7) + store_data_long(destoffset, destval); + } else { + u16 destval,imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm01_address(rl); + destval = fetch_data_word(destoffset); + imm = (s8) fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc83_word_operation[rh]) (destval, imm); + if (rh != 7) + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm10_address(rl); + destval = fetch_data_long(destoffset); + imm = (s8) fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc83_long_operation[rh]) (destval, imm); + if (rh != 7) + store_data_long(destoffset, destval); + } else { + u16 destval,imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm10_address(rl); + destval = fetch_data_word(destoffset); + imm = (s8) fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc83_word_operation[rh]) (destval, imm); + if (rh != 7) + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 destval,imm; + + destreg = DECODE_RM_LONG_REGISTER(rl); + imm = (s8) fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc83_long_operation[rh]) (*destreg, imm); + if (rh != 7) + *destreg = destval; + } else { + u16 *destreg; + u16 destval,imm; + + destreg = DECODE_RM_WORD_REGISTER(rl); + imm = (s8) fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + destval = (*opc83_word_operation[rh]) (*destreg, imm); + if (rh != 7) + *destreg = destval; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x84 +****************************************************************************/ +static void x86emuOp_test_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + u8 destval; + + START_OF_INSTR(); + DECODE_PRINTF("TEST\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_byte(destval, *srcreg); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_byte(destval, *srcreg); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_byte(destval, *srcreg); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_byte(*destreg, *srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x85 +****************************************************************************/ +static void x86emuOp_test_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("TEST\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_long(destval, *srcreg); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_word(destval, *srcreg); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_long(destval, *srcreg); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_word(destval, *srcreg); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_long(destval, *srcreg); + } else { + u16 destval; + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_word(destval, *srcreg); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_long(*destreg, *srcreg); + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + test_word(*destreg, *srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x86 +****************************************************************************/ +static void x86emuOp_xchg_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + u8 destval; + u8 tmp; + + START_OF_INSTR(); + DECODE_PRINTF("XCHG\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_byte(destoffset, destval); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_byte(destoffset, destval); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_byte(destoffset); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = *destreg; + *destreg = tmp; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x87 +****************************************************************************/ +static void x86emuOp_xchg_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("XCHG\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg; + u32 destval,tmp; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_long(destoffset, destval); + } else { + u16 *srcreg; + u16 destval,tmp; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg; + u32 destval,tmp; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_long(destoffset, destval); + } else { + u16 *srcreg; + u16 destval,tmp; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg; + u32 destval,tmp; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_long(destoffset); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_long(destoffset, destval); + } else { + u16 *srcreg; + u16 destval,tmp; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + destval = fetch_data_word(destoffset); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = destval; + destval = tmp; + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + u32 tmp; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = *destreg; + *destreg = tmp; + } else { + u16 *destreg,*srcreg; + u16 tmp; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + tmp = *srcreg; + *srcreg = *destreg; + *destreg = tmp; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x88 +****************************************************************************/ +static void x86emuOp_mov_byte_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_byte(destoffset, *srcreg); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_byte(destoffset, *srcreg); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_byte(destoffset, *srcreg); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x89 +****************************************************************************/ +static void x86emuOp_mov_word_RM_R(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u32 destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_long(destoffset, *srcreg); + } else { + u16 *srcreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_word(destoffset, *srcreg); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_long(destoffset, *srcreg); + } else { + u16 *srcreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_word(destoffset, *srcreg); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_long(destoffset, *srcreg); + } else { + u16 *srcreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_word(destoffset, *srcreg); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + } else { + u16 *destreg,*srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x8a +****************************************************************************/ +static void x86emuOp_mov_byte_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg, *srcreg; + uint srcoffset; + u8 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 1: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 2: + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x8b +****************************************************************************/ +static void x86emuOp_mov_word_R_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg, *srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + } else { + u16 *destreg, *srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x8c +****************************************************************************/ +static void x86emuOp_mov_word_RM_SR(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u16 *destreg, *srcreg; + uint destoffset; + u16 destval; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + srcreg = decode_rm_seg_register(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = *srcreg; + store_data_word(destoffset, destval); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + srcreg = decode_rm_seg_register(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = *srcreg; + store_data_word(destoffset, destval); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + srcreg = decode_rm_seg_register(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = *srcreg; + store_data_word(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcreg = decode_rm_seg_register(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x8d +****************************************************************************/ +static void x86emuOp_lea_word_R_M(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("LEA\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_ADDR) { + u32 *srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *srcreg = (u32)destoffset; + } else { + u16 *srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *srcreg = (u16)destoffset; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_ADDR) { + u32 *srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *srcreg = (u32)destoffset; + } else { + u16 *srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *srcreg = (u16)destoffset; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_ADDR) { + u32 *srcreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *srcreg = (u32)destoffset; + } else { + u16 *srcreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *srcreg = (u16)destoffset; + } + break; + case 3: /* register to register */ + /* undefined. Do nothing. */ + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x8e +****************************************************************************/ +static void x86emuOp_mov_word_SR_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u16 *destreg, *srcreg; + uint srcoffset; + u16 srcval; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = decode_rm_seg_register(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 1: + destreg = decode_rm_seg_register(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 2: + destreg = decode_rm_seg_register(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 3: /* register to register */ + destreg = decode_rm_seg_register(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + break; + } + /* + * Clean up, and reset all the R_xSP pointers to the correct + * locations. This is about 3x too much overhead (doing all the + * segreg ptrs when only one is needed, but this instruction + * *cannot* be that common, and this isn't too much work anyway. + */ + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x8f +****************************************************************************/ +static void x86emuOp_pop_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("POP\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + if (rh != 0) { + DECODE_PRINTF("ILLEGAL DECODE OF OPCODE 8F\n"); + HALT_SYS(); + } + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = pop_long(); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = pop_word(); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = pop_long(); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = pop_word(); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = pop_long(); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + destval = pop_word(); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = pop_long(); + } else { + u16 *destreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = pop_word(); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x90 +****************************************************************************/ +static void x86emuOp_nop(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("NOP\n"); + TRACE_AND_STEP(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x91 +****************************************************************************/ +static void x86emuOp_xchg_word_AX_CX(u8 X86EMU_UNUSED(op1)) +{ + u32 tmp; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("XCHG\tEAX,ECX\n"); + } else { + DECODE_PRINTF("XCHG\tAX,CX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + tmp = M.x86.R_EAX; + M.x86.R_EAX = M.x86.R_ECX; + M.x86.R_ECX = tmp; + } else { + tmp = M.x86.R_AX; + M.x86.R_AX = M.x86.R_CX; + M.x86.R_CX = (u16)tmp; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x92 +****************************************************************************/ +static void x86emuOp_xchg_word_AX_DX(u8 X86EMU_UNUSED(op1)) +{ + u32 tmp; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("XCHG\tEAX,EDX\n"); + } else { + DECODE_PRINTF("XCHG\tAX,DX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + tmp = M.x86.R_EAX; + M.x86.R_EAX = M.x86.R_EDX; + M.x86.R_EDX = tmp; + } else { + tmp = M.x86.R_AX; + M.x86.R_AX = M.x86.R_DX; + M.x86.R_DX = (u16)tmp; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x93 +****************************************************************************/ +static void x86emuOp_xchg_word_AX_BX(u8 X86EMU_UNUSED(op1)) +{ + u32 tmp; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("XCHG\tEAX,EBX\n"); + } else { + DECODE_PRINTF("XCHG\tAX,BX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + tmp = M.x86.R_EAX; + M.x86.R_EAX = M.x86.R_EBX; + M.x86.R_EBX = tmp; + } else { + tmp = M.x86.R_AX; + M.x86.R_AX = M.x86.R_BX; + M.x86.R_BX = (u16)tmp; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x94 +****************************************************************************/ +static void x86emuOp_xchg_word_AX_SP(u8 X86EMU_UNUSED(op1)) +{ + u32 tmp; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("XCHG\tEAX,ESP\n"); + } else { + DECODE_PRINTF("XCHG\tAX,SP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + tmp = M.x86.R_EAX; + M.x86.R_EAX = M.x86.R_ESP; + M.x86.R_ESP = tmp; + } else { + tmp = M.x86.R_AX; + M.x86.R_AX = M.x86.R_SP; + M.x86.R_SP = (u16)tmp; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x95 +****************************************************************************/ +static void x86emuOp_xchg_word_AX_BP(u8 X86EMU_UNUSED(op1)) +{ + u32 tmp; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("XCHG\tEAX,EBP\n"); + } else { + DECODE_PRINTF("XCHG\tAX,BP\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + tmp = M.x86.R_EAX; + M.x86.R_EAX = M.x86.R_EBP; + M.x86.R_EBP = tmp; + } else { + tmp = M.x86.R_AX; + M.x86.R_AX = M.x86.R_BP; + M.x86.R_BP = (u16)tmp; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x96 +****************************************************************************/ +static void x86emuOp_xchg_word_AX_SI(u8 X86EMU_UNUSED(op1)) +{ + u32 tmp; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("XCHG\tEAX,ESI\n"); + } else { + DECODE_PRINTF("XCHG\tAX,SI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + tmp = M.x86.R_EAX; + M.x86.R_EAX = M.x86.R_ESI; + M.x86.R_ESI = tmp; + } else { + tmp = M.x86.R_AX; + M.x86.R_AX = M.x86.R_SI; + M.x86.R_SI = (u16)tmp; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x97 +****************************************************************************/ +static void x86emuOp_xchg_word_AX_DI(u8 X86EMU_UNUSED(op1)) +{ + u32 tmp; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("XCHG\tEAX,EDI\n"); + } else { + DECODE_PRINTF("XCHG\tAX,DI\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + tmp = M.x86.R_EAX; + M.x86.R_EAX = M.x86.R_EDI; + M.x86.R_EDI = tmp; + } else { + tmp = M.x86.R_AX; + M.x86.R_AX = M.x86.R_DI; + M.x86.R_DI = (u16)tmp; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x98 +****************************************************************************/ +static void x86emuOp_cbw(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("CWDE\n"); + } else { + DECODE_PRINTF("CBW\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + if (M.x86.R_AX & 0x8000) { + M.x86.R_EAX |= 0xffff0000; + } else { + M.x86.R_EAX &= 0x0000ffff; + } + } else { + if (M.x86.R_AL & 0x80) { + M.x86.R_AH = 0xff; + } else { + M.x86.R_AH = 0x0; + } + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x99 +****************************************************************************/ +static void x86emuOp_cwd(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("CDQ\n"); + } else { + DECODE_PRINTF("CWD\n"); + } + DECODE_PRINTF("CWD\n"); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + if (M.x86.R_EAX & 0x80000000) { + M.x86.R_EDX = 0xffffffff; + } else { + M.x86.R_EDX = 0x0; + } + } else { + if (M.x86.R_AX & 0x8000) { + M.x86.R_DX = 0xffff; + } else { + M.x86.R_DX = 0x0; + } + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x9a +****************************************************************************/ +static void x86emuOp_call_far_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 farseg, faroff; + + START_OF_INSTR(); + DECODE_PRINTF("CALL\t"); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + faroff = fetch_long_imm(); + farseg = fetch_word_imm(); + } else { + faroff = fetch_word_imm(); + farseg = fetch_word_imm(); + } + DECODE_PRINTF2("%04x:", farseg); + DECODE_PRINTF2("%04x\n", faroff); + CALL_TRACE(M.x86.saved_cs, M.x86.saved_ip, farseg, faroff, "FAR "); + + /* XXX + * + * Hooked interrupt vectors calling into our "BIOS" will cause + * problems unless all intersegment stuff is checked for BIOS + * access. Check needed here. For moment, let it alone. + */ + TRACE_AND_STEP(); + push_word(M.x86.R_CS); + M.x86.R_CS = farseg; + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_EIP); + } else { + push_word(M.x86.R_IP); + } + M.x86.R_EIP = faroff & 0xffff; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x9b +****************************************************************************/ +static void x86emuOp_wait(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("WAIT"); + TRACE_AND_STEP(); + /* NADA. */ + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x9c +****************************************************************************/ +static void x86emuOp_pushf_word(u8 X86EMU_UNUSED(op1)) +{ + u32 flags; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("PUSHFD\n"); + } else { + DECODE_PRINTF("PUSHF\n"); + } + TRACE_AND_STEP(); + + /* clear out *all* bits not representing flags, and turn on real bits */ + flags = (M.x86.R_EFLG & F_MSK) | F_ALWAYS_ON; + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(flags); + } else { + push_word((u16)flags); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x9d +****************************************************************************/ +static void x86emuOp_popf_word(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("POPFD\n"); + } else { + DECODE_PRINTF("POPF\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EFLG = pop_long(); + } else { + M.x86.R_FLG = pop_word(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x9e +****************************************************************************/ +static void x86emuOp_sahf(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("SAHF\n"); + TRACE_AND_STEP(); + /* clear the lower bits of the flag register */ + M.x86.R_FLG &= 0xffffff00; + /* or in the AH register into the flags register */ + M.x86.R_FLG |= M.x86.R_AH; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x9f +****************************************************************************/ +static void x86emuOp_lahf(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("LAHF\n"); + TRACE_AND_STEP(); + M.x86.R_AH = (u8)(M.x86.R_FLG & 0xff); + /*undocumented TC++ behavior??? Nope. It's documented, but + you have too look real hard to notice it. */ + M.x86.R_AH |= 0x2; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa0 +****************************************************************************/ +static void x86emuOp_mov_AL_M_IMM(u8 X86EMU_UNUSED(op1)) +{ + u16 offset; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tAL,"); + offset = fetch_word_imm(); + DECODE_PRINTF2("[%04x]\n", offset); + TRACE_AND_STEP(); + M.x86.R_AL = fetch_data_byte(offset); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa1 +****************************************************************************/ +static void x86emuOp_mov_AX_M_IMM(u8 X86EMU_UNUSED(op1)) +{ + u16 offset; + + START_OF_INSTR(); + offset = fetch_word_imm(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF2("MOV\tEAX,[%04x]\n", offset); + } else { + DECODE_PRINTF2("MOV\tAX,[%04x]\n", offset); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = fetch_data_long(offset); + } else { + M.x86.R_AX = fetch_data_word(offset); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa2 +****************************************************************************/ +static void x86emuOp_mov_M_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u16 offset; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + offset = fetch_word_imm(); + DECODE_PRINTF2("[%04x],AL\n", offset); + TRACE_AND_STEP(); + store_data_byte(offset, M.x86.R_AL); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa3 +****************************************************************************/ +static void x86emuOp_mov_M_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u16 offset; + + START_OF_INSTR(); + offset = fetch_word_imm(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF2("MOV\t[%04x],EAX\n", offset); + } else { + DECODE_PRINTF2("MOV\t[%04x],AX\n", offset); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + store_data_long(offset, M.x86.R_EAX); + } else { + store_data_word(offset, M.x86.R_AX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa4 +****************************************************************************/ +static void x86emuOp_movs_byte(u8 X86EMU_UNUSED(op1)) +{ + u8 val; + u32 count; + int inc; + + START_OF_INSTR(); + DECODE_PRINTF("MOVS\tBYTE\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -1; + else + inc = 1; + TRACE_AND_STEP(); + count = 1; + if (M.x86.mode & (SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE)) { + /* dont care whether REPE or REPNE */ + /* move them until CX is ZERO. */ + count = M.x86.R_CX; + M.x86.R_CX = 0; + M.x86.mode &= ~(SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE); + } + while (count--) { + val = fetch_data_byte(M.x86.R_SI); + store_data_byte_abs(M.x86.R_ES, M.x86.R_DI, val); + M.x86.R_SI += inc; + M.x86.R_DI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa5 +****************************************************************************/ +static void x86emuOp_movs_word(u8 X86EMU_UNUSED(op1)) +{ + u32 val; + int inc; + u32 count; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOVS\tDWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -4; + else + inc = 4; + } else { + DECODE_PRINTF("MOVS\tWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -2; + else + inc = 2; + } + TRACE_AND_STEP(); + count = 1; + if (M.x86.mode & (SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE)) { + /* dont care whether REPE or REPNE */ + /* move them until CX is ZERO. */ + count = M.x86.R_CX; + M.x86.R_CX = 0; + M.x86.mode &= ~(SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE); + } + while (count--) { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + val = fetch_data_long(M.x86.R_SI); + store_data_long_abs(M.x86.R_ES, M.x86.R_DI, val); + } else { + val = fetch_data_word(M.x86.R_SI); + store_data_word_abs(M.x86.R_ES, M.x86.R_DI, (u16)val); + } + M.x86.R_SI += inc; + M.x86.R_DI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa6 +****************************************************************************/ +static void x86emuOp_cmps_byte(u8 X86EMU_UNUSED(op1)) +{ + s8 val1, val2; + int inc; + + START_OF_INSTR(); + DECODE_PRINTF("CMPS\tBYTE\n"); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -1; + else + inc = 1; + + if (M.x86.mode & SYSMODE_PREFIX_REPE) { + /* REPE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + val1 = fetch_data_byte(M.x86.R_SI); + val2 = fetch_data_byte_abs(M.x86.R_ES, M.x86.R_DI); + cmp_byte(val1, val2); + M.x86.R_CX -= 1; + M.x86.R_SI += inc; + M.x86.R_DI += inc; + if (ACCESS_FLAG(F_ZF) == 0) + break; + } + M.x86.mode &= ~SYSMODE_PREFIX_REPE; + } else if (M.x86.mode & SYSMODE_PREFIX_REPNE) { + /* REPNE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + val1 = fetch_data_byte(M.x86.R_SI); + val2 = fetch_data_byte_abs(M.x86.R_ES, M.x86.R_DI); + cmp_byte(val1, val2); + M.x86.R_CX -= 1; + M.x86.R_SI += inc; + M.x86.R_DI += inc; + if (ACCESS_FLAG(F_ZF)) + break; /* zero flag set means equal */ + } + M.x86.mode &= ~SYSMODE_PREFIX_REPNE; + } else { + val1 = fetch_data_byte(M.x86.R_SI); + val2 = fetch_data_byte_abs(M.x86.R_ES, M.x86.R_DI); + cmp_byte(val1, val2); + M.x86.R_SI += inc; + M.x86.R_DI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa7 +****************************************************************************/ +static void x86emuOp_cmps_word(u8 X86EMU_UNUSED(op1)) +{ + u32 val1,val2; + int inc; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("CMPS\tDWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -4; + else + inc = 4; + } else { + DECODE_PRINTF("CMPS\tWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -2; + else + inc = 2; + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_REPE) { + /* REPE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + val1 = fetch_data_long(M.x86.R_SI); + val2 = fetch_data_long_abs(M.x86.R_ES, M.x86.R_DI); + cmp_long(val1, val2); + } else { + val1 = fetch_data_word(M.x86.R_SI); + val2 = fetch_data_word_abs(M.x86.R_ES, M.x86.R_DI); + cmp_word((u16)val1, (u16)val2); + } + M.x86.R_CX -= 1; + M.x86.R_SI += inc; + M.x86.R_DI += inc; + if (ACCESS_FLAG(F_ZF) == 0) + break; + } + M.x86.mode &= ~SYSMODE_PREFIX_REPE; + } else if (M.x86.mode & SYSMODE_PREFIX_REPNE) { + /* REPNE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + val1 = fetch_data_long(M.x86.R_SI); + val2 = fetch_data_long_abs(M.x86.R_ES, M.x86.R_DI); + cmp_long(val1, val2); + } else { + val1 = fetch_data_word(M.x86.R_SI); + val2 = fetch_data_word_abs(M.x86.R_ES, M.x86.R_DI); + cmp_word((u16)val1, (u16)val2); + } + M.x86.R_CX -= 1; + M.x86.R_SI += inc; + M.x86.R_DI += inc; + if (ACCESS_FLAG(F_ZF)) + break; /* zero flag set means equal */ + } + M.x86.mode &= ~SYSMODE_PREFIX_REPNE; + } else { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + val1 = fetch_data_long(M.x86.R_SI); + val2 = fetch_data_long_abs(M.x86.R_ES, M.x86.R_DI); + cmp_long(val1, val2); + } else { + val1 = fetch_data_word(M.x86.R_SI); + val2 = fetch_data_word_abs(M.x86.R_ES, M.x86.R_DI); + cmp_word((u16)val1, (u16)val2); + } + M.x86.R_SI += inc; + M.x86.R_DI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa8 +****************************************************************************/ +static void x86emuOp_test_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + int imm; + + START_OF_INSTR(); + DECODE_PRINTF("TEST\tAL,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%04x\n", imm); + TRACE_AND_STEP(); + test_byte(M.x86.R_AL, (u8)imm); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xa9 +****************************************************************************/ +static void x86emuOp_test_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("TEST\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("TEST\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + test_long(M.x86.R_EAX, srcval); + } else { + test_word(M.x86.R_AX, (u16)srcval); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xaa +****************************************************************************/ +static void x86emuOp_stos_byte(u8 X86EMU_UNUSED(op1)) +{ + int inc; + + START_OF_INSTR(); + DECODE_PRINTF("STOS\tBYTE\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -1; + else + inc = 1; + TRACE_AND_STEP(); + if (M.x86.mode & (SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE)) { + /* dont care whether REPE or REPNE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + store_data_byte_abs(M.x86.R_ES, M.x86.R_DI, M.x86.R_AL); + M.x86.R_CX -= 1; + M.x86.R_DI += inc; + } + M.x86.mode &= ~(SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE); + } else { + store_data_byte_abs(M.x86.R_ES, M.x86.R_DI, M.x86.R_AL); + M.x86.R_DI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xab +****************************************************************************/ +static void x86emuOp_stos_word(u8 X86EMU_UNUSED(op1)) +{ + int inc; + u32 count; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("STOS\tDWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -4; + else + inc = 4; + } else { + DECODE_PRINTF("STOS\tWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -2; + else + inc = 2; + } + TRACE_AND_STEP(); + count = 1; + if (M.x86.mode & (SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE)) { + /* dont care whether REPE or REPNE */ + /* move them until CX is ZERO. */ + count = M.x86.R_CX; + M.x86.R_CX = 0; + M.x86.mode &= ~(SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE); + } + while (count--) { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + store_data_long_abs(M.x86.R_ES, M.x86.R_DI, M.x86.R_EAX); + } else { + store_data_word_abs(M.x86.R_ES, M.x86.R_DI, M.x86.R_AX); + } + M.x86.R_DI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xac +****************************************************************************/ +static void x86emuOp_lods_byte(u8 X86EMU_UNUSED(op1)) +{ + int inc; + + START_OF_INSTR(); + DECODE_PRINTF("LODS\tBYTE\n"); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -1; + else + inc = 1; + if (M.x86.mode & (SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE)) { + /* dont care whether REPE or REPNE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + M.x86.R_AL = fetch_data_byte(M.x86.R_SI); + M.x86.R_CX -= 1; + M.x86.R_SI += inc; + } + M.x86.mode &= ~(SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE); + } else { + M.x86.R_AL = fetch_data_byte(M.x86.R_SI); + M.x86.R_SI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xad +****************************************************************************/ +static void x86emuOp_lods_word(u8 X86EMU_UNUSED(op1)) +{ + int inc; + u32 count; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("LODS\tDWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -4; + else + inc = 4; + } else { + DECODE_PRINTF("LODS\tWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -2; + else + inc = 2; + } + TRACE_AND_STEP(); + count = 1; + if (M.x86.mode & (SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE)) { + /* dont care whether REPE or REPNE */ + /* move them until CX is ZERO. */ + count = M.x86.R_CX; + M.x86.R_CX = 0; + M.x86.mode &= ~(SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE); + } + while (count--) { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = fetch_data_long(M.x86.R_SI); + } else { + M.x86.R_AX = fetch_data_word(M.x86.R_SI); + } + M.x86.R_SI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xae +****************************************************************************/ +static void x86emuOp_scas_byte(u8 X86EMU_UNUSED(op1)) +{ + s8 val2; + int inc; + + START_OF_INSTR(); + DECODE_PRINTF("SCAS\tBYTE\n"); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -1; + else + inc = 1; + if (M.x86.mode & SYSMODE_PREFIX_REPE) { + /* REPE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + val2 = fetch_data_byte_abs(M.x86.R_ES, M.x86.R_DI); + cmp_byte(M.x86.R_AL, val2); + M.x86.R_CX -= 1; + M.x86.R_DI += inc; + if (ACCESS_FLAG(F_ZF) == 0) + break; + } + M.x86.mode &= ~SYSMODE_PREFIX_REPE; + } else if (M.x86.mode & SYSMODE_PREFIX_REPNE) { + /* REPNE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + val2 = fetch_data_byte_abs(M.x86.R_ES, M.x86.R_DI); + cmp_byte(M.x86.R_AL, val2); + M.x86.R_CX -= 1; + M.x86.R_DI += inc; + if (ACCESS_FLAG(F_ZF)) + break; /* zero flag set means equal */ + } + M.x86.mode &= ~SYSMODE_PREFIX_REPNE; + } else { + val2 = fetch_data_byte_abs(M.x86.R_ES, M.x86.R_DI); + cmp_byte(M.x86.R_AL, val2); + M.x86.R_DI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xaf +****************************************************************************/ +static void x86emuOp_scas_word(u8 X86EMU_UNUSED(op1)) +{ + int inc; + u32 val; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("SCAS\tDWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -4; + else + inc = 4; + } else { + DECODE_PRINTF("SCAS\tWORD\n"); + if (ACCESS_FLAG(F_DF)) /* down */ + inc = -2; + else + inc = 2; + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_REPE) { + /* REPE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + val = fetch_data_long_abs(M.x86.R_ES, M.x86.R_DI); + cmp_long(M.x86.R_EAX, val); + } else { + val = fetch_data_word_abs(M.x86.R_ES, M.x86.R_DI); + cmp_word(M.x86.R_AX, (u16)val); + } + M.x86.R_CX -= 1; + M.x86.R_DI += inc; + if (ACCESS_FLAG(F_ZF) == 0) + break; + } + M.x86.mode &= ~SYSMODE_PREFIX_REPE; + } else if (M.x86.mode & SYSMODE_PREFIX_REPNE) { + /* REPNE */ + /* move them until CX is ZERO. */ + while (M.x86.R_CX != 0) { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + val = fetch_data_long_abs(M.x86.R_ES, M.x86.R_DI); + cmp_long(M.x86.R_EAX, val); + } else { + val = fetch_data_word_abs(M.x86.R_ES, M.x86.R_DI); + cmp_word(M.x86.R_AX, (u16)val); + } + M.x86.R_CX -= 1; + M.x86.R_DI += inc; + if (ACCESS_FLAG(F_ZF)) + break; /* zero flag set means equal */ + } + M.x86.mode &= ~SYSMODE_PREFIX_REPNE; + } else { + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + val = fetch_data_long_abs(M.x86.R_ES, M.x86.R_DI); + cmp_long(M.x86.R_EAX, val); + } else { + val = fetch_data_word_abs(M.x86.R_ES, M.x86.R_DI); + cmp_word(M.x86.R_AX, (u16)val); + } + M.x86.R_DI += inc; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb0 +****************************************************************************/ +static void x86emuOp_mov_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tAL,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + M.x86.R_AL = imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb1 +****************************************************************************/ +static void x86emuOp_mov_byte_CL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tCL,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + M.x86.R_CL = imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb2 +****************************************************************************/ +static void x86emuOp_mov_byte_DL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tDL,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + M.x86.R_DL = imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb3 +****************************************************************************/ +static void x86emuOp_mov_byte_BL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tBL,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + M.x86.R_BL = imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb4 +****************************************************************************/ +static void x86emuOp_mov_byte_AH_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tAH,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + M.x86.R_AH = imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb5 +****************************************************************************/ +static void x86emuOp_mov_byte_CH_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tCH,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + M.x86.R_CH = imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb6 +****************************************************************************/ +static void x86emuOp_mov_byte_DH_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tDH,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + M.x86.R_DH = imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb7 +****************************************************************************/ +static void x86emuOp_mov_byte_BH_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\tBH,"); + imm = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", imm); + TRACE_AND_STEP(); + M.x86.R_BH = imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb8 +****************************************************************************/ +static void x86emuOp_mov_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOV\tEAX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("MOV\tAX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = srcval; + } else { + M.x86.R_AX = (u16)srcval; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xb9 +****************************************************************************/ +static void x86emuOp_mov_word_CX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOV\tECX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("MOV\tCX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ECX = srcval; + } else { + M.x86.R_CX = (u16)srcval; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xba +****************************************************************************/ +static void x86emuOp_mov_word_DX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOV\tEDX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("MOV\tDX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDX = srcval; + } else { + M.x86.R_DX = (u16)srcval; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xbb +****************************************************************************/ +static void x86emuOp_mov_word_BX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOV\tEBX,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("MOV\tBX,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EBX = srcval; + } else { + M.x86.R_BX = (u16)srcval; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xbc +****************************************************************************/ +static void x86emuOp_mov_word_SP_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOV\tESP,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("MOV\tSP,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ESP = srcval; + } else { + M.x86.R_SP = (u16)srcval; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xbd +****************************************************************************/ +static void x86emuOp_mov_word_BP_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOV\tEBP,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("MOV\tBP,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EBP = srcval; + } else { + M.x86.R_BP = (u16)srcval; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xbe +****************************************************************************/ +static void x86emuOp_mov_word_SI_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOV\tESI,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("MOV\tSI,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ESI = srcval; + } else { + M.x86.R_SI = (u16)srcval; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xbf +****************************************************************************/ +static void x86emuOp_mov_word_DI_IMM(u8 X86EMU_UNUSED(op1)) +{ + u32 srcval; + + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("MOV\tEDI,"); + srcval = fetch_long_imm(); + } else { + DECODE_PRINTF("MOV\tDI,"); + srcval = fetch_word_imm(); + } + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EDI = srcval; + } else { + M.x86.R_DI = (u16)srcval; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/* used by opcodes c0, d0, and d2. */ +static u8(*opcD0_byte_operation[])(u8 d, u8 s) = +{ + rol_byte, + ror_byte, + rcl_byte, + rcr_byte, + shl_byte, + shr_byte, + shl_byte, /* sal_byte === shl_byte by definition */ + sar_byte, +}; + +/**************************************************************************** +REMARKS: +Handles opcode 0xc0 +****************************************************************************/ +static void x86emuOp_opcC0_byte_RM_MEM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg; + uint destoffset; + u8 destval; + u8 amt; + + /* + * Yet another weirdo special case instruction format. Part of + * the opcode held below in "RH". Doubly nested case would + * result, except that the decoded instruction + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + + switch (rh) { + case 0: + DECODE_PRINTF("ROL\t"); + break; + case 1: + DECODE_PRINTF("ROR\t"); + break; + case 2: + DECODE_PRINTF("RCL\t"); + break; + case 3: + DECODE_PRINTF("RCR\t"); + break; + case 4: + DECODE_PRINTF("SHL\t"); + break; + case 5: + DECODE_PRINTF("SHR\t"); + break; + case 6: + DECODE_PRINTF("SAL\t"); + break; + case 7: + DECODE_PRINTF("SAR\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + switch (mod) { + case 0: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm00_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, amt); + store_data_byte(destoffset, destval); + break; + case 1: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm01_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, amt); + store_data_byte(destoffset, destval); + break; + case 2: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm10_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, amt); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (*destreg, amt); + *destreg = destval; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/* used by opcodes c1, d1, and d3. */ +static u16(*opcD1_word_operation[])(u16 s, u8 d) = +{ + rol_word, + ror_word, + rcl_word, + rcr_word, + shl_word, + shr_word, + shl_word, /* sal_byte === shl_byte by definition */ + sar_word, +}; + +/* used by opcodes c1, d1, and d3. */ +static u32 (*opcD1_long_operation[])(u32 s, u8 d) = +{ + rol_long, + ror_long, + rcl_long, + rcr_long, + shl_long, + shr_long, + shl_long, /* sal_byte === shl_byte by definition */ + sar_long, +}; + +/**************************************************************************** +REMARKS: +Handles opcode 0xc1 +****************************************************************************/ +static void x86emuOp_opcC1_word_RM_MEM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + u8 amt; + + /* + * Yet another weirdo special case instruction format. Part of + * the opcode held below in "RH". Doubly nested case would + * result, except that the decoded instruction + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + + switch (rh) { + case 0: + DECODE_PRINTF("ROL\t"); + break; + case 1: + DECODE_PRINTF("ROR\t"); + break; + case 2: + DECODE_PRINTF("RCL\t"); + break; + case 3: + DECODE_PRINTF("RCR\t"); + break; + case 4: + DECODE_PRINTF("SHL\t"); + break; + case 5: + DECODE_PRINTF("SHR\t"); + break; + case 6: + DECODE_PRINTF("SAL\t"); + break; + case 7: + DECODE_PRINTF("SAR\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm00_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, amt); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm00_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, amt); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm01_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, amt); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm01_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, amt); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm10_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, amt); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm10_address(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, amt); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + TRACE_AND_STEP(); + *destreg = (*opcD1_long_operation[rh]) (*destreg, amt); + } else { + u16 *destreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + amt = fetch_byte_imm(); + DECODE_PRINTF2(",%x\n", amt); + TRACE_AND_STEP(); + *destreg = (*opcD1_word_operation[rh]) (*destreg, amt); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xc2 +****************************************************************************/ +static void x86emuOp_ret_near_IMM(u8 X86EMU_UNUSED(op1)) +{ + u16 imm; + + START_OF_INSTR(); + DECODE_PRINTF("RET\t"); + imm = fetch_word_imm(); + DECODE_PRINTF2("%x\n", imm); + RETURN_TRACE("RET",M.x86.saved_cs,M.x86.saved_ip); + TRACE_AND_STEP(); + M.x86.R_IP = pop_word(); + M.x86.R_SP += imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xc3 +****************************************************************************/ +static void x86emuOp_ret_near(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("RET\n"); + RETURN_TRACE("RET",M.x86.saved_cs,M.x86.saved_ip); + TRACE_AND_STEP(); + M.x86.R_IP = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xc4 +****************************************************************************/ +static void x86emuOp_les_R_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rh, rl; + u16 *dstreg; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("LES\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_ES = fetch_data_word(srcoffset + 2); + break; + case 1: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_ES = fetch_data_word(srcoffset + 2); + break; + case 2: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_ES = fetch_data_word(srcoffset + 2); + break; + case 3: /* register to register */ + /* UNDEFINED! */ + TRACE_AND_STEP(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xc5 +****************************************************************************/ +static void x86emuOp_lds_R_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rh, rl; + u16 *dstreg; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("LDS\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_DS = fetch_data_word(srcoffset + 2); + break; + case 1: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_DS = fetch_data_word(srcoffset + 2); + break; + case 2: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_DS = fetch_data_word(srcoffset + 2); + break; + case 3: /* register to register */ + /* UNDEFINED! */ + TRACE_AND_STEP(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xc6 +****************************************************************************/ +static void x86emuOp_mov_byte_RM_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg; + uint destoffset; + u8 imm; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + if (rh != 0) { + DECODE_PRINTF("ILLEGAL DECODE OF OPCODE c6\n"); + HALT_SYS(); + } + switch (mod) { + case 0: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm00_address(rl); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%2x\n", imm); + TRACE_AND_STEP(); + store_data_byte(destoffset, imm); + break; + case 1: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm01_address(rl); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%2x\n", imm); + TRACE_AND_STEP(); + store_data_byte(destoffset, imm); + break; + case 2: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm10_address(rl); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%2x\n", imm); + TRACE_AND_STEP(); + store_data_byte(destoffset, imm); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + imm = fetch_byte_imm(); + DECODE_PRINTF2(",%2x\n", imm); + TRACE_AND_STEP(); + *destreg = imm; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xc7 +****************************************************************************/ +static void x86emuOp_mov_word_RM_IMM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("MOV\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + if (rh != 0) { + DECODE_PRINTF("ILLEGAL DECODE OF OPCODE 8F\n"); + HALT_SYS(); + } + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm00_address(rl); + imm = fetch_long_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + store_data_long(destoffset, imm); + } else { + u16 imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm00_address(rl); + imm = fetch_word_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + store_data_word(destoffset, imm); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm01_address(rl); + imm = fetch_long_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + store_data_long(destoffset, imm); + } else { + u16 imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm01_address(rl); + imm = fetch_word_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + store_data_word(destoffset, imm); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 imm; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm10_address(rl); + imm = fetch_long_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + store_data_long(destoffset, imm); + } else { + u16 imm; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm10_address(rl); + imm = fetch_word_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + store_data_word(destoffset, imm); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 imm; + + destreg = DECODE_RM_LONG_REGISTER(rl); + imm = fetch_long_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + *destreg = imm; + } else { + u16 *destreg; + u16 imm; + + destreg = DECODE_RM_WORD_REGISTER(rl); + imm = fetch_word_imm(); + DECODE_PRINTF2(",%x\n", imm); + TRACE_AND_STEP(); + *destreg = imm; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xc8 +****************************************************************************/ +static void x86emuOp_enter(u8 X86EMU_UNUSED(op1)) +{ + u16 local,frame_pointer; + u8 nesting; + int i; + + START_OF_INSTR(); + local = fetch_word_imm(); + nesting = fetch_byte_imm(); + DECODE_PRINTF2("ENTER %x\n", local); + DECODE_PRINTF2(",%x\n", nesting); + TRACE_AND_STEP(); + push_word(M.x86.R_BP); + frame_pointer = M.x86.R_SP; + if (nesting > 0) { + for (i = 1; i < nesting; i++) { + M.x86.R_BP -= 2; + push_word(fetch_data_word_abs(M.x86.R_SS, M.x86.R_BP)); + } + push_word(frame_pointer); + } + M.x86.R_BP = frame_pointer; + M.x86.R_SP = (u16)(M.x86.R_SP - local); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xc9 +****************************************************************************/ +static void x86emuOp_leave(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("LEAVE\n"); + TRACE_AND_STEP(); + M.x86.R_SP = M.x86.R_BP; + M.x86.R_BP = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xca +****************************************************************************/ +static void x86emuOp_ret_far_IMM(u8 X86EMU_UNUSED(op1)) +{ + u16 imm; + + START_OF_INSTR(); + DECODE_PRINTF("RETF\t"); + imm = fetch_word_imm(); + DECODE_PRINTF2("%x\n", imm); + RETURN_TRACE("RETF",M.x86.saved_cs,M.x86.saved_ip); + TRACE_AND_STEP(); + M.x86.R_IP = pop_word(); + M.x86.R_CS = pop_word(); + M.x86.R_SP += imm; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xcb +****************************************************************************/ +static void x86emuOp_ret_far(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("RETF\n"); + RETURN_TRACE("RETF",M.x86.saved_cs,M.x86.saved_ip); + TRACE_AND_STEP(); + M.x86.R_IP = pop_word(); + M.x86.R_CS = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xcc +****************************************************************************/ +static void x86emuOp_int3(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("INT 3\n"); + TRACE_AND_STEP(); + if (_X86EMU_intrTab[3]) { + (*_X86EMU_intrTab[3])(3); + } else { + push_word((u16)M.x86.R_FLG); + CLEAR_FLAG(F_IF); + CLEAR_FLAG(F_TF); + push_word(M.x86.R_CS); + M.x86.R_CS = mem_access_word(3 * 4 + 2); + push_word(M.x86.R_IP); + M.x86.R_IP = mem_access_word(3 * 4); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xcd +****************************************************************************/ +static void x86emuOp_int_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 intnum; + + START_OF_INSTR(); + DECODE_PRINTF("INT\t"); + intnum = fetch_byte_imm(); + DECODE_PRINTF2("%x\n", intnum); + TRACE_AND_STEP(); + if (_X86EMU_intrTab[intnum]) { + (*_X86EMU_intrTab[intnum])(intnum); + } else { + push_word((u16)M.x86.R_FLG); + CLEAR_FLAG(F_IF); + CLEAR_FLAG(F_TF); + push_word(M.x86.R_CS); + M.x86.R_CS = mem_access_word(intnum * 4 + 2); + push_word(M.x86.R_IP); + M.x86.R_IP = mem_access_word(intnum * 4); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xce +****************************************************************************/ +static void x86emuOp_into(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("INTO\n"); + TRACE_AND_STEP(); + if (ACCESS_FLAG(F_OF)) { + if (_X86EMU_intrTab[4]) { + (*_X86EMU_intrTab[4])(4); + } else { + push_word((u16)M.x86.R_FLG); + CLEAR_FLAG(F_IF); + CLEAR_FLAG(F_TF); + push_word(M.x86.R_CS); + M.x86.R_CS = mem_access_word(4 * 4 + 2); + push_word(M.x86.R_IP); + M.x86.R_IP = mem_access_word(4 * 4); + } + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xcf +****************************************************************************/ +static void x86emuOp_iret(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("IRET\n"); + + TRACE_AND_STEP(); + + M.x86.R_IP = pop_word(); + M.x86.R_CS = pop_word(); + M.x86.R_FLG = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xd0 +****************************************************************************/ +static void x86emuOp_opcD0_byte_RM_1(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg; + uint destoffset; + u8 destval; + + /* + * Yet another weirdo special case instruction format. Part of + * the opcode held below in "RH". Doubly nested case would + * result, except that the decoded instruction + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + switch (rh) { + case 0: + DECODE_PRINTF("ROL\t"); + break; + case 1: + DECODE_PRINTF("ROR\t"); + break; + case 2: + DECODE_PRINTF("RCL\t"); + break; + case 3: + DECODE_PRINTF("RCR\t"); + break; + case 4: + DECODE_PRINTF("SHL\t"); + break; + case 5: + DECODE_PRINTF("SHR\t"); + break; + case 6: + DECODE_PRINTF("SAL\t"); + break; + case 7: + DECODE_PRINTF("SAR\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + switch (mod) { + case 0: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, 1); + store_data_byte(destoffset, destval); + break; + case 1: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, 1); + store_data_byte(destoffset, destval); + break; + case 2: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, 1); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(",1\n"); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (*destreg, 1); + *destreg = destval; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xd1 +****************************************************************************/ +static void x86emuOp_opcD1_word_RM_1(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + /* + * Yet another weirdo special case instruction format. Part of + * the opcode held below in "RH". Doubly nested case would + * result, except that the decoded instruction + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + switch (rh) { + case 0: + DECODE_PRINTF("ROL\t"); + break; + case 1: + DECODE_PRINTF("ROR\t"); + break; + case 2: + DECODE_PRINTF("RCL\t"); + break; + case 3: + DECODE_PRINTF("RCR\t"); + break; + case 4: + DECODE_PRINTF("SHL\t"); + break; + case 5: + DECODE_PRINTF("SHR\t"); + break; + case 6: + DECODE_PRINTF("SAL\t"); + break; + case 7: + DECODE_PRINTF("SAR\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, 1); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, 1); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, 1); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, 1); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, 1); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(",1\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, 1); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *destreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(",1\n"); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (*destreg, 1); + *destreg = destval; + } else { + u16 destval; + u16 *destreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(",1\n"); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (*destreg, 1); + *destreg = destval; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xd2 +****************************************************************************/ +static void x86emuOp_opcD2_byte_RM_CL(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg; + uint destoffset; + u8 destval; + u8 amt; + + /* + * Yet another weirdo special case instruction format. Part of + * the opcode held below in "RH". Doubly nested case would + * result, except that the decoded instruction + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + switch (rh) { + case 0: + DECODE_PRINTF("ROL\t"); + break; + case 1: + DECODE_PRINTF("ROR\t"); + break; + case 2: + DECODE_PRINTF("RCL\t"); + break; + case 3: + DECODE_PRINTF("RCR\t"); + break; + case 4: + DECODE_PRINTF("SHL\t"); + break; + case 5: + DECODE_PRINTF("SHR\t"); + break; + case 6: + DECODE_PRINTF("SAL\t"); + break; + case 7: + DECODE_PRINTF("SAR\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + amt = M.x86.R_CL; + switch (mod) { + case 0: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, amt); + store_data_byte(destoffset, destval); + break; + case 1: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, amt); + store_data_byte(destoffset, destval); + break; + case 2: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (destval, amt); + store_data_byte(destoffset, destval); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = (*opcD0_byte_operation[rh]) (*destreg, amt); + *destreg = destval; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xd3 +****************************************************************************/ +static void x86emuOp_opcD3_word_RM_CL(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + u8 amt; + + /* + * Yet another weirdo special case instruction format. Part of + * the opcode held below in "RH". Doubly nested case would + * result, except that the decoded instruction + */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + switch (rh) { + case 0: + DECODE_PRINTF("ROL\t"); + break; + case 1: + DECODE_PRINTF("ROR\t"); + break; + case 2: + DECODE_PRINTF("RCL\t"); + break; + case 3: + DECODE_PRINTF("RCR\t"); + break; + case 4: + DECODE_PRINTF("SHL\t"); + break; + case 5: + DECODE_PRINTF("SHR\t"); + break; + case 6: + DECODE_PRINTF("SAL\t"); + break; + case 7: + DECODE_PRINTF("SAR\t"); + break; + } + } +#endif + /* know operation, decode the mod byte to find the addressing + mode. */ + amt = M.x86.R_CL; + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, amt); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, amt); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, amt); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, amt); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_long_operation[rh]) (destval, amt); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("WORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(",CL\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = (*opcD1_word_operation[rh]) (destval, amt); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + *destreg = (*opcD1_long_operation[rh]) (*destreg, amt); + } else { + u16 *destreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + *destreg = (*opcD1_word_operation[rh]) (*destreg, amt); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xd4 +****************************************************************************/ +static void x86emuOp_aam(u8 X86EMU_UNUSED(op1)) +{ + u8 a; + + START_OF_INSTR(); + DECODE_PRINTF("AAM\n"); + a = fetch_byte_imm(); /* this is a stupid encoding. */ + if (a != 10) { + /* fix: add base decoding + aam_word(u8 val, int base a) */ + DECODE_PRINTF("ERROR DECODING AAM\n"); + TRACE_REGS(); + HALT_SYS(); + } + TRACE_AND_STEP(); + /* note the type change here --- returning AL and AH in AX. */ + M.x86.R_AX = aam_word(M.x86.R_AL); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xd5 +****************************************************************************/ +static void x86emuOp_aad(u8 X86EMU_UNUSED(op1)) +{ + u8 a; + + START_OF_INSTR(); + DECODE_PRINTF("AAD\n"); + a = fetch_byte_imm(); + if (a != 10) { + /* fix: add base decoding + aad_word(u16 val, int base a) */ + DECODE_PRINTF("ERROR DECODING AAM\n"); + TRACE_REGS(); + HALT_SYS(); + } + TRACE_AND_STEP(); + M.x86.R_AX = aad_word(M.x86.R_AX); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/* opcode 0xd6 ILLEGAL OPCODE */ + +/**************************************************************************** +REMARKS: +Handles opcode 0xd7 +****************************************************************************/ +static void x86emuOp_xlat(u8 X86EMU_UNUSED(op1)) +{ + u16 addr; + + START_OF_INSTR(); + DECODE_PRINTF("XLAT\n"); + TRACE_AND_STEP(); + addr = (u16)(M.x86.R_BX + (u8)M.x86.R_AL); + M.x86.R_AL = fetch_data_byte(addr); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/* instuctions D8 .. DF are in i87_ops.c */ + +/**************************************************************************** +REMARKS: +Handles opcode 0xe0 +****************************************************************************/ +static void x86emuOp_loopne(u8 X86EMU_UNUSED(op1)) +{ + s16 ip; + + START_OF_INSTR(); + DECODE_PRINTF("LOOPNE\t"); + ip = (s8) fetch_byte_imm(); + ip += (s16) M.x86.R_IP; + DECODE_PRINTF2("%04x\n", ip); + TRACE_AND_STEP(); + M.x86.R_CX -= 1; + if (M.x86.R_CX != 0 && !ACCESS_FLAG(F_ZF)) /* CX != 0 and !ZF */ + M.x86.R_IP = ip; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe1 +****************************************************************************/ +static void x86emuOp_loope(u8 X86EMU_UNUSED(op1)) +{ + s16 ip; + + START_OF_INSTR(); + DECODE_PRINTF("LOOPE\t"); + ip = (s8) fetch_byte_imm(); + ip += (s16) M.x86.R_IP; + DECODE_PRINTF2("%04x\n", ip); + TRACE_AND_STEP(); + M.x86.R_CX -= 1; + if (M.x86.R_CX != 0 && ACCESS_FLAG(F_ZF)) /* CX != 0 and ZF */ + M.x86.R_IP = ip; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe2 +****************************************************************************/ +static void x86emuOp_loop(u8 X86EMU_UNUSED(op1)) +{ + s16 ip; + + START_OF_INSTR(); + DECODE_PRINTF("LOOP\t"); + ip = (s8) fetch_byte_imm(); + ip += (s16) M.x86.R_IP; + DECODE_PRINTF2("%04x\n", ip); + TRACE_AND_STEP(); + M.x86.R_CX -= 1; + if (M.x86.R_CX != 0) + M.x86.R_IP = ip; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe3 +****************************************************************************/ +static void x86emuOp_jcxz(u8 X86EMU_UNUSED(op1)) +{ + u16 target; + s8 offset; + + /* jump to byte offset if overflow flag is set */ + START_OF_INSTR(); + DECODE_PRINTF("JCXZ\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + if (M.x86.R_CX == 0) + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe4 +****************************************************************************/ +static void x86emuOp_in_byte_AL_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 port; + + START_OF_INSTR(); + DECODE_PRINTF("IN\t"); + port = (u8) fetch_byte_imm(); + DECODE_PRINTF2("%x,AL\n", port); + TRACE_AND_STEP(); + M.x86.R_AL = (*sys_inb)(port); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe5 +****************************************************************************/ +static void x86emuOp_in_word_AX_IMM(u8 X86EMU_UNUSED(op1)) +{ + u8 port; + + START_OF_INSTR(); + DECODE_PRINTF("IN\t"); + port = (u8) fetch_byte_imm(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF2("EAX,%x\n", port); + } else { + DECODE_PRINTF2("AX,%x\n", port); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = (*sys_inl)(port); + } else { + M.x86.R_AX = (*sys_inw)(port); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe6 +****************************************************************************/ +static void x86emuOp_out_byte_IMM_AL(u8 X86EMU_UNUSED(op1)) +{ + u8 port; + + START_OF_INSTR(); + DECODE_PRINTF("OUT\t"); + port = (u8) fetch_byte_imm(); + DECODE_PRINTF2("%x,AL\n", port); + TRACE_AND_STEP(); + (*sys_outb)(port, M.x86.R_AL); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe7 +****************************************************************************/ +static void x86emuOp_out_word_IMM_AX(u8 X86EMU_UNUSED(op1)) +{ + u8 port; + + START_OF_INSTR(); + DECODE_PRINTF("OUT\t"); + port = (u8) fetch_byte_imm(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF2("%x,EAX\n", port); + } else { + DECODE_PRINTF2("%x,AX\n", port); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + (*sys_outl)(port, M.x86.R_EAX); + } else { + (*sys_outw)(port, M.x86.R_AX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe8 +****************************************************************************/ +static void x86emuOp_call_near_IMM(u8 X86EMU_UNUSED(op1)) +{ + s16 ip16 = 0; + s32 ip32 = 0; + + START_OF_INSTR(); + DECODE_PRINTF("CALL\t"); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + ip32 = (s32) fetch_long_imm(); + ip32 += (s16) M.x86.R_IP; /* CHECK SIGN */ + DECODE_PRINTF2("%04x\n", (u16)ip32); + CALL_TRACE(M.x86.saved_cs, M.x86.saved_ip, M.x86.R_CS, ip32, ""); + } else { + ip16 = (s16) fetch_word_imm(); + ip16 += (s16) M.x86.R_IP; /* CHECK SIGN */ + DECODE_PRINTF2("%04x\n", (u16)ip16); + CALL_TRACE(M.x86.saved_cs, M.x86.saved_ip, M.x86.R_CS, ip16, ""); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + push_long(M.x86.R_EIP); + M.x86.R_EIP = ip32 & 0xffff; + } else { + push_word(M.x86.R_IP); + M.x86.R_EIP = ip16; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xe9 +****************************************************************************/ +static void x86emuOp_jump_near_IMM(u8 X86EMU_UNUSED(op1)) +{ + s32 ip; + + START_OF_INSTR(); + DECODE_PRINTF("JMP\t"); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + ip = (s32)fetch_long_imm(); + ip += (s32)M.x86.R_EIP; + DECODE_PRINTF2("%08x\n", (u32)ip); + TRACE_AND_STEP(); + M.x86.R_EIP = (u32)ip; + } else { + ip = (s16)fetch_word_imm(); + ip += (s16)M.x86.R_IP; + DECODE_PRINTF2("%04x\n", (u16)ip); + TRACE_AND_STEP(); + M.x86.R_IP = (u16)ip; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xea +****************************************************************************/ +static void x86emuOp_jump_far_IMM(u8 X86EMU_UNUSED(op1)) +{ + u16 cs; + u32 ip; + + START_OF_INSTR(); + DECODE_PRINTF("JMP\tFAR "); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + ip = fetch_long_imm(); + } else { + ip = fetch_word_imm(); + } + cs = fetch_word_imm(); + DECODE_PRINTF2("%04x:", cs); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF2("%08x\n", ip); + } else { + DECODE_PRINTF2("%04x\n", (u16)ip); + } + TRACE_AND_STEP(); + M.x86.R_EIP = ip & 0xffff; + M.x86.R_CS = cs; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xeb +****************************************************************************/ +static void x86emuOp_jump_byte_IMM(u8 X86EMU_UNUSED(op1)) +{ + u16 target; + s8 offset; + + START_OF_INSTR(); + DECODE_PRINTF("JMP\t"); + offset = (s8)fetch_byte_imm(); + target = (u16)(M.x86.R_IP + offset); + DECODE_PRINTF2("%x\n", target); + TRACE_AND_STEP(); + M.x86.R_IP = target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xec +****************************************************************************/ +static void x86emuOp_in_byte_AL_DX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("IN\tAL,DX\n"); + TRACE_AND_STEP(); + M.x86.R_AL = (*sys_inb)(M.x86.R_DX); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xed +****************************************************************************/ +static void x86emuOp_in_word_AX_DX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("IN\tEAX,DX\n"); + } else { + DECODE_PRINTF("IN\tAX,DX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_EAX = (*sys_inl)(M.x86.R_DX); + } else { + M.x86.R_AX = (*sys_inw)(M.x86.R_DX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xee +****************************************************************************/ +static void x86emuOp_out_byte_DX_AL(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("OUT\tDX,AL\n"); + TRACE_AND_STEP(); + (*sys_outb)(M.x86.R_DX, M.x86.R_AL); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xef +****************************************************************************/ +static void x86emuOp_out_word_DX_AX(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("OUT\tDX,EAX\n"); + } else { + DECODE_PRINTF("OUT\tDX,AX\n"); + } + TRACE_AND_STEP(); + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + (*sys_outl)(M.x86.R_DX, M.x86.R_EAX); + } else { + (*sys_outw)(M.x86.R_DX, M.x86.R_AX); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xf0 +****************************************************************************/ +static void x86emuOp_lock(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("LOCK:\n"); + TRACE_AND_STEP(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/*opcode 0xf1 ILLEGAL OPERATION */ + +/**************************************************************************** +REMARKS: +Handles opcode 0xf2 +****************************************************************************/ +static void x86emuOp_repne(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("REPNE\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_PREFIX_REPNE; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xf3 +****************************************************************************/ +static void x86emuOp_repe(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("REPE\n"); + TRACE_AND_STEP(); + M.x86.mode |= SYSMODE_PREFIX_REPE; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xf4 +****************************************************************************/ +static void x86emuOp_halt(u8 X86EMU_UNUSED(op1)) +{ + START_OF_INSTR(); + DECODE_PRINTF("HALT\n"); + TRACE_AND_STEP(); + HALT_SYS(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xf5 +****************************************************************************/ +static void x86emuOp_cmc(u8 X86EMU_UNUSED(op1)) +{ + /* complement the carry flag. */ + START_OF_INSTR(); + DECODE_PRINTF("CMC\n"); + TRACE_AND_STEP(); + TOGGLE_FLAG(F_CF); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xf6 +****************************************************************************/ +static void x86emuOp_opcF6_byte_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + u8 *destreg; + uint destoffset; + u8 destval, srcval; + + /* long, drawn out code follows. Double switch for a total + of 32 cases. */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: /* mod=00 */ + switch (rh) { + case 0: /* test byte imm */ + DECODE_PRINTF("TEST\tBYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%02x\n", srcval); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + test_byte(destval, srcval); + break; + case 1: + DECODE_PRINTF("ILLEGAL OP MOD=00 RH=01 OP=F6\n"); + HALT_SYS(); + break; + case 2: + DECODE_PRINTF("NOT\tBYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = not_byte(destval); + store_data_byte(destoffset, destval); + break; + case 3: + DECODE_PRINTF("NEG\tBYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = neg_byte(destval); + store_data_byte(destoffset, destval); + break; + case 4: + DECODE_PRINTF("MUL\tBYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + mul_byte(destval); + break; + case 5: + DECODE_PRINTF("IMUL\tBYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + imul_byte(destval); + break; + case 6: + DECODE_PRINTF("DIV\tBYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + div_byte(destval); + break; + case 7: + DECODE_PRINTF("IDIV\tBYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + idiv_byte(destval); + break; + } + break; /* end mod==00 */ + case 1: /* mod=01 */ + switch (rh) { + case 0: /* test byte imm */ + DECODE_PRINTF("TEST\tBYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%02x\n", srcval); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + test_byte(destval, srcval); + break; + case 1: + DECODE_PRINTF("ILLEGAL OP MOD=01 RH=01 OP=F6\n"); + HALT_SYS(); + break; + case 2: + DECODE_PRINTF("NOT\tBYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = not_byte(destval); + store_data_byte(destoffset, destval); + break; + case 3: + DECODE_PRINTF("NEG\tBYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = neg_byte(destval); + store_data_byte(destoffset, destval); + break; + case 4: + DECODE_PRINTF("MUL\tBYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + mul_byte(destval); + break; + case 5: + DECODE_PRINTF("IMUL\tBYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + imul_byte(destval); + break; + case 6: + DECODE_PRINTF("DIV\tBYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + div_byte(destval); + break; + case 7: + DECODE_PRINTF("IDIV\tBYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + idiv_byte(destval); + break; + } + break; /* end mod==01 */ + case 2: /* mod=10 */ + switch (rh) { + case 0: /* test byte imm */ + DECODE_PRINTF("TEST\tBYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%02x\n", srcval); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + test_byte(destval, srcval); + break; + case 1: + DECODE_PRINTF("ILLEGAL OP MOD=10 RH=01 OP=F6\n"); + HALT_SYS(); + break; + case 2: + DECODE_PRINTF("NOT\tBYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = not_byte(destval); + store_data_byte(destoffset, destval); + break; + case 3: + DECODE_PRINTF("NEG\tBYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = neg_byte(destval); + store_data_byte(destoffset, destval); + break; + case 4: + DECODE_PRINTF("MUL\tBYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + mul_byte(destval); + break; + case 5: + DECODE_PRINTF("IMUL\tBYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + imul_byte(destval); + break; + case 6: + DECODE_PRINTF("DIV\tBYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + div_byte(destval); + break; + case 7: + DECODE_PRINTF("IDIV\tBYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + idiv_byte(destval); + break; + } + break; /* end mod==10 */ + case 3: /* mod=11 */ + switch (rh) { + case 0: /* test byte imm */ + DECODE_PRINTF("TEST\t"); + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF(","); + srcval = fetch_byte_imm(); + DECODE_PRINTF2("%02x\n", srcval); + TRACE_AND_STEP(); + test_byte(*destreg, srcval); + break; + case 1: + DECODE_PRINTF("ILLEGAL OP MOD=00 RH=01 OP=F6\n"); + HALT_SYS(); + break; + case 2: + DECODE_PRINTF("NOT\t"); + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = not_byte(*destreg); + break; + case 3: + DECODE_PRINTF("NEG\t"); + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = neg_byte(*destreg); + break; + case 4: + DECODE_PRINTF("MUL\t"); + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + mul_byte(*destreg); /*!!! */ + break; + case 5: + DECODE_PRINTF("IMUL\t"); + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + imul_byte(*destreg); + break; + case 6: + DECODE_PRINTF("DIV\t"); + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + div_byte(*destreg); + break; + case 7: + DECODE_PRINTF("IDIV\t"); + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + idiv_byte(*destreg); + break; + } + break; /* end mod==11 */ + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xf7 +****************************************************************************/ +static void x86emuOp_opcF7_word_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rl, rh; + uint destoffset; + + /* long, drawn out code follows. Double switch for a total + of 32 cases. */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: /* mod=00 */ + switch (rh) { + case 0: /* test word imm */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,srcval; + + DECODE_PRINTF("TEST\tDWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + srcval = fetch_long_imm(); + DECODE_PRINTF2("%x\n", srcval); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + test_long(destval, srcval); + } else { + u16 destval,srcval; + + DECODE_PRINTF("TEST\tWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + srcval = fetch_word_imm(); + DECODE_PRINTF2("%x\n", srcval); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + test_word(destval, srcval); + } + break; + case 1: + DECODE_PRINTF("ILLEGAL OP MOD=00 RH=01 OP=F7\n"); + HALT_SYS(); + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("NOT\tDWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = not_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("NOT\tWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = not_word(destval); + store_data_word(destoffset, destval); + } + break; + case 3: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("NEG\tDWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = neg_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("NEG\tWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = neg_word(destval); + store_data_word(destoffset, destval); + } + break; + case 4: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("MUL\tDWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + mul_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("MUL\tWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + mul_word(destval); + } + break; + case 5: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("IMUL\tDWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + imul_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("IMUL\tWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + imul_word(destval); + } + break; + case 6: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DIV\tDWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + div_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("DIV\tWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + div_word(destval); + } + break; + case 7: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("IDIV\tDWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + idiv_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("IDIV\tWORD PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + idiv_word(destval); + } + break; + } + break; /* end mod==00 */ + case 1: /* mod=01 */ + switch (rh) { + case 0: /* test word imm */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,srcval; + + DECODE_PRINTF("TEST\tDWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + srcval = fetch_long_imm(); + DECODE_PRINTF2("%x\n", srcval); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + test_long(destval, srcval); + } else { + u16 destval,srcval; + + DECODE_PRINTF("TEST\tWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + srcval = fetch_word_imm(); + DECODE_PRINTF2("%x\n", srcval); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + test_word(destval, srcval); + } + break; + case 1: + DECODE_PRINTF("ILLEGAL OP MOD=01 RH=01 OP=F6\n"); + HALT_SYS(); + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("NOT\tDWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = not_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("NOT\tWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = not_word(destval); + store_data_word(destoffset, destval); + } + break; + case 3: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("NEG\tDWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = neg_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("NEG\tWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = neg_word(destval); + store_data_word(destoffset, destval); + } + break; + case 4: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("MUL\tDWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + mul_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("MUL\tWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + mul_word(destval); + } + break; + case 5: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("IMUL\tDWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + imul_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("IMUL\tWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + imul_word(destval); + } + break; + case 6: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DIV\tDWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + div_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("DIV\tWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + div_word(destval); + } + break; + case 7: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("IDIV\tDWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + idiv_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("IDIV\tWORD PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + idiv_word(destval); + } + break; + } + break; /* end mod==01 */ + case 2: /* mod=10 */ + switch (rh) { + case 0: /* test word imm */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval,srcval; + + DECODE_PRINTF("TEST\tDWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + srcval = fetch_long_imm(); + DECODE_PRINTF2("%x\n", srcval); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + test_long(destval, srcval); + } else { + u16 destval,srcval; + + DECODE_PRINTF("TEST\tWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + srcval = fetch_word_imm(); + DECODE_PRINTF2("%x\n", srcval); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + test_word(destval, srcval); + } + break; + case 1: + DECODE_PRINTF("ILLEGAL OP MOD=10 RH=01 OP=F6\n"); + HALT_SYS(); + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("NOT\tDWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = not_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("NOT\tWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = not_word(destval); + store_data_word(destoffset, destval); + } + break; + case 3: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("NEG\tDWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = neg_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + DECODE_PRINTF("NEG\tWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = neg_word(destval); + store_data_word(destoffset, destval); + } + break; + case 4: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("MUL\tDWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + mul_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("MUL\tWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + mul_word(destval); + } + break; + case 5: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("IMUL\tDWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + imul_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("IMUL\tWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + imul_word(destval); + } + break; + case 6: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("DIV\tDWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + div_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("DIV\tWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + div_word(destval); + } + break; + case 7: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + DECODE_PRINTF("IDIV\tDWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + idiv_long(destval); + } else { + u16 destval; + + DECODE_PRINTF("IDIV\tWORD PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + idiv_word(destval); + } + break; + } + break; /* end mod==10 */ + case 3: /* mod=11 */ + switch (rh) { + case 0: /* test word imm */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + DECODE_PRINTF("TEST\t"); + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + srcval = fetch_long_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + test_long(*destreg, srcval); + } else { + u16 *destreg; + u16 srcval; + + DECODE_PRINTF("TEST\t"); + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + srcval = fetch_word_imm(); + DECODE_PRINTF2("%x\n", srcval); + TRACE_AND_STEP(); + test_word(*destreg, srcval); + } + break; + case 1: + DECODE_PRINTF("ILLEGAL OP MOD=00 RH=01 OP=F6\n"); + HALT_SYS(); + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + DECODE_PRINTF("NOT\t"); + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = not_long(*destreg); + } else { + u16 *destreg; + + DECODE_PRINTF("NOT\t"); + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = not_word(*destreg); + } + break; + case 3: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + DECODE_PRINTF("NEG\t"); + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = neg_long(*destreg); + } else { + u16 *destreg; + + DECODE_PRINTF("NEG\t"); + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = neg_word(*destreg); + } + break; + case 4: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + DECODE_PRINTF("MUL\t"); + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + mul_long(*destreg); /*!!! */ + } else { + u16 *destreg; + + DECODE_PRINTF("MUL\t"); + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + mul_word(*destreg); /*!!! */ + } + break; + case 5: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + DECODE_PRINTF("IMUL\t"); + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + imul_long(*destreg); + } else { + u16 *destreg; + + DECODE_PRINTF("IMUL\t"); + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + imul_word(*destreg); + } + break; + case 6: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + DECODE_PRINTF("DIV\t"); + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + div_long(*destreg); + } else { + u16 *destreg; + + DECODE_PRINTF("DIV\t"); + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + div_word(*destreg); + } + break; + case 7: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + DECODE_PRINTF("IDIV\t"); + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + idiv_long(*destreg); + } else { + u16 *destreg; + + DECODE_PRINTF("IDIV\t"); + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + idiv_word(*destreg); + } + break; + } + break; /* end mod==11 */ + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xf8 +****************************************************************************/ +static void x86emuOp_clc(u8 X86EMU_UNUSED(op1)) +{ + /* clear the carry flag. */ + START_OF_INSTR(); + DECODE_PRINTF("CLC\n"); + TRACE_AND_STEP(); + CLEAR_FLAG(F_CF); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xf9 +****************************************************************************/ +static void x86emuOp_stc(u8 X86EMU_UNUSED(op1)) +{ + /* set the carry flag. */ + START_OF_INSTR(); + DECODE_PRINTF("STC\n"); + TRACE_AND_STEP(); + SET_FLAG(F_CF); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xfa +****************************************************************************/ +static void x86emuOp_cli(u8 X86EMU_UNUSED(op1)) +{ + /* clear interrupts. */ + START_OF_INSTR(); + DECODE_PRINTF("CLI\n"); + TRACE_AND_STEP(); + CLEAR_FLAG(F_IF); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xfb +****************************************************************************/ +static void x86emuOp_sti(u8 X86EMU_UNUSED(op1)) +{ + /* enable interrupts. */ + START_OF_INSTR(); + DECODE_PRINTF("STI\n"); + TRACE_AND_STEP(); + SET_FLAG(F_IF); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xfc +****************************************************************************/ +static void x86emuOp_cld(u8 X86EMU_UNUSED(op1)) +{ + /* clear interrupts. */ + START_OF_INSTR(); + DECODE_PRINTF("CLD\n"); + TRACE_AND_STEP(); + CLEAR_FLAG(F_DF); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xfd +****************************************************************************/ +static void x86emuOp_std(u8 X86EMU_UNUSED(op1)) +{ + /* clear interrupts. */ + START_OF_INSTR(); + DECODE_PRINTF("STD\n"); + TRACE_AND_STEP(); + SET_FLAG(F_DF); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xfe +****************************************************************************/ +static void x86emuOp_opcFE_byte_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rh, rl; + u8 destval; + uint destoffset; + u8 *destreg; + + /* Yet another special case instruction. */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + + switch (rh) { + case 0: + DECODE_PRINTF("INC\t"); + break; + case 1: + DECODE_PRINTF("DEC\t"); + break; + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + DECODE_PRINTF2("ILLEGAL OP MAJOR OP 0xFE MINOR OP %x \n", mod); + HALT_SYS(); + break; + } + } +#endif + switch (mod) { + case 0: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + switch (rh) { + case 0: /* inc word ptr ... */ + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = inc_byte(destval); + store_data_byte(destoffset, destval); + break; + case 1: /* dec word ptr ... */ + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = dec_byte(destval); + store_data_byte(destoffset, destval); + break; + } + break; + case 1: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + switch (rh) { + case 0: + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = inc_byte(destval); + store_data_byte(destoffset, destval); + break; + case 1: + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = dec_byte(destval); + store_data_byte(destoffset, destval); + break; + } + break; + case 2: + DECODE_PRINTF("BYTE PTR "); + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + switch (rh) { + case 0: + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = inc_byte(destval); + store_data_byte(destoffset, destval); + break; + case 1: + destval = fetch_data_byte(destoffset); + TRACE_AND_STEP(); + destval = dec_byte(destval); + store_data_byte(destoffset, destval); + break; + } + break; + case 3: + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + switch (rh) { + case 0: + TRACE_AND_STEP(); + *destreg = inc_byte(*destreg); + break; + case 1: + TRACE_AND_STEP(); + *destreg = dec_byte(*destreg); + break; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0xff +****************************************************************************/ +static void x86emuOp_opcFF_word_RM(u8 X86EMU_UNUSED(op1)) +{ + int mod, rh, rl; + uint destoffset = 0; + u16 *destreg; + u16 destval,destval2; + + /* Yet another special case instruction. */ + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); +#ifdef X86EMU_DEBUG + if (DEBUG_DECODE()) { + /* XXX DECODE_PRINTF may be changed to something more + general, so that it is important to leave the strings + in the same format, even though the result is that the + above test is done twice. */ + + switch (rh) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("INC\tDWORD PTR "); + } else { + DECODE_PRINTF("INC\tWORD PTR "); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + DECODE_PRINTF("DEC\tDWORD PTR "); + } else { + DECODE_PRINTF("DEC\tWORD PTR "); + } + break; + case 2: + DECODE_PRINTF("CALL\t"); + break; + case 3: + DECODE_PRINTF("CALL\tFAR "); + break; + case 4: + DECODE_PRINTF("JMP\t"); + break; + case 5: + DECODE_PRINTF("JMP\tFAR "); + break; + case 6: + DECODE_PRINTF("PUSH\t"); + break; + case 7: + DECODE_PRINTF("ILLEGAL DECODING OF OPCODE FF\t"); + HALT_SYS(); + break; + } + } +#endif + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + switch (rh) { + case 0: /* inc word ptr ... */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = inc_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = inc_word(destval); + store_data_word(destoffset, destval); + } + break; + case 1: /* dec word ptr ... */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = dec_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = dec_word(destval); + store_data_word(destoffset, destval); + } + break; + case 2: /* call word ptr ... */ + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + push_word(M.x86.R_IP); + M.x86.R_IP = destval; + break; + case 3: /* call far ptr ... */ + destval = fetch_data_word(destoffset); + destval2 = fetch_data_word(destoffset + 2); + TRACE_AND_STEP(); + push_word(M.x86.R_CS); + M.x86.R_CS = destval2; + push_word(M.x86.R_IP); + M.x86.R_IP = destval; + break; + case 4: /* jmp word ptr ... */ + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + M.x86.R_IP = destval; + break; + case 5: /* jmp far ptr ... */ + destval = fetch_data_word(destoffset); + destval2 = fetch_data_word(destoffset + 2); + TRACE_AND_STEP(); + M.x86.R_IP = destval; + M.x86.R_CS = destval2; + break; + case 6: /* push word ptr ... */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + push_long(destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + push_word(destval); + } + break; + } + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + switch (rh) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = inc_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = inc_word(destval); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = dec_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = dec_word(destval); + store_data_word(destoffset, destval); + } + break; + case 2: /* call word ptr ... */ + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + push_word(M.x86.R_IP); + M.x86.R_IP = destval; + break; + case 3: /* call far ptr ... */ + destval = fetch_data_word(destoffset); + destval2 = fetch_data_word(destoffset + 2); + TRACE_AND_STEP(); + push_word(M.x86.R_CS); + M.x86.R_CS = destval2; + push_word(M.x86.R_IP); + M.x86.R_IP = destval; + break; + case 4: /* jmp word ptr ... */ + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + M.x86.R_IP = destval; + break; + case 5: /* jmp far ptr ... */ + destval = fetch_data_word(destoffset); + destval2 = fetch_data_word(destoffset + 2); + TRACE_AND_STEP(); + M.x86.R_IP = destval; + M.x86.R_CS = destval2; + break; + case 6: /* push word ptr ... */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + push_long(destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + push_word(destval); + } + break; + } + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + switch (rh) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = inc_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = inc_word(destval); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + destval = dec_long(destval); + store_data_long(destoffset, destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + destval = dec_word(destval); + store_data_word(destoffset, destval); + } + break; + case 2: /* call word ptr ... */ + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + push_word(M.x86.R_IP); + M.x86.R_IP = destval; + break; + case 3: /* call far ptr ... */ + destval = fetch_data_word(destoffset); + destval2 = fetch_data_word(destoffset + 2); + TRACE_AND_STEP(); + push_word(M.x86.R_CS); + M.x86.R_CS = destval2; + push_word(M.x86.R_IP); + M.x86.R_IP = destval; + break; + case 4: /* jmp word ptr ... */ + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + M.x86.R_IP = destval; + break; + case 5: /* jmp far ptr ... */ + destval = fetch_data_word(destoffset); + destval2 = fetch_data_word(destoffset + 2); + TRACE_AND_STEP(); + M.x86.R_IP = destval; + M.x86.R_CS = destval2; + break; + case 6: /* push word ptr ... */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + + destval = fetch_data_long(destoffset); + TRACE_AND_STEP(); + push_long(destval); + } else { + u16 destval; + + destval = fetch_data_word(destoffset); + TRACE_AND_STEP(); + push_word(destval); + } + break; + } + break; + case 3: + switch (rh) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = inc_long(*destreg); + } else { + u16 *destreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = inc_word(*destreg); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = dec_long(*destreg); + } else { + u16 *destreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = dec_word(*destreg); + } + break; + case 2: /* call word ptr ... */ + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + push_word(M.x86.R_IP); + M.x86.R_IP = *destreg; + break; + case 3: /* jmp far ptr ... */ + DECODE_PRINTF("OPERATION UNDEFINED 0XFF \n"); + TRACE_AND_STEP(); + HALT_SYS(); + break; + + case 4: /* jmp ... */ + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + M.x86.R_IP = (u16) (*destreg); + break; + case 5: /* jmp far ptr ... */ + DECODE_PRINTF("OPERATION UNDEFINED 0XFF \n"); + TRACE_AND_STEP(); + HALT_SYS(); + break; + case 6: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + push_long(*destreg); + } else { + u16 *destreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + push_word(*destreg); + } + break; + case 7: + DECODE_PRINTF("OPERATION UNDEFINED 0XFF \n"); + TRACE_AND_STEP(); + HALT_SYS(); + break; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/*************************************************************************** + * Single byte operation code table: + **************************************************************************/ +void (*x86emu_optab[256])(u8) = +{ +/* 0x00 */ x86emuOp_add_byte_RM_R, +/* 0x01 */ x86emuOp_add_word_RM_R, +/* 0x02 */ x86emuOp_add_byte_R_RM, +/* 0x03 */ x86emuOp_add_word_R_RM, +/* 0x04 */ x86emuOp_add_byte_AL_IMM, +/* 0x05 */ x86emuOp_add_word_AX_IMM, +/* 0x06 */ x86emuOp_push_ES, +/* 0x07 */ x86emuOp_pop_ES, + +/* 0x08 */ x86emuOp_or_byte_RM_R, +/* 0x09 */ x86emuOp_or_word_RM_R, +/* 0x0a */ x86emuOp_or_byte_R_RM, +/* 0x0b */ x86emuOp_or_word_R_RM, +/* 0x0c */ x86emuOp_or_byte_AL_IMM, +/* 0x0d */ x86emuOp_or_word_AX_IMM, +/* 0x0e */ x86emuOp_push_CS, +/* 0x0f */ x86emuOp_two_byte, + +/* 0x10 */ x86emuOp_adc_byte_RM_R, +/* 0x11 */ x86emuOp_adc_word_RM_R, +/* 0x12 */ x86emuOp_adc_byte_R_RM, +/* 0x13 */ x86emuOp_adc_word_R_RM, +/* 0x14 */ x86emuOp_adc_byte_AL_IMM, +/* 0x15 */ x86emuOp_adc_word_AX_IMM, +/* 0x16 */ x86emuOp_push_SS, +/* 0x17 */ x86emuOp_pop_SS, + +/* 0x18 */ x86emuOp_sbb_byte_RM_R, +/* 0x19 */ x86emuOp_sbb_word_RM_R, +/* 0x1a */ x86emuOp_sbb_byte_R_RM, +/* 0x1b */ x86emuOp_sbb_word_R_RM, +/* 0x1c */ x86emuOp_sbb_byte_AL_IMM, +/* 0x1d */ x86emuOp_sbb_word_AX_IMM, +/* 0x1e */ x86emuOp_push_DS, +/* 0x1f */ x86emuOp_pop_DS, + +/* 0x20 */ x86emuOp_and_byte_RM_R, +/* 0x21 */ x86emuOp_and_word_RM_R, +/* 0x22 */ x86emuOp_and_byte_R_RM, +/* 0x23 */ x86emuOp_and_word_R_RM, +/* 0x24 */ x86emuOp_and_byte_AL_IMM, +/* 0x25 */ x86emuOp_and_word_AX_IMM, +/* 0x26 */ x86emuOp_segovr_ES, +/* 0x27 */ x86emuOp_daa, + +/* 0x28 */ x86emuOp_sub_byte_RM_R, +/* 0x29 */ x86emuOp_sub_word_RM_R, +/* 0x2a */ x86emuOp_sub_byte_R_RM, +/* 0x2b */ x86emuOp_sub_word_R_RM, +/* 0x2c */ x86emuOp_sub_byte_AL_IMM, +/* 0x2d */ x86emuOp_sub_word_AX_IMM, +/* 0x2e */ x86emuOp_segovr_CS, +/* 0x2f */ x86emuOp_das, + +/* 0x30 */ x86emuOp_xor_byte_RM_R, +/* 0x31 */ x86emuOp_xor_word_RM_R, +/* 0x32 */ x86emuOp_xor_byte_R_RM, +/* 0x33 */ x86emuOp_xor_word_R_RM, +/* 0x34 */ x86emuOp_xor_byte_AL_IMM, +/* 0x35 */ x86emuOp_xor_word_AX_IMM, +/* 0x36 */ x86emuOp_segovr_SS, +/* 0x37 */ x86emuOp_aaa, + +/* 0x38 */ x86emuOp_cmp_byte_RM_R, +/* 0x39 */ x86emuOp_cmp_word_RM_R, +/* 0x3a */ x86emuOp_cmp_byte_R_RM, +/* 0x3b */ x86emuOp_cmp_word_R_RM, +/* 0x3c */ x86emuOp_cmp_byte_AL_IMM, +/* 0x3d */ x86emuOp_cmp_word_AX_IMM, +/* 0x3e */ x86emuOp_segovr_DS, +/* 0x3f */ x86emuOp_aas, + +/* 0x40 */ x86emuOp_inc_AX, +/* 0x41 */ x86emuOp_inc_CX, +/* 0x42 */ x86emuOp_inc_DX, +/* 0x43 */ x86emuOp_inc_BX, +/* 0x44 */ x86emuOp_inc_SP, +/* 0x45 */ x86emuOp_inc_BP, +/* 0x46 */ x86emuOp_inc_SI, +/* 0x47 */ x86emuOp_inc_DI, + +/* 0x48 */ x86emuOp_dec_AX, +/* 0x49 */ x86emuOp_dec_CX, +/* 0x4a */ x86emuOp_dec_DX, +/* 0x4b */ x86emuOp_dec_BX, +/* 0x4c */ x86emuOp_dec_SP, +/* 0x4d */ x86emuOp_dec_BP, +/* 0x4e */ x86emuOp_dec_SI, +/* 0x4f */ x86emuOp_dec_DI, + +/* 0x50 */ x86emuOp_push_AX, +/* 0x51 */ x86emuOp_push_CX, +/* 0x52 */ x86emuOp_push_DX, +/* 0x53 */ x86emuOp_push_BX, +/* 0x54 */ x86emuOp_push_SP, +/* 0x55 */ x86emuOp_push_BP, +/* 0x56 */ x86emuOp_push_SI, +/* 0x57 */ x86emuOp_push_DI, + +/* 0x58 */ x86emuOp_pop_AX, +/* 0x59 */ x86emuOp_pop_CX, +/* 0x5a */ x86emuOp_pop_DX, +/* 0x5b */ x86emuOp_pop_BX, +/* 0x5c */ x86emuOp_pop_SP, +/* 0x5d */ x86emuOp_pop_BP, +/* 0x5e */ x86emuOp_pop_SI, +/* 0x5f */ x86emuOp_pop_DI, + +/* 0x60 */ x86emuOp_push_all, +/* 0x61 */ x86emuOp_pop_all, +/* 0x62 */ x86emuOp_illegal_op, /* bound */ +/* 0x63 */ x86emuOp_illegal_op, /* arpl */ +/* 0x64 */ x86emuOp_segovr_FS, +/* 0x65 */ x86emuOp_segovr_GS, +/* 0x66 */ x86emuOp_prefix_data, +/* 0x67 */ x86emuOp_prefix_addr, + +/* 0x68 */ x86emuOp_push_word_IMM, +/* 0x69 */ x86emuOp_imul_word_IMM, +/* 0x6a */ x86emuOp_push_byte_IMM, +/* 0x6b */ x86emuOp_imul_byte_IMM, +/* 0x6c */ x86emuOp_ins_byte, +/* 0x6d */ x86emuOp_ins_word, +/* 0x6e */ x86emuOp_outs_byte, +/* 0x6f */ x86emuOp_outs_word, + +/* 0x70 */ x86emuOp_jump_near_O, +/* 0x71 */ x86emuOp_jump_near_NO, +/* 0x72 */ x86emuOp_jump_near_B, +/* 0x73 */ x86emuOp_jump_near_NB, +/* 0x74 */ x86emuOp_jump_near_Z, +/* 0x75 */ x86emuOp_jump_near_NZ, +/* 0x76 */ x86emuOp_jump_near_BE, +/* 0x77 */ x86emuOp_jump_near_NBE, + +/* 0x78 */ x86emuOp_jump_near_S, +/* 0x79 */ x86emuOp_jump_near_NS, +/* 0x7a */ x86emuOp_jump_near_P, +/* 0x7b */ x86emuOp_jump_near_NP, +/* 0x7c */ x86emuOp_jump_near_L, +/* 0x7d */ x86emuOp_jump_near_NL, +/* 0x7e */ x86emuOp_jump_near_LE, +/* 0x7f */ x86emuOp_jump_near_NLE, + +/* 0x80 */ x86emuOp_opc80_byte_RM_IMM, +/* 0x81 */ x86emuOp_opc81_word_RM_IMM, +/* 0x82 */ x86emuOp_opc82_byte_RM_IMM, +/* 0x83 */ x86emuOp_opc83_word_RM_IMM, +/* 0x84 */ x86emuOp_test_byte_RM_R, +/* 0x85 */ x86emuOp_test_word_RM_R, +/* 0x86 */ x86emuOp_xchg_byte_RM_R, +/* 0x87 */ x86emuOp_xchg_word_RM_R, + +/* 0x88 */ x86emuOp_mov_byte_RM_R, +/* 0x89 */ x86emuOp_mov_word_RM_R, +/* 0x8a */ x86emuOp_mov_byte_R_RM, +/* 0x8b */ x86emuOp_mov_word_R_RM, +/* 0x8c */ x86emuOp_mov_word_RM_SR, +/* 0x8d */ x86emuOp_lea_word_R_M, +/* 0x8e */ x86emuOp_mov_word_SR_RM, +/* 0x8f */ x86emuOp_pop_RM, + +/* 0x90 */ x86emuOp_nop, +/* 0x91 */ x86emuOp_xchg_word_AX_CX, +/* 0x92 */ x86emuOp_xchg_word_AX_DX, +/* 0x93 */ x86emuOp_xchg_word_AX_BX, +/* 0x94 */ x86emuOp_xchg_word_AX_SP, +/* 0x95 */ x86emuOp_xchg_word_AX_BP, +/* 0x96 */ x86emuOp_xchg_word_AX_SI, +/* 0x97 */ x86emuOp_xchg_word_AX_DI, + +/* 0x98 */ x86emuOp_cbw, +/* 0x99 */ x86emuOp_cwd, +/* 0x9a */ x86emuOp_call_far_IMM, +/* 0x9b */ x86emuOp_wait, +/* 0x9c */ x86emuOp_pushf_word, +/* 0x9d */ x86emuOp_popf_word, +/* 0x9e */ x86emuOp_sahf, +/* 0x9f */ x86emuOp_lahf, + +/* 0xa0 */ x86emuOp_mov_AL_M_IMM, +/* 0xa1 */ x86emuOp_mov_AX_M_IMM, +/* 0xa2 */ x86emuOp_mov_M_AL_IMM, +/* 0xa3 */ x86emuOp_mov_M_AX_IMM, +/* 0xa4 */ x86emuOp_movs_byte, +/* 0xa5 */ x86emuOp_movs_word, +/* 0xa6 */ x86emuOp_cmps_byte, +/* 0xa7 */ x86emuOp_cmps_word, +/* 0xa8 */ x86emuOp_test_AL_IMM, +/* 0xa9 */ x86emuOp_test_AX_IMM, +/* 0xaa */ x86emuOp_stos_byte, +/* 0xab */ x86emuOp_stos_word, +/* 0xac */ x86emuOp_lods_byte, +/* 0xad */ x86emuOp_lods_word, +/* 0xac */ x86emuOp_scas_byte, +/* 0xad */ x86emuOp_scas_word, + + +/* 0xb0 */ x86emuOp_mov_byte_AL_IMM, +/* 0xb1 */ x86emuOp_mov_byte_CL_IMM, +/* 0xb2 */ x86emuOp_mov_byte_DL_IMM, +/* 0xb3 */ x86emuOp_mov_byte_BL_IMM, +/* 0xb4 */ x86emuOp_mov_byte_AH_IMM, +/* 0xb5 */ x86emuOp_mov_byte_CH_IMM, +/* 0xb6 */ x86emuOp_mov_byte_DH_IMM, +/* 0xb7 */ x86emuOp_mov_byte_BH_IMM, + +/* 0xb8 */ x86emuOp_mov_word_AX_IMM, +/* 0xb9 */ x86emuOp_mov_word_CX_IMM, +/* 0xba */ x86emuOp_mov_word_DX_IMM, +/* 0xbb */ x86emuOp_mov_word_BX_IMM, +/* 0xbc */ x86emuOp_mov_word_SP_IMM, +/* 0xbd */ x86emuOp_mov_word_BP_IMM, +/* 0xbe */ x86emuOp_mov_word_SI_IMM, +/* 0xbf */ x86emuOp_mov_word_DI_IMM, + +/* 0xc0 */ x86emuOp_opcC0_byte_RM_MEM, +/* 0xc1 */ x86emuOp_opcC1_word_RM_MEM, +/* 0xc2 */ x86emuOp_ret_near_IMM, +/* 0xc3 */ x86emuOp_ret_near, +/* 0xc4 */ x86emuOp_les_R_IMM, +/* 0xc5 */ x86emuOp_lds_R_IMM, +/* 0xc6 */ x86emuOp_mov_byte_RM_IMM, +/* 0xc7 */ x86emuOp_mov_word_RM_IMM, +/* 0xc8 */ x86emuOp_enter, +/* 0xc9 */ x86emuOp_leave, +/* 0xca */ x86emuOp_ret_far_IMM, +/* 0xcb */ x86emuOp_ret_far, +/* 0xcc */ x86emuOp_int3, +/* 0xcd */ x86emuOp_int_IMM, +/* 0xce */ x86emuOp_into, +/* 0xcf */ x86emuOp_iret, + +/* 0xd0 */ x86emuOp_opcD0_byte_RM_1, +/* 0xd1 */ x86emuOp_opcD1_word_RM_1, +/* 0xd2 */ x86emuOp_opcD2_byte_RM_CL, +/* 0xd3 */ x86emuOp_opcD3_word_RM_CL, +/* 0xd4 */ x86emuOp_aam, +/* 0xd5 */ x86emuOp_aad, +/* 0xd6 */ x86emuOp_illegal_op, /* Undocumented SETALC instruction */ +/* 0xd7 */ x86emuOp_xlat, +/* 0xd8 */ x86emuOp_esc_coprocess_d8, +/* 0xd9 */ x86emuOp_esc_coprocess_d9, +/* 0xda */ x86emuOp_esc_coprocess_da, +/* 0xdb */ x86emuOp_esc_coprocess_db, +/* 0xdc */ x86emuOp_esc_coprocess_dc, +/* 0xdd */ x86emuOp_esc_coprocess_dd, +/* 0xde */ x86emuOp_esc_coprocess_de, +/* 0xdf */ x86emuOp_esc_coprocess_df, + +/* 0xe0 */ x86emuOp_loopne, +/* 0xe1 */ x86emuOp_loope, +/* 0xe2 */ x86emuOp_loop, +/* 0xe3 */ x86emuOp_jcxz, +/* 0xe4 */ x86emuOp_in_byte_AL_IMM, +/* 0xe5 */ x86emuOp_in_word_AX_IMM, +/* 0xe6 */ x86emuOp_out_byte_IMM_AL, +/* 0xe7 */ x86emuOp_out_word_IMM_AX, + +/* 0xe8 */ x86emuOp_call_near_IMM, +/* 0xe9 */ x86emuOp_jump_near_IMM, +/* 0xea */ x86emuOp_jump_far_IMM, +/* 0xeb */ x86emuOp_jump_byte_IMM, +/* 0xec */ x86emuOp_in_byte_AL_DX, +/* 0xed */ x86emuOp_in_word_AX_DX, +/* 0xee */ x86emuOp_out_byte_DX_AL, +/* 0xef */ x86emuOp_out_word_DX_AX, + +/* 0xf0 */ x86emuOp_lock, +/* 0xf1 */ x86emuOp_illegal_op, +/* 0xf2 */ x86emuOp_repne, +/* 0xf3 */ x86emuOp_repe, +/* 0xf4 */ x86emuOp_halt, +/* 0xf5 */ x86emuOp_cmc, +/* 0xf6 */ x86emuOp_opcF6_byte_RM, +/* 0xf7 */ x86emuOp_opcF7_word_RM, + +/* 0xf8 */ x86emuOp_clc, +/* 0xf9 */ x86emuOp_stc, +/* 0xfa */ x86emuOp_cli, +/* 0xfb */ x86emuOp_sti, +/* 0xfc */ x86emuOp_cld, +/* 0xfd */ x86emuOp_std, +/* 0xfe */ x86emuOp_opcFE_byte_RM, +/* 0xff */ x86emuOp_opcFF_word_RM, +}; diff --git a/userspace/sdks/nv570/nvalloc/unix/x86emu/ops2.c b/userspace/sdks/nv570/nvalloc/unix/x86emu/ops2.c new file mode 100644 index 0000000..6053b48 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/x86emu/ops2.c @@ -0,0 +1,2877 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: This file includes subroutines to implement the decoding +* and emulation of all the x86 extended two-byte processor +* instructions. +* +****************************************************************************/ + +#include "x86emu/x86emui.h" + +/*----------------------------- Implementation ----------------------------*/ + +/**************************************************************************** +PARAMETERS: +op1 - Instruction op code + +REMARKS: +Handles illegal opcodes. +****************************************************************************/ +static void x86emuOp2_illegal_op( + u8 op2) +{ + START_OF_INSTR(); + DECODE_PRINTF("ILLEGAL EXTENDED X86 OPCODE\n"); + TRACE_REGS(); +DB( NV_PRINTF(LEVEL_INFO, + "%04x:%04x: %02X ILLEGAL EXTENDED X86 OPCODE!\n", + M.x86.R_CS, M.x86.R_IP - 2, op2);) + HALT_SYS(); + END_OF_INSTR(); +} + +#define xorl(a,b) ((a) && !(b)) || (!(a) && (b)) + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0x80-0x8F +****************************************************************************/ +static void x86emuOp2_long_jump(u8 op2) +{ + s32 target; + const char *name = 0; + int cond = 0; + + /* conditional jump to word offset. */ + START_OF_INSTR(); + switch (op2) { + case 0x80: + name = "JO\t"; + cond = ACCESS_FLAG(F_OF); + break; + case 0x81: + name = "JNO\t"; + cond = !ACCESS_FLAG(F_OF); + break; + case 0x82: + name = "JB\t"; + cond = ACCESS_FLAG(F_CF); + break; + case 0x83: + name = "JNB\t"; + cond = !ACCESS_FLAG(F_CF); + break; + case 0x84: + name = "JZ\t"; + cond = ACCESS_FLAG(F_ZF); + break; + case 0x85: + name = "JNZ\t"; + cond = !ACCESS_FLAG(F_ZF); + break; + case 0x86: + name = "JBE\t"; + cond = ACCESS_FLAG(F_CF) || ACCESS_FLAG(F_ZF); + break; + case 0x87: + name = "JNBE\t"; + cond = !(ACCESS_FLAG(F_CF) || ACCESS_FLAG(F_ZF)); + break; + case 0x88: + name = "JS\t"; + cond = ACCESS_FLAG(F_SF); + break; + case 0x89: + name = "JNS\t"; + cond = !ACCESS_FLAG(F_SF); + break; + case 0x8a: + name = "JP\t"; + cond = ACCESS_FLAG(F_PF); + break; + case 0x8b: + name = "JNP\t"; + cond = !ACCESS_FLAG(F_PF); + break; + case 0x8c: + name = "JL\t"; + cond = xorl(ACCESS_FLAG(F_SF), ACCESS_FLAG(F_OF)); + break; + case 0x8d: + name = "JNL\t"; + cond = !xorl(ACCESS_FLAG(F_SF), ACCESS_FLAG(F_OF)); + break; + case 0x8e: + name = "JLE\t"; + cond = (xorl(ACCESS_FLAG(F_SF), ACCESS_FLAG(F_OF)) || + ACCESS_FLAG(F_ZF)); + break; + case 0x8f: + name = "JNLE\t"; + cond = !(xorl(ACCESS_FLAG(F_SF), ACCESS_FLAG(F_OF)) || + ACCESS_FLAG(F_ZF)); + break; + } + DECODE_PRINTF(name); + (void)name; + target = (s16) fetch_word_imm(); + target += (s16) M.x86.R_IP; + DECODE_PRINTF2("%04x\n", target); + TRACE_AND_STEP(); + if (cond) + M.x86.R_IP = (u16)target; + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0x90-0x9F +****************************************************************************/ +static void x86emuOp2_set_byte(u8 op2) +{ + int mod, rl, rh; + uint destoffset; + u8 *destreg; + const char *name = 0; + int cond = 0; + + START_OF_INSTR(); + switch (op2) { + case 0x90: + name = "SETO\t"; + cond = ACCESS_FLAG(F_OF); + break; + case 0x91: + name = "SETNO\t"; + cond = !ACCESS_FLAG(F_OF); + break; + case 0x92: + name = "SETB\t"; + cond = ACCESS_FLAG(F_CF); + break; + case 0x93: + name = "SETNB\t"; + cond = !ACCESS_FLAG(F_CF); + break; + case 0x94: + name = "SETZ\t"; + cond = ACCESS_FLAG(F_ZF); + break; + case 0x95: + name = "SETNZ\t"; + cond = !ACCESS_FLAG(F_ZF); + break; + case 0x96: + name = "SETBE\t"; + cond = ACCESS_FLAG(F_CF) || ACCESS_FLAG(F_ZF); + break; + case 0x97: + name = "SETNBE\t"; + cond = !(ACCESS_FLAG(F_CF) || ACCESS_FLAG(F_ZF)); + break; + case 0x98: + name = "SETS\t"; + cond = ACCESS_FLAG(F_SF); + break; + case 0x99: + name = "SETNS\t"; + cond = !ACCESS_FLAG(F_SF); + break; + case 0x9a: + name = "SETP\t"; + cond = ACCESS_FLAG(F_PF); + break; + case 0x9b: + name = "SETNP\t"; + cond = !ACCESS_FLAG(F_PF); + break; + case 0x9c: + name = "SETL\t"; + cond = xorl(ACCESS_FLAG(F_SF), ACCESS_FLAG(F_OF)); + break; + case 0x9d: + name = "SETNL\t"; + cond = xorl(ACCESS_FLAG(F_SF), ACCESS_FLAG(F_OF)); + break; + case 0x9e: + name = "SETLE\t"; + cond = (xorl(ACCESS_FLAG(F_SF), ACCESS_FLAG(F_OF)) || + ACCESS_FLAG(F_ZF)); + break; + case 0x9f: + name = "SETNLE\t"; + cond = !(xorl(ACCESS_FLAG(F_SF), ACCESS_FLAG(F_OF)) || + ACCESS_FLAG(F_ZF)); + break; + } + DECODE_PRINTF(name); + (void)name; + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_byte(destoffset, cond ? 0x01 : 0x00); + break; + case 1: + destoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_byte(destoffset, cond ? 0x01 : 0x00); + break; + case 2: + destoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + store_data_byte(destoffset, cond ? 0x01 : 0x00); + break; + case 3: /* register to register */ + destreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = cond ? 0x01 : 0x00; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xa0 +****************************************************************************/ +static void x86emuOp2_push_FS(u8 X86EMU_UNUSED(op2)) +{ + START_OF_INSTR(); + DECODE_PRINTF("PUSH\tFS\n"); + TRACE_AND_STEP(); + push_word(M.x86.R_FS); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xa1 +****************************************************************************/ +static void x86emuOp2_pop_FS(u8 X86EMU_UNUSED(op2)) +{ + START_OF_INSTR(); + DECODE_PRINTF("POP\tFS\n"); + TRACE_AND_STEP(); + M.x86.R_FS = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xa3 +****************************************************************************/ +static void x86emuOp2_bt_R(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + int bit,disp; + + START_OF_INSTR(); + DECODE_PRINTF("BT\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval; + u32 *shiftreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + CONDITIONAL_SET_FLAG(srcval & (0x1 << bit),F_CF); + } else { + u16 srcval; + u16 *shiftreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + CONDITIONAL_SET_FLAG(srcval & (0x1 << bit),F_CF); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval; + u32 *shiftreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + CONDITIONAL_SET_FLAG(srcval & (0x1 << bit),F_CF); + } else { + u16 srcval; + u16 *shiftreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + CONDITIONAL_SET_FLAG(srcval & (0x1 << bit),F_CF); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval; + u32 *shiftreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + CONDITIONAL_SET_FLAG(srcval & (0x1 << bit),F_CF); + } else { + u16 srcval; + u16 *shiftreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + CONDITIONAL_SET_FLAG(srcval & (0x1 << bit),F_CF); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg,*shiftreg; + + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + CONDITIONAL_SET_FLAG(*srcreg & (0x1 << bit),F_CF); + } else { + u16 *srcreg,*shiftreg; + + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + CONDITIONAL_SET_FLAG(*srcreg & (0x1 << bit),F_CF); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xa4 +****************************************************************************/ +static void x86emuOp2_shld_IMM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint destoffset; + u8 shift; + + START_OF_INSTR(); + DECODE_PRINTF("SHLD\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shld_long(destval,*shiftreg,shift); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shld_word(destval,*shiftreg,shift); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shld_long(destval,*shiftreg,shift); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shld_word(destval,*shiftreg,shift); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shld_long(destval,*shiftreg,shift); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shld_word(destval,*shiftreg,shift); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*shiftreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + *destreg = shld_long(*destreg,*shiftreg,shift); + } else { + u16 *destreg,*shiftreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + *destreg = shld_word(*destreg,*shiftreg,shift); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xa5 +****************************************************************************/ +static void x86emuOp2_shld_CL(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("SHLD\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shld_long(destval,*shiftreg,M.x86.R_CL); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shld_word(destval,*shiftreg,M.x86.R_CL); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shld_long(destval,*shiftreg,M.x86.R_CL); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shld_word(destval,*shiftreg,M.x86.R_CL); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shld_long(destval,*shiftreg,M.x86.R_CL); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shld_word(destval,*shiftreg,M.x86.R_CL); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*shiftreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + *destreg = shld_long(*destreg,*shiftreg,M.x86.R_CL); + } else { + u16 *destreg,*shiftreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + *destreg = shld_word(*destreg,*shiftreg,M.x86.R_CL); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xa8 +****************************************************************************/ +static void x86emuOp2_push_GS(u8 X86EMU_UNUSED(op2)) +{ + START_OF_INSTR(); + DECODE_PRINTF("PUSH\tGS\n"); + TRACE_AND_STEP(); + push_word(M.x86.R_GS); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xa9 +****************************************************************************/ +static void x86emuOp2_pop_GS(u8 X86EMU_UNUSED(op2)) +{ + START_OF_INSTR(); + DECODE_PRINTF("POP\tGS\n"); + TRACE_AND_STEP(); + M.x86.R_GS = pop_word(); + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +#if 1 +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xab +****************************************************************************/ +static void x86emuOp2_bts_R(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + int bit,disp; + + START_OF_INSTR(); + DECODE_PRINTF("BTS\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval | mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, srcval | mask); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval | mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, srcval | mask); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval | mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, srcval | mask); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg,*shiftreg; + u32 mask; + + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(*srcreg & mask,F_CF); + *srcreg |= mask; + } else { + u16 *srcreg,*shiftreg; + u16 mask; + + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(*srcreg & mask,F_CF); + *srcreg |= mask; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} +#endif + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xac +****************************************************************************/ +static void x86emuOp2_shrd_IMM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint destoffset; + u8 shift; + + START_OF_INSTR(); + DECODE_PRINTF("SHLD\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shrd_long(destval,*shiftreg,shift); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shrd_word(destval,*shiftreg,shift); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shrd_long(destval,*shiftreg,shift); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shrd_word(destval,*shiftreg,shift); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shrd_long(destval,*shiftreg,shift); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shrd_word(destval,*shiftreg,shift); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*shiftreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + *destreg = shrd_long(*destreg,*shiftreg,shift); + } else { + u16 *destreg,*shiftreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF2("%d\n", shift); + TRACE_AND_STEP(); + *destreg = shrd_word(*destreg,*shiftreg,shift); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xad +****************************************************************************/ +static void x86emuOp2_shrd_CL(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint destoffset; + + START_OF_INSTR(); + DECODE_PRINTF("SHLD\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shrd_long(destval,*shiftreg,M.x86.R_CL); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shrd_word(destval,*shiftreg,M.x86.R_CL); + store_data_word(destoffset, destval); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shrd_long(destval,*shiftreg,M.x86.R_CL); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shrd_word(destval,*shiftreg,M.x86.R_CL); + store_data_word(destoffset, destval); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 destval; + u32 *shiftreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_long(destoffset); + destval = shrd_long(destval,*shiftreg,M.x86.R_CL); + store_data_long(destoffset, destval); + } else { + u16 destval; + u16 *shiftreg; + + destoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + destval = fetch_data_word(destoffset); + destval = shrd_word(destval,*shiftreg,M.x86.R_CL); + store_data_word(destoffset, destval); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*shiftreg; + + destreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + *destreg = shrd_long(*destreg,*shiftreg,M.x86.R_CL); + } else { + u16 *destreg,*shiftreg; + + destreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(",CL\n"); + TRACE_AND_STEP(); + *destreg = shrd_word(*destreg,*shiftreg,M.x86.R_CL); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xaf +****************************************************************************/ +static void x86emuOp2_imul_R_RM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("IMUL\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)*destreg,(s32)srcval); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + res = (s16)*destreg * (s16)srcval; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)*destreg,(s32)srcval); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + res = (s16)*destreg * (s16)srcval; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + u32 res_lo,res_hi; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_long(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)*destreg,(s32)srcval); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg; + u16 srcval; + u32 res; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + res = (s16)*destreg * (s16)srcval; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg,*srcreg; + u32 res_lo,res_hi; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + imul_long_direct(&res_lo,&res_hi,(s32)*destreg,(s32)*srcreg); + if (res_hi != 0) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u32)res_lo; + } else { + u16 *destreg,*srcreg; + u32 res; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + res = (s16)*destreg * (s16)*srcreg; + if (res > 0xFFFF) { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } + *destreg = (u16)res; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xb2 +****************************************************************************/ +static void x86emuOp2_lss_R_IMM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rh, rl; + u16 *dstreg; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("LSS\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_SS = fetch_data_word(srcoffset + 2); + break; + case 1: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_SS = fetch_data_word(srcoffset + 2); + break; + case 2: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_SS = fetch_data_word(srcoffset + 2); + break; + case 3: /* register to register */ + /* UNDEFINED! */ + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xb3 +****************************************************************************/ +static void x86emuOp2_btr_R(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + int bit,disp; + + START_OF_INSTR(); + DECODE_PRINTF("BTR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval & ~mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, (u16)(srcval & ~mask)); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval & ~mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, (u16)(srcval & ~mask)); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval & ~mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, (u16)(srcval & ~mask)); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg,*shiftreg; + u32 mask; + + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(*srcreg & mask,F_CF); + *srcreg &= ~mask; + } else { + u16 *srcreg,*shiftreg; + u16 mask; + + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(*srcreg & mask,F_CF); + *srcreg &= ~mask; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xb4 +****************************************************************************/ +static void x86emuOp2_lfs_R_IMM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rh, rl; + u16 *dstreg; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("LFS\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_FS = fetch_data_word(srcoffset + 2); + break; + case 1: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_FS = fetch_data_word(srcoffset + 2); + break; + case 2: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_FS = fetch_data_word(srcoffset + 2); + break; + case 3: /* register to register */ + /* UNDEFINED! */ + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xb5 +****************************************************************************/ +static void x86emuOp2_lgs_R_IMM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rh, rl; + u16 *dstreg; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("LGS\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_GS = fetch_data_word(srcoffset + 2); + break; + case 1: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_GS = fetch_data_word(srcoffset + 2); + break; + case 2: + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *dstreg = fetch_data_word(srcoffset); + M.x86.R_GS = fetch_data_word(srcoffset + 2); + break; + case 3: /* register to register */ + /* UNDEFINED! */ + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xb6 +****************************************************************************/ +static void x86emuOp2_movzx_byte_R_RM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("MOVZX\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_byte(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u8 *srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + } else { + u16 *destreg; + u8 *srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xb7 +****************************************************************************/ +static void x86emuOp2_movzx_word_R_RM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + u32 *destreg; + u32 srcval; + u16 *srcreg; + + START_OF_INSTR(); + DECODE_PRINTF("MOVZX\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 1: + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 2: + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = fetch_data_word(srcoffset); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 3: /* register to register */ + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = *srcreg; + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xba +****************************************************************************/ +static void x86emuOp2_btX_I(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + int bit; + + START_OF_INSTR(); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (rh) { + case 4: + DECODE_PRINTF("BT\t"); + break; + case 5: + DECODE_PRINTF("BTS\t"); + break; + case 6: + DECODE_PRINTF("BTR\t"); + break; + case 7: + DECODE_PRINTF("BTC\t"); + break; + default: + DECODE_PRINTF("ILLEGAL EXTENDED X86 OPCODE\n"); + TRACE_REGS(); +DB( NV_PRINTF(LEVEL_INFO, + "%04x:%04x: %02X%02X ILLEGAL EXTENDED X86 OPCODE EXTENSION!\n", + M.x86.R_CS, M.x86.R_IP - 3, op2, + (mod << 6) | (rh << 3) | rl);) + HALT_SYS(); + } + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, mask; + u8 shift; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = shift & 0x1F; + srcval = fetch_data_long(srcoffset); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + switch (rh) { + case 5: + store_data_long(srcoffset, srcval | mask); + break; + case 6: + store_data_long(srcoffset, srcval & ~mask); + break; + case 7: + store_data_long(srcoffset, srcval ^ mask); + break; + default: + break; + } + } else { + u16 srcval, mask; + u8 shift; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = shift & 0xF; + srcval = fetch_data_word(srcoffset); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + switch (rh) { + case 5: + store_data_word(srcoffset, srcval | mask); + break; + case 6: + store_data_word(srcoffset, srcval & ~mask); + break; + case 7: + store_data_word(srcoffset, srcval ^ mask); + break; + default: + break; + } + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, mask; + u8 shift; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = shift & 0x1F; + srcval = fetch_data_long(srcoffset); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + switch (rh) { + case 5: + store_data_long(srcoffset, srcval | mask); + break; + case 6: + store_data_long(srcoffset, srcval & ~mask); + break; + case 7: + store_data_long(srcoffset, srcval ^ mask); + break; + default: + break; + } + } else { + u16 srcval, mask; + u8 shift; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = shift & 0xF; + srcval = fetch_data_word(srcoffset); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + switch (rh) { + case 5: + store_data_word(srcoffset, srcval | mask); + break; + case 6: + store_data_word(srcoffset, srcval & ~mask); + break; + case 7: + store_data_word(srcoffset, srcval ^ mask); + break; + default: + break; + } + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, mask; + u8 shift; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = shift & 0x1F; + srcval = fetch_data_long(srcoffset); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + switch (rh) { + case 5: + store_data_long(srcoffset, srcval | mask); + break; + case 6: + store_data_long(srcoffset, srcval & ~mask); + break; + case 7: + store_data_long(srcoffset, srcval ^ mask); + break; + default: + break; + } + } else { + u16 srcval, mask; + u8 shift; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = shift & 0xF; + srcval = fetch_data_word(srcoffset); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + switch (rh) { + case 5: + store_data_word(srcoffset, srcval | mask); + break; + case 6: + store_data_word(srcoffset, srcval & ~mask); + break; + case 7: + store_data_word(srcoffset, srcval ^ mask); + break; + default: + break; + } + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg; + u32 mask; + u8 shift; + + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = shift & 0x1F; + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(*srcreg & mask,F_CF); + switch (rh) { + case 5: + *srcreg |= mask; + break; + case 6: + *srcreg &= ~mask; + break; + case 7: + *srcreg ^= mask; + break; + default: + break; + } + } else { + u16 *srcreg; + u16 mask; + u8 shift; + + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shift = fetch_byte_imm(); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = shift & 0xF; + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(*srcreg & mask,F_CF); + switch (rh) { + case 5: + *srcreg |= mask; + break; + case 6: + *srcreg &= ~mask; + break; + case 7: + *srcreg ^= mask; + break; + default: + break; + } + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xbb +****************************************************************************/ +static void x86emuOp2_btc_R(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + int bit,disp; + + START_OF_INSTR(); + DECODE_PRINTF("BTC\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval ^ mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, (u16)(srcval ^ mask)); + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval ^ mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, (u16)(srcval ^ mask)); + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval,mask; + u32 *shiftreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + disp = (s16)*shiftreg >> 5; + srcval = fetch_data_long(srcoffset+disp); + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_long(srcoffset+disp, srcval ^ mask); + } else { + u16 srcval,mask; + u16 *shiftreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + disp = (s16)*shiftreg >> 4; + srcval = fetch_data_word(srcoffset+disp); + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(srcval & mask,F_CF); + store_data_word(srcoffset+disp, (u16)(srcval ^ mask)); + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *srcreg,*shiftreg; + u32 mask; + + srcreg = DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0x1F; + mask = (0x1 << bit); + CONDITIONAL_SET_FLAG(*srcreg & mask,F_CF); + *srcreg ^= mask; + } else { + u16 *srcreg,*shiftreg; + u16 mask; + + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + shiftreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + bit = *shiftreg & 0xF; + mask = (u16)(0x1 << bit); + CONDITIONAL_SET_FLAG(*srcreg & mask,F_CF); + *srcreg ^= mask; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xbc +****************************************************************************/ +static void x86emuOp2_bsf(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("BSF\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch(mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, *dstreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_long(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 0; *dstreg < 32; (*dstreg)++) + if ((srcval >> *dstreg) & 1) break; + } else { + u16 srcval, *dstreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_word(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 0; *dstreg < 16; (*dstreg)++) + if ((srcval >> *dstreg) & 1) break; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, *dstreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_long(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 0; *dstreg < 32; (*dstreg)++) + if ((srcval >> *dstreg) & 1) break; + } else { + u16 srcval, *dstreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_word(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 0; *dstreg < 16; (*dstreg)++) + if ((srcval >> *dstreg) & 1) break; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, *dstreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_long(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 0; *dstreg < 32; (*dstreg)++) + if ((srcval >> *dstreg) & 1) break; + } else { + u16 srcval, *dstreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_word(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 0; *dstreg < 16; (*dstreg)++) + if ((srcval >> *dstreg) & 1) break; + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, *dstreg; + + srcval = *DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 0; *dstreg < 32; (*dstreg)++) + if ((srcval >> *dstreg) & 1) break; + } else { + u16 srcval, *dstreg; + + srcval = *DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 0; *dstreg < 16; (*dstreg)++) + if ((srcval >> *dstreg) & 1) break; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xbd +****************************************************************************/ +static void x86emuOp2_bsr(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("BSR\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch(mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, *dstreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_long(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 31; *dstreg > 0; (*dstreg)--) + if ((srcval >> *dstreg) & 1) break; + } else { + u16 srcval, *dstreg; + + srcoffset = decode_rm00_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_word(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 15; *dstreg > 0; (*dstreg)--) + if ((srcval >> *dstreg) & 1) break; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, *dstreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_long(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 31; *dstreg > 0; (*dstreg)--) + if ((srcval >> *dstreg) & 1) break; + } else { + u16 srcval, *dstreg; + + srcoffset = decode_rm01_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_word(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 15; *dstreg > 0; (*dstreg)--) + if ((srcval >> *dstreg) & 1) break; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, *dstreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_long(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 31; *dstreg > 0; (*dstreg)--) + if ((srcval >> *dstreg) & 1) break; + } else { + u16 srcval, *dstreg; + + srcoffset = decode_rm10_address(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + srcval = fetch_data_word(srcoffset); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 15; *dstreg > 0; (*dstreg)--) + if ((srcval >> *dstreg) & 1) break; + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 srcval, *dstreg; + + srcval = *DECODE_RM_LONG_REGISTER(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 31; *dstreg > 0; (*dstreg)--) + if ((srcval >> *dstreg) & 1) break; + } else { + u16 srcval, *dstreg; + + srcval = *DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF(","); + dstreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + CONDITIONAL_SET_FLAG(srcval == 0, F_ZF); + for(*dstreg = 15; *dstreg > 0; (*dstreg)--) + if ((srcval >> *dstreg) & 1) break; + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xbe +****************************************************************************/ +static void x86emuOp2_movsx_byte_R_RM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + + START_OF_INSTR(); + DECODE_PRINTF("MOVSX\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = (s32)((s8)fetch_data_byte(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = (s16)((s8)fetch_data_byte(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 1: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = (s32)((s8)fetch_data_byte(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = (s16)((s8)fetch_data_byte(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 2: + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u32 srcval; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = (s32)((s8)fetch_data_byte(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } else { + u16 *destreg; + u16 srcval; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = (s16)((s8)fetch_data_byte(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + } + break; + case 3: /* register to register */ + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + u32 *destreg; + u8 *srcreg; + + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = (s32)((s8)*srcreg); + } else { + u16 *destreg; + u8 *srcreg; + + destreg = DECODE_RM_WORD_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_BYTE_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = (s16)((s8)*srcreg); + } + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/**************************************************************************** +REMARKS: +Handles opcode 0x0f,0xbf +****************************************************************************/ +static void x86emuOp2_movsx_word_R_RM(u8 X86EMU_UNUSED(op2)) +{ + int mod, rl, rh; + uint srcoffset; + u32 *destreg; + u32 srcval; + u16 *srcreg; + + START_OF_INSTR(); + DECODE_PRINTF("MOVSX\t"); + FETCH_DECODE_MODRM(mod, rh, rl); + switch (mod) { + case 0: + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm00_address(rl); + srcval = (s32)((s16)fetch_data_word(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 1: + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm01_address(rl); + srcval = (s32)((s16)fetch_data_word(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 2: + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcoffset = decode_rm10_address(rl); + srcval = (s32)((s16)fetch_data_word(srcoffset)); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = srcval; + break; + case 3: /* register to register */ + destreg = DECODE_RM_LONG_REGISTER(rh); + DECODE_PRINTF(","); + srcreg = DECODE_RM_WORD_REGISTER(rl); + DECODE_PRINTF("\n"); + TRACE_AND_STEP(); + *destreg = (s32)((s16)*srcreg); + break; + } + DECODE_CLEAR_SEGOVR(); + END_OF_INSTR(); +} + +/*************************************************************************** + * Double byte operation code table: + **************************************************************************/ +void (*x86emu_optab2[256])(u8) = +{ +/* 0x00 */ x86emuOp2_illegal_op, /* Group F (ring 0 PM) */ +/* 0x01 */ x86emuOp2_illegal_op, /* Group G (ring 0 PM) */ +/* 0x02 */ x86emuOp2_illegal_op, /* lar (ring 0 PM) */ +/* 0x03 */ x86emuOp2_illegal_op, /* lsl (ring 0 PM) */ +/* 0x04 */ x86emuOp2_illegal_op, +/* 0x05 */ x86emuOp2_illegal_op, /* loadall (undocumented) */ +/* 0x06 */ x86emuOp2_illegal_op, /* clts (ring 0 PM) */ +/* 0x07 */ x86emuOp2_illegal_op, /* loadall (undocumented) */ +/* 0x08 */ x86emuOp2_illegal_op, /* invd (ring 0 PM) */ +/* 0x09 */ x86emuOp2_illegal_op, /* wbinvd (ring 0 PM) */ +/* 0x0a */ x86emuOp2_illegal_op, +/* 0x0b */ x86emuOp2_illegal_op, +/* 0x0c */ x86emuOp2_illegal_op, +/* 0x0d */ x86emuOp2_illegal_op, +/* 0x0e */ x86emuOp2_illegal_op, +/* 0x0f */ x86emuOp2_illegal_op, + +/* 0x10 */ x86emuOp2_illegal_op, +/* 0x11 */ x86emuOp2_illegal_op, +/* 0x12 */ x86emuOp2_illegal_op, +/* 0x13 */ x86emuOp2_illegal_op, +/* 0x14 */ x86emuOp2_illegal_op, +/* 0x15 */ x86emuOp2_illegal_op, +/* 0x16 */ x86emuOp2_illegal_op, +/* 0x17 */ x86emuOp2_illegal_op, +/* 0x18 */ x86emuOp2_illegal_op, +/* 0x19 */ x86emuOp2_illegal_op, +/* 0x1a */ x86emuOp2_illegal_op, +/* 0x1b */ x86emuOp2_illegal_op, +/* 0x1c */ x86emuOp2_illegal_op, +/* 0x1d */ x86emuOp2_illegal_op, +/* 0x1e */ x86emuOp2_illegal_op, +/* 0x1f */ x86emuOp2_illegal_op, + +/* 0x20 */ x86emuOp2_illegal_op, /* mov reg32,creg (ring 0 PM) */ +/* 0x21 */ x86emuOp2_illegal_op, /* mov reg32,dreg (ring 0 PM) */ +/* 0x22 */ x86emuOp2_illegal_op, /* mov creg,reg32 (ring 0 PM) */ +/* 0x23 */ x86emuOp2_illegal_op, /* mov dreg,reg32 (ring 0 PM) */ +/* 0x24 */ x86emuOp2_illegal_op, /* mov reg32,treg (ring 0 PM) */ +/* 0x25 */ x86emuOp2_illegal_op, +/* 0x26 */ x86emuOp2_illegal_op, /* mov treg,reg32 (ring 0 PM) */ +/* 0x27 */ x86emuOp2_illegal_op, +/* 0x28 */ x86emuOp2_illegal_op, +/* 0x29 */ x86emuOp2_illegal_op, +/* 0x2a */ x86emuOp2_illegal_op, +/* 0x2b */ x86emuOp2_illegal_op, +/* 0x2c */ x86emuOp2_illegal_op, +/* 0x2d */ x86emuOp2_illegal_op, +/* 0x2e */ x86emuOp2_illegal_op, +/* 0x2f */ x86emuOp2_illegal_op, + +/* 0x30 */ x86emuOp2_illegal_op, +/* 0x31 */ x86emuOp2_illegal_op, +/* 0x32 */ x86emuOp2_illegal_op, +/* 0x33 */ x86emuOp2_illegal_op, +/* 0x34 */ x86emuOp2_illegal_op, +/* 0x35 */ x86emuOp2_illegal_op, +/* 0x36 */ x86emuOp2_illegal_op, +/* 0x37 */ x86emuOp2_illegal_op, +/* 0x38 */ x86emuOp2_illegal_op, +/* 0x39 */ x86emuOp2_illegal_op, +/* 0x3a */ x86emuOp2_illegal_op, +/* 0x3b */ x86emuOp2_illegal_op, +/* 0x3c */ x86emuOp2_illegal_op, +/* 0x3d */ x86emuOp2_illegal_op, +/* 0x3e */ x86emuOp2_illegal_op, +/* 0x3f */ x86emuOp2_illegal_op, + +/* 0x40 */ x86emuOp2_illegal_op, +/* 0x41 */ x86emuOp2_illegal_op, +/* 0x42 */ x86emuOp2_illegal_op, +/* 0x43 */ x86emuOp2_illegal_op, +/* 0x44 */ x86emuOp2_illegal_op, +/* 0x45 */ x86emuOp2_illegal_op, +/* 0x46 */ x86emuOp2_illegal_op, +/* 0x47 */ x86emuOp2_illegal_op, +/* 0x48 */ x86emuOp2_illegal_op, +/* 0x49 */ x86emuOp2_illegal_op, +/* 0x4a */ x86emuOp2_illegal_op, +/* 0x4b */ x86emuOp2_illegal_op, +/* 0x4c */ x86emuOp2_illegal_op, +/* 0x4d */ x86emuOp2_illegal_op, +/* 0x4e */ x86emuOp2_illegal_op, +/* 0x4f */ x86emuOp2_illegal_op, + +/* 0x50 */ x86emuOp2_illegal_op, +/* 0x51 */ x86emuOp2_illegal_op, +/* 0x52 */ x86emuOp2_illegal_op, +/* 0x53 */ x86emuOp2_illegal_op, +/* 0x54 */ x86emuOp2_illegal_op, +/* 0x55 */ x86emuOp2_illegal_op, +/* 0x56 */ x86emuOp2_illegal_op, +/* 0x57 */ x86emuOp2_illegal_op, +/* 0x58 */ x86emuOp2_illegal_op, +/* 0x59 */ x86emuOp2_illegal_op, +/* 0x5a */ x86emuOp2_illegal_op, +/* 0x5b */ x86emuOp2_illegal_op, +/* 0x5c */ x86emuOp2_illegal_op, +/* 0x5d */ x86emuOp2_illegal_op, +/* 0x5e */ x86emuOp2_illegal_op, +/* 0x5f */ x86emuOp2_illegal_op, + +/* 0x60 */ x86emuOp2_illegal_op, +/* 0x61 */ x86emuOp2_illegal_op, +/* 0x62 */ x86emuOp2_illegal_op, +/* 0x63 */ x86emuOp2_illegal_op, +/* 0x64 */ x86emuOp2_illegal_op, +/* 0x65 */ x86emuOp2_illegal_op, +/* 0x66 */ x86emuOp2_illegal_op, +/* 0x67 */ x86emuOp2_illegal_op, +/* 0x68 */ x86emuOp2_illegal_op, +/* 0x69 */ x86emuOp2_illegal_op, +/* 0x6a */ x86emuOp2_illegal_op, +/* 0x6b */ x86emuOp2_illegal_op, +/* 0x6c */ x86emuOp2_illegal_op, +/* 0x6d */ x86emuOp2_illegal_op, +/* 0x6e */ x86emuOp2_illegal_op, +/* 0x6f */ x86emuOp2_illegal_op, + +/* 0x70 */ x86emuOp2_illegal_op, +/* 0x71 */ x86emuOp2_illegal_op, +/* 0x72 */ x86emuOp2_illegal_op, +/* 0x73 */ x86emuOp2_illegal_op, +/* 0x74 */ x86emuOp2_illegal_op, +/* 0x75 */ x86emuOp2_illegal_op, +/* 0x76 */ x86emuOp2_illegal_op, +/* 0x77 */ x86emuOp2_illegal_op, +/* 0x78 */ x86emuOp2_illegal_op, +/* 0x79 */ x86emuOp2_illegal_op, +/* 0x7a */ x86emuOp2_illegal_op, +/* 0x7b */ x86emuOp2_illegal_op, +/* 0x7c */ x86emuOp2_illegal_op, +/* 0x7d */ x86emuOp2_illegal_op, +/* 0x7e */ x86emuOp2_illegal_op, +/* 0x7f */ x86emuOp2_illegal_op, + +/* 0x80 */ x86emuOp2_long_jump, +/* 0x81 */ x86emuOp2_long_jump, +/* 0x82 */ x86emuOp2_long_jump, +/* 0x83 */ x86emuOp2_long_jump, +/* 0x84 */ x86emuOp2_long_jump, +/* 0x85 */ x86emuOp2_long_jump, +/* 0x86 */ x86emuOp2_long_jump, +/* 0x87 */ x86emuOp2_long_jump, +/* 0x88 */ x86emuOp2_long_jump, +/* 0x89 */ x86emuOp2_long_jump, +/* 0x8a */ x86emuOp2_long_jump, +/* 0x8b */ x86emuOp2_long_jump, +/* 0x8c */ x86emuOp2_long_jump, +/* 0x8d */ x86emuOp2_long_jump, +/* 0x8e */ x86emuOp2_long_jump, +/* 0x8f */ x86emuOp2_long_jump, + +/* 0x90 */ x86emuOp2_set_byte, +/* 0x91 */ x86emuOp2_set_byte, +/* 0x92 */ x86emuOp2_set_byte, +/* 0x93 */ x86emuOp2_set_byte, +/* 0x94 */ x86emuOp2_set_byte, +/* 0x95 */ x86emuOp2_set_byte, +/* 0x96 */ x86emuOp2_set_byte, +/* 0x97 */ x86emuOp2_set_byte, +/* 0x98 */ x86emuOp2_set_byte, +/* 0x99 */ x86emuOp2_set_byte, +/* 0x9a */ x86emuOp2_set_byte, +/* 0x9b */ x86emuOp2_set_byte, +/* 0x9c */ x86emuOp2_set_byte, +/* 0x9d */ x86emuOp2_set_byte, +/* 0x9e */ x86emuOp2_set_byte, +/* 0x9f */ x86emuOp2_set_byte, + +/* 0xa0 */ x86emuOp2_push_FS, +/* 0xa1 */ x86emuOp2_pop_FS, +/* 0xa2 */ x86emuOp2_illegal_op, +/* 0xa3 */ x86emuOp2_bt_R, +/* 0xa4 */ x86emuOp2_shld_IMM, +/* 0xa5 */ x86emuOp2_shld_CL, +/* 0xa6 */ x86emuOp2_illegal_op, +/* 0xa7 */ x86emuOp2_illegal_op, +/* 0xa8 */ x86emuOp2_push_GS, +/* 0xa9 */ x86emuOp2_pop_GS, +/* 0xaa */ x86emuOp2_illegal_op, +/* 0xab */ x86emuOp2_bts_R, +/* 0xac */ x86emuOp2_shrd_IMM, +/* 0xad */ x86emuOp2_shrd_CL, +/* 0xae */ x86emuOp2_illegal_op, +/* 0xaf */ x86emuOp2_imul_R_RM, + +/* 0xb0 */ x86emuOp2_illegal_op, /* TODO: cmpxchg */ +/* 0xb1 */ x86emuOp2_illegal_op, /* TODO: cmpxchg */ +/* 0xb2 */ x86emuOp2_lss_R_IMM, +/* 0xb3 */ x86emuOp2_btr_R, +/* 0xb4 */ x86emuOp2_lfs_R_IMM, +/* 0xb5 */ x86emuOp2_lgs_R_IMM, +/* 0xb6 */ x86emuOp2_movzx_byte_R_RM, +/* 0xb7 */ x86emuOp2_movzx_word_R_RM, +/* 0xb8 */ x86emuOp2_illegal_op, +/* 0xb9 */ x86emuOp2_illegal_op, +/* 0xba */ x86emuOp2_btX_I, +/* 0xbb */ x86emuOp2_btc_R, +/* 0xbc */ x86emuOp2_bsf, +/* 0xbd */ x86emuOp2_bsr, +/* 0xbe */ x86emuOp2_movsx_byte_R_RM, +/* 0xbf */ x86emuOp2_movsx_word_R_RM, + +/* 0xc0 */ x86emuOp2_illegal_op, /* TODO: xadd */ +/* 0xc1 */ x86emuOp2_illegal_op, /* TODO: xadd */ +/* 0xc2 */ x86emuOp2_illegal_op, +/* 0xc3 */ x86emuOp2_illegal_op, +/* 0xc4 */ x86emuOp2_illegal_op, +/* 0xc5 */ x86emuOp2_illegal_op, +/* 0xc6 */ x86emuOp2_illegal_op, +/* 0xc7 */ x86emuOp2_illegal_op, +/* 0xc8 */ x86emuOp2_illegal_op, /* TODO: bswap */ +/* 0xc9 */ x86emuOp2_illegal_op, /* TODO: bswap */ +/* 0xca */ x86emuOp2_illegal_op, /* TODO: bswap */ +/* 0xcb */ x86emuOp2_illegal_op, /* TODO: bswap */ +/* 0xcc */ x86emuOp2_illegal_op, /* TODO: bswap */ +/* 0xcd */ x86emuOp2_illegal_op, /* TODO: bswap */ +/* 0xce */ x86emuOp2_illegal_op, /* TODO: bswap */ +/* 0xcf */ x86emuOp2_illegal_op, /* TODO: bswap */ + +/* 0xd0 */ x86emuOp2_illegal_op, +/* 0xd1 */ x86emuOp2_illegal_op, +/* 0xd2 */ x86emuOp2_illegal_op, +/* 0xd3 */ x86emuOp2_illegal_op, +/* 0xd4 */ x86emuOp2_illegal_op, +/* 0xd5 */ x86emuOp2_illegal_op, +/* 0xd6 */ x86emuOp2_illegal_op, +/* 0xd7 */ x86emuOp2_illegal_op, +/* 0xd8 */ x86emuOp2_illegal_op, +/* 0xd9 */ x86emuOp2_illegal_op, +/* 0xda */ x86emuOp2_illegal_op, +/* 0xdb */ x86emuOp2_illegal_op, +/* 0xdc */ x86emuOp2_illegal_op, +/* 0xdd */ x86emuOp2_illegal_op, +/* 0xde */ x86emuOp2_illegal_op, +/* 0xdf */ x86emuOp2_illegal_op, + +/* 0xe0 */ x86emuOp2_illegal_op, +/* 0xe1 */ x86emuOp2_illegal_op, +/* 0xe2 */ x86emuOp2_illegal_op, +/* 0xe3 */ x86emuOp2_illegal_op, +/* 0xe4 */ x86emuOp2_illegal_op, +/* 0xe5 */ x86emuOp2_illegal_op, +/* 0xe6 */ x86emuOp2_illegal_op, +/* 0xe7 */ x86emuOp2_illegal_op, +/* 0xe8 */ x86emuOp2_illegal_op, +/* 0xe9 */ x86emuOp2_illegal_op, +/* 0xea */ x86emuOp2_illegal_op, +/* 0xeb */ x86emuOp2_illegal_op, +/* 0xec */ x86emuOp2_illegal_op, +/* 0xed */ x86emuOp2_illegal_op, +/* 0xee */ x86emuOp2_illegal_op, +/* 0xef */ x86emuOp2_illegal_op, + +/* 0xf0 */ x86emuOp2_illegal_op, +/* 0xf1 */ x86emuOp2_illegal_op, +/* 0xf2 */ x86emuOp2_illegal_op, +/* 0xf3 */ x86emuOp2_illegal_op, +/* 0xf4 */ x86emuOp2_illegal_op, +/* 0xf5 */ x86emuOp2_illegal_op, +/* 0xf6 */ x86emuOp2_illegal_op, +/* 0xf7 */ x86emuOp2_illegal_op, +/* 0xf8 */ x86emuOp2_illegal_op, +/* 0xf9 */ x86emuOp2_illegal_op, +/* 0xfa */ x86emuOp2_illegal_op, +/* 0xfb */ x86emuOp2_illegal_op, +/* 0xfc */ x86emuOp2_illegal_op, +/* 0xfd */ x86emuOp2_illegal_op, +/* 0xfe */ x86emuOp2_illegal_op, +/* 0xff */ x86emuOp2_illegal_op, +}; diff --git a/userspace/sdks/nv570/nvalloc/unix/x86emu/prim_ops.c b/userspace/sdks/nv570/nvalloc/unix/x86emu/prim_ops.c new file mode 100644 index 0000000..27c1948 --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/x86emu/prim_ops.c @@ -0,0 +1,2655 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: This file contains the code to implement the primitive +* machine operations used by the emulation code in ops.c +* +* Carry Chain Calculation +* +* This represents a somewhat expensive calculation which is +* apparently required to emulate the setting of the OF and AF flag. +* The latter is not so important, but the former is. The overflow +* flag is the XOR of the top two bits of the carry chain for an +* addition (similar for subtraction). Since we do not want to +* simulate the addition in a bitwise manner, we try to calculate the +* carry chain given the two operands and the result. +* +* So, given the following table, which represents the addition of two +* bits, we can derive a formula for the carry chain. +* +* a b cin r cout +* 0 0 0 0 0 +* 0 0 1 1 0 +* 0 1 0 1 0 +* 0 1 1 0 1 +* 1 0 0 1 0 +* 1 0 1 0 1 +* 1 1 0 0 1 +* 1 1 1 1 1 +* +* Construction of table for cout: +* +* ab +* r \ 00 01 11 10 +* |------------------ +* 0 | 0 1 1 1 +* 1 | 0 0 1 0 +* +* By inspection, one gets: cc = ab + r'(a + b) +* +* That represents alot of operations, but NO CHOICE.... +* +* Borrow Chain Calculation. +* +* The following table represents the subtraction of two bits, from +* which we can derive a formula for the borrow chain. +* +* a b bin r bout +* 0 0 0 0 0 +* 0 0 1 1 1 +* 0 1 0 1 1 +* 0 1 1 0 1 +* 1 0 0 1 0 +* 1 0 1 0 0 +* 1 1 0 0 0 +* 1 1 1 1 1 +* +* Construction of table for cout: +* +* ab +* r \ 00 01 11 10 +* |------------------ +* 0 | 0 1 0 0 +* 1 | 1 1 1 0 +* +* By inspection, one gets: bc = a'b + r(a' + b) +* +****************************************************************************/ + +#define PRIM_OPS_NO_REDEFINE_ASM +#include "x86emu/x86emui.h" + +/*------------------------- Global Variables ------------------------------*/ + +static u32 x86emu_parity_tab[8] = +{ + 0x96696996, + 0x69969669, + 0x69969669, + 0x96696996, + 0x69969669, + 0x96696996, + 0x96696996, + 0x69969669, +}; + +#define PARITY(x) (((x86emu_parity_tab[(x) / 32] >> ((x) % 32)) & 1) == 0) +#define XOR2(x) (((x) ^ ((x)>>1)) & 0x1) + +/*----------------------------- Implementation ----------------------------*/ + +/**************************************************************************** +REMARKS: +Implements the AAA instruction and side effects. +****************************************************************************/ +u16 aaa_word(u16 d) +{ + u16 res; + if ((d & 0xf) > 0x9 || ACCESS_FLAG(F_AF)) { + d += 0x6; + d += 0x100; + SET_FLAG(F_AF); + SET_FLAG(F_CF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + } + res = (u16)(d & 0xFF0F); + CLEAR_FLAG(F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the AAA instruction and side effects. +****************************************************************************/ +u16 aas_word(u16 d) +{ + u16 res; + if ((d & 0xf) > 0x9 || ACCESS_FLAG(F_AF)) { + d -= 0x6; + d -= 0x100; + SET_FLAG(F_AF); + SET_FLAG(F_CF); + } else { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + } + res = (u16)(d & 0xFF0F); + CLEAR_FLAG(F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the AAD instruction and side effects. +****************************************************************************/ +u16 aad_word(u16 d) +{ + u16 l; + u8 hb, lb; + + hb = (u8)((d >> 8) & 0xff); + lb = (u8)((d & 0xff)); + l = (u16)((lb + 10 * hb) & 0xFF); + + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CLEAR_FLAG(F_OF); + CONDITIONAL_SET_FLAG(l & 0x80, F_SF); + CONDITIONAL_SET_FLAG(l == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(l & 0xff), F_PF); + return l; +} + +/**************************************************************************** +REMARKS: +Implements the AAM instruction and side effects. +****************************************************************************/ +u16 aam_word(u8 d) +{ + u16 h, l; + + h = (u16)(d / 10); + l = (u16)(d % 10); + l |= (u16)(h << 8); + + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CLEAR_FLAG(F_OF); + CONDITIONAL_SET_FLAG(l & 0x80, F_SF); + CONDITIONAL_SET_FLAG(l == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(l & 0xff), F_PF); + return l; +} + +/**************************************************************************** +REMARKS: +Implements the ADC instruction and side effects. +****************************************************************************/ +u8 adc_byte(u8 d, u8 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 cc; + + if (ACCESS_FLAG(F_CF)) + res = 1 + d + s; + else + res = d + s; + + CONDITIONAL_SET_FLAG(res & 0x100, F_CF); + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = (s & d) | ((~res) & (s | d)); + CONDITIONAL_SET_FLAG(XOR2(cc >> 6), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the ADC instruction and side effects. +****************************************************************************/ +u16 adc_word(u16 d, u16 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 cc; + + if (ACCESS_FLAG(F_CF)) + res = 1 + d + s; + else + res = d + s; + + CONDITIONAL_SET_FLAG(res & 0x10000, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = (s & d) | ((~res) & (s | d)); + CONDITIONAL_SET_FLAG(XOR2(cc >> 14), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the ADC instruction and side effects. +****************************************************************************/ +u32 adc_long(u32 d, u32 s) +{ + register u32 lo; /* all operands in native machine order */ + register u32 hi; + register u32 res; + register u32 cc; + + if (ACCESS_FLAG(F_CF)) { + lo = 1 + (d & 0xFFFF) + (s & 0xFFFF); + res = 1 + d + s; + } + else { + lo = (d & 0xFFFF) + (s & 0xFFFF); + res = d + s; + } + hi = (lo >> 16) + (d >> 16) + (s >> 16); + + CONDITIONAL_SET_FLAG(hi & 0x10000, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = (s & d) | ((~res) & (s | d)); + CONDITIONAL_SET_FLAG(XOR2(cc >> 30), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the ADD instruction and side effects. +****************************************************************************/ +u8 add_byte(u8 d, u8 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 cc; + + res = d + s; + CONDITIONAL_SET_FLAG(res & 0x100, F_CF); + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = (s & d) | ((~res) & (s | d)); + CONDITIONAL_SET_FLAG(XOR2(cc >> 6), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the ADD instruction and side effects. +****************************************************************************/ +u16 add_word(u16 d, u16 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 cc; + + res = d + s; + CONDITIONAL_SET_FLAG(res & 0x10000, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = (s & d) | ((~res) & (s | d)); + CONDITIONAL_SET_FLAG(XOR2(cc >> 14), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the ADD instruction and side effects. +****************************************************************************/ +u32 add_long(u32 d, u32 s) +{ + register u32 lo; /* all operands in native machine order */ + register u32 hi; + register u32 res; + register u32 cc; + + lo = (d & 0xFFFF) + (s & 0xFFFF); + res = d + s; + hi = (lo >> 16) + (d >> 16) + (s >> 16); + + CONDITIONAL_SET_FLAG(hi & 0x10000, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = (s & d) | ((~res) & (s | d)); + CONDITIONAL_SET_FLAG(XOR2(cc >> 30), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + + return res; +} + +/**************************************************************************** +REMARKS: +Implements the AND instruction and side effects. +****************************************************************************/ +u8 and_byte(u8 d, u8 s) +{ + register u8 res; /* all operands in native machine order */ + + res = d & s; + + /* set the flags */ + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res), F_PF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the AND instruction and side effects. +****************************************************************************/ +u16 and_word(u16 d, u16 s) +{ + register u16 res; /* all operands in native machine order */ + + res = d & s; + + /* set the flags */ + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the AND instruction and side effects. +****************************************************************************/ +u32 and_long(u32 d, u32 s) +{ + register u32 res; /* all operands in native machine order */ + + res = d & s; + + /* set the flags */ + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the CMP instruction and side effects. +****************************************************************************/ +u8 cmp_byte(u8 d, u8 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - s; + CLEAR_FLAG(F_CF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x80, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 6), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return d; +} + +/**************************************************************************** +REMARKS: +Implements the CMP instruction and side effects. +****************************************************************************/ +u16 cmp_word(u16 d, u16 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - s; + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x8000, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 14), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return d; +} + +/**************************************************************************** +REMARKS: +Implements the CMP instruction and side effects. +****************************************************************************/ +u32 cmp_long(u32 d, u32 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - s; + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x80000000, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 30), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return d; +} + +/**************************************************************************** +REMARKS: +Implements the DAA instruction and side effects. +****************************************************************************/ +u8 daa_byte(u8 d) +{ + u32 res = d; + if ((d & 0xf) > 9 || ACCESS_FLAG(F_AF)) { + res += 6; + SET_FLAG(F_AF); + } + if (res > 0x9F || ACCESS_FLAG(F_CF)) { + res += 0x60; + SET_FLAG(F_CF); + } + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG((res & 0xFF) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the DAS instruction and side effects. +****************************************************************************/ +u8 das_byte(u8 d) +{ + if ((d & 0xf) > 9 || ACCESS_FLAG(F_AF)) { + d -= 6; + SET_FLAG(F_AF); + } + if (d > 0x9F || ACCESS_FLAG(F_CF)) { + d -= 0x60; + SET_FLAG(F_CF); + } + CONDITIONAL_SET_FLAG(d & 0x80, F_SF); + CONDITIONAL_SET_FLAG(d == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(d & 0xff), F_PF); + return d; +} + +/**************************************************************************** +REMARKS: +Implements the DEC instruction and side effects. +****************************************************************************/ +u8 dec_byte(u8 d) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - 1; + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + /* based on sub_byte, uses s==1. */ + bc = (res & (~d | 1)) | (~d & 1); + /* carry flag unchanged */ + CONDITIONAL_SET_FLAG(XOR2(bc >> 6), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the DEC instruction and side effects. +****************************************************************************/ +u16 dec_word(u16 d) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - 1; + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + /* based on the sub_byte routine, with s==1 */ + bc = (res & (~d | 1)) | (~d & 1); + /* carry flag unchanged */ + CONDITIONAL_SET_FLAG(XOR2(bc >> 14), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the DEC instruction and side effects. +****************************************************************************/ +u32 dec_long(u32 d) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - 1; + + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | 1)) | (~d & 1); + /* carry flag unchanged */ + CONDITIONAL_SET_FLAG(XOR2(bc >> 30), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the INC instruction and side effects. +****************************************************************************/ +u8 inc_byte(u8 d) +{ + register u32 res; /* all operands in native machine order */ + register u32 cc; + + res = d + 1; + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = ((1 & d) | (~res)) & (1 | d); + CONDITIONAL_SET_FLAG(XOR2(cc >> 6), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the INC instruction and side effects. +****************************************************************************/ +u16 inc_word(u16 d) +{ + register u32 res; /* all operands in native machine order */ + register u32 cc; + + res = d + 1; + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = (1 & d) | ((~res) & (1 | d)); + CONDITIONAL_SET_FLAG(XOR2(cc >> 14), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the INC instruction and side effects. +****************************************************************************/ +u32 inc_long(u32 d) +{ + register u32 res; /* all operands in native machine order */ + register u32 cc; + + res = d + 1; + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the carry chain SEE NOTE AT TOP. */ + cc = (1 & d) | ((~res) & (1 | d)); + CONDITIONAL_SET_FLAG(XOR2(cc >> 30), F_OF); + CONDITIONAL_SET_FLAG(cc & 0x8, F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the OR instruction and side effects. +****************************************************************************/ +u8 or_byte(u8 d, u8 s) +{ + register u8 res; /* all operands in native machine order */ + + res = d | s; + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res), F_PF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the OR instruction and side effects. +****************************************************************************/ +u16 or_word(u16 d, u16 s) +{ + register u16 res; /* all operands in native machine order */ + + res = d | s; + /* set the carry flag to be bit 8 */ + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the OR instruction and side effects. +****************************************************************************/ +u32 or_long(u32 d, u32 s) +{ + register u32 res; /* all operands in native machine order */ + + res = d | s; + + /* set the carry flag to be bit 8 */ + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the OR instruction and side effects. +****************************************************************************/ +u8 neg_byte(u8 s) +{ + register u8 res; + register u8 bc; + + CONDITIONAL_SET_FLAG(s != 0, F_CF); + res = (u8)-s; + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res), F_PF); + /* calculate the borrow chain --- modified such that d=0. + substitutiing d=0 into bc= res&(~d|s)|(~d&s); + (the one used for sub) and simplifying, since ~d=0xff..., + ~d|s == 0xffff..., and res&0xfff... == res. Similarly + ~d&s == s. So the simplified result is: */ + bc = res | s; + CONDITIONAL_SET_FLAG(XOR2(bc >> 6), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the OR instruction and side effects. +****************************************************************************/ +u16 neg_word(u16 s) +{ + register u16 res; + register u16 bc; + + CONDITIONAL_SET_FLAG(s != 0, F_CF); + res = (u16)-s; + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain --- modified such that d=0. + substitutiing d=0 into bc= res&(~d|s)|(~d&s); + (the one used for sub) and simplifying, since ~d=0xff..., + ~d|s == 0xffff..., and res&0xfff... == res. Similarly + ~d&s == s. So the simplified result is: */ + bc = res | s; + CONDITIONAL_SET_FLAG(XOR2(bc >> 14), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the OR instruction and side effects. +****************************************************************************/ +u32 neg_long(u32 s) +{ + register u32 res; + register u32 bc; + + CONDITIONAL_SET_FLAG(s != 0, F_CF); + res = (u32)-s; + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain --- modified such that d=0. + substitutiing d=0 into bc= res&(~d|s)|(~d&s); + (the one used for sub) and simplifying, since ~d=0xff..., + ~d|s == 0xffff..., and res&0xfff... == res. Similarly + ~d&s == s. So the simplified result is: */ + bc = res | s; + CONDITIONAL_SET_FLAG(XOR2(bc >> 30), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the NOT instruction and side effects. +****************************************************************************/ +u8 not_byte(u8 s) +{ + return ~s; +} + +/**************************************************************************** +REMARKS: +Implements the NOT instruction and side effects. +****************************************************************************/ +u16 not_word(u16 s) +{ + return ~s; +} + +/**************************************************************************** +REMARKS: +Implements the NOT instruction and side effects. +****************************************************************************/ +u32 not_long(u32 s) +{ + return ~s; +} + +/**************************************************************************** +REMARKS: +Implements the RCL instruction and side effects. +****************************************************************************/ +u8 rcl_byte(u8 d, u8 s) +{ + register unsigned int res, cnt, mask, cf; + + /* s is the rotate distance. It varies from 0 - 8. */ + /* have + + CF B_7 B_6 B_5 B_4 B_3 B_2 B_1 B_0 + + want to rotate through the carry by "s" bits. We could + loop, but that's inefficient. So the width is 9, + and we split into three parts: + + The new carry flag (was B_n) + the stuff in B_n-1 .. B_0 + the stuff in B_7 .. B_n+1 + + The new rotate is done mod 9, and given this, + for a rotation of n bits (mod 9) the new carry flag is + then located n bits from the MSB. The low part is + then shifted up cnt bits, and the high part is or'd + in. Using CAPS for new values, and lowercase for the + original values, this can be expressed as: + + IF n > 0 + 1) CF <- b_(8-n) + 2) B_(7) .. B_(n) <- b_(8-(n+1)) .. b_0 + 3) B_(n-1) <- cf + 4) B_(n-2) .. B_0 <- b_7 .. b_(8-(n-1)) + */ + res = d; + if ((cnt = s % 9) != 0) { + /* extract the new CARRY FLAG. */ + /* CF <- b_(8-n) */ + cf = (d >> (8 - cnt)) & 0x1; + + /* get the low stuff which rotated + into the range B_7 .. B_cnt */ + /* B_(7) .. B_(n) <- b_(8-(n+1)) .. b_0 */ + /* note that the right hand side done by the mask */ + res = (d << cnt) & 0xff; + + /* now the high stuff which rotated around + into the positions B_cnt-2 .. B_0 */ + /* B_(n-2) .. B_0 <- b_7 .. b_(8-(n-1)) */ + /* shift it downward, 7-(n-2) = 9-n positions. + and mask off the result before or'ing in. + */ + mask = (1 << (cnt - 1)) - 1; + res |= (d >> (9 - cnt)) & mask; + + /* if the carry flag was set, or it in. */ + if (ACCESS_FLAG(F_CF)) { /* carry flag is set */ + /* B_(n-1) <- cf */ + res |= 1 << (cnt - 1); + } + /* set the new carry flag, based on the variable "cf" */ + CONDITIONAL_SET_FLAG(cf, F_CF); + /* OVERFLOW is set *IFF* cnt==1, then it is the + xor of CF and the most significant bit. Blecck. */ + /* parenthesized this expression since it appears to + be causing OF to be misset */ + CONDITIONAL_SET_FLAG(cnt == 1 && XOR2(cf + ((res >> 6) & 0x2)), + F_OF); + + } + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the RCL instruction and side effects. +****************************************************************************/ +u16 rcl_word(u16 d, u8 s) +{ + register unsigned int res, cnt, mask, cf; + + res = d; + if ((cnt = s % 17) != 0) { + cf = (d >> (16 - cnt)) & 0x1; + res = (d << cnt) & 0xffff; + mask = (1 << (cnt - 1)) - 1; + res |= (d >> (17 - cnt)) & mask; + if (ACCESS_FLAG(F_CF)) { + res |= 1 << (cnt - 1); + } + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG(cnt == 1 && XOR2(cf + ((res >> 14) & 0x2)), + F_OF); + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the RCL instruction and side effects. +****************************************************************************/ +u32 rcl_long(u32 d, u8 s) +{ + register u32 res, cnt, mask, cf; + + res = d; + if ((cnt = s % 33) != 0) { + cf = (d >> (32 - cnt)) & 0x1; + res = (d << cnt) & 0xffffffff; + mask = (1 << (cnt - 1)) - 1; + res |= (d >> (33 - cnt)) & mask; + if (ACCESS_FLAG(F_CF)) { /* carry flag is set */ + res |= 1 << (cnt - 1); + } + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG(cnt == 1 && XOR2(cf + ((res >> 30) & 0x2)), + F_OF); + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the RCR instruction and side effects. +****************************************************************************/ +u8 rcr_byte(u8 d, u8 s) +{ + u32 res, cnt; + u32 mask, cf, ocf = 0; + + /* rotate right through carry */ + /* + s is the rotate distance. It varies from 0 - 8. + d is the byte object rotated. + + have + + CF B_7 B_6 B_5 B_4 B_3 B_2 B_1 B_0 + + The new rotate is done mod 9, and given this, + for a rotation of n bits (mod 9) the new carry flag is + then located n bits from the LSB. The low part is + then shifted up cnt bits, and the high part is or'd + in. Using CAPS for new values, and lowercase for the + original values, this can be expressed as: + + IF n > 0 + 1) CF <- b_(n-1) + 2) B_(8-(n+1)) .. B_(0) <- b_(7) .. b_(n) + 3) B_(8-n) <- cf + 4) B_(7) .. B_(8-(n-1)) <- b_(n-2) .. b_(0) + */ + res = d; + if ((cnt = s % 9) != 0) { + /* extract the new CARRY FLAG. */ + /* CF <- b_(n-1) */ + if (cnt == 1) { + cf = d & 0x1; + /* note hackery here. Access_flag(..) evaluates to either + 0 if flag not set + non-zero if flag is set. + doing access_flag(..) != 0 casts that into either + 0..1 in any representation of the flags register + (i.e. packed bit array or unpacked.) + */ + ocf = ACCESS_FLAG(F_CF) != 0; + } else + cf = (d >> (cnt - 1)) & 0x1; + + /* B_(8-(n+1)) .. B_(0) <- b_(7) .. b_n */ + /* note that the right hand side done by the mask + This is effectively done by shifting the + object to the right. The result must be masked, + in case the object came in and was treated + as a negative number. Needed??? */ + + mask = (1 << (8 - cnt)) - 1; + res = (d >> cnt) & mask; + + /* now the high stuff which rotated around + into the positions B_cnt-2 .. B_0 */ + /* B_(7) .. B_(8-(n-1)) <- b_(n-2) .. b_(0) */ + /* shift it downward, 7-(n-2) = 9-n positions. + and mask off the result before or'ing in. + */ + res |= (d << (9 - cnt)); + + /* if the carry flag was set, or it in. */ + if (ACCESS_FLAG(F_CF)) { /* carry flag is set */ + /* B_(8-n) <- cf */ + res |= 1 << (8 - cnt); + } + /* set the new carry flag, based on the variable "cf" */ + CONDITIONAL_SET_FLAG(cf, F_CF); + /* OVERFLOW is set *IFF* cnt==1, then it is the + xor of CF and the most significant bit. Blecck. */ + /* parenthesized... */ + if (cnt == 1) { + CONDITIONAL_SET_FLAG(XOR2(ocf + ((d >> 6) & 0x2)), + F_OF); + } + } + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the RCR instruction and side effects. +****************************************************************************/ +u16 rcr_word(u16 d, u8 s) +{ + u32 res, cnt; + u32 mask, cf, ocf = 0; + + /* rotate right through carry */ + res = d; + if ((cnt = s % 17) != 0) { + if (cnt == 1) { + cf = d & 0x1; + ocf = ACCESS_FLAG(F_CF) != 0; + } else + cf = (d >> (cnt - 1)) & 0x1; + mask = (1 << (16 - cnt)) - 1; + res = (d >> cnt) & mask; + res |= (d << (17 - cnt)); + if (ACCESS_FLAG(F_CF)) { + res |= 1 << (16 - cnt); + } + CONDITIONAL_SET_FLAG(cf, F_CF); + if (cnt == 1) { + CONDITIONAL_SET_FLAG(XOR2(ocf + ((d >> 14) & 0x2)), + F_OF); + } + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the RCR instruction and side effects. +****************************************************************************/ +u32 rcr_long(u32 d, u8 s) +{ + u32 res, cnt; + u32 mask, cf, ocf = 0; + + /* rotate right through carry */ + res = d; + if ((cnt = s % 33) != 0) { + if (cnt == 1) { + cf = d & 0x1; + ocf = ACCESS_FLAG(F_CF) != 0; + } else + cf = (d >> (cnt - 1)) & 0x1; + mask = (1 << (32 - cnt)) - 1; + res = (d >> cnt) & mask; + if (cnt != 1) + res |= (d << (33 - cnt)); + if (ACCESS_FLAG(F_CF)) { /* carry flag is set */ + res |= 1 << (32 - cnt); + } + CONDITIONAL_SET_FLAG(cf, F_CF); + if (cnt == 1) { + CONDITIONAL_SET_FLAG(XOR2(ocf + ((d >> 30) & 0x2)), + F_OF); + } + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the ROL instruction and side effects. +****************************************************************************/ +u8 rol_byte(u8 d, u8 s) +{ + register unsigned int res, cnt, mask; + + /* rotate left */ + /* + s is the rotate distance. It varies from 0 - 8. + d is the byte object rotated. + + have + + CF B_7 ... B_0 + + The new rotate is done mod 8. + Much simpler than the "rcl" or "rcr" operations. + + IF n > 0 + 1) B_(7) .. B_(n) <- b_(8-(n+1)) .. b_(0) + 2) B_(n-1) .. B_(0) <- b_(7) .. b_(8-n) + */ + res = d; + if ((cnt = s % 8) != 0) { + /* B_(7) .. B_(n) <- b_(8-(n+1)) .. b_(0) */ + res = (d << cnt); + + /* B_(n-1) .. B_(0) <- b_(7) .. b_(8-n) */ + mask = (1 << cnt) - 1; + res |= (d >> (8 - cnt)) & mask; + + /* set the new carry flag, Note that it is the low order + bit of the result!!! */ + CONDITIONAL_SET_FLAG(res & 0x1, F_CF); + /* OVERFLOW is set *IFF* s==1, then it is the + xor of CF and the most significant bit. Blecck. */ + CONDITIONAL_SET_FLAG(s == 1 && + XOR2((res & 0x1) + ((res >> 6) & 0x2)), + F_OF); + } if (s != 0) { + /* set the new carry flag, Note that it is the low order + bit of the result!!! */ + CONDITIONAL_SET_FLAG(res & 0x1, F_CF); + } + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the ROL instruction and side effects. +****************************************************************************/ +u16 rol_word(u16 d, u8 s) +{ + register unsigned int res, cnt, mask; + + res = d; + if ((cnt = s % 16) != 0) { + res = (d << cnt); + mask = (1 << cnt) - 1; + res |= (d >> (16 - cnt)) & mask; + CONDITIONAL_SET_FLAG(res & 0x1, F_CF); + CONDITIONAL_SET_FLAG(s == 1 && + XOR2((res & 0x1) + ((res >> 14) & 0x2)), + F_OF); + } if (s != 0) { + /* set the new carry flag, Note that it is the low order + bit of the result!!! */ + CONDITIONAL_SET_FLAG(res & 0x1, F_CF); + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the ROL instruction and side effects. +****************************************************************************/ +u32 rol_long(u32 d, u8 s) +{ + register u32 res, cnt, mask; + + res = d; + if ((cnt = s % 32) != 0) { + res = (d << cnt); + mask = (1 << cnt) - 1; + res |= (d >> (32 - cnt)) & mask; + CONDITIONAL_SET_FLAG(res & 0x1, F_CF); + CONDITIONAL_SET_FLAG(s == 1 && + XOR2((res & 0x1) + ((res >> 30) & 0x2)), + F_OF); + } if (s != 0) { + /* set the new carry flag, Note that it is the low order + bit of the result!!! */ + CONDITIONAL_SET_FLAG(res & 0x1, F_CF); + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the ROR instruction and side effects. +****************************************************************************/ +u8 ror_byte(u8 d, u8 s) +{ + register unsigned int res, cnt, mask; + + /* rotate right */ + /* + s is the rotate distance. It varies from 0 - 8. + d is the byte object rotated. + + have + + B_7 ... B_0 + + The rotate is done mod 8. + + IF n > 0 + 1) B_(8-(n+1)) .. B_(0) <- b_(7) .. b_(n) + 2) B_(7) .. B_(8-n) <- b_(n-1) .. b_(0) + */ + res = d; + if ((cnt = s % 8) != 0) { /* not a typo, do nada if cnt==0 */ + /* B_(7) .. B_(8-n) <- b_(n-1) .. b_(0) */ + res = (d << (8 - cnt)); + + /* B_(8-(n+1)) .. B_(0) <- b_(7) .. b_(n) */ + mask = (1 << (8 - cnt)) - 1; + res |= (d >> (cnt)) & mask; + + /* set the new carry flag, Note that it is the low order + bit of the result!!! */ + CONDITIONAL_SET_FLAG(res & 0x80, F_CF); + /* OVERFLOW is set *IFF* s==1, then it is the + xor of the two most significant bits. Blecck. */ + CONDITIONAL_SET_FLAG(s == 1 && XOR2(res >> 6), F_OF); + } else if (s != 0) { + /* set the new carry flag, Note that it is the low order + bit of the result!!! */ + CONDITIONAL_SET_FLAG(res & 0x80, F_CF); + } + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the ROR instruction and side effects. +****************************************************************************/ +u16 ror_word(u16 d, u8 s) +{ + register unsigned int res, cnt, mask; + + res = d; + if ((cnt = s % 16) != 0) { + res = (d << (16 - cnt)); + mask = (1 << (16 - cnt)) - 1; + res |= (d >> (cnt)) & mask; + CONDITIONAL_SET_FLAG(res & 0x8000, F_CF); + CONDITIONAL_SET_FLAG(s == 1 && XOR2(res >> 14), F_OF); + } else if (s != 0) { + /* set the new carry flag, Note that it is the low order + bit of the result!!! */ + CONDITIONAL_SET_FLAG(res & 0x8000, F_CF); + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the ROR instruction and side effects. +****************************************************************************/ +u32 ror_long(u32 d, u8 s) +{ + register u32 res, cnt, mask; + + res = d; + if ((cnt = s % 32) != 0) { + res = (d << (32 - cnt)); + mask = (1 << (32 - cnt)) - 1; + res |= (d >> (cnt)) & mask; + CONDITIONAL_SET_FLAG(res & 0x80000000, F_CF); + CONDITIONAL_SET_FLAG(s == 1 && XOR2(res >> 30), F_OF); + } else if (s != 0) { + /* set the new carry flag, Note that it is the low order + bit of the result!!! */ + CONDITIONAL_SET_FLAG(res & 0x80000000, F_CF); + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the SHL instruction and side effects. +****************************************************************************/ +u8 shl_byte(u8 d, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 8) { + cnt = s % 8; + + /* last bit shifted out goes into carry flag */ + if (cnt > 0) { + res = d << cnt; + cf = d & (1 << (8 - cnt)); + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = (u8) d; + } + + if (cnt == 1) { + /* Needs simplification. */ + CONDITIONAL_SET_FLAG( + (((res & 0x80) == 0x80) ^ + (ACCESS_FLAG(F_CF) != 0)), + /* was (M.x86.R_FLG&F_CF)==F_CF)), */ + F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CONDITIONAL_SET_FLAG((d << (s-1)) & 0x80, F_CF); + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_SF); + SET_FLAG(F_PF); + SET_FLAG(F_ZF); + } + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the SHL instruction and side effects. +****************************************************************************/ +u16 shl_word(u16 d, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 16) { + cnt = s % 16; + if (cnt > 0) { + res = d << cnt; + cf = d & (1 << (16 - cnt)); + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = (u16) d; + } + + if (cnt == 1) { + CONDITIONAL_SET_FLAG( + (((res & 0x8000) == 0x8000) ^ + (ACCESS_FLAG(F_CF) != 0)), + F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CONDITIONAL_SET_FLAG((d << (s-1)) & 0x8000, F_CF); + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_SF); + SET_FLAG(F_PF); + SET_FLAG(F_ZF); + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the SHL instruction and side effects. +****************************************************************************/ +u32 shl_long(u32 d, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 32) { + cnt = s % 32; + if (cnt > 0) { + res = d << cnt; + cf = d & (1 << (32 - cnt)); + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = d; + } + if (cnt == 1) { + CONDITIONAL_SET_FLAG((((res & 0x80000000) == 0x80000000) ^ + (ACCESS_FLAG(F_CF) != 0)), F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CONDITIONAL_SET_FLAG((d << (s-1)) & 0x80000000, F_CF); + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_SF); + SET_FLAG(F_PF); + SET_FLAG(F_ZF); + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the SHR instruction and side effects. +****************************************************************************/ +u8 shr_byte(u8 d, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 8) { + cnt = s % 8; + if (cnt > 0) { + cf = d & (1 << (cnt - 1)); + res = d >> cnt; + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = (u8) d; + } + + if (cnt == 1) { + CONDITIONAL_SET_FLAG(XOR2(res >> 6), F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CONDITIONAL_SET_FLAG((d >> (s-1)) & 0x1, F_CF); + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_SF); + SET_FLAG(F_PF); + SET_FLAG(F_ZF); + } + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the SHR instruction and side effects. +****************************************************************************/ +u16 shr_word(u16 d, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 16) { + cnt = s % 16; + if (cnt > 0) { + cf = d & (1 << (cnt - 1)); + res = d >> cnt; + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = d; + } + + if (cnt == 1) { + CONDITIONAL_SET_FLAG(XOR2(res >> 14), F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + SET_FLAG(F_ZF); + CLEAR_FLAG(F_SF); + CLEAR_FLAG(F_PF); + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the SHR instruction and side effects. +****************************************************************************/ +u32 shr_long(u32 d, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 32) { + cnt = s % 32; + if (cnt > 0) { + cf = d & (1 << (cnt - 1)); + res = d >> cnt; + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = d; + } + if (cnt == 1) { + CONDITIONAL_SET_FLAG(XOR2(res >> 30), F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + SET_FLAG(F_ZF); + CLEAR_FLAG(F_SF); + CLEAR_FLAG(F_PF); + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the SAR instruction and side effects. +****************************************************************************/ +u8 sar_byte(u8 d, u8 s) +{ + unsigned int cnt, res, cf, mask, sf; + + res = d; + sf = d & 0x80; + cnt = s % 8; + if (cnt > 0 && cnt < 8) { + mask = (1 << (8 - cnt)) - 1; + cf = d & (1 << (cnt - 1)); + res = (d >> cnt) & mask; + CONDITIONAL_SET_FLAG(cf, F_CF); + if (sf) { + res |= ~mask; + } + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + } else if (cnt >= 8) { + if (sf) { + res = 0xff; + SET_FLAG(F_CF); + CLEAR_FLAG(F_ZF); + SET_FLAG(F_SF); + SET_FLAG(F_PF); + } else { + res = 0; + CLEAR_FLAG(F_CF); + SET_FLAG(F_ZF); + CLEAR_FLAG(F_SF); + CLEAR_FLAG(F_PF); + } + } + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the SAR instruction and side effects. +****************************************************************************/ +u16 sar_word(u16 d, u8 s) +{ + unsigned int cnt, res, cf, mask, sf; + + sf = d & 0x8000; + cnt = s % 16; + res = d; + if (cnt > 0 && cnt < 16) { + mask = (1 << (16 - cnt)) - 1; + cf = d & (1 << (cnt - 1)); + res = (d >> cnt) & mask; + CONDITIONAL_SET_FLAG(cf, F_CF); + if (sf) { + res |= ~mask; + } + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else if (cnt >= 16) { + if (sf) { + res = 0xffff; + SET_FLAG(F_CF); + CLEAR_FLAG(F_ZF); + SET_FLAG(F_SF); + SET_FLAG(F_PF); + } else { + res = 0; + CLEAR_FLAG(F_CF); + SET_FLAG(F_ZF); + CLEAR_FLAG(F_SF); + CLEAR_FLAG(F_PF); + } + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the SAR instruction and side effects. +****************************************************************************/ +u32 sar_long(u32 d, u8 s) +{ + u32 cnt, res, cf, mask, sf; + + sf = d & 0x80000000; + cnt = s % 32; + res = d; + if (cnt > 0 && cnt < 32) { + mask = (1 << (32 - cnt)) - 1; + cf = d & (1 << (cnt - 1)); + res = (d >> cnt) & mask; + CONDITIONAL_SET_FLAG(cf, F_CF); + if (sf) { + res |= ~mask; + } + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else if (cnt >= 32) { + if (sf) { + res = 0xffffffff; + SET_FLAG(F_CF); + CLEAR_FLAG(F_ZF); + SET_FLAG(F_SF); + SET_FLAG(F_PF); + } else { + res = 0; + CLEAR_FLAG(F_CF); + SET_FLAG(F_ZF); + CLEAR_FLAG(F_SF); + CLEAR_FLAG(F_PF); + } + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the SHLD instruction and side effects. +****************************************************************************/ +u16 shld_word (u16 d, u16 fill, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 16) { + cnt = s % 16; + if (cnt > 0) { + res = (d << cnt) | (fill >> (16-cnt)); + cf = d & (1 << (16 - cnt)); + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = d; + } + if (cnt == 1) { + CONDITIONAL_SET_FLAG((((res & 0x8000) == 0x8000) ^ + (ACCESS_FLAG(F_CF) != 0)), F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CONDITIONAL_SET_FLAG((d << (s-1)) & 0x8000, F_CF); + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_SF); + SET_FLAG(F_PF); + SET_FLAG(F_ZF); + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the SHLD instruction and side effects. +****************************************************************************/ +u32 shld_long (u32 d, u32 fill, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 32) { + cnt = s % 32; + if (cnt > 0) { + res = (d << cnt) | (fill >> (32-cnt)); + cf = d & (1 << (32 - cnt)); + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = d; + } + if (cnt == 1) { + CONDITIONAL_SET_FLAG((((res & 0x80000000) == 0x80000000) ^ + (ACCESS_FLAG(F_CF) != 0)), F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CONDITIONAL_SET_FLAG((d << (s-1)) & 0x80000000, F_CF); + CLEAR_FLAG(F_OF); + CLEAR_FLAG(F_SF); + SET_FLAG(F_PF); + SET_FLAG(F_ZF); + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the SHRD instruction and side effects. +****************************************************************************/ +u16 shrd_word (u16 d, u16 fill, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 16) { + cnt = s % 16; + if (cnt > 0) { + cf = d & (1 << (cnt - 1)); + res = (d >> cnt) | (fill << (16 - cnt)); + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = d; + } + + if (cnt == 1) { + CONDITIONAL_SET_FLAG(XOR2(res >> 14), F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + SET_FLAG(F_ZF); + CLEAR_FLAG(F_SF); + CLEAR_FLAG(F_PF); + } + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the SHRD instruction and side effects. +****************************************************************************/ +u32 shrd_long (u32 d, u32 fill, u8 s) +{ + unsigned int cnt, res, cf; + + if (s < 32) { + cnt = s % 32; + if (cnt > 0) { + cf = d & (1 << (cnt - 1)); + res = (d >> cnt) | (fill << (32 - cnt)); + CONDITIONAL_SET_FLAG(cf, F_CF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + } else { + res = d; + } + if (cnt == 1) { + CONDITIONAL_SET_FLAG(XOR2(res >> 30), F_OF); + } else { + CLEAR_FLAG(F_OF); + } + } else { + res = 0; + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + SET_FLAG(F_ZF); + CLEAR_FLAG(F_SF); + CLEAR_FLAG(F_PF); + } + return res; +} + +/**************************************************************************** +REMARKS: +Implements the SBB instruction and side effects. +****************************************************************************/ +u8 sbb_byte(u8 d, u8 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + if (ACCESS_FLAG(F_CF)) + res = d - s - 1; + else + res = d - s; + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x80, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 6), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the SBB instruction and side effects. +****************************************************************************/ +u16 sbb_word(u16 d, u16 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + if (ACCESS_FLAG(F_CF)) + res = d - s - 1; + else + res = d - s; + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x8000, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 14), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the SBB instruction and side effects. +****************************************************************************/ +u32 sbb_long(u32 d, u32 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + if (ACCESS_FLAG(F_CF)) + res = d - s - 1; + else + res = d - s; + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x80000000, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 30), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the SUB instruction and side effects. +****************************************************************************/ +u8 sub_byte(u8 d, u8 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - s; + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG((res & 0xff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x80, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 6), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return (u8)res; +} + +/**************************************************************************** +REMARKS: +Implements the SUB instruction and side effects. +****************************************************************************/ +u16 sub_word(u16 d, u16 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - s; + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG((res & 0xffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x8000, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 14), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return (u16)res; +} + +/**************************************************************************** +REMARKS: +Implements the SUB instruction and side effects. +****************************************************************************/ +u32 sub_long(u32 d, u32 s) +{ + register u32 res; /* all operands in native machine order */ + register u32 bc; + + res = d - s; + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG((res & 0xffffffff) == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + + /* calculate the borrow chain. See note at top */ + bc = (res & (~d | s)) | (~d & s); + CONDITIONAL_SET_FLAG(bc & 0x80000000, F_CF); + CONDITIONAL_SET_FLAG(XOR2(bc >> 30), F_OF); + CONDITIONAL_SET_FLAG(bc & 0x8, F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the TEST instruction and side effects. +****************************************************************************/ +void test_byte(u8 d, u8 s) +{ + register u32 res; /* all operands in native machine order */ + + res = d & s; + + CLEAR_FLAG(F_OF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + /* AF == dont care */ + CLEAR_FLAG(F_CF); +} + +/**************************************************************************** +REMARKS: +Implements the TEST instruction and side effects. +****************************************************************************/ +void test_word(u16 d, u16 s) +{ + register u32 res; /* all operands in native machine order */ + + res = d & s; + + CLEAR_FLAG(F_OF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + /* AF == dont care */ + CLEAR_FLAG(F_CF); +} + +/**************************************************************************** +REMARKS: +Implements the TEST instruction and side effects. +****************************************************************************/ +void test_long(u32 d, u32 s) +{ + register u32 res; /* all operands in native machine order */ + + res = d & s; + + CLEAR_FLAG(F_OF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + /* AF == dont care */ + CLEAR_FLAG(F_CF); +} + +/**************************************************************************** +REMARKS: +Implements the XOR instruction and side effects. +****************************************************************************/ +u8 xor_byte(u8 d, u8 s) +{ + register u8 res; /* all operands in native machine order */ + + res = d ^ s; + CLEAR_FLAG(F_OF); + CONDITIONAL_SET_FLAG(res & 0x80, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res), F_PF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the XOR instruction and side effects. +****************************************************************************/ +u16 xor_word(u16 d, u16 s) +{ + register u16 res; /* all operands in native machine order */ + + res = d ^ s; + CLEAR_FLAG(F_OF); + CONDITIONAL_SET_FLAG(res & 0x8000, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the XOR instruction and side effects. +****************************************************************************/ +u32 xor_long(u32 d, u32 s) +{ + register u32 res; /* all operands in native machine order */ + + res = d ^ s; + CLEAR_FLAG(F_OF); + CONDITIONAL_SET_FLAG(res & 0x80000000, F_SF); + CONDITIONAL_SET_FLAG(res == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(res & 0xff), F_PF); + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + return res; +} + +/**************************************************************************** +REMARKS: +Implements the IMUL instruction and side effects. +****************************************************************************/ +void imul_byte(u8 s) +{ + s16 res = (s16)((s8)M.x86.R_AL * (s8)s); + + M.x86.R_AX = res; + if (((M.x86.R_AL & 0x80) == 0 && M.x86.R_AH == 0x00) || + ((M.x86.R_AL & 0x80) != 0 && M.x86.R_AH == 0xFF)) { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } else { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } +} + +/**************************************************************************** +REMARKS: +Implements the IMUL instruction and side effects. +****************************************************************************/ +void imul_word(u16 s) +{ + s32 res = (s16)M.x86.R_AX * (s16)s; + + M.x86.R_AX = (u16)res; + M.x86.R_DX = (u16)(res >> 16); + if (((M.x86.R_AX & 0x8000) == 0 && M.x86.R_DX == 0x00) || + ((M.x86.R_AX & 0x8000) != 0 && M.x86.R_DX == 0xFF)) { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } else { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } +} + +/**************************************************************************** +REMARKS: +Implements the IMUL instruction and side effects. +****************************************************************************/ +void imul_long_direct(u32 *res_lo, u32* res_hi,u32 d, u32 s) +{ +#ifdef __HAS_LONG_LONG__ + s64 res = (s64)(s32)d * (s32)s; + + *res_lo = (u32)res; + *res_hi = (u32)(res >> 32); +#else + u32 d_lo,d_hi,d_sign; + u32 s_lo,s_hi,s_sign; + u32 rlo_lo,rlo_hi,rhi_lo; + + if ((d_sign = d & 0x80000000) != 0) + d = -d; + d_lo = d & 0xFFFF; + d_hi = d >> 16; + if ((s_sign = s & 0x80000000) != 0) + s = -s; + s_lo = s & 0xFFFF; + s_hi = s >> 16; + rlo_lo = d_lo * s_lo; + rlo_hi = (d_hi * s_lo + d_lo * s_hi) + (rlo_lo >> 16); + rhi_lo = d_hi * s_hi + (rlo_hi >> 16); + *res_lo = (rlo_hi << 16) | (rlo_lo & 0xFFFF); + *res_hi = rhi_lo; + if (d_sign != s_sign) { + d = ~*res_lo; + s = (((d & 0xFFFF) + 1) >> 16) + (d >> 16); + *res_lo = ~*res_lo+1; + *res_hi = ~*res_hi+(s >> 16); + } +#endif +} + +/**************************************************************************** +REMARKS: +Implements the IMUL instruction and side effects. +****************************************************************************/ +void imul_long(u32 s) +{ + imul_long_direct(&M.x86.R_EAX,&M.x86.R_EDX,M.x86.R_EAX,s); + if (((M.x86.R_EAX & 0x80000000) == 0 && M.x86.R_EDX == 0x00) || + ((M.x86.R_EAX & 0x80000000) != 0 && M.x86.R_EDX == 0xFF)) { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } else { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } +} + +/**************************************************************************** +REMARKS: +Implements the MUL instruction and side effects. +****************************************************************************/ +void mul_byte(u8 s) +{ + u16 res = (u16)(M.x86.R_AL * s); + + M.x86.R_AX = res; + if (M.x86.R_AH == 0) { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } else { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } +} + +/**************************************************************************** +REMARKS: +Implements the MUL instruction and side effects. +****************************************************************************/ +void mul_word(u16 s) +{ + u32 res = M.x86.R_AX * s; + + M.x86.R_AX = (u16)res; + M.x86.R_DX = (u16)(res >> 16); + if (M.x86.R_DX == 0) { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } else { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } +} + +/**************************************************************************** +REMARKS: +Implements the MUL instruction and side effects. +****************************************************************************/ +void mul_long(u32 s) +{ +#ifdef __HAS_LONG_LONG__ + u64 res = (u64)M.x86.R_EAX * s; + + M.x86.R_EAX = (u32)res; + M.x86.R_EDX = (u32)(res >> 32); +#else + u32 a,a_lo,a_hi; + u32 s_lo,s_hi; + u32 rlo_lo,rlo_hi,rhi_lo; + + a = M.x86.R_EAX; + a_lo = a & 0xFFFF; + a_hi = a >> 16; + s_lo = s & 0xFFFF; + s_hi = s >> 16; + rlo_lo = a_lo * s_lo; + rlo_hi = (a_hi * s_lo + a_lo * s_hi) + (rlo_lo >> 16); + rhi_lo = a_hi * s_hi + (rlo_hi >> 16); + M.x86.R_EAX = (rlo_hi << 16) | (rlo_lo & 0xFFFF); + M.x86.R_EDX = rhi_lo; +#endif + + if (M.x86.R_EDX == 0) { + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_OF); + } else { + SET_FLAG(F_CF); + SET_FLAG(F_OF); + } +} + +/**************************************************************************** +REMARKS: +Implements the IDIV instruction and side effects. +****************************************************************************/ +void idiv_byte(u8 s) +{ + s32 dvd, div, mod; + + dvd = (s16)M.x86.R_AX; + if (s == 0) { + x86emu_intr_raise(0); + return; + } + div = dvd / (s8)s; + mod = dvd % (s8)s; + if (abs(div) > 0x7f) { + x86emu_intr_raise(0); + return; + } + M.x86.R_AL = (s8) div; + M.x86.R_AH = (s8) mod; +} + +/**************************************************************************** +REMARKS: +Implements the IDIV instruction and side effects. +****************************************************************************/ +void idiv_word(u16 s) +{ + s32 dvd, div, mod; + + dvd = (((s32)M.x86.R_DX) << 16) | M.x86.R_AX; + if (s == 0) { + x86emu_intr_raise(0); + return; + } + div = dvd / (s16)s; + mod = dvd % (s16)s; + if (abs(div) > 0x7fff) { + x86emu_intr_raise(0); + return; + } + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_SF); + CONDITIONAL_SET_FLAG(div == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(mod & 0xff), F_PF); + + M.x86.R_AX = (u16)div; + M.x86.R_DX = (u16)mod; +} + +/**************************************************************************** +REMARKS: +Implements the IDIV instruction and side effects. +****************************************************************************/ +void idiv_long(u32 s) +{ +#ifdef __HAS_LONG_LONG__ + s64 dvd, div, mod; + + dvd = (((s64)M.x86.R_EDX) << 32) | M.x86.R_EAX; + if (s == 0) { + x86emu_intr_raise(0); + return; + } + div = dvd / (s32)s; + mod = dvd % (s32)s; + if (((u64)div >> 32) != 0 && + ((u64)div >> 32) != 0xFFFFFFFFUL) { + x86emu_intr_raise(0); + return; + } +#else + s32 div = 0, mod; + s32 h_dvd = M.x86.R_EDX; + u32 l_dvd = M.x86.R_EAX; + u32 abs_s = s & 0x7FFFFFFF; + u32 abs_h_dvd = h_dvd & 0x7FFFFFFF; + u32 h_s = abs_s >> 1; + u32 l_s = abs_s << 31; + int counter = 31; + int carry; + + if (s == 0) { + x86emu_intr_raise(0); + return; + } + do { + div <<= 1; + carry = (l_dvd >= l_s) ? 0 : 1; + + if (abs_h_dvd < (h_s + carry)) { + h_s >>= 1; + l_s = abs_s << (--counter); + continue; + } else { + abs_h_dvd -= (h_s + carry); + l_dvd = carry ? ((0xFFFFFFFF - l_s) + l_dvd + 1) + : (l_dvd - l_s); + h_s >>= 1; + l_s = abs_s << (--counter); + div |= 1; + continue; + } + + } while (counter > -1); + /* overflow */ + if (abs_h_dvd || (l_dvd > abs_s)) { + x86emu_intr_raise(0); + return; + } + /* sign */ + div |= ((h_dvd & 0x10000000) ^ (s & 0x10000000)); + mod = l_dvd; + +#endif + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CLEAR_FLAG(F_SF); + SET_FLAG(F_ZF); + CONDITIONAL_SET_FLAG(PARITY(mod & 0xff), F_PF); + + M.x86.R_EAX = (u32)div; + M.x86.R_EDX = (u32)mod; +} + +/**************************************************************************** +REMARKS: +Implements the DIV instruction and side effects. +****************************************************************************/ +void div_byte(u8 s) +{ + u32 dvd, div, mod; + + dvd = M.x86.R_AX; + if (s == 0) { + x86emu_intr_raise(0); + return; + } + div = dvd / (u8)s; + mod = dvd % (u8)s; + if (div > 0xff) { + x86emu_intr_raise(0); + return; + } + M.x86.R_AL = (u8)div; + M.x86.R_AH = (u8)mod; +} + +/**************************************************************************** +REMARKS: +Implements the DIV instruction and side effects. +****************************************************************************/ +void div_word(u16 s) +{ + u32 dvd, div, mod; + + dvd = (((u32)M.x86.R_DX) << 16) | M.x86.R_AX; + if (s == 0) { + x86emu_intr_raise(0); + return; + } + div = dvd / (u16)s; + mod = dvd % (u16)s; + if (div > 0xffff) { + x86emu_intr_raise(0); + return; + } + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_SF); + CONDITIONAL_SET_FLAG(div == 0, F_ZF); + CONDITIONAL_SET_FLAG(PARITY(mod & 0xff), F_PF); + + M.x86.R_AX = (u16)div; + M.x86.R_DX = (u16)mod; +} + +/**************************************************************************** +REMARKS: +Implements the DIV instruction and side effects. +****************************************************************************/ +void div_long(u32 s) +{ +#ifdef __HAS_LONG_LONG__ + u64 dvd, div, mod; + + dvd = (((u64)M.x86.R_EDX) << 32) | M.x86.R_EAX; + if (s == 0) { + x86emu_intr_raise(0); + return; + } + div = dvd / (u32)s; + mod = dvd % (u32)s; + if (div > 0xffffffffULL) { + x86emu_intr_raise(0); + return; + } +#else + s32 div = 0, mod; + s32 h_dvd = M.x86.R_EDX; + u32 l_dvd = M.x86.R_EAX; + + u32 h_s = s; + u32 l_s = 0; + int counter = 32; + int carry; + + if (s == 0) { + x86emu_intr_raise(0); + return; + } + do { + div <<= 1; + carry = (l_dvd >= l_s) ? 0 : 1; + + if (h_dvd < (h_s + carry)) { + h_s >>= 1; + l_s = s << (--counter); + continue; + } else { + h_dvd -= (h_s + carry); + l_dvd = carry ? ((0xFFFFFFFF - l_s) + l_dvd + 1) + : (l_dvd - l_s); + h_s >>= 1; + l_s = s << (--counter); + div |= 1; + continue; + } + + } while (counter > -1); + /* overflow */ + if (h_dvd || (l_dvd > s)) { + x86emu_intr_raise(0); + return; + } + mod = l_dvd; +#endif + CLEAR_FLAG(F_CF); + CLEAR_FLAG(F_AF); + CLEAR_FLAG(F_SF); + SET_FLAG(F_ZF); + CONDITIONAL_SET_FLAG(PARITY(mod & 0xff), F_PF); + + M.x86.R_EAX = (u32)div; + M.x86.R_EDX = (u32)mod; +} + +/**************************************************************************** +REMARKS: +Implements the IN string instruction and side effects. +****************************************************************************/ +void ins(int size) +{ + int inc = size; + + if (ACCESS_FLAG(F_DF)) { + inc = -size; + } + if (M.x86.mode & (SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE)) { + /* dont care whether REPE or REPNE */ + /* in until CX is ZERO. */ + u32 count = ((M.x86.mode & SYSMODE_PREFIX_DATA) ? + M.x86.R_ECX : M.x86.R_CX); + switch (size) { + case 1: + while (count--) { + store_data_byte_abs(M.x86.R_ES, M.x86.R_DI, + (*sys_inb)(M.x86.R_DX)); + M.x86.R_DI += inc; + } + break; + + case 2: + while (count--) { + store_data_word_abs(M.x86.R_ES, M.x86.R_DI, + (*sys_inw)(M.x86.R_DX)); + M.x86.R_DI += inc; + } + break; + case 4: + while (count--) { + store_data_long_abs(M.x86.R_ES, M.x86.R_DI, + (*sys_inl)(M.x86.R_DX)); + M.x86.R_DI += inc; + break; + } + } + M.x86.R_CX = 0; + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ECX = 0; + } + M.x86.mode &= ~(SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE); + } else { + switch (size) { + case 1: + store_data_byte_abs(M.x86.R_ES, M.x86.R_DI, + (*sys_inb)(M.x86.R_DX)); + break; + case 2: + store_data_word_abs(M.x86.R_ES, M.x86.R_DI, + (*sys_inw)(M.x86.R_DX)); + break; + case 4: + store_data_long_abs(M.x86.R_ES, M.x86.R_DI, + (*sys_inl)(M.x86.R_DX)); + break; + } + M.x86.R_DI += inc; + } +} + +/**************************************************************************** +REMARKS: +Implements the OUT string instruction and side effects. +****************************************************************************/ +void outs(int size) +{ + int inc = size; + + if (ACCESS_FLAG(F_DF)) { + inc = -size; + } + if (M.x86.mode & (SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE)) { + /* dont care whether REPE or REPNE */ + /* out until CX is ZERO. */ + u32 count = ((M.x86.mode & SYSMODE_PREFIX_DATA) ? + M.x86.R_ECX : M.x86.R_CX); + switch (size) { + case 1: + while (count--) { + (*sys_outb)(M.x86.R_DX, + fetch_data_byte_abs(M.x86.R_ES, M.x86.R_SI)); + M.x86.R_SI += inc; + } + break; + + case 2: + while (count--) { + (*sys_outw)(M.x86.R_DX, + fetch_data_word_abs(M.x86.R_ES, M.x86.R_SI)); + M.x86.R_SI += inc; + } + break; + case 4: + while (count--) { + (*sys_outl)(M.x86.R_DX, + fetch_data_long_abs(M.x86.R_ES, M.x86.R_SI)); + M.x86.R_SI += inc; + break; + } + } + M.x86.R_CX = 0; + if (M.x86.mode & SYSMODE_PREFIX_DATA) { + M.x86.R_ECX = 0; + } + M.x86.mode &= ~(SYSMODE_PREFIX_REPE | SYSMODE_PREFIX_REPNE); + } else { + switch (size) { + case 1: + (*sys_outb)(M.x86.R_DX, + fetch_data_byte_abs(M.x86.R_ES, M.x86.R_SI)); + break; + case 2: + (*sys_outw)(M.x86.R_DX, + fetch_data_word_abs(M.x86.R_ES, M.x86.R_SI)); + break; + case 4: + (*sys_outl)(M.x86.R_DX, + fetch_data_long_abs(M.x86.R_ES, M.x86.R_SI)); + break; + } + M.x86.R_SI += inc; + } +} + +/**************************************************************************** +PARAMETERS: +addr - Address to fetch word from + +REMARKS: +Fetches a word from emulator memory using an absolute address. +****************************************************************************/ +u16 mem_access_word(int addr) +{ +DB( if (CHECK_MEM_ACCESS()) + x86emu_check_mem_access(addr);) + return (*sys_rdw)(addr); +} + +/**************************************************************************** +REMARKS: +Pushes a word onto the stack. + +NOTE: Do not inline this, as (*sys_wrX) is already inline! +****************************************************************************/ +void push_word(u16 w) +{ +DB( if (CHECK_SP_ACCESS()) + x86emu_check_sp_access();) + M.x86.R_SP -= 2; + (*sys_wrw)(((u32)M.x86.R_SS << 4) + M.x86.R_SP, w); +} + +/**************************************************************************** +REMARKS: +Pushes a long onto the stack. + +NOTE: Do not inline this, as (*sys_wrX) is already inline! +****************************************************************************/ +void push_long(u32 w) +{ +DB( if (CHECK_SP_ACCESS()) + x86emu_check_sp_access();) + M.x86.R_SP -= 4; + (*sys_wrl)(((u32)M.x86.R_SS << 4) + M.x86.R_SP, w); +} + +/**************************************************************************** +REMARKS: +Pops a word from the stack. + +NOTE: Do not inline this, as (*sys_rdX) is already inline! +****************************************************************************/ +u16 pop_word(void) +{ + register u16 res; + +DB( if (CHECK_SP_ACCESS()) + x86emu_check_sp_access();) + res = (*sys_rdw)(((u32)M.x86.R_SS << 4) + M.x86.R_SP); + M.x86.R_SP += 2; + return res; +} + +/**************************************************************************** +REMARKS: +Pops a long from the stack. + +NOTE: Do not inline this, as (*sys_rdX) is already inline! +****************************************************************************/ +u32 pop_long(void) +{ + register u32 res; + +DB( if (CHECK_SP_ACCESS()) + x86emu_check_sp_access();) + res = (*sys_rdl)(((u32)M.x86.R_SS << 4) + M.x86.R_SP); + M.x86.R_SP += 4; + return res; +} + diff --git a/userspace/sdks/nv570/nvalloc/unix/x86emu/sys.c b/userspace/sdks/nv570/nvalloc/unix/x86emu/sys.c new file mode 100644 index 0000000..9a5527f --- /dev/null +++ b/userspace/sdks/nv570/nvalloc/unix/x86emu/sys.c @@ -0,0 +1,663 @@ +/**************************************************************************** +* +* Realmode X86 Emulator Library +* +* Copyright (C) 1996-1999 SciTech Software, Inc. +* Copyright (C) David Mosberger-Tang +* Copyright (C) 1999 Egbert Eich +* +* ======================================================================== +* +* Permission to use, copy, modify, distribute, and sell this software and +* its documentation for any purpose is hereby granted without fee, +* provided that the above copyright notice appear in all copies and that +* both that copyright notice and this permission notice appear in +* supporting documentation, and that the name of the authors not be used +* in advertising or publicity pertaining to distribution of the software +* without specific, written prior permission. The authors makes no +* representations about the suitability of this software for any purpose. +* It is provided "as is" without express or implied warranty. +* +* THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +* EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR +* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +* PERFORMANCE OF THIS SOFTWARE. +* +* ======================================================================== +* +* Language: ANSI C +* Environment: Any +* Developer: Kendall Bennett +* +* Description: This file includes subroutines which are related to +* programmed I/O and memory access. Included in this module +* are default functions with limited usefulness. For real +* uses these functions will most likely be overriden by the +* user library. +* +****************************************************************************/ + +#include "x86emu/x86emu.h" +#include "x86emu/x86emui.h" +#include "x86emu/regs.h" +#include "x86emu/debug.h" +#include "x86emu/prim_ops.h" +#ifndef NO_SYS_HEADERS +#include <string.h> +#endif +/*------------------------- Global Variables ------------------------------*/ + +X86EMU_sysEnv _X86EMU_env; /* Global emulator machine state */ +X86EMU_intrFuncs _X86EMU_intrTab[256]; + +/*----------------------------- Implementation ----------------------------*/ +#if defined(__alpha__) || defined(__alpha) +/* to cope with broken egcs-1.1.2 :-(((( */ + +#define ALPHA_UALOADS +/* + * inline functions to do unaligned accesses + * from linux/include/asm-alpha/unaligned.h + */ + +/* + * EGCS 1.1 knows about arbitrary unaligned loads. Define some + * packed structures to talk about such things with. + */ + +#if defined(__GNUC__) && ((__GNUC__ > 2) || (__GNUC_MINOR__ >= 91)) +struct __una_u64 { unsigned long x __attribute__((packed)); }; +struct __una_u32 { unsigned int x __attribute__((packed)); }; +struct __una_u16 { unsigned short x __attribute__((packed)); }; +#endif + +static __inline__ unsigned long ldq_u(unsigned long * r11) +{ +#if defined(__GNUC__) && ((__GNUC__ > 2) || (__GNUC_MINOR__ >= 91)) + const struct __una_u64 *ptr = (const struct __una_u64 *) r11; + return ptr->x; +#else + unsigned long r1,r2; + __asm__("ldq_u %0,%3\n\t" + "ldq_u %1,%4\n\t" + "extql %0,%2,%0\n\t" + "extqh %1,%2,%1" + :"=&r" (r1), "=&r" (r2) + :"r" (r11), + "m" (*r11), + "m" (*(const unsigned long *)(7+(char *) r11))); + return r1 | r2; +#endif +} + +static __inline__ unsigned long ldl_u(unsigned int * r11) +{ +#if defined(__GNUC__) && ((__GNUC__ > 2) || (__GNUC_MINOR__ >= 91)) + const struct __una_u32 *ptr = (const struct __una_u32 *) r11; + return ptr->x; +#else + unsigned long r1,r2; + __asm__("ldq_u %0,%3\n\t" + "ldq_u %1,%4\n\t" + "extll %0,%2,%0\n\t" + "extlh %1,%2,%1" + :"=&r" (r1), "=&r" (r2) + :"r" (r11), + "m" (*r11), + "m" (*(const unsigned long *)(3+(char *) r11))); + return r1 | r2; +#endif +} + +static __inline__ unsigned long ldw_u(unsigned short * r11) +{ +#if defined(__GNUC__) && ((__GNUC__ > 2) || (__GNUC_MINOR__ >= 91)) + const struct __una_u16 *ptr = (const struct __una_u16 *) r11; + return ptr->x; +#else + unsigned long r1,r2; + __asm__("ldq_u %0,%3\n\t" + "ldq_u %1,%4\n\t" + "extwl %0,%2,%0\n\t" + "extwh %1,%2,%1" + :"=&r" (r1), "=&r" (r2) + :"r" (r11), + "m" (*r11), + "m" (*(const unsigned long *)(1+(char *) r11))); + return r1 | r2; +#endif +} + +/* + * Elemental unaligned stores + */ + +static __inline__ void stq_u(unsigned long r5, unsigned long * r11) +{ +#if defined(__GNUC__) && ((__GNUC__ > 2) || (__GNUC_MINOR__ >= 91)) + struct __una_u64 *ptr = (struct __una_u64 *) r11; + ptr->x = r5; +#else + unsigned long r1,r2,r3,r4; + + __asm__("ldq_u %3,%1\n\t" + "ldq_u %2,%0\n\t" + "insqh %6,%7,%5\n\t" + "insql %6,%7,%4\n\t" + "mskqh %3,%7,%3\n\t" + "mskql %2,%7,%2\n\t" + "bis %3,%5,%3\n\t" + "bis %2,%4,%2\n\t" + "stq_u %3,%1\n\t" + "stq_u %2,%0" + :"=m" (*r11), + "=m" (*(unsigned long *)(7+(char *) r11)), + "=&r" (r1), "=&r" (r2), "=&r" (r3), "=&r" (r4) + :"r" (r5), "r" (r11)); +#endif +} + +static __inline__ void stl_u(unsigned long r5, unsigned int * r11) +{ +#if defined(__GNUC__) && ((__GNUC__ > 2) || (__GNUC_MINOR__ >= 91)) + struct __una_u32 *ptr = (struct __una_u32 *) r11; + ptr->x = r5; +#else + unsigned long r1,r2,r3,r4; + + __asm__("ldq_u %3,%1\n\t" + "ldq_u %2,%0\n\t" + "inslh %6,%7,%5\n\t" + "insll %6,%7,%4\n\t" + "msklh %3,%7,%3\n\t" + "mskll %2,%7,%2\n\t" + "bis %3,%5,%3\n\t" + "bis %2,%4,%2\n\t" + "stq_u %3,%1\n\t" + "stq_u %2,%0" + :"=m" (*r11), + "=m" (*(unsigned long *)(3+(char *) r11)), + "=&r" (r1), "=&r" (r2), "=&r" (r3), "=&r" (r4) + :"r" (r5), "r" (r11)); +#endif +} + +static __inline__ void stw_u(unsigned long r5, unsigned short * r11) +{ +#if defined(__GNUC__) && ((__GNUC__ > 2) || (__GNUC_MINOR__ >= 91)) + struct __una_u16 *ptr = (struct __una_u16 *) r11; + ptr->x = r5; +#else + unsigned long r1,r2,r3,r4; + + __asm__("ldq_u %3,%1\n\t" + "ldq_u %2,%0\n\t" + "inswh %6,%7,%5\n\t" + "inswl %6,%7,%4\n\t" + "mskwh %3,%7,%3\n\t" + "mskwl %2,%7,%2\n\t" + "bis %3,%5,%3\n\t" + "bis %2,%4,%2\n\t" + "stq_u %3,%1\n\t" + "stq_u %2,%0" + :"=m" (*r11), + "=m" (*(unsigned long *)(1+(char *) r11)), + "=&r" (r1), "=&r" (r2), "=&r" (r3), "=&r" (r4) + :"r" (r5), "r" (r11)); +#endif +} + +#elif defined(__GNUC__) && ((__GNUC__ < 3)) && \ + (defined (__ia64__) || defined (ia64__)) +#define IA64_UALOADS +/* + * EGCS 1.1 knows about arbitrary unaligned loads. Define some + * packed structures to talk about such things with. + */ +struct __una_u64 { unsigned long x __attribute__((packed)); }; +struct __una_u32 { unsigned int x __attribute__((packed)); }; +struct __una_u16 { unsigned short x __attribute__((packed)); }; + +static __inline__ unsigned long +__uldq (const unsigned long * r11) +{ + const struct __una_u64 *ptr = (const struct __una_u64 *) r11; + return ptr->x; +} + +static __inline__ unsigned long +uldl (const unsigned int * r11) +{ + const struct __una_u32 *ptr = (const struct __una_u32 *) r11; + return ptr->x; +} + +static __inline__ unsigned long +uldw (const unsigned short * r11) +{ + const struct __una_u16 *ptr = (const struct __una_u16 *) r11; + return ptr->x; +} + +static __inline__ void +ustq (unsigned long r5, unsigned long * r11) +{ + struct __una_u64 *ptr = (struct __una_u64 *) r11; + ptr->x = r5; +} + +static __inline__ void +ustl (unsigned long r5, unsigned int * r11) +{ + struct __una_u32 *ptr = (struct __una_u32 *) r11; + ptr->x = r5; +} + +static __inline__ void +ustw (unsigned long r5, unsigned short * r11) +{ + struct __una_u16 *ptr = (struct __una_u16 *) r11; + ptr->x = r5; +} + +#endif + +/**************************************************************************** +PARAMETERS: +addr - Emulator memory address to read + +RETURNS: +Byte value read from emulator memory. + +REMARKS: +Reads a byte value from the emulator memory. +****************************************************************************/ +u8 X86API rdb( + u32 addr) +{ + u8 val; + + if (addr > M.mem_size - 1) { + DB(NV_PRINTF(LEVEL_INFO, "mem_read: address %#x out of range!\n", addr);) + HALT_SYS(); + } + val = *(u8*)(M.mem_base + addr); +DB( if (DEBUG_MEM_TRACE()) + NV_PRINTF(LEVEL_INFO, "%#08x 1 -> %#x\n", addr, val);) + return val; +} + +/**************************************************************************** +PARAMETERS: +addr - Emulator memory address to read + +RETURNS: +Word value read from emulator memory. + +REMARKS: +Reads a word value from the emulator memory. +****************************************************************************/ +u16 X86API rdw( + u32 addr) +{ + u16 val = 0; + + if (addr > M.mem_size - 2) { + DB(NV_PRINTF(LEVEL_INFO, "mem_read: address %#x out of range!\n", addr);) + HALT_SYS(); + } +#ifdef __BIG_ENDIAN__ + if (addr & 0x1) { + val = (*(u8*)(M.mem_base + addr) | + (*(u8*)(M.mem_base + addr + 1) << 8)); + } + else +#endif +#if defined(ALPHA_UALOADS) + val = ldw_u((u16*)(M.mem_base + addr)); +#elif defined(IA64_UALOADS) + val = uldw((u16*)(M.mem_base + addr)); +#else + val = *(u16*)(M.mem_base + addr); +#endif +DB( if (DEBUG_MEM_TRACE()) + NV_PRINTF(LEVEL_INFO, "%#08x 2 -> %#x\n", addr, val);) + return val; +} + +/**************************************************************************** +PARAMETERS: +addr - Emulator memory address to read + +RETURNS: +Long value read from emulator memory. +REMARKS: +Reads a long value from the emulator memory. +****************************************************************************/ +u32 X86API rdl( + u32 addr) +{ + u32 val = 0; + + if (addr > M.mem_size - 4) { + DB(NV_PRINTF(LEVEL_INFO, "mem_read: address %#x out of range!\n", addr);) + HALT_SYS(); + } +#ifdef __BIG_ENDIAN__ + if (addr & 0x3) { + val = (*(u8*)(M.mem_base + addr + 0) | + (*(u8*)(M.mem_base + addr + 1) << 8) | + (*(u8*)(M.mem_base + addr + 2) << 16) | + (*(u8*)(M.mem_base + addr + 3) << 24)); + } + else +#endif +#if defined(ALPHA_UALOADS) + val = ldl_u((u32*)(M.mem_base + addr)); +#elif defined(IA64_UALOADS) + val = uldl((u32*)(M.mem_base + addr)); +#else + val = *(u32*)(M.mem_base + addr); +#endif +DB( if (DEBUG_MEM_TRACE()) + NV_PRINTF(LEVEL_INFO, "%#08x 4 -> %#x\n", addr, val);) + return val; +} + +/**************************************************************************** +PARAMETERS: +addr - Emulator memory address to read +val - Value to store + +REMARKS: +Writes a byte value to emulator memory. +****************************************************************************/ +void X86API wrb( + u32 addr, + u8 val) +{ +DB( if (DEBUG_MEM_TRACE()) + NV_PRINTF(LEVEL_INFO, "%#08x 1 <- %#x\n", addr, val);) + if (addr > M.mem_size - 1) { + DB(NV_PRINTF(LEVEL_INFO, "mem_write: address %#x out of range!\n", addr);) + HALT_SYS(); + } + *(u8*)(M.mem_base + addr) = val; +} + +/**************************************************************************** +PARAMETERS: +addr - Emulator memory address to read +val - Value to store + +REMARKS: +Writes a word value to emulator memory. +****************************************************************************/ +void X86API wrw( + u32 addr, + u16 val) +{ +DB( if (DEBUG_MEM_TRACE()) + NV_PRINTF(LEVEL_INFO, "%#08x 2 <- %#x\n", addr, val);) + if (addr > M.mem_size - 2) { + DB(NV_PRINTF(LEVEL_INFO, "mem_write: address %#x out of range!\n", addr);) + HALT_SYS(); + } +#ifdef __BIG_ENDIAN__ + if (addr & 0x1) { + *(u8*)(M.mem_base + addr + 0) = (val >> 0) & 0xff; + *(u8*)(M.mem_base + addr + 1) = (val >> 8) & 0xff; + } + else +#endif +#if defined(ALPHA_UALOADS) + stw_u(val,(u16*)(M.mem_base + addr)); +#elif defined(IA64_UALOADS) + ustw(val,(u16*)(M.mem_base + addr)); +#else + *(u16*)(M.mem_base + addr) = val; +#endif +} + +/**************************************************************************** +PARAMETERS: +addr - Emulator memory address to read +val - Value to store + +REMARKS: +Writes a long value to emulator memory. +****************************************************************************/ +void X86API wrl( + u32 addr, + u32 val) +{ +DB( if (DEBUG_MEM_TRACE()) + NV_PRINTF(LEVEL_INFO, "%#08x 4 <- %#x\n", addr, val);) + if (addr > M.mem_size - 4) { + DB(NV_PRINTF(LEVEL_INFO, "mem_write: address %#x out of range!\n", addr);) + HALT_SYS(); + } +#ifdef __BIG_ENDIAN__ + if (addr & 0x1) { + *(u8*)(M.mem_base + addr + 0) = (val >> 0) & 0xff; + *(u8*)(M.mem_base + addr + 1) = (val >> 8) & 0xff; + *(u8*)(M.mem_base + addr + 2) = (val >> 16) & 0xff; + *(u8*)(M.mem_base + addr + 3) = (val >> 24) & 0xff; + } + else +#endif +#if defined(ALPHA_UALOADS) + stl_u(val,(u32*)(M.mem_base + addr)); +#elif defined(IA64_UALOADS) + ustl(val,(u32*)(M.mem_base + addr)); +#else + *(u32*)(M.mem_base + addr) = val; +#endif +} + +/**************************************************************************** +PARAMETERS: +addr - PIO address to read +RETURN: +0 +REMARKS: +Default PIO byte read function. Doesn't perform real inb. +****************************************************************************/ +static u8 X86API p_inb( + X86EMU_pioAddr addr) +{ +DB( if (DEBUG_IO_TRACE()) + NV_PRINTF(LEVEL_INFO, "inb %#04x \n", addr);) + return 0; +} + +/**************************************************************************** +PARAMETERS: +addr - PIO address to read +RETURN: +0 +REMARKS: +Default PIO word read function. Doesn't perform real inw. +****************************************************************************/ +static u16 X86API p_inw( + X86EMU_pioAddr addr) +{ +DB( if (DEBUG_IO_TRACE()) + NV_PRINTF(LEVEL_INFO, "inw %#04x \n", addr);) + return 0; +} + +/**************************************************************************** +PARAMETERS: +addr - PIO address to read +RETURN: +0 +REMARKS: +Default PIO long read function. Doesn't perform real inl. +****************************************************************************/ +static u32 X86API p_inl( + X86EMU_pioAddr addr) +{ +DB( if (DEBUG_IO_TRACE()) + NV_PRINTF(LEVEL_INFO, "inl %#04x \n", addr);) + return 0; +} + +/**************************************************************************** +PARAMETERS: +addr - PIO address to write +val - Value to store +REMARKS: +Default PIO byte write function. Doesn't perform real outb. +****************************************************************************/ +static void X86API p_outb( + X86EMU_pioAddr addr, + u8 val) +{ +DB( if (DEBUG_IO_TRACE()) + NV_PRINTF(LEVEL_INFO, "outb %#02x -> %#04x \n", val, addr);) + return; +} + +/**************************************************************************** +PARAMETERS: +addr - PIO address to write +val - Value to store +REMARKS: +Default PIO word write function. Doesn't perform real outw. +****************************************************************************/ +static void X86API p_outw( + X86EMU_pioAddr addr, + u16 val) +{ +DB( if (DEBUG_IO_TRACE()) + NV_PRINTF(LEVEL_INFO, "outw %#04x -> %#04x \n", val, addr);) + return; +} + +/**************************************************************************** +PARAMETERS: +addr - PIO address to write +val - Value to store +REMARKS: +Default PIO ;ong write function. Doesn't perform real outl. +****************************************************************************/ +static void X86API p_outl( + X86EMU_pioAddr addr, + u32 val) +{ +DB( if (DEBUG_IO_TRACE()) + NV_PRINTF(LEVEL_INFO, "outl %#08x -> %#04x \n", val, addr);) + return; +} + +/*------------------------- Global Variables ------------------------------*/ + +u8 (X86APIP sys_rdb)(u32 addr) = rdb; +u16 (X86APIP sys_rdw)(u32 addr) = rdw; +u32 (X86APIP sys_rdl)(u32 addr) = rdl; +void (X86APIP sys_wrb)(u32 addr,u8 val) = wrb; +void (X86APIP sys_wrw)(u32 addr,u16 val) = wrw; +void (X86APIP sys_wrl)(u32 addr,u32 val) = wrl; +u8 (X86APIP sys_inb)(X86EMU_pioAddr addr) = p_inb; +u16 (X86APIP sys_inw)(X86EMU_pioAddr addr) = p_inw; +u32 (X86APIP sys_inl)(X86EMU_pioAddr addr) = p_inl; +void (X86APIP sys_outb)(X86EMU_pioAddr addr, u8 val) = p_outb; +void (X86APIP sys_outw)(X86EMU_pioAddr addr, u16 val) = p_outw; +void (X86APIP sys_outl)(X86EMU_pioAddr addr, u32 val) = p_outl; + +/*----------------------------- Setup -------------------------------------*/ + +/**************************************************************************** +PARAMETERS: +funcs - New memory function pointers to make active + +REMARKS: +This function is used to set the pointers to functions which access +memory space, allowing the user application to override these functions +and hook them out as necessary for their application. +****************************************************************************/ +void X86EMU_setupMemFuncs( + X86EMU_memFuncs *funcs) +{ + sys_rdb = funcs->rdb; + sys_rdw = funcs->rdw; + sys_rdl = funcs->rdl; + sys_wrb = funcs->wrb; + sys_wrw = funcs->wrw; + sys_wrl = funcs->wrl; +} + +/**************************************************************************** +PARAMETERS: +funcs - New programmed I/O function pointers to make active + +REMARKS: +This function is used to set the pointers to functions which access +I/O space, allowing the user application to override these functions +and hook them out as necessary for their application. +****************************************************************************/ +void X86EMU_setupPioFuncs( + X86EMU_pioFuncs *funcs) +{ + sys_inb = funcs->inb; + sys_inw = funcs->inw; + sys_inl = funcs->inl; + sys_outb = funcs->outb; + sys_outw = funcs->outw; + sys_outl = funcs->outl; +} + +/**************************************************************************** +PARAMETERS: +funcs - New interrupt vector table to make active + +REMARKS: +This function is used to set the pointers to functions which handle +interrupt processing in the emulator, allowing the user application to +hook interrupts as necessary for their application. Any interrupts that +are not hooked by the user application, and reflected and handled internally +in the emulator via the interrupt vector table. This allows the application +to get control when the code being emulated executes specific software +interrupts. +****************************************************************************/ +void X86EMU_setupIntrFuncs( + X86EMU_intrFuncs funcs[]) +{ + int i; + + for (i=0; i < 256; i++) + _X86EMU_intrTab[i] = NULL; + if (funcs) { + for (i = 0; i < 256; i++) + _X86EMU_intrTab[i] = funcs[i]; + } +} + +/**************************************************************************** +PARAMETERS: +int - New software interrupt to prepare for + +REMARKS: +This function is used to set up the emulator state to exceute a software +interrupt. This can be used by the user application code to allow an +interrupt to be hooked, examined and then reflected back to the emulator +so that the code in the emulator will continue processing the software +interrupt as per normal. This essentially allows system code to actively +hook and handle certain software interrupts as necessary. +****************************************************************************/ +void X86EMU_prepareForInt( + int num) +{ + push_word((u16)M.x86.R_FLG); + CLEAR_FLAG(F_IF); + CLEAR_FLAG(F_TF); + push_word(M.x86.R_CS); + M.x86.R_CS = mem_access_word(num * 4 + 2); + push_word(M.x86.R_IP); + M.x86.R_IP = mem_access_word(num * 4); + M.x86.intr = 0; +} |