diff options
| author | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-09-13 18:36:21 -0400 |
|---|---|---|
| committer | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-09-13 18:36:21 -0400 |
| commit | 00e54c13c7c9030c4d5f7151467f2d85a3730e29 (patch) | |
| tree | 17a45239df8d2e4912ff3697ed06e85a10b969de /userspace/sdks/nv525/nvalloc | |
| download | open-iov-00e54c13c7c9030c4d5f7151467f2d85a3730e29.tar.gz open-iov-00e54c13c7c9030c4d5f7151467f2d85a3730e29.zip | |
Initial commit: Open-IOV vGPU-IO complete stack
- Firmware decompilation: GSP 570.144W (28.5MB ELF, RV64IMAC), 132 decompiled C files
- Kernel modules: nvidia.ko (34MB, OpenRM), open-iov-vfio.ko (2.8MB, VFIO mdev), gsp-vgpu.ko (1.8MB, guest driver)
- Userspace: libiov.so/libvgpu.so (76KB), example_vgpu (28KB), rpc_emulator.c
- PCI config space emulation (256B with PM, MSI, MSI-X, PCIe caps)
- 12 RPC types fully mapped from firmware decompilation
- BAR trap, WPR, scheduling, notification, migration emulation
- Helper scripts: load_vgpu_stack, create_mdev, launch_qemu_vgpu, test_vgpu_stack, setup_guest_driver, unload_vgpu_stack
- Architecture documentation
- Wiki: 10 technical documents
- Git LFS for large binaries (.ko, .bin, .iso, .img, .o, ghidra repos)
- nvidia-open-gpu-kernel-modules as submodule
Diffstat (limited to 'userspace/sdks/nv525/nvalloc')
65 files changed, 30918 insertions, 0 deletions
diff --git a/userspace/sdks/nv525/nvalloc/common/inc/dev_ctrl_defines.h b/userspace/sdks/nv525/nvalloc/common/inc/dev_ctrl_defines.h new file mode 100644 index 0000000..1644e3d --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/dev_ctrl_defines.h @@ -0,0 +1,86 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @brief Defines for simplifying SW accesses to the dev_ctrl interrupt tree. + * These are generic defines ued in addition to architecure-specific + * defines in dev_vm_addendum.h + * + */ + +// +// Notes on the terms used below: +// Subtree: The HW tree is a 64-way tree that consists of 2 TOP level interrupt +// registers, 32 bits each. Each of these 64 is referred to as a subtree. +// Leaf: Each of these 64 subtrees are associated with a pair of LEAF registers +// giving us a total of 128 LEAF registers. +// GPU vector: The 128 LEAF registers give us a total of (128*32) GPU vectors +// giving us a total of 4096 GPU vectors +// + +// +// Given a subtree index, the below macros give us the index of the TOP level +// register and the bit within the TOP level register to program for that +// subtree. +// +#define NV_CTRL_INTR_SUBTREE_TO_TOP_IDX(i) ((i) / 32) +#define NV_CTRL_INTR_SUBTREE_TO_TOP_BIT(i) ((i) % 32) + +// +// Given a subtree index, the below macros give us the two LEAF register indices +// that correspond to that subtree. +// +#define NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_START(i) ((i)*2) +#define NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_END(i) (((i)*2) + 1) + +// +// Given a LEAF register index, the below macros give us the range of GPU +// interrupt vectors that correspond to those leafs. +// +#define NV_CTRL_INTR_LEAF_IDX_TO_GPU_VECTOR_START(i) ((i)*32) +#define NV_CTRL_INTR_LEAF_IDX_TO_GPU_VECTOR_END(i) (((i)*32) + 31) + +// +// Given a GPU interrupt vector, the below macros give us the index of the +// LEAF register and the bit within the LEAF register to program for that +// GPU interrupt vector. +// +#define NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(i) ((i) / 32) +#define NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_BIT(i) ((i) % 32) + +// +// Given a GPU interrupt vector, the below macro gives us the subtree in which +// it belongs. +// +#define NV_CTRL_INTR_GPU_VECTOR_TO_SUBTREE(i) ((NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(i)) / 2) + +// +// The max number of leaf registers we expect +// This is enforced to be greater than or equal to +// (NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_END(NV_CPU_INTR_STALL_SUBTREE_LAST) + 1) +// for the largest NV_CPU_INTR_STALL_SUBTREE_LAST +// +#define NV_MAX_INTR_LEAVES 12 + +// In SW, this specifies an invalid interrupt vector +#define NV_INTR_VECTOR_INVALID (NV_U32_MAX) diff --git a/userspace/sdks/nv525/nvalloc/common/inc/flcnifcmn.h b/userspace/sdks/nv525/nvalloc/common/inc/flcnifcmn.h new file mode 100644 index 0000000..854c84e --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/flcnifcmn.h @@ -0,0 +1,149 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2014-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * flcn Command/Message Interfaces - Common + */ + +#ifndef FLCNIFCMN_H +#define FLCNIFCMN_H + +#include "nvtypes.h" +#include "flcnretval.h" + +#ifndef NV_SIZEOF32 +#define NV_SIZEOF32(v) (sizeof(v)) +#endif + +#ifndef NV_ARRAY_ELEMENTS +#define NV_ARRAY_ELEMENTS(x) ((sizeof(x)/sizeof((x)[0]))) +#endif + +/*! + * @ref NvU64_ALIGN32 , NvU64_ALIGN32_PACK, NvU64_ALIGN32_UNPACK + * TODO: Use NV types directly + */ +typedef NvU64_ALIGN32 RM_FLCN_U64; +typedef NvU64_ALIGN32 *PRM_FLCN_U64; + +#define RM_FLCN_U64_IS_ZERO NvU64_ALIGN32_IS_ZERO +#define RM_FLCN_U64_PACK NvU64_ALIGN32_PACK +#define RM_FLCN_U64_UNPACK NvU64_ALIGN32_UNPACK + +/*! + * @brief Header preceding each CMD/MSG exchanged through falcon's queues. + * + * @note Data package sent to falcon are referred as CMDs (commands). + * Data package sent by falcon are referred as MSGs (messages). + */ +typedef struct +{ + /*! + * Unit ID identifies falcon's task/unit receiving/issuing this message. + */ + NvU8 unitId; + /*! + * Total CMD/MSG size (including header). + */ + NvU8 size; + /*! + * Flags identifying state of CMD/MSG. + */ + NvU8 ctrlFlags; + /*! + * Sequence # ID to track each request sent to falcon (where applicable). + */ + NvU8 seqNumId; +} RM_FLCN_QUEUE_HDR, +*PRM_FLCN_QUEUE_HDR; + +/*! + * Convenience macro for determining the size of the falcon's queue header: + */ +#define RM_FLCN_QUEUE_HDR_SIZE sizeof(RM_FLCN_QUEUE_HDR) + +/*! + * Generic Falcon rewind unit ID. + */ +#define RM_FLCN_UNIT_ID_REWIND (0x00U) + +/*! + * Generic CMD structure to hold the header. + */ +typedef struct +{ + RM_FLCN_QUEUE_HDR hdr; + NvU32 cmd; +} RM_FLCN_CMD_GEN; + +/*! + * Generic MSG structure to hold the header. + */ +typedef struct +{ + RM_FLCN_QUEUE_HDR hdr; + NvU32 msg; +} RM_FLCN_MSG_GEN; + +/*! + * Convenience macros for determining the size of body for a command or message: + */ +#define RM_FLCN_CMD_BODY_SIZE(u,t) sizeof(RM_FLCN_##u##_CMD_##t) +#define RM_FLCN_MSG_BODY_SIZE(u,t) sizeof(RM_FLCN_##u##_MSG_##t) + +/*! + * Convenience macros for determining the size of a command or message: + */ +#define RM_FLCN_CMD_SIZE(u,t) \ + (RM_FLCN_QUEUE_HDR_SIZE + RM_FLCN_CMD_BODY_SIZE(u,t)) + +#define RM_FLCN_MSG_SIZE(u,t) \ + (RM_FLCN_QUEUE_HDR_SIZE + RM_FLCN_MSG_BODY_SIZE(u,t)) + +/*! + * Convenience macros for determining the type of a command or message + * (intended to be used symmetrically with the CMD and MSG _SIZE macros): + */ +#define RM_FLCN_CMD_TYPE(u,t) RM_FLCN_##u##_CMD_ID_##t +#define RM_FLCN_MSG_TYPE(u,t) RM_FLCN_##u##_MSG_ID_##t + +/*! + * @brief Falcons' queue header flags (@ref RM_FLCN_QUEUE_HDR::ctrlFlags). + * + * RM_FLCN_QUEUE_FLAGS_STATUS + * Set by command's sender to request back message confirming the completion of + * command's execution. In RM->FLCN communication response is required to free + * command related data tracked within RM (***_SEQ_INFO). + * + * RM_FLCN_QUEUE_HDR_FLAGS_EVENT + * Set by the falcon to distinguish messages sent to RM from command responses. + * + * RM_FLCN_QUEUE_HDR_FLAGS_UNIT_ACK + * Used internally within the falcon to track completed commands when updating + * get (tail) command queue pointers. + */ +#define RM_FLCN_QUEUE_HDR_FLAGS_STATUS NVBIT(0) +#define RM_FLCN_QUEUE_HDR_FLAGS_EVENT NVBIT(2) +#define RM_FLCN_QUEUE_HDR_FLAGS_UNIT_ACK NVBIT(5) + +#endif // FLCNIFCMN_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/flcnretval.h b/userspace/sdks/nv525/nvalloc/common/inc/flcnretval.h new file mode 100644 index 0000000..c43cbb2 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/flcnretval.h @@ -0,0 +1,292 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2008-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @file flcnretval.h + * @brief Defines various status codes that are convenient to relay status + * information in functions. + */ + +#ifndef FLCNRETVAL_H +#define FLCNRETVAL_H + +#include "nvtypes.h" +/*! + * Defines a generic type that may be used to convey status information. This + * is very similar to the RM_STATUS type but smaller in width to save DMEM and + * stack space. + */ +typedef NvU8 FLCN_STATUS; + +#define FLCN_OK (0x00U) +#define FLCN_ERR_BINARY_NOT_STARTED (0xFEU) +#define FLCN_ERROR (0xFFU) + +#define FLCN_ERR_NOT_SUPPORTED (0x01U) +#define FLCN_ERR_INVALID_ARGUMENT (0x02U) +#define FLCN_ERR_ILLEGAL_OPERATION (0x03U) +#define FLCN_ERR_TIMEOUT (0x04U) +#define FLCN_ERR_NO_FREE_MEM (0x05U) +#define FLCN_ERR_HDCP_INVALID_SRM (0x06U) +#define FLCN_ERR_HDCP_RECV_REVOKED (0x07U) +#define FLCN_ERR_RPC_INVALID_INPUT (0x08U) +#define FLCN_ERR_DMA_SUSPENDED (0x09U) +#define FLCN_ERR_MUTEX_ACQUIRED (0x10U) +#define FLCN_ERR_MUTEX_ID_NOT_AVAILABLE (0x11U) +#define FLCN_ERR_OBJECT_NOT_FOUND (0x12U) +#define FLCN_ERR_MSGBOX_TIMEOUT (0x13U) +#define FLCN_ERR_INVALID_INDEX (0x14U) +#define FLCN_ERR_INVALID_FUNCTION (0x15U) +#define FLCN_ERR_INSUFFICIENT_PMB_PLM_PROTECTION (0x16U) +#define FLCN_ERR_DMA_NACK (0x17U) +#define FLCN_ERR_CHIP_NOT_SUPPORTED_FOR_PR (0x18U) +#define FLCN_ERR_BAR0_PRIV_READ_ERROR (0x19U) +#define FLCN_ERR_BAR0_PRIV_WRITE_ERROR (0x1AU) +#define FLCN_ERR_HDCP22_ABORT_AUTHENTICATION (0x1BU) +#define FLCN_ERR_DPU_IS_BUSY (0x1CU) +#define FLCN_ERR_DPU_TIMEOUT_FOR_HDCP_TYPE1_LOCK_REQUEST (0x1DU) +#define FLCN_ERR_HDCP_TYPE1_LOCK_FAILED (0x1EU) +#define FLCN_ERR_HDCP_TYPE1_LOCK_UNKNOWN (0x1FU) +#define FLCN_ERR_WAIT_FOR_BAR0_IDLE_FAILED (0x20U) +#define FLCN_ERR_CSB_PRIV_READ_ERROR (0x21U) +#define FLCN_ERR_CSB_PRIV_WRITE_ERROR (0x22U) +#define FLCN_ERR_DMA_UNEXPECTED_DMAIDX (0x23U) +#define FLCN_ERR_PRIV_SEC_VIOLATION (0x24U) +#define FLCN_ERR_INVALID_VERSION (0x25U) +#define FLCN_ERR_PR_SHARED_STRUCT_INIT_FAILED (0x26U) +#define FLCN_ERR_GPU_IN_DEBUG_MODE (0x27U) +#define FLCN_ERR_HPD_UNPLUG (0x28U) +#define FLCN_ERR_HDCP22_DELAY_ABORT_AUTHENTICATION (0x29U) +#define FLCN_ERR_SECUREBUS_REGISTER_READ_ERROR (0x2AU) +#define FLCN_ERR_SECUREBUS_REGISTER_WRITE_ERROR (0x2BU) +#define FLCN_ERR_HDCP22_FLUSH_TYPE_LOCK_ACTIVE (0x2CU) +#define FLCN_ERR_HDCP22_FLUSH_TYPE_IN_PROGRESS (0x2DU) +#define FLCN_ERR_FEATURE_NOT_ENABLED (0x2EU) +#define FLCN_ERR_OUT_OF_RANGE (0x2FU) + +// I2C Errors +#define FLCN_ERR_I2C_BUSY (0x30U) +#define FLCN_ERR_I2C_NACK_ADDRESS (0x31U) +#define FLCN_ERR_I2C_NACK_BYTE (0x32U) +#define FLCN_ERR_I2C_SIZE (0x33U) +#define FLCN_ERR_I2C_BUS_INVALID (0x34U) + +#define FLCN_ERR_INVALID_STATE (0x35U) +#define FLCN_ERR_RECURSION_LIMIT_EXCEEDED (0x36U) +#define FLCN_ERR_INVALID_CAST (0x37U) + +// AUX Errors +#define FLCN_ERR_AUX_ERROR (0x3AU) +#define FLCN_ERR_AUX_SEMA_ACQUIRED (0x3BU) +#define FLCN_ERR_AUX_SEMA_INVALID_RELEASE (0x3CU) + +#define FLCN_ERR_MORE_PROCESSING_REQUIRED (0x3EU) +#define FLCN_ERR_DMA_ALIGN (0x3FU) + +// Power-Device Errors +#define FLCN_ERR_PWR_DEVICE_TAMPERED (0x40U) + +#define FLCN_ERR_ITERATION_END (0x41U) + +// Perf change sequence Errors +#define FLCN_ERR_LOCK_NOT_AVAILABLE (0x42U) + +#define FLCN_ERR_STATE_RESET_NEEDED (0x43U) +#define FLCN_ERR_DMA_GENERIC (0x44U) +#define FLCN_ERR_LS_CHK_UCODE_REVOKED (0x45U) +#define FLCN_ERR_ACC_SEQ_MISMATCH (0x46U) +#define FLCN_ERR_SSP_STACK_CHECK_FAILED (0x47U) +#define FLCN_ERR_SE_TRNG_FAILED (0x48U) +#define FLCN_ERR_PROD_MODE_NOT_YET_SUPPORTED (0x49U) + +// SHA HW errors +#define FLCN_ERR_SHA_HW_CHECK_INT_STATUS (0x4AU) +#define FLCN_ERR_SHA_HW_SOFTRESET_REQUIRED (0x4BU) +#define FLCN_ERR_SHA_HW_SOFTRESET_FAILED (0x4CU) +#define FLCN_ERR_SHA_HW_BUSY (0x4DU) + +// +// Add new generic error codes here, do not changes values of exiting error codes, +// because that will affect other binaries and their signatures +// +#define FLCN_ERR_CTXSW_ERROR (0x4EU) +#define FLCN_ERR_SE_SNPKA_HW_CRYPTO_OPERATION_FAILED (0x4FU) + +// VPR SEC2 task errors +#define FLCN_ERR_VPR_APP_INVALID_REQUEST_END_ADDR (0x51U) +#define FLCN_ERR_VPR_APP_INVALID_REQUEST_START_ADDR (0x52U) +#define FLCN_ERR_VPR_APP_SCRUB_VERIF_FAILED (0x53U) +#define FLCN_ERR_VPR_APP_MEMLOCK_ALREADY_SET (0x54U) +#define FLCN_ERR_VPR_APP_INVALID_INDEX (0x55U) +#define FLCN_ERR_VPR_APP_UNEXPECTED_VPR_HANDOFF_FROM_SCRUBBER (0x56U) +#define FLCN_ERR_VPR_APP_CBC_RANGE_CLASH (0x57U) +#define FLCN_ERR_VPR_APP_NOT_SUPPORTED_BY_HW (0x58U) +#define FLCN_ERR_VPR_APP_NOT_SUPPORTED_BY_SW (0x59U) +#define FLCN_ERR_VPR_APP_DISPLAY_VERSION_NOT_SUPPORTED (0x5AU) +#define FLCN_ERR_VPR_APP_VPR_WPR_WRITE_FAILED (0x5BU) +#define FLCN_ERR_VPR_APP_NOTHING_TO_DO (0x5CU) +#define FLCN_ERR_VPR_APP_DISPLAY_NOT_PRESENT (0x5DU) +#define FLCN_ERR_VPR_APP_PREVIOUS_CMD_FAILED_AS_MAX_VPR_IS_0 (0x5EU) +#define FLCN_ERR_VPR_APP_PLM_PROTECTION_NOT_RAISED (0x5FU) +#define FLCN_ERR_VPR_APP_PLM_PROTECTION_ALREADY_RAISED (0x60U) +#define FLCN_ERR_VPR_APP_DISP_FALCON_IS_NOT_IN_LS_MODE (0x61U) +#define FLCN_ERR_VPR_APP_VPR_IS_ALREADY_ENABLED (0x62U) +#define FLCN_ERR_VPR_APP_UNEXPECTEDLY_RUNNING_ON_RISCV (0x63U) + +// Clocks Errors +#define FLCN_ERR_CYCLE_DETECTED (0x70U) +#define FLCN_ERR_INVALID_PATH (0x71U) +#define FLCN_ERR_MISMATCHED_TARGET (0x72U) +#define FLCN_ERR_FREQ_NOT_SUPPORTED (0x73U) +#define FLCN_ERR_INVALID_SOURCE (0x74U) +#define FLCN_ERR_NOT_INITIALIZED (0x75U) + +// HDCP2.2 Errors +#define FLCN_ERR_HDCP22_GETDKEY_FAILED (0x80U) +#define FLCN_ERR_HDCP22_H_PRIME (0x81U) +#define FLCN_ERR_HDCP22_CERT_RX (0x82U) +#define FLCN_ERR_HDCP22_PAIRING (0x83U) +#define FLCN_ERR_HDCP22_L_PRIME (0x84U) +#define FLCN_ERR_HDCP22_V_PRIME (0x85U) +#define FLCN_ERR_HDCP22_INVALID_RXIDLIST (0x86U) +#define FLCN_ERR_HDCP22_M_PRIME (0x87U) +#define FLCN_ERR_HDCP22_SEQ_ROLLOVER (0x88U) +#define FLCN_ERR_HDCP22_RSA_HW (0x89U) +#define FLCN_ERR_HDCP22_ECF_TIMESLOT_MISMATCH (0x90U) + +// LibCCC Errors +#define FLCN_ERR_INIT_CRYPTO_DEVICE_FAILED (0x91U) +#define FLCN_ERR_NVPKA_SELECT_ENGINE_FAILED (0x92U) +#define FLCN_ERR_NVPKA_ACQUIRE_MUTEX_FAILED (0x93U) +#define FLCN_ERR_NVPKA_MODULAR_EXP_LOCK_FAILED (0x94U) +#define FLCN_ERR_NVRNG_INIT_CRYPTO_DEVICE_FAILED (0x95U) +#define FLCN_ERR_NVRNG_SELECT_ENGINE_FAILED (0x96U) +#define FLCN_ERR_NVRNG_GENERATE_FAILED (0x97U) + +// Heavy Secure Errors +#define FLCN_ERR_HS_CHK_INVALID_INPUT (0xA0U) +#define FLCN_ERR_HS_CHK_CHIP_NOT_SUPPORTED (0xA1U) +#define FLCN_ERR_HS_CHK_UCODE_REVOKED (0xA2U) +#define FLCN_ERR_HS_CHK_NOT_IN_LSMODE (0xA3U) +#define FLCN_ERR_HS_CHK_INVALID_LS_PRIV_LEVEL (0xA4U) +#define FLCN_ERR_HS_CHK_INVALID_REGIONCFG (0xA5U) +#define FLCN_ERR_HS_CHK_PRIV_SEC_DISABLED_ON_PROD (0xA6U) +#define FLCN_ERR_HS_CHK_SW_FUSING_ALLOWED_ON_PROD (0xA7U) +#define FLCN_ERR_HS_CHK_INTERNAL_SKU_ON_PROD (0xA8U) +#define FLCN_ERR_HS_CHK_DEVID_OVERRIDE_ENABLED_ON_PROD (0xA9U) +#define FLCN_ERR_HS_CHK_INCONSISTENT_PROD_MODES (0xAAU) +#define FLCN_ERR_HS_CHK_HUB_ENCRPTION_DISABLED (0xABU) +#define FLCN_ERR_HS_PR_ILLEGAL_LASSAHS_STATE_AT_HS_ENTRY (0xACU) +#define FLCN_ERR_HS_PR_ILLEGAL_LASSAHS_STATE_AT_MPK_DECRYPT (0xADU) +#define FLCN_ERR_HS_PR_ILLEGAL_LASSAHS_STATE_AT_HS_EXIT (0xAEU) +#define FLCN_ERR_HS_PROD_MODE_NOT_YET_SUPPORTED (0xAFU) +#define FLCN_ERR_HS_DEV_VERSION_ON_PROD (0xB0U) +#define FLCN_ERR_HS_PR_LASSAHS_LS_SIG_GRP_MISMATCH (0xB1U) +#define FLCN_ERR_HS_PR_LASSAHS_LS_SIG_GRP_OVERLAYS_CNT (0xB2U) +#define FLCN_ERR_HS_PR_LASSAHS_LS_SIG_GRP_INVALID_VA (0xB3U) +#define FLCN_ERR_HS_MUTEX_ACQUIRE_FAILED (0xB4U) +#define FLCN_ERR_HS_MUTEX_RELEASE_FAILED (0xB5U) +#define FLCN_ERR_HS_PR_MPK_DEC_NEEDS_NEWER_ACR_UDE_SCRUBBER (0xB6U) +#define FLCN_ERR_HS_CHK_ENGID_MISMATCH (0xB7U) +#define FLCN_ERR_HS_OPT_INTERNAL_SKU_CHECK_FAILED (0xB8U) +#define FLCN_ERR_HS_CHK_BOARD_MISMATCH (0xB9U) +#define FLCN_ERR_HS_CHK_DISP_ENG_DISABLED (0xBAU) +#define FLCN_ERR_HS_GEN_RANDOM (0xBBU) +#define FLCN_ERR_HS_CHK_IMPROPERLY_FUSED_BOARD (0xBCU) +#define FLCN_ERR_HS_CHK_HDCP_DISABLED (0xBDU) +#define FLCN_ERR_HS_CHK_HDCP_BLACKLISTED_SKU (0XBEU) +#define FLCN_ERR_HS_SECURE_ACTION_ARG_CHECK_FAILED (0xBFU) +#define FLCN_ERR_HS_CHK_RETURN_PC_AT_HS_ENTRY_IS_OF_HS (0xC0U) +#define FLCN_ERR_HS_CHK_HS_LIB_ENTRY_CALLED_BY_NON_HS (0xC1U) +#define FLCN_ERR_HS_DECODE_TRAP_ALREADY_IN_USE (0xC2U) +#define FLCN_ERR_HS_REGISTER_READ_WRITE_ERROR (0xC3U) +#define FLCN_ERR_HS_CHK_CPUCTL_ALIAS_FALSE (0xC4U) +#define FLCN_ERR_HS_UPDATE_RESET_PLM_ERROR (0xC5U) +#define FLCN_ERR_HS_RNG_CONFIG_ERROR (0xC6U) +#define FLCN_ERR_HS_CHK_NOT_IN_HSMODE (0xC7U) +#define FLCN_ERR_HS_CHK_GFW_CHAIN_OF_TRUST_BROKEN (0xC8U) +#define FLCN_ERR_HS_HDCP22_WRONG_SEQUENCE (0xC9U) +#define FLCN_ERR_HS_HDCP22_INTEGRITY_ERROR (0xCAU) +#define FLCN_ERR_HS_HDCP22_WRONG_TYPE (0xCBU) +#define FLCN_ERR_HS_APM_NOT_ENABLED (0xCCU) +#define FLCN_ERR_HS_APM_SMC_ENABLED (0xCDU) +#define FLCN_ERR_HS_APM_FECS_NOT_HALTED (0xCEU) +#define FLCN_ERR_HS_APM_SCRATCH_PLM_INVALID (0xCFU) +#define FLCN_ERR_HS_APM_SCRATCH_INIT_INVALID (0xD0U) +#define FLCN_ERR_HS_INVALID_KEY (0xD1U) +#define FLCN_ERR_HS_SWKG_INVALID_SIGNATURE (0xD2U) + + +// +// BAR0/CSB Priv Read/Write Error Handling Defines +// These need to be defined by HW - NV Bug 200198584 +// +#define FLCN_BAR0_PRIV_PRI_ERROR_MASK 0xFFF00000U +#define FLCN_BAR0_PRIV_PRI_ERROR_CODE 0xBAD00000U +#define FLCN_BAR0_PRIV_PRI_RETURN_VAL 0x00BADBADU + +#define FLCN_CSB_PRIV_PRI_ERROR_MASK 0xFFFF0000U +#define FLCN_CSB_PRIV_PRI_ERROR_CODE 0xBADF0000U + +// +// Macro to check FALCON return status +// +#define CHECK_FLCN_STATUS(expr) do { \ + flcnStatus = (expr); \ + if (flcnStatus != FLCN_OK) \ + { \ + goto ErrorExit; \ + } \ + } while (NV_FALSE) + +// Warnings. +#define FLCN_WARN_NOTHING_TO_DO (0xD0U) +#define FLCN_WARN_NOT_QUERIED (0xD1U) + +// Queue handling Errors +#define FLCN_ERR_QUEUE_MGMT_INVALID_UNIT_ID (0xE0U) +#define FLCN_ERR_QUEUE_MGMT_HEAP_MIRROR_ERR (0xE1U) +#define FLCN_ERR_QUEUE_TASK_INVALID_EVENT_TYPE (0xE2U) +#define FLCN_ERR_QUEUE_TASK_INVALID_UNIT_ID (0xE3U) +#define FLCN_ERR_QUEUE_TASK_INVALID_CMD_TYPE (0xE4U) + +// Posted write errors +#define FLCN_ERR_POSTED_WRITE_FAILURE (0xF0U) +#define FLCN_ERR_POSTED_WRITE_INTERRUPTS_ENABLED (0xF1U) +#define FLCN_ERR_POSTED_WRITE_PRI_CLUSTER_COUNT_MISMATCH (0xF2U) +#define FLCN_ERR_POSTED_WRITE_INCORRECT_PARAMS (0xF3U) + +// Lane Margining errors +#define FLCN_ERR_LM_INVALID_RECEIVER_NUMBER (0xF5U) + +// APM errors +#define FLCN_ERR_APM_NOT_FUSED_FOR_EK (0xF6U) +#define FLCN_ERR_APM_BROM_SIGN_FAIL (0xF7U) +// Booter Reload on SEC2-RTOS errors +#define FLCN_ERR_AUTH_GSP_RM_HANDOFF_FAILED (0xF8U) +#define FLCN_ERR_INVALID_WPRMETA_MAGIC_OR_REVISION (0xF9U) + +// Arithmetic errors +#define FLCN_ERR_ARITHMETIC_OVERFLOW (0xFAU) + +#endif // FLCNRETVAL_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_emem_channels.h b/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_emem_channels.h new file mode 100644 index 0000000..c4b9949 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_emem_channels.h @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _FSP_EMEM_CHANNELS_H_ +#define _FSP_EMEM_CHANNELS_H_ + +// +// NVDM (NVIDIA Data Model) overlayed on MCTP (Management Component Transport +// Protocol) and sent over EMEM is the communication mechanism used between FSP +// management partition and CPU-RM/other uprocs. +// + +// RM uses channel 0 for FSP EMEM. +#define FSP_EMEM_CHANNEL_RM 0x0 + +// PMU/SOE use channel 4 for FSP EMEM. +#define FSP_EMEM_CHANNEL_PMU_SOE 0x4 +#define FSP_EMEM_CHANNEL_MAX 0x8 + +// EMEM channel 0 (RM) is allocated 1K bytes. +#define FSP_EMEM_CHANNEL_RM_SIZE 1024 + +// EMEM channel 4 (PMU/SOE) is allocated 1K bytes. +#define FSP_EMEM_CHANNEL_PMU_SOE_SIZE 1024 +#define FSP_EMEM_CHANNEL_PMU_SOE_OFFSET 4096 + +#endif // _FSP_EMEM_CHANNELS_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_mctp_format.h b/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_mctp_format.h new file mode 100644 index 0000000..c087dc2 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_mctp_format.h @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _FSP_MCTP_FORMAT_H_ +#define _FSP_MCTP_FORMAT_H_ + +// +// NVDM (NVIDIA Data Model) overlayed on MCTP (Management Component Transport +// Protocol) and sent over EMEM is the communication mechanism used between FSP +// management partition and CPU-RM/other uprocs. +// + + +#define MCTP_HEADER_VERSION 3:0 +#define MCTP_HEADER_RSVD 7:4 + +#define MCTP_HEADER_DEID 15:8 +#define MCTP_HEADER_SEID 23:16 + +#define MCTP_HEADER_TAG 26:24 +#define MCTP_HEADER_TO 27:27 +#define MCTP_HEADER_SEQ 29:28 +#define MCTP_HEADER_EOM 30:30 +#define MCTP_HEADER_SOM 31:31 + +#define MCTP_MSG_HEADER_TYPE 6:0 +#define MCTP_MSG_HEADER_IC 7:7 + +#define MCTP_MSG_HEADER_VENDOR_ID 23:8 +#define MCTP_MSG_HEADER_NVDM_TYPE 31:24 + +#define MCTP_MSG_HEADER_TYPE_VENDOR_PCI 0x7e +#define MCTP_MSG_HEADER_VENDOR_ID_NV 0x10de + +#endif // _FSP_MCTP_FORMAT_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_nvdm_format.h b/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_nvdm_format.h new file mode 100644 index 0000000..dc389f1 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/fsp/fsp_nvdm_format.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _FSP_NVDM_FORMAT_H_ +#define _FSP_NVDM_FORMAT_H_ + +#include "fsp/fsp_mctp_format.h" +#include "fsp/fsp_emem_channels.h" + +// +// NVDM (NVIDIA Data Model) overlayed on MCTP (Management Component Transport +// Protocol) and sent over EMEM is the communication mechanism used between FSP +// management partition and CPU-RM/other uprocs. +// + +#define NVDM_TYPE_HULK 0x11 +#define NVDM_TYPE_FIRMWARE_UPDATE 0x12 +#define NVDM_TYPE_COT 0x14 +#define NVDM_TYPE_FSP_RESPONSE 0x15 +#define NVDM_TYPE_INFOROM 0x17 +#define NVDM_TYPE_SMBPBI 0x18 + +#endif // _FSP_NVDM_FORMAT_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/fsp/nvdm_payload_cmd_response.h b/userspace/sdks/nv525/nvalloc/common/inc/fsp/nvdm_payload_cmd_response.h new file mode 100644 index 0000000..fa117aa --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/fsp/nvdm_payload_cmd_response.h @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/* + * @file nvdm_payload_cmd_response.h + * @brief Header file containing the definition for the + * NVDM_PAYLOAD_COMMAND_RESPONSE structure, which + * is required for SMBPBI-specific payloads and needs + * to be visible by bothe the PMU and the driver sides. + */ + +#ifndef _NVDM_PAYLOAD_CMD_RESPONSE_H +#define _NVDM_PAYLOAD_CMD_RESPONSE_H + +#pragma pack(1) +typedef struct +{ + NvU32 taskId; + NvU32 commandNvdmType; + NvU32 errorCode; +} NVDM_PAYLOAD_COMMAND_RESPONSE; +#pragma pack() + +#endif // _NVDM_PAYLOAD_CMD_RESPONSE_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/gsp/gsp_error.h b/userspace/sdks/nv525/nvalloc/common/inc/gsp/gsp_error.h new file mode 100644 index 0000000..2401c78 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/gsp/gsp_error.h @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef GSP_ERROR_H +#define GSP_ERROR_H + +// Definitions for GSP-RM to report errors to CPU-RM via mailbox +#define NV_GSP_ERROR_CODE 7:0 +#define NV_GSP_ERROR_REASON 15:8 +#define NV_GSP_ERROR_TASK 23:16 +#define NV_GSP_ERROR_SKIPPED 27:24 +#define NV_GSP_ERROR_TAG 31:28 +#define NV_GSP_ERROR_TAG_VAL 0xE + +#endif // GSP_ERROR_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h b/userspace/sdks/nv525/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h new file mode 100644 index 0000000..c056000 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#ifndef GSP_FW_WPR_META_H_ +#define GSP_FW_WPR_META_H_ + +/*! + * GSP firmware WPR metadata + * + * Initialized by CPU-RM and DMA'd to FB, at the end of what will be WPR2. + * Verified, and locked in WPR2 by Booter. + * + * Firmware scrubs the last 256mb of FB, no memory outside of this region + * may be used until the FW RM has scrubbed the remainder of memory. + * + * ---------------------------- <- fbSize (end of FB, 1M aligned) + * | VGA WORKSPACE | + * ---------------------------- <- vbiosReservedOffset (64K? aligned) + * | (potential align. gap) | + * ---------------------------- <- gspFwWprEnd (128K aligned) + * | FRTS data | (frtsSize is 0 on GA100) + * | ------------------------ | <- frtsOffset + * | BOOT BIN (e.g. SK + BL) | + * ---------------------------- <- bootBinOffset + * | GSP FW ELF | + * ---------------------------- <- gspFwOffset + * | GSP FW (WPR) HEAP | + * ---------------------------- <- gspFwHeapOffset + * | Booter-placed metadata | + * | (struct GspFwWprMeta) | + * ---------------------------- <- gspFwWprStart (128K aligned) + * | GSP FW (non-WPR) HEAP | + * ---------------------------- <- nonWprHeapOffset, gspFwRsvdStart + * (GSP_CARVEOUT_SIZE bytes from end of FB) + */ +typedef struct +{ + // Magic + // BL to use for verification (i.e. Booter locked it in WPR2) + NvU64 magic; // = 0xdc3aae21371a60b3; + + // Revision number of Booter-BL-Sequencer handoff interface + // Bumped up when we change this interface so it is not backward compatible. + // Bumped up when we revoke GSP-RM ucode + NvU64 revision; // = 1; + + // ---- Members regarding data in SYSMEM ---------------------------- + // Consumed by Booter for DMA + + NvU64 sysmemAddrOfRadix3Elf; + NvU64 sizeOfRadix3Elf; + + NvU64 sysmemAddrOfBootloader; + NvU64 sizeOfBootloader; + + // Offsets inside bootloader image needed by Booter + NvU64 bootloaderCodeOffset; + NvU64 bootloaderDataOffset; + NvU64 bootloaderManifestOffset; + + NvU64 sysmemAddrOfSignature; + NvU64 sizeOfSignature; + + // ---- Members describing FB layout -------------------------------- + NvU64 gspFwRsvdStart; + + NvU64 nonWprHeapOffset; + NvU64 nonWprHeapSize; + + NvU64 gspFwWprStart; + + // GSP-RM to use to setup heap. + NvU64 gspFwHeapOffset; + NvU64 gspFwHeapSize; + + // BL to use to find ELF for jump + NvU64 gspFwOffset; + // Size is sizeOfRadix3Elf above. + + NvU64 bootBinOffset; + // Size is sizeOfBootloader above. + + NvU64 frtsOffset; + NvU64 frtsSize; + + NvU64 gspFwWprEnd; + + // GSP-RM to use for fbRegionInfo? + NvU64 fbSize; + + // ---- Other members ----------------------------------------------- + + // GSP-RM to use for fbRegionInfo? + NvU64 vgaWorkspaceOffset; + NvU64 vgaWorkspaceSize; + + // Boot count. Used to determine whether to load the firmware image. + NvU64 bootCount; + + // TODO: the partitionRpc* fields below do not really belong in this + // structure. The values are patched in by the partition bootstrapper + // when GSP-RM is booted in a partition, and this structure was a + // convenient place for the bootstrapper to access them. These should + // be moved to a different comm. mechanism between the bootstrapper + // and the GSP-RM tasks. + + // Shared partition RPC memory (physical address) + NvU64 partitionRpcAddr; + + // Offsets relative to partitionRpcAddr + NvU16 partitionRpcRequestOffset; + NvU16 partitionRpcReplyOffset; + + // Code section and dataSection offset and size. + NvU32 elfCodeOffset; + NvU32 elfDataOffset; + NvU32 elfCodeSize; + NvU32 elfDataSize; + + // Pad structure to exactly 256 bytes. Can replace padding with additional + // fields without incrementing revision. Padding initialized to 0. + NvU32 padding[3]; + + // BL to use for verification (i.e. Booter says OK to boot) + NvU64 verified; // 0x0 -> unverified, 0xa0a0a0a0a0a0a0a0 -> verified + +} GspFwWprMeta; + +#define GSP_FW_WPR_META_VERIFIED 0xa0a0a0a0a0a0a0a0ULL +#define GSP_FW_WPR_META_REVISION 1 +#define GSP_FW_WPR_META_MAGIC 0xdc3aae21371a60b3ULL + +#endif // GSP_FW_WPR_META_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/gsp/gspifpub.h b/userspace/sdks/nv525/nvalloc/common/inc/gsp/gspifpub.h new file mode 100644 index 0000000..46733a6 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/gsp/gspifpub.h @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef GSPIFPUB_H +#define GSPIFPUB_H + +#include <nvtypes.h> + +/*! + * @file gspifpub.h + * @brief GSP Command/Message Interfaces - Published + */ + +/*! + * Target physical memory apertures supported for DMA + */ +typedef enum { + GSP_DMA_TARGET_LOCAL_FB, + GSP_DMA_TARGET_COHERENT_SYSTEM, + GSP_DMA_TARGET_NONCOHERENT_SYSTEM, + GSP_DMA_TARGET_COUNT +} GSP_DMA_TARGET; + +/*! + * @brief GSP-CC Microcode Initialization Parameters + */ +typedef struct GSP_FMC_INIT_PARAMS +{ + // CC initialization "registry keys" + NvU32 regkeys; +} GSP_FMC_INIT_PARAMS; + +/*! + * @brief GSP-ACR BOOT_GSP_RM Command Parameters + * + * The wprCarveout fields have no effect in environments where the WPR can be allocated + * implicitly by ACR. + */ +typedef struct GSP_ACR_BOOT_GSP_RM_PARAMS +{ + // Physical memory aperture through which gspRmDescPa is accessed + GSP_DMA_TARGET target; + // Size in bytes of the GSP-RM descriptor structure + NvU32 gspRmDescSize; + // Physical offset in the target aperture of the GSP-RM descriptor structure + NvU64 gspRmDescOffset; + // Physical offset in FB to set the start of the WPR containing GSP-RM + NvU64 wprCarveoutOffset; + // Size in bytes of the WPR containing GSP-RM + NvU32 wprCarveoutSize; + // Whether to boot GSP-RM or GSP-Proxy through ACR + NvBool bIsGspRmBoot; +} GSP_ACR_BOOT_GSP_RM_PARAMS; + +/*! + * @brief GSP-RM Parameters + */ +typedef struct GSP_RM_PARAMS +{ + // Physical memory aperture through which bootArgsPa is accessed + GSP_DMA_TARGET target; + // Physical address that will be stuffed in NV_PGSP_FALCON_MAILBOX(0|1) + NvU64 bootArgsOffset; +} GSP_RM_PARAMS; + +/*! + * @brief GSP-CC Microcode Parameters for Boot Partitions + */ +typedef struct GSP_FMC_BOOT_PARAMS +{ + GSP_FMC_INIT_PARAMS initParams; + GSP_ACR_BOOT_GSP_RM_PARAMS bootGspRmParams; + GSP_RM_PARAMS gspRmParams; +} GSP_FMC_BOOT_PARAMS; + +#endif // GSPIFPUB_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrdem.h b/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrdem.h new file mode 100644 index 0000000..26395e8 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrdem.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) <year> NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _IFR_DEM_H_ +#define _IFR_DEM_H_ + +#define INFOROM_DEM_OBJECT_V1_00_PACKED_SIZE 4144 +#define INFOROM_DEM_OBJECT_V1_00_FIFO_SIZE 4096 + +struct INFOROM_DEM_OBJECT_V1_00 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U032 seqNumber; + inforom_U016 writeOffset; + inforom_U016 readOffset; + inforom_X008 fifoBuffer[INFOROM_DEM_OBJECT_V1_00_FIFO_SIZE]; + inforom_U032 reserved[8]; +}; + +#define INFOROM_DEM_OBJECT_V1_00_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d2w4096x8d" +typedef struct INFOROM_DEM_OBJECT_V1_00 INFOROM_DEM_OBJECT_V1_00; + +#endif // _IFR_DEM_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrecc.h b/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrecc.h new file mode 100644 index 0000000..337537d --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrecc.h @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2017-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _IFRECC_H_ +#define _IFRECC_H_ + +#include "nvtypes.h" +#include "inforom/types.h" + +// NVSwitch ECC v6 object definition +#define INFOROM_ECC_OBJECT_V6_S0_PACKED_SIZE 3808 + +//Used to determine if the entry is empty or not +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_VALID 0:0 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_VALID_FALSE 0 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_VALID_TRUE 1 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_CORRUPT_TIMEDATA 1:1 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_CORRUPT_TIMEDATA_FALSE 0 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_CORRUPT_TIMEDATA_TRUE 1 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_ADDR_VALID 2:2 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_ADDR_VALID_FALSE 0 +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_HEADER_ADDR_VALID_TRUE 1 + +#define NV_INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_LOCATION_LINK_ID 7:0 + +typedef struct INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER +{ + inforom_U008 header; + inforom_U032 errId; + inforom_U032 lastErrorTimestamp; + inforom_U032 averageEventDelta; + inforom_U016 location; + inforom_U016 sublocation; + inforom_U032 correctedCount; + inforom_U032 uncorrectedCount; + inforom_U032 address; + +} INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER; + +#define INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_MAX_COUNT 128 + +typedef struct INFOROM_ECC_OBJECT_V6_S0 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + + NvU64_ALIGN32 uncorrectedTotal; + NvU64_ALIGN32 correctedTotal; + inforom_U032 lastClearedTimestamp; + + INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER errorEntries[INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_MAX_COUNT]; + + inforom_U008 padding[68]; +} INFOROM_ECC_OBJECT_V6_S0; + +#define INFOROM_ECC_OBJECT_V6_S0_HEADER_FMT INFOROM_OBJECT_HEADER_V1_00_FMT + +#define INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_FMT "b3d2w3d" + +#define INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_ARRAY_FMT \ + INFOROM_FMT_REP128(INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_FMT) + +#define INFOROM_ECC_OBJECT_V6_S0_PADDING_FMT "68b" + +#define INFOROM_ECC_OBJECT_V6_S0_FMT INFOROM_ECC_OBJECT_V6_S0_HEADER_FMT "qqd" \ + INFOROM_ECC_OBJECT_V6_S0_ERROR_COUNTER_ARRAY_FMT \ + INFOROM_ECC_OBJECT_V6_S0_PADDING_FMT + +// Error event structure for NVSwitch ECC errors +typedef struct +{ + NvU32 sxid; + NvU32 linkId; + NvBool bAddressValid; + NvU32 address; + + // The timestamp is filled in by the inforom ECC error logging API + NvU32 timestamp; + + NvBool bUncErr; + NvU32 errorCount; +} INFOROM_NVS_ECC_ERROR_EVENT; + +typedef union +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_ECC_OBJECT_V6_S0 v6s; +} INFOROM_ECC_OBJECT; + +typedef struct +{ + const char *pFmt; + NvU8 *pPackedObject; + INFOROM_ECC_OBJECT *pEcc; + + // Signals if there are pending updates to be flushed to InfoROM + NvBool bDirty; +} INFOROM_ECC_STATE, *PINFOROM_ECC_STATE; + +#endif // _IFRECC_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrstruct.h b/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrstruct.h new file mode 100644 index 0000000..0ea7235 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/inforom/ifrstruct.h @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _IFRSTRUCT_H_ +#define _IFRSTRUCT_H_ + +#include "inforom/types.h" + +#include "inforom/ifrecc.h" +#include "inforom/ifrdem.h" + +#define INFOROM_OBD_OBJECT_V1_XX_PACKED_SIZE 128 + +struct INFOROM_OBD_OBJECT_V1_XX +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U032 buildDate; + inforom_U008 marketingName[24]; + inforom_U008 serialNumber[16]; + inforom_U008 memoryManufacturer; + inforom_U008 memoryPartID[20]; + inforom_U008 memoryDateCode[6]; + inforom_U008 productPartNumber[20]; + inforom_U008 boardRevision[3]; + inforom_U008 boardType; + inforom_U008 board699PartNumber[20]; + inforom_U008 reserved[5]; +}; +#define INFOROM_OBD_OBJECT_V1_XX_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d116b" +typedef struct INFOROM_OBD_OBJECT_V1_XX INFOROM_OBD_OBJECT_V1_XX; + +// +// OEM 1.0 +// + +#define INFOROM_OEM_OBJECT_V1_00_PACKED_SIZE 512 +#define INFOROM_OEM_OBJECT_HEADER_VERSION 1 + +struct INFOROM_OEM_OBJECT_V1_00 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U008 oemInfo[504]; +}; +#define INFOROM_OEM_OBJECT_V1_00_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "504b" +typedef struct INFOROM_OEM_OBJECT_V1_00 INFOROM_OEM_OBJECT_V1_00; + +#define INFOROM_IMG_OBJECT_V1_00_PACKED_SIZE 64 +#define INFOROM_IMG_OBJECT_V1_00_VERSION_LENGTH 16 + +struct INFOROM_IMG_OBJECT_V1_00 +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U008 version[INFOROM_IMG_OBJECT_V1_00_VERSION_LENGTH]; + inforom_U016 pciDeviceId; + inforom_U016 pciVendorId; + inforom_U016 pciSubsystemId; + inforom_U016 pciSubsystemVendorId; + inforom_U008 reserved[32]; +}; +#define INFOROM_IMG_OBJECT_V1_00_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "16b4w32b" +typedef struct INFOROM_IMG_OBJECT_V1_00 INFOROM_IMG_OBJECT_V1_00; + +#endif // _IFRSTRUCT_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/inforom/omsdef.h b/userspace/sdks/nv525/nvalloc/common/inc/inforom/omsdef.h new file mode 100644 index 0000000..921482e --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/inforom/omsdef.h @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _INFOROM_OMSDEF_H_ +#define _INFOROM_OMSDEF_H_ + +#include "inforom/types.h" + +#define INFOROM_OMS_OBJECT_V1_PACKED_SIZE 112 + +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_ENTRY_AVAILABLE 0:0 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_ENTRY_AVAILABLE_NO 0 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_ENTRY_AVAILABLE_YES 1 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_FORCE_DEVICE_DISABLE 1:1 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_FORCE_DEVICE_DISABLE_NO 0 +#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_FORCE_DEVICE_DISABLE_YES 1 + +#define INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY_DATA_RESERVED 7:2 +#define INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY_DATA_ENTRY_CHECKSUM 15:8 + +typedef struct INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY +{ + inforom_U016 data; +} INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY; + +#define INFOROM_OMS_OBJECT_V1S_NUM_SETTINGS_ENTRIES 50 + +typedef struct INFOROM_OMS_OBJECT_V1S +{ + INFOROM_OBJECT_HEADER_V1_00 header; + inforom_U032 lifetimeRefreshCount; + INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY settings[ + INFOROM_OMS_OBJECT_V1S_NUM_SETTINGS_ENTRIES]; +} INFOROM_OMS_OBJECT_V1S; + +#define INFOROM_OMS_OBJECT_V1S_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d50w" + +typedef struct INFOROM_OMS_V1S_DATA +{ + INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY *pIter; + INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY prev; + INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY *pNext; +} INFOROM_OMS_V1S_DATA; + +typedef union +{ + INFOROM_OBJECT_HEADER_V1_00 header; + INFOROM_OMS_OBJECT_V1S v1s; +} INFOROM_OMS_OBJECT; + +typedef union +{ + INFOROM_OMS_V1S_DATA v1s; +} INFOROM_OMS_DATA; + +typedef struct +{ + const char *pFmt; + NvU8 *pPackedObject; + INFOROM_OMS_OBJECT *pOms; + INFOROM_OMS_DATA omsData; +} INFOROM_OMS_STATE; + +#endif /* _INFOROM_OMSDEF_H_ */ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/inforom/types.h b/userspace/sdks/nv525/nvalloc/common/inc/inforom/types.h new file mode 100644 index 0000000..c062cb0 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/inforom/types.h @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _INFOROM_TYPES_H_ +#define _INFOROM_TYPES_H_ + +/*! + * @file types.h + * @brief Common types and definitions used by InfoROM objects + */ + +typedef NvS32 inforom_S008; +typedef NvU32 inforom_U004; +typedef NvU32 inforom_U008; +typedef NvU32 inforom_U016; +typedef NvU32 inforom_U024; +typedef NvU32 inforom_U032; +typedef NvU64 inforom_U064; +typedef NvU8 inforom_X008; + +#define INFOROM_FMT_S08 's' +#define INFOROM_FMT_U04 'n' +#define INFOROM_FMT_U08 'b' +#define INFOROM_FMT_U16 'w' +#define INFOROM_FMT_U24 't' +#define INFOROM_FMT_U32 'd' +#define INFOROM_FMT_U64 'q' +#define INFOROM_FMT_BINARY 'x' + +// Helper macros for generating repeating format sequences +#define INFOROM_FMT_REP02(fmt) fmt fmt +#define INFOROM_FMT_REP04(fmt) INFOROM_FMT_REP02(fmt) INFOROM_FMT_REP02(fmt) +#define INFOROM_FMT_REP08(fmt) INFOROM_FMT_REP04(fmt) INFOROM_FMT_REP04(fmt) +#define INFOROM_FMT_REP16(fmt) INFOROM_FMT_REP08(fmt) INFOROM_FMT_REP08(fmt) +#define INFOROM_FMT_REP32(fmt) INFOROM_FMT_REP16(fmt) INFOROM_FMT_REP16(fmt) +#define INFOROM_FMT_REP64(fmt) INFOROM_FMT_REP32(fmt) INFOROM_FMT_REP32(fmt) +#define INFOROM_FMT_REP128(fmt) INFOROM_FMT_REP64(fmt) INFOROM_FMT_REP64(fmt) +#define INFOROM_FMT_REP256(fmt) INFOROM_FMT_REP128(fmt) INFOROM_FMT_REP128(fmt) +#define INFOROM_FMT_REP512(fmt) INFOROM_FMT_REP256(fmt) INFOROM_FMT_REP256(fmt) + +#define INFOROM_OBJECT_SUBVERSION_SUPPORTS_NVSWITCH(sv) \ + ((((sv) & 0xF0) == 0) || (((sv) & 0xF0) == 0x20)) + +#define INFOROM_OBJECT_HEADER_V1_00_SIZE_OFFSET 0x05 +#define INFOROM_OBJECT_HEADER_V1_00_CHECKSUM_OFFSET 0x07 +#define INFOROM_OBJECT_HEADER_V1_00_PACKED_SIZE 8 + +typedef struct INFOROM_OBJECT_HEADER_V1_00 +{ + inforom_S008 type[3]; + inforom_U008 version; + inforom_U008 subversion; + inforom_U016 size; + inforom_U008 checksum; +} INFOROM_OBJECT_HEADER_V1_00; + +#define INFOROM_OBJECT_HEADER_V1_00_FMT "3s2bwb" + +#endif // _INFOROM_TYPES_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/nv-firmware-chip-family-select.h b/userspace/sdks/nv525/nvalloc/common/inc/nv-firmware-chip-family-select.h new file mode 100644 index 0000000..068cc44 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/nv-firmware-chip-family-select.h @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NV_FIRMWARE_CHIP_FAMILY_SELECT_H +#define NV_FIRMWARE_CHIP_FAMILY_SELECT_H + + + +#include <published/nv_arch.h> +#include <nv-firmware.h> + +static inline nv_firmware_chip_family_t nv_firmware_get_chip_family( + NvU32 gpuArch, + NvU32 gpuImpl +) +{ + switch (gpuArch) + { + case GPU_ARCHITECTURE_TURING: + if (gpuImpl <= GPU_IMPLEMENTATION_TU106) + return NV_FIRMWARE_CHIP_FAMILY_TU10X; + else + return NV_FIRMWARE_CHIP_FAMILY_TU11X; + + case GPU_ARCHITECTURE_AMPERE: + if (gpuImpl == GPU_IMPLEMENTATION_GA100) + return NV_FIRMWARE_CHIP_FAMILY_GA100; + else + return NV_FIRMWARE_CHIP_FAMILY_GA10X; + + case GPU_ARCHITECTURE_ADA: + return NV_FIRMWARE_CHIP_FAMILY_AD10X; + + case GPU_ARCHITECTURE_HOPPER: + if (gpuImpl == GPU_IMPLEMENTATION_GH100) + return NV_FIRMWARE_CHIP_FAMILY_GH100; + } + + return NV_FIRMWARE_CHIP_FAMILY_NULL; +} + +#endif // NV_FIRMWARE_CHIP_FAMILY_SELECT_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/nv-firmware.h b/userspace/sdks/nv525/nvalloc/common/inc/nv-firmware.h new file mode 100644 index 0000000..3ddade5 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/nv-firmware.h @@ -0,0 +1,132 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NV_FIRMWARE_H +#define NV_FIRMWARE_H + + + +#include <nvtypes.h> +#include <nvmisc.h> + +typedef enum +{ + NV_FIRMWARE_TYPE_GSP, + NV_FIRMWARE_TYPE_GSP_LOG +} nv_firmware_type_t; + +typedef enum +{ + NV_FIRMWARE_CHIP_FAMILY_NULL = 0, + NV_FIRMWARE_CHIP_FAMILY_TU10X = 1, + NV_FIRMWARE_CHIP_FAMILY_TU11X = 2, + NV_FIRMWARE_CHIP_FAMILY_GA100 = 3, + NV_FIRMWARE_CHIP_FAMILY_GA10X = 4, + NV_FIRMWARE_CHIP_FAMILY_AD10X = 5, + NV_FIRMWARE_CHIP_FAMILY_GH100 = 6, + NV_FIRMWARE_CHIP_FAMILY_END, +} nv_firmware_chip_family_t; + +static inline const char *nv_firmware_chip_family_to_string( + nv_firmware_chip_family_t fw_chip_family +) +{ + switch (fw_chip_family) { + case NV_FIRMWARE_CHIP_FAMILY_GH100: return "gh100"; + case NV_FIRMWARE_CHIP_FAMILY_AD10X: return "ad10x"; + case NV_FIRMWARE_CHIP_FAMILY_GA10X: return "ga10x"; + case NV_FIRMWARE_CHIP_FAMILY_GA100: return "ga100"; + case NV_FIRMWARE_CHIP_FAMILY_TU11X: return "tu11x"; + case NV_FIRMWARE_CHIP_FAMILY_TU10X: return "tu10x"; + + case NV_FIRMWARE_CHIP_FAMILY_END: // fall through + case NV_FIRMWARE_CHIP_FAMILY_NULL: + return NULL; + } + return NULL; +} + +// The includer (presumably nv.c) may optionally define +// NV_FIRMWARE_PATH_FOR_FILENAME(filename) +// to return a string "path" given a gsp_*.bin or gsp_log_*.bin filename. +// +// The function nv_firmware_path will then be available. +#if defined(NV_FIRMWARE_PATH_FOR_FILENAME) +static inline const char *nv_firmware_path( + nv_firmware_type_t fw_type, + nv_firmware_chip_family_t fw_chip_family +) +{ + if (fw_type == NV_FIRMWARE_TYPE_GSP) + { + switch (fw_chip_family) + { + case NV_FIRMWARE_CHIP_FAMILY_AD10X: + return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_ad10x.bin"); + + case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GA100: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GA10X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_TU11X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_TU10X: + return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_tu10x.bin"); + + case NV_FIRMWARE_CHIP_FAMILY_END: // fall through + case NV_FIRMWARE_CHIP_FAMILY_NULL: + return ""; + } + } + else if (fw_type == NV_FIRMWARE_TYPE_GSP_LOG) + { + switch (fw_chip_family) + { + case NV_FIRMWARE_CHIP_FAMILY_AD10X: + return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_log_ad10x.bin"); + + case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GA100: // fall through + case NV_FIRMWARE_CHIP_FAMILY_GA10X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_TU11X: // fall through + case NV_FIRMWARE_CHIP_FAMILY_TU10X: + return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_log_tu10x.bin"); + + case NV_FIRMWARE_CHIP_FAMILY_END: // fall through + case NV_FIRMWARE_CHIP_FAMILY_NULL: + return ""; + } + } + + return ""; +} +#endif // defined(NV_FIRMWARE_PATH_FOR_FILENAME) + +// The includer (presumably nv.c) may optionally define +// NV_FIRMWARE_DECLARE_GSP_FILENAME(filename) +// which will then be invoked (at the top-level) for each +// gsp_*.bin (but not gsp_log_*.bin) +#if defined(NV_FIRMWARE_DECLARE_GSP_FILENAME) +NV_FIRMWARE_DECLARE_GSP_FILENAME("gsp_ad10x.bin") +NV_FIRMWARE_DECLARE_GSP_FILENAME("gsp_tu10x.bin") +#endif // defined(NV_FIRMWARE_DECLARE_GSP_FILENAME) + +#endif // NV_FIRMWARE_DECLARE_GSP_FILENAME diff --git a/userspace/sdks/nv525/nvalloc/common/inc/nvcst.h b/userspace/sdks/nv525/nvalloc/common/inc/nvcst.h new file mode 100644 index 0000000..9a86d5e --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/nvcst.h @@ -0,0 +1,353 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2004-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVCST_H +#define NVCST_H + +#include <platform/chipset/chipset.h> +#include <platform/chipset/chipset_info.h> +#include <nvpcie.h> +#include <nvdevid.h> + +#define CHIPSET_SETUP_FUNC(name) static NV_STATUS name(OBJCL *pCl); + +CHIPSET_SETUP_FUNC(Intel_25XX_setupFunc) +CHIPSET_SETUP_FUNC(Intel_27XX_setupFunc) +CHIPSET_SETUP_FUNC(Intel_2A40_setupFunc) +CHIPSET_SETUP_FUNC(Intel_0040_setupFunc) +CHIPSET_SETUP_FUNC(Intel_2E00_setupFunc) +CHIPSET_SETUP_FUNC(Intel_25E0_setupFunc) +CHIPSET_SETUP_FUNC(Intel_29XX_setupFunc) +CHIPSET_SETUP_FUNC(Intel_29X0_setupFunc) +CHIPSET_SETUP_FUNC(Intel_29E0_setupFunc) +CHIPSET_SETUP_FUNC(Intel_359E_setupFunc) +CHIPSET_SETUP_FUNC(Intel_4000_setupFunc) +CHIPSET_SETUP_FUNC(Intel_4003_setupFunc) +CHIPSET_SETUP_FUNC(Intel_3400_setupFunc) +CHIPSET_SETUP_FUNC(Intel_3B42_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1C46_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1C10_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1C4B_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1C49_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1D40_setupFunc) +CHIPSET_SETUP_FUNC(Intel_8D47_setupFunc) +CHIPSET_SETUP_FUNC(Intel_8D44_setupFunc) +CHIPSET_SETUP_FUNC(Intel_1E10_setupFunc) +CHIPSET_SETUP_FUNC(Intel_8C4B_setupFunc) +CHIPSET_SETUP_FUNC(Intel_8CC4_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A145_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A2C5_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A242_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A2D2_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A2C9_setupFunc) +CHIPSET_SETUP_FUNC(Intel_A301_setupFunc) +CHIPSET_SETUP_FUNC(Intel_0685_setupFunc) +CHIPSET_SETUP_FUNC(Intel_IceLake_setupFunc) +CHIPSET_SETUP_FUNC(Intel_4381_setupFunc) +CHIPSET_SETUP_FUNC(Intel_7A82_setupFunc) +CHIPSET_SETUP_FUNC(Intel_7A04_setupFunc) +CHIPSET_SETUP_FUNC(SiS_656_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RS400_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RS480_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RD870_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RD890_setupFunc) +CHIPSET_SETUP_FUNC(ATI_RX780_setupFunc) +CHIPSET_SETUP_FUNC(ATI_FX990_setupFunc) +CHIPSET_SETUP_FUNC(AMD_RS780_setupFunc) +CHIPSET_SETUP_FUNC(AMD_FX790_setupFunc) +CHIPSET_SETUP_FUNC(AMD_FX890_setupFunc) +CHIPSET_SETUP_FUNC(AMD_X370_setupFunc) +CHIPSET_SETUP_FUNC(VIA_VX900_setupFunc) +CHIPSET_SETUP_FUNC(APM_Storm_setupFunc) +CHIPSET_SETUP_FUNC(ARMV8_generic_setupFunc) +CHIPSET_SETUP_FUNC(Marvell_ThunderX2_setupFunc) +CHIPSET_SETUP_FUNC(QEMU_setupFunc) +CHIPSET_SETUP_FUNC(Ampere_eMag_setupFunc) +CHIPSET_SETUP_FUNC(Huawei_Kunpeng920_setupFunc) +CHIPSET_SETUP_FUNC(Mellanox_BlueField_setupFunc) +CHIPSET_SETUP_FUNC(Amazon_Gravitron2_setupFunc) +CHIPSET_SETUP_FUNC(Fujitsu_A64FX_setupFunc) +CHIPSET_SETUP_FUNC(Ampere_Altra_setupFunc) +CHIPSET_SETUP_FUNC(Arm_NeoverseN1_setupFunc) +CHIPSET_SETUP_FUNC(Nvidia_T210_setupFunc) +CHIPSET_SETUP_FUNC(Nvidia_T194_setupFunc) + + +// Keep string length <=32 (including termination) to avoid string copy overflow +CSINFO chipsetInfo[] = +{ + // PCI Express chipset + + {PCI_VENDOR_ID_INTEL, 0x2580, CS_INTEL_2580, "Grantsdale", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2584, CS_INTEL_2584, "Alderwood", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2588, CS_INTEL_2588, "Intel2588", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2590, CS_INTEL_2590, "Alviso", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x25C0, CS_INTEL_25E0, "Greencreek", Intel_25E0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x25E0, CS_INTEL_25E0, "Greencreek", Intel_25E0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x29B0, CS_INTEL_29X0, "IntelQ35", Intel_29X0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x29C0, CS_INTEL_29X0, "BearlakeB", Intel_29X0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x29D0, CS_INTEL_29X0, "IntelQ33", Intel_29X0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x29E0, CS_INTEL_29E0, "BearlakeX", Intel_29E0_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x359E, CS_INTEL_359E, "Tumwater", Intel_359E_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x4000, CS_INTEL_4000, "Stoakley", Intel_4000_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x4003, CS_INTEL_4003, "SkullTrail", Intel_4003_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3400, CS_INTEL_3400, "IntelX58", Intel_3400_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3403, CS_INTEL_3400, "IntelX58", Intel_3400_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3405, CS_INTEL_3400, "IntelX58", Intel_3400_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3406, CS_INTEL_3400, "Tylersburg", Intel_3400_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2770, CS_INTEL_2770, "Lakeport", Intel_25XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2774, CS_INTEL_2774, "Glenwood", Intel_27XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x277C, CS_INTEL_277C, "Glenwood", Intel_27XX_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2A40, CS_INTEL_2A40, "Montevina", Intel_2A40_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2E00, CS_INTEL_2E00, "Eaglelake", Intel_2E00_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2E10, CS_INTEL_2E00, "Eaglelake", Intel_2E00_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2E20, CS_INTEL_2E00, "Eaglelake", Intel_2E00_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x2E30, CS_INTEL_2E00, "Eaglelake", Intel_2E00_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0044, CS_INTEL_0040, "Arrandale/Auburndale", Intel_0040_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0062, CS_INTEL_0040, "Arrandale/Auburndale", Intel_0040_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xD130, CS_INTEL_3B42, "Clarksfield", Intel_3B42_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xD132, CS_INTEL_3B42, "Clarksfield", Intel_3B42_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x3B42, CS_INTEL_3B42, "P55/PM55/H57", Intel_3B42_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C46, CS_INTEL_1C46, "IntelP67-CougarPoint", Intel_1C46_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C4B, CS_INTEL_1C46, "HuronRiver-HM67", Intel_1C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C4F, CS_INTEL_1C46, "HuronRiver-QM67", Intel_1C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C49, CS_INTEL_1C49, "HuronRiver-HM65", Intel_1C49_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C44, CS_INTEL_1C46, "IntelZ68", Intel_1C46_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1C10, CS_INTEL_1C10, "IntelP67", Intel_1C10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1D40, CS_INTEL_1D40, "IntelX79", Intel_1D40_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1D41, CS_INTEL_1D40, "IntelX79", Intel_1D40_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1E10, CS_INTEL_1E10, "IntelZ75", Intel_1E10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0150, CS_INTEL_1E10, "IntelZ77A-GD55", Intel_1E10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0151, CS_INTEL_1E10, "IntelZ77A-GD55", Intel_1E10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0100, CS_INTEL_1E10, "IntelZ77A-GD55", Intel_1E10_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C4B, CS_INTEL_8C4B, "SharkBay-HM87", Intel_8C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C44, CS_INTEL_8C4B, "SharkBay-Z87", Intel_8C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C41, CS_INTEL_8C4B, "SharkBay-H8x/P8x", Intel_8C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C49, CS_INTEL_8C4B, "SharkBay-HM86", Intel_8C4B_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8C52, CS_INTEL_8C4B, "SharkBay-E3", Intel_8C4B_setupFunc}, // Does not support SLI + {PCI_VENDOR_ID_INTEL, 0x8CC4, CS_INTEL_8CC4, "IntelZ97", Intel_8CC4_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8CC3, CS_INTEL_8CC4, "IntelHM97", Intel_8CC4_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA145, CS_INTEL_A145, "IntelZ170", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA14E, CS_INTEL_A145, "IntelHM170", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA150, CS_INTEL_A145, "IntelHM170", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA149, CS_INTEL_A145, "SkyLake C236", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA14A, CS_INTEL_A145, "SkyLake C232", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA14D, CS_INTEL_A145, "SkyLake-H", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA244, CS_INTEL_A145, "SkyLake C620", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C8, CS_INTEL_A145, "SkyLake C620", Intel_A145_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8D47, CS_INTEL_8D47, "IntelX99", Intel_8D47_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x8D44, CS_INTEL_8D47, "IntelC612", Intel_8D44_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA2C5, CS_INTEL_A2C5, "IntelZ270", Intel_A2C5_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA154, CS_INTEL_A2C5, "IntelZ270", Intel_A2C5_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA152, CS_INTEL_A2C5, "IntelRX9S", Intel_A2C5_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA242, CS_INTEL_A242, "IntelC422", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA241, CS_INTEL_A242, "IntelC422", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA243, CS_INTEL_A242, "IntelC422", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA2D2, CS_INTEL_A2D2, "IntelX299", Intel_A2D2_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA2D3, CS_INTEL_A242, "IntelC422", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C1, CS_INTEL_A242, "IntelC621", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C2, CS_INTEL_A242, "IntelC622", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C3, CS_INTEL_A242, "IntelC624", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C4, CS_INTEL_A242, "IntelC625", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C5, CS_INTEL_A242, "IntelC626", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C6, CS_INTEL_A242, "IntelC627", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1C7, CS_INTEL_A242, "IntelC628", Intel_A242_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA2C9, CS_INTEL_A2C9, "IntelZ370", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA310, CS_INTEL_A2C9, "IntelZ370", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA30E, CS_INTEL_A2C9, "IntelZ370", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA305, CS_INTEL_A2C9, "IntelZ390", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA30D, CS_INTEL_A2C9, "IntelH370", Intel_A2C9_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA301, CS_INTEL_A301, "Intel-CannonLake", Intel_A301_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x0685, CS_INTEL_0685, "Intel-CometLake", Intel_0685_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0xA1CB, CS_INTEL_C620, "Intel-IceLake", Intel_IceLake_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x4381, CS_INTEL_4381, "Intel-RocketLake", Intel_4381_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x4385, CS_INTEL_4381, "Intel-RocketLake", Intel_4381_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7A82, CS_INTEL_7A82, "Intel-AlderLake", Intel_7A82_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7A84, CS_INTEL_7A82, "Intel-AlderLake", Intel_7A82_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x1B81, CS_INTEL_1B81, "Intel-SapphireRapids", NULL}, + {PCI_VENDOR_ID_INTEL, 0x18DC, CS_INTEL_18DC, "Intel-IceLake", Intel_IceLake_setupFunc}, + {PCI_VENDOR_ID_INTEL, 0x7A04, CS_INTEL_7A04, "Intel-RaptorLake", Intel_7A04_setupFunc}, + + {PCI_VENDOR_ID_NVIDIA, 0x0FAE, CS_NVIDIA_T210, "T210", Nvidia_T210_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x0FAF, CS_NVIDIA_T210, "T210", Nvidia_T210_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x10E5, CS_NVIDIA_T186, "T186", Nvidia_T210_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x10E6, CS_NVIDIA_T186, "T186", Nvidia_T210_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x1AD0, CS_NVIDIA_T194, "T194", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x1AD1, CS_NVIDIA_T194, "T194", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x1AD2, CS_NVIDIA_T194, "T194", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x229A, CS_NVIDIA_T234, "T234", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x229C, CS_NVIDIA_T234, "T234", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x229E, CS_NVIDIA_T234, "T234", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22C2, CS_NVIDIA_T23x, "T23x", Nvidia_T194_setupFunc}, + {PCI_VENDOR_ID_NVIDIA, 0x22C3, CS_NVIDIA_T23x, "T23x", Nvidia_T194_setupFunc}, + + {PCI_VENDOR_ID_SIS, 0x0649, CS_SIS_649, "649", SiS_656_setupFunc}, + {PCI_VENDOR_ID_SIS, 0x0656, CS_SIS_656, "656", SiS_656_setupFunc}, + + {PCI_VENDOR_ID_ATI, 0x5A31, CS_ATI_RS400, "RS400", ATI_RS400_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5A33, CS_ATI_RS400, "RS400", ATI_RS400_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5950, CS_ATI_RS480, "RS480", ATI_RS480_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5951, CS_ATI_RS480, "RS480", ATI_RS480_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5956, CS_ATI_FX790, "FX790" ,AMD_FX790_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5A11, CS_ATI_FX890, "FX890" ,AMD_FX890_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5a13, CS_ATI_RD850, "RD850" ,ATI_RD870_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5a12, CS_ATI_RD870, "RD870" ,ATI_RD870_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5a10, CS_ATI_RD890, "RD890" ,ATI_RD890_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5957, CS_ATI_RX780, "RX780" ,ATI_RX780_setupFunc}, + {PCI_VENDOR_ID_ATI, 0x5A14, CS_ATI_FX990, "FX990/X990/970",ATI_FX990_setupFunc}, + + {PCI_VENDOR_ID_AMD, 0x9601, CS_AMD_GX890, "GX890" ,AMD_FX890_setupFunc}, + {PCI_VENDOR_ID_AMD, 0x9600, CS_AMD_RS780, "RS780" ,AMD_RS780_setupFunc}, + {PCI_VENDOR_ID_AMD, 0x790e, CS_AMD_X370, "X370/X399/X470/ TRX40/X570/WRX80", AMD_X370_setupFunc }, + + {PCI_VENDOR_ID_VIA, 0x0308, CS_VIA_VT8369B, "VT8369B", NULL}, + {PCI_VENDOR_ID_VIA, 0x0410, CS_VIA_VX900, "VX900", VIA_VX900_setupFunc}, + {PCI_VENDOR_ID_APM, 0xe004, CS_APM_STORM, "X-Gene Storm", APM_Storm_setupFunc}, + {PCI_VENDOR_ID_IBM, 0x03DC, CS_IBM_VENICE, "Venice", NULL}, + {PCI_VENDOR_ID_MARVELL, 0xAF00, CS_MARVELL_THUNDERX2, "Marvell ThunderX2", Marvell_ThunderX2_setupFunc}, + {PCI_VENDOR_ID_REDHAT, 0x0008, CS_REDHAT_QEMU, "QEMU Redhat", QEMU_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE005, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE006, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE007, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE008, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE009, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00A, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00B, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00C, CS_AMPERE_EMAG, "AMPERE eMag", Ampere_eMag_setupFunc}, + {PCI_VENDOR_ID_HUAWEI, 0xA120, CS_HUAWEI_KUNPENG920, "Huawei Kunpeng920", Huawei_Kunpeng920_setupFunc}, + {PCI_VENDOR_ID_MELLANOX, 0xA2D0, CS_MELLANOX_BLUEFIELD, "Mellanox BlueField", Mellanox_BlueField_setupFunc}, + {PCI_VENDOR_ID_MELLANOX, 0xA2D4, CS_MELLANOX_BLUEFIELD2, "Mellanox BlueField 2", NULL}, + {PCI_VENDOR_ID_MELLANOX, 0xA2D5, CS_MELLANOX_BLUEFIELD2, "Mellanox BlueField 2 Crypto disabled", NULL}, + {PCI_VENDOR_ID_AMAZON, 0x0200, CS_AMAZON_GRAVITRON2, "Amazon Gravitron2", Amazon_Gravitron2_setupFunc}, + {PCI_VENDOR_ID_FUJITSU, 0x1952, CS_FUJITSU_A64FX, "Fujitsu A64FX", Fujitsu_A64FX_setupFunc}, + {PCI_VENDOR_ID_CADENCE, 0xDC01, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xDC08, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xDC16, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xFC01, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xFC08, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_CADENCE, 0xFC16, CS_PHYTIUM_S2500, "Phytium S2500", NULL}, + {PCI_VENDOR_ID_AMPERE, 0xE000, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00D, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE00E, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE010, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE100, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_AMPERE, 0xE110, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc}, + {PCI_VENDOR_ID_ARM, 0x0100, CS_ARM_NEOVERSEN1, "Arm Neoverse N1", Arm_NeoverseN1_setupFunc}, + {PCI_VENDOR_ID_HYGON, 0x790E, CS_HYGON_C86, "Hygon-C86-7151", NULL}, + {PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX, "Marvell Octeon CN96xx", ARMV8_generic_setupFunc}, + {PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX, "Marvell Octeon CN98xx", ARMV8_generic_setupFunc}, + +/////////////////////////////////////////////////////////////////////////////////////////////////// + + // last element must have chipset CS_UNKNOWN (zero) + {0, 0, CS_UNKNOWN, "Unknown", NULL} +}; + + +VENDORNAME vendorName[] = +{ + {PCI_VENDOR_ID_NVIDIA, "NVIDIA"}, + {PCI_VENDOR_ID_INTEL, "Intel"}, + {PCI_VENDOR_ID_VIA, "VIA"}, + {PCI_VENDOR_ID_RCC, "ServerWorks"}, + {PCI_VENDOR_ID_MICRON_1, "Micron"}, + {PCI_VENDOR_ID_MICRON_2, "Micron"}, + {PCI_VENDOR_ID_APPLE, "Apple"}, + {PCI_VENDOR_ID_SIS, "SiS"}, + {PCI_VENDOR_ID_ATI, "ATI"}, + {PCI_VENDOR_ID_TRANSMETA, "Transmeta"}, + {PCI_VENDOR_ID_HP, "HP"}, + {PCI_VENDOR_ID_AMD, "AMD"}, + {PCI_VENDOR_ID_ALI, "ALi"}, + {PCI_VENDOR_ID_APM, "AppliedMicro"}, + {PCI_VENDOR_ID_IBM, "IBM"}, + {PCI_VENDOR_ID_MARVELL, "MarvellThunderX2"}, + {PCI_VENDOR_ID_REDHAT, "QemuRedhat"}, + {PCI_VENDOR_ID_AMPERE, "AmpereComputing"}, + {PCI_VENDOR_ID_HUAWEI, "Huawei"}, + {PCI_VENDOR_ID_MELLANOX, "Mellanox"}, + {PCI_VENDOR_ID_AMAZON, "Amazon"}, + {PCI_VENDOR_ID_FUJITSU, "Fujitsu"}, + {PCI_VENDOR_ID_CADENCE, "Cadence"}, + {PCI_VENDOR_ID_ARM, "ARM"}, + {0, "Unknown"} // Indicates end of the table +}; + + +// +// Allowlist all chipsets with which dGPU over PCIe is supported on ARM +// (both v7 and v8) platforms +// +ARMCSALLOWLISTINFO armChipsetAllowListInfo[] = +{ + {PCI_VENDOR_ID_NVIDIA, 0x0FAE, CS_NVIDIA_T210}, // NVIDIA Tegra X1 RP0 + {PCI_VENDOR_ID_NVIDIA, 0x0FAF, CS_NVIDIA_T210}, // NVIDIA Tegra X1 RP1 + {PCI_VENDOR_ID_NVIDIA, 0x10E5, CS_NVIDIA_T186}, // NVIDIA Tegra P1 RP0 + {PCI_VENDOR_ID_NVIDIA, 0x10E6, CS_NVIDIA_T186}, // NVIDIA Tegra P1 RP1 + {PCI_VENDOR_ID_NVIDIA, 0x1AD0, CS_NVIDIA_T194}, // NVIDIA Tegra V1 RP0 + {PCI_VENDOR_ID_NVIDIA, 0x1AD1, CS_NVIDIA_T194}, // NVIDIA Tegra V1 RP1 + {PCI_VENDOR_ID_NVIDIA, 0x1AD2, CS_NVIDIA_T194}, // NVIDIA Tegra V1 RP2 + {PCI_VENDOR_ID_NVIDIA, 0x229A, CS_NVIDIA_T234}, // NVIDIA Tegra Orin RP0 + {PCI_VENDOR_ID_NVIDIA, 0x229C, CS_NVIDIA_T234}, // NVIDIA Tegra Orin RP1 + {PCI_VENDOR_ID_NVIDIA, 0x229E, CS_NVIDIA_T234}, // NVIDIA Tegra Orin RP2 + {PCI_VENDOR_ID_NVIDIA, 0x22C2, CS_NVIDIA_T23x}, // NVIDIA Tegra RP0 + {PCI_VENDOR_ID_NVIDIA, 0x22C3, CS_NVIDIA_T23x}, // NVIDIA Tegra RP1 + + {PCI_VENDOR_ID_APM, 0xe004, CS_APM_STORM}, // Applied Micro X-Gene "Storm" + {PCI_VENDOR_ID_MARVELL, 0xAF00, CS_MARVELL_THUNDERX2}, // Marvell ThunderX2 + {PCI_VENDOR_ID_REDHAT, 0x0008, CS_REDHAT_QEMU}, // Redhat QEMU + {PCI_VENDOR_ID_AMPERE, 0xE005, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE006, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE007, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE008, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE009, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE00A, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE00B, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_AMPERE, 0xE00C, CS_AMPERE_EMAG}, // Ampere eMag + {PCI_VENDOR_ID_HUAWEI, 0xA120, CS_HUAWEI_KUNPENG920}, // Huawei Kunpeng 920 + {PCI_VENDOR_ID_MELLANOX, 0xA2D0, CS_MELLANOX_BLUEFIELD}, // Mellanox BlueField + {PCI_VENDOR_ID_MELLANOX, 0xA2D4, CS_MELLANOX_BLUEFIELD2},// Mellanox BlueField 2 + {PCI_VENDOR_ID_MELLANOX, 0xA2D5, CS_MELLANOX_BLUEFIELD2},// Mellanox BlueField 2 Crypto disabled + {PCI_VENDOR_ID_AMAZON, 0x0200, CS_AMAZON_GRAVITRON2}, // Amazon Gravitron2 + {PCI_VENDOR_ID_FUJITSU, 0x1952, CS_FUJITSU_A64FX}, // Fujitsu A64FX + {PCI_VENDOR_ID_CADENCE, 0xDC01, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xDC08, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xDC16, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xFC01, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xFC08, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_CADENCE, 0xFC16, CS_PHYTIUM_S2500}, // Phytium S2500 + {PCI_VENDOR_ID_AMPERE, 0xE000, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE00D, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE00E, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE010, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE100, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_AMPERE, 0xE110, CS_AMPERE_ALTRA}, // Ampere Altra + {PCI_VENDOR_ID_ARM, 0x0100, CS_ARM_NEOVERSEN1}, // Arm Neoverse N1 + {PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX}, // Marvell OCTEON CN96xx + {PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX}, // Marvell OCTEON CN98xx + + // last element must have chipset CS_UNKNOWN (zero) + {0, 0, CS_UNKNOWN} +}; + +#endif /* NVCST_H */ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/nvdevid.h b/userspace/sdks/nv525/nvalloc/common/inc/nvdevid.h new file mode 100644 index 0000000..559bcb7 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/nvdevid.h @@ -0,0 +1,697 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 200-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVDEVID_H +#define NVDEVID_H + + + +/**************** Resource Manager Defines and Structures ******************\ +* * +* Private device ids defines - only defines ! * +* * +\***************************************************************************/ + +/////////////////////////////////////////////////////////////////////////////////////////// +// +// VENDOR/SUBVENDOR IDS +// XXX Cleanup to do: change PCI_VENDOR_* to NV_PCI_SUBID_VENDOR_* +// +/////////////////////////////////////////////////////////////////////////////////////////// +#define NV_PCI_SUBID_VENDOR 15:0 /* RW--F */ +#define NV_PCI_SUBID_VENDOR_AMD 0x1022 +#define NV_PCI_SUBID_VENDOR_ALI 0x10B9 +#define NV_PCI_SUBID_VENDOR_NVIDIA 0x10DE +#define NV_PCI_SUBID_VENDOR_INTEL 0x8086 +#define NV_PCI_SUBID_VENDOR_VIA 0x1106 +#define NV_PCI_SUBID_VENDOR_RCC 0x1166 +#define NV_PCI_SUBID_VENDOR_MICRON_1 0x1042 +#define NV_PCI_SUBID_VENDOR_MICRON_2 0x1344 +#define NV_PCI_SUBID_VENDOR_APPLE 0x106B +#define NV_PCI_SUBID_VENDOR_SIS 0x1039 +#define NV_PCI_SUBID_VENDOR_ATI 0x1002 +#define NV_PCI_SUBID_VENDOR_TRANSMETA 0x1279 +#define NV_PCI_SUBID_VENDOR_HP 0x103C +#define NV_PCI_SUBID_VENDOR_DELL 0x1028 +#define NV_PCI_SUBID_VENDOR_FUJITSU 0x10cf +#define NV_PCI_SUBID_VENDOR_ASUS 0x1043 +#define NV_PCI_SUBID_VENDOR_MSI 0x1462 +#define NV_PCI_SUBID_VENDOR_FOXCONN 0x105B +#define NV_PCI_SUBID_VENDOR_ECS 0x1019 +#define NV_PCI_SUBID_VENDOR_DFI_1 0x106E +#define NV_PCI_SUBID_VENDOR_TOSHIBA 0x1179 +#define NV_PCI_SUBID_VENDOR_DFI_2 0x15BD +#define NV_PCI_SUBID_VENDOR_ACER 0x1025 +#define NV_PCI_SUBID_VENDOR_GIGABYTE 0x1458 +#define NV_PCI_SUBID_VENDOR_EVGA 0x3842 +#define NV_PCI_SUBID_VENDOR_BROADCOM 0x1166 +#define NV_PCI_SUBID_VENDOR_SUPERMICRO 0x15D9 +#define NV_PCI_SUBID_VENDOR_BIOSTAR 0x1565 +#define NV_PCI_SUBID_VENDOR_XFX 0x1682 +#define NV_PCI_SUBID_VENDOR_PCPARTNER 0x19DA +#define NV_PCI_SUBID_VENDOR_LENOVO 0x17AA +#define NV_PCI_SUBID_VENDOR_FSC 0x1734 +#define NV_PCI_SUBID_VENDOR_FTS 0x1734 +#define NV_PCI_SUBID_VENDOR_COLORFUL 0x7377 +#define NV_PCI_SUBID_VENDOR_ASROCK 0x1849 +#define NV_PCI_SUBID_VENDOR_SHUTTLE 0x1297 +#define NV_PCI_SUBID_VENDOR_CLEVO 0x1558 +#define NV_PCI_SUBID_VENDOR_PEGATRON 0x1B0A +#define NV_PCI_SUBID_VENDOR_JETWAY 0x16F3 +#define NV_PCI_SUBID_VENDOR_HIGHGRADE 0x1C6C +#define NV_PCI_SUBID_VENDOR_GALAXY 0x1B4C +#define NV_PCI_SUBID_VENDOR_ZOTAC 0x19DA +#define NV_PCI_SUBID_VENDOR_ARIMA 0x161F +#define NV_PCI_SUBID_VENDOR_BFG 0x19F1 +#define NV_PCI_SUBID_VENDOR_SONY 0x104D +#define NV_PCI_SUBID_VENDOR_BITLAND 0x1642 +#define NV_PCI_SUBID_VENDOR_PC_PARTNER 0x174B +#define NV_PCI_SUBID_VENDOR_NZXT 0x1D96 + +// XXX CKEANUP TO REMOVE IN FAVOR OF NV_PCI_SUBID_VENDOR_* +#define PCI_VENDOR_ID_AMD 0x1022 +#define PCI_VENDOR_ID_ALI 0x10B9 +#define PCI_VENDOR_ID_NVIDIA 0x10DE +#define PCI_VENDOR_ID_INTEL 0x8086 +#define PCI_VENDOR_ID_VIA 0x1106 +#define PCI_VENDOR_ID_RCC 0x1166 +#define PCI_VENDOR_ID_MICRON_1 0x1042 +#define PCI_VENDOR_ID_MICRON_2 0x1344 +#define PCI_VENDOR_ID_APPLE 0x106B +#define PCI_VENDOR_ID_SIS 0x1039 +#define PCI_VENDOR_ID_ATI 0x1002 +#define PCI_VENDOR_ID_TRANSMETA 0x1279 +#define PCI_VENDOR_ID_HP 0x103C +#define PCI_VENDOR_ID_DELL 0x1028 +#define PCI_VENDOR_ID_FUJITSU 0x10cf +#define PCI_VENDOR_ID_ASUS 0x1043 +#define PCI_VENDOR_ID_MSI 0x1462 +#define PCI_VENDOR_ID_FOXCONN 0x105B +#define PCI_VENDOR_ID_ECS 0x1019 +#define PCI_VENDOR_ID_DFI_1 0x106E +#define PCI_VENDOR_ID_TOSHIBA 0x1179 +#define PCI_VENDOR_ID_DFI_2 0x15BD +#define PCI_VENDOR_ID_ACER 0x1025 +#define PCI_VENDOR_ID_GIGABYTE 0x1458 +#define PCI_VENDOR_ID_EVGA 0x3842 +#define PCI_VENDOR_ID_BROADCOM 0x1166 +#define PCI_VENDOR_ID_SUPERMICRO 0x15D9 +#define PCI_VENDOR_ID_BIOSTAR 0x1565 +#define PCI_VENDOR_ID_XFX 0x1682 +#define PCI_VENDOR_ID_PCPARTNER 0x19DA +#define PCI_VENDOR_ID_LENOVO 0x17AA +#define PCI_VENDOR_ID_FSC 0x1734 +#define PCI_VENDOR_ID_FTS 0x1734 +#define PCI_VENDOR_ID_COLORFUL 0x7377 +#define PCI_VENDOR_ID_ASROCK 0x1849 +#define PCI_VENDOR_ID_SHUTTLE 0x1297 +#define PCI_VENDOR_ID_CLEVO 0x1558 +#define PCI_VENDOR_ID_PEGATRON 0x1B0A +#define PCI_VENDOR_ID_JETWAY 0x16F3 +#define PCI_VENDOR_ID_HIGHGRADE 0x1C6C +#define PCI_VENDOR_ID_GALAXY 0x1B4C +#define PCI_VENDOR_ID_ZOTAC 0x19DA +#define PCI_VENDOR_ID_ARIMA 0x161F +#define PCI_VENDOR_ID_PC_PARTNER 0x174B +#define PCI_VENDOR_ID_APM 0x10E8 +#define PCI_VENDOR_ID_IBM 0x1014 +#define PCI_VENDOR_ID_NZXT 0x1D96 +#define PCI_VENDOR_ID_MARVELL 0x177D +#define PCI_VENDOR_ID_REDHAT 0x1B36 +#define PCI_VENDOR_ID_AMPERE 0x1DEF +#define PCI_VENDOR_ID_HUAWEI 0x19E5 +#define PCI_VENDOR_ID_MELLANOX 0x15B3 +#define PCI_VENDOR_ID_AMAZON 0x1D0F +#define PCI_VENDOR_ID_CADENCE 0x17CD +#define PCI_VENDOR_ID_ARM 0x13B5 +#define PCI_VENDOR_ID_HYGON 0x1D94 + +#define NV_PCI_DEVID_DEVICE 31:16 /* RW--F */ +#define NV_PCI_SUBID_DEVICE 31:16 /* RW--F */ + +/////////////////////////////////////////////////////////////////////////////////////////// +// +// GPU DEVICE IDS +// +/////////////////////////////////////////////////////////////////////////////////////////// + +#define NV_PCI_DEVID_DEVICE_PG171_SKU200_PG179_SKU220 0x25B6 /* NVIDIA A16 / NVIDIA A2 */ +#define NV_PCI_DEVID_DEVICE_PG189_SKU600 0x1EBA + +/////////////////////////////////////////////////////////////////////////////////////////// +// +// SUBDEVICE IDs +// +/////////////////////////////////////////////////////////////////////////////////////////// + +// A16 +#define NV_PCI_SUBID_DEVICE_PG171_SKU200 0x14A9 + +/////////////////////////////////////////////////////////////////////////////////////////// +// +// CHIPSET IDs +// +/////////////////////////////////////////////////////////////////////////////////////////// +// Desktop flavor of X58 +#define X58_DESKTOP_DEVIDS 0x3400, 0x3405 +// Mobile version of X58 +#define X58_MOBILE_DEVID 0x3405 +#define X58_MOBILE_CLEVO_7200_SSDEVID 0x7200 + +// Sandy bridge CLEVO platform +#define SANDYBRIDGE_P180HM_SSDEVID 0x8000 +#define SandyBridge_E_X79_P270WM_SSDEVID 0x270 +#define IvyBridge_Z75_P370EM_SSDEVID 0x371 + +// Device ID's of Devices present on Patsburg's PCIE bus. +#define PATSBURG_PCIE_DEVICE_MIN_DEVID 0x1D10 +#define PATSBURG_PCIE_DEVICE_MAX_DEVID 0x1D1F +#define PATSBURG_PCIE_DEVICE_DEVID 0x244E + +//Tylersburg Congurations +#define TYLERSBURG_DEVID 0x3406 + +// Intel Grantsdale definitions +#define DEVICE_ID_INTEL_2580_HOST_BRIDGE 0x2580 +#define DEVICE_ID_INTEL_2581_ROOT_PORT 0x2581 + +// Intel Alderwood definitions +#define DEVICE_ID_INTEL_2584_HOST_BRIDGE 0x2584 +#define DEVICE_ID_INTEL_2585_ROOT_PORT 0x2585 + +// Intel Alviso definitions +#define DEVICE_ID_INTEL_2590_HOST_BRIDGE 0x2590 +#define DEVICE_ID_INTEL_2591_ROOT_PORT 0x2591 + +// Intel Tumwater definitions +#define DEVICE_ID_INTEL_359E_HOST_BRIDGE 0x359E +#define DEVICE_ID_INTEL_3597_ROOT_PORT 0x3597 + +// Intel Stoakley definitions +#define INTEL_4000_SUBDEVICE_ID 0x021D + +// Intel SkullTrail definitions +#define INTEL_4003_SUBDEVICE_ID 0x5358 + +// Intel Core I7 CPU +#define INTEL_QUICKPATH_SYSTEM_ADDRESS_DECODER_I7 0x2C01 + +// Intel Core I5 CPU Lynnfield +#define INTEL_QUICKPATH_SYSTEM_ADDRESS_DECODER_I5_L 0x2C81 +#define INTEL_LYNNFIELD_ROOTPORT_CPU1 0xD138 +#define INTEL_LYNNFIELD_ROOTPORT_CPU2 0xD13A + +// Intel Core I5 CPU Auburndale +#define INTEL_QUICKPATH_SYSTEM_ADDRESS_DECODER_I5_A 0x2D41 + +// Intel Core I5 CPU 650 +#define INTEL_QUICKPATH_SYSTEM_ADDRESS_DECODER_I5_6 0x2D01 + +// Intel Poulsbo definitions +#define DEVICE_ID_INTEL_8100_HOST_BRIDGE 0x8100 +#define DEVICE_ID_INTEL_8110_ROOT_PORT 0x8110 +#define DEVICE_ID_INTEL_8112_ROOT_PORT 0x8112 + +// Intel TunnelCreek definitions +#define DEVICE_ID_INTEL_8180_ROOT_PORT 0x8180 +#define DEVICE_ID_INTEL_8181_ROOT_PORT 0x8181 +#define DEVICE_ID_INTEL_8184_ROOT_PORT 0x8184 +#define DEVICE_ID_INTEL_8185_ROOT_PORT 0x8185 + +// Intel I/O Hub definitions +#define DEVICE_ID_INTEL_3408_ROOT_PORT 0x3408 +#define DEVICE_ID_INTEL_3411_ROOT_PORT 0x3411 +#define DEVICE_ID_INTEL_3420_ROOT_PORT 0x3420 +#define DEVICE_ID_INTEL_3421_ROOT_PORT 0x3421 + +// Intel SandyBridge IIO definitions +#define DEVICE_ID_INTEL_3C02_ROOT_PORT 0x3c02 +#define DEVICE_ID_INTEL_3C03_ROOT_PORT 0x3c03 +#define DEVICE_ID_INTEL_3C04_ROOT_PORT 0x3c04 +#define DEVICE_ID_INTEL_3C05_ROOT_PORT 0x3c05 +#define DEVICE_ID_INTEL_3C06_ROOT_PORT 0x3c06 +#define DEVICE_ID_INTEL_3C07_ROOT_PORT 0x3c07 +#define DEVICE_ID_INTEL_3C08_ROOT_PORT 0x3c08 +#define DEVICE_ID_INTEL_3C09_ROOT_PORT 0x3c09 +#define DEVICE_ID_INTEL_3C0A_ROOT_PORT 0x3c0a +#define DEVICE_ID_INTEL_3C0B_ROOT_PORT 0x3c0b + +// Intel Haswell-E definitions +#define DEVICE_ID_INTEL_2F00_HOST_BRIDGE 0x2f00 +#define DEVICE_ID_INTEL_2F01_ROOT_PORT 0x2f01 +#define DEVICE_ID_INTEL_2F02_ROOT_PORT 0x2f02 +#define DEVICE_ID_INTEL_2F03_ROOT_PORT 0x2f03 +#define DEVICE_ID_INTEL_2F04_ROOT_PORT 0x2f04 +#define DEVICE_ID_INTEL_2F05_ROOT_PORT 0x2f05 +#define DEVICE_ID_INTEL_2F06_ROOT_PORT 0x2f06 +#define DEVICE_ID_INTEL_2F07_ROOT_PORT 0x2f07 +#define DEVICE_ID_INTEL_2F08_ROOT_PORT 0x2f08 +#define DEVICE_ID_INTEL_2F09_ROOT_PORT 0x2f09 +#define DEVICE_ID_INTEL_2F0A_ROOT_PORT 0x2f0a +#define DEVICE_ID_INTEL_2F0B_ROOT_PORT 0x2f0b + +#define DEVICE_ID_INTEL_0C01_ROOT_PORT 0x0c01 + +// Intel IvyTown definitions + +#define DEVICE_ID_INTEL_0E02_ROOT_PORT 0x0e02 +#define DEVICE_ID_INTEL_0E03_ROOT_PORT 0x0e03 +#define DEVICE_ID_INTEL_0E04_ROOT_PORT 0x0e04 +#define DEVICE_ID_INTEL_0E05_ROOT_PORT 0x0e05 +#define DEVICE_ID_INTEL_0E06_ROOT_PORT 0x0e06 +#define DEVICE_ID_INTEL_0E07_ROOT_PORT 0x0e07 +#define DEVICE_ID_INTEL_0E08_ROOT_PORT 0x0e08 +#define DEVICE_ID_INTEL_0E09_ROOT_PORT 0x0e09 +#define DEVICE_ID_INTEL_0E0A_ROOT_PORT 0x0e0a +#define DEVICE_ID_INTEL_0E0B_ROOT_PORT 0x0e0b +// Intel Ivy Bridge E definitions +#define DEVICE_ID_INTEL_0E00_HOST_BRIDGE 0x0E00 + +// Intel Haswell definitions +#define DEVICE_ID_INTEL_0C00_HASWELL_HOST_BRIDGE 0x0C00 +#define DEVICE_ID_INTEL_0C04_HASWELL_HOST_BRIDGE 0x0C04 + +// Intel PCH definitions +#define DEVICE_ID_INTEL_9D10_PCH_BRIDGE 0x9d10 +#define DEVICE_ID_INTEL_9D18_PCH_BRIDGE 0x9d18 +#define DEVICE_ID_INTEL_A117_PCH_BRIDGE 0xa117 +#define DEVICE_ID_INTEL_A118_PCH_BRIDGE 0xa118 +#define DEVICE_ID_INTEL_9C98_PCH_BRIDGE 0x9c98 + +// Intel Broadwell definitions +#define DEVICE_ID_INTEL_6F00_HOST_BRIDGE 0x6f00 +#define DEVICE_ID_INTEL_6F01_ROOT_PORT 0x6f01 +#define DEVICE_ID_INTEL_6F02_ROOT_PORT 0x6f02 +#define DEVICE_ID_INTEL_6F03_ROOT_PORT 0x6f03 +#define DEVICE_ID_INTEL_6F04_ROOT_PORT 0x6f04 +#define DEVICE_ID_INTEL_6F05_ROOT_PORT 0x6f05 +#define DEVICE_ID_INTEL_6F06_ROOT_PORT 0x6f06 +#define DEVICE_ID_INTEL_6F07_ROOT_PORT 0x6f07 +#define DEVICE_ID_INTEL_6F08_ROOT_PORT 0x6f08 +#define DEVICE_ID_INTEL_6F09_ROOT_PORT 0x6f09 +#define DEVICE_ID_INTEL_6F0A_ROOT_PORT 0x6f0A +#define DEVICE_ID_INTEL_6F0B_ROOT_PORT 0x6f0B +#define DEVICE_ID_INTEL_1601_ROOT_PORT 0x1601 +#define DEVICE_ID_INTEL_1605_ROOT_PORT 0x1605 +#define DEVICE_ID_INTEL_1609_ROOT_PORT 0x1609 +#define DEVICE_ID_INTEL_BROADWELL_U_HOST_BRIDGE 0x1604 +#define DEVICE_ID_INTEL_BROADWELL_H_HOST_BRIDGE 0x1614 + +// Intel Skylake definitions +#define DEVICE_ID_INTEL_1901_ROOT_PORT 0x1901 +#define DEVICE_ID_INTEL_1905_ROOT_PORT 0x1905 +#define DEVICE_ID_INTEL_1909_ROOT_PORT 0x1909 +#define DEVICE_ID_INTEL_SKYLAKE_U_HOST_BRIDGE 0x1904 +#define DEVICE_ID_INTEL_SKYLAKE_S_HOST_BRIDGE 0x191F +#define DEVICE_ID_INTEL_SKYLAKE_H_HOST_BRIDGE 0x1910 + +// Intel Skylake-E definitions +#define DEVICE_ID_INTEL_2030_ROOT_PORT 0x2030 +#define DEVICE_ID_INTEL_2033_ROOT_PORT 0x2033 + +// Intel Kabylake definitions +#define DEVICE_ID_INTEL_KABYLAKE_U_HOST_BRIDGE 0x5904 +#define DEVICE_ID_INTEL_KABYLAKE_H_HOST_BRIDGE 0x5910 + +// AMD Matisse, Rome definitions +#define DEVICE_ID_AMD_1483_ROOT_PORT 0x1483 +// AMD Castle Peak definition +#define DEVICE_ID_AMD_1480_ROOT_PORT 0x1480 +// AMD Renoir-H definition +#define DEVICE_ID_AMD_1630_ROOT_PORT 0x1630 + +// Dell SkullTrail definitions +#define DELL_4003_SUBDEVICE_ID 0x021D + +// Dell Quicksilver MLK definitions +#define DELL_0040_SUBDEVICE_ID 0x043a + +// HP Tylersburg definitions +#define TYLERSBURG_Z800_SSDEVID 0x130B + +// HP Romley definitions +#define ROMLEY_Z820_SSDEVID 0x158B +#define ROMLEY_Z620_SSDEVID 0x158A +#define ROMLEY_Z420_SSDEVID 0x1589 + +// HP Grantley definitions +#define GRANTLEY_Z840_SSDEVID 0x2129 +#define GRANTLEY_Z640_SSDEVID 0x212A +#define GRANTLEY_Z440_SSDEVID 0x212B + +// HP PURELY definitions +#define HP_QUADRO_Z4GEN4_DEVID 0xA2D2 +#define PURLEY_Z8GEN4_SSDEVID 0x81C7 +#define PURLEY_Z6GEN4_SSDEVID 0x81C6 +#define PURLEY_Z4GEN4_SSDEVID 0x81C5 + +// Lenovo Romley definitions +#define ROMLEY_C30_SSDEVID 0x1028 +#define ROMLEY_D30_SSDEVID 0x1027 +#define ROMLEY_S30_SSDEVID 0x1026 + +// Dell Romley definitions +#define ROMLEY_T7600_SSDEVID 0x0495 +#define ROMLEY_T5600_SSDEVID 0x0496 +#define ROMLEY_T3600_SSDEVID 0x0497 + +// Dell Romley + IVB-EP CPU Refresh +#define IVYTOWN_T7610_SSDEVID 0x05D4 +#define IVYTOWN_T5610_SSDEVID 0x05D3 + +// Dell Romley (Ipanema) +#define ROMLEY_R7610_SSDEVID 0x05A1 + +// FTS Romley definitions +#define ROMLEY_R920_SSDEVID 0x11B6 + +// Lenovo Grantley (Messi, Pele, Ronaldo) +#define GRANTLEY_V40_SSDEVID 0x1031 +#define GRANTLEY_D40_SSDEVID 0x1030 +#define GRANTLEY_S40_SSDEVID 0x102F + +// Dell Grantley (Avalon) +#define GRANTLEY_T7810_SSDEVID 0x0618 +#define GRANTLEY_T7910_SSDEVID 0x0619 + +// Lenovo Purley (Nile, Volga) +#define PURLEY_P920_SSDEVID 0x1038 +#define PURLEY_P720_SSDEVID 0x1037 +#define PURLEY_P520_SSDEVID 0x1036 + +// Lenovo P520c +#define LENOVO_P520C_SSDEVID 0x103C + +// Dell Purley(Matira) +#define PURLEY_MATIRA3X_DEVID 0xA2D2 +#define PURLEY_MATIRA3X_SSDEVID 0x08B1 +#define PURLEY_MATIRA3_SSDEVID 0x0738 +#define PURLEY_MATIRA5_SSDEVID 0x0739 +#define PURLEY_MATIRA7_SSDEVID 0x073A + +//FTS Grantley +#define GRANTLEY_R940_SSDEVID 0x1201 + +//FTS Purley +#define PURLEY_R970_SSDEVID 0x1230 +#define PURLEY_M770_SSDEVID 0x1231 + +// HP Arrandale, Clarksfield, X58 workstation definitions +#define ARRANDALE_Z200SFF_SSDEVID 0x304A +#define CLARKSFIELD_Z200_SSDEVID 0x170B +#define X58_Z400_SSDEVID 0x1309 + +// GIGABYTE Sniper 3 (Z77) +#define GIGABYTE_SNIPER_3_SSDEVID_1 0x5000 +#define GIGABYTE_SNIPER_3_SSDEVID_2 0x5001 + +// Supermicro Quadro VCA definitions +#define SUPERMICRO_QUADRO_VCA_DEVID 0x8D44 +#define SUPERMICRO_QUADRO_VCA_SSDEVID 0x7270 + +// Supermicro SYS-4027GR-TRT +#define SUPERMICRO_SYS_4027GR_TRT_DEVID 0x1D41 +#define SUPERMICRO_SYS_4027GR_TRT_SSDEVID 0x0732 + +// Supermicro SYS-4029GP-TRT2 +#define SUPERMICRO_SYS_4029GP_TRT2_DEVID 0xA1C2 +#define SUPERMICRO_SYS_4029GP_TRT2_SSDEVID 0x7270 + +// Asus Quadro BOXX definitions +#define ASUS_QUADRO_BOXX_DEVID 0x8D44 +#define ASUS_QUADRO_BOXX_SSDEVID 0x85F6 + +// APEXX8 Quadro BOXX definitions +#define APEXX8_QUADRO_BOXX_DEVID 0xA2D3 +#define APEXX8_QUADRO_BOXX_SSDEVID 0x098e + +// APEXX5 Quadro BOXX definitions +#define APEXX5_QUADRO_BOXX_DEVID 0xA2D3 +#define APEXX5_QUADRO_BOXX_SSDEVID 0x1000 + +// ASUS X99-E-10G +#define ASUS_X99_E_10G_SSDEVID 0x8600 + +// VIA definitions +#define DEVICE_ID_VIA_VT8369B_HOST_BRIDGE 0x0308 + +// Foxconn Einstein 64 [8086:a1c1][105b:7270] +#define FOXCONN_EINSTEIN_64_DEVID 0xA1C1 +#define FOXCONN_EINSTEIN_64_SSDEVID 0x7270 + +// Tyan Workstation +#define TYAN_B7100_DEVID 0xA1C1 +#define TYAN_B7100_SSDEVID 0x7270 + +// ESC 4000 Series Workstation +#define ESC_4000_G4_DEVID 0xA1C1 +#define ESC_4000_G4_SSDEVID 0x871E + + +// NVIDIA C51 +#define NVIDIA_C51_DEVICE_ID_MIN 0x2F0 +#define NVIDIA_C51_ULDT_CFG_0_DEVICE_ID_0 0x2F0 +#define NVIDIA_C51_ULDT_CFG_0_DEVICE_ID_1 0x2F1 +#define NVIDIA_C51_ULDT_CFG_0_DEVICE_ID_2 0x2F2 +#define NVIDIA_C51_ULDT_CFG_0_DEVICE_ID_3 0x2F3 +#define NVIDIA_C51_ULDT_CFG_0_DEVICE_ID_IGPU_DISABLE_0 0x2F4 +#define NVIDIA_C51_ULDT_CFG_0_DEVICE_ID_IGPU_DISABLE_1 0x2F5 +#define NVIDIA_C51_ULDT_CFG_0_DEVICE_ID_IGPU_DISABLE_2 0x2F6 +#define NVIDIA_C51_ULDT_CFG_0_DEVICE_ID_IGPU_DISABLE_3 0x2F7 +#define NVIDIA_C51_DEVICE_ID_MAX 0x2F7 + +// NVIDIA MCP55 +#define NVIDIA_MCP55_ULDT_CFG_0_DEVICE_ID_DEFAULT 0x0369 + +// NVIDIA MCP61 +#define NVIDIA_MCP61_ULDT_CFG_0_DEVICE_ID_DEFAULT 0x03EA +#define NVIDIA_MCP61_ULDT_CFG_0_DEVICE_ID_PA 0x03E2 + +// NVIDIA C55 +#define NVIDIA_C55_CPU_PCI_0_DEVICE_ID_PRO 0x03A0 +#define NVIDIA_C55_CPU_PCI_0_DEVICE_ID_PRO 0x03A0 +#define NVIDIA_C55_CPU_PCI_0_DEVICE_ID_SLIX16 0x03A1 +#define NVIDIA_C55_CPU_PCI_0_DEVICE_ID_SLI 0x03A3 +#define NVIDIA_C55_CPU_PCI_0_DEVICE_ID_U 0x03A2 + +// NVIDIA MCP65 +#define NVIDIA_MCP65_ULDT_CFG_0_DEVICE_ID_DEFAULT 0x0444 + +// NVIDIA MCP67/MCP68 +#define NVIDIA_MCP67_ULDT_CFG_0_DEVICE_ID_DEFAULT 0x0547 + +// NVIDIA MCP73 +#define NVIDIA_MCP73_CPU_PCI_0_DEVICE_ID_PV 0x07C0 +#define NVIDIA_MCP73_CPU_PCI_0_DEVICE_ID_O 0x07C1 +#define NVIDIA_MCP73_CPU_PCI_0_DEVICE_ID_S 0x07C2 +#define NVIDIA_MCP73_CPU_PCI_0_DEVICE_ID_V 0x07C3 +#define NVIDIA_MCP73_CPU_PCI_0_DEVICE_ID_RSVD_0 0x07C4 +#define NVIDIA_MCP73_CPU_PCI_0_DEVICE_ID_RSVD_1 0x07C5 +#define NVIDIA_MCP73_CPU_PCI_0_DEVICE_ID_RSVD_2 0x07C6 +#define NVIDIA_MCP73_CPU_PCI_0_DEVICE_ID_D 0x07C7 + +// NVIDIA C73 +#define NVIDIA_C73_CPU_PCI_0_DEVICE_ID_SLI2 0x0800 +#define NVIDIA_C73_CPU_PCI_0_DEVICE_ID_SLI_ALL 0x0801 +#define NVIDIA_C73_CPU_PCI_0_DEVICE_ID_SLIX8 0x0802 +#define NVIDIA_C73_CPU_PCI_0_DEVICE_ID_U 0x0803 +#define NVIDIA_C73_CPU_PCI_0_DEVICE_ID_RESERVED_0 0x0804 +#define NVIDIA_C73_CPU_PCI_0_DEVICE_ID_RESERVED_1 0x0805 +#define NVIDIA_C73_CPU_PCI_0_DEVICE_ID_RESERVED_2 0x0806 +#define NVIDIA_C73_CPU_PCI_0_DEVICE_ID_RESERVED_3 0x0807 + +// NVIDIA MCP77/78 +#define NVIDIA_MCP77_ULDT_CFG_0_DEVICE_ID_DEFAULT 0x0754 +#define NVIDIA_MCP77_ULDT_CFG_0_DEVICE_ID_1 0x0755 +#define NVIDIA_MCP77_ULDT_CFG_0_DEVICE_ID_2 0x0756 +#define NVIDIA_MCP77_ULDT_CFG_0_DEVICE_ID_3 0x0757 +#define NVIDIA_MCP77_MCP_SM_CFG_0_DEVICE_ID_UNIT_SM 0x0752 + +// NVIDIA MCP79/7A +#define NVIDIA_MCP79_CPU_PCI_0_DEVICE_ID_DEFAULT 0x0A80 +#define NVIDIA_MCP79_CPU_PCI_0_DEVICE_ID_SLIX16 0x0A81 +#define NVIDIA_MCP79_CPU_PCI_0_DEVICE_ID_SLI 0x0A82 +#define NVIDIA_MCP79_CPU_PCI_0_DEVICE_ID_U 0x0A83 +#define NVIDIA_MCP79_CPU_PCI_0_DEVICE_ID_GM 0x0A84 +#define NVIDIA_MCP79_CPU_PCI_0_DEVICE_ID_GVM 0x0A85 +#define NVIDIA_MCP79_MCP_SM_CFG_0_DEVICE_ID_UNIT_SM 0x0AA2 + +// NVIDIA MCP89/P83 +#define NVIDIA_MCP89_CPU_PCI_0_DEVICE_ID_DEFAULT 0x00000D60 + +/////////////////////////////////////////////////////////////////////////////////////////// +// +// enumeration of chipset families +// +/////////////////////////////////////////////////////////////////////////////////////////// + +// +// When adding a variable to the following enum, please +// add it also to the following chipset_names[]. +// +enum { + CS_UNKNOWN = 0x0000, + CS_UNKNOWN_PCIE = 0x1000 +, CS_INTEL_2580 +, CS_INTEL_2584 +, CS_INTEL_2588 +, CS_INTEL_2590 +, CS_INTEL_25E0 +, CS_INTEL_29X0 +, CS_INTEL_29E0 +, CS_INTEL_359E +, CS_INTEL_4000 +, CS_INTEL_4003 +, CS_INTEL_3400 +, CS_INTEL_3B42 +, CS_INTEL_2770 +, CS_INTEL_2774 +, CS_INTEL_277C +, CS_INTEL_2A40 +, CS_INTEL_2E00 +, CS_INTEL_0040 +, CS_INTEL_1C10 +, CS_INTEL_1C46 +, CS_INTEL_1C49 +, CS_INTEL_1D40 +, CS_INTEL_8D47 +, CS_INTEL_1E10 +, CS_INTEL_8C4B +, CS_INTEL_8CC4 +, CS_INTEL_A145 +, CS_INTEL_A2C5 +, CS_INTEL_A242 +, CS_INTEL_A2D2 +, CS_INTEL_A2C9 +, CS_INTEL_A301 +, CS_INTEL_0685 +, CS_INTEL_4381 +, CS_INTEL_7A82 +, CS_NVIDIA_CK804 +, CS_NVIDIA_C19 +, CS_NVIDIA_C51 +, CS_NVIDIA_MCP55 +, CS_NVIDIA_MCP61 +, CS_NVIDIA_C55 +, CS_NVIDIA_MCP65 +, CS_NVIDIA_MCP67 +, CS_NVIDIA_MCP73 +, CS_NVIDIA_C73 +, CS_NVIDIA_MCP77 +, CS_NVIDIA_MCP79 +, CS_NVIDIA_MCP89 +, CS_NVIDIA_TEGRA3 +, CS_SIS_649 +, CS_SIS_656 +, CS_ATI_RS400 +, CS_ATI_RS400_A21 +, CS_ATI_RS480 +, CS_ATI_RS480_A21 +, CS_AMD_RS780 +, CS_VIA_VT8369B +, CS_ATI_FX790 +, CS_ATI_RD850 +, CS_ATI_RD870 +, CS_ATI_RD890 +, CS_ATI_FX890 +, CS_ATI_RX780 +, CS_ATI_FX990 +, CS_AMD_GX890 +, CS_AMD_X370 +, CS_VIA_VX900 +, CS_APM_STORM +, CS_IBM_VENICE +, CS_NVIDIA_T124 +, CS_NVIDIA_T210 +, CS_NVIDIA_T186 +, CS_NVIDIA_T194 +, CS_NVIDIA_T234 +, CS_NVIDIA_T23x +, CS_MARVELL_THUNDERX2 +, CS_REDHAT_QEMU +, CS_AMPERE_EMAG +, CS_HUAWEI_KUNPENG920 +, CS_MELLANOX_BLUEFIELD +, CS_AMAZON_GRAVITRON2 +, CS_FUJITSU_A64FX +, CS_AMPERE_ALTRA +, CS_ARM_NEOVERSEN1 +, CS_MARVELL_OCTEON_CN96XX +, CS_MARVELL_OCTEON_CN98XX +, CS_INTEL_C620 +, CS_HYGON_C86 +, CS_PHYTIUM_S2500 +, CS_MELLANOX_BLUEFIELD2 +, CS_INTEL_1B81 +, CS_INTEL_18DC +, CS_INTEL_7A04 +, CS_MAX_PCIE +}; + +enum { + RP_UNKNOWN = 0 +, RP_BROADCOM_HT2100 +, RP_INTEL_2581 +, RP_INTEL_2585 +, RP_INTEL_2589 +, RP_INTEL_2591 +, RP_INTEL_3597 +, RP_INTEL_2775 +, RP_INTEL_2771 +, RP_INTEL_8110 +, RP_INTEL_8112 +, RP_INTEL_8180 +, RP_INTEL_8181 +, RP_INTEL_8184 +, RP_INTEL_8185 +, RP_INTEL_3C02 +, RP_INTEL_3C03 +, RP_INTEL_3C04 +, RP_INTEL_3C05 +, RP_INTEL_3C06 +, RP_INTEL_3C07 +, RP_INTEL_3C08 +, RP_INTEL_3C09 +, RP_INTEL_3C0A +, RP_INTEL_3C0B +, RP_INTEL_2F04 +, RP_INTEL_2F08 +, RP_INTEL_0C01 +, RP_INTEL_1601 +, RP_INTEL_1605 +, RP_INTEL_1609 +, RP_INTEL_1901 +, RP_INTEL_1905 +, RP_INTEL_1909 +, RP_INTEL_5904 +, RP_NVIDIA_CK804 +, RP_NVIDIA_C19 +, RP_NVIDIA_C51 +, RP_NVIDIA_MCP55 +, RP_NVIDIA_MCP61 +, RP_NVIDIA_C55 +, RP_NVIDIA_MCP65 +}; + +#endif //NVDEVID_H + diff --git a/userspace/sdks/nv525/nvalloc/common/inc/nvpcie.h b/userspace/sdks/nv525/nvalloc/common/inc/nvpcie.h new file mode 100644 index 0000000..6b5c7ca --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/nvpcie.h @@ -0,0 +1,499 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2000-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVPCIE_H +#define NVPCIE_H + +/**************** Resource Manager Defines and Structures ******************\ +* * +* Private PCI Express related defines and structures. * +* * +\***************************************************************************/ + +#define PCI_VENDOR_ID 0x00 +#ifndef PCI_DEVICE_ID +#define PCI_DEVICE_ID 0x02 +#endif +#define PCI_BASE_ADDRESS_1 0x14 /* Aperture Base */ +#define PCI_BASE_ADDRESS_2 0x18 /* Aperture Base */ +#define PCI_CAPABILITY_LIST 0x34 +#define PCI_DEVICE_SPECIFIC 0x40 + +#define NV_PCI_ID 0x0 +#define NV_PCI_ID_VENDOR 15:0 +#define NV_PCI_ID_VENDOR_NVIDIA 0x10DE +#define NV_PCI_ID_DEVICE 31:16 + +#define PCI_MAX_SLOTS 255 +#define PCI_MAX_LANE_WIDTH 32 + +#define PCI_MAX_FUNCTION 8 +#define PCI_INVALID_VENDORID 0xFFFF +#define PCI_INVALID_DEVICEID 0xFFFF +#define PCI_INVALID_SUBVENDORID 0xFFFF +#define PCI_INVALID_SUBDEVICEID 0xFFFF + +#define PCI_CLASS_BRIDGE_DEV 0x06 +#define PCI_SUBCLASS_BR_HOST 0x00 +#define PCI_MULTIFUNCTION 0x80 + +// From PCI Local Bus Specification, Revision 3.0 +// and PCI Express Base Specification 6.0 +// numbers in comments to right of values indicate +// the referenced section in the PCIE spec + + +#define CAP_ID_MASK 0xFF + +#define CAP_ID_NULL 0x00 // 7.9.28.1 +#define CAP_ID_PMI 0x01 // 7.5.2.1 +#define CAP_ID_AGP 0x02 +#define CAP_ID_VPD 0x03 // 7.9.18.1 +#define CAP_ID_SLOT_ID 0x04 +#define CAP_ID_MSI 0x05 // 7.7.1.1 +#define CAP_ID_HOT_SWAP 0x06 +#define CAP_ID_PCI_X 0x07 +#define CAP_ID_HYPER_TRANSPORT 0x08 +#define CAP_ID_VENDOR_SPECIFIC 0x09 +#define CAP_ID_DEBUG_PORT 0x0A +#define CAP_ID_CRC 0x0B +#define CAP_ID_HOT_PLUG 0x0C +#define CAP_ID_SUBSYSTEM_ID 0x0D // 7.9.23.1 +#define CAP_ID_AGP8X 0x0E +#define CAP_ID_SECURE 0x0F +#define CAP_ID_PCI_EXPRESS 0x10 // 7.5.3.1 +#define CAP_ID_MSI_X 0x11 // 7.7.2.1 +#define CAP_ID_ENHANCED_ALLOCATION 0x14 // 7.8.5.1 +#define CAP_ID_FPB 0x15 // 7.8.11.1 +#define CAP_ID_AF 0x13 // 7.9.21.1 + +// +// sizes for static PCI capabilities structure +// +#define CAP_NULL_SIZE 0x04 // 7.9.28 +#define CAP_PMI_SIZE 0x08 // 7.5.2 +#define CAP_VPD_SIZE 0x08 // 7.9.18 +#define CAP_PCI_X_SIZE 0x3C // 7.5.3 +#define CAP_PCI_EXPRESS_SIZE 0x3C // 7.5.3 +#define CAP_FPB_SIZE 0x08 // 7.8.11.1 +#define CAP_AF_SIZE 0x08 // 7.9.21 +#define CAP_SUBSYSTEM_ID_SIZE 0x08 // 7.9.23 + +// MSI capability size related fields +#define PCI_MSI_CONTROL 0x02 // 7.7.1.2 +#define PCI_MSI_CONTROL_64BIT_CAPABLE 7:7 +#define PCI_MSI_CONTROL_64BIT_CAPABLE_FALSE 0 +#define PCI_MSI_CONTROL_64BIT_CAPABLE_TRUE 1 +#define PCI_MSI_CONTROL_PVM_CAPABLE 8:8 +#define PCI_MSI_CONTROL_PVM_CAPABLE_FALSE 0 +#define PCI_MSI_CONTROL_PVM_CAPABLE_TRUE 1 +#define PCI_MSI_BASE_SIZE 0x0C // 7.7.1 +#define PCI_MSI_64BIT_ADDR_CAPABLE_ADJ_SIZE 0x04 // 7.7.1 +#define PCI_MSI_PVM_CAPABLE_ADJ_SIZE 0x08 // 7.7.1 + +// MSI-X capability size related fields +#define PCI_MSI_X_BASE_SIZE 0x0C // 7.7.2 +#define PCI_MSI_X_CONTROL 0x02 // 7.7.2.2 +#define PCI_MSI_X_CONTROL_TABLE_SIZE 10:0 +#define PCI_MSI_X_TABLE_OFFSET_BIR 0x04 // 7.7.2.3 +#define PCI_MSI_X_TABLE_OFFSET 31:3 +#define PCI_MSI_X_PBR_OFFSET_BIR 0x08 // 7.7.2.4 +#define PCI_MSI_X_PBR_OFFSET 31:3 +#define PCI_MSI_X_TABLE_ENTRY_SIZE 0x10 // 7.7.2 +#define PCI_MSI_X_PBR_ENTRY_SIZE 0x10 // 7.7.2 + +// Enhanced Allocation Capability size related fields +#define PCI_ENHANCED_ALLOCATION_FIRST_DW 0x00 // 7.8.5.1 +#define PCI_ENHANCED_ALLOCATION_FIRST_DW_NUM_ENTRIES 21:16 +#define PCI_ENHANCED_ALLOCATION_TYPE_0_BASE_SIZE 0x04 // 7.8.5.1 +#define PCI_ENHANCED_ALLOCATION_TYPE_1_BASE_SIZE 0x08 // 7.8.5.2 +#define PCI_ENHANCED_ALLOCATION_ENTRY_HEADER 0x00 // 7.8.5.3 +#define PCI_ENHANCED_ALLOCATION_ENTRY_HEADER_ENTRY_SIZE 2:0 + +// PCI Vendor Specific Capability size related fields +#define PCI_VENDOR_SPECIFIC_CAP_HEADER 0x00 // 7.9.4 +#define PCI_VENDOR_SPECIFIC_CAP_HEADER_LENGTH 23:16 + +// +// Extended config space size is 4096 bytes. +// +#define PCI_EXTENDED_CONFIG_SPACE_LENGTH 4096 + +// +// From PCI Local Bus Specification, Revision 3.0 +// HEADER TYPE0 Definitions - Byte offsets +// +#define PCI_HEADER_TYPE0_VENDOR_ID 0x00 +#define PCI_HEADER_TYPE0_DEVICE_ID 0x02 +#define PCI_HEADER_TYPE0_COMMAND 0x04 +#define PCI_HEADER_TYPE0_STATUS 0x06 +#define PCI_HEADER_TYPE0_REVISION_ID 0x08 +#define PCI_HEADER_TYPE0_PROGIF 0x09 +#define PCI_HEADER_TYPE0_SUBCLASS 0x0A +#define PCI_HEADER_TYPE0_BASECLASS 0x0B +#define PCI_HEADER_TYPE0_CACHE_LINE_SIZE 0x0C +#define PCI_HEADER_TYPE0_LATENCY_TIMER 0x0D +#define PCI_HEADER_TYPE0_HEADER_TYPE 0x0E +#define PCI_HEADER_TYPE0_HEADER_TYPE_0 0 +#define PCI_HEADER_TYPE0_HEADER_TYPE_1 1 +#define PCI_HEADER_TYPE0_BIST 0x0F +#define PCI_HEADER_TYPE0_BAR0 0x10 +#define PCI_HEADER_TYPE0_BAR1 0x14 +#define PCI_HEADER_TYPE0_BAR2 0x18 +#define PCI_HEADER_TYPE0_BAR3 0x1C +#define PCI_HEADER_TYPE0_BAR4 0x20 +#define PCI_HEADER_TYPE0_BAR5 0x24 +#define PCI_HEADER_TYPE0_CBCIS_PTR 0x28 +#define PCI_HEADER_TYPE0_SUBSYS_VEN_ID 0x2C +#define PCI_HEADER_TYPE0_SUBSYS_ID 0x2E +#define PCI_HEADER_TYPE0_ROMBAR 0x30 +#define PCI_HEADER_TYPE0_CAP_PTR 0x34 +#define PCI_HEADER_TYPE0_INT_LINE 0x3C +#define PCI_HEADER_TYPE0_INT_PIN 0x3D +#define PCI_HEADER_TYPE0_MIN_GNT 0x3E +#define PCI_HEADER_TYPE0_MAX_LAT 0x3F + +// +// From PCI Express Base Specification Revision 2.0 +// HEADER TYPE1 Definitions +#define PCI_HEADER_TYPE1_BRIDGE_CONTROL 0x3E +#define PCI_HEADER_TYPE1_BRIDGE_CONTROL_VGA_EN 0x08 + +#define PCIE_LINK_CAPABILITIES_2 0x000000A4 /* R--4R */ +#define PCIE_LINK_CAPABILITIES_2_RSVD 0:0 /* C--VF */ +#define PCIE_LINK_CAPABILITIES_2_RSVD_INIT 0x00000000 /* C---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED 7:1 /* R-EVF */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1_GEN2_GEN3_GEN4_GEN5 0x0000001F /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_HIDDEN 0x00000000 /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1 0x00000001 /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1_GEN2 0x00000003 /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1_GEN2_GEN3 0x00000007 /* R-E-V */ +#define PCIE_LINK_CAPABILITIES_2_SUPPORTED_LINK_SPEED_GEN1_GEN2_GEN3_GEN4 0x0000000F /* R---V */ +#define PCIE_LINK_CAPABILITIES_2_CROSS_LINK_SUPPORT 8:8 /* C--VF */ +#define PCIE_LINK_CAPABILITIES_2_CROSS_LINK_SUPPORT_DISABLED 0x00000000 /* C---V */ +#define PCIE_LINK_CAPABILITIES_2_RET_PRESENCE_DET_SUPP 23:23 /* R-EVF */ +#define PCIE_LINK_CAPABILITIES_2_RET_PRESENCE_DET_SUPP_UNSET 0x00000000 /* R-E-V */ +#define PCIE_LINK_CAPABILITIES_2_2RET_PRESENCE_DET_SUPP 24:24 /* R-EVF */ +#define PCIE_LINK_CAPABILITIES_2_2RET_PRESENCE_DET_SUPP_UNSET 0x00000000 /* R-E-V */ +#define PCIE_LINK_CAPABILITIES_2_RSVD1 31:25 /* C--VF */ +#define PCIE_LINK_CAPABILITIES_2_RSVD1_INIT 0x00000000 /* C---V */ + +// +// PCI Express Virtual Peer-to-Peer Approval Definition +// +// These offsets are unused in hardware on existing chips and are reserved on +// future chips. Software has defined a virtual PCI capability that may be +// emulated by hypervisors at these offsets, and this capability is not tied +// to any specific hardware. +// +// +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0 0x000000C8 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_ID 7:0 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_NEXT 15:8 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_LENGTH 23:16 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_SIG_LO 31:24 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1 0x000000CC +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_SIG_HI 15:0 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_VERSION 18:16 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_PEER_CLIQUE_ID 22:19 +#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_RSVD 31:23 + +#define NV_PCI_VIRTUAL_P2P_APPROVAL_SIGNATURE 0x00503250 + +// Chipset-specific definitions. +// Intel SantaRosa definitions +#define INTEL_2A00_CONFIG_SPACE_BASE 0x60 + +// Intel Montevina definitions +#define INTEL_2A40_CONFIG_SPACE_BASE 0x60 +#define INTEL_2A40_ASLM_CAPABLE_REVID 0x05 + +// Intel EagleLake definitions +#define INTEL_2E00_CONFIG_SPACE_BASE 0x60 + +// Intel Bearlake definitions +#define INTEL_29XX_CONFIG_SPACE_BASE 0x60 + +// Intel BroadWater definitions +#define INTEL_29A0_CONFIG_SPACE_BASE 0x60 + +// Intel Grantsdale definitions +#define INTEL_25XX_CONFIG_SPACE_BASE 0x48 + +// Intel Tumwater definitions +#define INTEL_359E_CONFIG_SPACE_BASE 0xCC + +// Intel Greencreek definitions +#define INTEL_25E0_CONFIG_SPACE_BASE_ADDRESS 0xE0000000 + +// Intel Stoakley definitions +#define INTEL_4000_CONFIG_SPACE_BASE_ADDRESS 0xE0000000 + +// Intel SkullTrail definitions +#define INTEL_4003_CONFIG_SPACE_BASE_ADDRESS_F 0xF0000000 +#define INTEL_4003_CONFIG_SPACE_BASE_ADDRESS_E 0xE0000000 +#define INTEL_4003_CONFIG_SPACE_BASE_ADDRESS INTEL_4003_CONFIG_SPACE_BASE_ADDRESS_F +#define INTEL_4003_CONFIG_SPACE_BASE 0x64 + +// SiS 656 +#define SIS_656_CONFIG_SPACE_BASE 0xE0 +#define SIS_656_CONFIG_SPACE_BASE_ADDRESS 3:0 // mapped to 31:28 + +// PCI/PCIE definitions +#define PCI_MAX_CAPS 20 // max caps to parse +#define PCI_MAX_DEVICES 32 // max devices on bus +#define PCI_MAX_FUNCTIONS 8 // max functions for a device +#define PCI_CAP_HEADER_ID 7:0 // PCI cap header id +#define PCI_CAP_HEADER_NEXT 15:8 // PCI cap header next +#define PCI_COMMON_CLASS_SUBCLASS 0x0a // PCI class/subclass (word) +#define PCI_COMMON_CLASS_SUBBASECLASS_HOST 0x0600 // Host bridge (connect PCI to CPU) [00] + Bridge Device [06] +#define PCI_COMMON_CLASS_SUBBASECLASS_P2P 0x0604 // PCI-to-PCI bridge (connects PCI buses) [04] + Bridge Device [06] +#define PCI_COMMON_CLASS_SUBBASECLASS_VGA 0x0300 +#define PCI_COMMON_CLASS_SUBBASECLASS_3DCTRL 0x0302 +#define PCI_COMMON_CAP_PTR 0x34 // PCI common cap ptr (byte) +#define PCI_TYPE_1_SECONDARY_BUS_NUMBER 0x19 // PCI type 1 sec bus (byte) +#define PCI_TYPE_1_SUBORDINATE_BUS_NUMBER 0x1a // PCI type 1 sub bus (byte) +#define PCIE_CAP_HEADER_ID 15:0 // PCIE cap header id +#define PCIE_CAP_HEADER_ID_INVALID 0xffff +#define PCIE_CAP_HEADER_NEXT 31:20 // PCIE cap header next +#define PCIE_BUS_SHIFT 20 // PCIE cfg space bus shift +#define PCIE_DEVICE_SHIFT 15 // PCIE cfg space dev shift +#define PCIE_FUNC_SHIFT 12 // PCIE cfg space func shift +#define PCIE_CAP_VERSION 19:16 // PCIE cap version +#define PCIE_CAP_VERSION_2P0 2 // PCIE 2.0 version +#define PCIE_LINK_CNTRL_STATUS_2_OFFSET 0x30 // PCIE Link Control/Status 2 offset +#define PCIE_LINK_STATUS_2 31:16 // PCIE Link Status 2 Register +#define PCIE_LINK_STATUS_2_DE_EMPHASIS 0:0 // PCIE De-Emphasis Level +#define PCI_COMMON_SUBSYSTEM_VENDOR_ID 0x2c // PCI subsystem Vendor Id +#define PCI_COMMON_SUBSYSTEM_ID 0x2e // PCI subsystem Id +#define PCIE_CAPABILITY_BASE 0x100 // 1st PCIE capability. + +// PCI Express Capability ID in the enhanced configuration space +#define PCIE_CAP_ID_NULL 0x00 // 7.9.28.1 +#define PCIE_CAP_ID_ERROR 0x01 // 7.8.4.1 +#define PCIE_CAP_ID_VC 0x02 // 7.9.1.1 +#define PCIE_CAP_ID_SERIAL 0x03 // 7.9.3.1 +#define PCIE_CAP_ID_POWER 0x04 // 7.8.1.1 +#define PCIE_CAP_ID_ROOT_COMPLEX 0x05 // 7.9.8.1 +#define PCIE_CAP_ID_ROOT_COMPLEX_INTERNAL_LINK_CTRL 0x06 // 7.9.9.1 +#define PCIE_CAP_ID_ROOT_COMPLEX_EVENT_COLLECTOR_ENDPOINT 0x07 // 7.9.10.1 +#define PCIE_CAP_ID_PCIE_CAP_ID_MFVC 0x08 // 7.9.2.1 +#define PCIE_CAP_ID_RCRB 0x0A // 7.9.7.1 +#define PCIE_CAP_ID_ACS 0x0D // 7.7.11.1 +#define PCIE_CAP_ID_ARI 0x0E // 7.8.8.1 +#define PCIE_CAP_ID_MULTICAST 0x12 // 7.9.11.1 +#define PCIE_CAP_ID_RESIZABLE_BAR 0x15 // 7.8.6.1 +#define PCIE_CAP_ID_DYNAMIC_POWER_ALLOCATION 0x16 // 7.9.12.1 +#define PCIE_CAP_ID_TPH 0x17 // 7.9.13.1 +#define PCIE_CAP_ID_LATENCY_TOLERANCE 0x18 // 7.8.2.1 +#define PCIE_CAP_ID_SECONDARY_PCIE_CAPABILITY 0x19 // 7.7.3.1 +#define PCIE_CAP_ID_PASID 0x1B // 7.8.9.1 +#define PCIE_CAP_ID_DPC 0x1D // 7.9.14.1 +#define PCIE_CAP_ID_L1_PM_SUBSTATES 0x1E // 7.8.3.1 +#define PCIE_CAP_ID_PTM 0x1F // 7.9.15.1 +#define PCIE_CAP_ID_FRS_QUEUING 0x21 // 7.8.10.1 +#define PCIE_CAP_ID_READINESS_TIME_REPORTING 0x22 // 7.9.16.1 +#define PCIE_CAP_ID_VENDOR_SPECIFIC 0x23 // 7.9.6.1 +#define PCIE_CAP_ID_VF_RESIZABLE_BAR 0x24 // 7.8.7.1 +#define PCIE_CAP_ID_DATA_LINK 0x25 // 7.7.4.1 +#define PCIE_CAP_ID_PHYSLAYER_16_GT 0x26 // 7.7.5.1 +#define PCIE_CAP_ID_LANE_MARGINING_AT_RECEVER 0x27 // 7.7.10.1 +#define PCIE_CAP_ID_HIERARCHY_ID 0x28 // 7.9.17.1 +#define PCIE_CAP_ID_NPEM 0x29 // 7.9.19.1 +#define PCIE_CAP_ID_PHYSLAYER_32_GT 0x2A // 7.7.6.1 +#define PCIE_CAP_ID_ALTERNATE_PROTOCOL 0x2B // 7.9.20.1 +#define PCIE_CAP_ID_SFI 0x2C // 7.9.22.1 +#define PCIE_CAP_ID_SHADOW_FUNCTIONS 0x2D // 7.9.25.1 +#define PCIE_CAP_ID_DATA_OBJECT_EXCHANGE 0x2E // 7.9.24.1 +#define PCIE_CAP_ID_DEVICE_3 0x2F // 7.7.9.1 +#define PCIE_CAP_ID_IDE 0x30 // 7.9.26.1 +#define PCIE_CAP_ID_PHYSLAYER_64_GT 0x31 // 7.7.7.1 +#define PCIE_CAP_ID_FLT_LOGGING 0x32 // 7.7.8.1 +#define PCIE_CAP_ID_FLIT_PERF_MEASURMENT 0x33 // 7.8.12.1 +#define PCIE_CAP_ID_FLIT_ERROR_INJECTION 0x34 // 7.8.13.1 + +// static sized structure sizes +#define PCIE_CAP_HEADER_SIZE 0x04 // 7.6.3 +#define PCIE_CAP_NULL_SIZE 0x04 // 7.9.28 +#define PCIE_CAP_ERROR_SIZE 0x48 // 7.8.4 +#define PCIE_CAP_POWER_SIZE 0x10 // 7.8.1 +#define PCIE_CAP_ROOT_COMPLEX_INTERNAL_LINK_CTRL_SIZE 0x0C // 7.9.9 +#define PCIE_CAP_ROOT_COMPLEX_EVENT_COLLECTOR_ENDPOINT_SIZE 0x0C // 7.9.10 +#define PCIE_CAP_SECONDARY_PCIE_SIZE 0x4C // 7.7.3.1 +#define PCIE_CAP_DATA_LINK_SIZE 0x0C // 7.7.4 +#define PCIE_CAP_PHYSLAYER_16_GT_SIZE 0x40 // 7.7.5 +#define PCIE_CAP_PHYSLAYER_32_GT_SIZE 0x40 // 7.7.6 +#define PCIE_CAP_PHYSLAYER_64_GT_SIZE 0x20 // 7.7.7 +#define PCIE_CAP_FLT_LOGGING_SIZE 0x3C // 7.7.8 +#define PCIE_CAP_DEVICE_3_SIZE 0x10 // 7.7.9 +#define PCIE_CAP_LANE_MARGINING_AT_RECEVER_SIZE 0x88 // 7.7.10 +#define PCIE_CAP_ACS_SIZE 0x10 // 7.7.11 +#define PCIE_CAP_LATENCY_TOLERANCE_SIZE 0x08 // 7.8.2 +#define PCIE_CAP_L1_PM_SUBSTATE_SIZE 0x14 // 7.8.3 +#define PCIE_CAP_RESIZABLE_BAR_SIZE 0x34 // 7.8.6 +#define PCIE_CAP_VF_RESIZABLE_BAR_SIZE 0x34 // 7.8.7 +#define PCIE_CAP_ARI_SIZE 0x08 // 7.8.8 +#define PCIE_CAP_PASID_SIZE 0x08 // 7.8.9 +#define PCIE_CAP_FRS_QUEUING_SIZE 0x10 // 7.8.10 +#define PCIE_CAP_FPB_SIZE 0x24 // 7.8.11 +#define PCIE_CAP_FLIT_PERF_MEASURMENT_SIZE 0x24 // 7.8.12 +#define PCIE_CAP_FLIT_ERROR_INJECTION_SIZE 0x24 // 7.8.13 +#define PCIE_CAP_DEV_SERIAL_SIZE 0x0C // 7.9.3 +#define PCIE_CAP_RCRB_SIZE 0x14 // 7.9.7 +#define PCIE_CAP_MULTICAST_SIZE 0x30 // 7.9.11 +#define PCIE_CAP_DYNAMIC_POWER_ALLOCATION_SIZE 0x30 // 7.9.12 +#define PCIE_CAP_DPC_SIZE 0x5C // 7.9.14 +#define PCIE_CAP_PTM_SIZE 0x0C // 7.9.15 +#define PCIE_CAP_READINESS_TIME_REPORTING_SIZE 0x0C // 7.9.16 +#define PCIE_CAP_HIERARCHY_ID_SIZE 0x0C // 7.9.17 +#define PCIE_CAP_NPEM_SIZE 0x10 // 7.9.19 +#define PCIE_CAP_ALTERNATE_PROTOCOL_SIZE 0x14 // 7.9.20 +#define PCIE_CAP_SFI_SIZE 0x14 // 7.9.22 +#define PCIE_CAP_DATA_OBJECT_EXCHANGE_SIZE 0x18 // 7.9.24 +#define PCIE_CAP_SHADOW_FUNCTIONS_SIZE 0x1C // 7.9.25 +#define PCIE_CAP_IDE_SIZE 0x34 // 7.9.26 + +// Virtual Channel Capability size related fields +#define PCIE_VC_REGISTER_1 0x04 // 7.9.1.2 +#define PCIE_VC_REGISTER_1_EXTENDED_VC_COUNT 2:0 +#define PCIE_VIRTUAL_CHANNELS_BASE_SIZE 0x18 // 7.9.1 +#define PCIE_VIRTUAL_CHANNELS_EXTENDED_VC_ENTRY_SIZE 0x10 // 7.9.1 + +// Multi Function Virtual Channel Capability size related fields +#define PCIE_MFVC_REGISTER_1 0x04 // 7.9.2.2 +#define PCIE_MFVC_REGISTER_1_EXTENDED_VC_COUNT 2:0 +#define PCIE_PCIE_CAP_ID_MFVC_BASE_SIZE 0x18 // 7.9.2 +#define PCIE_PCIE_CAP_ID_MFVC_EXTENDED_VC_ENTRY_SIZE 0x10 // 7.9.2 + +// Vendor Specific Capability size related fields +#define PCIE_VENDOR_SPECIFIC_HEADER_1 0x04 // 7.9.6.2 +#define PCIE_VENDOR_SPECIFIC_HEADER_1_LENGTH 31:20 + +// Root Complex Capability size related fields +#define PCIE_ROOT_COMPLEX_SELF_DESC_REGISTER 0x04 // 7.9.8.2 +#define PCIE_ROOT_COMPLEX_SELF_DESC_REGISTER_NUM_LINK_ENTRIES 15:8 +#define PCIE_ROOT_COMPLEX_BASE_SIZE 0x0C // 7.9.8 +#define PCIE_ROOT_COMPLEX_LINK_ENTRY_SIZE 0x10 // 7.9.8.3 + +// TPH capability size related fields +#define PCIE_TPH_REQUESTOR_REGISTER 0x04 // 7.9.13.2 +#define PCIE_TPH_REQUESTOR_REGISTER_ST_TABLE_SIZE 26:16 +#define PCIE_TPH_BASE_SIZE 0x0C // 7.9.13 +#define PCIE_TPH_ST_ENTRY_SIZE 0x02 // 7.9.13.4 + +// Intel CPU family. +#define INTEL_CPU_FAMILY_06 0x06 +#define INTEL_CPU_FAMILY_15 0x0f +#define INTEL_CPU_FAMILY_16 0x10 +#define INTEL_CPU_FAMILY_21 0x15 + +// Intel CPU Model. Calculated as Model += (extModel << 4). +#define INTEL_CPU_MODEL_2A 0x2a +#define INTEL_CPU_MODEL_2D 0x2d +#define INTEL_CPU_MODEL_3A 0x3a +#define INTEL_CPU_MODEL_3F 0x3f + +// Symbolic defines for each possible virtual channel +enum +{ + RM_PCIE_VIRTUAL_CHANNEL_0 = 0, + RM_PCIE_VIRTUAL_CHANNEL_1, + RM_PCIE_VIRTUAL_CHANNEL_2, + RM_PCIE_VIRTUAL_CHANNEL_3, + RM_PCIE_VIRTUAL_CHANNEL_4, + RM_PCIE_VIRTUAL_CHANNEL_5, + RM_PCIE_VIRTUAL_CHANNEL_6, + RM_PCIE_VIRTUAL_CHANNEL_7, + RM_PCIE_VIRTUAL_CHANNEL_INVALID +}; + +// Diagnostic collection actions. +#define RM_PCIE_ACTION_NOP 0 +#define RM_PCIE_ACTION_COLLECT_CONFIG_SPACE 1 +#define RM_PCIE_ACTION_COLLECT_PCI_CAP_STRUCT 2 +#define RM_PCIE_ACTION_COLLECT_PCIE_CAP_STRUCT 3 +#define RM_PCIE_ACTION_COLLECT_ALL_PCI_CAPS 4 +#define RM_PCIE_ACTION_COLLECT_ALL_PCIE_CAPS 5 +#define RM_PCIE_ACTION_REPORT_PCI_CAPS_COUNT 6 +#define RM_PCIE_ACTION_REPORT_PCIE_CAPS_COUNT 7 +#define RM_PCIE_ACTION_EOS 0xff + + +// Diagnostic collection device Type ids +#define RM_PCIE_DEVICE_TYPE_NONE 0xff +#define RM_PCIE_DEVICE_TYPE_GPU 0 +#define RM_PCIE_DEVICE_TYPE_UPSTREAM_BRIDGE 1 +#define RM_PCIE_DEVICE_COUNT 2 + +// Diagnostic collection capability Type ids +#define RM_PCIE_DC_CAP_TYPE_NONE 0xff +#define RM_PCIE_DC_CAP_TYPE_PCI 0 +#define RM_PCIE_DC_CAP_TYPE_PCIE 1 +#define RM_PCIE_DC_CAP_TYPE_COUNT 2 + +typedef struct _def_bif_dc_diagnostic_collection_command +{ + NvU8 action; + NvU8 deviceType; + NvU16 locator; + NvU16 length; +} CL_PCIE_DC_DIAGNOSTIC_COLLECTION_ENTRY; + +typedef struct _def_cl_pcie_dc_capability_map_entry +{ + NvU16 id; + NvU16 blkOffset; +} CL_PCIE_DC_CAPABILITY_MAP_ENTRY; +typedef struct +{ + NvU16 count; + CL_PCIE_DC_CAPABILITY_MAP_ENTRY entries[PCI_MAX_CAPS]; +} CL_PCIE_DC_CAPABILITY_MAP; + +struct OBJCL; +// root port setup functions +NV_STATUS Broadcom_HT2100_setupFunc(OBJGPU *, OBJCL*); + +NV_STATUS Intel_RP25XX_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_RP81XX_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_RP3C0X_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_RP2F0X_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_RP0C0X_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Broadwell_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Skylake_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Skylake_U_Pch_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Skylake_H_Pch_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Intel_Kabylake_Y_setupFunc(OBJGPU *, OBJCL*); + +NV_STATUS Nvidia_RPC19_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Nvidia_RPC51_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS Nvidia_RPC55_setupFunc(OBJGPU *, OBJCL*); + +NV_STATUS AMD_RP1480_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS AMD_RP1630_setupFunc(OBJGPU *, OBJCL*); +NV_STATUS AMD_RP1483_setupFunc(OBJGPU *, OBJCL*); + +// Determines if the GPU is in a multi-GPU board based on devid checks +NvBool gpuIsMultiGpuBoard(OBJGPU *, NvBool *); + +#endif // NVPCIE_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/nvrangetypes.h b/userspace/sdks/nv525/nvalloc/common/inc/nvrangetypes.h new file mode 100644 index 0000000..1ff7df6 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/nvrangetypes.h @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @file nvrangetypes.h + * @brief Range types and operator macros + * @note #include a header to define NvUxx and NvSxx before sourcing this file. + */ + +#ifndef _NVRANGETYPES_H_ +#define _NVRANGETYPES_H_ + + +// +// Define range types by convention +// +#define __NV_DEFINE_RANGE_TYPE(T) \ +typedef struct NvRange ## T \ +{ \ + Nv ## T min; \ + Nv ## T max; \ +} NvRange ## T; + + +__NV_DEFINE_RANGE_TYPE(U64) // NvRangeU64 +__NV_DEFINE_RANGE_TYPE(S64) // NvRangeS64 +__NV_DEFINE_RANGE_TYPE(U32) // NvRangeU32 +__NV_DEFINE_RANGE_TYPE(S32) // NvRangeS32 +__NV_DEFINE_RANGE_TYPE(U16) // NvRangeU16 +__NV_DEFINE_RANGE_TYPE(S16) // NvRangeS16 +__NV_DEFINE_RANGE_TYPE(U8) // NvRangeU8 +__NV_DEFINE_RANGE_TYPE(S8) // NvRangeS8 + + +// +// Operator macros +// +// Macros are named xxx_RANGE (rather than xxx_RANGEU32, etc.) since they work +// properly on ranges with any number of bits, signed or unsigned. +// + +#define NV_EQUAL_RANGE(r1, r2) ((r1).min == (r2).min && (r1).max == (r2).max) +#define NV_EMPTY_INCLUSIVE_RANGE(r) ((r).min > (r).max) +#define NV_EMPTY_EXCLUSIVE_RANGE(r) ((r).min + 1 > (r).max - 1) +#define NV_WITHIN_INCLUSIVE_RANGE(r, x) ((r).min <= (x) && (x) <= (r).max) +#define NV_WITHIN_EXCLUSIVE_RANGE(r, x) ((r).min < (x) && (x) < (r).max) +#define NV_IS_SUBSET_RANGE(r1, r2) ((r1).min >= (r2).min && (r2).max >= (r1).max) +#define NV_IS_SUPERSET_RANGE(r1, r2) ((r1).min <= (r2).min && (r2).max <= (r1).max) +#define NV_CENTER_OF_RANGE(r) ((r).min / 2 + ((r).max + 1) / 2) // Avoid overflow and rounding anomalies. +#define NV_IS_OVERLAPPING_RANGE(r1, r2) \ + (NV_WITHIN_INCLUSIVE_RANGE((r1), (r2).min) || \ + NV_WITHIN_INCLUSIVE_RANGE((r1), (r2).max)) + +#define NV_DISTANCE_FROM_RANGE(r, x) ((x) < (r).min? (r).min - (x): ((x) > (r).max? (x) - (r).max: 0)) +#define NV_VALUE_WITHIN_INCLUSIVE_RANGE(r, x) ((x) < (r).min? (r).min : ((x) > (r).max? (r).max : (x))) +#define NV_VALUE_WITHIN_EXCLUSIVE_RANGE(r, x) ((x) <= (r).min? (r).min + 1 : ((x) >= (r).max? (r).max - 1 : (x))) + +#define NV_INIT_RANGE(r, x, y) \ +do \ +{ \ + (r).min = (x); \ + (r).max = (y); \ +} while(0) + +#define NV_ASSIGN_DELTA_RANGE(r, x, d) \ +do \ +{ \ + (r).min = (x) - (d); \ + (r).max = (x) + (d); \ +} while(0) + +#define NV_ASSIGN_INTERSECTION_RANGE(r1, r2) \ +do \ +{ \ + if ((r1).min < (r2).min) \ + (r1).min = (r2).min; \ + if ((r1).max > (r2).max) \ + (r1).max = (r2).max; \ +} while(0) + +#define NV_ASSIGN_UNION_RANGE(r1, r2) \ +do \ +{ \ + if ((r1).min > (r2).min) \ + (r1).min = (r2).min; \ + if ((r1).max < (r2).max) \ + (r1).max = (r2).max; \ +} while(0) + +#define NV_MULTIPLY_RANGE(r, x) \ +do \ +{ \ + (r).min *= (x); \ + (r).max *= (x); \ +} while(0) + +#define NV_DIVIDE_FLOOR_RANGE(r, x) \ +do \ +{ \ + (r).min /= (x); \ + (r).max /= (x); \ +} while(0) + +#define NV_DIVIDE_CEILING_RANGE(r, x) \ +do \ +{ \ + (r).min = ((r).min + (x) - 1) / (x); \ + (r).max = ((r).max + (x) - 1) / (x); \ +} while(0) + +#define NV_DIVIDE_ROUND_RANGE(r, x) \ +do \ +{ \ + (r).min = ((r).min + (x) / 2) / (x); \ + (r).max = ((r).max + (x) / 2) / (x); \ +} while(0) + +#define NV_DIVIDE_WIDE_RANGE(r, x) \ +do \ +{ \ + (r).min /= (x); \ + (r).max = ((r).max + (x) - 1) / (x); \ +} while(0) + +#define NV_DIVIDE_NARROW_RANGE(r, x) \ +do \ +{ \ + (r).min = ((r).min + (x) - 1) / (x); \ + (r).max /= (x); \ +} while(0) + +#define NV_VALUE_WITHIN_INCLUSIVE_RANGE(r, x) \ + ((x) < (r).min? (r).min : ((x) > (r).max? (r).max : (x))) + +#define NV_WITHIN_INCLUSIVE_RANGE(r, x) \ + ((r).min <= (x) && (x) <= (r).max) + +#define NV_DISTANCE_FROM_RANGE(r, x) \ + ((x) < (r).min? (r).min - (x): ((x) > (r).max? (x) - (r).max: 0)) + +#endif // _NVRANGETYPES_H_ + diff --git a/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi.h b/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi.h new file mode 100644 index 0000000..e5b17a0 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi.h @@ -0,0 +1,2684 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2010-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _SMBPBI_H +#define _SMBPBI_H + +#include "oob/smbpbi_impl.h" + +/*! + * This file contains definitions for SMBPBI - SMBUS Out-Of-Band (OOB) interface + */ +/*! + * NV_MSGBOX_CMD() macro below and its derivatives depend on DRF_DEF() and DRF_NUM() + * macros, that are defined in "nvdrf.h" header file, which is a part of + * smbus/msgbox SDK. That file needs to be #included before this (smbpbi.h) + * header file. + */ + +/******************************************************************************** + * * + * GPU COMMANDS * + * * + ********************************************************************************/ + +// +// Encoding: +// 30 29 23 16 15 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |I|C|R| STATUS | ARG2 | ARG1 | OPCODE | COMMAND +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// 31 28 24 +// + +#define NV_MSGBOX_CMD_OPCODE 7:0 +#define NV_MSGBOX_CMD_OPCODE_NULL_CMD 0x00000000 +#define NV_MSGBOX_CMD_OPCODE_GET_CAP_DWORD 0x00000001 +#define NV_MSGBOX_CMD_OPCODE_GET_TEMP 0x00000002 +#define NV_MSGBOX_CMD_OPCODE_GET_EXT_TEMP 0x00000003 /* With 8 fractional bits! */ +#define NV_MSGBOX_CMD_OPCODE_GET_POWER 0x00000004 +#define NV_MSGBOX_CMD_OPCODE_GET_SYS_ID_DATA 0x00000005 +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V1 0x00000006 +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V2 0x00000007 +#define NV_MSGBOX_CMD_OPCODE_GPU_PCONTROL 0x00000008 +#define NV_MSGBOX_CMD_OPCODE_GPU_SYSCONTROL 0x00000009 +#define NV_MSGBOX_CMD_OPCODE_SET_MASTER_CAPS 0x0000000a +#define NV_MSGBOX_CMD_OPCODE_SET_PRIMARY_CAPS 0x0000000a // Duplicate value till bug 3190552 is addressed. +#define NV_MSGBOX_CMD_OPCODE_GPU_REQUEST_CPL 0x0000000b +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V3 0x0000000c +#define NV_MSGBOX_CMD_OPCODE_SCRATCH_READ 0x0000000d +#define NV_MSGBOX_CMD_OPCODE_SCRATCH_WRITE 0x0000000e +#define NV_MSGBOX_CMD_OPCODE_SCRATCH_COPY 0x0000000f +#define NV_MSGBOX_CMD_OPCODE_ASYNC_REQUEST 0x00000010 +#define NV_MSGBOX_CMD_OPCODE_REGISTER_ACCESS 0x00000011 +#define NV_MSGBOX_CMD_OPCODE_GET_POWER_CONNECTOR_STATE 0x00000012 +#define NV_MSGBOX_CMD_OPCODE_GET_PAGE_RETIREMENT_STATS 0x00000013 +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V4 0x00000014 +#define NV_MSGBOX_CMD_OPCODE_GET_THERM_PARAM 0x00000015 +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V5 0x00000016 +#define NV_MSGBOX_CMD_OPCODE_ACCESS_WP_MODE 0x00000017 + +// +// TODO: Get rid of MISC_GPU_FLAGS once all SMBPBI implementations have made +// the switch to MISC_DEVICE_FLAGS. +// +#define NV_MSGBOX_CMD_OPCODE_GET_MISC_GPU_FLAGS 0x00000018 +#define NV_MSGBOX_CMD_OPCODE_GET_MISC_DEVICE_FLAGS 0x00000018 +#define NV_MSGBOX_CMD_OPCODE_GPU_UTIL_COUNTERS 0x00000019 +#define NV_MSGBOX_CMD_OPCODE_GET_NVLINK_INFO 0x0000001a +#define NV_MSGBOX_CMD_OPCODE_GET_CLOCK_FREQ_INFO 0x0000001b +#define NV_MSGBOX_CMD_OPCODE_BUNDLE_LAUNCH 0x0000001c +#define NV_MSGBOX_CMD_OPCODE_GET_DRIVER_EVENT_MSG 0x0000001d +#define NV_MSGBOX_CMD_OPCODE_GET_ECC_V6 0x0000001e +#define NV_MSGBOX_CMD_OPCODE_REMAP_ROW_STATS 0x00000020 +#define NV_MSGBOX_CMD_OPCODE_GET_PCIE_LINK_INFO 0x00000021 +#define NV_MSGBOX_CMD_OPCODE_GET_ENERGY_COUNTER 0x00000022 +#define NV_MSGBOX_CMD_OPCODE_RESERVED_0 0x00000024 +#define NV_MSGBOX_CMD_OPCODE_RESERVED_1 0x00000025 +#define NV_MSGBOX_CMD_OPCODE_GET_POWER_HINT_INFO 0x00000026 +#define NV_MSGBOX_CMD_OPCODE_DYNAMIC_SYSTEM_INFORMATION 0x00000027 +#define NV_MSGBOX_CMD_OPCODE_GPU_PERFORMANCE_MONITORING 0x00000028 + +#define NV_MSGBOX_CMD_ARG1 15:8 +#define NV_MSGBOX_CMD_ARG1_NULL 0x00000000 +#define NV_MSGBOX_CMD_ARG1_TEMP_GPU_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_TEMP_NVSWITCH_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_TEMP_GPU_1 0x00000001 /* For Gemini boards */ +#define NV_MSGBOX_CMD_ARG1_TEMP_BOARD 0x00000004 +#define NV_MSGBOX_CMD_ARG1_TEMP_MEMORY 0x00000005 +#define NV_MSGBOX_CMD_ARG1_TEMP_PWR_SUPPLY 0x00000006 +#define NV_MSGBOX_CMD_ARG1_TEMP_T_LIMIT 0x00000007 +#define NV_MSGBOX_CMD_ARG1_TEMP_NUM_SENSORS 8 +#define NV_MSGBOX_CMD_ARG1_POWER_TOTAL 0x00000000 +#define NV_MSGBOX_CMD_ARG1_SMBPBI_POWER 0x00000001 +/* SysId info type encodings for opcode NV_MSGBOX_CMD_OPCODE_GET_SYS_ID_DATA (0x05) */ +#define NV_MSGBOX_CMD_ARG1_BOARD_PART_NUM_V1 0x00000000 +#define NV_MSGBOX_CMD_ARG1_OEM_INFO_V1 0x00000001 +#define NV_MSGBOX_CMD_ARG1_SERIAL_NUM_V1 0x00000002 +#define NV_MSGBOX_CMD_ARG1_MARKETING_NAME_V1 0x00000003 +#define NV_MSGBOX_CMD_ARG1_GPU_PART_NUM_V1 0x00000004 +#define NV_MSGBOX_CMD_ARG1_MEMORY_VENDOR_V1 0x00000005 +#define NV_MSGBOX_CMD_ARG1_MEMORY_PART_NUM_V1 0x00000006 +#define NV_MSGBOX_CMD_ARG1_BUILD_DATE_V1 0x00000007 +#define NV_MSGBOX_CMD_ARG1_FIRMWARE_VER_V1 0x00000008 +#define NV_MSGBOX_CMD_ARG1_VENDOR_ID_V1 0x00000009 +#define NV_MSGBOX_CMD_ARG1_DEV_ID_V1 0x0000000a +#define NV_MSGBOX_CMD_ARG1_SUB_VENDOR_ID_V1 0x0000000b +#define NV_MSGBOX_CMD_ARG1_SUB_ID_V1 0x0000000c +#define NV_MSGBOX_CMD_ARG1_GPU_GUID_V1 0x0000000d +#define NV_MSGBOX_CMD_ARG1_INFOROM_VER_V1 0x0000000e +#define NV_MSGBOX_CMD_ARG1_PRODUCT_LENGTH_V1 0x0000000f +#define NV_MSGBOX_CMD_ARG1_PRODUCT_WIDTH_V1 0x00000010 +#define NV_MSGBOX_CMD_ARG1_PRODUCT_HEIGHT_V1 0x00000011 +#define NV_MSGBOX_CMD_ARG1_PCIE_SPEED_V1 0x00000012 +#define NV_MSGBOX_CMD_ARG1_PCIE_WIDTH_V1 0x00000013 +#define NV_MSGBOX_CMD_ARG1_TGP_LIMIT_V1 0x00000014 +#define NV_MSGBOX_CMD_ARG1_SYS_ID_DATA_TYPE_MAX 0x00000014 /* Adjust, when adding new types */ +#define NV_MSGBOX_CMD_ARG1_REGISTER_ACCESS_WRITE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_REGISTER_ACCESS_READ 0x00000001 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_TARGET 0x00000000 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_HW_SLOWDN 0x00000001 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_SHUTDN 0x00000002 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_HBM_SLOWDN 0x00000003 +#define NV_MSGBOX_CMD_ARG1_THERM_PARAM_TEMP_SW_SLOWDN 0x00000004 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_ECC_ENABLED_STATE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_RESET_REQUIRED 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_1 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_DEVICE_FLAGS_PAGE_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_MISC_DEVICE_FLAGS_PAGE_1 0x00000001 + +#define NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_CONTEXT_TIME 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_SM_TIME 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_RESET_COUNTERS 0x000000FF +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_COUNT 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_STATE_V1 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_BANDWIDTH 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER_REPLAY 0x00000003 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER_RECOVERY 0x00000004 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER_FLIT_CRC 0x00000005 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER_DATA_CRC 0x00000006 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_STATE_V2 0x00000007 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_SUBLINK_WIDTH 0x00000008 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_DATA_TX 0x00000009 +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_DATA_RX 0x0000000a +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_RAW_TX 0x0000000b +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_RAW_RX 0x0000000c +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_TRAINING_ERROR_STATE 0x0000000d +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_RUNTIME_ERROR_STATE 0x0000000e +#define NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_AVAILABILITY 0x0000000f +#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_CURRENT 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_MIN 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_MAX 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_PAGE_3 0x00000003 +#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_RAW_COUNTS 0x00000000 +#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_STATE_FLAGS 0x00000001 +#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_0 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_1 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_2 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_3 0x00000003 + +/* Async requests */ +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_GET \ + 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_SET \ + 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_INFO_GET \ + 0x00000002 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_COUNT_GET \ + 0x00000003 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_INFO_GET \ + 0x00000004 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_STATUS_GET \ + 0x00000005 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OVERCLOCKING_LIMIT_CONTROL_GET \ + 0x00000006 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OVERCLOCKING_LIMIT_CONTROL_SET \ + 0x00000007 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_ENERGY_COUNTER_STATUS_GET \ + 0x00000008 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_VIOLATION_COUNTERS_STATUS_GET \ + 0x00000009 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_UTILIZATION_RATE_GET \ + 0x0000000a +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OOB_CLOCK_LIMIT_SET \ + 0x0000000b +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_DEVICE_MODE_CONTROL \ + 0x0000000c +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_TEST_MESSAGE_SEND \ + 0x0000000e +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_CLOCK_LIMIT_GET \ + 0x0000000d +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_FAN_CURVE_POINTS_GET_SET \ + 0x0000000f +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POWER_HINT_GET \ + 0x00000010 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_CONFIGURE_PROGRAMMABLE_EDPP \ + 0x00000011 +#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POLL 0x000000ff + + +#define NV_MSGBOX_CMD_ARG1_ECC_V2_SEL 15:14 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_SEL_CNT 0x00000000 /* counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_SEL_FBA 0x00000001 /* FB addresses */ +/* Counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_DEV 13:12 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_DEV_FB 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_DEV_GR 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_FB_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_FB_SRC_LTC 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_FB_SRC_FB 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_GR_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_GR_SRC_L1 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_GR_SRC_RF 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_TYPE 9:8 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_TYPE_SBE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V2_CNT_TYPE_DBE 0x00000001 +/* FB addresses */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_SC 13:13 /* scope */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_SC_IND 0x00000000 /* individual addresses */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_SC_CUMUL 0x00000001 /* cumulative counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_IND_FLD 12:12 /* field */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_IND_FLD_CTRS 0x00000000 /* counters: sbe, dbe */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_IND_FLD_ADDR 0x00000001 /* address value */ +#define NV_MSGBOX_CMD_ARG1_ECC_V2_FBA_IND_INDEX 11:8 /* address array index: 0-13 */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V3_SEL 15:14 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_SEL_CNT 0x00000000 /* counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_SEL_FBA 0x00000001 /* FB addresses */ +/* Counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_DEV 13:12 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_DEV_FB 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_DEV_GR 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_FB_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_FB_SRC_LTC 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_FB_SRC_FB 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_GR_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_GR_SRC_L1 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_GR_SRC_RF 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_GR_SRC_TEX 0x00000002 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_TYPE 9:8 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_TYPE_SBE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V3_CNT_TYPE_DBE 0x00000001 +/* FB addresses */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_SC 13:13 /* scope */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_SC_IND 0x00000000 /* individual addresses */ +// These are not present in V3 +//#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_SC_CUMUL 0x00000001 /* cumulative counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_IND_FLD 12:12 /* field */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_IND_FLD_CTRS 0x00000000 /* counters: sbe, dbe */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_IND_FLD_ADDR 0x00000001 /* address value */ +#define NV_MSGBOX_CMD_ARG1_ECC_V3_FBA_IND_INDEX 11:8 /* address array index: 0-13 */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V4_SEL 15:14 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_SEL_CNT 0x00000000 /* counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_SEL_FBA 0x00000001 /* FB addresses */ +/* Counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_DEV 13:12 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_DEV_FB 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_DEV_GR 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_FB_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_FB_SRC_LTC 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_FB_SRC_FB 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_GR_SRC 11:10 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_GR_SRC_SHM 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_GR_SRC_RF 0x00000001 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_GR_SRC_TEX 0x00000002 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_TYPE 9:8 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_TYPE_SBE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ECC_V4_CNT_TYPE_DBE 0x00000001 +/* FB addresses */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_SC 13:13 /* scope */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_SC_IND 0x00000000 /* individual addresses */ +// These are not present in V4 +//#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_SC_CUMUL 0x00000001 /* cumulative counters */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_IND_FLD 12:12 /* field */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_IND_FLD_CTRS 0x00000000 /* counters: sbe, dbe */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_IND_FLD_ADDR 0x00000001 /* address value */ +#define NV_MSGBOX_CMD_ARG1_ECC_V4_FBA_IND_INDEX 11:8 /* address array index: 0-13 */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V5_SEL 15:15 +#define NV_MSGBOX_CMD_ARG1_ECC_V5_SEL_COUNTS 0x00000000 /* address/region counts */ +#define NV_MSGBOX_CMD_ARG1_ECC_V5_SEL_ERR_BUFFER 0x00000001 /* SRAM error buffer */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V5_COUNTS_INDEX_LO 14:8 /* 7 least significant index bits */ + +#define NV_MSGBOX_CMD_ARG1_ECC_V5_ERR_BUF_INDEX 11:8 /* buffer entry index */ + +#define NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION 8:8 +#define NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_GET 0x00000000 +#define NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_SET 0x00000001 + +#define NV_MSGBOX_CMD_ARG1_BUNDLE_REQUEST_COUNT 11:8 +#define NV_MSGBOX_CMD_ARG1_BUNDLE_DISP_RULE_COUNT 15:12 + +#define NV_MSGBOX_CMD_ARG1_GET_DEM_OLDEST 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_DEM_BY_SEQ_NUMBER 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_DEM_BY_TIMESTAMP 0x00000002 + +#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE 15:8 +#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE_CORRECTABLE_ERROR 0 +#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE_UNCORRECTABLE_ERROR 1 + +// Query type of _GET_POWER_HINT_INFO +#define NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_CLK 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_TEMP 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_PROFILES 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_NUM \ + (NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_PROFILES + 1) + +// Arg1 for _GPU_PERFORMANCE_MONITORING +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION 15:14 +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION_GET_INSTANTANEOUS_METRIC 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION_GET_SNAPSHOT_METRIC 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION_CAPTURE_SNAPSHOT 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GPM_ACTION_SET_MULTIPLIER 0x00000003 + +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC 13:8 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_GRAPHICS_ENGINE 0x00000000 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_SM_ACTIVITY 0x00000001 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_SM_OCCUPANCY 0x00000002 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_TENSOR_CORE_ACTIVITY 0x00000003 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_DRAM_USAGE 0x00000004 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_FP64_ACTIVITY 0x00000005 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_FP32_ACTIVITY 0x00000006 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_FP16_ACTIVITY 0x00000007 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_PCIE_TX_BANDWIDTH 0x00000008 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_PCIE_RX_BANDWIDTH 0x00000009 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_TX_BANDWIDTH 0x0000000A +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_TX_BANDWIDTH 0x0000000B +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_RX_BANDWIDTH 0x0000000C +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_RX_BANDWIDTH 0x0000000D +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVDEC_UTILIZATION 0x0000000E +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVJPG_UTILIZATION 0x0000000F +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVOFA_UTILIZATION 0x00000010 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_TX_BW_PER_LINK 0x00000011 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_TX_BW_PER_LINK 0x00000012 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_RX_BW_PER_LINK 0x00000013 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_RX_BW_PER_LINK 0x00000014 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVDEC_UTIL_PER_INSTANCE 0x00000015 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVJPG_UTIL_PER_INSTANCE 0x00000016 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_INTEGER_UTILIZATION 0x00000017 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_DMMA_UTILIZATION 0x00000018 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_HMMA_UTILIZATION 0x00000019 +#define NV_MSGBOX_CMD_ARG1_GPM_METRIC_IMMA_UTILIZATION 0x0000001A + +#define NV_MSGBOX_CMD_ARG1_DYN_SYS_INFO_DRIVER_VERSION_V1 0x00000000 + +#define NV_MSGBOX_CMD_ARG2 23:16 +#define NV_MSGBOX_CMD_ARG2_NULL 0x00000000 + +#define NV_MSGBOX_CMD_ARG2_ECC_V2_FB_PARTITION 23:20 +#define NV_MSGBOX_CMD_ARG2_ECC_V2_FB_SLICE 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V2_GR_GPC 23:20 +#define NV_MSGBOX_CMD_ARG2_ECC_V2_GR_TPC 19:16 + +#define NV_MSGBOX_CMD_ARG2_ECC_V3_FB_PARTITION 23:20 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_FB_SLICE 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_FB_SUBPARTITION 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_GR_TEX 23:22 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_GR_GPC 21:19 +#define NV_MSGBOX_CMD_ARG2_ECC_V3_GR_TPC 18:16 + +#define NV_MSGBOX_CMD_ARG2_ECC_V4_FB_PARTITION 23:20 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_FB_SLICE 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_FB_SUBPARTITION 19:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_GR_TEX 23:22 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_GR_GPC 21:19 +#define NV_MSGBOX_CMD_ARG2_ECC_V4_GR_TPC 18:16 + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_INDEX_HI 23:21 +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE 18:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_HDR_META 0x0 /* header and metadata */ + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_ADDR_ADDR 0x1 /* addr: address */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_ADDR_UNCORRECTED_COUNTS 0x2 /* addr: uncorrectedCounts */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_ADDR_CORRECTED_TOTAL 0x3 /* addr: correctedTotal */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_ADDR_CORRECTED_UNIQUE 0x4 /* addr: correctedUnique */ + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_REGN_CORRECTED_TOTAL 0x1 /* region: correctedTotal */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_REGN_CORRECTED_UNIQUE 0x2 /* region: correctedUnique */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_REGN_UNCORRECTED_TOTAL 0x3 /* region: uncorrectedTotal */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_REGN_UNCORRECTED_UNIQUE 0x4 /* region: uncorrectedUnique */ + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_COUNTS_TYPE_MAX 0x4 + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE 17:16 /* entry element type */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE_ERR_TYPE_META 0x0 /* errorType and metadata */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE_TIME_STAMP 0x1 /* time stamp */ +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE_ADDR 0x2 /* address */ + +#define NV_MSGBOX_CMD_ARG2_ECC_V5_ERR_BUF_TYPE_MAX 0x2 + +#define NV_MSGBOX_CMD_ARG2_REG_INDEX_SCRATCH_PAGE 0x00000000 +#define NV_MSGBOX_CMD_ARG2_REG_INDEX_EVENTS_PENDING 0x00000001 +#define NV_MSGBOX_CMD_ARG2_REG_INDEX_EVENT_MASK 0x00000002 +#define NV_MSGBOX_CMD_ARG2_REG_INDEX_MAX 0x00000002 /* increase when adding + new registers */ + +#define NV_MSGBOX_CMD_ARG2_NVLINK_INFO_AGGREGATE 0x000000FF + +#define NV_MSGBOX_CMD_ARG2_ACCESS_WP_MODE_SET_STATE 23:16 +#define NV_MSGBOX_CMD_ARG2_ACCESS_WP_MODE_SET_STATE_DISABLED 0x0000005A +#define NV_MSGBOX_CMD_ARG2_ACCESS_WP_MODE_SET_STATE_ENABLED 0x000000A5 + +#define NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_GPCCLK 0x00000000 +#define NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_MEMCLK 0x00000001 + +#define NV_MSGBOX_CMD_ARG2_ECC_V6_COUNTER_TYPE 23:16 +#define NV_MSGBOX_CMD_ARG2_ECC_V6_COUNTER_TYPE_SRAM 0 +#define NV_MSGBOX_CMD_ARG2_ECC_V6_COUNTER_TYPE_DRAM 1 + +#define NV_MSGBOX_CMD_ARG2_REMAP_ROW_RAW_CNT_COMBINED 0x00000000 +#define NV_MSGBOX_CMD_ARG2_REMAP_ROW_RAW_CNT_UNCORR 0x00000001 +#define NV_MSGBOX_CMD_ARG2_REMAP_ROW_RAW_CNT_CORR 0x00000002 + +#define NV_MSGBOX_CMD_ARG2_REMAP_ROWS_STATE_FLAGS_PAGE0 0x00000000 + +/*! + * Arg2 for _GET_POWER_HINT_INFO + * ARG1 == _GET_POWER_HINT_INFO_CLK + * Return Min/Max frequency in MHz + */ +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_CLK_GR 0x00000000 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_CLK_MEM 0x00000001 + +/*! + * Arg2 for _GET_POWER_HINT_INFO + * ARG1 == _GET_POWER_HINT_INFO_PROFILES + * Return bits represents which profiles are available + */ +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_0 0x00000000 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_1 0x00000001 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_2 0x00000002 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_3 0x00000003 +#define NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_TOTAL_PAGES \ + (NV_MSGBOX_CMD_ARG2_GET_POWER_HINT_INFO_PROFILES_PAGE_3 + 1) + +// Arg2 for _GPU_PERFORMANCE_MONITORING +#define NV_MSGBOX_CMD_ARG2_GPM_PARTITION_AGGREGATE 0x000000FF +#define NV_MSGBOX_CMD_ARG2_GPM_PARTITION_INDEX 21:16 +#define NV_MSGBOX_CMD_ARG2_GPM_CI_METRICS_REQUESTED 23:23 +#define NV_MSGBOX_CMD_ARG2_GPM_CI_METRICS_REQUESTED_YES 0x00000001 +#define NV_MSGBOX_CMD_ARG2_GPM_CI_METRICS_REQUESTED_NO 0x00000000 + +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER 23:16 +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_1X 0x00000001 +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_2X 0x00000002 +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_5X 0x00000005 +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_10X 0x0000000A +#define NV_MSGBOX_CMD_ARG2_GPM_MULTIPLIER_100X 0x00000064 + + +#define NV_MSGBOX_CMD_STATUS 28:24 +#define NV_MSGBOX_CMD_STATUS_NULL 0x00000000 +#define NV_MSGBOX_CMD_STATUS_ERR_REQUEST 0x00000001 +#define NV_MSGBOX_CMD_STATUS_ERR_OPCODE 0x00000002 +#define NV_MSGBOX_CMD_STATUS_ERR_ARG1 0x00000003 +#define NV_MSGBOX_CMD_STATUS_ERR_ARG2 0x00000004 +#define NV_MSGBOX_CMD_STATUS_ERR_DATA 0x00000005 +#define NV_MSGBOX_CMD_STATUS_ERR_MISC 0x00000006 +#define NV_MSGBOX_CMD_STATUS_ERR_I2C_ACCESS 0x00000007 +#define NV_MSGBOX_CMD_STATUS_ERR_NOT_SUPPORTED 0x00000008 +#define NV_MSGBOX_CMD_STATUS_ERR_NOT_AVAILABLE 0x00000009 +#define NV_MSGBOX_CMD_STATUS_ERR_BUSY 0x0000000a +#define NV_MSGBOX_CMD_STATUS_ERR_AGAIN 0x0000000b +#define NV_MSGBOX_CMD_STATUS_ERR_SENSOR_DATA 0x0000000c +#define NV_MSGBOX_CMD_STATUS_ERR_DISPOSITION 0x0000000d +#define NV_MSGBOX_CMD_STATUS_PARTIAL_FAILURE 0x0000001b +#define NV_MSGBOX_CMD_STATUS_ACCEPTED 0x0000001c +#define NV_MSGBOX_CMD_STATUS_INACTIVE 0x0000001d +#define NV_MSGBOX_CMD_STATUS_READY 0x0000001e +#define NV_MSGBOX_CMD_STATUS_SUCCESS 0x0000001f + +#define NV_MSGBOX_CMD_RSVD 29:29 +#define NV_MSGBOX_CMD_RSVD_INIT 0x00000000 + +#define NV_MSGBOX_CMD_COPY_DATA 30:30 +#define NV_MSGBOX_CMD_COPY_DATA_OFF 0x00000000 +#define NV_MSGBOX_CMD_COPY_DATA_ON 0x00000001 + +#define NV_MSGBOX_CMD_EVENT_PENDING 30:30 +#define NV_MSGBOX_CMD_EVENT_PENDING_OFF 0x00000000 +#define NV_MSGBOX_CMD_EVENT_PENDING_ON 0x00000001 + +#define NV_MSGBOX_CMD_INTR 31:31 +#define NV_MSGBOX_CMD_INTR_NOT_PENDING 0x00000000 +#define NV_MSGBOX_CMD_INTR_PENDING 0x00000001 +#define NV_MSGBOX_CMD_INTR_CLEAR 0x00000000 + +/* For individual requests in a bundle */ + +#define NV_MSGBOX_CMD_ON_BUNDLE_FAILURE 31:31 +#define NV_MSGBOX_CMD_ON_BUNDLE_FAILURE_CONTINUE 0x00000000 +#define NV_MSGBOX_CMD_ON_BUNDLE_FAILURE_STOP 0x00000001 + +#define NV_MSGBOX_CMD_DATA_COPY 23:0 +#define NV_MSGBOX_CMD_DATA_COPY_INIT 0x00000000 +#define NV_MSGBOX_CMD_EXT_STATUS 23:0 + +// NV_MSGBOX_CMD_COPY_SIZE_ENCODING used for certain commands to embed size +// indicators in the output when copy-bit is set in request + +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_DATA_OUT 0:0 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_DATA_OUT_SET 1 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_DATA_OUT_NOT_SET 0 + +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_EXT_DATA_OUT 1:1 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_EXT_DATA_OUT_SET 1 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_READ_EXT_DATA_OUT_NOT_SET 0 + +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_DATA 23:2 +#define NV_MSGBOX_CMD_COPY_SIZE_ENCODING_DATA_INIT 0x00000000 + +/* Response to NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM */ +#define NV_MSGBOX_CMD_REMAP_ROW_HISTOGRAM_NONE_AVAILABILITY 15:0 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_* */ +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY 1:0 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY_KIB 0x00000000 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY_MIB 0x00000001 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY_GIB 0x00000002 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_GRANULARITY_TIB 0x00000003 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_NEW_SAMPLE 2:2 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_NEW_SAMPLE_FALSE 0x00000000 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_NEW_SAMPLE_TRUE 0x00000001 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_INVALID_DELTA 3:3 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_INVALID_DELTA_FALSE 0x00000000 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_INVALID_DELTA_TRUE 0x00000001 +#define NV_MSGBOX_CMD_NVLINK_INFO_THROUGHPUT_DELTA 23:4 + +/* + * Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_TRAINING_ERR_STATE + * NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_RUNTIME_ERR_STATE + */ +#define NV_MSGBOX_CMD_NVLINK_INFO_TRAINING_ERROR_COUNT 11:0 +#define NV_MSGBOX_CMD_NVLINK_INFO_RUNTIME_ERROR_COUNT 23:12 +#define NV_MSGBOX_CMD_NVLINK_INFO_ERROR_COUNT_MAX \ + DRF_MASK(NV_MSGBOX_CMD_NVLINK_INFO_TRAINING_ERROR_COUNT) + +/** + * Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_AVAILABILITY + */ +#define NV_MSGBOX_CMD_NVLINK_INFO_GET_NVLINK_INFO_AVAILABILTY_PAGE_0 0x00000000 + +/** + * This field is populated as response to the following metrics: + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_DRAM_USAGE + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_PCIE_TX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_PCIE_RX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_TX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_TX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_RAW_RX_BANDWIDTH + * - NV_MSGBOX_CMD_ARG1_GPM_METRIC_NVLINK_DATA_RX_BANDWIDTH + */ +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY 2:0 +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY_B 0x00000000 +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY_KIB 0x00000001 +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY_MIB 0x00000002 +#define NV_MSGBOX_CMD_GPM_DATA_GRANULARITY_GIB 0x00000003 + +/* MSGBOX data, capability dword structure */ + +#define NV_MSGBOX_DATA_REG 31:0 +#define NV_MSGBOX_DATA_CAP_COUNT 5 + +#define NV_MSGBOX_DATA_CAP_0 0 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_0 0:0 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_0_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_0_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_NVSWITCH_0 0:0 +#define NV_MSGBOX_DATA_CAP_0_TEMP_NVSWITCH_0_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_NVSWITCH_0_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_1 1:1 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_BOARD 4:4 +#define NV_MSGBOX_DATA_CAP_0_TEMP_BOARD_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_BOARD_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_MEMORY 5:5 +#define NV_MSGBOX_DATA_CAP_0_TEMP_MEMORY_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_MEMORY_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_PWR_SUPPLY 6:6 +#define NV_MSGBOX_DATA_CAP_0_TEMP_PWR_SUPPLY_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_PWR_SUPPLY_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_TEMP_T_LIMIT 7:7 +#define NV_MSGBOX_DATA_CAP_0_TEMP_T_LIMIT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_TEMP_T_LIMIT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS 11:8 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS_ZERO 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS_ADT7473 0x00000002 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS_SFXP11_5 0x00000005 +#define NV_MSGBOX_DATA_CAP_0_EXT_TEMP_BITS_SFXP24_8 0x00000008 +#define NV_MSGBOX_DATA_CAP_0_POWER_TOTAL 16:16 +#define NV_MSGBOX_DATA_CAP_0_POWER_TOTAL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_POWER_TOTAL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_GPU_PCONTROL 17:17 +#define NV_MSGBOX_DATA_CAP_0_GPU_PCONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_GPU_PCONTROL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_GPU_SYSCONTROL 18:18 +#define NV_MSGBOX_DATA_CAP_0_GPU_SYSCONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_GPU_SYSCONTROL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_BITS 28:24 // Adjust when adding new bits +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC 24:24 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_ACOUSTIC_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN 25:25 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SLOWDN_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN 26:26 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_SHUTDN_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY 27:27 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_MEMORY_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN 28:28 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_THERMP_TEMP_GPU_SW_SLOWDOWN_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS 29:29 +#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS_AVAILABLE 0x00000001 + +#define NV_MSGBOX_DATA_CAP_1 1 +#define NV_MSGBOX_DATA_CAP_1_BOARD_PART_NUM_V1 0:0 +#define NV_MSGBOX_DATA_CAP_1_BOARD_PART_NUM_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_BOARD_PART_NUM_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_OEM_INFO_V1 1:1 +#define NV_MSGBOX_DATA_CAP_1_OEM_INFO_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_OEM_INFO_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_SERIAL_NUM_V1 2:2 +#define NV_MSGBOX_DATA_CAP_1_SERIAL_NUM_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_SERIAL_NUM_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_MARKETING_NAME_V1 3:3 +#define NV_MSGBOX_DATA_CAP_1_MARKETING_NAME_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_MARKETING_NAME_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GPU_PART_NUM_V1 4:4 +#define NV_MSGBOX_DATA_CAP_1_GPU_PART_NUM_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GPU_PART_NUM_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_VENDOR_V1 5:5 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_VENDOR_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_VENDOR_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_PART_NUM_V1 6:6 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_PART_NUM_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_MEMORY_PART_NUM_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_BUILD_DATE_V1 7:7 +#define NV_MSGBOX_DATA_CAP_1_BUILD_DATE_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_BUILD_DATE_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_FIRMWARE_VER_V1 8:8 +#define NV_MSGBOX_DATA_CAP_1_FIRMWARE_VER_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_FIRMWARE_VER_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_VENDOR_ID_V1 9:9 +#define NV_MSGBOX_DATA_CAP_1_VENDOR_ID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_VENDOR_ID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_DEV_ID_V1 10:10 +#define NV_MSGBOX_DATA_CAP_1_DEV_ID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_DEV_ID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_SUB_VENDOR_ID_V1 11:11 +#define NV_MSGBOX_DATA_CAP_1_SUB_VENDOR_ID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_SUB_VENDOR_ID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_SUB_ID_V1 12:12 +#define NV_MSGBOX_DATA_CAP_1_SUB_ID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_SUB_ID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GPU_GUID_V1 13:13 +#define NV_MSGBOX_DATA_CAP_1_GPU_GUID_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GPU_GUID_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_INFOROM_VER_V1 14:14 +#define NV_MSGBOX_DATA_CAP_1_INFOROM_VER_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_INFOROM_VER_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V1 16:16 +#define NV_MSGBOX_DATA_CAP_1_ECC_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V2 17:17 +#define NV_MSGBOX_DATA_CAP_1_ECC_V2_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V2_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V3 18:18 +#define NV_MSGBOX_DATA_CAP_1_ECC_V3_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V3_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_RET_PG_CNT 19:19 +#define NV_MSGBOX_DATA_CAP_1_RET_PG_CNT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_RET_PG_CNT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V4 20:20 +#define NV_MSGBOX_DATA_CAP_1_ECC_V4_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V4_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V5 21:21 +#define NV_MSGBOX_DATA_CAP_1_ECC_V5_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V5_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ACCESS_WP_MODE 22:22 +#define NV_MSGBOX_DATA_CAP_1_ACCESS_WP_MODE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ACCESS_WP_MODE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GET_ECC_ENABLED_STATE 23:23 +#define NV_MSGBOX_DATA_CAP_1_GET_ECC_ENABLED_STATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GET_ECC_ENABLED_STATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GET_GPU_RESET_REQUIRED 24:24 +#define NV_MSGBOX_DATA_CAP_1_GET_GPU_RESET_REQUIRED_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GET_GPU_RESET_REQUIRED_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GPU_UTIL_COUNTERS 25:25 +#define NV_MSGBOX_DATA_CAP_1_GPU_UTIL_COUNTERS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GPU_UTIL_COUNTERS_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_INFO_STATE_V1 26:26 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_INFO_STATE_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_INFO_STATE_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_ERROR_COUNTERS 27:27 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_ERROR_COUNTERS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_NVLINK_ERROR_COUNTERS_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_FREQ_INFO 28:28 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_FREQ_INFO_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_CLOCK_FREQ_INFO_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_GET_MIG_ENABLED_STATE 29:29 +#define NV_MSGBOX_DATA_CAP_1_GET_MIG_ENABLED_STATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_GET_MIG_ENABLED_STATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_1_ECC_V6 30:30 +#define NV_MSGBOX_DATA_CAP_1_ECC_V6_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_1_ECC_V6_AVAILABLE 0x00000001 + +#define NV_MSGBOX_DATA_CAP_2 2 +#define NV_MSGBOX_DATA_CAP_2_GPU_DRIVER 0:0 +#define NV_MSGBOX_DATA_CAP_2_GPU_DRIVER_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GPU_DRIVER_NOT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVSWITCH_DRIVER 0:0 +#define NV_MSGBOX_DATA_CAP_2_NVSWITCH_DRIVER_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVSWITCH_DRIVER_NOT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GPU_REQUEST 1:1 +#define NV_MSGBOX_DATA_CAP_2_GPU_REQUEST_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GPU_REQUEST_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NUM_SCRATCH_BANKS 4:2 +#define NV_MSGBOX_DATA_CAP_2_NUM_SCRATCH_BANKS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_FAN_V1_CONTROL 5:5 +#define NV_MSGBOX_DATA_CAP_2_FAN_V1_CONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_FAN_V1_CONTROL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_LENGTH_V1 6:6 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_LENGTH_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_LENGTH_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_WIDTH_V1 7:7 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_WIDTH_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_WIDTH_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_HEIGHT_V1 8:8 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_HEIGHT_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PRODUCT_HEIGHT_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PCIE_SPEED_V1 9:9 +#define NV_MSGBOX_DATA_CAP_2_PCIE_SPEED_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PCIE_SPEED_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_PCIE_WIDTH_V1 10:10 +#define NV_MSGBOX_DATA_CAP_2_PCIE_WIDTH_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_PCIE_WIDTH_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_TGP_LIMIT_V1 11:11 +#define NV_MSGBOX_DATA_CAP_2_TGP_LIMIT_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_TGP_LIMIT_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_SCRATCH_PAGE_SIZE 12:12 +#define NV_MSGBOX_DATA_CAP_2_SCRATCH_PAGE_SIZE_1024B 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_SCRATCH_PAGE_SIZE_256B 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_STATS 13:13 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_STATS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_STATS_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_INFO 14:14 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_INFO_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_INFO_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GET_GPU_DRAIN_AND_RESET_RECOMMENDED 15:15 +#define NV_MSGBOX_DATA_CAP_2_GET_GPU_DRAIN_AND_RESET_RECOMMENDED_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GET_GPU_DRAIN_AND_RESET_RECOMMENDED_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_THROUGHPUT 16:16 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_THROUGHPUT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_THROUGHPUT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_LINK_STATE_V2 17:17 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_LINK_STATE_V2_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_LINK_STATE_V2_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_SUBLINK_WIDTH 18:18 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_SUBLINK_WIDTH_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_SUBLINK_WIDTH_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GET_ENERGY_COUNTER 19:19 +#define NV_MSGBOX_DATA_CAP_2_GET_ENERGY_COUNTER_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GET_ENERGY_COUNTER_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_PENDING 20:20 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_PENDING_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_PENDING_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_HISTOGRAM 21:21 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_HISTOGRAM_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_REMAP_ROW_HISTOGRAM_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_TRAINING_ERROR_STATE 22:22 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_TRAINING_ERROR_STATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_TRAINING_ERROR_STATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_RUNTIME_ERROR_STATE 23:23 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_RUNTIME_ERROR_STATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_RUNTIME_ERROR_STATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_TARGET_SPEED 25:25 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_TARGET_SPEED_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_GET_PCIE_LINK_TARGET_SPEED_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_CURRENT_PSTATE 26:26 +#define NV_MSGBOX_DATA_CAP_2_CURRENT_PSTATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_CURRENT_PSTATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_AVAILABILITY 27:27 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_AVAILABILITY_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_2_NVLINK_INFO_AVAILABILITY_AVAILABLE 0x00000001 + +#define NV_MSGBOX_DATA_CAP_4 4 +#define NV_MSGBOX_DATA_CAP_4_HW_VIOLATION_TIME 0:0 +#define NV_MSGBOX_DATA_CAP_4_HW_VIOLATION_TIME_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_HW_VIOLATION_TIME_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SW_VIOLATION_TIME 1:1 +#define NV_MSGBOX_DATA_CAP_4_SW_VIOLATION_TIME_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SW_VIOLATION_TIME_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SW_POWER_VIOLATION_TIME 2:2 +#define NV_MSGBOX_DATA_CAP_4_SW_POWER_VIOLATION_TIME_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SW_POWER_VIOLATION_TIME_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SW_THERMAL_VIOLATION_TIME 3:3 +#define NV_MSGBOX_DATA_CAP_4_SW_THERMAL_VIOLATION_TIME_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SW_THERMAL_VIOLATION_TIME_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_UTILIZATION_RATE 4:4 +#define NV_MSGBOX_DATA_CAP_4_UTILIZATION_RATE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_UTILIZATION_RATE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_GET_DRIVER_EVENT_MSG 5:5 +#define NV_MSGBOX_DATA_CAP_4_GET_DRIVER_EVENT_MSG_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_GET_DRIVER_EVENT_MSG_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_REQUEST_BUNDLING 6:6 +#define NV_MSGBOX_DATA_CAP_4_REQUEST_BUNDLING_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_REQUEST_BUNDLING_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SET_DEVICE_DISABLE 7:7 +#define NV_MSGBOX_DATA_CAP_4_SET_DEVICE_DISABLE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SET_DEVICE_DISABLE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SET_ECC_MODE 8:8 +#define NV_MSGBOX_DATA_CAP_4_SET_ECC_MODE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SET_ECC_MODE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_SET_MIG_MODE 10:10 +#define NV_MSGBOX_DATA_CAP_4_SET_MIG_MODE_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_SET_MIG_MODE_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_FAN_CURVE_POINTS_GET_SET 11:11 +#define NV_MSGBOX_DATA_CAP_4_FAN_CURVE_POINTS_GET_SET_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_FAN_CURVE_POINTS_GET_SET_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_POWER_HINT 22:22 +#define NV_MSGBOX_DATA_CAP_4_POWER_HINT_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_POWER_HINT_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_DRIVER_VERSION_V1 23:23 +#define NV_MSGBOX_DATA_CAP_4_DRIVER_VERSION_V1_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_DRIVER_VERSION_V1_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_GPU_PERFORMANCE_MONITORING 24:24 +#define NV_MSGBOX_DATA_CAP_4_GPU_PERFORMANCE_MONITORING_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_GPU_PERFORMANCE_MONITORING_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_CAP_4_CONFIGURE_PROGRAMMABLE_EDPP 30:30 +#define NV_MSGBOX_DATA_CAP_4_CONFIGURE_PROGRAMMABLE_EDPP_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_CAP_4_CONFIGURE_PROGRAMMABLE_EDPP_AVAILABLE 0x00000001 + +/* ECC counters */ +#define NV_MSGBOX_DATA_ECC_CNT_16BIT_DBE 31:16 +#define NV_MSGBOX_DATA_ECC_CNT_16BIT_SBE 16:0 +#define NV_MSGBOX_DATA_ECC_CNT_8BIT_DBE 23:16 +#define NV_MSGBOX_DATA_ECC_CNT_8BIT_SBE 7:0 + +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HEADER 7:0 + +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HDR_ID 1:0 +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HDR_ID_INVAL 0 +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HDR_ID_ADDR 1 +#define NV_MSGBOX_DATA_ECC_V5_COUNT_HDR_ID_REGN 2 + +#define NV_MSGBOX_DATA_ECC_V5_COUNT_METADATA 31:16 + +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_ERR_TYPE 7:0 + +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_ERR_TYPE_ID 0:0 +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_ERR_TYPE_ID_UNCORR 0 +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_ERR_TYPE_ID_CORR 1 + +#define NV_MSGBOX_DATA_ECC_V5_ERR_BUF_METADATA 31:16 + +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID 21:16 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_LRF 0 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_L1DATA 1 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_L1TAG 2 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_CBU 3 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_LTC 4 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_STRUCT_ID_DRAM 5 + +#define NV_MSGBOX_DATA_ECC_V5_METADATA_LOCATION_ID 26:22 +#define NV_MSGBOX_DATA_ECC_V5_METADATA_SUBLOCATION_ID 31:27 + +/* NV_MSGBOX_CMD_OPCODE_SCRATCH_COPY src offset argument */ +#define NV_MSGBOX_DATA_COPY_SRC_OFFSET 7:0 + +/* NV_MSGBOX_CMD_ARG2_REG_INDEX_SCRATCH_PAGE register layout */ +#define NV_MSGBOX_DATA_SCRATCH_PAGE_DST 7:0 +#define NV_MSGBOX_DATA_SCRATCH_PAGE_SRC 15:8 + +/* Async request status codes returned in the data register */ +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS 7:0 + +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_SUCCESS 0x00000000 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_CARD_NOT_PRESENT 0x00000001 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_DUAL_LINK_INUSE 0x00000002 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_GENERIC 0x00000003 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_GPU_NOT_FULL_POWER 0x00000004 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_IN_USE 0x00000005 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INSUFFICIENT_RESOURCES 0x00000006 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_ACCESS_TYPE 0x00000007 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_ARGUMENT 0x00000008 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_BASE 0x00000009 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_CHANNEL 0x0000000A +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_CLASS 0x0000000B +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_CLIENT 0x0000000C +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_COMMAND 0x0000000D +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_DATA 0x0000000E +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_DEVICE 0x0000000F +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_DMA_SPECIFIER 0x00000010 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_EVENT 0x00000011 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_FLAGS 0x00000012 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_FUNCTION 0x00000013 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_HEAP 0x00000014 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_INDEX 0x00000015 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_LIMIT 0x00000016 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_METHOD 0x00000017 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_BUFFER 0x00000018 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_ERROR 0x00000019 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_HANDLE 0x0000001A +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_NEW 0x0000001B +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_OLD 0x0000001C +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OBJECT_PARENT 0x0000001D +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OFFSET 0x0000001E +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OWNER 0x0000001F +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_PARAM_STRUCT 0x00000020 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_PARAMETER 0x00000021 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_POINTER 0x00000022 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_REGISTRY_KEY 0x00000023 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_STATE 0x00000024 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_STRING_LENGTH 0x00000025 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_XLATE 0x00000026 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_IRQ_NOT_FIRING 0x00000027 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_MULTIPLE_MEMORY_TYPES 0x00000028 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_NOT_SUPPORTED 0x00000029 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_OPERATING_SYSTEM 0x0000002A +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_PROTECTION_FAULT 0x0000002B +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_TIMEOUT 0x0000002C +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_TOO_MANY_PRIMARIES 0x0000002D +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_IRQ_EDGE_TRIGGERED 0x0000002E +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_OPERATION 0x0000002F +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_NOT_COMPATIBLE 0x00000030 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_MORE_PROCESSING_REQUIRED 0x00000031 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INSUFFICIENT_PERMISSIONS 0x00000032 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_TIMEOUT_RETRY 0x00000033 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_NOT_READY 0x00000034 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_GPU_IS_LOST 0x00000035 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_IN_FULLCHIP_RESET 0x00000036 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_LOCK_STATE 0x00000037 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_ADDRESS 0x00000038 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INVALID_IRQ_LEVEL 0x00000039 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_MEMORY_TRAINING_FAILED 0x00000040 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_BUSY_RETRY 0x00000041 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_INSUFFICIENT_POWER 0x00000042 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_OBJECT_NOT_FOUND 0x00000043 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_BUFFER_TOO_SMALL 0x00000044 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_RESET_REQUIRED 0x00000045 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_PRIV_SEC_VIOLATION 0x00000046 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_DEFERRED 0x00000047 +#define NV_MSGBOX_DATA_ASYNC_REQ_STATUS_ERROR_FREQ_NOT_SUPPORTED 0x00000048 + +/* Response to NV_MSGBOX_CMD_OPCODE_GET_POWER_CONNECTOR_STATE */ +#define NV_MSGBOX_DATA_POWER_CONNECTED 0:0 +#define NV_MSGBOX_DATA_POWER_CONNECTED_SUFFICIENT 0x00000000 +#define NV_MSGBOX_DATA_POWER_CONNECTED_INSUFFICIENT 0x00000001 + +/* Response to NV_MSGBOX_CMD_OPCODE_GET_PAGE_RETIREMENT_STATS */ +#define NV_MSGBOX_DATA_RETIRED_PAGES_CNT_SBE 7:0 +#define NV_MSGBOX_DATA_RETIRED_PAGES_CNT_DBE 15:8 + +/* + * Response to + * NV_MSGBOX_CMD_ARG1_GET_MISC_ECC_ENABLED_STATE + * NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_0 + */ +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_SUPPORTED 0:0 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_SUPPORTED_OFF 0x00000000 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_SUPPORTED_ON 0x00000001 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_CURRENT 1:1 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_CURRENT_OFF 0x00000000 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_CURRENT_ON 0x00000001 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_PENDING 2:2 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_PENDING_OFF 0x00000000 +#define NV_MSGBOX_DATA_ECC_ENABLED_STATE_PENDING_ON 0x00000001 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_0 */ +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_SUPPORTED 3:3 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_SUPPORTED_OFF 0x00000000 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_SUPPORTED_ON 0x00000001 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_CURRENT 4:4 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_CURRENT_OFF 0x00000000 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_CURRENT_ON 0x00000001 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_PENDING 5:5 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_PENDING_OFF 0x00000000 +#define NV_MSGBOX_DATA_MIG_ENABLED_STATE_PENDING_ON 0x00000001 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_MISC_DEVICE_FLAGS_PAGE_0 */ +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_STATUS 6:6 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_STATUS_NOT_RUNNING 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_STATUS_RUNNING 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DEVICE_STATUS 7:7 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DEVICE_STATUS_ENABLED 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DEVICE_STATUS_DISABLED 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS 10:8 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS_NOT_RUNNING 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS_RUNNING 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS_TIMEOUT 0x00000002 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_FABRIC_MANAGER_STATUS_ERROR 0x00000003 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE 13:11 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_NONE 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_LOCAL_DEVICE 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_PEER_DEVICE 0x00000002 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_IN_BAND 0x00000003 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_SOURCE_OUT_OF_BAND 0x00000004 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON 16:14 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_NONE 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_CLIENT_OVERRIDE 0x00000001 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_TRUNK_LINK_FAILED 0x00000002 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_ACCESS_LINK_FAILED 0x00000003 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DISABLE_REASON_UNSPECIFIED_FAILURE 0x00000004 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_RELOAD 17:17 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_RELOAD_NOT_PENDING 0x00000000 +#define NV_MSGBOX_DATA_DEVICE_DISABLE_STATE_DRIVER_RELOAD_PENDING 0x00000001 + +/* Response to NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_GET */ +#define NV_MSGBOX_DATA_ACCESS_WP_MODE_GET_STATE 7:0 +#define NV_MSGBOX_DATA_ACCESS_WP_MODE_GET_STATE_DISABLED 0x0000005A +#define NV_MSGBOX_DATA_ACCESS_WP_MODE_GET_STATE_ENABLED 0x000000A5 + +/* + * Response to + * NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_RESET_REQUIRED + * NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_1 + */ +#define NV_MSGBOX_DATA_GPU_RESET_REQUIRED 0:0 +#define NV_MSGBOX_DATA_GPU_RESET_REQUIRED_OFF 0x00000000 +#define NV_MSGBOX_DATA_GPU_RESET_REQUIRED_ON 0x00000001 +#define NV_MSGBOX_DATA_GPU_DRAIN_AND_RESET_RECOMMENDED 1:1 +#define NV_MSGBOX_DATA_GPU_DRAIN_AND_RESET_RECOMMENDED_NO 0x00000000 +#define NV_MSGBOX_DATA_GPU_DRAIN_AND_RESET_RECOMMENDED_YES 0x00000001 + +/** + * Response to + * NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_PAGE_3 + */ +#define NV_MSGBOX_DATA_GET_CLOCK_FREQ_INFO_PAGE_3_CURRENT_PSTATE 3:0 +#define NV_MSGBOX_DATA_GET_CLOCK_FREQ_INFO_PAGE_3_CURRENT_PSTATE_INVALID 0x0000000F + +/* + * Number of Nvlink data outputs (dataOut, extData) for + * NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK queries + */ +#define NV_MSGBOX_DATA_NVLINK_INFO_DATA_WIDTH 2 + +/* + * Encoding for Invalid data + */ +#define NV_MSGBOX_DATA_NVLINK_INFO_DATA_INVALID 0xFFFFFFFF + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_STATE_V1 */ +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V1__SIZE 32 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V1_DOWN 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V1_UP 0x00000001 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V1_INVALID 0x000000ff + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_STATE_V2 */ +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2(i) (3+(i)*4):(0+(i)*4) +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2__SIZE 8 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_OFF 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_SAFE 0x00000001 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_ACTIVE 0x00000002 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_ERROR 0x00000003 +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_STATE_V2_INVALID 0x000000ff + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK_BANDWIDTH (in Mps) */ +#define NV_MSGBOX_DATA_NVLINK_INFO_LINK_BANDWIDTH__SIZE 1 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_INVALID 0xFFFFFFFF +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_00000_MBPS 0 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_16000_MBPS 16000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_19200_MBPS 19200 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_20000_MBPS 20000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_25000_MBPS 25000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_25781_MBPS 25781 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_32000_MBPS 32000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_40000_MBPS 40000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_50000_MBPS 50000 +#define NV_MSGBOX_DATA_NVLINK_LINE_RATE_53125_MBPS 53125 + +/* Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_SUBLINK_WIDTH */ +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_WIDTH(i) (5+(i)*6):(0+(i)*6) +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_WIDTH__SIZE 5 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_WIDTH_INVALID 0x000000ff +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH 2:0 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_0 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_1 0x00000001 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_2 0x00000002 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_4 0x00000003 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_8 0x00000004 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_TX_WIDTH_INVALID 0x00000007 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH 5:3 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_0 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_1 0x00000001 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_2 0x00000002 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_4 0x00000003 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_8 0x00000004 +#define NV_MSGBOX_DATA_NVLINK_INFO_SUBLINK_RX_WIDTH_INVALID 0x00000007 + +/* + * Response to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_TRAINING_ERR_STATE + * NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_RUNTIME_ERR_STATE + */ +#define NV_MSGBOX_DATA_NVLINK_INFO_ERROR_STATE__SIZE 32 +#define NV_MSGBOX_DATA_NVLINK_INFO_ERROR_STATE_INVALID 0x000000ff +#define NV_MSGBOX_DATA_NVLINK_INFO_ERROR_STATE_NO_ERROR 0x00000000 +#define NV_MSGBOX_DATA_NVLINK_INFO_ERROR_STATE_ERROR 0x00000001 + +/* Respones to NV_MSGBOX_CMD_ARG2_REMAP_ROW_RAW_CNT_COMBINED */ +#define NV_MSGBOX_DATA_REMAP_ROW_UNCORR_CNT 10:0 +#define NV_MSGBOX_DATA_REMAP_ROW_UNCORR_CNT_EXCESS 11:11 +#define NV_MSGBOX_DATA_REMAP_ROW_UNCORR_CNT_EXCESS_FALSE 0 +#define NV_MSGBOX_DATA_REMAP_ROW_UNCORR_CNT_EXCESS_TRUE 1 +#define NV_MSGBOX_DATA_REMAP_ROW_CORR_CNT 22:12 +#define NV_MSGBOX_DATA_REMAP_ROW_CORR_CNT_EXCESS 23:23 +#define NV_MSGBOX_DATA_REMAP_ROW_CORR_CNT_EXCESS_FALSE 0 +#define NV_MSGBOX_DATA_REMAP_ROW_CORR_CNT_EXCESS_TRUE 1 + +/* Response to NV_MSGBOX_CMD_ARG1_REMAP_ROWS_STATE_FLAGS */ +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_FAILED_REMAPPING 0:0 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_FAILED_REMAPPING_FALSE 0 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_FAILED_REMAPPING_TRUE 1 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_PENDING_REMAPPING 1:1 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_PENDING_REMAPPING_FALSE 0 +#define NV_MSGBOX_DATA_REMAP_ROW_STATE_FLAGS_PAGE0_PENDING_REMAPPING_TRUE 1 + +/* Response to NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM */ +#define NV_MSGBOX_DATA_REMAP_ROW_HISTOGRAM_LOW_AVAILABILITY 15:0 +#define NV_MSGBOX_DATA_REMAP_ROW_HISTOGRAM_PARTIAL_AVAILABILITY 31:16 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_0 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED 2:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_UNKNOWN 0x00000000 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_2500_MTPS 0x00000001 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_5000_MTPS 0x00000002 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_8000_MTPS 0x00000003 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_16000_MTPS 0x00000004 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_SPEED_32000_MTPS 0x00000005 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH 6:4 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_UNKNOWN 0x00000000 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X1 0x00000001 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X2 0x00000002 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X4 0x00000003 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X8 0x00000004 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X16 0x00000005 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_LINK_WIDTH_X32 0x00000006 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_NONFATAL_ERROR_COUNT 15:8 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_FATAL_ERROR_COUNT 23:16 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_0_UNSUPP_REQ_COUNT 31:24 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_1 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_1_L0_TO_RECOVERY_COUNT 31:0 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_2 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_2_REPLAY_ROLLOVER_COUNT 15:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_2_NAKS_RCVD_COUNT 31:16 + +/* + * Input for NV_MSGBOX_CMD_OPCODE_GPU_PERFORMANCE_MONITORING. Value is valid + * only if Arg2 != GPM_PARTITION_AGGREGATE and Arg2.Bit7 == 1 + */ +#define NV_MSGBOX_DATA_GPM_COMPUTE_INSTANCE_INDEX 15:8 + +/* + * The following three fields correspond to the data which is interpreted + * differently depending on GPM Metric specified in Arg1. + */ +#define NV_MSGBOX_DATA_GPM_NVLINK_INDEX 7:0 +#define NV_MSGBOX_DATA_GPM_NVDEC_INSTANCE 7:0 +#define NV_MSGBOX_DATA_GPM_NVJPG_INSTANCE 7:0 + +/* MSGBOX Extended Data Register */ +#define NV_MSGBOX_EXT_DATA_REG 31:0 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_0 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_0_CORRECTABLE_ERROR_COUNT 15:0 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_1 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_1_REPLAY_COUNT 31:0 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_2 */ +#define NV_MSGBOX_EXT_DATA_PCIE_LINK_INFO_PAGE_2_NAKS_SENT_COUNT 15:0 + +/* Respones to NV_MSGBOX_CMD_ARG1_GET_PCIE_LINK_INFO_PAGE_3 */ +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED 2:0 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_UNKNOWN 0x00000000 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_2500_MTPS 0x00000001 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_5000_MTPS 0x00000002 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_8000_MTPS 0x00000003 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_16000_MTPS 0x00000004 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_32000_MTPS 0x00000005 +#define NV_MSGBOX_DATA_PCIE_LINK_INFO_PAGE_3_TARGET_LINK_SPEED_RESERVED 2:0 + +/* Response to NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM */ +#define NV_MSGBOX_EXT_DATA_REMAP_ROW_HISTOGRAM_MAX_AVAILABILITY 31:16 +#define NV_MSGBOX_EXT_DATA_REMAP_ROW_HISTOGRAM_HIGH_AVAILABILITY 15:0 + +/* SysId info per type sizes */ +#define NV_MSGBOX_SYSID_DATA_SIZE_BOARD_PART_NUM_V1 24 +#define NV_MSGBOX_SYSID_DATA_SIZE_OEM_INFO_V1 24 +#define NV_MSGBOX_SYSID_DATA_SIZE_SERIAL_NUM_V1 16 +#define NV_MSGBOX_SYSID_DATA_SIZE_MARKETING_NAME_V1 24 +#define NV_MSGBOX_SYSID_DATA_SIZE_GPU_PART_NUM_V1 16 +#define NV_MSGBOX_SYSID_DATA_SIZE_MEMORY_VENDOR_V1 1 +#define NV_MSGBOX_SYSID_DATA_SIZE_MEMORY_PART_NUM_V1 20 +#define NV_MSGBOX_SYSID_DATA_SIZE_BUILD_DATE_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_FIRMWARE_VER_V1 14 +#define NV_MSGBOX_SYSID_DATA_SIZE_VENDOR_ID_V1 2 +#define NV_MSGBOX_SYSID_DATA_SIZE_DEV_ID_V1 2 +#define NV_MSGBOX_SYSID_DATA_SIZE_SUB_VENDOR_ID_V1 2 +#define NV_MSGBOX_SYSID_DATA_SIZE_SUB_ID_V1 2 +#define NV_MSGBOX_SYSID_DATA_SIZE_GPU_GUID_V1 16 +#define NV_MSGBOX_SYSID_DATA_SIZE_INFOROM_VER_V1 16 +#define NV_MSGBOX_SYSID_DATA_SIZE_PRODUCT_LENGTH_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_PRODUCT_WIDTH_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_PRODUCT_HEIGHT_V1 4 +#define NV_MSGBOX_SYSID_DATA_SIZE_PCIE_SPEED_V1 1 +#define NV_MSGBOX_SYSID_DATA_SIZE_PCIE_WIDTH_V1 1 + +/*! + * Response to NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_CLK + * 16-bit frequency in MHz + */ +#define NV_MSGBOX_DATA_POWER_HINT_INFO_CLK_GR_MIN 15:0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_CLK_GR_MAX 31:16 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_CLK_MEM_MIN 15:0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_CLK_MEM_MAX 31:16 + +/*! + * Response to NV_MSGBOX_CMD_ARG1_GET_POWER_HINT_INFO_TEMP + * 16-bit frequency in MHz + */ +#define NV_MSGBOX_DATA_POWER_HINT_INFO_TEMP_MIN 15:0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_TEMP_MAX 31:16 + +/*! + * Response to _ARG1_GET_POWER_HINT_INFO_PROFILES, 1 page consists of data and + * ext data, then we can support up to 256 profiles + * Bit index represents profile ID + */ +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_PERF 0:0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_PERF_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_PERF_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_HIGH_K 1:1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_HIGH_K_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_HIGH_K_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_LOW_K 2:2 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_LOW_K_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_DMMA_LOW_K_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_HMMA 3:3 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_HMMA_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_HMMA_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_IMMA 4:4 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_IMMA_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_IMMA_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_SGEMM 5:5 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_SGEMM_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_SGEMM_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_TRANSFORMER 6:6 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_TRANSFORMER_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_TRANSFORMER_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_0 7:7 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_0_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_0_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_1 8:8 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_1_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_1_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_2 9:9 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_2_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_2_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_3 10:10 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_3_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_3_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_4 11:11 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_4_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_4_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_5 12:12 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_5_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_5_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_6 13:13 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_6_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_6_AVAILABLE 1 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_7 14:14 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_7_NOT_AVAILABLE 0 +#define NV_MSGBOX_DATA_POWER_HINT_INFO_PROFILES_PAGE_0_CUSTOMER_CUSTOM_7_AVAILABLE 1 + +#define NV_MSGBOX_DATA_DYN_SYS_INFO_SIZE_DRIVER_VERSION 64 + +/* Event types */ +typedef enum +{ + NV_MSGBOX_EVENT_TYPE_SERVER_RESTART_COLD = 0, + NV_MSGBOX_EVENT_TYPE_GPU_RESET_REQUIRED, + NV_MSGBOX_EVENT_TYPE_DRIVER_ERROR_MESSAGE_OLDEST, + NV_MSGBOX_EVENT_TYPE_TGP_LIMIT_SET_SUCCESS, + NV_MSGBOX_EVENT_TYPE_CLOCK_LIMIT_SET_SUCCESS, + NV_MSGBOX_EVENT_TYPE_ECC_TOGGLE_SUCCESS, + NV_MSGBOX_EVENT_TYPE_MIG_TOGGLE_SUCCESS, + NV_MSGBOX_EVENT_TYPE_SERVER_RESTART_WARM, + NV_MSGBOX_EVENT_TYPE_DRIVER_ERROR_MESSAGE_NEW, + NV_MSGBOX_NUM_EVENTS, /* insert new event types before this line */ +} NvMsgboxEventType; + +/* Legacy event names for compatipility */ +#define NV_MSGBOX_EVENT_TYPE_SERVER_RESTART NV_MSGBOX_EVENT_TYPE_SERVER_RESTART_COLD +#define NV_MSGBOX_EVENT_TYPE_DRIVER_ERROR_MESSAGE NV_MSGBOX_EVENT_TYPE_DRIVER_ERROR_MESSAGE_OLDEST + +/* Bit mask of all defined events */ +#define NV_MSGBOX_EVENT_TYPE__ALL (NVBIT(NV_MSGBOX_NUM_EVENTS) - 1) + +/* ----------------------- Set Master Capabilities ---------------------------- */ + +// +// SET_MASTER_CAPS +// +// Descripiton: +// Set/declare SMBPBI Master capabilities. +// +// Common Encoding: +// 7 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | CLASS | ARG1 | CAP_INDEX | ARG2 +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +// The SMBPBI Master capabilities are logically grouped into classes (specified +// in arg1). Capabilities are sent 32 at a time (dword index stored in arg2) +// per-class (typically once during system-initialization). +// +// Classes: +// _GPU_REQ - Class of capability bits corresponding 1:1 with the GPU- +// Request opcodes the master supports. +// +#define NV_MSGBOX_CMD_SET_MASTER_CAPS_ARG1_CLASS 7:0 +#define NV_MSGBOX_CMD_SET_MASTER_CAPS_ARG1_CLASS_GPU_REQ 0x00000000 + +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_COUNT 1 + +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0 31:0 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_CPU_PCONTROL 0:0 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_CPU_PCONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_CPU_PCONTROL_AVAILABLE 0x00000001 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_SYS_PCONTROL 1:1 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_SYS_PCONTROL_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_DATA_MASTER_CAPS_GPU_REQ_0_SYS_PCONTROL_AVAILABLE 0x00000001 + +/* ------------------ GPU Performance Control (PCONTROL) ---------------------- */ + +// +// GPU_PCONTROL arg1, arg2, and data encoding (fields are relative to zero, not +// relative to their position in the COMMAND/DATA registers). +// + +// +// ARG1/ARG2 Encoding +// +// 7 3 2 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | TARGET | ACT | ARG1 |0 0 0 0 0 0 0 0| ARG2 (reserved) +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION 2:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION_GET_INFO 0x00000000 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION_GET_LIMIT 0x00000001 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION_SET_LIMIT 0x00000002 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_ACTION_GET_STATUS 0x00000003 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_TARGET 7:3 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_TARGET_VPSTATE 0x00000000 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_TARGET_POWER_SMBIDX 0x00000001 +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG2_RSVD 7:0 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_SET_ACTION(arg1, action) \ + (arg1) = FLD_SET_DRF(_MSGBOX_CMD, _GPU_PCONTROL_ARG1, _ACTION, \ + action, (arg1)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_GET_ACTION(arg1) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_ARG1, _ACTION, (arg1)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_SET_TARGET(arg1, target) \ + (arg1) = FLD_SET_DRF(_MSGBOX_CMD, _GPU_PCONTROL_ARG1, _TARGET, \ + target, (arg1)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_ARG1_GET_TARGET(arg1) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_ARG1, _TARGET, (arg1)) + +// +// Getting GPU vPstate Information +// +// Inputs: +// ARG1 - {TARGET=VPSTATE, ACTION=GET_INFO} +// ARG2 - Unused/reserved (must be zero) +// +// Outputs: +// DATA - Current min/max range of externally available vpstates +// (vPmin=fastest, vPmax=slowest) +// +// Encoding: +// 31 16 15 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0| MIN | MAX | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_MAX 7:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_MIN 15:8 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_RSVD 31:16 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_SET_MAX(data, max) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_GET_INFO_MAX, (max), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_GET_MAX(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_GET_INFO_MAX, (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_SET_MIN(data, min) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_GET_INFO_MIN, (min), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_INFO_GET_MIN(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_GET_INFO_MIN, (data)) + +// +// Getting GPU vPstate Limits +// +// Inputs: +// ARG1 - {TARGET=VPSTATE, ACTION=GET_LIMIT} +// ARG2 - Unused/reserved (must be zero) +// +// Outputs: +// DATA - Last requested vpstate limit/value, zero when no limit has been set. +// +// Encoding: +// 31 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0| VALUE | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_LIMIT_VALUE 7:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_LIMIT_RSVD 31:8 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_LIMIT_SET_VALUE(data, value) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_GET_LIMIT_VALUE, (value), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_LIMIT_GET_VALUE(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_GET_LIMIT_VALUE, (data)) + +// +// Setting GPU vPstate Limits: +// +// Inputs: +// ARG1 - {TARGET=VPSTATE, ACTION=SET_LIMIT} +// ARG2 - Unused/reserved (must be zero) +// DATA - Desired vpstate limit/value between vPmin and vPmax (inclusive), +// zero to clear active limits. +// Outputs: +// none +// +// Encoding: +// 31 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0| VALUE | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_VALUE 7:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_VALUE_CLEAR 0x00000000 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_VALUE_INIT 0x00000000 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_RSVD 31:8 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_SET_VALUE(data, limit) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_SET_LIMIT_VALUE, (limit), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_SET_LIMIT_GET_VALUE(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_SET_LIMIT_VALUE, (data)) + +// +// Getting GPU vPstate Status +// +// Inputs: +// ARG1 - {TARGET=VPSTATE, ACTION=GET_STATUS} +// ARG2 - Unused/reserved (must be zero) +// +// Outputs: +// DATA - Current vPstate +// +// Encoding: +// 31 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0| CURRENT | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_STATUS_CURRENT 7:0 +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_STATUS_RSVD 31:8 + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_STATUS_SET_CURRENT(data, curr) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_PCONTROL_DATA, \ + _VPSTATE_GET_STATUS_CURRENT, (curr), (data)) + +#define NV_MSGBOX_CMD_GPU_PCONTROL_DATA_VPSTATE_GET_STATUS_GET_CURRENT(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_PCONTROL_DATA, _VPSTATE_GET_STATUS_CURRENT, (data)) + + +/* ------------------------- GPU System Control ------------------------------- */ + +// +// GPU_SYSCONTROL +// +// Description: +// Command sent to the GPU to set/remove a system-imposed operating limit +// on a specific system-parameter. +// +// Common Encoding: +// 7 2 1 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | TARGET |ACT| ARG1 |0 0 0 0 0 0 0 0| ARG2 (reserved) +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +// Notes: +// - TARGET accepts NV0000_CTRL_SYSTEM_PARAM_* definitions +// - arg1, arg2, and data encoding (fields are relative to zero, not relative +// to their position in the DATA_OUT register). +// +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_ACTION 1:0 +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_ACTION_CLEAR_LIMIT 0x00000000 +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_ACTION_SET_LIMIT 0x00000001 +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_TARGET 7:2 +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG2_RSVD 7:0 + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_SET_ACTION(arg1, action) \ + (arg1) = FLD_SET_DRF(_MSGBOX_CMD, _GPU_SYSCONTROL_ARG1, _ACTION, \ + action, (arg1)) + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_GET_ACTION(arg1) \ + DRF_VAL(_MSGBOX_CMD, _GPU_SYSCONTROL_ARG1, _ACTION, (arg1)) + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_SET_TARGET(arg1, target) \ + (arg1) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_SYSCONTROL_ARG1, _TARGET, \ + target, (arg1)) + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_ARG1_GET_TARGET(arg1) \ + DRF_VAL(_MSGBOX_CMD, _GPU_SYSCONTROL_ARG1, _TARGET, (arg1)) + +// +// Setting a System-Parameter Operating-Limit: +// +// Inputs: +// ARG1 - {TARGET=NV_MSGBOX_SYS_PARAM_*, ACTION=SET_LIMIT} +// ARG2 - Unused/reserved (must be zero) +// DATA - Desired limit/value. +// +// Outputs: +// none +// +// Encoding: +// 31 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// | VALUE | DATA +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_DATA_SET_LIMIT_VALUE 31:0 + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_DATA_SET_LIMIT_SET_VALUE(data, limit) \ + (data) = FLD_SET_DRF_NUM(_MSGBOX_CMD, _GPU_SYSCONTROL_DATA, \ + _SET_LIMIT_VALUE, (limit), (data)) + +#define NV_MSGBOX_CMD_GPU_SYSCONTROL_DATA_PARAM_SET_LIMIT_GET_VALUE(data) \ + DRF_VAL(_MSGBOX_CMD, _GPU_SYSCONTROL_DATA, _SET_LIMIT_VALUE, (data)) + +// +// Host side scratch memory buffer +// This memory buffer is allocated by the host CPU in its system memory +// and is used for passing additional arguments and results, that don't +// fit in the command/status and data registers due to their size. +// This memory can be accessed by the BMC SMBus master through +// dedicated requests. +// +// Page size is various on different SMBPBI servers. +// Capability dword[2] bit 12 shows size in bytes +// +// _1024 --> 0xFF dwords +// _256 --> 0x40 dowrds +// +#define NV_MSGBOX_SCRATCH_PAGE_SIZE_1024_D 0xFF +#define NV_MSGBOX_SCRATCH_PAGE_SIZE_256_D 0x40 + +// +// TODO Once switch soc/pmu/oobtest remove the reference to this, this will be +// removed. +// +#define NV_MSGBOX_SCRATCH_PAGE_SIZE 1024 +#define NV_MSGBOX_SCRATCH_NUM_PAGES_P 2 // expressed as a power of 2 + // must be >= 2 + +// This cap code plugs into NV_MSGBOX_DATA_CAP_2_NUM_SCRATCH_BANKS +#define NV_MSGBOX_SCRATCH_NUM_PAGES_CAP_CODE \ + (NV_MSGBOX_SCRATCH_NUM_PAGES_P > 0 ? \ + NV_MSGBOX_SCRATCH_NUM_PAGES_P - 1 : 0) + +#define NV_MSGBOX_SCRATCH_NUM_PAGES (1 << NV_MSGBOX_SCRATCH_NUM_PAGES_P) +#define NV_MSGBOX_SCRATCH_BUFFER_SIZE (NV_MSGBOX_SCRATCH_NUM_PAGES \ + * NV_MSGBOX_SCRATCH_PAGE_SIZE) + +// Auxiliary definitions related to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_STATE_SPEED +#define NV_MSGBOX_NVLINK_STATE_BITS_PER_NVLINK 4 +#define NV_MSGBOX_NVLINK_STATE_NVLINKS_PER_REGISTER (32 / NV_MSGBOX_NVLINK_STATE_BITS_PER_NVLINK) +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_VOLTA 6 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_TURING 2 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_GA100 3 +#define NV_MSGBOX_NVLINK_STATE_NUM_LINKS_PER_NVLIPT_NVL30 4 +#define NV_MSGBOX_NVLINK_SUBLINK_WIDTH_MAX_NVL30 4 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_GA100 \ + (NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_GA100 * \ + NV_MSGBOX_NVLINK_STATE_NUM_LINKS_PER_NVLIPT_NVL30) +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_GA10X 1 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_GA10X \ + (NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_GA10X * \ + NV_MSGBOX_NVLINK_STATE_NUM_LINKS_PER_NVLIPT_NVL30) +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_LR10 9 +#define NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS_LR10 \ + (NV_MSGBOX_NVLINK_STATE_NUM_NVLIPT_LR10 * \ + NV_MSGBOX_NVLINK_STATE_NUM_LINKS_PER_NVLIPT_NVL30) +#define NV_MSGBOX_NVLINK_STATE_NUM_REGISTERS(arch) \ + ((NV_MSGBOX_NVLINK_STATE_NUM_NVLINKS##arch + NV_MSGBOX_NVLINK_STATE_NVLINKS_PER_REGISTER - 1) \ + / NV_MSGBOX_NVLINK_STATE_NVLINKS_PER_REGISTER) + +// Auxiliary definitions related to NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_THROUGHPUT_* +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_DATA_TX 0x00000000 +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_DATA_RX 0x00000001 +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_RAW_TX 0x00000002 +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_RAW_RX 0x00000003 +#define NV_MSGBOX_NVLINK_THROUGHPUT_COUNTER_INDEX_MAX 0x00000004 + +#if !defined(NV_MSGBOX_NO_PARAM_STRUCTS) +// +// Parameter structures for asynchronous requests +// passed in scratch space +// + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_GET and + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_SET + */ +typedef struct +{ + NvU32 flags; + /*! + * If flags:_PERSIST is _ON, make limitCurrInput persistent, + * surviving the driver reload and the system reboot. + * + * If flags:_CLEAR is _ON, it will clear the TGP limit. The + * persistence still depends on persist flag. + */ +#define NV_MSGBOX_PMGR_PWR_TGP_LIMIT_CONTROL_PARAMS_FLAGS_PERSIST 0:0 +#define NV_MSGBOX_PMGR_PWR_TGP_LIMIT_CONTROL_PARAMS_FLAGS_PERSIST_OFF 0x00000000 +#define NV_MSGBOX_PMGR_PWR_TGP_LIMIT_CONTROL_PARAMS_FLAGS_PERSIST_ON 0x00000001 +#define NV_MSGBOX_PMGR_PWR_TGP_LIMIT_CONTROL_PARAMS_FLAGS_CLEAR 1:1 +#define NV_MSGBOX_PMGR_PWR_TGP_LIMIT_CONTROL_PARAMS_FLAGS_CLEAR_OFF 0x00000000 +#define NV_MSGBOX_PMGR_PWR_TGP_LIMIT_CONTROL_PARAMS_FLAGS_CLEAR_ON 0x00000001 + + /*! + * Current total GPU power limit value to enforce, requested by the + * SMBPBI client, expressed in milliwatts. + * Must always be within range imposed by the current policy. + * 0xffffffff is returned, if the limit has not been set by the SMBPBI client. + */ + NvU32 limitCurrInput; + + /*! + * Currently arbitrated total GPU power limit value after taking + * into account limits, requested by all clients, expressed + * in milliwatts. + */ + NvU32 limitCurrOutput; +} NV_MSGBOX_PMGR_PWR_TGP_LIMIT_CONTROL_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_INFO_GET + */ +typedef struct +{ + /*! + * Current total GPU power limit lower and upper bounds and the + * default setting, expressed in milliwatts. + * These constraints must be observed, when the limit + * is being set with + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_TGP_LIMIT_CONTROL_SET. + */ + NvU32 limitMin; + NvU32 limitMax; + NvU32 limitDefault; +} NV_MSGBOX_PMGR_PWR_TGP_LIMIT_INFO_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_COUNT_GET + */ +typedef struct +{ + NvU32 fanCount; //<! number of fans on the board +} NV_MSGBOX_THERMAL_FAN_V1_COUNT_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_INFO_GET + */ +typedef struct +{ + NvU32 fanIndex; //<! fan ID to be filled by the caller + NvU32 fanProperties; //<! fan properties, detailed below + +//! Fan level metric +#define NV_MSGBOX_FAN_V1_PROP_LEVEL_METRIC 0:0 +#define NV_MSGBOX_FAN_V1_PROP_LEVEL_METRIC_PWM 0x00000000 +#define NV_MSGBOX_FAN_V1_PROP_LEVEL_METRIC_RPM 0x00000001 + +//! Fan tachometer availability +#define NV_MSGBOX_FAN_V1_PROP_TACH 1:1 +#define NV_MSGBOX_FAN_V1_PROP_TACH_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_FAN_V1_PROP_TACH_AVAILABLE 0x00000001 + +//! Fan tachometer bounds (min/max) availability +#define NV_MSGBOX_FAN_V1_PROP_TACH_BOUNDS 2:2 +#define NV_MSGBOX_FAN_V1_PROP_TACH_BOUNDS_NOT_AVAILABLE 0x00000000 +#define NV_MSGBOX_FAN_V1_PROP_TACH_BOUNDS_AVAILABLE 0x00000001 + + NvU16 fanLevelMin; //<! minimum fan level + NvU16 fanLevelMax; //<! maximum fan level + NvU16 fanTachMin; //<! tachometer minimum + NvU16 fanTachMax; //<! tachometer maximum +} NV_MSGBOX_THERMAL_FAN_V1_INFO_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_THERMAL_FAN_V1_STATUS_GET + */ +typedef struct +{ + NvU32 fanIndex; //<! fan ID to be filled by the caller + /*! + * Current fan level and speed + */ + NvU8 levelCurr; //<! current fan level, 0 if n/a + NvU16 speedCurr; //<! current fan speed (RPM), 0 if no tach + +} NV_MSGBOX_THERMAL_FAN_V1_STATUS_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OVERCLOCKING_LIMIT_CONTROL + */ +typedef struct +{ + NvU32 limitType; //<! Argument representing VPSTATE/CLOCK + +//! CLOCK/VPSTATE types - add types below +#define NV_MSGBOX_OVERCLOCKING_LIMIT_TYPE_MAX_CUSTOMER_BOOST_VPSTATE 0x00000001 + + NvU32 maxClkLimitMHz; //<! Clock limit + +} NV_MSGBOX_OVERCLOCKING_LIMIT_CONTROL_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_ENERGY_COUNTER_STATUS_GET + */ +typedef struct +{ + NV_DECLARE_ALIGNED(NvU64 energyCountermJ, 8); //<! Energy counter value + +} NV_MSGBOX_ENERGY_COUNTER_STATUS_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_VIOLATION_COUNTERS_STATUS_GET + */ +typedef struct +{ + NvU32 policyMask; + /*! + * Argument representing policies whose violation times + * are being requested/provided + */ + +//! NVBIT OFFSET <> POLICY - add types below +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_ALL_SUPPORTED_POLICIES 0xFFFFFFFF //-1 represents All Supported policies +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_HW_VIOLATION 0 +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_SW_GLOBAL_VIOLATION 1 +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_SW_POWER_VIOLATION 2 +#define NV_MSGBOX_VIOLATION_COUNTERS_TYPE_SW_THERMAL_VIOLATION 3 + + NV_DECLARE_ALIGNED(NvU64 violationTime[32], 8); //<! Violation counter value + +} NV_MSGBOX_VIOLATION_COUNTERS_STATUS_PARAMS; + +//! Number of clock domains to track for gathering utilization data. +#define NV_MSGBOX_UTILIZATION_RATE_CLOCK_DOMAIN_COUNT 2 + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_UTILIZATION_RATE_GET + */ +typedef struct +{ + NvU32 gpuUtil; //<! Percent of sample that the Graphics/SM unit remains busy. + NvU32 memUtil; //<! Percent of sample that the memory unit remains busy. + +} NV_MSGBOX_UTILIZATION_RATE_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_OOB_CLOCK_LIMIT_SET + * + * Masks of NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_<XYZ>. + */ +typedef struct +{ + NvU32 flags; + /*! + * If flags: _CLEAR is _ON, it will clear the limits. Otherwise, + * set the clock limits. + * + * If flags: _PERSIST is _ON, set/clear limits persistently, + * surviving the driver reload and the system reboot. + * + */ +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_PERSIST 0:0 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_PERSIST_OFF 0x00000000 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_PERSIST_ON 0x00000001 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_CLEAR 1:1 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_CLEAR_OFF 0x00000000 +#define NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_CLEAR_ON 0x00000001 + + /*! + * Graphics clock limit in MHz + */ + NvU32 clkMinFreqMHz; + NvU32 clkMaxFreqMHz; +} NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS; + +/* + * Parameters for NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_TEST_MESSAGE_SEND + */ +#define NV_MSGBOX_TEST_MESSAGE_SEND_STRING_SIZE_MAX 32 +/* + * If the message starts with this character, the message + * will not be sent to the kernel. + */ +#define NV_MSGBOX_TEST_MESSAGE_SEND_SILENT_CHAR '@' +typedef struct +{ + NvU8 string[NV_MSGBOX_TEST_MESSAGE_SEND_STRING_SIZE_MAX]; +} NV_MSGBOX_TEST_MESSAGE_SEND_PARAMS; + +/* + * Individual request structure definition for bundled requests + */ +typedef struct +{ + NvU32 cmdStatus; //<! Command/Status + NvU32 dataIn; //<! Data In + NvU32 dataOut; //<! Data Out + NvU32 extData; //<! Extended Data +} NV_MSGBOX_IND_REQUEST; + +/*! + * Driver Event Message record + */ +typedef struct +{ + NvU8 recordSize; //<! in 32-bit words + NvU8 xidId; //<! event type Id (Xid) + NvU8 flags; //<! bit 0: some DEMs after this one + //<! have been lost + //<! bit 1: the text message has been + //<! truncated + NvU32 seqNumber; //<! record sequential number + NvU32 timeStamp; //<! seconds since the epoch UTC + + /* + * Some Windows builds include this file, and older Microsoft + * compilers won't accept a zero size array in this context: + * "C4200 nonstandard extension used: zero-sized array in struct/union", + * despite this being a legitimate C99. + * We need to find a way to fix this, if we ever decide to + * implement DEM/Xid logging for Windows. + */ + NvU8 textMessage[]; //<! NUL terminated + +} NV_MSGBOX_DEM_RECORD; +#define NV_MSGBOX_DEM_RECORD_FLAGS_OVFL 0:0 +#define NV_MSGBOX_DEM_RECORD_FLAGS_OVFL_NOT_SET 0x00000000 +#define NV_MSGBOX_DEM_RECORD_FLAGS_OVFL_SET 0x00000001 +#define NV_MSGBOX_DEM_RECORD_FLAGS_TRUNC 1:1 +#define NV_MSGBOX_DEM_RECORD_FLAGS_TRUNC_NOT_SET 0x00000000 +#define NV_MSGBOX_DEM_RECORD_FLAGS_TRUNC_SET 0x00000001 + +/*! + * Result Disposition Rule word layout + */ +#define NV_MSGBOX_DISP_RULE_REQ_IDX 2:0 +#define NV_MSGBOX_DISP_RULE_SRC_REG 4:3 +#define NV_MSGBOX_DISP_RULE_SRC_REG_STATUS 0x00000000 +#define NV_MSGBOX_DISP_RULE_SRC_REG_DATA 0x00000001 +#define NV_MSGBOX_DISP_RULE_SRC_REG_EXT_DATA 0x00000002 +#define NV_MSGBOX_DISP_RULE_SRC_REG_MAX 0x00000002 +#define NV_MSGBOX_DISP_RULE_SRC_RIGHT_BIT 9:5 +#define NV_MSGBOX_DISP_RULE_SRC_WIDTH 14:10 +#define NV_MSGBOX_DISP_RULE_DST_REG 16:15 +#define NV_MSGBOX_DISP_RULE_DST_REG_STATUS 0x00000000 +#define NV_MSGBOX_DISP_RULE_DST_REG_DATA 0x00000001 +#define NV_MSGBOX_DISP_RULE_DST_REG_EXT_DATA 0x00000002 +#define NV_MSGBOX_DISP_RULE_DST_REG_MAX 0x00000002 +#define NV_MSGBOX_DISP_RULE_DST_RIGHT_BIT 21:17 + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_CLOCK_LIMIT_GET + * Masks of NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS_FLAGS_<XYZ>. + */ +typedef struct { + /*! + * Graphics clock limit in MHz + * if there is no cap, return + * NV_MSGBOX_CLOCK_LIMIT_GET_PARAMS_CLOCK_LIMIT_NOT_SET + */ + NvU16 oobClientMin; + NvU16 oobClientMax; + NvU16 enforcedMin; + NvU16 enforcedMax; +#define NV_MSGBOX_CLOCK_LIMIT_GET_PARAMS_CLOCK_LIMIT_NOT_SET 0xffff +} NV_MSGBOX_CLOCK_LIMIT_GET_PARAMS; + +/*! + * This structure is used to hold parameters for + * NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_DEVICE_MODE_CONTROL + */ +typedef struct +{ + NvU16 modeType; //<! Mode type +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_MODE_TYPE_ECC 0x00000000 +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_MODE_TYPE_MIG 0x00000001 +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_MODE_TYPE_DISABLE 0x00000002 + + NvU16 flags; //<! flags +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_FLAGS_CONTROL 0:0 +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_FLAGS_CONTROL_CLEAR 0x00000000 +#define NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS_FLAGS_CONTROL_SET 0x00000001 + +} NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS; + +// +// This is the maximum number of curve points this API will be able to carry. +// With the understanding that the maximum number of points currently +// implemented in the RM is 3, we are leaving 2 additional slots as a room +// for possible future growth, so that we don't have to spin the API definition +// in case we ever want to implement more than 3 points. +// +#define NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_MAX_POINTS 5 + +typedef struct +{ + NvS16 temperature; //<! degrees C, signed 16 bit value + + union { + NvU16 level; //<! Fan PWM setting in the range 0..100 + //<! 0: no rotation + //<! 100: full speed + NvU16 rpmValue; //<! tach speed in RPM + } fanSetting; +} NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINT; + +#define NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_GET 0 +#define NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_GET_DEFAULT 1 +#define NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_SET 2 +typedef struct +{ + NvU8 action; //<! [in] + //<! NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_GET: + //<! get currently active point values + //<! NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_GET_DEFAULT: + //<! get VBIOS default point values + //<! NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_ACTION_SET: + //<! set new active point values + + NvU8 fanIndex; //<! [in] + + NvU8 numEffectivePoints; //<! [out] for fanGetCurvePoints. This is the + //<! number of points actually implemented + //<! and filled in the array below + //<! [in] for fanSetCurvePoints. This is how + //<! many points the caller is sending to + //<! the API. Must not exceed what was + //<! returned by fanGetCurvePoints. + + NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINT + curvePoints[NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_MAX_POINTS]; + //<! [out] for fanGetCurvePoints. + //<! [in] for fanSetCurvePoints. +} NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_PARAMS; + +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_DMMA_PERF 0 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_DMMA_HIGH_K 1 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_DMMA_LOW_K 2 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_HMMA 3 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_IMMA 4 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_SGEMM 5 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_TRANSFORMER 6 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_0 7 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_1 8 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_2 9 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_3 10 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_4 11 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_5 12 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_6 13 +#define NV_MSGBOX_POWER_HINT_PROFILES_ID_CUSTOMER_CUSTOM_7 14 +/*! + * This param is used for NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POWER_HINT_GET + */ +typedef struct +{ + /*! + * [out] Power Hint in mW + */ + NvU32 powermW; + /*! + * [in] Graphics Clock in MHz + */ + NvU16 grClkMHz; + /*! + * [in] Memory Clock + */ + NvU16 memClkMHz; + /*! + * [in] GPU Temperature + */ + NvS16 temperature; + /*! + * [in] Profile ID + */ + NvU8 profile; + // new param starting from here. +} NV_MSGBOX_POWER_HINT_PARAMS; + +/*! + * This param is used for NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_CONFIGURE_PROGRAMMABLE_EDPP + */ +typedef struct +{ + /*! + * [out] Power Hint in mW + */ + NvU32 data; +} NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS; + +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA 31:0 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_OPERATION 1:0 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_OPERATION_GET 0x0 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_OPERATION_SET 0x1 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_OPERATION_CLEAR 0x2 + +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_MODE 3:2 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_MODE_PERSISTENT 0x0 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_MODE_ONESHOT 0x1 + +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_LIMIT_CURRENT 10:4 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_LIMIT_RATED 17:11 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_LIMIT_MAXIMUM 24:18 +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_DATA_LIMIT_MINIMUM 31:25 + +#define NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS_LIMIT_NOT_SET 0x7F + +/*! + * @brief Union of all possible parameter struct. Used to determine the maximum + * amount of space parameter blocks can take. + */ +typedef union { + NV_MSGBOX_PMGR_PWR_TGP_LIMIT_CONTROL_PARAMS tgpLimitControl; + NV_MSGBOX_THERMAL_FAN_V1_COUNT_PARAMS fanCountV1Get; + NV_MSGBOX_THERMAL_FAN_V1_INFO_PARAMS fanCountV1Info; + NV_MSGBOX_THERMAL_FAN_V1_STATUS_PARAMS fanCountV1Status; + NV_MSGBOX_OVERCLOCKING_LIMIT_CONTROL_PARAMS overclockingLimitControl; + NV_MSGBOX_ENERGY_COUNTER_STATUS_PARAMS energyCounterStatus; + NV_MSGBOX_VIOLATION_COUNTERS_STATUS_PARAMS violationCountersStatus; + NV_MSGBOX_UTILIZATION_RATE_PARAMS utilizationRate; + NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS oobClockLimitCtrlParams; + NV_MSGBOX_DEVICE_MODE_CONTROL_PARAMS deviceModeControlParams; + NV_MSGBOX_TEST_MESSAGE_SEND_PARAMS testMessageSend; + NV_MSGBOX_CLOCK_LIMIT_GET_PARAMS clockLimitGet; + NV_MSGBOX_THERMAL_FAN_V3_FAN_CURVE_POINTS_PARAMS fanCurvePointsV3; + NV_MSGBOX_POWER_HINT_PARAMS powerHintParams; + NV_MSGBOX_PROGRAMMABLE_EDPP_PARAMS programmableEdppParams; +} NV_MSGBOX_ASYNC_REQ_PARAMS_UNION; + +#endif // !NV_MSGBOX_NO_PARAM_STRUCTS + +/* Utility command constructor macros */ + +#define NV_MSGBOX_CMD(opcode, arg1, arg2) \ + ( \ + DRF_DEF(_MSGBOX, _CMD, _OPCODE, opcode) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1, (arg1)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2, (arg2)) | \ + DRF_DEF(_MSGBOX, _CMD, _STATUS, _NULL) | \ + DRF_DEF(_MSGBOX, _CMD, _RSVD, _INIT) | \ + DRF_DEF(_MSGBOX, _CMD, _INTR, _PENDING) \ + ) +#define NV_MSGBOX_CMD_NULL_CMD() NV_MSGBOX_CMD(_NULL_CMD, 0, 0) +#define NV_MSGBOX_CMD_GET_CAP_DWORD(index) NV_MSGBOX_CMD(_GET_CAP_DWORD, (index), 0) +#define NV_MSGBOX_CMD_GET_TEMP(source) NV_MSGBOX_CMD(_GET_TEMP, (source), 0) +#define NV_MSGBOX_CMD_GET_EXT_TEMP(source) NV_MSGBOX_CMD(_GET_EXT_TEMP, (source), 0) +#define NV_MSGBOX_CMD_GET_POWER(source) NV_MSGBOX_CMD(_GET_POWER, (source), 0) +#define NV_MSGBOX_CMD_GET_SYS_ID_DATA(type, offset) \ + NV_MSGBOX_CMD(_GET_SYS_ID_DATA, (type), (offset) / 4) +#define NV_MSGBOX_CMD_SCRATCH_READ(off) NV_MSGBOX_CMD(_SCRATCH_READ, (off), 0) +#define NV_MSGBOX_CMD_SCRATCH_WRITE(off, cnt) NV_MSGBOX_CMD(_SCRATCH_WRITE, (off), (cnt) - 1) +#define NV_MSGBOX_CMD_SCRATCH_COPY(off, cnt) NV_MSGBOX_CMD(_SCRATCH_COPY, (off), (cnt) - 1) +#define NV_MSGBOX_CMD_ASYNC_REQUEST(req_type, scratch_offset) \ + NV_MSGBOX_CMD(_ASYNC_REQUEST, (req_type), (scratch_offset) / 4) +#define NV_MSGBOX_CMD_ASYNC_REQUEST_POLL(req_id) \ + NV_MSGBOX_CMD(_ASYNC_REQUEST, \ + NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POLL, \ + (req_id)) +#define NV_MSGBOX_CMD_REGISTER_READ(addr) NV_MSGBOX_CMD(_REGISTER_ACCESS, \ + NV_MSGBOX_CMD_ARG1_REGISTER_ACCESS_READ, \ + (addr)) +#define NV_MSGBOX_CMD_REGISTER_WRITE(addr) NV_MSGBOX_CMD(_REGISTER_ACCESS, \ + NV_MSGBOX_CMD_ARG1_REGISTER_ACCESS_WRITE, \ + (addr)) +#define NV_MSGBOX_CMD_GET_THERM_PARAM(param) NV_MSGBOX_CMD(_GET_THERM_PARAM, \ + NV_MSGBOX_CMD_ARG1_THERM_PARAM##param, 0) + +#define NV_MSGBOX_CMD_ACCESS_WP_MODE(action, state) \ + NV_MSGBOX_CMD(_ACCESS_WP_MODE, (action), (state)) + +#define NV_MSGBOX_CMD_GET_WP_MODE() \ + NV_MSGBOX_CMD_ACCESS_WP_MODE(NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_GET, 0) + +#define NV_MSGBOX_CMD_SET_WP_MODE(state) \ + NV_MSGBOX_CMD_ACCESS_WP_MODE(NV_MSGBOX_CMD_ARG1_ACCESS_WP_MODE_ACTION_SET, (state)) + +#define NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(subtype, arg2) \ + NV_MSGBOX_CMD(_GET_MISC_GPU_FLAGS, (subtype), (arg2)) + +#define NV_MSGBOX_CMD_GET_ECC_ENABLED_STATE() \ + NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(NV_MSGBOX_CMD_ARG1_GET_MISC_ECC_ENABLED_STATE, 0) + +#define NV_MSGBOX_CMD_GET_GPU_RESET_REQUIRED() \ + NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_RESET_REQUIRED, 0) + +#define NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS_PAGE_0() \ + NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_0, 0) + +#define NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS_PAGE_1() \ + NV_MSGBOX_CMD_GET_MISC_GPU_FLAGS(NV_MSGBOX_CMD_ARG1_GET_MISC_GPU_FLAGS_PAGE_1, 0) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO(subtype, arg2) \ + NV_MSGBOX_CMD(_GET_NVLINK_INFO, (subtype), (arg2)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(errtype, link_index) \ + NV_MSGBOX_CMD(_GET_NVLINK_INFO, \ + NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_ERR_COUNTER##errtype,\ + (link_index)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_STATE(errtype, link_page) \ + NV_MSGBOX_CMD(_GET_NVLINK_INFO, \ + NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO##errtype##_ERROR_STATE, \ + (link_page)) + +#define NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS(subtype) \ + NV_MSGBOX_CMD(_GPU_UTIL_COUNTERS, (subtype), 0) + +#define NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS_CONTEXT_TIME() \ + NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS(NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_CONTEXT_TIME) + +#define NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS_SM_TIME() \ + NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS(NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_SM_TIME) + +#define NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS_RESET_COUNTERS() \ + NV_MSGBOX_CMD_GET_GPU_UTIL_COUNTERS(NV_MSGBOX_CMD_ARG1_GPU_UTIL_COUNTERS_RESET_COUNTERS) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT_REPLAY(link_index) \ + NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(_REPLAY, (link_index)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT_RECOVERY(link_index) \ + NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(_RECOVERY, (link_index)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT_FLIT_CRC(link_index) \ + NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(_FLIT_CRC, (link_index)) + +#define NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT_DATA_CRC(link_index) \ + NV_MSGBOX_CMD_GET_NVLINK_INFO_ERR_CNT(_DATA_CRC, (link_index)) + +#define NV_MSGBOX_CMD_GET_CLOCK_FREQ_INFO(subtype, clkdomain) \ + NV_MSGBOX_CMD(_GET_CLOCK_FREQ_INFO, (subtype), (clkdomain)) + +#define NV_MSGBOX_CMD_GET_CLOCK_FREQ_INFO_GPC_CURRENT() \ + NV_MSGBOX_CMD_GET_CLOCK_FREQ_RANGE(NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_RANGE_GPC_CURRENT, \ + NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_GPCCLK) + +#define NV_MSGBOX_CMD_GET_CLOCK_FREQ_INFO_GPC_MIN() \ + NV_MSGBOX_CMD_GET_CLOCK_FREQ_RANGE(NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_GPC_MIN, \ + NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_GPCCLK) + +#define NV_MSGBOX_CMD_GET_CLOCK_FREQ_INFO_GPC_MAX() \ + NV_MSGBOX_CMD_GET_CLOCK_FREQ_RANGE(NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_GPC_MAX, \ + NV_MSGBOX_CMD_ARG2_GET_CLOCK_FREQ_INFO_GPCCLK) + +#define NV_MSGBOX_CMD_GET_DRIVER_EVENT_MSG(ptr) NV_MSGBOX_CMD(_GET_DRIVER_EVENT_MSG, 0, (ptr)) + +// +// ECC V2 +// +#define NV_MSGBOX_CMD_ECC_V2_CNT_FB(src, type, partition, slice) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_FB_SRC, src) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V2_CNT_GR(src, type, gpc, tpc) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_GR_SRC, src) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_GR_TPC, (tpc)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V2_FBA_CUMUL(partition, slice) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_SC, _CUMUL) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V2_FBA_IND_ADDR(partition, slice, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_IND_FLD, _ADDR) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V2_FBA_IND_CTRS(partition, slice, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V2, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_IND_FLD, _CTRS) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V2_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V2_FB_SLICE, (slice)) \ + ) + +// +// ECC V3 +// +#define NV_MSGBOX_CMD_ECC_V3_CNT_FB_LTC(type, partition, slice) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_FB_SRC, _LTC) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_CNT_FB_FB(type, partition, subpart) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_FB_SRC, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_SUBPARTITION, (subpart)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_CNT_GR(src, type, gpc, tpc) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_GR_SRC, src) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_TPC, (tpc)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_CNT_GR_TEX(type, gpc, tpc, tex) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_GR_SRC, _TEX) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_TPC, (tpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_GR_TEX, (tex)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_FBA_IND_ADDR(partition, subpart, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_IND_FLD, _ADDR) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_SUBPARTITION, (subpart)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V3_FBA_IND_CTRS(partition, subpart, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V3, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_IND_FLD, _CTRS) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V3_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V3_FB_SUBPARTITION, (subpart)) \ + ) + +// +// ECC V4 +// +#define NV_MSGBOX_CMD_ECC_V4_CNT_FB_LTC(type, partition, slice) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_FB_SRC, _LTC) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_SLICE, (slice)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_CNT_FB_FB(type, partition, subpart) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_DEV, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_FB_SRC, _FB) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_SUBPARTITION, (subpart)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_CNT_GR(src, type, gpc, tpc) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_GR_SRC, src) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_TPC, (tpc)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_CNT_GR_TEX(type, gpc, tpc, tex) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _CNT) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_DEV, _GR) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_GR_SRC, _TEX) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_CNT_TYPE, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_GPC, (gpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_TPC, (tpc)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_GR_TEX, (tex)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_FBA_IND_ADDR(partition, subpart, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_IND_FLD, _ADDR) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_SUBPARTITION, (subpart)) \ + ) + +#define NV_MSGBOX_CMD_ECC_V4_FBA_IND_CTRS(partition, subpart, index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V4, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_SEL, _FBA) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_SC, _IND) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_IND_FLD, _CTRS) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V4_FBA_IND_INDEX, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_PARTITION, (partition)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V4_FB_SUBPARTITION, (subpart)) \ + ) + +// +// ECC V5 +// +#define NV_MSGBOX_CMD_ECC_V5_CNT(index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V5, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V5_SEL, _COUNTS) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V5_COUNTS_INDEX_LO, (index)) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_INDEX_HI, (index) >> 7) \ + ) +#define NV_MSGBOX_ECC_V5_COUNTS_NUM_ENTRIES 600 + +#define NV_MSGBOX_CMD_ECC_V5_CNT_HDR_META(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _HDR_META) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_REGN_CORRECTED_TOTAL(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _REGN_CORRECTED_TOTAL) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_REGN_CORRECTED_UNIQUE(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _REGN_CORRECTED_UNIQUE) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_REGN_UNCORRECTED_TOTAL(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _REGN_UNCORRECTED_TOTAL) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_REGN_UNCORRECTED_UNIQUE(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _REGN_UNCORRECTED_UNIQUE) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_ADDR_ADDR(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _ADDR_ADDR) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_ADDR_UNCORRECTED_COUNTS(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _ADDR_UNCORRECTED_COUNTS) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_ADDR_CORRECTED_TOTAL(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _ADDR_CORRECTED_TOTAL) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_CNT_ADDR_CORRECTED_UNIQUE(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_CNT(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_COUNTS_TYPE, _ADDR_CORRECTED_UNIQUE) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_ERR_BUF(index) \ + ( \ + NV_MSGBOX_CMD(_GET_ECC_V5, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_ECC_V5_SEL, _ERR_BUFFER) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_ECC_V5_ERR_BUF_INDEX, (index)) \ + ) +#define NV_MSGBOX_ECC_V5_ERR_BUF_NUM_ENTRIES 16 + +#define NV_MSGBOX_CMD_ECC_V5_ERR_BUF_ERR_TYPE_META(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_ERR_BUF(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_ERR_BUF_TYPE, _ERR_TYPE_META) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_ERR_BUF_TIME_STAMP(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_ERR_BUF(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_ERR_BUF_TYPE, _TIME_STAMP) \ + ) + +#define NV_MSGBOX_CMD_ECC_V5_ERR_BUF_ADDR(index) \ + ( \ + NV_MSGBOX_CMD_ECC_V5_ERR_BUF(index) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG2_ECC_V5_ERR_BUF_TYPE, _ADDR) \ + ) + +#define NV_MSGBOX_CMD_SET_COPY_DATA(cmd) \ + ( \ + FLD_SET_DRF(_MSGBOX, _CMD, _COPY_DATA, _ON, (cmd)) \ + ) + +#define NV_MSGBOX_CMD_GPM_GET_METRIC(type, metric, partition) \ + ( \ + NV_MSGBOX_CMD(_GPU_PERFORMANCE_MONITORING, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_GPM_ACTION, type) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG1_GPM_METRIC, metric) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_GPM_PARTITION, partition) \ + ) + +#define NV_MSGBOX_CMD_GPM_SET_MULTIPLIER(multiplier) \ + ( \ + NV_MSGBOX_CMD(_GPU_PERFORMANCE_MONITORING, 0, 0) | \ + DRF_DEF(_MSGBOX, _CMD, _ARG1_GPM_ACTION, _SET_MULTIPLIER) | \ + DRF_NUM(_MSGBOX, _CMD, _ARG2_GPM_MULTIPLIER, (multiplier)) \ + ) + +#define NV_MSGBOX_GET_CMD_OPCODE(cmd) DRF_VAL(_MSGBOX, _CMD, _OPCODE, (cmd)) +#define NV_MSGBOX_GET_CMD_ARG1(cmd) DRF_VAL(_MSGBOX, _CMD, _ARG1 , (cmd)) +#define NV_MSGBOX_GET_CMD_ARG2(cmd) DRF_VAL(_MSGBOX, _CMD, _ARG2 , (cmd)) +#define NV_MSGBOX_GET_CMD_STATUS(cmd) DRF_VAL(_MSGBOX, _CMD, _STATUS, (cmd)) +#define NV_MSGBOX_CMD_IS_COPY_DATA_SET(cmd) \ + FLD_TEST_DRF(_MSGBOX, _CMD, _COPY_DATA, _ON, (cmd)) + +#define NV_MSGBOX_CAP_IS_AVAILABLE(caps, idx, name) \ + !(FLD_TEST_DRF(_MSGBOX, _DATA_CAP_##idx, name, _NOT_AVAILABLE, (caps)[idx])) + +#define NV_MSGBOX_CAP_SET_AVAILABLE(caps, idx, name) \ + (caps)[idx] = FLD_SET_DRF(_MSGBOX, _DATA_CAP_##idx, name, _AVAILABLE, \ + (caps)[idx]) + +/******************************************************************************** + * * + * GPU REQUESTS * + * * + ********************************************************************************/ + +// +// Encoding: +// 31 24 23 16 15 8 7 0 +// .-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. +// | REQUEST ID | ARG2 | ARG1 | OPCODE | DATA_OUT +// `-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_REQ_OPCODE 7:0 +#define NV_MSGBOX_REQ_OPCODE_CPU_PCONTROL 0x00000000 +#define NV_MSGBOX_REQ_OPCODE_SYS_PCONTROL 0x00000001 +#define NV_MSGBOX_REQ_ARG1 15:8 +#define NV_MSGBOX_REQ_ARG2 23:16 +#define NV_MSGBOX_REQ_REQUEST_ID 31:24 +#define NV_MSGBOX_REQ(opcode, arg1, arg2, requestId) \ + ( \ + DRF_NUM(_MSGBOX, _REQ, _OPCODE , (opcode)) | \ + DRF_NUM(_MSGBOX, _REQ, _ARG1 , (arg1)) | \ + DRF_NUM(_MSGBOX, _REQ, _ARG2 , (arg2)) | \ + DRF_NUM(_MSGBOX, _REQ, _REQUEST_ID, (requestId)) \ + ) + +/* ---------------- CPU Performance Control (PCONTROL) ------------------------ */ + +// +// CPU_PCONTROL +// +// Description: +// GPU-Request used to retrieve information and control various aspects of +// CPU performance. +// +// Common Encoding: +// 7 3 2 0 +// .-+-+-+-+-+-+-+-. +// | TARGET | ACT | ARG1 +// `-+-+-+-+-+-+-+-' +// +// Notes: +// arg1, arg2, and data encoding (fields are relative to zero, not relative +// to their position in the DATA_OUT register). +// +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_ACTION 2:0 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_ACTION_RSVD1 0x00000000 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_ACTION_RSVD2 0x00000001 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_ACTION_SET_LIMIT 0x00000002 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_TARGET 7:3 +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_TARGET_PSTATE 0x00000000 + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_SET_ACTION(arg1, action) \ + (arg1) = FLD_SET_DRF(_MSGBOX_REQ, _CPU_PCONTROL_ARG1, _ACTION, \ + action, (arg1)) + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_GET_ACTION(arg1) \ + DRF_VAL(_MSGBOX_REQ, _CPU_PCONTROL_ARG1, _ACTION, (arg1)) + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_SET_TARGET(arg1, target) \ + (arg1) = FLD_SET_DRF(_MSGBOX_REQ, _CPU_PCONTROL_ARG1, _TARGET, \ + target, (arg1)) + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG1_GET_TARGET(arg1) \ + DRF_VAL(_MSGBOX_REQ, _CPU_PCONTROL_ARG1, _TARGET, (arg1)) + +// +// Setting CPU Pstate Limits: +// +// Inputs: +// ARG1 - {TARGET=PSTATE, ACTION=SET_LIMIT} +// ARG2 - Stores desired CPU Pstate limit +// +// Outputs: +// none +// +// Encoding: +// 7 3 2 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | TARGET | ACT | ARG1 | VALUE | ARG2 +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG2_PSTATE_SET_LIMIT_VALUE 7:0 + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG2_PSTATE_SET_LIMIT_SET_VALUE(arg2, limit) \ + (arg2) = FLD_SET_DRF_NUM(_MSGBOX_REQ, _CPU_PCONTROL_ARG2, \ + _PSTATE_SET_LIMIT_VALUE, (limit), (arg2)) + +#define NV_MSGBOX_REQ_CPU_PCONTROL_ARG2_PSTATE_SET_LIMIT_GET_VALUE(arg2) \ + DRF_VAL(_MSGBOX_REQ, _CPU_PCONTROL_ARG2, _PSTATE_SET_LIMIT_VALUE, (arg2)) + +/* ----------------------- System Performance Control ------------------------- */ + +// +// SYS_PCONTROL +// +// Description: +// GPU-Request used to retrieve information and control various aspects of +// system-performance. +// +// Common Encoding: +// 7 3 2 0 7 0 +// .-+-+-+-+-+-+-+-. .-+-+-+-+-+-+-+-. +// | TARGET | ACT | ARG1 |0 0 0 0 0 0 0 0| ARG2 (reserved) +// `-+-+-+-+-+-+-+-' `-+-+-+-+-+-+-+-' +// +// Notes: +// - TARGET accepts NV0000_CTRL_SYSTEM_PARAM_* definitions +// - arg1, arg2, and data encoding (fields are relative to zero, not relative +// to their position in the DATA_OUT register). +// +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_ACTION 2:0 +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_ACTION_GET_STATUS 0x00000000 +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_ACTION_GET_LIMIT 0x00000001 +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_TARGET 7:3 +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG2_RSVD 7:0 + +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_SET_ACTION(arg1, action) \ + (arg1) = FLD_SET_DRF(_MSGBOX_REQ, _SYS_PCONTROL_ARG1, _ACTION, \ + action, (arg1)) + +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_GET_ACTION(arg1) \ + DRF_VAL(_MSGBOX_REQ, _SYS_PCONTROL_ARG1, _ACTION, (arg1)) + +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_SET_TARGET(arg1, target) \ + (arg1) = FLD_SET_DRF_NUM(_MSGBOX_REQ, _SYS_PCONTROL_ARG1, _TARGET, \ + (target), (arg1)) + +#define NV_MSGBOX_REQ_SYS_PCONTROL_ARG1_GET_TARGET(arg1) \ + DRF_VAL(_MSGBOX_REQ, _SYS_PCONTROL_ARG1, _TARGET, (arg1)) + +#endif /* _SMBPBI_H */ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi_impl.h b/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi_impl.h new file mode 100644 index 0000000..e7052c8 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi_impl.h @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef _SMBPBI_IMPL_H +#define _SMBPBI_IMPL_H +/*! + * This header file stores implementation dependent parameters of the SMBPBI server. + */ + +/*! + * Maximum number of individual requests in a bundle + */ +#define NV_MSGBOX_PARAM_MAX_BUNDLE_SIZE 4 + +/*! + * Maximum number of Result Disposition Rules + */ +#define NV_MSGBOX_PARAM_MAX_DISP_RULES 10 + +/*! + * Maximum length of the Driver Event Message text string is 80, including + * the terminating NUL character. + */ +#define NV_MSGBOX_MAX_DRIVER_EVENT_MSG_TXT_SIZE 80 + +#endif // _SMBPBI_IMPL_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi_priv.h b/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi_priv.h new file mode 100644 index 0000000..2055aeb --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/oob/smbpbi_priv.h @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2010-2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _SMBPBI_PRIV_H_ +#define _SMBPBI_PRIV_H_ + +/*! + * This file contains NVIDIA private defines for the SMBPBI + * interface. + */ + +#include "oob/smbpbi.h" + +// MSGBOX command word structure + +// +// Reserving opcodes above 0xC0 for internal/private functionationality. +// +// These opcodes should not be included in any documentation we release outside +// of NVIDIA! +// + +// +// Only for internal use (should not be written to the command register). Used +// for internal tracking when commands are redirected to the RM from the PMU. +// +#define NV_MSGBOX_CMD_ERR_MORE_PROCESSING_REQUIRED 0x000000F0 + +// +// Alternative encodings of the command word +// These are distinguished by a non-zero value in the 29:29 bit, +// previously known as _RSVD. +// +#define NV_MSGBOX_CMD_ENCODING 29:29 +#define NV_MSGBOX_CMD_ENCODING_STANDARD 0x00000000 +#define NV_MSGBOX_CMD_ENCODING_DEBUG 0x00000001 + +// Debug command structure +#define NV_MSGBOX_DEBUG_CMD_OPCODE 1:0 +#define NV_MSGBOX_DEBUG_CMD_OPCODE_READ_PRIV 0x00000000 + +#define NV_MSGBOX_DEBUG_CMD_ARG 23:2 + +/* Utility command constructor macros */ + +#define NV_MSGBOX_DEBUG_CMD(opcode, arg) \ + ( \ + DRF_DEF(_MSGBOX, _DEBUG_CMD, _OPCODE, opcode) | \ + DRF_NUM(_MSGBOX, _DEBUG_CMD, _ARG, (arg)) | \ + DRF_DEF(_MSGBOX, _CMD, _STATUS, _NULL) | \ + DRF_DEF(_MSGBOX, _CMD, _ENCODING, _DEBUG) | \ + DRF_DEF(_MSGBOX, _CMD, _INTR, _PENDING) \ + ) + +#define NV_MSGBOX_DEBUG_CMD_READ_PRIV(offset) \ + NV_MSGBOX_DEBUG_CMD(_READ_PRIV, (offset) >> 2) + +#endif // _SMBPBI_PRIV_H_ + diff --git a/userspace/sdks/nv525/nvalloc/common/inc/rmRiscvUcode.h b/userspace/sdks/nv525/nvalloc/common/inc/rmRiscvUcode.h new file mode 100644 index 0000000..22a4c88 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/rmRiscvUcode.h @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/* + * << DO NOT EDIT >> + * + * This file describes the format of generated ucode binary. Please do not change the + * content unless the same change is applied to the target ucode builds. + */ + +#ifndef RM_RISCV_UCODE_H +#define RM_RISCV_UCODE_H + +#include "nvtypes.h" + +typedef struct { + // + // Version 1 + // Version 2 + // Version 3 = for Partition boot + // Version 4 = for eb riscv boot + // Version 5 = Support signing entire RISC-V image as "code" in code section for hopper and later. + // + NvU32 version; // structure version + NvU32 bootloaderOffset; + NvU32 bootloaderSize; + NvU32 bootloaderParamOffset; + NvU32 bootloaderParamSize; + NvU32 riscvElfOffset; + NvU32 riscvElfSize; + NvU32 appVersion; // Changelist number associated with the image + // + // Manifest contains information about Monitor and it is + // input to BR + // + NvU32 manifestOffset; + NvU32 manifestSize; + // + // Monitor Data offset within RISCV image and size + // + NvU32 monitorDataOffset; + NvU32 monitorDataSize; + // + // Monitor Code offset withtin RISCV image and size + // + NvU32 monitorCodeOffset; + NvU32 monitorCodeSize; + NvU32 bIsMonitorEnabled; + // + // Swbrom Code offset within RISCV image and size + // + NvU32 swbromCodeOffset; + NvU32 swbromCodeSize; + // + // Swbrom Data offset within RISCV image and size + // + NvU32 swbromDataOffset; + NvU32 swbromDataSize; + // + // Total size of FB carveout (image and reserved space). + // + NvU32 fbReservedSize; + // + // Indicates whether the entire RISC-V image is signed as "code" in code section. + // + NvU32 bSignedAsCode; +} RM_RISCV_UCODE_DESC; + +#endif // RM_RISCV_UCODE_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/rmflcnbl.h b/userspace/sdks/nv525/nvalloc/common/inc/rmflcnbl.h new file mode 100644 index 0000000..7a49306 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/rmflcnbl.h @@ -0,0 +1,147 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2013-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * Data structures and interfaces used for generic falcon boot-loader. + * + * This generic boot-loader is designed to load both non-secure and + * secure code taking care of signature as well. This bootloader + * should be loaded at the end of the IMEM so that it doesnt overwrite + * itself when it tries to load the code into IMEM starting at blk 0. + * The data will be loaded into DMEM offset 0. + */ + +#ifndef RMFLCNBL_H +#define RMFLCNBL_H + +#include "flcnifcmn.h" + +/*! + * @brief Structure used by the boot-loader to load the rest of the code. + * + * This has to be filled by the GPU driver and copied into DMEM at offset + * provided in the @ref _def_rm_flcn_bl_desc.blDmemDescLoadOff. + */ +typedef struct _def_rm_flcn_bl_dmem_desc +{ + /*! + * reserved should be always first element + */ + NvU32 reserved[4]; + /*! + * signature should follow reserved 16B signature for secure code. + * 0s if no secure code + */ + NvU32 signature[4]; + /*! + * ctxDma is used by the bootloader while loading code/data. + */ + NvU32 ctxDma; + /*! + * 256B aligned physical FB address where code is located. + */ + RM_FLCN_U64 codeDmaBase; + /*! + * Offset from codeDmaBase where the nonSecure code is located. + * The offset must be multiple of 256 to help performance. + */ + NvU32 nonSecureCodeOff; + /*! + * The size of the nonSecure code part. + */ + NvU32 nonSecureCodeSize; + /*! + * Offset from codeDmaBase where the secure code is located. + * The offset must be multiple of 256 to help performance. + */ + NvU32 secureCodeOff; + /*! + * The size of the ecure code part. + */ + NvU32 secureCodeSize; + /*! + * Code entry point which will be invoked by BL after code is loaded. + */ + NvU32 codeEntryPoint; + /*! + * 256B aligned Physical FB Address where data is located. + */ + RM_FLCN_U64 dataDmaBase; + /*! + * Size of data block. Should be multiple of 256B. + */ + NvU32 dataSize; + /*! + * Arguments to be passed to the target firmware being loaded. + */ + NvU32 argc; + /*! + * Number of arguments to be passed to the target firmware being loaded. + */ + NvU32 argv; +} RM_FLCN_BL_DMEM_DESC, *PRM_FLCN_BL_DMEM_DESC; + +/*! + * @brief The header used by the GPU driver to figure out code and data + * sections of bootloader. + */ +typedef struct _def_rm_flcn_bl_img_header +{ + /*! + * Offset of code section in the image. + */ + NvU32 blCodeOffset; + /*! + * Size of code section in the image. + */ + NvU32 blCodeSize; + /*! + * Offset of data section in the image. + */ + NvU32 blDataOffset; + /*! + * Size of data section in the image. + */ + NvU32 blDataSize; +} RM_FLCN_BL_IMG_HEADER, *PRM_FLCN_BL_IMG_HEADER; + +/*! + * @brief The descriptor used by RM to figure out the requirements of boot loader. + */ +typedef struct _def_rm_flcn_bl_desc +{ + /*! + * Starting tag of bootloader + */ + NvU32 blStartTag; + /*! + * Dmem offset where _def_rm_flcn_bl_dmem_desc to be loaded + */ + NvU32 blDmemDescLoadOff; + /*! + * Description of the image + */ + RM_FLCN_BL_IMG_HEADER blImgHeader; +} RM_FLCN_BL_DESC, *PRM_FLCN_BL_DESC; + +#endif // RMFLCNBL_H diff --git a/userspace/sdks/nv525/nvalloc/common/inc/rmgspseq.h b/userspace/sdks/nv525/nvalloc/common/inc/rmgspseq.h new file mode 100644 index 0000000..cee7e65 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/rmgspseq.h @@ -0,0 +1,202 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * Top-level header-file that defines the GSP sequencer structures + */ + +#ifndef _RMGSPSEQ_H_ +#define _RMGSPSEQ_H_ + +/*! + * GSP sequencer structures and defines + * + * Here are the supported opcodes. + * the sequencer buffer is a stream of commands composed of + * an op-code directly followed by the exact number of arguments it needs. + * + * The supported op-codes are: + * GSP_SEQUENCER_BUFFER_OPCODE_REG_WRITE: + * arg[0]: the register offset + * arg[1]: the register value + * + * GSP_SEQUENCER_BUFFER_OPCODE_REG_MODIFY: + * arg[0]: the register offset + * arg[1]: the mask where to apply the modification + * arg[2]: the value to apply. The value needs to be shifted to fit inside the mask, + * + * GSP_SEQUENCER_BUFFER_OPCODE_REG_POLL: + * arg[0]: the register offset + * arg[1]: the mask where to apply the modification + * arg[2]: the value to apply. The value needs to be shifted to fit inside the mask. + * arg[3]: the timeout in MS + * arg[4]: an unique error code from GSP_SEQUENCER_BUFFER_ERR. Helps map to the failing GSP code. + * + * GSP_SEQUENCER_BUFFER_OPCODE_DELAY_US + * arg[0]: the delay in micoseconds. + * + * GSP_SEQUENCER_REG_STORE + * This operation stores the specified register at the specified index in + * the sequencer buffer register storage area. + * arg[0]: the reg offset to store in the save area + * arg[1]: index in save area to store value of reg offset + * + * GSP_SEQUENCER_CORE_RESET + * This operation resets the core. This operation takes no arguments. + * + * GSP_SEQUENCER_CORE_START + * This operation starts the core. This operation takes no arguments. + * + * GSP_SEQUENCER_CORE_WAIT_FOR_HALT + * This operation waits for the core to halt after completing execution. + * This operation takes no arguments. + * + * GSP_SEQUENCER_CORE_RESUME + * This operation resumes the core in preparation for switching back to + * the GSP-RM. + */ +typedef enum GSP_SEQ_BUF_OPCODE +{ + GSP_SEQ_BUF_OPCODE_REG_WRITE = 0, + GSP_SEQ_BUF_OPCODE_REG_MODIFY, + GSP_SEQ_BUF_OPCODE_REG_POLL, + GSP_SEQ_BUF_OPCODE_DELAY_US, + GSP_SEQ_BUF_OPCODE_REG_STORE, + GSP_SEQ_BUF_OPCODE_CORE_RESET, + GSP_SEQ_BUF_OPCODE_CORE_START, + GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT, + GSP_SEQ_BUF_OPCODE_CORE_RESUME, +} GSP_SEQ_BUF_OPCODE; + +#define GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(opcode) \ + ((opcode == GSP_SEQ_BUF_OPCODE_REG_WRITE) ? (sizeof(GSP_SEQ_BUF_PAYLOAD_REG_WRITE) / sizeof(NvU32)) : \ + (opcode == GSP_SEQ_BUF_OPCODE_REG_MODIFY) ? (sizeof(GSP_SEQ_BUF_PAYLOAD_REG_MODIFY) / sizeof(NvU32)) : \ + (opcode == GSP_SEQ_BUF_OPCODE_REG_POLL) ? (sizeof(GSP_SEQ_BUF_PAYLOAD_REG_POLL) / sizeof(NvU32)) : \ + (opcode == GSP_SEQ_BUF_OPCODE_DELAY_US) ? (sizeof(GSP_SEQ_BUF_PAYLOAD_DELAY_US) / sizeof(NvU32)) : \ + (opcode == GSP_SEQ_BUF_OPCODE_REG_STORE) ? (sizeof(GSP_SEQ_BUF_PAYLOAD_REG_STORE) / sizeof(NvU32)) : \ + /* GSP_SEQ_BUF_OPCODE_CORE_RESET */ \ + /* GSP_SEQ_BUF_OPCODE_CORE_START */ \ + /* GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT */ \ + /* GSP_SEQ_BUF_OPCODE_CORE_RESUME */ \ + 0) + +// The size of the structure must be DWord aligned! +typedef struct +{ + NvU32 addr; + NvU32 val; +} GSP_SEQ_BUF_PAYLOAD_REG_WRITE; + +// The size of the structure must be DWord aligned! +typedef struct +{ + NvU32 addr; + NvU32 mask; + NvU32 val; +} GSP_SEQ_BUF_PAYLOAD_REG_MODIFY; + +// The size of the structure must be DWord aligned! +typedef struct +{ + NvU32 addr; + NvU32 mask; + NvU32 val; + NvU32 timeout; + NvU32 error; +} GSP_SEQ_BUF_PAYLOAD_REG_POLL; + +// The size of the structure must be DWord aligned! +typedef struct +{ + NvU32 val; +} GSP_SEQ_BUF_PAYLOAD_DELAY_US; + +// The size of the structure must be DWord aligned! +typedef struct +{ + NvU32 addr; + NvU32 index; +} GSP_SEQ_BUF_PAYLOAD_REG_STORE; + +typedef struct GSP_SEQUENCER_BUFFER_CMD +{ + GSP_SEQ_BUF_OPCODE opCode; + union + { + GSP_SEQ_BUF_PAYLOAD_REG_WRITE regWrite; + GSP_SEQ_BUF_PAYLOAD_REG_MODIFY regModify; + GSP_SEQ_BUF_PAYLOAD_REG_POLL regPoll; + GSP_SEQ_BUF_PAYLOAD_DELAY_US delayUs; + GSP_SEQ_BUF_PAYLOAD_REG_STORE regStore; + } payload; +} GSP_SEQUENCER_BUFFER_CMD; + +// +// These error codes printed by the client help us +// see to which GSP uCode line it corresponds. +// +typedef enum GSP_SEQUENCER_BUFFER_ERR +{ + GSP_SEQUENCER_BUFFER_ERR_OK = 0, + GSP_SEQUENCER_BUFFER_ERR_TIMEOUT1, + GSP_SEQUENCER_BUFFER_ERR_TIMEOUT2, + GSP_SEQUENCER_BUFFER_ERR_TIMEOUT3 +} GSP_SEQUENCER_BUFFER_ERR; + + +// Sequencer implementation of FLD_WR_DRF_DEF() +#define GSP_SEQ_FLD_WR_DRF_DEF(gpu, gsp, d, r, f, c) \ +{ \ + GSP_SEQUENCER_BUFFER_CMD cmd; \ + cmd.opCode = GSP_SEQ_BUF_OPCODE_REG_MODIFY; \ + cmd.payload.regModify.addr = NV##d##r; \ + cmd.payload.regModify.mask = DRF_MASK(NV##d##r##f) << DRF_SHIFT(NV##d##r##f); \ + cmd.payload.regModify.val = DRF_DEF(d, r, f, c); \ + (void)gspAppendToSequencerBuffer(gpu, gsp, &cmd); \ +} + +/*! + * Forward references + */ +struct rpc_run_cpu_sequencer_v17_00; + +/*! + * Structure tracking all information relevant to GSP sequencer bufferfor GSP-RM + */ +typedef struct +{ + /*! Pointer to RM-GSP CPU sequencer parameter block */ + struct rpc_run_cpu_sequencer_v17_00 *pRunCpuSeqParam; +} GSP_SEQUENCER_BUFFER; + +/*! + * RM-GSP sequencer buffer register macros. + * GSP_SEQ_BUF_REG_SAVE_SIZE : specifies size of save area in reg values + * GSP_SEQ_BUF_REG_SAVE_MAILBOX0 : index for saving of mailbox0 reg + * GSP_SEQ_BUF_REG_SAVE_MAILBOX1 : index for saving of mailbox1 reg + */ +#define GSP_SEQ_BUF_REG_SAVE_SIZE (8) +#define GSP_SEQ_BUF_REG_SAVE_MAILBOX0 (0) +#define GSP_SEQ_BUF_REG_SAVE_MAILBOX1 (1) + +#endif // _RMGSPSEQ_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/rmifrif.h b/userspace/sdks/nv525/nvalloc/common/inc/rmifrif.h new file mode 100644 index 0000000..048fcf8 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/rmifrif.h @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2012 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _RMIFRIF_H_ +#define _RMIFRIF_H_ + +/*! + * @file rmifrif.h + * @brief Defines structures and interfaces common between RM and + * Init-From-Rom (IFR). + * + * For systems supporting GC6 that have on-board VBIOS ROMs, IFR is used + * to expedite several parts of GC6 exit in parallel with PEX init. + * + * After running devinit using a PMU ucode image loaded from the ROM itself, + * parts of RM stateLoad can be done using RM's ucode image. This is + * achieved by loading RM PMU ucode directly from FB. The primary difficulties + * are how to find RM's PMU ucode and how to bootstrap it. + * + * We use the simple approach of allocating a fixed buffer near the + * top of FB that contains the information required to bootstrap RM's PMU + * image. This buffer is called the RM_IFR_GC6_CTX. + * + * The buffer is allocated within RM's reserved memory space, directly before + * the VBIOS workspace (if any is present). Since the VBIOS workspace is + * always a multiple of 64K, RM enforces that the offset between top of memory + * and the end of the buffer is 64K. This way the IFR code can start + * from the top of memory and search downwards in 64K decrements. + * + * A small header is placed at the end of the buffer which contains a + * string signature identifying the buffer and other data needed to find the + * remaining context data. + * + * Top_Of-FB /---------------------\ <- + * | | \ + * | (VBIOS_Workspace) | | END_OFFSET + * | | / + * |---------------------| <- + | | \ + * | GSP FW (if present) | | pFbHalPvtInfo->gspFwSizeBytes + * | | / + * |---------------------| <- + * | RM_IFR_GC6_CTX_HDR | \ + * |---------------------| | + * | (Padding) | | RM_IFR_GC6_CTX_HDR.bufferSize + * |---------------------| | + * | Sequence Data | / + * |---------------------| <- + * | | + * | | + * | | + * | | + * 0x00000000 \---------------------/ + * + * To simplify the RM PMU bootstrap process and decrease IFR maintainence + * cost, the bootstrap process is encoded as a sequence script, leveraging + * a small subset of RM's PMU_SEQ_INST interface (see pmuseqinst.h). + * Register writes are captured during the initial (CPU-driven) RM PMU bootstrap + * and saved into a sequence for replay during GC6 exit. + * + * Only the following opcodes are supported currently: + * NV_PMU_SEQ_WRITE_REG_OPC - (multi-)register write + * NV_PMU_SEQ_EXIT_OPC - sequence done + * + */ + +/*! + * Header structure which identifies the GC6 context buffer. + */ +typedef struct +{ + NvU8 signature[12]; // RM_IFR_GC6_CTX_SIGNATURE + NvU32 bufferSize; // Size of the entire context buffer in bytes + NvU32 seqSizeWords; // Number of 32-bit words of sequence data + NvU32 checksum; // 32-bit chunk checksum of the sequence data +} RM_IFR_GC6_CTX_HDR, *PRM_IFR_GC6_CTX_HDR; + +/*! + * String signature that IFR searches for to find the GC6 context buffer. + */ +#define RM_IFR_GC6_CTX_SIGNATURE "GC6_CTX_HDR" // 12 bytes + +/*! + * Alignment of the offset between top of memory and the end of the + * GC6 context buffer (which is also the end of the header). + */ +#define RM_IFR_GC6_CTX_END_OFFSET_ALIGNMENT 0x10000 // 64KB + +/*! + * Maximum offset between top of memory and the end of the + * GC6 context buffer. This is meant to be a loose upper bound preventing + * scanning of the whole of memory (e.g. when something goes wrong). + */ +#define RM_IFR_GC6_CTX_END_OFFSET_MAX 0x1000000 // 16MB +#define RM_IFR_GC6_CTX_END_OFFSET_MAX_WITH_GSP 0x10000000 // 256MB + +/*! + * Maximum size of the context data in bytes. + * This is limited by FECS falcon DMEM size (4K on Kepler). + * The buffer must fit within DMEM together with stack and other global data. + */ +#define RM_IFR_GC6_CTX_DATA_MAX_SIZE 2048 // 2KB + +#endif // _RMIFRIF_H_ diff --git a/userspace/sdks/nv525/nvalloc/common/inc/rmlsfm.h b/userspace/sdks/nv525/nvalloc/common/inc/rmlsfm.h new file mode 100644 index 0000000..2e71260 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/rmlsfm.h @@ -0,0 +1,525 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2011-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef RMLSFM_H +#define RMLSFM_H + +/*****************************************************************************/ +/* This file is shared between ACR, SEC2 Binaries */ +/* Do not update this file without updating ACR/SEC2 */ +/*****************************************************************************/ + +/*! + * @file rmlsfm.h + * @brief Top-level header-file that defines Light Secure Falcon Managment + SW shared interfaces. + */ + +/*! + * READ/WRITE masks for WPR region + */ +#define LSF_WPR_REGION_RMASK (0xCU) // Readable only from level 2 and 3 client +#define LSF_WPR_REGION_WMASK (0xCU) // Writable only from level 2 and 3 client +#define LSF_WPR_REGION_RMASK_SUB_WPR_ENABLED (0x8) // Readable only from level 3 client +#define LSF_WPR_REGION_WMASK_SUB_WPR_ENABLED (0x8) // Writable only from level 3 client +#define LSF_WPR_REGION_ALLOW_READ_MISMATCH_NO (0x0) // Disallow read mis-match for all clients +#define LSF_WPR_REGION_ALLOW_WRITE_MISMATCH_NO (0x0) // Disallow write mis-match for all clients + +/*! + * READ mask for WPR region on Tegra + * This is required until we update tegra binaries, Bug 200281517 + * TODO: dgoyal - Remove this once tegra binaries are updated + */ +#define LSF_WPR_REGION_RMASK_FOR_TEGRA (0xFU) + +/*! + * Expected REGION ID to be used for the unprotected FB region (region that + * does not have read or write protections) + */ +#define LSF_UNPROTECTED_REGION_ID (0x0U) + +/*! + * Expected REGION ID to be used for the WPR region for the falcon microcode (includes data). + * ACR allots client requests to each region based on read/write masks and it is supposed + * to provide first priority to requests from LSFM. Providing first priority will naturally assign + * region ID 1 to LSFM and this define will provide a way for different parties to sanity check + * this fact. Also there are other falcons (FECS/video falcons) which depends on this define, so please + * be aware while modifying this. + */ +#define LSF_WPR_EXPECTED_REGION_ID (0x1U) + +/*! + * Expected REGION ID to be used for the unused WPR region. + */ +#define LSF_WPR_UNUSED_REGION_ID (0x2U) + +/*! + * Invalid LS falcon subWpr ID + */ +#define LSF_SUB_WPR_ID_INVALID (0xFFFFFFFFU) + +/*! + * Expected REGION ID to be used for the VPR region. + */ +#define LSF_VPR_REGION_ID (0x3U) + +/*! + * Size of the separate bootloader data that could be present in WPR region. + */ +#define LSF_LS_BLDATA_EXPECTED_SIZE (0x100U) + +/*! + * since we dont check signatures in GC6 exit, we need to hardcode the WPR offset + */ +#define LSF_WPR_EXPECTED_OFFSET (0x0U) + +/*! + * CTXDMA to be used while loading code/data in target falcons + */ +#define LSF_BOOTSTRAP_CTX_DMA_FECS (0x0) + +/*! + * Context DMA ID 6 is reserved for Video UCODE + */ +#define LSF_BOOTSTRAP_CTX_DMA_VIDEO (0x6) +#define LSF_BOOTSTRAP_CTX_DMA_BOOTSTRAP_OWNER (0x0) +#define LSF_BOOTSTRAP_CTX_DMA_FBFLCN (0x0) + +/*! + * Falcon Id Defines + * Defines a common Light Secure Falcon identifier. + * Codesigning infra. assumes LSF_FALCON_ID_ prefix for units, + * Changes to the define needs to be reflected in path [1] + * For new Falcon Id adding, we need to append to the end; + * don't insert the new falcon Id in the middle. + */ +#define LSF_FALCON_ID_PMU (0U) +#define LSF_FALCON_ID_DPU (1U) +#define LSF_FALCON_ID_GSPLITE LSF_FALCON_ID_DPU +#define LSF_FALCON_ID_FECS (2U) +#define LSF_FALCON_ID_GPCCS (3U) +#define LSF_FALCON_ID_NVDEC (4U) +#define LSF_FALCON_ID_NVENC (5U) +#define LSF_FALCON_ID_NVENC0 (5U) +#define LSF_FALCON_ID_NVENC1 (6U) +#define LSF_FALCON_ID_SEC2 (7U) +#define LSF_FALCON_ID_NVENC2 (8U) +#define LSF_FALCON_ID_MINION (9U) +#define LSF_FALCON_ID_FBFALCON (10U) +#define LSF_FALCON_ID_XUSB (11U) +#define LSF_FALCON_ID_GSP_RISCV (12U) +#define LSF_FALCON_ID_PMU_RISCV (13U) +#define LSF_FALCON_ID_SOE (14U) +#define LSF_FALCON_ID_NVDEC1 (15U) +#define LSF_FALCON_ID_OFA (16U) +#define LSF_FALCON_ID_SEC2_RISCV (17U) +#define LSF_FALCON_ID_NVDEC_RISCV (18U) +#define LSF_FALCON_ID_NVDEC_RISCV_EB (19U) +#define LSF_FALCON_ID_NVJPG (20U) +#define LSF_FALCON_ID_END (21U) + +#define LSF_FALCON_ID_INVALID (0xFFFFFFFFU) + +// +// ************************ NOTIFICATION ********************************* +// In case anyone needs to add new LSF falconId, please must calculate +// WPR header size per LSF_FALCON_ID_END. RM needs to call lsfmGetWprHeaderSizeMax_HAL +// to align with acrReadSubWprHeader_HAL in ACR. Otherwise, ACR can't get correct +// address to read sub wpr headers. +// We observer in case LSF_FALCON_ID_END > 32 will cause SEC2 IMEM tag missing error; +// but don't get the root cause yet. +// +#define LSF_FALCON_ID_END_15 (15U) +#define LSF_FALCON_ID_END_17 (17U) +#define LSF_FALCON_ID_END_18 (18U) +#define LSF_FALCON_ID_END_21 (21U) + +#define LSF_FALCON_INSTANCE_DEFAULT_0 (0x0) +#define LSF_FALCON_INSTANCE_COUNT_DEFAULT_1 (0x1) + +// Currently max supported instance is 8 for FECS/GPCCS SMC +#define LSF_FALCON_INSTANCE_FECS_GPCCS_MAX (0x8) +#define LSF_FALCON_INSTANCE_INVALID (0xFFFFFFFFU) +#define LSF_FALCON_INDEX_MASK_DEFAULT_0 (0x0) + + + +/*! + * Size in entries of the ucode descriptor's dependency map. + * This used to be LSF_FALCON_ID_END, but since that had to grow and we did not want to break any + * existing binaries, they had to be split. + * + * Increasing this number should be done with care. + */ +#define LSF_FALCON_DEPMAP_SIZE (11) + +/*! + * Falcon Binaries version defines + */ +#define LSF_FALCON_BIN_VERSION_INVALID (0xFFFFFFFFU) + + +/*! + * Light Secure Falcon Ucode Description Defines + * This stucture is prelim and may change as the ucode signing flow evolves. + */ +typedef struct +{ + NvU8 prdKeys[2][16]; + NvU8 dbgKeys[2][16]; + NvU32 bPrdPresent; + NvU32 bDbgPresent; + NvU32 falconId; + NvU32 bSupportsVersioning; + NvU32 version; + NvU32 depMapCount; + NvU8 depMap[LSF_FALCON_DEPMAP_SIZE * 2 * 4]; + NvU8 kdf[16]; +} LSF_UCODE_DESC, *PLSF_UCODE_DESC; + +/*! + * Light Secure WPR Header + * Defines state allowing Light Secure Falcon bootstrapping. + * + * falconId - LS falcon ID + * lsbOffset - Offset into WPR region holding LSB header + * bootstrapOwner - Bootstrap OWNER (either PMU or SEC2) + * bLazyBootstrap - Skip bootstrapping by ACR + * status - Bootstrapping status + */ +typedef struct +{ + NvU32 falconId; + NvU32 lsbOffset; + NvU32 bootstrapOwner; + NvU32 bLazyBootstrap; + NvU32 binVersion; + NvU32 status; +} LSF_WPR_HEADER, *PLSF_WPR_HEADER; + +/*! + * LSF shared SubWpr Header + * + * useCaseId - Shared SubWpr se case ID (updated by RM) + * startAddr - start address of subWpr (updated by RM) + * size4K - size of subWpr in 4K (updated by RM) + */ +typedef struct +{ + NvU32 useCaseId; + NvU32 startAddr; + NvU32 size4K; +} LSF_SHARED_SUB_WPR_HEADER, *PLSF_SHARED_SUB_WPR_HEADER; + +// Shared SubWpr use case IDs +typedef enum +{ + LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_FRTS_VBIOS_TABLES = 1, + LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_PLAYREADY_SHARED_DATA = 2, + LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_APM_RTS = 3 +} LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_ENUM; + +#define LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_APM_RTS +#define LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_INVALID (0xFFFFFFFFU) + +#define MAX_SUPPORTED_SHARED_SUB_WPR_USE_CASES LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX + +// +// Static sizes of shared subWPRs +// Minimum granularity supported is 4K +// +#define LSF_SHARED_DATA_SUB_WPR_FRTS_VBIOS_TABLES_SIZE_IN_4K (0x100) // 1MB in 4K +#define LSF_SHARED_DATA_SUB_WPR_PLAYREADY_SHARED_DATA_SIZE_IN_4K (0x1) // 4K +#define LSF_SHARED_DATA_SUB_WPR_APM_RTS_SIZE_IN_4K (0x1) // 4K + +/*! + * Bootstrap Owner Defines + */ +#define LSF_BOOTSTRAP_OWNER_PMU (LSF_FALCON_ID_PMU) +#define LSF_BOOTSTRAP_OWNER_SEC2 (LSF_FALCON_ID_SEC2) +#define LSF_BOOTSTRAP_OWNER_GSPLITE (LSF_FALCON_ID_GSPLITE) +#define LSF_BOOTSTRAP_OWNER_DEFAULT LSF_BOOTSTRAP_OWNER_PMU + +/*! + * Image Status Defines + */ +#define LSF_IMAGE_STATUS_NONE (0U) +#define LSF_IMAGE_STATUS_COPY (1U) +#define LSF_IMAGE_STATUS_VALIDATION_CODE_FAILED (2U) +#define LSF_IMAGE_STATUS_VALIDATION_DATA_FAILED (3U) +#define LSF_IMAGE_STATUS_VALIDATION_DONE (4U) +#define LSF_IMAGE_STATUS_VALIDATION_SKIPPED (5U) +#define LSF_IMAGE_STATUS_BOOTSTRAP_READY (6U) +#define LSF_IMAGE_STATUS_REVOCATION_CHECK_FAILED (7U) + +/*! + * Light Secure Bootstrap Header + * Defines state allowing Light Secure Falcon bootstrapping. + * + * signature - Code/data signature details for this LS falcon + * ucodeOffset - Offset into WPR region where UCODE is located + * ucodeSize - Size of ucode + * dataSize - Size of ucode data + * blCodeSize - Size of bootloader that needs to be loaded by bootstrap owner + * blImemOffset - BL starting virtual address. Need for tagging. + * blDataOffset - Offset into WPR region holding the BL data + * blDataSize - Size of BL data + * appCodeOffset - Offset into WPR region where Application UCODE is located + * appCodeSize - Size of Application UCODE + * appDataOffset - Offset into WPR region where Application DATA is located + * appDataSize - Size of Application DATA + * blLoadCodeAt0 - Load BL at 0th IMEM offset + * bSetVACtx - Make sure to set the code/data loading CTX DMA to be virtual before exiting + * bDmaReqCtx - This falcon requires a ctx before issuing DMAs + * bForcePrivLoad- Use priv loading method instead of bootloader/DMAs + */ + +#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0 0:0 +#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0_FALSE 0 +#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0_TRUE 1 +#define NV_FLCN_ACR_LSF_FLAG_SET_VA_CTX 1:1 +#define NV_FLCN_ACR_LSF_FLAG_SET_VA_CTX_FALSE 0 +#define NV_FLCN_ACR_LSF_FLAG_SET_VA_CTX_TRUE 1 +#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX 2:2 +#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_FALSE 0 +#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_TRUE 1 +#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD 3:3 +#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_FALSE 0 +#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_TRUE 1 +typedef struct +{ + LSF_UCODE_DESC signature; + NvU32 ucodeOffset; + NvU32 ucodeSize; + NvU32 dataSize; + NvU32 blCodeSize; + NvU32 blImemOffset; + NvU32 blDataOffset; + NvU32 blDataSize; + NvU32 appCodeOffset; + NvU32 appCodeSize; + NvU32 appDataOffset; + NvU32 appDataSize; + NvU32 flags; + /* + * TODO: Uncomment this once Sanket's changes + * of HALifying is done + NvU32 monitorCodeOffset; + NvU32 monitorDataOffset; + NvU32 manifestOffset; + */ +} LSF_LSB_HEADER, *PLSF_LSB_HEADER; + + +/*! + * Light Secure WPR Content Alignments + */ +#define LSF_WPR_HEADER_ALIGNMENT (256U) +#define LSF_SUB_WPR_HEADER_ALIGNMENT (256U) +#define LSF_LSB_HEADER_ALIGNMENT (256U) +#define LSF_BL_DATA_ALIGNMENT (256U) +#define LSF_BL_DATA_SIZE_ALIGNMENT (256U) +#define LSF_BL_CODE_SIZE_ALIGNMENT (256U) +#define LSF_DATA_SIZE_ALIGNMENT (256U) +#define LSF_CODE_SIZE_ALIGNMENT (256U) + +// MMU excepts subWpr sizes in units of 4K +#define SUB_WPR_SIZE_ALIGNMENT (4096U) + +/*! + * Maximum WPR Header size + */ +#define LSF_WPR_HEADERS_TOTAL_SIZE_MAX (NV_ALIGN_UP((sizeof(LSF_WPR_HEADER) * LSF_FALCON_ID_END), LSF_WPR_HEADER_ALIGNMENT)) +#define LSF_LSB_HEADER_TOTAL_SIZE_MAX (NV_ALIGN_UP(sizeof(LSF_LSB_HEADER), LSF_LSB_HEADER_ALIGNMENT)) + +// +// PMU OBJACR_ALIGNED_256 size will vary with LSF_FALCON_ID_END. +// PMU could run out of DMEM in case we increase LSF_FALCON_ID_END more and more. +// The PMU supports the ACR task on GM20X_thru_VOLTA profiles only. +// In order to prevent LSF_FALCON_ID_END changes to affect older / shipped PMU ucodes (increase of DMEM footprint) +// adding PMU specific ***_END define capturing value covering all PMU profiles that this with the ACR task. +// +#define LSF_FALCON_ID_END_PMU (LSF_FALCON_ID_FBFALCON + 1) +#define LSF_WPR_HEADERS_TOTAL_SIZE_MAX_PMU (NV_ALIGN_UP((sizeof(LSF_WPR_HEADER) * LSF_FALCON_ID_END_PMU), LSF_WPR_HEADER_ALIGNMENT)) + +// Maximum SUB WPR header size +#define LSF_SUB_WPR_HEADERS_TOTAL_SIZE_MAX (NV_ALIGN_UP((sizeof(LSF_SHARED_SUB_WPR_HEADER) * LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX), LSF_SUB_WPR_HEADER_ALIGNMENT)) + +/*! + * For the ucode surface alignment, We align to RM_PAGE_SIZE because of + * busMapRmAperture issues, not because of Falcon ucode alignment requirements + * which currently are that it be at least 256. + */ +#define LSF_UCODE_DATA_ALIGNMENT RM_PAGE_SIZE + +/*! + * ACR Descriptors used by ACR UC + */ + +/*! + * Supporting maximum of 2 regions. + * This is needed to pre-allocate space in DMEM + */ +#define RM_FLCN_ACR_MAX_REGIONS (2) +#define LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE (0x200) + +/*! + * startAddress - Starting address of region + * endAddress - Ending address of region + * regionID - Region ID + * readMask - Read Mask + * writeMask - WriteMask + * clientMask - Bit map of all clients currently using this region + * shadowMemStartAddress- FB location from where contents need to be copied to startAddress + */ +typedef struct _def_acr_dmem_region_prop +{ + NvU32 startAddress; + NvU32 endAddress; + NvU32 regionID; + NvU32 readMask; + NvU32 writeMask; + NvU32 clientMask; + NvU32 shadowMemStartAddress; +} RM_FLCN_ACR_REGION_PROP, *PRM_FLCN_ACR_REGION_PROP; + + +/*! + * noOfRegions - Number of regions used by RM. + * regionProps - Region properties + */ +typedef struct _def_acr_regions +{ + NvU32 noOfRegions; + RM_FLCN_ACR_REGION_PROP regionProps[RM_FLCN_ACR_MAX_REGIONS]; +} RM_FLCN_ACR_REGIONS, *PRM_FLCN_ACR_REGIONS; + +/*! + * bVprEnabled : When set, ACR_LOCKDOWN phase programs VPR range. Needs to be + : NvU32 because of alignment + * vprStartAddress : Start address of VPR region. SEC2 binary updates this value + * vprEndAddress : End address of VPR region. SEC2 binary updates this value + * hdcpPolicies : VPR display policies. SEC2 binary updates this value + */ +typedef struct _def_acr_vpr_dmem_desc +{ + NvU32 bVprEnabled; + NvU32 vprStartAddress; + NvU32 vprEndAddress; + NvU32 hdcpPolicies; +} ACR_BSI_VPR_DESC, *PACR_BSI_VPR_DESC; + +/*! + * reservedDmem - When the bootstrap owner has done bootstrapping other falcons, + * and need to switch into LS mode, it needs to have its own actual + * DMEM image copied into DMEM as part of LS setup. If ACR desc is at + * location 0, it will definitely get overwritten causing data corruption. + * Hence we are reserving 0x200 bytes to give room for any loading data. + * NOTE: This has to be the first member always + * signature - Signature of ACR ucode. + * wprRegionID - Region ID holding the WPR header and its details + * wprOffset - Offset from the WPR region holding the wpr header + * regions - Region descriptors + * ucodeBlobBase- Used for Tegra, stores non-WPR start address where kernel stores ucode blob + * ucodeBlobSize- Used for Tegra, stores the size of the ucode blob + */ +typedef struct _def_acr_dmem_desc +{ + NvU32 signatures[4]; + NvU32 wprRegionID; + NvU32 wprOffset; + NvU32 mmuMemoryRange; + RM_FLCN_ACR_REGIONS regions; + NvU32 ucodeBlobSize; + // uCodeBlobBase is moved after ucodeBlobSize to inherently align to qword (8 bytes) + NvU64 NV_DECLARE_ALIGNED(ucodeBlobBase, 8); + + /*! + * Do not change the offset of this descriptor as it shared between + * ACR_REGION_LOCKDOWN HS binary and SEC2. Any change in this structure + * need recompilation of SEC2 and ACR_LOCKDOWN HS binary + */ + ACR_BSI_VPR_DESC vprDesc; +} RM_FLCN_ACR_DESC, *PRM_FLCN_ACR_DESC; + +/*! +* Hub keys/nonce Structure in BSI +*/ +#define MAX_SFBHUB_ENCRYPTION_REGION_KEY_SIZE 4 + +typedef struct _def_acr_hub_scratch_data +{ + NvU32 key[MAX_SFBHUB_ENCRYPTION_REGION_KEY_SIZE]; + NvU32 nonce[MAX_SFBHUB_ENCRYPTION_REGION_KEY_SIZE]; +} ACR_BSI_HUB_DESC, *PACR_BSI_HUB_DESC; + +#define MAX_HUB_ENCRYPTION_REGION_COUNT 3 +typedef struct _def_acr_hub_scratch_array +{ + ACR_BSI_HUB_DESC entries[MAX_HUB_ENCRYPTION_REGION_COUNT]; +} ACR_BSI_HUB_DESC_ARRAY, *PACR_BSI_HUB_DESC_ARRAY; + +typedef struct _def_acr_reserved_dmem +{ + NvU32 reservedDmem[(LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE/4)]; // Always first.. +} ACR_RESERVED_DMEM, *PACR_RESERVED_DMEM; + +typedef struct _def_booter_reserved_dmem +{ + NvU32 reservedDmem[(LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE/4)]; // Always first.. +} BOOTER_RESERVED_DMEM; + +#define NV_FLCN_ACR_DESC_FLAGS_SIG_VERIF 0:0 +#define NV_FLCN_ACR_DESC_FLAGS_SIG_VERIF_DISABLE 0 +#define NV_FLCN_ACR_DESC_FLAGS_SIG_VERIF_ENABLE 1 + +// Macro defines to be consumed by RM to get GH100 GSP Inst_in_sys FMC boot status. +#define GSP_INST_IN_SYS_COMPLETION_STATUS_OK 0x55 +#define GSP_INST_IN_SYS_COMPLETION_STATUS_ERROR 0xAA +#define GSP_INST_IN_SYS_COMPLETION_STATUS_IN_PROGRESS 0x00 + +/*! + * Size of ACR phase in dword + */ +#define ACR_PHASE_SIZE_DWORD sizeof(RM_FLCN_ACR_DESC)/sizeof(NvU32) + +/*! + * Falcon Mode Tokens + * This is the value logged to a mailbox register to indicate that the + * falcon isn't booted in secure mode. + */ +#define LSF_FALCON_MODE_TOKEN_FLCN_INSECURE (0xDEADDEADU) + + +// LS encryption context, to store data to decrypt LS images. +#define LS_ENCRYPTION_AES128_CBC_IV_SIZE_BYTE (16) + +typedef struct +{ + NvU8 bLsEncrypted; + NvU8 rsvd[3]; + NvU8 lsEncIV[LS_ENCRYPTION_AES128_CBC_IV_SIZE_BYTE]; +} LSF_ENCRYPTION_CONTEXT, *PLSF_ENCRYPTION_CONTEXT; + +#endif // RMLSFM_H + diff --git a/userspace/sdks/nv525/nvalloc/common/inc/rmpbicmdif.h b/userspace/sdks/nv525/nvalloc/common/inc/rmpbicmdif.h new file mode 100644 index 0000000..0efac2f --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/common/inc/rmpbicmdif.h @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2009-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * Shared postbox interface defines for RM and PMU. + */ + +#ifndef RMPBICMDIF_H +#define RMPBICMDIF_H + +/* + * Define the status of postbox interface at different instances. + * + * The values should be in accordance to the spec and must not be changed. + * A new PBI command must be submitted with the status NV_PBI_COMMAND_STATUS_UNDEFINED. + */ + +#define NV_PBI_COMMAND_STATUS 7:0 +#define NV_PBI_COMMAND_STATUS_UNDEFINED 0x00 // command submitted to PMU +#define NV_PBI_COMMAND_STATUS_SUCCESS 0x01 // command successfully completed by PMU +#define NV_PBI_COMMAND_STATUS_PENDING 0x02 // command accepted by PMU +#define NV_PBI_COMMAND_STATUS_BUSY 0x03 // command processing in PMU +#define NV_PBI_COMMAND_STATUS_UNSPECIFIED_FAILURE 0x04 // unknown failure or hang +#define NV_PBI_COMMAND_STATUS_INVALID_ADDRESS 0x05 // invalid address submitted to PMU +#define NV_PBI_COMMAND_STATUS_MORE_DATA 0x06 // user needs to send more data to PMU +#define NV_PBI_COMMAND_STATUS_INVALID_COMMAND 0x07 // invalid command submitted + +/* + * This corresponds to reserved bits of command register + */ +#define NV_PBI_COMMAND_RSVD_0 15:8 +#define NV_PBI_COMMAND_RSVD_0_VAL 0x00 + +/* + * This specifies the dword index if client is sending multiple dwords + * for single Dword this should be zero + */ +#define NV_PBI_COMMAND_BUFFER_INDEX 19:16 + +/* + * This specifies the total no. of dwords passed by client + */ +#define NV_PBI_COMMAND_BUFFER_SIZE 23:20 + +/* + * These corresponds the different function ID's supported out of PBI + */ +#define NV_PBI_COMMAND_FUNC_ID 27:24 +#define NV_PBI_COMMAND_FUNC_ID_GET_CAPABILITIES (0) +#define NV_PBI_COMMAND_FUNC_ID_EXECUTE_ROUTINE (11) + +// +// Definitions for common 'Execute Routine' calls that are +// shared across all GPUs. +// +// Common routine IDs must be in the range 0x80 to 0xff. +// +// Get GID: +#define NV_PBI_EXECUTE_ROUTINE_GET_GID 0x80 +// Get Feature: +#define NV_PBI_EXECUTE_ROUTINE_GET_FEATURE 0x81 +#define NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION 5:0 +#define NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION_ALLOWED 0x2 + +/* + * This corresponds to reserved field of command register + */ +#define NV_PBI_COMMAND_RSVD_1 28:28 +#define NV_PBI_COMMAND_RSVD_1_VAL 0x00 + +/* + * If this bit is set system will be notified on command completion + */ +#define NV_PBI_COMMAND_SYS_NOTIFY 29:29 +#define NV_PBI_COMMAND_SYS_NOTIFY_TRUE 0x01 +#define NV_PBI_COMMAND_SYS_NOTIFY_FALSE 0x00 + +/* + * If this bit is set driver will be notified of command completion status + */ +#define NV_PBI_COMMAND_DRV_NOTIFY 30:30 +#define NV_PBI_COMMAND_DRV_NOTIFY_TRUE 0x01 +#define NV_PBI_COMMAND_DRV_NOTIFY_FALSE 0x00 + +/* + * Defines the interrupt state of the PBI command + */ +#define NV_PBI_COMMAND_INTERRUPT 31:31 +#define NV_PBI_COMMAND_INTERRUPT_TRUE 0x01 +#define NV_PBI_COMMAND_INTERRUPT_FALSE 0x00 + +/* + * This sets the different fields of command register + */ +#define PBI_SET_COMMAND_PARAMS(status, r0, index, sz, cmd, r1, sys, \ + drv, intr, val) \ +{ \ + val = DRF_NUM(_PBI, _COMMAND, _STATUS, status) | \ + DRF_NUM(_PBI, _COMMAND, _RSVD_0, r0) | \ + DRF_NUM(_PBI, _COMMAND, _BUFFER_INDEX, index) | \ + DRF_NUM(_PBI, _COMMAND, _BUFFER_SIZE, sz) | \ + DRF_NUM(_PBI, _COMMAND, _FUNC_ID, cmd) | \ + DRF_NUM(_PBI, _COMMAND, _RSVD_1, r1) | \ + DRF_NUM(_PBI, _COMMAND, _SYS_NOTIFY, sys) | \ + DRF_NUM(_PBI, _COMMAND, _DRV_NOTIFY, drv) | \ + DRF_NUM(_PBI, _COMMAND, _INTERRUPT, intr); \ +} + +/* + * postbox_clients_mutex_id + * + * Define the Mutex ID for different PBI clients + */ +enum postbox_clients_mutex_id +{ + PBI_CLIENT_NONE = 0x00, + PBI_CLIENT_DRIVER_PCIPBI_SHIM = 0x15 +}; + +#endif /* RMPBICMDIF_H */ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/efi-console.h b/userspace/sdks/nv525/nvalloc/unix/include/efi-console.h new file mode 100644 index 0000000..7971565 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/efi-console.h @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef EFI_CONSOLE_H +#define EFI_CONSOLE_H + +#include "gpu/disp/kern_disp_max.h" + +struct OBJGPU; + +typedef struct +{ + NvBool isDispStateSave; + + NvU32 activeDisplayId[OBJ_MAX_HEADS]; + + struct + { + NvU32 displayId; + + struct { + NvU32 index; + NvU32 subLinkMask; + } sorXBar; + + struct { + NvU32 linkBw; + NvU32 laneCount; + NvU32 linkCtl; + } displayPort; + } activeDfpState[OBJ_MAX_DFPS]; + NvU32 numDfps; + + struct + { + NvU32 coreChannelClass; + NvU32 cacheSize; + NvU32 *pCache; + } display; +} nv_efi_t; + +void RmSaveEFIDisplayState (OBJGPU *pGpu); +void RmRestoreEFIDisplayState (OBJGPU *pGpu); + +#endif // EFI_CONSOLE_H diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-caps.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-caps.h new file mode 100644 index 0000000..35bbf7c --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-caps.h @@ -0,0 +1,94 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_CAPS_H_ +#define _NV_CAPS_H_ + +#include <nv-kernel-interface-api.h> + +/* + * Opaque OS-specific struct; on Linux, this has member + * 'struct proc_dir_entry'. + */ +typedef struct nv_cap nv_cap_t; + +/* + * Creates directory named "capabilities" under the provided path. + * + * @param[in] path Absolute path + * + * Returns a valid nv_cap_t upon success. Otherwise, returns NULL. + */ +nv_cap_t* NV_API_CALL nv_cap_init(const char *path); + +/* + * Creates capability directory entry + * + * @param[in] parent_cap Parent capability directory + * @param[in] name Capability directory's name + * @param[in] mode Capability directory's access mode + * + * Returns a valid nv_cap_t upon success. Otherwise, returns NULL. + */ +nv_cap_t* NV_API_CALL nv_cap_create_dir_entry(nv_cap_t *parent_cap, const char *name, int mode); + +/* + * Creates capability file entry + * + * @param[in] parent_cap Parent capability directory + * @param[in] name Capability file's name + * @param[in] mode Capability file's access mode + * + * Returns a valid nv_cap_t upon success. Otherwise, returns NULL. + */ +nv_cap_t* NV_API_CALL nv_cap_create_file_entry(nv_cap_t *parent_cap, const char *name, int mode); + +/* + * Destroys capability entry + * + * @param[in] cap Capability entry + */ +void NV_API_CALL nv_cap_destroy_entry(nv_cap_t *cap); + +/* + * Validates and duplicates the provided file descriptor + * + * @param[in] cap Capability entry + * @param[in] fd File descriptor to be validated + * + * Returns duplicate fd upon success. Otherwise, returns -1. + */ +int NV_API_CALL nv_cap_validate_and_dup_fd(const nv_cap_t *cap, int fd); + +/* + * Closes file descriptor + * + * This function should be used to close duplicate file descriptors + * returned by nv_cap_validate_and_dup_fd. + * + * @param[in] fd File descriptor to be validated + * + */ +void NV_API_CALL nv_cap_close_fd(int fd); + +#endif /* _NV_CAPS_H_ */ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-gpu-info.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-gpu-info.h new file mode 100644 index 0000000..a8c0c0a --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-gpu-info.h @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2015 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_GPU_INFO_H_ +#define _NV_GPU_INFO_H_ + +typedef struct { + NvU32 gpu_id; + + struct { + NvU32 domain; + NvU8 bus, slot, function; + } pci_info; + + /* + * opaque OS-specific pointer; on Linux, this is a pointer to the + * 'struct device' for the GPU. + */ + void *os_device_ptr; +} nv_gpu_info_t; + +#define NV_MAX_GPUS 32 + +#endif /* _NV_GPU_INFO_H_ */ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-ioctl-numbers.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-ioctl-numbers.h new file mode 100644 index 0000000..cb0b6a2 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-ioctl-numbers.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef NV_IOCTL_NUMBERS_H +#define NV_IOCTL_NUMBERS_H + +/* NOTE: using an ioctl() number > 55 will overflow! */ +#define NV_IOCTL_MAGIC 'F' +#define NV_IOCTL_BASE 200 +#define NV_ESC_CARD_INFO (NV_IOCTL_BASE + 0) +#define NV_ESC_REGISTER_FD (NV_IOCTL_BASE + 1) +#define NV_ESC_ALLOC_OS_EVENT (NV_IOCTL_BASE + 6) +#define NV_ESC_FREE_OS_EVENT (NV_IOCTL_BASE + 7) +#define NV_ESC_STATUS_CODE (NV_IOCTL_BASE + 9) +#define NV_ESC_CHECK_VERSION_STR (NV_IOCTL_BASE + 10) +#define NV_ESC_IOCTL_XFER_CMD (NV_IOCTL_BASE + 11) +#define NV_ESC_ATTACH_GPUS_TO_FD (NV_IOCTL_BASE + 12) +#define NV_ESC_QUERY_DEVICE_INTR (NV_IOCTL_BASE + 13) +#define NV_ESC_SYS_PARAMS (NV_IOCTL_BASE + 14) +#define NV_ESC_EXPORT_TO_DMABUF_FD (NV_IOCTL_BASE + 17) + +#endif diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-ioctl.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-ioctl.h new file mode 100644 index 0000000..ffd1dee --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-ioctl.h @@ -0,0 +1,145 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef NV_IOCTL_H +#define NV_IOCTL_H + +#include <nv-ioctl-numbers.h> +#include <nvtypes.h> + +typedef struct { + NvU32 domain; /* PCI domain number */ + NvU8 bus; /* PCI bus number */ + NvU8 slot; /* PCI slot number */ + NvU8 function; /* PCI function number */ + NvU16 vendor_id; /* PCI vendor ID */ + NvU16 device_id; /* PCI device ID */ +} nv_pci_info_t; + +/* + * ioctl()'s with parameter structures too large for the + * _IOC cmd layout use the nv_ioctl_xfer_t structure + * and the NV_ESC_IOCTL_XFER_CMD ioctl() to pass the actual + * size and user argument pointer into the RM, which + * will then copy it to/from kernel space in separate steps. + */ +typedef struct nv_ioctl_xfer +{ + NvU32 cmd; + NvU32 size; + NvP64 ptr NV_ALIGN_BYTES(8); +} nv_ioctl_xfer_t; + +typedef struct nv_ioctl_card_info +{ + NvBool valid; + nv_pci_info_t pci_info; /* PCI config information */ + NvU32 gpu_id; + NvU16 interrupt_line; + NvU64 reg_address NV_ALIGN_BYTES(8); + NvU64 reg_size NV_ALIGN_BYTES(8); + NvU64 fb_address NV_ALIGN_BYTES(8); + NvU64 fb_size NV_ALIGN_BYTES(8); + NvU32 minor_number; + NvU8 dev_name[10]; /* device names such as vmgfx[0-32] for vmkernel */ +} nv_ioctl_card_info_t; + +/* alloc event */ +typedef struct nv_ioctl_alloc_os_event +{ + NvHandle hClient; + NvHandle hDevice; + NvU32 fd; + NvU32 Status; +} nv_ioctl_alloc_os_event_t; + +/* free event */ +typedef struct nv_ioctl_free_os_event +{ + NvHandle hClient; + NvHandle hDevice; + NvU32 fd; + NvU32 Status; +} nv_ioctl_free_os_event_t; + +/* status code */ +typedef struct nv_ioctl_status_code +{ + NvU32 domain; + NvU8 bus; + NvU8 slot; + NvU32 status; +} nv_ioctl_status_code_t; + +/* check version string */ +#define NV_RM_API_VERSION_STRING_LENGTH 64 + +typedef struct nv_ioctl_rm_api_version +{ + NvU32 cmd; + NvU32 reply; + char versionString[NV_RM_API_VERSION_STRING_LENGTH]; +} nv_ioctl_rm_api_version_t; + +#define NV_RM_API_VERSION_CMD_STRICT 0 +#define NV_RM_API_VERSION_CMD_RELAXED '1' +#define NV_RM_API_VERSION_CMD_OVERRIDE '2' + +#define NV_RM_API_VERSION_REPLY_UNRECOGNIZED 0 +#define NV_RM_API_VERSION_REPLY_RECOGNIZED 1 + +typedef struct nv_ioctl_query_device_intr +{ + NvU32 intrStatus NV_ALIGN_BYTES(4); + NvU32 status; +} nv_ioctl_query_device_intr; + +/* system parameters that the kernel driver may use for configuration */ +typedef struct nv_ioctl_sys_params +{ + NvU64 memblock_size NV_ALIGN_BYTES(8); +} nv_ioctl_sys_params_t; + +typedef struct nv_ioctl_register_fd +{ + int ctl_fd; +} nv_ioctl_register_fd_t; + +#define NV_DMABUF_EXPORT_MAX_HANDLES 128 + +typedef struct nv_ioctl_export_to_dma_buf_fd +{ + int fd; + NvHandle hClient; + NvU32 totalObjects; + NvU32 numObjects; + NvU32 index; + NvU64 totalSize NV_ALIGN_BYTES(8); + NvHandle handles[NV_DMABUF_EXPORT_MAX_HANDLES]; + NvU64 offsets[NV_DMABUF_EXPORT_MAX_HANDLES] NV_ALIGN_BYTES(8); + NvU64 sizes[NV_DMABUF_EXPORT_MAX_HANDLES] NV_ALIGN_BYTES(8); + NvU32 status; +} nv_ioctl_export_to_dma_buf_fd_t; + +#endif diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-kernel-rmapi-ops.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-kernel-rmapi-ops.h new file mode 100644 index 0000000..f44799b --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-kernel-rmapi-ops.h @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2015-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_KERNEL_RMAPI_OPS_H_ +#define _NV_KERNEL_RMAPI_OPS_H_ + +/* + * Define the RMAPI provided to kernel-level RM clients. + * + * Kernel-level RM clients should populate nvidia_kernel_rmapi_ops_t + * by assigning nvidia_kernel_rmapi_ops_t::op and the corresponding + * parameter structure in nvidia_kernel_rmapi_ops_t's params union. + * Then, pass a pointer to the nvidia_kernel_rmapi_ops_t to + * rm_kernel_rmapi_op(). + */ + +#include "nvtypes.h" +#include "nvos.h" + +typedef struct { + NvU32 op; /* One of the NV0[14]_XXXX operations listed below. */ + + union { + NVOS00_PARAMETERS free; /* NV01_FREE */ + NVOS02_PARAMETERS allocMemory64; /* NV01_ALLOC_MEMORY */ + NVOS21_PARAMETERS alloc; /* NV04_ALLOC */ + NVOS32_PARAMETERS *pVidHeapControl; /* NV04_VID_HEAP_CONTROL */ + NVOS33_PARAMETERS mapMemory; /* NV04_MAP_MEMORY */ + NVOS34_PARAMETERS unmapMemory; /* NV04_UNMAP_MEMORY */ + NVOS39_PARAMETERS allocContextDma2; /* NV04_ALLOC_CONTEXT_DMA */ + NVOS46_PARAMETERS mapMemoryDma; /* NV04_MAP_MEMORY_DMA */ + NVOS47_PARAMETERS unmapMemoryDma; /* NV04_UNMAP_MEMORY_DMA */ + NVOS49_PARAMETERS bindContextDma; /* NV04_BIND_CONTEXT_DMA */ + NVOS54_PARAMETERS control; /* NV04_CONTROL*/ + NVOS55_PARAMETERS dupObject; /* NV04_DUP_OBJECT */ + NVOS57_PARAMETERS share; /* NV04_SHARE */ + NVOS61_PARAMETERS addVblankCallback; /* NV04_ADD_VBLANK_CALLBACK */ + } params; +} nvidia_kernel_rmapi_ops_t; + +#endif /* _NV_KERNEL_RMAPI_OPS_H_ */ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-nb-regs.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-nb-regs.h new file mode 100644 index 0000000..6282b35 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-nb-regs.h @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2007-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_NB_REGS_H_ +#define _NV_NB_REGS_H_ + +#include "nvdevid.h" + +typedef struct +{ + NvU32 subsystem_vendor_id; + NvU32 subsystem_device_id; + NvU16 gpu_device_id; +} nv_nb_id_t; + +typedef struct +{ + NvU32 vendor_id; + const char *name; + NvU32 data; +} nv_nb_reg_t; + +/* + * nb_id_table contains the OEM vendor ID, the subsystem ID and the + * GPU device ID of the notebooks for which we need to enable + * vendor specific registry keys. nb_reg_table contains the vendor + * specific registry key values. The initVendorSpecificRegistry() + * function compares the present notebooks OEM subsystem ID and the + * GPU device ID with the values present in id_tables. If a match + * is found, initVendorSpecificRegistry() extracts the vendor + * ID and sets any associated registry key listed in nb_reg_table. + */ + +static nv_nb_id_t nb_id_table[] = { + { PCI_VENDOR_ID_PC_PARTNER, 0x0620, 0x1284 }, // Acer GT 630 + { PCI_VENDOR_ID_PC_PARTNER, 0x0620, 0x124b }, // Acer GT 640 + { 0, 0, 0 } +}; + +static nv_nb_reg_t nb_reg_table[] = { + { PCI_VENDOR_ID_PC_PARTNER, "RmOverrideSupportChipsetAspm", 2 }, + { 0, NULL, 0 } +}; + +#endif //_NV_NB_REGS_H_ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-priv.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-priv.h new file mode 100644 index 0000000..b605a73 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-priv.h @@ -0,0 +1,377 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_PRIV_H_ +#define _NV_PRIV_H_ + +#include <nv.h> +#include <os/os.h> +#include <ctrl/ctrl402c.h> +#include <gpu/disp/kern_disp_max.h> +#include <gpu/disp/kern_disp_type.h> + +#include <efi-console.h> + +#define NV_PRIV_REG_WR08(b,o,d) (*((volatile NvV8*)&(b)->Reg008[(o)/1])=(NvV8)(d)) +#define NV_PRIV_REG_WR16(b,o,d) (*((volatile NvV16*)&(b)->Reg016[(o)/2])=(NvV16)(d)) +#define NV_PRIV_REG_WR32(b,o,d) (*((volatile NvV32*)&(b)->Reg032[(o)/4])=(NvV32)(d)) + +#define NV_PRIV_REG_RD08(b,o) ((b)->Reg008[(o)/1]) +#define NV_PRIV_REG_RD16(b,o) ((b)->Reg016[(o)/2]) +#define NV_PRIV_REG_RD32(b,o) ((b)->Reg032[(o)/4]) + +struct OBJGPU; + +typedef struct +{ + NvBool baseValid; + VGAADDRDESC base; + NvBool workspaceBaseValid; + VGAADDRDESC workspaceBase; + NvU32 vesaMode; +} nv_vga_t; + +/* +* device state during Power Management +*/ +typedef struct nv_pm_state_s +{ + NvU32 IntrEn; + NvBool InHibernate; +} nv_pm_state_t; + +/* +* data structure for the UNIX workqueues +*/ +typedef struct nv_work_item_s +{ + NvU32 flags; + NvU32 gpuInstance; + union + { + OSWorkItemFunction *pGpuFunction; + OSSystemWorkItemFunction *pSystemFunction; + } func; + void *pData; +} nv_work_item_t; + +#define NV_WORK_ITEM_FLAGS_NONE 0x0 +#define NV_WORK_ITEM_FLAGS_REQUIRES_GPU 0x1 +#define NV_WORK_ITEM_FLAGS_DONT_FREE_DATA 0x2 + +/* + * pseudo-registry data structure + */ + +typedef enum +{ + NV_REGISTRY_ENTRY_TYPE_UNKNOWN = 0, + NV_REGISTRY_ENTRY_TYPE_DWORD, + NV_REGISTRY_ENTRY_TYPE_BINARY, + NV_REGISTRY_ENTRY_TYPE_STRING +} nv_reg_type_t; + +typedef struct nv_reg_entry_s +{ + char *regParmStr; + NvU32 type; + NvU32 data; // used when type == NV_REGISTRY_ENTRY_TYPE_DWORD + NvU8 *pdata; // used when type == NV_REGISTRY_ENTRY_TYPE_{BINARY,STRING} + NvU32 len; // used when type == NV_REGISTRY_ENTRY_TYPE_{BINARY,STRING} + struct nv_reg_entry_s *next; +} nv_reg_entry_t; + +#define INVALID_DISP_ID 0xFFFFFFFF +#define MAX_DISP_ID_PER_ADAPTER 0x2 + +typedef struct nv_i2c_adapter_entry_s +{ + void *pOsAdapter; + NvU32 port; + NvU32 displayId[MAX_DISP_ID_PER_ADAPTER]; +} nv_i2c_adapter_entry_t; + +#define NV_INIT_FLAG_HAL 0x0001 +#define NV_INIT_FLAG_HAL_COMPONENTS 0x0002 +#define NV_INIT_FLAG_GPU_STATE 0x0004 +#define NV_INIT_FLAG_GPU_STATE_LOAD 0x0008 +#define NV_INIT_FLAG_FIFO_WATCHDOG 0x0010 +#define NV_INIT_FLAG_CORE_LOGIC 0x0020 +#define NV_INIT_FLAG_GPUMGR_ATTACH 0x0040 +#define NV_INIT_FLAG_PUBLIC_I2C 0x0080 +#define NV_INIT_FLAG_SCALABILITY 0x0100 +#define NV_INIT_FLAG_DMA 0x0200 + +#define MAX_I2C_ADAPTERS NV402C_CTRL_NUM_I2C_PORTS + +/* + * GPU dynamic power state machine. + * + * The GPU is in exactly one of these states at at time. Only certain state + * transitions are valid, as documented by the DAGs below. + * + * When in "instant idle" or COARSE mode: + * + * +----------------------+ + * v | + * +---------+ +----------------+ +--------+ + * | UNKNOWN | --> | IDLE_INDICATED | --> | IN_USE | + * +---------+ +----------------+ +--------+ + * + * The transition from UNKNOWN to IDLE_INDICATED happens in + * rm_init_dynamic_power_management(). + * + * Thereafter, transitions from IDLE_INDICATED to IN_USE happen when + * os_ref_dynamic_power() is called and the refcount transitions from 0 to 1; + * transitions from IN_USE to IDLE_INDICATED happen when + * os_unref_dynamic_power() is called and the refcount transitions from 1 to 0. + * Note that only calls to os_(un)ref_dynamic_power() with the mode == COARSE + * are considered in this mode; calls with mode == FINE are ignored. Since + * COARSE calls are placed only in rm_init_adapter/rm_shutdown_adapter, the GPU + * effectively stays in the IN_USE state any time any client has initialized + * it. + * + * + * When in "deferred idle" or FINE mode: + * + * +----------------------------------------------------------------+ + * | | + * | | + * | +-------------------------------------------+----------------------+ + * | | | v + * | +---------+ +----------------+ +--------------+ +----------------+ +--------+ + * | | UNKNOWN | --> | IDLE_INDICATED | --> | | --> | IDLE_SUSTAINED | --> | IN_USE | -+ + * | +---------+ +----------------+ | | +----------------+ +--------+ | + * | ^ | | | ^ | + * +--------------------+ | IDLE_INSTANT | ------+----------------------+ | + * | | | | + * | | | | + * | | <-----+ | + * +--------------+ | + * ^ | + * +-----------------------------------------------------+ + * + * As before, the transition from UNKNOWN to IDLE_INDICATED happens in + * rm_init_dynamic_power_management(). This is not ideal: it means the GPU may + * be powered down immediately upon loading the RM module, even if + * rm_init_adapter() is going to be called soon thereafter. However, we can't + * rely on deferred idle callbacks yet, since those currently rely on core RM + * being initialized. + * + * At the beginning of rm_init_adapter(), the GPU transitions to the IN_USE + * state; during the rm_init_adapter() sequence, + * RmInitDeferredDynamicPowerManagement() will be called which will schedule + * timer callbacks and set the "deferred_idle_enabled" boolean. + * + * While in "deferred idle" mode, one of the callbacks + * timerCallbackForIdlePreConditions(), timerCallbackToIndicateIdle(), or + * RmIndicateIdle() should be scheduled when in the states: + * - IN_USE + * - IDLE_INSTANT + * - IDLE_SUSTAINED + * Note that since we may transition from IN_USE to IDLE_INSTANT rapidly (e.g., + * for a series of RM calls), we don't attempt to schedule the callbacks and + * cancel them on each of these transitions. The + * timerCallbackForIdlePreConditions() callback will simply exit early if in + * the IN_USE state. + * + * As before, the GPU will remain in the IN_USE state until + * os_unref_dynamic_power() is called and the count transitions from 1 to 0 + * (calls with mode == FINE are honored, in this mode, and these transitions + * can happen frequently). When the refcount reaches 0, rather than going + * directly to the IDLE_INDICATED state, it transitions to the IDLE_INSTANT + * state. + * + * Then, when the next timerCallbackForIdlePreConditions() callback executes, + * if all preconditions are met, the state will transition to IDLE_SUSTAINED. + * + * If, when in the IDLE_SUSTAINED state, os_ref_dynamic_power() is called, the + * GPU will transition back to the IN_USE state and return to the IDLE_INSTANT + * state. This ensures that there is a suitable delay between any activity + * that requires bumping the refcount and indicating idleness. + * + * If the timerCallbackForIdlePreConditions() callback executes again and the + * GPU is still in the IDLE_SUSTAINED state, userspace mappings will be revoked + * and the timerCallbackToIndicateIdle() callback will be scheduled. + * + * If, before the timerCallbackToIndicateIdle() callback executes, either + * os_ref_dynamic_power() is called or a mapping which has been revoked is + * accessed (which triggers the RmForceGpuNotIdle() callback), the GPU will + * transition back to the IN_USE or IDLE_INSTANT state, respectively. + * + * Then, when the timerCallbackToIndicateIdle() callback executes, if all + * mappings are still revoked, and the GPU is still in the IDLE_SUSTAINED + * state, and all GPU idleness preconditions remain satisfied, the + * RmIndicateIdle() work item will be enqueued. (Else, the GPU will transition + * back to the IDLE_INSTANT state and the callback for preconditions is + * scheduled again.) + * + * Finally, once the RmIndicateIdle() work item is called, if all of the same + * conditions still hold, the state will transition to IDLE_INDICATED. No + * callbacks will be scheduled from here; the callbacks for preconditions + * should be re-scheduled when transitioning out of the IDLE_INDICATED state. + * + * Once in the IDLE_INDICATED state, the kernel is free to call the RM to + * perform the GC6 entry sequence then turn off power to the GPU (although it + * may not, if the audio function is being used for example). + * + * There are two paths to exit the IDLE_INDICATED state: + * (a) If os_ref_dynamic_power() is called, in which case it transitions + * directly to the IN_USE state; + * (b) If RmForceGpuNotIdle() is called, in which case it transitions back to + * the IDLE_INSTANT state. + */ +typedef enum +{ + NV_DYNAMIC_POWER_STATE_UNKNOWN = 0, + + NV_DYNAMIC_POWER_STATE_IN_USE, + + NV_DYNAMIC_POWER_STATE_IDLE_INSTANT, + NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED, + NV_DYNAMIC_POWER_STATE_IDLE_INDICATED, +} nv_dynamic_power_state_t; + +typedef struct nv_dynamic_power_s +{ + /* + * mode is read without the mutex -- should be read-only outside of + * rm_init_dynamic_power_management, called during probe only. + */ + nv_dynamic_power_mode_t mode; + /* + * Whether to indicate idle immediately when the refcount reaches 0, or + * only go to the IDLE_INSTANT state, and expect timer callbacks to + * transition through IDLE_SUSTAINED -> IDLE_INDICATED. + */ + NvBool deferred_idle_enabled; + + nv_dynamic_power_state_t state; + NvS32 refcount; + + /* + * A word on lock ordering. These locks must be taken in the order: + * + * RM API lock > this dynamic_power mutex > RM GPUs lock + * + * Skipping any of those locks is fine (if they aren't required to protect + * whatever state is being accessed or modified), so long as the order is + * not violated. + */ + PORT_MUTEX *mutex; + + /* + * callback handles for deferred dynamic power management. + */ + NvP64 idle_precondition_check_event; + NvP64 indicate_idle_event; + NvBool idle_precondition_check_callback_scheduled; + + /* + * callback handle for kernel initiated gc6 entry/exit. + * these will be protected by the gpu lock. + */ + NvP64 remove_idle_holdoff; + NvBool b_idle_holdoff; + + /* + * flag set if the platform does not support fine grain dynamic power + * management. + */ + NvBool b_fine_not_supported; + + /* + * This flag is used to check if a workitem is queued for + * RmQueueIdleSustainedWorkitem(). + */ + NvBool b_idle_sustained_workitem_queued; + + /* + * Counter to track clients disallowing GCOFF. + */ + NvU32 clients_gcoff_disallow_refcount; + + /* + * Maximum FB allocation size which can be saved in system memory + * while doing GCOFF based dynamic PM. + */ + NvU64 gcoff_max_fb_size; + + /* + * NVreg_DynamicPowerManagement regkey value set by the user + */ + NvU32 dynamic_power_regkey; +} nv_dynamic_power_t; + +typedef struct +{ + OBJGPU *pGpu; + + NvU32 pmc_boot_0; + + nv_vga_t vga; + + nv_efi_t efi; + + NvU32 flags; + NvU32 status; + + nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS]; + + void *pVbiosCopy; + NvU32 vbiosSize; + + nv_pm_state_t pm_state; + + nv_reg_entry_t *pRegistry; + + nv_dynamic_power_t dynamic_power; + + /* Flag to check if the GPU needs 4K page isolation. */ + NvBool b_4k_page_isolation_required; + + /* Flag to check if GPU mobile config is enabled */ + NvBool b_mobile_config_enabled; + + /* Flag to check if S0ix-based power management is enabled. */ + NvBool s0ix_pm_enabled; + + /* + * Maximum FB allocation size which can be saved in system memory + * during system supened with S0ix-based power management. + */ + NvU64 s0ix_gcoff_max_fb_size; + + NvU32 pmc_boot_42; +} nv_priv_t; + +#define NV_SET_NV_PRIV(nv,p) ((nv)->priv = (p)) +#define NV_GET_NV_PRIV(nv) ((nv) ? (nv)->priv : NULL) + +/* + * Make sure that your stack has taken API Lock before using this macro. + */ +#define NV_GET_NV_PRIV_PGPU(nv) \ + (NV_GET_NV_PRIV(nv) ? ((nv_priv_t *)NV_GET_NV_PRIV(nv))->pGpu : NULL) + +#endif // _NV_PRIV_H_ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-reg.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-reg.h new file mode 100644 index 0000000..31d3560 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-reg.h @@ -0,0 +1,927 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2006-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _RM_REG_H_ +#define _RM_REG_H_ + +#include "nvtypes.h" + +/* + * use NV_REG_STRING to stringify a registry key when using that registry key + */ + +#define __NV_REG_STRING(regkey) #regkey +#define NV_REG_STRING(regkey) __NV_REG_STRING(regkey) + +/* + * use NV_DEFINE_REG_ENTRY and NV_DEFINE_PARAMS_TABLE_ENTRY to simplify definition + * of registry keys in the kernel module source code. + */ + +#define __NV_REG_VAR(regkey) NVreg_##regkey + +#if defined(NV_MODULE_PARAMETER) +#define NV_DEFINE_REG_ENTRY(regkey, default_value) \ + static NvU32 __NV_REG_VAR(regkey) = (default_value); \ + NV_MODULE_PARAMETER(__NV_REG_VAR(regkey)) +#define NV_DEFINE_REG_ENTRY_GLOBAL(regkey, default_value) \ + NvU32 __NV_REG_VAR(regkey) = (default_value); \ + NV_MODULE_PARAMETER(__NV_REG_VAR(regkey)) +#else +#define NV_DEFINE_REG_ENTRY(regkey, default_value) \ + static NvU32 __NV_REG_VAR(regkey) = (default_value) +#define NV_DEFINE_REG_ENTRY_GLOBAL(regkey, default_value) \ + NvU32 __NV_REG_VAR(regkey) = (default_value) +#endif + +#if defined(NV_MODULE_STRING_PARAMETER) +#define NV_DEFINE_REG_STRING_ENTRY(regkey, default_value) \ + char *__NV_REG_VAR(regkey) = (default_value); \ + NV_MODULE_STRING_PARAMETER(__NV_REG_VAR(regkey)) +#else +#define NV_DEFINE_REG_STRING_ENTRY(regkey, default_value) \ + char *__NV_REG_VAR(regkey) = (default_value) +#endif + +#define NV_DEFINE_PARAMS_TABLE_ENTRY(regkey) \ + { NV_REG_STRING(regkey), &__NV_REG_VAR(regkey) } + +/* + * Like NV_DEFINE_PARMS_TABLE_ENTRY, but allows a mismatch between the name of + * the regkey and the name of the module parameter. When using this macro, the + * name of the parameter is passed to the extra "parameter" argument, and it is + * this name that must be used in the NV_DEFINE_REG_ENTRY() macro. + */ + +#define NV_DEFINE_PARAMS_TABLE_ENTRY_CUSTOM_NAME(regkey, parameter) \ + { NV_REG_STRING(regkey), &__NV_REG_VAR(parameter)} + +/* + *----------------- registry key definitions-------------------------- + */ + +/* + * Option: ModifyDeviceFiles + * + * Description: + * + * When this option is enabled, the NVIDIA driver will verify the validity + * of the NVIDIA device files in /dev and attempt to dynamically modify + * and/or (re-)create them, if necessary. If you don't wish for the NVIDIA + * driver to touch the device files, you can use this registry key. + * + * This module parameter is only honored by the NVIDIA GPU driver and NVIDIA + * capability driver. Furthermore, the NVIDIA capability driver provides + * modifiable /proc file entry (DeviceFileModify=0/1) to alter the behavior of + * this module parameter per device file. + * + * Possible Values: + * 0 = disable dynamic device file management + * 1 = enable dynamic device file management (default) + */ + +#define __NV_MODIFY_DEVICE_FILES ModifyDeviceFiles +#define NV_REG_MODIFY_DEVICE_FILES NV_REG_STRING(__NV_MODIFY_DEVICE_FILES) + +/* + * Option: DeviceFileUID + * + * Description: + * + * This registry key specifies the UID assigned to the NVIDIA device files + * created and/or modified by the NVIDIA driver when dynamic device file + * management is enabled. + * + * This module parameter is only honored by the NVIDIA GPU driver. + * + * The default UID is 0 ('root'). + */ + +#define __NV_DEVICE_FILE_UID DeviceFileUID +#define NV_REG_DEVICE_FILE_UID NV_REG_STRING(__NV_DEVICE_FILE_UID) + +/* + * Option: DeviceFileGID + * + * Description: + * + * This registry key specifies the GID assigned to the NVIDIA device files + * created and/or modified by the NVIDIA driver when dynamic device file + * management is enabled. + * + * This module parameter is only honored by the NVIDIA GPU driver. + * + * The default GID is 0 ('root'). + */ + +#define __NV_DEVICE_FILE_GID DeviceFileGID +#define NV_REG_DEVICE_FILE_GID NV_REG_STRING(__NV_DEVICE_FILE_GID) + +/* + * Option: DeviceFileMode + * + * Description: + * + * This registry key specifies the device file mode assigned to the NVIDIA + * device files created and/or modified by the NVIDIA driver when dynamic + * device file management is enabled. + * + * This module parameter is only honored by the NVIDIA GPU driver. + * + * The default mode is 0666 (octal, rw-rw-rw-). + */ + +#define __NV_DEVICE_FILE_MODE DeviceFileMode +#define NV_REG_DEVICE_FILE_MODE NV_REG_STRING(__NV_DEVICE_FILE_MODE) + +/* + * Option: ResmanDebugLevel + * + * Default value: ~0 + */ + +#define __NV_RESMAN_DEBUG_LEVEL ResmanDebugLevel +#define NV_REG_RESMAN_DEBUG_LEVEL NV_REG_STRING(__NV_RESMAN_DEBUG_LEVEL) + +/* + * Option: RmLogonRC + * + * Default value: 1 + */ + +#define __NV_RM_LOGON_RC RmLogonRC +#define NV_REG_RM_LOGON_RC NV_REG_STRING(__NV_RM_LOGON_RC) + +/* + * Option: InitializeSystemMemoryAllocations + * + * Description: + * + * The NVIDIA Linux driver normally clears system memory it allocates + * for use with GPUs or within the driver stack. This is to ensure + * that potentially sensitive data is not rendered accessible by + * arbitrary user applications. + * + * Owners of single-user systems or similar trusted configurations may + * choose to disable the aforementioned clears using this option and + * potentially improve performance. + * + * Possible values: + * + * 1 = zero out system memory allocations (default) + * 0 = do not perform memory clears + */ + +#define __NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS \ + InitializeSystemMemoryAllocations +#define NV_REG_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS \ + NV_REG_STRING(__NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS) + +/* + * Option: RegistryDwords + * + * Description: + * + * This option accepts a semicolon-separated list of key=value pairs. Each + * key name is checked against the table of static options; if a match is + * found, the static option value is overridden, but invalid options remain + * invalid. Pairs that do not match an entry in the static option table + * are passed on to the RM directly. + * + * Format: + * + * NVreg_RegistryDwords="<key=value>;<key=value>;..." + */ + +#define __NV_REGISTRY_DWORDS RegistryDwords +#define NV_REG_REGISTRY_DWORDS NV_REG_STRING(__NV_REGISTRY_DWORDS) + +/* + * Option: RegistryDwordsPerDevice + * + * Description: + * + * This option allows to specify registry keys per GPU device. It helps to + * control registry at GPU level of granularity. It accepts a semicolon + * separated list of key=value pairs. The first key value pair MUST be + * "pci=DDDD:BB:DD.F;" where DDDD is Domain, BB is Bus Id, DD is device slot + * number and F is the Function. This PCI BDF is used to identify which GPU to + * assign the registry keys that follows next. + * If a GPU corresponding to the value specified in "pci=DDDD:BB:DD.F;" is NOT + * found, then all the registry keys that follows are skipped, until we find next + * valid pci identified "pci=DDDD:BB:DD.F;". Following are the valid formats for + * the value of the "pci" string: + * 1) bus:slot : Domain and function defaults to 0. + * 2) domain:bus:slot : Function defaults to 0. + * 3) domain:bus:slot.func : Complete PCI dev id string. + * + * For each of the registry keys that follows, key name is checked against the + * table of static options; if a match is found, the static option value is + * overridden, but invalid options remain invalid. Pairs that do not match an + * entry in the static option table are passed on to the RM directly. + * + * Format: + * + * NVreg_RegistryDwordsPerDevice="pci=DDDD:BB:DD.F;<key=value>;<key=value>;..; \ + * pci=DDDD:BB:DD.F;<key=value>;..;" + */ + +#define __NV_REGISTRY_DWORDS_PER_DEVICE RegistryDwordsPerDevice +#define NV_REG_REGISTRY_DWORDS_PER_DEVICE NV_REG_STRING(__NV_REGISTRY_DWORDS_PER_DEVICE) + +#define __NV_RM_MSG RmMsg +#define NV_RM_MSG NV_REG_STRING(__NV_RM_MSG) + +/* + * Option: UsePageAttributeTable + * + * Description: + * + * Enable/disable use of the page attribute table (PAT) available in + * modern x86/x86-64 processors to set the effective memory type of memory + * mappings to write-combining (WC). + * + * If enabled, an x86 processor with PAT support is present and the host + * system's Linux kernel did not configure one of the PAT entries to + * indicate the WC memory type, the driver will change the second entry in + * the PAT from its default (write-through (WT)) to WC at module load + * time. If the kernel did update one of the PAT entries, the driver will + * not modify the PAT. + * + * In both cases, the driver will honor attempts to map memory with the WC + * memory type by selecting the appropriate PAT entry using the correct + * set of PTE flags. + * + * Possible values: + * + * ~0 = use the NVIDIA driver's default logic (default) + * 1 = enable use of the PAT for WC mappings. + * 0 = disable use of the PAT for WC mappings. + */ + +#define __NV_USE_PAGE_ATTRIBUTE_TABLE UsePageAttributeTable +#define NV_USE_PAGE_ATTRIBUTE_TABLE NV_REG_STRING(__NV_USE_PAGE_ATTRIBUTE_TABLE) + +/* + * Option: EnableMSI + * + * Description: + * + * When this option is enabled and the host kernel supports the MSI feature, + * the NVIDIA driver will enable the PCI-E MSI capability of GPUs with the + * support for this feature instead of using PCI-E wired interrupt. + * + * Possible Values: + * + * 0 = disable MSI interrupt + * 1 = enable MSI interrupt (default) + * + */ + +#define __NV_ENABLE_MSI EnableMSI +#define NV_REG_ENABLE_MSI NV_REG_STRING(__NV_ENABLE_MSI) + +/* + * Option: EnablePCIeGen3 + * + * Description: + * + * Due to interoperability problems seen with Kepler PCIe Gen3 capable GPUs + * when configured on SandyBridge E desktop platforms, NVIDIA feels that + * delivering a reliable, high-quality experience is not currently possible in + * PCIe Gen3 mode on all PCIe Gen3 platforms. Therefore, Quadro, Tesla and + * NVS Kepler products operate in PCIe Gen2 mode by default. You may use this + * option to enable PCIe Gen3 support. + * + * This is completely unsupported! + * + * Possible Values: + * + * 0: disable PCIe Gen3 support (default) + * 1: enable PCIe Gen3 support + */ + +#define __NV_ENABLE_PCIE_GEN3 EnablePCIeGen3 +#define NV_REG_ENABLE_PCIE_GEN3 NV_REG_STRING(__NV_ENABLE_PCIE_GEN3) + +/* + * Option: MemoryPoolSize + * + * Description: + * + * When set to a non-zero value, this option specifies the size of the + * memory pool, given as a multiple of 1 GB, created on VMware ESXi to + * satisfy any system memory allocations requested by the NVIDIA kernel + * module. + */ + +#define __NV_MEMORY_POOL_SIZE MemoryPoolSize +#define NV_REG_MEMORY_POOL_SIZE NV_REG_STRING(__NV_MEMORY_POOL_SIZE) + +/* + * Option: KMallocHeapMaxSize + * + * Description: + * + * When set to a non-zero value, this option specifies the maximum size of the + * heap memory space reserved for kmalloc operations. Given as a + * multiple of 1 MB created on VMware ESXi to satisfy any system memory + * allocations requested by the NVIDIA kernel module. + */ + +#define __NV_KMALLOC_HEAP_MAX_SIZE KMallocHeapMaxSize +#define NV_KMALLOC_HEAP_MAX_SIZE NV_REG_STRING(__NV_KMALLOC_HEAP_MAX_SIZE) + +/* + * Option: VMallocHeapMaxSize + * + * Description: + * + * When set to a non-zero value, this option specifies the maximum size of the + * heap memory space reserved for vmalloc operations. Given as a + * multiple of 1 MB created on VMware ESXi to satisfy any system memory + * allocations requested by the NVIDIA kernel module. + */ + +#define __NV_VMALLOC_HEAP_MAX_SIZE VMallocHeapMaxSize +#define NV_VMALLOC_HEAP_MAX_SIZE NV_REG_STRING(__NV_VMALLOC_HEAP_MAX_SIZE) + +/* + * Option: IgnoreMMIOCheck + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will ignore + * MMIO limit check during device probe on VMWare ESXi kernel. This is + * typically necessary when VMware ESXi MMIO limit differs between any + * base version and its updates. Customer using updates can set regkey + * to avoid probe failure. + */ + +#define __NV_IGNORE_MMIO_CHECK IgnoreMMIOCheck +#define NV_REG_IGNORE_MMIO_CHECK NV_REG_STRING(__NV_IGNORE_MMIO_CHECK) + +/* + * Option: TCEBypassMode + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will attempt to setup + * all GPUs in "TCE bypass mode", in which DMA mappings of system memory bypass + * the IOMMU/TCE remapping hardware on IBM POWER systems. This is typically + * necessary for CUDA applications in which large system memory mappings may + * exceed the default TCE remapping capacity when operated in non-bypass mode. + * + * This option has no effect on non-POWER platforms. + * + * Possible Values: + * + * 0: system default TCE mode on all GPUs + * 1: enable TCE bypass mode on all GPUs + * 2: disable TCE bypass mode on all GPUs + */ +#define __NV_TCE_BYPASS_MODE TCEBypassMode +#define NV_REG_TCE_BYPASS_MODE NV_REG_STRING(__NV_TCE_BYPASS_MODE) + +#define NV_TCE_BYPASS_MODE_DEFAULT 0 +#define NV_TCE_BYPASS_MODE_ENABLE 1 +#define NV_TCE_BYPASS_MODE_DISABLE 2 + +/* + * Option: pci + * + * Description: + * + * On Unix platforms, per GPU based registry key can be specified as: + * NVreg_RegistryDwordsPerDevice="pci=DDDD:BB:DD.F,<per-gpu registry keys>". + * where DDDD:BB:DD.F refers to Domain:Bus:Device.Function. + * We need this key "pci" to identify what follows next is a PCI BDF identifier, + * for which the registry keys are to be applied. + * + * This define is not used on non-UNIX platforms. + * + * Possible Formats for value: + * + * 1) bus:slot : Domain and function defaults to 0. + * 2) domain:bus:slot : Function defaults to 0. + * 3) domain:bus:slot.func : Complete PCI BDF identifier string. + */ +#define __NV_PCI_DEVICE_BDF pci +#define NV_REG_PCI_DEVICE_BDF NV_REG_STRING(__NV_PCI_DEVICE_BDF) + +/* + * Option: EnableStreamMemOPs + * + * Description: + * + * When this option is enabled, the CUDA driver will enable support for + * CUDA Stream Memory Operations in user-mode applications, which are so + * far required to be disabled by default due to limited support in + * devtools. + * + * Note: this is treated as a hint. MemOPs may still be left disabled by CUDA + * driver for other reasons. + * + * Possible Values: + * + * 0 = disable feature (default) + * 1 = enable feature + */ +#define __NV_ENABLE_STREAM_MEMOPS EnableStreamMemOPs +#define NV_REG_ENABLE_STREAM_MEMOPS NV_REG_STRING(__NV_ENABLE_STREAM_MEMOPS) + +/* + * Option: EnableUserNUMAManagement + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will require the + * user-mode NVIDIA Persistence daemon to manage the onlining and offlining + * of its NUMA device memory. + * + * This option has no effect on platforms that do not support onlining + * device memory to a NUMA node (this feature is only supported on certain + * POWER9 systems). + * + * Possible Values: + * + * 0: disable user-mode NUMA management + * 1: enable user-mode NUMA management (default) + */ +#define __NV_ENABLE_USER_NUMA_MANAGEMENT EnableUserNUMAManagement +#define NV_REG_ENABLE_USER_NUMA_MANAGEMENT NV_REG_STRING(__NV_ENABLE_USER_NUMA_MANAGEMENT) + +/* + * Option: GpuBlacklist + * + * Description: + * + * This option accepts a list of blacklisted GPUs, separated by commas, that + * cannot be attached or used. Each blacklisted GPU is identified by a UUID in + * the ASCII format with leading "GPU-". An exact match is required; no partial + * UUIDs. This regkey is deprecated and will be removed in the future. Use + * NV_REG_EXCLUDED_GPUS instead. + */ +#define __NV_GPU_BLACKLIST GpuBlacklist +#define NV_REG_GPU_BLACKLIST NV_REG_STRING(__NV_GPU_BLACKLIST) + +/* + * Option: ExcludedGpus + * + * Description: + * + * This option accepts a list of excluded GPUs, separated by commas, that + * cannot be attached or used. Each excluded GPU is identified by a UUID in + * the ASCII format with leading "GPU-". An exact match is required; no partial + * UUIDs. + */ +#define __NV_EXCLUDED_GPUS ExcludedGpus +#define NV_REG_EXCLUDED_GPUS NV_REG_STRING(__NV_EXCLUDED_GPUS) + +/* + * Option: NvLinkDisable + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will not attempt to + * initialize or train NVLink connections for any GPUs. System reboot is required + * for changes to take affect. + * + * This option has no effect if no GPUs support NVLink. + * + * Possible Values: + * + * 0: Do not disable NVLink (default) + * 1: Disable NVLink + */ +#define __NV_NVLINK_DISABLE NvLinkDisable +#define NV_REG_NVLINK_DISABLE NV_REG_STRING(__NV_NVLINK_DISABLE) + +/* + * Option: RestrictProfilingToAdminUsers + * + * Description: + * + * When this option is enabled, the NVIDIA kernel module will prevent users + * without administrative access (i.e., the CAP_SYS_ADMIN capability) from + * using GPU performance counters. + * + * Possible Values: + * + * 0: Do not restrict GPU counters (default) + * 1: Restrict GPU counters to system administrators only + */ + +#define __NV_RM_PROFILING_ADMIN_ONLY RmProfilingAdminOnly +#define __NV_RM_PROFILING_ADMIN_ONLY_PARAMETER RestrictProfilingToAdminUsers +#define NV_REG_RM_PROFILING_ADMIN_ONLY NV_REG_STRING(__NV_RM_PROFILING_ADMIN_ONLY) + +/* + * Option: TemporaryFilePath + * + * Description: + * + * When specified, this option changes the location in which the + * NVIDIA kernel module will create unnamed temporary files (e.g. to + * save the contents of video memory in). The indicated file must + * be a directory. By default, temporary files are created in /tmp. + */ +#define __NV_TEMPORARY_FILE_PATH TemporaryFilePath +#define NV_REG_TEMPORARY_FILE_PATH NV_REG_STRING(__NV_TEMPORARY_FILE_PATH) + +/* + * Option: PreserveVideoMemoryAllocations + * + * If enabled, this option prompts the NVIDIA kernel module to save and + * restore all video memory allocations across system power management + * cycles, i.e. suspend/resume and hibernate/restore. Otherwise, + * only select allocations are preserved. + * + * Possible Values: + * + * 0: Preserve only select video memory allocations (default) + * 1: Preserve all video memory allocations + */ +#define __NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS PreserveVideoMemoryAllocations +#define NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS \ + NV_REG_STRING(__NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS) + +/* + * Option: EnableS0ixPowerManagement + * + * When this option is enabled, the NVIDIA driver will use S0ix-based + * power management for system suspend/resume, if both the platform and + * the GPU support S0ix. + * + * During system suspend, if S0ix is enabled and + * video memory usage is above the threshold configured by + * 'S0ixPowerManagementVideoMemoryThreshold', video memory will be kept + * in self-refresh mode while the rest of the GPU is powered down. + * + * Otherwise, the driver will copy video memory contents to system memory + * and power off the video memory along with the GPU. + * + * Possible Values: + * + * 0: Disable S0ix based power management (default) + * 1: Enable S0ix based power management + */ + +#define __NV_ENABLE_S0IX_POWER_MANAGEMENT EnableS0ixPowerManagement +#define NV_REG_ENABLE_S0IX_POWER_MANAGEMENT \ + NV_REG_STRING(__NV_ENABLE_S0IX_POWER_MANAGEMENT) + +/* + * Option: S0ixPowerManagementVideoMemoryThreshold + * + * This option controls the threshold that the NVIDIA driver will use during + * S0ix-based system power management. + * + * When S0ix is enabled and the system is suspended, the driver will + * compare the amount of video memory in use with this threshold, + * to decide whether to keep video memory in self-refresh or copy video + * memory content to system memory. + * + * See the 'EnableS0ixPowerManagement' option. + * + * Values are expressed in Megabytes (1048576 bytes). + * + * Default value for this option is 256MB. + * + */ +#define __NV_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD \ + S0ixPowerManagementVideoMemoryThreshold +#define NV_REG_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD \ + NV_REG_STRING(__NV_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD) + +/* + * Option: DynamicPowerManagement + * + * This option controls how aggressively the NVIDIA kernel module will manage + * GPU power through kernel interfaces. + * + * Possible Values: + * + * 0: Never allow the GPU to be powered down (default). + * 1: Power down the GPU when it is not initialized. + * 2: Power down the GPU after it has been inactive for some time. + * 3: (Default) Power down the GPU after a period of inactivity (i.e., + * mode 2) on Ampere or later notebooks. Otherwise, do not power down + * the GPU. + */ +#define __NV_DYNAMIC_POWER_MANAGEMENT DynamicPowerManagement +#define NV_REG_DYNAMIC_POWER_MANAGEMENT \ + NV_REG_STRING(__NV_DYNAMIC_POWER_MANAGEMENT) + +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER 0 +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_COARSE 1 +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_FINE 2 +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT 3 + +/* + * Option: DynamicPowerManagementVideoMemoryThreshold + * + * This option controls the threshold that the NVIDIA driver will use + * when selecting the dynamic power management scheme. + * + * When the driver detects that the GPU is idle, it will compare the amount + * of video memory in use with this threshold. + * + * If the current video memory usage is less than the threshold, the + * driver may preserve video memory contents in system memory and power off + * the video memory along with the GPU itself, if supported. Otherwise, + * the video memory will be kept in self-refresh mode while powering down + * the rest of the GPU, if supported. + * + * Values are expressed in Megabytes (1048576 bytes). + * + * If the requested value is greater than 200MB (the default), then it + * will be capped to 200MB. + */ +#define __NV_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD \ + DynamicPowerManagementVideoMemoryThreshold +#define NV_REG_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD \ + NV_REG_STRING(__NV_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD) + +/* + * Option: RegisterPCIDriver + * + * Description: + * + * When this option is enabled, the NVIDIA driver will register with + * PCI subsystem. + * + * Possible values: + * + * 1 - register as PCI driver (default) + * 0 - do not register as PCI driver + */ + +#define __NV_REGISTER_PCI_DRIVER RegisterPCIDriver +#define NV_REG_REGISTER_PCI_DRIVER NV_REG_STRING(__NV_REGISTER_PCI_DRIVER) + +/* + * Option: EnablePCIERelaxedOrderingMode + * + * Description: + * + * When this option is enabled, the registry key RmSetPCIERelaxedOrdering will + * be set to NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING_FORCE_ENABLE, causing + * every device to set the relaxed ordering bit to 1 in all outbound MWr + * transaction-layer packets. This is equivalent to setting the regkey to + * FORCE_ENABLE as a non-per-device registry key. + * + * Possible values: + * 0 - Do not enable PCIe TLP relaxed ordering bit-setting (default) + * 1 - Enable PCIe TLP relaxed ordering bit-setting + */ +#define __NV_ENABLE_PCIE_RELAXED_ORDERING_MODE EnablePCIERelaxedOrderingMode +#define NV_REG_ENABLE_PCIE_RELAXED_ORDERING_MODE \ + NV_REG_STRING(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE) + +/* + * Option: EnableGpuFirmware + * + * Description: + * + * When this option is enabled, the NVIDIA driver will enable use of GPU + * firmware. + * + * Possible mode values: + * 0 - Do not enable GPU firmware + * 1 - Enable GPU firmware + * 2 - (Default) Use the default enablement policy for GPU firmware + * + * Setting this to anything other than 2 will alter driver firmware- + * enablement policies, possibly disabling GPU firmware where it would + * have otherwise been enabled by default. + * + * If this key is set globally to the system, the driver may still attempt + * to apply some policies to maintain uniform firmware modes across all + * GPUS. This may result in the driver failing initialization on some GPUs + * to maintain such a policy. + * + * If this key is set using NVreg_RegistryDwordsPerDevice, then the driver + * will attempt to honor whatever configuration is specified without applying + * additional policies. This may also result in failed GPU initialzations if + * the configuration is not possible (for example if the firmware is missing + * from the filesystem, or the GPU is not capable). + * + * Policy bits: + * + * POLICY_ALLOW_FALLBACK: + * As the normal behavior is to fail GPU initialization if this registry + * entry is set in such a way that results in an invalid configuration, if + * instead the user would like the driver to automatically try to fallback + * to initializing the failing GPU with firmware disabled, then this bit can + * be set (ex: 0x11 means try to enable GPU firmware but fall back if needed). + * Note that this can result in a mixed mode configuration (ex: GPU0 has + * firmware enabled, but GPU1 does not). + * + */ + +#define __NV_ENABLE_GPU_FIRMWARE EnableGpuFirmware +#define NV_REG_ENABLE_GPU_FIRMWARE NV_REG_STRING(__NV_ENABLE_GPU_FIRMWARE) + +#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_MASK 0x0000000F +#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_DISABLED 0x00000000 +#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_ENABLED 0x00000001 +#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_DEFAULT 0x00000002 + +#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_MASK 0x000000F0 +#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_ALLOW_FALLBACK 0x00000010 + +#define NV_REG_ENABLE_GPU_FIRMWARE_DEFAULT_VALUE 0x00000012 +#define NV_REG_ENABLE_GPU_FIRMWARE_INVALID_VALUE 0xFFFFFFFF + +/* + * Option: EnableGpuFirmwareLogs + * + * When this option is enabled, the NVIDIA driver will send GPU firmware logs + * to the system log, when possible. + * + * Possible values: + * 0 - Do not send GPU firmware logs to the system log + * 1 - Enable sending of GPU firmware logs to the system log + * 2 - (Default) Enable sending of GPU firmware logs to the system log for + * the debug kernel driver build only + */ +#define __NV_ENABLE_GPU_FIRMWARE_LOGS EnableGpuFirmwareLogs +#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS NV_REG_STRING(__NV_ENABLE_GPU_FIRMWARE_LOGS) + +#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_DISABLE 0x00000000 +#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE 0x00000001 +#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG 0x00000002 + +/* + * Option: EnableDbgBreakpoint + * + * When this option is set to a non-zero value, and the kernel is configured + * appropriately, assertions within resman will trigger a CPU breakpoint (e.g., + * INT3 on x86_64), assumed to be caught by an attached debugger. + * + * When this option is set to the value zero (the default), assertions within + * resman will print to the system log, but no CPU breakpoint will be triggered. + */ +#define __NV_ENABLE_DBG_BREAKPOINT EnableDbgBreakpoint + + +/* + * Option: OpenRmEnableUnsupportedGpus + * + * Open nvidia.ko support for features beyond what is used on Data Center GPUs + * is still fairly immature, so for now require users to opt into use of open + * nvidia.ko with a special registry key, if not on a Data Center GPU. + */ + +#define __NV_OPENRM_ENABLE_UNSUPPORTED_GPUS OpenRmEnableUnsupportedGpus +#define NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS NV_REG_STRING(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS) +#define NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DISABLE 0x00000000 +#define NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_ENABLE 0x00000001 +#define NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DEFAULT NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DISABLE + +/* + * Option: NVreg_DmaRemapPeerMmio + * + * Description: + * + * When this option is enabled, the NVIDIA driver will use device driver + * APIs provided by the Linux kernel for DMA-remapping part of a device's + * MMIO region to another device, creating e.g., IOMMU mappings as necessary. + * When this option is disabled, the NVIDIA driver will instead only apply a + * fixed offset, which may be zero, to CPU physical addresses to produce the + * DMA address for the peer's MMIO region, and no IOMMU mappings will be + * created. + * + * This option only affects peer MMIO DMA mappings, and not system memory + * mappings. + * + * Possible Values: + * 0 = disable dynamic DMA remapping of peer MMIO regions + * 1 = enable dynamic DMA remapping of peer MMIO regions (default) + */ +#define __NV_DMA_REMAP_PEER_MMIO DmaRemapPeerMmio +#define NV_DMA_REMAP_PEER_MMIO NV_REG_STRING(__NV_DMA_REMAP_PEER_MMIO) +#define NV_DMA_REMAP_PEER_MMIO_DISABLE 0x00000000 +#define NV_DMA_REMAP_PEER_MMIO_ENABLE 0x00000001 + +#if defined(NV_DEFINE_REGISTRY_KEY_TABLE) + +/* + *---------registry key parameter declarations-------------- + */ + +NV_DEFINE_REG_ENTRY(__NV_RESMAN_DEBUG_LEVEL, ~0); +NV_DEFINE_REG_ENTRY(__NV_RM_LOGON_RC, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_MODIFY_DEVICE_FILES, 1); +NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_UID, 0); +NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_GID, 0); +NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_MODE, 0666); +NV_DEFINE_REG_ENTRY(__NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, 1); +NV_DEFINE_REG_ENTRY(__NV_USE_PAGE_ATTRIBUTE_TABLE, ~0); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_PCIE_GEN3, 0); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_MSI, 1); +NV_DEFINE_REG_ENTRY(__NV_TCE_BYPASS_MODE, NV_TCE_BYPASS_MODE_DEFAULT); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_STREAM_MEMOPS, 0); +NV_DEFINE_REG_ENTRY(__NV_RM_PROFILING_ADMIN_ONLY_PARAMETER, 1); +NV_DEFINE_REG_ENTRY(__NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS, 0); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_S0IX_POWER_MANAGEMENT, 0); +NV_DEFINE_REG_ENTRY(__NV_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD, 256); +NV_DEFINE_REG_ENTRY(__NV_DYNAMIC_POWER_MANAGEMENT, 3); +NV_DEFINE_REG_ENTRY(__NV_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD, 200); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_GPU_FIRMWARE, NV_REG_ENABLE_GPU_FIRMWARE_DEFAULT_VALUE); +NV_DEFINE_REG_ENTRY(__NV_ENABLE_GPU_FIRMWARE_LOGS, NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG); +NV_DEFINE_REG_ENTRY(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS, NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DEFAULT); + +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_USER_NUMA_MANAGEMENT, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_MEMORY_POOL_SIZE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_KMALLOC_HEAP_MAX_SIZE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_VMALLOC_HEAP_MAX_SIZE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_IGNORE_MMIO_CHECK, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_NVLINK_DISABLE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PCI_DRIVER, 1); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_DBG_BREAKPOINT, 0); + +NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS_PER_DEVICE, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_RM_MSG, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_GPU_BLACKLIST, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_TEMPORARY_FILE_PATH, NULL); +NV_DEFINE_REG_STRING_ENTRY(__NV_EXCLUDED_GPUS, NULL); +NV_DEFINE_REG_ENTRY(__NV_DMA_REMAP_PEER_MMIO, NV_DMA_REMAP_PEER_MMIO_ENABLE); + +/* + *----------------registry database definition---------------------- + */ + +/* + * You can enable any of the registry options disabled by default by + * editing their respective entries in the table below. The last field + * determines if the option is considered valid - in order for the + * changes to take effect, you need to recompile and reload the NVIDIA + * kernel module. + */ +nv_parm_t nv_parms[] = { + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RESMAN_DEBUG_LEVEL), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RM_LOGON_RC), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_MODIFY_DEVICE_FILES), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_UID), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_GID), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_MODE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_USE_PAGE_ATTRIBUTE_TABLE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_MSI), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_PCIE_GEN3), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_MEMORY_POOL_SIZE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_KMALLOC_HEAP_MAX_SIZE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_VMALLOC_HEAP_MAX_SIZE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_IGNORE_MMIO_CHECK), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_TCE_BYPASS_MODE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_STREAM_MEMOPS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_USER_NUMA_MANAGEMENT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_NVLINK_DISABLE), + NV_DEFINE_PARAMS_TABLE_ENTRY_CUSTOM_NAME(__NV_RM_PROFILING_ADMIN_ONLY, + __NV_RM_PROFILING_ADMIN_ONLY_PARAMETER), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_S0IX_POWER_MANAGEMENT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DYNAMIC_POWER_MANAGEMENT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_REGISTER_PCI_DRIVER), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_GPU_FIRMWARE), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_GPU_FIRMWARE_LOGS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_DBG_BREAKPOINT), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DMA_REMAP_PEER_MMIO), + {NULL, NULL} +}; + +#elif defined(NVRM) + +extern nv_parm_t nv_parms[]; + +#endif /* NV_DEFINE_REGISTRY_KEY_TABLE */ + +#endif /* _RM_REG_H_ */ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv-unix-nvos-params-wrappers.h b/userspace/sdks/nv525/nvalloc/unix/include/nv-unix-nvos-params-wrappers.h new file mode 100644 index 0000000..02b0156 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv-unix-nvos-params-wrappers.h @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_UNIX_NVOS_PARAMS_WRAPPERS_H_ +#define _NV_UNIX_NVOS_PARAMS_WRAPPERS_H_ + +#include <nvos.h> + +/* + * This is a wrapper for NVOS02_PARAMETERS with file descriptor + */ + +typedef struct +{ + NVOS02_PARAMETERS params; + int fd; +} nv_ioctl_nvos02_parameters_with_fd; + +/* + * This is a wrapper for NVOS33_PARAMETERS with file descriptor + */ +typedef struct +{ + NVOS33_PARAMETERS params; + int fd; +} nv_ioctl_nvos33_parameters_with_fd; + +#endif // _NV_UNIX_NVOS_PARAMS_WRAPPERS_H_ + diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv.h b/userspace/sdks/nv525/nvalloc/unix/include/nv.h new file mode 100644 index 0000000..5cb4d1b --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv.h @@ -0,0 +1,1058 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef _NV_H_ +#define _NV_H_ + + + +#include <nvlimits.h> + +#if defined(NV_KERNEL_INTERFACE_LAYER) && defined(__FreeBSD__) + #include <sys/stddef.h> // NULL +#elif defined(NV_KERNEL_INTERFACE_LAYER) && defined(NV_LINUX) + #include <linux/stddef.h> // NULL +#else + #include <stddef.h> // NULL +#endif + +#include <nvstatus.h> +#include "nv_stdarg.h" +#include <nv-caps.h> +#include <nv-firmware.h> +#include <nv-ioctl.h> +#include <nvmisc.h> + +extern nv_cap_t *nvidia_caps_root; + +extern const NvBool nv_is_rm_firmware_supported_os; + +#include <nv-kernel-interface-api.h> + +/* NVIDIA's reserved major character device number (Linux). */ +#define NV_MAJOR_DEVICE_NUMBER 195 + +#define GPU_UUID_LEN (16) + +/* + * Buffer size for an ASCII UUID: We need 2 digits per byte, plus space + * for "GPU", 5 dashes, and '\0' termination: + */ +#define GPU_UUID_ASCII_LEN (GPU_UUID_LEN * 2 + 9) + +/* + * #define an absolute maximum used as a sanity check for the + * NV_ESC_IOCTL_XFER_CMD ioctl() size argument. + */ +#define NV_ABSOLUTE_MAX_IOCTL_SIZE 16384 + +/* + * Solaris provides no more than 8 bits for the argument size in + * the ioctl() command encoding; make sure we don't exceed this + * limit. + */ +#define __NV_IOWR_ASSERT(type) ((sizeof(type) <= NV_PLATFORM_MAX_IOCTL_SIZE) ? 1 : -1) +#define __NV_IOWR(nr, type) ({ \ + typedef char __NV_IOWR_TYPE_SIZE_ASSERT[__NV_IOWR_ASSERT(type)]; \ + _IOWR(NV_IOCTL_MAGIC, (nr), type); \ +}) + +#define NV_PCI_DEV_FMT "%04x:%02x:%02x.%x" +#define NV_PCI_DEV_FMT_ARGS(nv) (nv)->pci_info.domain, (nv)->pci_info.bus, \ + (nv)->pci_info.slot, (nv)->pci_info.function + +#define NV_RM_DEVICE_INTR_ADDRESS 0x100 + +/*! + * @brief The order of the display clocks in the below defined enum + * should be synced with below mapping array and macro. + * All four should be updated simultaneously in case + * of removal or addition of clocks in below order. + * Also, TEGRASOC_WHICH_CLK_MAX is used in various places + * in below mentioned files. + * arch/nvalloc/unix/Linux/nv-linux.h + * + * arch/nvalloc/unix/src/os.c + * dispClkMapRmToOsArr[] = {...}; + * + * arch/nvalloc/unix/Linux/nv-clk.c + * osMapClk[] = {...}; + * + */ +typedef enum _TEGRASOC_WHICH_CLK +{ + TEGRASOC_WHICH_CLK_NVDISPLAYHUB, + TEGRASOC_WHICH_CLK_NVDISPLAY_DISP, + TEGRASOC_WHICH_CLK_NVDISPLAY_P0, + TEGRASOC_WHICH_CLK_NVDISPLAY_P1, + TEGRASOC_WHICH_CLK_DPAUX0, + TEGRASOC_WHICH_CLK_FUSE, + TEGRASOC_WHICH_CLK_DSIPLL_VCO, + TEGRASOC_WHICH_CLK_DSIPLL_CLKOUTPN, + TEGRASOC_WHICH_CLK_DSIPLL_CLKOUTA, + TEGRASOC_WHICH_CLK_SPPLL0_VCO, + TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTPN, + TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTA, + TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTB, + TEGRASOC_WHICH_CLK_SPPLL0_DIV10, + TEGRASOC_WHICH_CLK_SPPLL0_DIV25, + TEGRASOC_WHICH_CLK_SPPLL0_DIV27, + TEGRASOC_WHICH_CLK_SPPLL1_VCO, + TEGRASOC_WHICH_CLK_SPPLL1_CLKOUTPN, + TEGRASOC_WHICH_CLK_SPPLL1_DIV27, + TEGRASOC_WHICH_CLK_VPLL0_REF, + TEGRASOC_WHICH_CLK_VPLL0, + TEGRASOC_WHICH_CLK_VPLL1, + TEGRASOC_WHICH_CLK_NVDISPLAY_P0_REF, + TEGRASOC_WHICH_CLK_RG0, + TEGRASOC_WHICH_CLK_RG1, + TEGRASOC_WHICH_CLK_DISPPLL, + TEGRASOC_WHICH_CLK_DISPHUBPLL, + TEGRASOC_WHICH_CLK_DSI_LP, + TEGRASOC_WHICH_CLK_DSI_CORE, + TEGRASOC_WHICH_CLK_DSI_PIXEL, + TEGRASOC_WHICH_CLK_PRE_SOR0, + TEGRASOC_WHICH_CLK_PRE_SOR1, + TEGRASOC_WHICH_CLK_DP_LINK_REF, + TEGRASOC_WHICH_CLK_SOR_LINKA_INPUT, + TEGRASOC_WHICH_CLK_SOR_LINKA_AFIFO, + TEGRASOC_WHICH_CLK_SOR_LINKA_AFIFO_M, + TEGRASOC_WHICH_CLK_RG0_M, + TEGRASOC_WHICH_CLK_RG1_M, + TEGRASOC_WHICH_CLK_SOR0_M, + TEGRASOC_WHICH_CLK_SOR1_M, + TEGRASOC_WHICH_CLK_PLLHUB, + TEGRASOC_WHICH_CLK_SOR0, + TEGRASOC_WHICH_CLK_SOR1, + TEGRASOC_WHICH_CLK_SOR_PAD_INPUT, + TEGRASOC_WHICH_CLK_PRE_SF0, + TEGRASOC_WHICH_CLK_SF0, + TEGRASOC_WHICH_CLK_SF1, + TEGRASOC_WHICH_CLK_DSI_PAD_INPUT, + TEGRASOC_WHICH_CLK_PRE_SOR0_REF, + TEGRASOC_WHICH_CLK_PRE_SOR1_REF, + TEGRASOC_WHICH_CLK_SOR0_PLL_REF, + TEGRASOC_WHICH_CLK_SOR1_PLL_REF, + TEGRASOC_WHICH_CLK_SOR0_REF, + TEGRASOC_WHICH_CLK_SOR1_REF, + TEGRASOC_WHICH_CLK_OSC, + TEGRASOC_WHICH_CLK_DSC, + TEGRASOC_WHICH_CLK_MAUD, + TEGRASOC_WHICH_CLK_AZA_2XBIT, + TEGRASOC_WHICH_CLK_AZA_BIT, + TEGRASOC_WHICH_CLK_MIPI_CAL, + TEGRASOC_WHICH_CLK_UART_FST_MIPI_CAL, + TEGRASOC_WHICH_CLK_SOR0_DIV, + TEGRASOC_WHICH_CLK_DISP_ROOT, + TEGRASOC_WHICH_CLK_HUB_ROOT, + TEGRASOC_WHICH_CLK_PLLA_DISP, + TEGRASOC_WHICH_CLK_PLLA_DISPHUB, + TEGRASOC_WHICH_CLK_PLLA, + TEGRASOC_WHICH_CLK_MAX, // TEGRASOC_WHICH_CLK_MAX is defined for boundary checks only. +} TEGRASOC_WHICH_CLK; + +#ifdef NVRM + +extern const char *pNVRM_ID; + +/* + * ptr arithmetic convenience + */ + +typedef union +{ + volatile NvV8 Reg008[1]; + volatile NvV16 Reg016[1]; + volatile NvV32 Reg032[1]; +} nv_hwreg_t, * nv_phwreg_t; + + +#define NVRM_PCICFG_NUM_BARS 6 +#define NVRM_PCICFG_BAR_OFFSET(i) (0x10 + (i) * 4) +#define NVRM_PCICFG_BAR_REQTYPE_MASK 0x00000001 +#define NVRM_PCICFG_BAR_REQTYPE_MEMORY 0x00000000 +#define NVRM_PCICFG_BAR_MEMTYPE_MASK 0x00000006 +#define NVRM_PCICFG_BAR_MEMTYPE_64BIT 0x00000004 +#define NVRM_PCICFG_BAR_ADDR_MASK 0xfffffff0 + +#define NVRM_PCICFG_NUM_DWORDS 16 + +#define NV_GPU_NUM_BARS 3 +#define NV_GPU_BAR_INDEX_REGS 0 +#define NV_GPU_BAR_INDEX_FB 1 +#define NV_GPU_BAR_INDEX_IMEM 2 + +typedef struct +{ + NvU64 cpu_address; + NvU64 size; + NvU32 offset; + NvU32 *map; + nv_phwreg_t map_u; +} nv_aperture_t; + +typedef struct +{ + char *name; + NvU32 *data; +} nv_parm_t; + +#define NV_RM_PAGE_SHIFT 12 +#define NV_RM_PAGE_SIZE (1 << NV_RM_PAGE_SHIFT) +#define NV_RM_PAGE_MASK (NV_RM_PAGE_SIZE - 1) + +#define NV_RM_TO_OS_PAGE_SHIFT (os_page_shift - NV_RM_PAGE_SHIFT) +#define NV_RM_PAGES_PER_OS_PAGE (1U << NV_RM_TO_OS_PAGE_SHIFT) +#define NV_RM_PAGES_TO_OS_PAGES(count) \ + ((((NvUPtr)(count)) >> NV_RM_TO_OS_PAGE_SHIFT) + \ + ((((count) & ((1 << NV_RM_TO_OS_PAGE_SHIFT) - 1)) != 0) ? 1 : 0)) + +#if defined(NVCPU_X86_64) +#define NV_STACK_SIZE (NV_RM_PAGE_SIZE * 3) +#else +#define NV_STACK_SIZE (NV_RM_PAGE_SIZE * 2) +#endif + +typedef struct nvidia_stack_s +{ + NvU32 size; + void *top; + NvU8 stack[NV_STACK_SIZE-16] __attribute__ ((aligned(16))); +} nvidia_stack_t; + +/* + * TODO: Remove once all UNIX layers have been converted to use nvidia_stack_t + */ +typedef nvidia_stack_t nv_stack_t; + +typedef struct nv_file_private_t nv_file_private_t; + +/* + * this is a wrapper for unix events + * unlike the events that will be returned to clients, this includes + * kernel-specific data, such as file pointer, etc.. + */ +typedef struct nv_event_s +{ + NvHandle hParent; + NvHandle hObject; + NvU32 index; + NvU32 info32; + NvU16 info16; + nv_file_private_t *nvfp; /* per file-descriptor data pointer */ + NvU32 fd; + NvBool active; /* whether the event should be signaled */ + NvU32 refcount; /* count of associated RM events */ + struct nv_event_s *next; +} nv_event_t; + +typedef struct nv_kern_mapping_s +{ + void *addr; + NvU64 size; + NvU32 modeFlag; + struct nv_kern_mapping_s *next; +} nv_kern_mapping_t; + +typedef struct nv_usermap_access_params_s +{ + NvU64 addr; + NvU64 size; + NvU64 offset; + NvU64 *page_array; + NvU64 num_pages; + NvU64 mmap_start; + NvU64 mmap_size; + NvU64 access_start; + NvU64 access_size; + NvU64 remap_prot_extra; + NvBool contig; + NvU32 caching; +} nv_usermap_access_params_t; + +/* + * It stores mapping context per mapping + */ +typedef struct nv_alloc_mapping_context_s { + void *alloc; + NvU64 page_index; + NvU64 *page_array; + NvU64 num_pages; + NvU64 mmap_start; + NvU64 mmap_size; + NvU64 access_start; + NvU64 access_size; + NvU64 remap_prot_extra; + NvU32 prot; + NvBool valid; + NvU32 caching; +} nv_alloc_mapping_context_t; + +typedef enum +{ + NV_SOC_IRQ_DISPLAY_TYPE = 0x1, + NV_SOC_IRQ_DPAUX_TYPE, + NV_SOC_IRQ_GPIO_TYPE, + NV_SOC_IRQ_HDACODEC_TYPE, + NV_SOC_IRQ_INVALID_TYPE +} nv_soc_irq_type_t; + +/* + * It stores interrupt numbers and interrupt type and private data + */ +typedef struct nv_soc_irq_info_s { + NvU32 irq_num; + nv_soc_irq_type_t irq_type; + NvBool bh_pending; + union { + NvU32 gpio_num; + NvU32 dpaux_instance; + } irq_data; +} nv_soc_irq_info_t; + +#define NV_MAX_SOC_IRQS 6 +#define NV_MAX_DPAUX_NUM_DEVICES 4 +#define NV_MAX_SOC_DPAUX_NUM_DEVICES 2 // From SOC_DEV_MAPPING + +#define NV_IGPU_LEGACY_STALL_IRQ 70 +#define NV_IGPU_MAX_STALL_IRQS 3 +#define NV_IGPU_MAX_NONSTALL_IRQS 1 +/* + * per device state + */ + +/* DMA-capable device data, defined by kernel interface layer */ +typedef struct nv_dma_device nv_dma_device_t; + +typedef struct nv_state_t +{ + void *priv; /* private data */ + void *os_state; /* os-specific device state */ + + int flags; + + /* PCI config info */ + nv_pci_info_t pci_info; + NvU16 subsystem_id; + NvU16 subsystem_vendor; + NvU32 gpu_id; + NvU32 iovaspace_id; + struct + { + NvBool valid; + NvU8 uuid[GPU_UUID_LEN]; + } nv_uuid_cache; + void *handle; + + NvU32 pci_cfg_space[NVRM_PCICFG_NUM_DWORDS]; + + /* physical characteristics */ + nv_aperture_t bars[NV_GPU_NUM_BARS]; + nv_aperture_t *regs; + nv_aperture_t *dpaux[NV_MAX_DPAUX_NUM_DEVICES]; + nv_aperture_t *hdacodec_regs; + nv_aperture_t *mipical_regs; + nv_aperture_t *fb, ud; + nv_aperture_t *simregs; + nv_aperture_t *emc_regs; + + NvU32 num_dpaux_instance; + NvU32 interrupt_line; + NvU32 dpaux_irqs[NV_MAX_DPAUX_NUM_DEVICES]; + nv_soc_irq_info_t soc_irq_info[NV_MAX_SOC_IRQS]; + NvS32 current_soc_irq; + NvU32 num_soc_irqs; + NvU32 hdacodec_irq; + NvU8 *soc_dcb_blob; + NvU32 soc_dcb_size; + NvU32 disp_sw_soc_chip_id; + + NvU32 igpu_stall_irq[NV_IGPU_MAX_STALL_IRQS]; + NvU32 igpu_nonstall_irq; + NvU32 num_stall_irqs; + NvU64 dma_mask; + + NvBool primary_vga; + + NvU32 sim_env; + + NvU32 rc_timer_enabled; + + /* list of events allocated for this device */ + nv_event_t *event_list; + + /* lock to protect event_list */ + void *event_spinlock; + + nv_kern_mapping_t *kern_mappings; + + /* Kernel interface DMA device data */ + nv_dma_device_t *dma_dev; + nv_dma_device_t *niso_dma_dev; + + /* + * Per-GPU queue. The actual queue object is usually allocated in the + * arch-specific parent structure (e.g. nv_linux_state_t), and this + * pointer just points to it. + */ + struct os_work_queue *queue; + + /* For loading RM as a firmware (DCE or GSP) client */ + NvBool request_firmware; /* request firmware from the OS */ + NvBool request_fw_client_rm; /* attempt to init RM as FW a client */ + NvBool allow_fallback_to_monolithic_rm; /* allow fallback to monolithic RM if FW client RM doesn't work out */ + NvBool enable_firmware_logs; /* attempt to enable firmware log decoding/printing */ + + /* Variable to track, if nvidia_remove is called */ + NvBool removed; + + NvBool console_device; + + /* Variable to track, if GPU is external GPU */ + NvBool is_external_gpu; + + /* Variable to track, if regkey PreserveVideoMemoryAllocations is set */ + NvBool preserve_vidmem_allocations; + + /* Variable to force allocation of 32-bit addressable memory */ + NvBool force_dma32_alloc; + + /* PCI power state should be D0 during system suspend */ + NvBool d0_state_in_suspend; + + /* Current cyclestats client and context */ + NvU32 profiler_owner; + void *profiler_context; + + /* + * RMAPI objects to use in the OS layer to talk to core RM. + * + * Note that we only need to store one subdevice handle: in SLI, we will + * have a separate nv_state_t per physical GPU. + */ + struct { + NvHandle hClient; + NvHandle hDevice; + NvHandle hSubDevice; + NvHandle hI2C; + NvHandle hDisp; + } rmapi; + + /* Bool to check if ISO iommu enabled */ + NvBool iso_iommu_present; + + /* Bool to check if dma-buf is supported */ + NvBool dma_buf_supported; + + NvBool printed_openrm_enable_unsupported_gpus_error; + + /* Check if NVPCF DSM function is implemented under NVPCF or GPU device scope */ + NvBool nvpcf_dsm_in_gpu_scope; + + /* Bool to check if the device received a shutdown notification */ + NvBool is_shutdown; + +} nv_state_t; + +// These define need to be in sync with defines in system.h +#define OS_TYPE_LINUX 0x1 +#define OS_TYPE_FREEBSD 0x2 +#define OS_TYPE_SUNOS 0x3 +#define OS_TYPE_VMWARE 0x4 + +#define NVFP_TYPE_NONE 0x0 +#define NVFP_TYPE_REFCOUNTED 0x1 +#define NVFP_TYPE_REGISTERED 0x2 + +struct nv_file_private_t +{ + NvHandle *handles; + NvU16 maxHandles; + NvU32 deviceInstance; + NvU8 metadata[64]; + + nv_file_private_t *ctl_nvfp; + void *ctl_nvfp_priv; + NvU32 register_or_refcount; +}; + +// Forward define the gpu ops structures +typedef struct gpuSession *nvgpuSessionHandle_t; +typedef struct gpuDevice *nvgpuDeviceHandle_t; +typedef struct gpuAddressSpace *nvgpuAddressSpaceHandle_t; +typedef struct gpuChannel *nvgpuChannelHandle_t; +typedef struct UvmGpuChannelInfo_tag *nvgpuChannelInfo_t; +typedef struct UvmGpuChannelAllocParams_tag nvgpuChannelAllocParams_t; +typedef struct UvmGpuCaps_tag *nvgpuCaps_t; +typedef struct UvmGpuCopyEnginesCaps_tag *nvgpuCesCaps_t; +typedef struct UvmGpuAddressSpaceInfo_tag *nvgpuAddressSpaceInfo_t; +typedef struct UvmGpuAllocInfo_tag *nvgpuAllocInfo_t; +typedef struct UvmGpuP2PCapsParams_tag *nvgpuP2PCapsParams_t; +typedef struct UvmGpuFbInfo_tag *nvgpuFbInfo_t; +typedef struct UvmGpuEccInfo_tag *nvgpuEccInfo_t; +typedef struct UvmGpuFaultInfo_tag *nvgpuFaultInfo_t; +typedef struct UvmGpuAccessCntrInfo_tag *nvgpuAccessCntrInfo_t; +typedef struct UvmGpuAccessCntrConfig_tag *nvgpuAccessCntrConfig_t; +typedef struct UvmGpuInfo_tag nvgpuInfo_t; +typedef struct UvmGpuClientInfo_tag nvgpuClientInfo_t; +typedef struct UvmPmaAllocationOptions_tag *nvgpuPmaAllocationOptions_t; +typedef struct UvmPmaStatistics_tag *nvgpuPmaStatistics_t; +typedef struct UvmGpuMemoryInfo_tag *nvgpuMemoryInfo_t; +typedef struct UvmGpuExternalMappingInfo_tag *nvgpuExternalMappingInfo_t; +typedef struct UvmGpuChannelResourceInfo_tag *nvgpuChannelResourceInfo_t; +typedef struct UvmGpuChannelInstanceInfo_tag *nvgpuChannelInstanceInfo_t; +typedef struct UvmGpuChannelResourceBindParams_tag *nvgpuChannelResourceBindParams_t; +typedef struct UvmGpuPagingChannelAllocParams_tag nvgpuPagingChannelAllocParams_t; +typedef struct UvmGpuPagingChannel_tag *nvgpuPagingChannelHandle_t; +typedef struct UvmGpuPagingChannelInfo_tag *nvgpuPagingChannelInfo_t; +typedef enum UvmPmaGpuMemoryType_tag nvgpuGpuMemoryType_t; +typedef NV_STATUS (*nvPmaEvictPagesCallback)(void *, NvU32, NvU64 *, NvU32, NvU64, NvU64, nvgpuGpuMemoryType_t); +typedef NV_STATUS (*nvPmaEvictRangeCallback)(void *, NvU64, NvU64, nvgpuGpuMemoryType_t); + +/* + * flags + */ + +#define NV_FLAG_OPEN 0x0001 +#define NV_FLAG_EXCLUDE 0x0002 +#define NV_FLAG_CONTROL 0x0004 +// Unused 0x0008 +#define NV_FLAG_SOC_DISPLAY 0x0010 +#define NV_FLAG_USES_MSI 0x0020 +#define NV_FLAG_USES_MSIX 0x0040 +#define NV_FLAG_PASSTHRU 0x0080 +#define NV_FLAG_SUSPENDED 0x0100 +#define NV_FLAG_SOC_IGPU 0x0200 +// Unused 0x0400 +#define NV_FLAG_PERSISTENT_SW_STATE 0x0800 +#define NV_FLAG_IN_RECOVERY 0x1000 +// Unused 0x2000 +#define NV_FLAG_UNBIND_LOCK 0x4000 +/* To be set when GPU is not present on the bus, to help device teardown */ +#define NV_FLAG_IN_SURPRISE_REMOVAL 0x8000 + +typedef enum +{ + NV_PM_ACTION_HIBERNATE, + NV_PM_ACTION_STANDBY, + NV_PM_ACTION_RESUME +} nv_pm_action_t; + +typedef enum +{ + NV_PM_ACTION_DEPTH_DEFAULT, + NV_PM_ACTION_DEPTH_MODESET, + NV_PM_ACTION_DEPTH_UVM +} nv_pm_action_depth_t; + +typedef enum +{ + NV_DYNAMIC_PM_NEVER, + NV_DYNAMIC_PM_COARSE, + NV_DYNAMIC_PM_FINE +} nv_dynamic_power_mode_t; + +typedef enum +{ + NV_POWER_STATE_IN_HIBERNATE, + NV_POWER_STATE_IN_STANDBY, + NV_POWER_STATE_RUNNING +} nv_power_state_t; + +#define NV_PRIMARY_VGA(nv) ((nv)->primary_vga) + +#define NV_IS_CTL_DEVICE(nv) ((nv)->flags & NV_FLAG_CONTROL) +#define NV_IS_SOC_DISPLAY_DEVICE(nv) \ + ((nv)->flags & NV_FLAG_SOC_DISPLAY) + +#define NV_IS_SOC_IGPU_DEVICE(nv) \ + ((nv)->flags & NV_FLAG_SOC_IGPU) + +#define NV_IS_DEVICE_IN_SURPRISE_REMOVAL(nv) \ + (((nv)->flags & NV_FLAG_IN_SURPRISE_REMOVAL) != 0) + +#define NV_SOC_IS_ISO_IOMMU_PRESENT(nv) \ + ((nv)->iso_iommu_present) + +/* + * GPU add/remove events + */ +#define NV_SYSTEM_GPU_ADD_EVENT 0x9001 +#define NV_SYSTEM_GPU_REMOVE_EVENT 0x9002 + +/* + * NVIDIA ACPI sub-event IDs (event types) to be passed into + * to core NVIDIA driver for ACPI events. + */ +#define NV_SYSTEM_ACPI_EVENT_VALUE_DISPLAY_SWITCH_DEFAULT 0 +#define NV_SYSTEM_ACPI_EVENT_VALUE_DOCK_EVENT_UNDOCKED 0 +#define NV_SYSTEM_ACPI_EVENT_VALUE_DOCK_EVENT_DOCKED 1 + +#define NV_ACPI_NVIF_HANDLE_PRESENT 0x01 +#define NV_ACPI_DSM_HANDLE_PRESENT 0x02 +#define NV_ACPI_WMMX_HANDLE_PRESENT 0x04 + +#define NV_EVAL_ACPI_METHOD_NVIF 0x01 +#define NV_EVAL_ACPI_METHOD_WMMX 0x02 + +typedef enum { + NV_I2C_CMD_READ = 1, + NV_I2C_CMD_WRITE, + NV_I2C_CMD_SMBUS_READ, + NV_I2C_CMD_SMBUS_WRITE, + NV_I2C_CMD_SMBUS_QUICK_WRITE, + NV_I2C_CMD_SMBUS_QUICK_READ, + NV_I2C_CMD_SMBUS_BLOCK_READ, + NV_I2C_CMD_SMBUS_BLOCK_WRITE, + NV_I2C_CMD_BLOCK_READ, + NV_I2C_CMD_BLOCK_WRITE +} nv_i2c_cmd_t; + +// Flags needed by OSAllocPagesNode +#define NV_ALLOC_PAGES_NODE_NONE 0x0 +#define NV_ALLOC_PAGES_NODE_SKIP_RECLAIM 0x1 + +/* +** where we hide our nv_state_t * ... +*/ +#define NV_SET_NV_STATE(pgpu,p) ((pgpu)->pOsGpuInfo = (p)) +#define NV_GET_NV_STATE(pGpu) \ + (nv_state_t *)((pGpu) ? (pGpu)->pOsGpuInfo : NULL) + +static inline NvBool IS_REG_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length) +{ + return ((offset >= nv->regs->cpu_address) && + ((offset + (length - 1)) >= offset) && + ((offset + (length - 1)) <= (nv->regs->cpu_address + (nv->regs->size - 1)))); +} + +static inline NvBool IS_FB_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length) +{ + return ((nv->fb) && (offset >= nv->fb->cpu_address) && + ((offset + (length - 1)) >= offset) && + ((offset + (length - 1)) <= (nv->fb->cpu_address + (nv->fb->size - 1)))); +} + +static inline NvBool IS_UD_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length) +{ + return ((nv->ud.cpu_address != 0) && (nv->ud.size != 0) && + (offset >= nv->ud.cpu_address) && + ((offset + (length - 1)) >= offset) && + ((offset + (length - 1)) <= (nv->ud.cpu_address + (nv->ud.size - 1)))); +} + +static inline NvBool IS_IMEM_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length) +{ + return ((nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address != 0) && + (nv->bars[NV_GPU_BAR_INDEX_IMEM].size != 0) && + (offset >= nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address) && + ((offset + (length - 1)) >= offset) && + ((offset + (length - 1)) <= (nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + + (nv->bars[NV_GPU_BAR_INDEX_IMEM].size - 1)))); +} + +#define NV_RM_MAX_MSIX_LINES 8 + +#define NV_MAX_ISR_DELAY_US 20000 +#define NV_MAX_ISR_DELAY_MS (NV_MAX_ISR_DELAY_US / 1000) + +#define NV_TIMERCMP(a, b, CMP) \ + (((a)->tv_sec == (b)->tv_sec) ? \ + ((a)->tv_usec CMP (b)->tv_usec) : ((a)->tv_sec CMP (b)->tv_sec)) + +#define NV_TIMERADD(a, b, result) \ + { \ + (result)->tv_sec = (a)->tv_sec + (b)->tv_sec; \ + (result)->tv_usec = (a)->tv_usec + (b)->tv_usec; \ + if ((result)->tv_usec >= 1000000) \ + { \ + ++(result)->tv_sec; \ + (result)->tv_usec -= 1000000; \ + } \ + } + +#define NV_TIMERSUB(a, b, result) \ + { \ + (result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \ + (result)->tv_usec = (a)->tv_usec - (b)->tv_usec; \ + if ((result)->tv_usec < 0) \ + { \ + --(result)->tv_sec; \ + (result)->tv_usec += 1000000; \ + } \ + } + +#define NV_TIMEVAL_TO_US(tv) ((NvU64)(tv).tv_sec * 1000000 + (tv).tv_usec) + +#ifndef NV_ALIGN_UP +#define NV_ALIGN_UP(v,g) (((v) + ((g) - 1)) & ~((g) - 1)) +#endif +#ifndef NV_ALIGN_DOWN +#define NV_ALIGN_DOWN(v,g) ((v) & ~((g) - 1)) +#endif + +/* + * driver internal interfaces + */ + +/* + * --------------------------------------------------------------------------- + * + * Function prototypes for UNIX specific OS interface. + * + * --------------------------------------------------------------------------- + */ + +NvU32 NV_API_CALL nv_get_dev_minor (nv_state_t *); +void* NV_API_CALL nv_alloc_kernel_mapping (nv_state_t *, void *, NvU64, NvU32, NvU64, void **); +NV_STATUS NV_API_CALL nv_free_kernel_mapping (nv_state_t *, void *, void *, void *); +NV_STATUS NV_API_CALL nv_alloc_user_mapping (nv_state_t *, void *, NvU64, NvU32, NvU64, NvU32, NvU64 *, void **); +NV_STATUS NV_API_CALL nv_free_user_mapping (nv_state_t *, void *, NvU64, void *); +NV_STATUS NV_API_CALL nv_add_mapping_context_to_file (nv_state_t *, nv_usermap_access_params_t*, NvU32, void *, NvU64, NvU32); + +NvU64 NV_API_CALL nv_get_kern_phys_address (NvU64); +NvU64 NV_API_CALL nv_get_user_phys_address (NvU64); +nv_state_t* NV_API_CALL nv_get_adapter_state (NvU32, NvU8, NvU8); +nv_state_t* NV_API_CALL nv_get_ctl_state (void); + +void NV_API_CALL nv_set_dma_address_size (nv_state_t *, NvU32 ); + +NV_STATUS NV_API_CALL nv_alias_pages (nv_state_t *, NvU32, NvU32, NvU32, NvU64, NvU64 *, void **); +NV_STATUS NV_API_CALL nv_alloc_pages (nv_state_t *, NvU32, NvBool, NvU32, NvBool, NvBool, NvU64 *, void **); +NV_STATUS NV_API_CALL nv_free_pages (nv_state_t *, NvU32, NvBool, NvU32, void *); + +NV_STATUS NV_API_CALL nv_register_user_pages (nv_state_t *, NvU64, NvU64 *, void *, void **); +void NV_API_CALL nv_unregister_user_pages (nv_state_t *, NvU64, void **, void **); + +NV_STATUS NV_API_CALL nv_register_peer_io_mem (nv_state_t *, NvU64 *, NvU64, void **); +void NV_API_CALL nv_unregister_peer_io_mem(nv_state_t *, void *); + +struct sg_table; + +NV_STATUS NV_API_CALL nv_register_sgt (nv_state_t *, NvU64 *, NvU64, NvU32, void **, struct sg_table *, void *); +void NV_API_CALL nv_unregister_sgt (nv_state_t *, struct sg_table **, void **, void *); +NV_STATUS NV_API_CALL nv_register_phys_pages (nv_state_t *, NvU64 *, NvU64, NvU32, void **); +void NV_API_CALL nv_unregister_phys_pages (nv_state_t *, void *); + +NV_STATUS NV_API_CALL nv_dma_map_sgt (nv_dma_device_t *, NvU64, NvU64 *, NvU32, void **); +NV_STATUS NV_API_CALL nv_dma_map_pages (nv_dma_device_t *, NvU64, NvU64 *, NvBool, NvU32, void **); +NV_STATUS NV_API_CALL nv_dma_unmap_pages (nv_dma_device_t *, NvU64, NvU64 *, void **); + +NV_STATUS NV_API_CALL nv_dma_map_alloc (nv_dma_device_t *, NvU64, NvU64 *, NvBool, void **); +NV_STATUS NV_API_CALL nv_dma_unmap_alloc (nv_dma_device_t *, NvU64, NvU64 *, void **); + +NV_STATUS NV_API_CALL nv_dma_map_peer (nv_dma_device_t *, nv_dma_device_t *, NvU8, NvU64, NvU64 *); +void NV_API_CALL nv_dma_unmap_peer (nv_dma_device_t *, NvU64, NvU64); + +NV_STATUS NV_API_CALL nv_dma_map_mmio (nv_dma_device_t *, NvU64, NvU64 *); +void NV_API_CALL nv_dma_unmap_mmio (nv_dma_device_t *, NvU64, NvU64); + +void NV_API_CALL nv_dma_cache_invalidate (nv_dma_device_t *, void *); +void NV_API_CALL nv_dma_enable_nvlink (nv_dma_device_t *); + +NvS32 NV_API_CALL nv_start_rc_timer (nv_state_t *); +NvS32 NV_API_CALL nv_stop_rc_timer (nv_state_t *); + +void NV_API_CALL nv_post_event (nv_event_t *, NvHandle, NvU32, NvU32, NvU16, NvBool); +NvS32 NV_API_CALL nv_get_event (nv_file_private_t *, nv_event_t *, NvU32 *); + +void* NV_API_CALL nv_i2c_add_adapter (nv_state_t *, NvU32); +void NV_API_CALL nv_i2c_del_adapter (nv_state_t *, void *); + +void NV_API_CALL nv_acpi_methods_init (NvU32 *); +void NV_API_CALL nv_acpi_methods_uninit (void); + +NV_STATUS NV_API_CALL nv_acpi_method (NvU32, NvU32, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *); +NV_STATUS NV_API_CALL nv_acpi_dsm_method (nv_state_t *, NvU8 *, NvU32, NvBool, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *); +NV_STATUS NV_API_CALL nv_acpi_ddc_method (nv_state_t *, void *, NvU32 *, NvBool); +NV_STATUS NV_API_CALL nv_acpi_dod_method (nv_state_t *, NvU32 *, NvU32 *); +NV_STATUS NV_API_CALL nv_acpi_rom_method (nv_state_t *, NvU32 *, NvU32 *); +NV_STATUS NV_API_CALL nv_acpi_get_powersource (NvU32 *); +NvBool NV_API_CALL nv_acpi_is_battery_present(void); + +NV_STATUS NV_API_CALL nv_acpi_mux_method (nv_state_t *, NvU32 *, NvU32, const char *); + +NV_STATUS NV_API_CALL nv_log_error (nv_state_t *, NvU32, const char *, va_list); + +NvU64 NV_API_CALL nv_get_dma_start_address (nv_state_t *); +NV_STATUS NV_API_CALL nv_set_primary_vga_status(nv_state_t *); +NV_STATUS NV_API_CALL nv_pci_trigger_recovery (nv_state_t *); +NvBool NV_API_CALL nv_requires_dma_remap (nv_state_t *); + +NvBool NV_API_CALL nv_is_rm_firmware_active(nv_state_t *); +const void*NV_API_CALL nv_get_firmware(nv_state_t *, nv_firmware_type_t, nv_firmware_chip_family_t, const void **, NvU32 *); +void NV_API_CALL nv_put_firmware(const void *); + +nv_file_private_t* NV_API_CALL nv_get_file_private(NvS32, NvBool, void **); +void NV_API_CALL nv_put_file_private(void *); + +NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU32 *, NvS32 *); + +NV_STATUS NV_API_CALL nv_get_ibmnpu_genreg_info(nv_state_t *, NvU64 *, NvU64 *, void**); +NV_STATUS NV_API_CALL nv_get_ibmnpu_relaxed_ordering_mode(nv_state_t *nv, NvBool *mode); + +void NV_API_CALL nv_wait_for_ibmnpu_rsync(nv_state_t *nv); + +void NV_API_CALL nv_ibmnpu_cache_flush_range(nv_state_t *nv, NvU64, NvU64); + +void NV_API_CALL nv_p2p_free_platform_data(void *data); + +#if defined(NVCPU_PPC64LE) +NV_STATUS NV_API_CALL nv_get_nvlink_line_rate (nv_state_t *, NvU32 *); +#endif + +NV_STATUS NV_API_CALL nv_revoke_gpu_mappings (nv_state_t *); +void NV_API_CALL nv_acquire_mmap_lock (nv_state_t *); +void NV_API_CALL nv_release_mmap_lock (nv_state_t *); +NvBool NV_API_CALL nv_get_all_mappings_revoked_locked (nv_state_t *); +void NV_API_CALL nv_set_safe_to_mmap_locked (nv_state_t *, NvBool); + +NV_STATUS NV_API_CALL nv_indicate_idle (nv_state_t *); +NV_STATUS NV_API_CALL nv_indicate_not_idle (nv_state_t *); +void NV_API_CALL nv_idle_holdoff (nv_state_t *); + +NvBool NV_API_CALL nv_dynamic_power_available (nv_state_t *); +void NV_API_CALL nv_audio_dynamic_power (nv_state_t *); + +void NV_API_CALL nv_control_soc_irqs (nv_state_t *, NvBool bEnable); +NV_STATUS NV_API_CALL nv_get_current_irq_priv_data(nv_state_t *, NvU32 *); + +NV_STATUS NV_API_CALL nv_acquire_fabric_mgmt_cap (int, int*); +int NV_API_CALL nv_cap_drv_init(void); +void NV_API_CALL nv_cap_drv_exit(void); +NvBool NV_API_CALL nv_is_gpu_accessible(nv_state_t *); +NvBool NV_API_CALL nv_match_gpu_os_info(nv_state_t *, void *); + +NvU32 NV_API_CALL nv_get_os_type(void); + +void NV_API_CALL nv_get_updated_emu_seg(NvU32 *start, NvU32 *end); +struct dma_buf; +typedef struct nv_dma_buf nv_dma_buf_t; +struct drm_gem_object; + +NV_STATUS NV_API_CALL nv_dma_import_sgt (nv_dma_device_t *, struct sg_table *, struct drm_gem_object *); +void NV_API_CALL nv_dma_release_sgt(struct sg_table *, struct drm_gem_object *); +NV_STATUS NV_API_CALL nv_dma_import_dma_buf (nv_dma_device_t *, struct dma_buf *, NvU32 *, void **, struct sg_table **, nv_dma_buf_t **); +NV_STATUS NV_API_CALL nv_dma_import_from_fd (nv_dma_device_t *, NvS32, NvU32 *, void **, struct sg_table **, nv_dma_buf_t **); +void NV_API_CALL nv_dma_release_dma_buf (void *, nv_dma_buf_t *); + +void NV_API_CALL nv_schedule_uvm_isr (nv_state_t *); + +NvBool NV_API_CALL nv_platform_supports_s0ix (void); +NvBool NV_API_CALL nv_s2idle_pm_configured (void); + +NvBool NV_API_CALL nv_is_chassis_notebook (void); +void NV_API_CALL nv_allow_runtime_suspend (nv_state_t *nv); +void NV_API_CALL nv_disallow_runtime_suspend (nv_state_t *nv); + +typedef void (*nvTegraDceClientIpcCallback)(NvU32, NvU32, NvU32, void *, void *); + +NV_STATUS NV_API_CALL nv_get_num_phys_pages (void *, NvU32 *); +NV_STATUS NV_API_CALL nv_get_phys_pages (void *, void *, NvU32 *); + +/* + * --------------------------------------------------------------------------- + * + * Function prototypes for Resource Manager interface. + * + * --------------------------------------------------------------------------- + */ + +NvBool NV_API_CALL rm_init_rm (nvidia_stack_t *); +void NV_API_CALL rm_shutdown_rm (nvidia_stack_t *); +NvBool NV_API_CALL rm_init_private_state (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_free_private_state (nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_init_adapter (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_disable_adapter (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_shutdown_adapter (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_exclude_adapter (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_acquire_api_lock (nvidia_stack_t *); +NV_STATUS NV_API_CALL rm_release_api_lock (nvidia_stack_t *); +NV_STATUS NV_API_CALL rm_acquire_gpu_lock (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_release_gpu_lock (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_acquire_all_gpus_lock (nvidia_stack_t *); +NV_STATUS NV_API_CALL rm_release_all_gpus_lock (nvidia_stack_t *); +NV_STATUS NV_API_CALL rm_ioctl (nvidia_stack_t *, nv_state_t *, nv_file_private_t *, NvU32, void *, NvU32); +NvBool NV_API_CALL rm_isr (nvidia_stack_t *, nv_state_t *, NvU32 *); +void NV_API_CALL rm_isr_bh (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_isr_bh_unlocked (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_power_management (nvidia_stack_t *, nv_state_t *, nv_pm_action_t); +NV_STATUS NV_API_CALL rm_stop_user_channels (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_restart_user_channels (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_save_low_res_mode (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_get_vbios_version (nvidia_stack_t *, nv_state_t *, char *); +char* NV_API_CALL rm_get_gpu_uuid (nvidia_stack_t *, nv_state_t *); +const NvU8* NV_API_CALL rm_get_gpu_uuid_raw (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_set_rm_firmware_requested(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_get_firmware_version (nvidia_stack_t *, nv_state_t *, char *, NvLength); +void NV_API_CALL rm_cleanup_file_private (nvidia_stack_t *, nv_state_t *, nv_file_private_t *); +void NV_API_CALL rm_unbind_lock (nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_read_registry_dword (nvidia_stack_t *, nv_state_t *, const char *, NvU32 *); +NV_STATUS NV_API_CALL rm_write_registry_dword (nvidia_stack_t *, nv_state_t *, const char *, NvU32); +NV_STATUS NV_API_CALL rm_write_registry_binary (nvidia_stack_t *, nv_state_t *, const char *, NvU8 *, NvU32); +NV_STATUS NV_API_CALL rm_write_registry_string (nvidia_stack_t *, nv_state_t *, const char *, const char *, NvU32); +void NV_API_CALL rm_parse_option_string (nvidia_stack_t *, const char *); +char* NV_API_CALL rm_remove_spaces (const char *); +char* NV_API_CALL rm_string_token (char **, const char); + +NV_STATUS NV_API_CALL rm_run_rc_callback (nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_execute_work_item (nvidia_stack_t *, void *); +const char* NV_API_CALL rm_get_device_name (NvU16, NvU16, NvU16); + +NV_STATUS NV_API_CALL rm_is_supported_device (nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_is_supported_pci_device(NvU8 pci_class, + NvU8 pci_subclass, + NvU16 vendor, + NvU16 device, + NvU16 subsystem_vendor, + NvU16 subsystem_device, + NvBool print_legacy_warning); + +void NV_API_CALL rm_i2c_remove_adapters (nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_i2c_is_smbus_capable (nvidia_stack_t *, nv_state_t *, void *); +NV_STATUS NV_API_CALL rm_i2c_transfer (nvidia_stack_t *, nv_state_t *, void *, nv_i2c_cmd_t, NvU8, NvU8, NvU32, NvU8 *); + +NV_STATUS NV_API_CALL rm_perform_version_check (nvidia_stack_t *, void *, NvU32); + +void NV_API_CALL rm_power_source_change_event (nvidia_stack_t *, NvU32); + +void NV_API_CALL rm_disable_gpu_state_persistence (nvidia_stack_t *sp, nv_state_t *); +NV_STATUS NV_API_CALL rm_p2p_init_mapping (nvidia_stack_t *, NvU64, NvU64 *, NvU64 *, NvU64 *, NvU64 *, NvU64, NvU64, NvU64, NvU64, void (*)(void *), void *); +NV_STATUS NV_API_CALL rm_p2p_destroy_mapping (nvidia_stack_t *, NvU64); +NV_STATUS NV_API_CALL rm_p2p_get_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, NvU64, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU8 **, void *); +NV_STATUS NV_API_CALL rm_p2p_get_gpu_info (nvidia_stack_t *, NvU64, NvU64, NvU8 **, void **); +NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent (nvidia_stack_t *, NvU64, NvU64, void **, NvU64 *, NvU32 *, void *, void *); +NV_STATUS NV_API_CALL rm_p2p_register_callback (nvidia_stack_t *, NvU64, NvU64, NvU64, void *, void (*)(void *), void *); +NV_STATUS NV_API_CALL rm_p2p_put_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, void *); +NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *); +NV_STATUS NV_API_CALL rm_p2p_dma_map_pages (nvidia_stack_t *, nv_dma_device_t *, NvU8 *, NvU32, NvU32, NvU64 *, void **); +NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, NvHandle, void *, NvHandle, NvU64, NvU64, NvHandle *); +void NV_API_CALL rm_dma_buf_undup_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle); +NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvU64, NvU64, NvU64 *); +NV_STATUS NV_API_CALL rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvU64, NvU64); +NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle *, NvHandle *, NvHandle *, void **); +void NV_API_CALL rm_dma_buf_put_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, void *); +NV_STATUS NV_API_CALL rm_log_gpu_crash (nv_stack_t *, nv_state_t *); + +void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd); +NvBool NV_API_CALL rm_get_device_remove_flag(nvidia_stack_t *sp, NvU32 gpu_id); +NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults(nvidia_stack_t *, nv_state_t *, NvU32 *); +NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults_unlocked(nvidia_stack_t *, nv_state_t *, NvU32 *); +NV_STATUS NV_API_CALL rm_gpu_handle_mmu_faults(nvidia_stack_t *, nv_state_t *, NvU32 *); +NvBool NV_API_CALL rm_gpu_need_4k_page_isolation(nv_state_t *); +NvBool NV_API_CALL rm_is_chipset_io_coherent(nv_stack_t *); +NvBool NV_API_CALL rm_init_event_locks(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_destroy_event_locks(nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_get_gpu_numa_info(nvidia_stack_t *, nv_state_t *, NvS32 *, NvU64 *, NvU64 *, NvU64 *, NvU32 *); +NV_STATUS NV_API_CALL rm_gpu_numa_online(nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_gpu_numa_offline(nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_is_device_sequestered(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_check_for_gpu_surprise_removal(nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_set_external_kernel_client_count(nvidia_stack_t *, nv_state_t *, NvBool); +NV_STATUS NV_API_CALL rm_schedule_gpu_wakeup(nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_is_iommu_needed_for_sriov(nvidia_stack_t *, nv_state_t *); +NvBool NV_API_CALL rm_disable_iomap_wc(void); + +void NV_API_CALL rm_init_dynamic_power_management(nvidia_stack_t *, nv_state_t *, NvBool); +void NV_API_CALL rm_cleanup_dynamic_power_management(nvidia_stack_t *, nv_state_t *); +void NV_API_CALL rm_enable_dynamic_power_management(nvidia_stack_t *, nv_state_t *); +NV_STATUS NV_API_CALL rm_ref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t); +void NV_API_CALL rm_unref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t); +NV_STATUS NV_API_CALL rm_transition_dynamic_power(nvidia_stack_t *, nv_state_t *, NvBool); +const char* NV_API_CALL rm_get_vidmem_power_status(nvidia_stack_t *, nv_state_t *); +const char* NV_API_CALL rm_get_dynamic_power_management_status(nvidia_stack_t *, nv_state_t *); +const char* NV_API_CALL rm_get_gpu_gcx_support(nvidia_stack_t *, nv_state_t *, NvBool); + +void NV_API_CALL rm_acpi_notify(nvidia_stack_t *, nv_state_t *, NvU32); + +NvBool NV_API_CALL rm_is_altstack_in_use(void); + +/* vGPU VFIO specific functions */ +NV_STATUS NV_API_CALL nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *, NvU32, NvBool *); +NV_STATUS NV_API_CALL nv_vgpu_delete(nvidia_stack_t *, const NvU8 *, NvU16); +NV_STATUS NV_API_CALL nv_vgpu_get_type_ids(nvidia_stack_t *, nv_state_t *, NvU32 *, NvU32 *, NvBool, NvU8, NvBool); +NV_STATUS NV_API_CALL nv_vgpu_get_type_info(nvidia_stack_t *, nv_state_t *, NvU32, char *, int, NvU8); +NV_STATUS NV_API_CALL nv_vgpu_get_bar_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU64 *, NvU32, void *); +NV_STATUS NV_API_CALL nv_vgpu_start(nvidia_stack_t *, const NvU8 *, void *, NvS32 *, NvU8 *, NvU32); +NV_STATUS NV_API_CALL nv_vgpu_get_sparse_mmap(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU64 **, NvU64 **, NvU32 *); +NV_STATUS NV_API_CALL nv_vgpu_process_vf_info(nvidia_stack_t *, nv_state_t *, NvU8, NvU32, NvU8, NvU8, NvU8, NvBool, void *); +NV_STATUS NV_API_CALL nv_vgpu_update_request(nvidia_stack_t *, const NvU8 *, NvU32, NvU64 *, NvU64 *, const char *); +NV_STATUS NV_API_CALL nv_gpu_bind_event(nvidia_stack_t *); + +NV_STATUS NV_API_CALL nv_get_usermap_access_params(nv_state_t*, nv_usermap_access_params_t*); +nv_soc_irq_type_t NV_API_CALL nv_get_current_irq_type(nv_state_t*); +void NV_API_CALL nv_flush_coherent_cpu_cache_range(nv_state_t *nv, NvU64 cpu_virtual, NvU64 size); + +/* Callbacks should occur roughly every 10ms. */ +#define NV_SNAPSHOT_TIMER_HZ 100 +void NV_API_CALL nv_start_snapshot_timer(void (*snapshot_callback)(void *context)); +void NV_API_CALL nv_flush_snapshot_timer(void); +void NV_API_CALL nv_stop_snapshot_timer(void); + +static inline const NvU8 *nv_get_cached_uuid(nv_state_t *nv) +{ + return nv->nv_uuid_cache.valid ? nv->nv_uuid_cache.uuid : NULL; +} + +/* nano second resolution timer callback structure */ +typedef struct nv_nano_timer nv_nano_timer_t; + +/* nano timer functions */ +void NV_API_CALL nv_create_nano_timer(nv_state_t *, void *pTmrEvent, nv_nano_timer_t **); +void NV_API_CALL nv_start_nano_timer(nv_state_t *nv, nv_nano_timer_t *, NvU64 timens); +NV_STATUS NV_API_CALL rm_run_nano_timer_callback(nvidia_stack_t *, nv_state_t *, void *pTmrEvent); +void NV_API_CALL nv_cancel_nano_timer(nv_state_t *, nv_nano_timer_t *); +void NV_API_CALL nv_destroy_nano_timer(nv_state_t *nv, nv_nano_timer_t *); + +#if defined(NVCPU_X86_64) + +static inline NvU64 nv_rdtsc(void) +{ + NvU64 val; + __asm__ __volatile__ ("rdtsc \t\n" + "shlq $0x20,%%rdx \t\n" + "orq %%rdx,%%rax \t\n" + : "=A" (val)); + return val; +} + +#endif + +#endif /* NVRM */ + +static inline int nv_count_bits(NvU64 word) +{ + NvU64 bits; + + bits = (word & 0x5555555555555555ULL) + ((word >> 1) & 0x5555555555555555ULL); + bits = (bits & 0x3333333333333333ULL) + ((bits >> 2) & 0x3333333333333333ULL); + bits = (bits & 0x0f0f0f0f0f0f0f0fULL) + ((bits >> 4) & 0x0f0f0f0f0f0f0f0fULL); + bits = (bits & 0x00ff00ff00ff00ffULL) + ((bits >> 8) & 0x00ff00ff00ff00ffULL); + bits = (bits & 0x0000ffff0000ffffULL) + ((bits >> 16) & 0x0000ffff0000ffffULL); + bits = (bits & 0x00000000ffffffffULL) + ((bits >> 32) & 0x00000000ffffffffULL); + + return (int)(bits); +} + +#endif diff --git a/userspace/sdks/nv525/nvalloc/unix/include/nv_escape.h b/userspace/sdks/nv525/nvalloc/unix/include/nv_escape.h new file mode 100644 index 0000000..629af5a --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/nv_escape.h @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NV_ESCAPE_H_INCLUDED +#define NV_ESCAPE_H_INCLUDED + +#define NV_ESC_RM_ALLOC_MEMORY 0x27 +#define NV_ESC_RM_ALLOC_OBJECT 0x28 +#define NV_ESC_RM_FREE 0x29 +#define NV_ESC_RM_CONTROL 0x2A +#define NV_ESC_RM_ALLOC 0x2B +#define NV_ESC_RM_CONFIG_GET 0x32 +#define NV_ESC_RM_CONFIG_SET 0x33 +#define NV_ESC_RM_DUP_OBJECT 0x34 +#define NV_ESC_RM_SHARE 0x35 +#define NV_ESC_RM_CONFIG_GET_EX 0x37 +#define NV_ESC_RM_CONFIG_SET_EX 0x38 +#define NV_ESC_RM_I2C_ACCESS 0x39 +#define NV_ESC_RM_IDLE_CHANNELS 0x41 +#define NV_ESC_RM_VID_HEAP_CONTROL 0x4A +#define NV_ESC_RM_ACCESS_REGISTRY 0x4D +#define NV_ESC_RM_MAP_MEMORY 0x4E +#define NV_ESC_RM_UNMAP_MEMORY 0x4F +#define NV_ESC_RM_GET_EVENT_DATA 0x52 +#define NV_ESC_RM_ALLOC_CONTEXT_DMA2 0x54 +#define NV_ESC_RM_ADD_VBLANK_CALLBACK 0x56 +#define NV_ESC_RM_MAP_MEMORY_DMA 0x57 +#define NV_ESC_RM_UNMAP_MEMORY_DMA 0x58 +#define NV_ESC_RM_BIND_CONTEXT_DMA 0x59 +#define NV_ESC_RM_EXPORT_OBJECT_TO_FD 0x5C +#define NV_ESC_RM_IMPORT_OBJECT_FROM_FD 0x5D +#define NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO 0x5E + +#endif // NV_ESCAPE_H_INCLUDED diff --git a/userspace/sdks/nv525/nvalloc/unix/include/os-interface.h b/userspace/sdks/nv525/nvalloc/unix/include/os-interface.h new file mode 100644 index 0000000..9ebeeed --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/os-interface.h @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +/* + * Os interface definitions needed by os-interface.c + */ + +#ifndef OS_INTERFACE_H +#define OS_INTERFACE_H + +/******************* Operating System Interface Routines *******************\ +* * +* Operating system wrapper functions used to abstract the OS. * +* * +\***************************************************************************/ + +#include <nvtypes.h> +#include <nvstatus.h> +#include "nv_stdarg.h" +#include <nv-kernel-interface-api.h> +#include <os/nv_memory_type.h> +#include <nv-caps.h> + + + +typedef struct +{ + NvU32 os_major_version; + NvU32 os_minor_version; + NvU32 os_build_number; + const char * os_build_version_str; + const char * os_build_date_plus_str; +}os_version_info; + +/* Each OS defines its own version of this opaque type */ +struct os_work_queue; + +/* Each OS defines its own version of this opaque type */ +typedef struct os_wait_queue os_wait_queue; + +/* + * --------------------------------------------------------------------------- + * + * Function prototypes for OS interface. + * + * --------------------------------------------------------------------------- + */ + +NvU64 NV_API_CALL os_get_num_phys_pages (void); +NV_STATUS NV_API_CALL os_alloc_mem (void **, NvU64); +void NV_API_CALL os_free_mem (void *); +NV_STATUS NV_API_CALL os_get_current_time (NvU32 *, NvU32 *); +NvU64 NV_API_CALL os_get_current_tick (void); +NvU64 NV_API_CALL os_get_current_tick_hr (void); +NvU64 NV_API_CALL os_get_tick_resolution (void); +NV_STATUS NV_API_CALL os_delay (NvU32); +NV_STATUS NV_API_CALL os_delay_us (NvU32); +NvU64 NV_API_CALL os_get_cpu_frequency (void); +NvU32 NV_API_CALL os_get_current_process (void); +void NV_API_CALL os_get_current_process_name (char *, NvU32); +NV_STATUS NV_API_CALL os_get_current_thread (NvU64 *); +char* NV_API_CALL os_string_copy (char *, const char *); +NvU32 NV_API_CALL os_string_length (const char *); +NvU32 NV_API_CALL os_strtoul (const char *, char **, NvU32); +NvS32 NV_API_CALL os_string_compare (const char *, const char *); +NvS32 NV_API_CALL os_snprintf (char *, NvU32, const char *, ...); +NvS32 NV_API_CALL os_vsnprintf (char *, NvU32, const char *, va_list); +void NV_API_CALL os_log_error (const char *, va_list); +void* NV_API_CALL os_mem_copy (void *, const void *, NvU32); +NV_STATUS NV_API_CALL os_memcpy_from_user (void *, const void *, NvU32); +NV_STATUS NV_API_CALL os_memcpy_to_user (void *, const void *, NvU32); +void* NV_API_CALL os_mem_set (void *, NvU8, NvU32); +NvS32 NV_API_CALL os_mem_cmp (const NvU8 *, const NvU8 *, NvU32); +void* NV_API_CALL os_pci_init_handle (NvU32, NvU8, NvU8, NvU8, NvU16 *, NvU16 *); +NV_STATUS NV_API_CALL os_pci_read_byte (void *, NvU32, NvU8 *); +NV_STATUS NV_API_CALL os_pci_read_word (void *, NvU32, NvU16 *); +NV_STATUS NV_API_CALL os_pci_read_dword (void *, NvU32, NvU32 *); +NV_STATUS NV_API_CALL os_pci_write_byte (void *, NvU32, NvU8); +NV_STATUS NV_API_CALL os_pci_write_word (void *, NvU32, NvU16); +NV_STATUS NV_API_CALL os_pci_write_dword (void *, NvU32, NvU32); +NvBool NV_API_CALL os_pci_remove_supported (void); +void NV_API_CALL os_pci_remove (void *); +void* NV_API_CALL os_map_kernel_space (NvU64, NvU64, NvU32); +void NV_API_CALL os_unmap_kernel_space (void *, NvU64); +void* NV_API_CALL os_map_user_space (NvU64, NvU64, NvU32, NvU32, void **); +void NV_API_CALL os_unmap_user_space (void *, NvU64, void *); +NV_STATUS NV_API_CALL os_flush_cpu_cache (void); +NV_STATUS NV_API_CALL os_flush_cpu_cache_all (void); +NV_STATUS NV_API_CALL os_flush_user_cache (void); +void NV_API_CALL os_flush_cpu_write_combine_buffer(void); +NvU8 NV_API_CALL os_io_read_byte (NvU32); +NvU16 NV_API_CALL os_io_read_word (NvU32); +NvU32 NV_API_CALL os_io_read_dword (NvU32); +void NV_API_CALL os_io_write_byte (NvU32, NvU8); +void NV_API_CALL os_io_write_word (NvU32, NvU16); +void NV_API_CALL os_io_write_dword (NvU32, NvU32); +NvBool NV_API_CALL os_is_administrator (void); +NvBool NV_API_CALL os_allow_priority_override (void); +void NV_API_CALL os_dbg_init (void); +void NV_API_CALL os_dbg_breakpoint (void); +void NV_API_CALL os_dbg_set_level (NvU32); +NvU32 NV_API_CALL os_get_cpu_count (void); +NvU32 NV_API_CALL os_get_cpu_number (void); +void NV_API_CALL os_disable_console_access (void); +void NV_API_CALL os_enable_console_access (void); +NV_STATUS NV_API_CALL os_registry_init (void); +NvU64 NV_API_CALL os_get_max_user_va (void); +NV_STATUS NV_API_CALL os_schedule (void); +NV_STATUS NV_API_CALL os_alloc_spinlock (void **); +void NV_API_CALL os_free_spinlock (void *); +NvU64 NV_API_CALL os_acquire_spinlock (void *); +void NV_API_CALL os_release_spinlock (void *, NvU64); +NV_STATUS NV_API_CALL os_queue_work_item (struct os_work_queue *, void *); +NV_STATUS NV_API_CALL os_flush_work_queue (struct os_work_queue *); +NV_STATUS NV_API_CALL os_alloc_mutex (void **); +void NV_API_CALL os_free_mutex (void *); +NV_STATUS NV_API_CALL os_acquire_mutex (void *); +NV_STATUS NV_API_CALL os_cond_acquire_mutex (void *); +void NV_API_CALL os_release_mutex (void *); +void* NV_API_CALL os_alloc_semaphore (NvU32); +void NV_API_CALL os_free_semaphore (void *); +NV_STATUS NV_API_CALL os_acquire_semaphore (void *); +NV_STATUS NV_API_CALL os_cond_acquire_semaphore (void *); +NV_STATUS NV_API_CALL os_release_semaphore (void *); +void* NV_API_CALL os_alloc_rwlock (void); +void NV_API_CALL os_free_rwlock (void *); +NV_STATUS NV_API_CALL os_acquire_rwlock_read (void *); +NV_STATUS NV_API_CALL os_acquire_rwlock_write (void *); +NV_STATUS NV_API_CALL os_cond_acquire_rwlock_read (void *); +NV_STATUS NV_API_CALL os_cond_acquire_rwlock_write(void *); +void NV_API_CALL os_release_rwlock_read (void *); +void NV_API_CALL os_release_rwlock_write (void *); +NvBool NV_API_CALL os_semaphore_may_sleep (void); +NV_STATUS NV_API_CALL os_get_version_info (os_version_info*); +NvBool NV_API_CALL os_is_isr (void); +NvBool NV_API_CALL os_pat_supported (void); +void NV_API_CALL os_dump_stack (void); +NvBool NV_API_CALL os_is_efi_enabled (void); +NvBool NV_API_CALL os_is_xen_dom0 (void); +NvBool NV_API_CALL os_is_vgx_hyper (void); +NV_STATUS NV_API_CALL os_inject_vgx_msi (NvU16, NvU64, NvU32); +NvBool NV_API_CALL os_is_grid_supported (void); +NvU32 NV_API_CALL os_get_grid_csp_support (void); +void NV_API_CALL os_get_screen_info (NvU64 *, NvU16 *, NvU16 *, NvU16 *, NvU16 *, NvU64, NvU64); +void NV_API_CALL os_bug_check (NvU32, const char *); +NV_STATUS NV_API_CALL os_lock_user_pages (void *, NvU64, void **, NvU32); +NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **, void**); +NV_STATUS NV_API_CALL os_unlock_user_pages (NvU64, void *); +NV_STATUS NV_API_CALL os_match_mmap_offset (void *, NvU64, NvU64 *); +NV_STATUS NV_API_CALL os_get_euid (NvU32 *); +NV_STATUS NV_API_CALL os_get_smbios_header (NvU64 *pSmbsAddr); +NV_STATUS NV_API_CALL os_get_acpi_rsdp_from_uefi (NvU32 *); +void NV_API_CALL os_add_record_for_crashLog (void *, NvU32); +void NV_API_CALL os_delete_record_for_crashLog (void *); +NV_STATUS NV_API_CALL os_call_vgpu_vfio (void *, NvU32); +NV_STATUS NV_API_CALL os_numa_memblock_size (NvU64 *); +NV_STATUS NV_API_CALL os_alloc_pages_node (NvS32, NvU32, NvU32, NvU64 *); +NV_STATUS NV_API_CALL os_get_page (NvU64 address); +NV_STATUS NV_API_CALL os_put_page (NvU64 address); +NvU32 NV_API_CALL os_get_page_refcount (NvU64 address); +NvU32 NV_API_CALL os_count_tail_pages (NvU64 address); +void NV_API_CALL os_free_pages_phys (NvU64, NvU32); +NV_STATUS NV_API_CALL os_call_nv_vmbus (NvU32, void *); +NV_STATUS NV_API_CALL os_open_temporary_file (void **); +void NV_API_CALL os_close_file (void *); +NV_STATUS NV_API_CALL os_write_file (void *, NvU8 *, NvU64, NvU64); +NV_STATUS NV_API_CALL os_read_file (void *, NvU8 *, NvU64, NvU64); +NV_STATUS NV_API_CALL os_open_readonly_file (const char *, void **); +NV_STATUS NV_API_CALL os_open_and_read_file (const char *, NvU8 *, NvU64); +NvBool NV_API_CALL os_is_nvswitch_present (void); +void NV_API_CALL os_get_random_bytes (NvU8 *, NvU16); +NV_STATUS NV_API_CALL os_alloc_wait_queue (os_wait_queue **); +void NV_API_CALL os_free_wait_queue (os_wait_queue *); +void NV_API_CALL os_wait_uninterruptible (os_wait_queue *); +void NV_API_CALL os_wait_interruptible (os_wait_queue *); +void NV_API_CALL os_wake_up (os_wait_queue *); +nv_cap_t* NV_API_CALL os_nv_cap_init (const char *); +nv_cap_t* NV_API_CALL os_nv_cap_create_dir_entry (nv_cap_t *, const char *, int); +nv_cap_t* NV_API_CALL os_nv_cap_create_file_entry (nv_cap_t *, const char *, int); +void NV_API_CALL os_nv_cap_destroy_entry (nv_cap_t *); +int NV_API_CALL os_nv_cap_validate_and_dup_fd(const nv_cap_t *, int); +void NV_API_CALL os_nv_cap_close_fd (int); + +enum os_pci_req_atomics_type { + OS_INTF_PCIE_REQ_ATOMICS_32BIT, + OS_INTF_PCIE_REQ_ATOMICS_64BIT, + OS_INTF_PCIE_REQ_ATOMICS_128BIT +}; +NV_STATUS NV_API_CALL os_enable_pci_req_atomics (void *, enum os_pci_req_atomics_type); + +extern NvU32 os_page_size; +extern NvU64 os_page_mask; +extern NvU8 os_page_shift; +extern NvU32 os_sev_status; +extern NvBool os_sev_enabled; +extern NvBool os_dma_buf_enabled; + +/* + * --------------------------------------------------------------------------- + * + * Debug macros. + * + * --------------------------------------------------------------------------- + */ + +#define NV_DBG_INFO 0x0 +#define NV_DBG_SETUP 0x1 +#define NV_DBG_USERERRORS 0x2 +#define NV_DBG_WARNINGS 0x3 +#define NV_DBG_ERRORS 0x4 + + +void NV_API_CALL out_string(const char *str); +int NV_API_CALL nv_printf(NvU32 debuglevel, const char *printf_format, ...); + +#define NV_DEV_PRINTF(debuglevel, nv, format, ... ) \ + nv_printf(debuglevel, "NVRM: GPU " NV_PCI_DEV_FMT ": " format, NV_PCI_DEV_FMT_ARGS(nv), ## __VA_ARGS__) + +#define NV_DEV_PRINTF_STATUS(debuglevel, nv, status, format, ... ) \ + nv_printf(debuglevel, "NVRM: GPU " NV_PCI_DEV_FMT ": " format " (0x%x)\n", NV_PCI_DEV_FMT_ARGS(nv), ## __VA_ARGS__, status) + +/* + * Fields for os_lock_user_pages flags parameter + */ +#define NV_LOCK_USER_PAGES_FLAGS_WRITE 0:0 +#define NV_LOCK_USER_PAGES_FLAGS_WRITE_NO 0x00000000 +#define NV_LOCK_USER_PAGES_FLAGS_WRITE_YES 0x00000001 + +#endif /* OS_INTERFACE_H */ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/os_custom.h b/userspace/sdks/nv525/nvalloc/unix/include/os_custom.h new file mode 100644 index 0000000..37df06f --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/os_custom.h @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _OS_CUSTOM_H_ +#define _OS_CUSTOM_H_ + +/*! + * @file os_custom.h + * @brief OS module specific definitions for this OS + */ + +#include <os-interface.h> +#include <osfuncs.h> + +// File modes, added for NVIDIA capabilities. +#define OS_RUSR 00400 // read permission, owner +#define OS_WUSR 00200 // write permission, owner +#define OS_XUSR 00100 // execute/search permission, owner +#define OS_RWXU (OS_RUSR | OS_WUSR | OS_XUSR) // read, write, execute/search, owner +#define OS_RGRP 00040 // read permission, group +#define OS_WGRP 00020 // write permission, group +#define OS_XGRP 00010 // execute/search permission, group +#define OS_RWXG (OS_RGRP | OS_WGRP | OS_XGRP) // read, write, execute/search, group +#define OS_ROTH 00004 // read permission, other +#define OS_WOTH 00002 // write permission, other +#define OS_XOTH 00001 // execute/search permission, other +#define OS_RWXO (OS_ROTH | OS_WOTH | OS_XOTH) // read, write, execute/search, other +#define OS_RUGO (OS_RUSR | OS_RGRP | OS_ROTH) +#define OS_WUGO (OS_WUSR | OS_WGRP | OS_WOTH) +#define OS_XUGO (OS_XUSR | OS_XGRP | OS_XOTH) + +// Trigger for collecting GPU state for later extraction. +NV_STATUS RmLogGpuCrash(OBJGPU *); + +// This is callback function in the miniport. +// The argument is a device extension, and must be cast as such to be useful. +typedef void (*MINIPORT_CALLBACK)(void*); + +NV_STATUS osPackageRegistry(OBJGPU *pGpu, PACKED_REGISTRY_TABLE *, NvU32 *); + +#endif // _OS_CUSTOM_H_ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/osapi.h b/userspace/sdks/nv525/nvalloc/unix/include/osapi.h new file mode 100644 index 0000000..7c71874 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/osapi.h @@ -0,0 +1,192 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _OSAPI_H_ +#define _OSAPI_H_ + +#include "core/system.h" +#include "gpu/gpu.h" + +#include <os-interface.h> // NV_DBG_ERRORS +#include <rmapi/rmapi.h> +#include <core/thread_state.h> + +#if defined(__use_altstack__) +#if defined(QA_BUILD) +//--------------------------------------------------------------------------- +// +// 32 bit debug marker values. +// +//--------------------------------------------------------------------------- + +#define NV_MARKER1 (NvU32)(('M' << 24) | ('R' << 16) | ('V' << 8) | 'N') +#define NV_MARKER2 (NvU32)(('N' << 24) | ('V' << 16) | ('R' << 8) | 'M') + +// +// The two macros below implement a simple alternate stack usage sanity +// check for QA_BUILD RM builds. NV_ALTSTACK_WRITE_MARKERS() fills +// altstacks with NV_MARKER1, which enables NV_ALTSTACK_CHECK_MARKERS() +// to determine the stack usage fairly reliably by looking for the +// first clobbered marker. If more than 7/8 of the alternate stack were +// used, NV_ALTSTACK_CHECK_MARKERS() prints an error and asserts. +// +#define NV_ALTSTACK_WRITE_MARKERS(sp) \ +{ \ + NvU32 i, *stack = (void *)(sp)->stack; \ + for (i = 0; i < ((sp)->size / sizeof(NvU32)); i++) \ + stack[i] = NV_MARKER1; \ +} + +#define NV_ALTSTACK_CHECK_MARKERS(sp) \ +{ \ + NvU32 i, *stack = (void *)(sp)->stack; \ + for (i = 0; i < ((sp)->size / sizeof(NvU32)); i++) \ + { \ + if (stack[i] != NV_MARKER1) \ + break; \ + } \ + if ((i * sizeof(NvU32)) < ((sp)->size / 8)) \ + { \ + nv_printf(NV_DBG_ERRORS, "NVRM: altstack: used %d of %d bytes!\n", \ + ((sp)->size - (i * sizeof(NvU32))), (sp)->size); \ + NV_ASSERT_PRECOMP((i * sizeof(NvU32)) >= ((sp)->size / 8)); \ + } \ +} +#else +#define NV_ALTSTACK_WRITE_MARKERS(sp) +#define NV_ALTSTACK_CHECK_MARKERS(sp) +#endif +#if defined(NVCPU_X86_64) +#define NV_ENTER_RM_RUNTIME(sp,fp) \ +{ \ + NV_ALTSTACK_WRITE_MARKERS(sp); \ + __asm__ __volatile__ ("movq %%rbp,%0" : "=r" (fp)); /* save %rbp */ \ + __asm__ __volatile__ ("movq %0,%%rbp" :: "r" ((sp)->top)); \ +} + +#define NV_EXIT_RM_RUNTIME(sp,fp) \ +{ \ + register void *__rbp __asm__ ("rbp"); \ + if (__rbp != (sp)->top) \ + { \ + nv_printf(NV_DBG_ERRORS, "NVRM: detected corrupted runtime stack!\n"); \ + NV_ASSERT_PRECOMP(__rbp == (sp)->top); \ + } \ + NV_ALTSTACK_CHECK_MARKERS(sp); \ + __asm__ __volatile__ ("movq %0,%%rbp" :: "r" (fp)); /* restore %rbp */ \ +} +#else +#error "gcc \"altstacks\" support is not implemented on this platform!" +#endif +#else +#define NV_ENTER_RM_RUNTIME(sp,fp) { (void)sp; (void)fp; } +#define NV_EXIT_RM_RUNTIME(sp,fp) +#endif + +void RmShutdownRm (void); + +NvBool RmInitPrivateState (nv_state_t *); +void RmFreePrivateState (nv_state_t *); + +NvBool RmInitAdapter (nv_state_t *); +NvBool RmPartiallyInitAdapter (nv_state_t *); +void RmShutdownAdapter (nv_state_t *); +void RmDisableAdapter (nv_state_t *); +void RmPartiallyDisableAdapter(nv_state_t *); +NV_STATUS RmGetAdapterStatus (nv_state_t *, NvU32 *); +NV_STATUS RmExcludeAdapter (nv_state_t *); + +NvBool RmGpuHasIOSpaceEnabled (nv_state_t *); + +void RmFreeUnusedClients (nv_state_t *, nv_file_private_t *); +NV_STATUS RmIoctl (nv_state_t *, nv_file_private_t *, NvU32, void *, NvU32); + +NV_STATUS RmAllocOsEvent (NvHandle, nv_file_private_t *, NvU32); +NV_STATUS RmFreeOsEvent (NvHandle, NvU32); + +void RmI2cAddGpuPorts(nv_state_t *); + +NV_STATUS RmInitX86EmuState(OBJGPU *); +void RmFreeX86EmuState(OBJGPU *); +NV_STATUS RmPowerSourceChangeEvent(nv_state_t *, NvU32); + +const NvU8 *RmGetGpuUuidRaw(nv_state_t *); + +NV_STATUS nv_vbios_call(OBJGPU *, NvU32 *, NvU32 *); + +int amd_adv_spec_cache_feature(OBJOS *); +int amd_msr_c0011022_incompatible(OBJOS *); + +NV_STATUS rm_get_adapter_status (nv_state_t *, NvU32 *); + +NV_STATUS rm_alloc_os_event (NvHandle, nv_file_private_t *, NvU32); +NV_STATUS rm_free_os_event (NvHandle, NvU32); +NV_STATUS rm_get_event_data (nv_file_private_t *, NvP64, NvU32 *); +void rm_client_free_os_events (NvHandle); + +NV_STATUS rm_create_mmap_context (NvHandle, NvHandle, NvHandle, NvP64, NvU64, NvU64, NvU32, NvU32); +NV_STATUS rm_update_device_mapping_info (NvHandle, NvHandle, NvHandle, void *, void *); + +NV_STATUS rm_access_registry (NvHandle, NvHandle, NvU32, NvP64, NvU32, NvP64, NvU32, NvP64, NvU32 *, NvU32 *, NvU32 *); + +// registry management +NV_STATUS RmInitRegistry (void); +NV_STATUS RmDestroyRegistry (nv_state_t *); + +NV_STATUS RmWriteRegistryDword (nv_state_t *, const char *, NvU32 ); +NV_STATUS RmReadRegistryDword (nv_state_t *, const char *, NvU32 *); +NV_STATUS RmWriteRegistryString (nv_state_t *, const char *, const char *, NvU32); +NV_STATUS RmReadRegistryBinary (nv_state_t *, const char *, NvU8 *, NvU32 *); +NV_STATUS RmWriteRegistryBinary (nv_state_t *, const char *, NvU8 *, NvU32); +NV_STATUS RmReadRegistryString (nv_state_t *, const char *, NvU8 *, NvU32 *); + +NV_STATUS RmPackageRegistry (nv_state_t *, PACKED_REGISTRY_TABLE *, NvU32 *); + +NvBool RmIsNvifFunctionSupported(NvU32, NvU32); +void RmInitAcpiMethods (OBJOS *, OBJSYS *, OBJGPU *); +void RmUnInitAcpiMethods (OBJSYS *); + +void RmInflateOsToRmPageArray (RmPhysAddr *, NvU64); +void RmDeflateRmToOsPageArray (RmPhysAddr *, NvU64); + +void RmInitS0ixPowerManagement (nv_state_t *); +void RmInitDeferredDynamicPowerManagement (nv_state_t *); +void RmDestroyDeferredDynamicPowerManagement(nv_state_t *); + +NV_STATUS os_ref_dynamic_power (nv_state_t *, nv_dynamic_power_mode_t); +void os_unref_dynamic_power (nv_state_t *, nv_dynamic_power_mode_t); +void RmHandleDisplayChange (nvidia_stack_t *, nv_state_t *); +void RmUpdateGc6ConsoleRefCount (nv_state_t *, NvBool); + +NvBool rm_get_uefi_console_status (nv_state_t *); +NvU64 rm_get_uefi_console_size (nv_state_t *, NvU64 *); + +RM_API *RmUnixRmApiPrologue (nv_state_t *, THREAD_STATE_NODE *, NvU32 module); +void RmUnixRmApiEpilogue (nv_state_t *, THREAD_STATE_NODE *); + +static inline NvBool rm_is_system_notebook(void) +{ + return (nv_is_chassis_notebook() || nv_acpi_is_battery_present()); +} + +#endif // _OSAPI_H_ diff --git a/userspace/sdks/nv525/nvalloc/unix/include/osfuncs.h b/userspace/sdks/nv525/nvalloc/unix/include/osfuncs.h new file mode 100644 index 0000000..7afbf63 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/osfuncs.h @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef OSFUNCS_H +#define OSFUNCS_H + +/**************** Resource Manager Defines and Structures ******************\ +* * +* Declarations for the Operating System Specific Functions. * +* * +\***************************************************************************/ + +#include <os/os.h> + +OSQueueWorkItem osQueueWorkItem; +OSQueueWorkItemWithFlags osQueueWorkItemWithFlags; +OSQueueSystemWorkItem osQueueSystemWorkItem; +OSDbgBreakpointEnabled osDbgBreakpointEnabled; + +void* osGetStereoDongleInterface(void); + +OSCallACPI_DSM osCallACPI_DSM; +OSCallACPI_DDC osCallACPI_DDC; +OSCallACPI_NVHG_ROM osCallACPI_NVHG_ROM; +OSCallACPI_DOD osCallACPI_DOD; +OSCallACPI_MXDS osCallACPI_MXDS; +OSCallACPI_MXDM osCallACPI_MXDM; + +#if defined(NVCPU_X86_64) +OSnv_rdcr4 nv_rdcr4; +NvU64 nv_rdcr3(OBJOS *); +OSnv_cpuid nv_cpuid; +#endif + +#endif // OSFUNCS_H diff --git a/userspace/sdks/nv525/nvalloc/unix/include/rmobjexportimport.h b/userspace/sdks/nv525/nvalloc/unix/include/rmobjexportimport.h new file mode 100644 index 0000000..1330ad8 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/include/rmobjexportimport.h @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _RMOBJEXPORTIMPORT_H_ +#define _RMOBJEXPORTIMPORT_H_ + +#include "nvstatus.h" + +typedef NvHandle RmObjExportHandle; + +NV_STATUS RmExportObject(NvHandle hSrcClient, NvHandle hSrcObject, + RmObjExportHandle *pDstObject, NvU32 *pDeviceInstance); + +void RmFreeObjExportHandle(RmObjExportHandle hObject); + +NV_STATUS RmImportObject(NvHandle hDstClient, NvHandle hDstParent, + NvHandle *phDstObject, RmObjExportHandle hSrcObject, + NvU8 *pObjectType); + +NV_STATUS RmGetExportObjectInfo(RmObjExportHandle hSrcObject, NvU32 *deviceInstance); +#endif // _RMOBJEXPORTIMPORT_H_ + diff --git a/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_cpuid.c b/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_cpuid.c new file mode 100644 index 0000000..0ebb1c8 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_cpuid.c @@ -0,0 +1,52 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <os/os.h> + +/* inline assembler routines for UNIX platforms */ + +#if defined(NVCPU_X86_64) + +NvS32 nv_cpuid( + OBJOS *pOS, + NvS32 op, + NvS32 subop, + NvU32 *eax, + NvU32 *ebx, + NvU32 *ecx, + NvU32 *edx +) +{ + asm volatile (" cpuid \n" + : "=a" (*eax), + "=b" (*ebx), + "=c" (*ecx), + "=d" (*edx) + : "a" (op), + "c" (subop) + : "cc"); + + return 1; +} + +#endif diff --git a/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_rdcr.c b/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_rdcr.c new file mode 100644 index 0000000..56e6bfd --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_rdcr.c @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <os/os.h> + +/* inline assembler routines for UNIX platforms */ + +#if defined(NVCPU_X86_64) + +NvU32 nv_rdcr4(OBJOS *pOS) +{ + NvU64 val; + asm volatile ("movq %%cr4,%0" : "=r" (val)); + return (NvU32)val; +} + +NvU64 nv_rdcr3(OBJOS *pOS) +{ + NvU64 val; + asm volatile ("movq %%cr3,%0" : "=r" (val)); + return val; +} + +#endif diff --git a/userspace/sdks/nv525/nvalloc/unix/src/escape.c b/userspace/sdks/nv525/nvalloc/unix/src/escape.c new file mode 100644 index 0000000..c7b451a --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/escape.c @@ -0,0 +1,938 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + + +//***************************** Module Header ********************************** +// +// This code is linked into the resource manager proper. It receives the +// ioctl from the resource manager's customer, unbundles the args and +// calls the correct resman routines. +// +//****************************************************************************** + +#include <core/prelude.h> +#include <core/locks.h> +#include <nv.h> +#include <nv_escape.h> +#include <osapi.h> +#include <rmapi/exports.h> +#include <nv-unix-nvos-params-wrappers.h> + +#include <nvos.h> +#include <class/cl0000.h> // NV01_ROOT +#include <class/cl0001.h> // NV01_ROOT_NON_PRIV +#include <class/cl0005.h> // NV01_EVENT +#include <class/cl003e.h> // NV01_MEMORY_SYSTEM +#include <class/cl0071.h> // NV01_MEMORY_SYSTEM_OS_DESCRIPTOR + +#include <ctrl/ctrl00fd.h> + +#define NV_CTL_DEVICE_ONLY(nv) \ +{ \ + if (((nv)->flags & NV_FLAG_CONTROL) == 0) \ + { \ + rmStatus = NV_ERR_INVALID_ARGUMENT; \ + goto done; \ + } \ +} + +#define NV_ACTUAL_DEVICE_ONLY(nv) \ +{ \ + if (((nv)->flags & NV_FLAG_CONTROL) != 0) \ + { \ + rmStatus = NV_ERR_INVALID_ARGUMENT; \ + goto done; \ + } \ +} + +static NvBool RmIsDeviceRefNeeded(NVOS54_PARAMETERS *pApi) +{ + switch(pApi->cmd) + { + case NV00FD_CTRL_CMD_ATTACH_MEM: + return NV_TRUE; + default: + return NV_FALSE; + } +} + +static NV_STATUS RmGetDeviceFd(NVOS54_PARAMETERS *pApi, NvS32 *pFd) +{ + RMAPI_PARAM_COPY paramCopy; + void *pKernelParams; + NvU32 paramSize; + NV_STATUS status; + + *pFd = -1; + + switch(pApi->cmd) + { + case NV00FD_CTRL_CMD_ATTACH_MEM: + paramSize = sizeof(NV00FD_CTRL_ATTACH_MEM_PARAMS); + break; + default: + return NV_ERR_INVALID_ARGUMENT; + } + + RMAPI_PARAM_COPY_INIT(paramCopy, pKernelParams, pApi->params, paramSize, 1); + + status = rmapiParamsAcquire(¶mCopy, NV_TRUE); + if (status != NV_OK) + return status; + + switch(pApi->cmd) + { + case NV00FD_CTRL_CMD_ATTACH_MEM: + *pFd = (NvS32)((NV00FD_CTRL_ATTACH_MEM_PARAMS *)pKernelParams)->devDescriptor; + break; + default: + NV_ASSERT(0); + break; + } + + NV_ASSERT(rmapiParamsRelease(¶mCopy) == NV_OK); + + return status; +} + +// Only return errors through pApi->status +static void RmCreateOsDescriptor(NVOS32_PARAMETERS *pApi, API_SECURITY_INFO secInfo) +{ + NV_STATUS rmStatus; + NvBool writable; + NvU32 flags = 0; + NvU64 allocSize, pageCount, *pPteArray = NULL; + void *pDescriptor, *pPageArray = NULL; + + pDescriptor = NvP64_VALUE(pApi->data.AllocOsDesc.descriptor); + if (((NvUPtr)pDescriptor & ~os_page_mask) != 0) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + goto done; + } + + // Check to prevent an NvU64 overflow + if ((pApi->data.AllocOsDesc.limit + 1) == 0) + { + rmStatus = NV_ERR_INVALID_LIMIT; + goto done; + } + + allocSize = (pApi->data.AllocOsDesc.limit + 1); + pageCount = (1 + ((allocSize - 1) / os_page_size)); + + writable = FLD_TEST_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_WRITE, pApi->data.AllocOsDesc.attr2); + + flags = FLD_SET_DRF_NUM(_LOCK_USER_PAGES, _FLAGS, _WRITE, writable, flags); + rmStatus = os_lock_user_pages(pDescriptor, pageCount, &pPageArray, flags); + if (rmStatus == NV_OK) + { + pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPageArray; + pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY; + } + else if (rmStatus == NV_ERR_INVALID_ADDRESS) + { + rmStatus = os_lookup_user_io_memory(pDescriptor, pageCount, + &pPteArray, &pPageArray); + if (rmStatus == NV_OK) + { + if (pPageArray != NULL) + { + pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPageArray; + pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY; + } + else if (pPteArray != NULL) + { + pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPteArray; + pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY; + } + else + { + NV_ASSERT_FAILED("unknown memory import type"); + rmStatus = NV_ERR_NOT_SUPPORTED; + } + } + } + if (rmStatus != NV_OK) + goto done; + + Nv04VidHeapControlWithSecInfo(pApi, secInfo); + + if (pApi->status != NV_OK) + { + switch (pApi->data.AllocOsDesc.descriptorType) + { + default: + break; + case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY: + os_unlock_user_pages(pageCount, pPageArray); + break; + } + } + +done: + if (rmStatus != NV_OK) + pApi->status = rmStatus; +} + +// Only return errors through pApi->status +static void RmAllocOsDescriptor(NVOS02_PARAMETERS *pApi, API_SECURITY_INFO secInfo) +{ + NV_STATUS rmStatus = NV_OK; + NvU32 flags, attr, attr2; + NVOS32_PARAMETERS *pVidHeapParams; + + if (!FLD_TEST_DRF(OS02, _FLAGS, _LOCATION, _PCI, pApi->flags) || + !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NO_MAP, pApi->flags)) + { + rmStatus = NV_ERR_INVALID_FLAGS; + goto done; + } + + attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI); + + if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, pApi->flags) || + FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, pApi->flags)) + { + attr = FLD_SET_DRF(OS32, _ATTR, _COHERENCY, _WRITE_BACK, attr); + } + else if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, pApi->flags)) + attr = FLD_SET_DRF(OS32, _ATTR, _COHERENCY, _UNCACHED, attr); + else { + rmStatus = NV_ERR_INVALID_FLAGS; + goto done; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, pApi->flags)) + attr = FLD_SET_DRF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS, attr); + else + attr = FLD_SET_DRF(OS32, _ATTR, _PHYSICALITY, _NONCONTIGUOUS, attr); + + if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, pApi->flags)) + attr2 = DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _YES); + else + attr2 = DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _NO); + + pVidHeapParams = portMemAllocNonPaged(sizeof(NVOS32_PARAMETERS)); + if (pVidHeapParams == NULL) + { + rmStatus = NV_ERR_NO_MEMORY; + goto done; + } + portMemSet(pVidHeapParams, 0, sizeof(NVOS32_PARAMETERS)); + + pVidHeapParams->hRoot = pApi->hRoot; + pVidHeapParams->hObjectParent = pApi->hObjectParent; + pVidHeapParams->function = NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR; + + flags = (NVOS32_ALLOC_FLAGS_MEMORY_HANDLE_PROVIDED | + NVOS32_ALLOC_FLAGS_MAP_NOT_REQUIRED); + + if (DRF_VAL(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, pApi->flags)) + attr2 = FLD_SET_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_ONLY, attr2); + + // Currently CPU-RO memory implies GPU-RO as well + if (DRF_VAL(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, pApi->flags) || + DRF_VAL(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, pApi->flags)) + attr2 = FLD_SET_DRF(OS32, _ATTR2, _PROTECTION_DEVICE, _READ_ONLY, attr2); + + pVidHeapParams->data.AllocOsDesc.hMemory = pApi->hObjectNew; + pVidHeapParams->data.AllocOsDesc.flags = flags; + pVidHeapParams->data.AllocOsDesc.attr = attr; + pVidHeapParams->data.AllocOsDesc.attr2 = attr2; + pVidHeapParams->data.AllocOsDesc.descriptor = pApi->pMemory; + pVidHeapParams->data.AllocOsDesc.limit = pApi->limit; + pVidHeapParams->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS; + + RmCreateOsDescriptor(pVidHeapParams, secInfo); + + pApi->status = pVidHeapParams->status; + + portMemFree(pVidHeapParams); + +done: + if (rmStatus != NV_OK) + pApi->status = rmStatus; +} + +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hRoot) == NV_OFFSETOF(NVOS64_PARAMETERS, hRoot)); +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hObjectParent) == NV_OFFSETOF(NVOS64_PARAMETERS, hObjectParent)); +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hObjectNew) == NV_OFFSETOF(NVOS64_PARAMETERS, hObjectNew)); +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hClass) == NV_OFFSETOF(NVOS64_PARAMETERS, hClass)); +ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, pAllocParms) == NV_OFFSETOF(NVOS64_PARAMETERS, pAllocParms)); + +NV_STATUS RmIoctl( + nv_state_t *nv, + nv_file_private_t *nvfp, + NvU32 cmd, + void *data, + NvU32 dataSize +) +{ + NV_STATUS rmStatus = NV_ERR_GENERIC; + API_SECURITY_INFO secInfo = { }; + + secInfo.privLevel = osIsAdministrator() ? RS_PRIV_LEVEL_USER_ROOT : RS_PRIV_LEVEL_USER; + secInfo.paramLocation = PARAM_LOCATION_USER; + secInfo.pProcessToken = NULL; + secInfo.gpuOsInfo = NULL; + secInfo.clientOSInfo = nvfp->ctl_nvfp; + if (secInfo.clientOSInfo == NULL) + secInfo.clientOSInfo = nvfp; + + switch (cmd) + { + case NV_ESC_RM_ALLOC_MEMORY: + { + nv_ioctl_nvos02_parameters_with_fd *pApi; + NVOS02_PARAMETERS *pParms; + + pApi = data; + pParms = &pApi->params; + + NV_ACTUAL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(nv_ioctl_nvos02_parameters_with_fd)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + if (pParms->hClass == NV01_MEMORY_SYSTEM_OS_DESCRIPTOR) + RmAllocOsDescriptor(pParms, secInfo); + else + { + NvU32 flags = pParms->flags; + + Nv01AllocMemoryWithSecInfo(pParms, secInfo); + + // + // If the system memory is going to be mapped immediately, + // create the mmap context for it now. + // + if ((pParms->hClass == NV01_MEMORY_SYSTEM) && + (!FLD_TEST_DRF(OS02, _FLAGS, _ALLOC, _NONE, flags)) && + (!FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NO_MAP, flags)) && + (pParms->status == NV_OK)) + { + if (rm_create_mmap_context(pParms->hRoot, + pParms->hObjectParent, pParms->hObjectNew, + pParms->pMemory, pParms->limit + 1, 0, + NV_MEMORY_DEFAULT, + pApi->fd) != NV_OK) + { + NV_PRINTF(LEVEL_WARNING, + "could not create mmap context for %p\n", + NvP64_VALUE(pParms->pMemory)); + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + } + } + + break; + } + + case NV_ESC_RM_ALLOC_OBJECT: + { + NVOS05_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS05_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv01AllocObjectWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_ALLOC: + { + NVOS21_PARAMETERS *pApi = data; + NVOS64_PARAMETERS *pApiAccess = data; + NvBool bAccessApi = (dataSize == sizeof(NVOS64_PARAMETERS)); + + if ((dataSize != sizeof(NVOS21_PARAMETERS)) && + (dataSize != sizeof(NVOS64_PARAMETERS))) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + switch (pApi->hClass) + { + case NV01_ROOT: + case NV01_ROOT_CLIENT: + case NV01_ROOT_NON_PRIV: + { + NV_CTL_DEVICE_ONLY(nv); + + // Force userspace client allocations to be the _CLIENT class. + pApi->hClass = NV01_ROOT_CLIENT; + break; + } + case NV01_EVENT: + case NV01_EVENT_OS_EVENT: + case NV01_EVENT_KERNEL_CALLBACK: + case NV01_EVENT_KERNEL_CALLBACK_EX: + { + break; + } + default: + { + NV_CTL_DEVICE_ONLY(nv); + break; + } + } + + if (!bAccessApi) + { + Nv04AllocWithSecInfo(pApi, secInfo); + } + else + { + Nv04AllocWithAccessSecInfo(pApiAccess, secInfo); + } + + break; + } + + case NV_ESC_RM_FREE: + { + NVOS00_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS00_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv01FreeWithSecInfo(pApi, secInfo); + + if (pApi->status == NV_OK && + pApi->hObjectOld == pApi->hRoot) + { + rm_client_free_os_events(pApi->hRoot); + } + + break; + } + + case NV_ESC_RM_VID_HEAP_CONTROL: + { + NVOS32_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS32_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + if (pApi->function == NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR) + RmCreateOsDescriptor(pApi, secInfo); + else + Nv04VidHeapControlWithSecInfo(pApi, secInfo); + + break; + } + + case NV_ESC_RM_I2C_ACCESS: + { + NVOS_I2C_ACCESS_PARAMS *pApi = data; + + NV_ACTUAL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS_I2C_ACCESS_PARAMS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04I2CAccessWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_IDLE_CHANNELS: + { + NVOS30_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS30_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04IdleChannelsWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_MAP_MEMORY: + { + nv_ioctl_nvos33_parameters_with_fd *pApi; + NVOS33_PARAMETERS *pParms; + + pApi = data; + pParms = &pApi->params; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(nv_ioctl_nvos33_parameters_with_fd)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + // Don't allow userspace to override the caching type + pParms->flags = FLD_SET_DRF(OS33, _FLAGS, _CACHING_TYPE, _DEFAULT, pParms->flags); + Nv04MapMemoryWithSecInfo(pParms, secInfo); + + if (pParms->status == NV_OK) + { + pParms->status = rm_create_mmap_context(pParms->hClient, + pParms->hDevice, pParms->hMemory, + pParms->pLinearAddress, pParms->length, + pParms->offset, + DRF_VAL(OS33, _FLAGS, _CACHING_TYPE, pParms->flags), + pApi->fd); + if (pParms->status != NV_OK) + { + NVOS34_PARAMETERS params; + portMemSet(¶ms, 0, sizeof(NVOS34_PARAMETERS)); + params.hClient = pParms->hClient; + params.hDevice = pParms->hDevice; + params.hMemory = pParms->hMemory; + params.pLinearAddress = pParms->pLinearAddress; + params.flags = pParms->flags; + Nv04UnmapMemoryWithSecInfo(¶ms, secInfo); + } + } + break; + } + + case NV_ESC_RM_UNMAP_MEMORY: + { + NVOS34_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS34_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04UnmapMemoryWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_ACCESS_REGISTRY: + { + NVOS38_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS38_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pApi->status = rm_access_registry(pApi->hClient, + pApi->hObject, + pApi->AccessType, + pApi->pDevNode, + pApi->DevNodeLength, + pApi->pParmStr, + pApi->ParmStrLength, + pApi->pBinaryData, + &pApi->BinaryDataLength, + &pApi->Data, + &pApi->Entry); + break; + } + + case NV_ESC_RM_ALLOC_CONTEXT_DMA2: + { + NVOS39_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS39_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04AllocContextDmaWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_BIND_CONTEXT_DMA: + { + NVOS49_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS49_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04BindContextDmaWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_MAP_MEMORY_DMA: + { + NVOS46_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS46_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04MapMemoryDmaWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_UNMAP_MEMORY_DMA: + { + NVOS47_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS47_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04UnmapMemoryDmaWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_DUP_OBJECT: + { + NVOS55_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS55_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04DupObjectWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_RM_SHARE: + { + NVOS57_PARAMETERS *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS57_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + Nv04ShareWithSecInfo(pApi, secInfo); + break; + } + + case NV_ESC_ALLOC_OS_EVENT: + { + nv_ioctl_alloc_os_event_t *pApi = data; + + if (dataSize != sizeof(nv_ioctl_alloc_os_event_t)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pApi->Status = rm_alloc_os_event(pApi->hClient, + nvfp, + pApi->fd); + break; + } + + case NV_ESC_FREE_OS_EVENT: + { + nv_ioctl_free_os_event_t *pApi = data; + + if (dataSize != sizeof(nv_ioctl_free_os_event_t)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pApi->Status = rm_free_os_event(pApi->hClient, pApi->fd); + break; + } + + case NV_ESC_RM_GET_EVENT_DATA: + { + NVOS41_PARAMETERS *pApi = data; + + if (dataSize != sizeof(NVOS41_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pApi->status = rm_get_event_data(nvfp, + pApi->pEvent, + &pApi->MoreEvents); + break; + } + + case NV_ESC_STATUS_CODE: + { + nv_state_t *pNv; + nv_ioctl_status_code_t *pApi = data; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(nv_ioctl_status_code_t)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pNv = nv_get_adapter_state(pApi->domain, pApi->bus, pApi->slot); + if (pNv == NULL) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + rmStatus = rm_get_adapter_status(pNv, &pApi->status); + + if (rmStatus != NV_OK) + goto done; + + break; + } + + case NV_ESC_RM_CONTROL: + { + NVOS54_PARAMETERS *pApi = data; + void *priv = NULL; + nv_file_private_t *dev_nvfp = NULL; + NvS32 fd; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS54_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + if (RmIsDeviceRefNeeded(pApi)) + { + rmStatus = RmGetDeviceFd(pApi, &fd); + if (rmStatus != NV_OK) + { + goto done; + } + + dev_nvfp = nv_get_file_private(fd, NV_FALSE, &priv); + if (dev_nvfp == NULL) + { + rmStatus = NV_ERR_INVALID_DEVICE; + goto done; + } + + // Check to avoid cyclic dependency with NV_ESC_REGISTER_FD + if (!portAtomicCompareAndSwapU32(&dev_nvfp->register_or_refcount, + NVFP_TYPE_REFCOUNTED, + NVFP_TYPE_NONE)) + { + // Is this already refcounted... + if (dev_nvfp->register_or_refcount != NVFP_TYPE_REFCOUNTED) + { + nv_put_file_private(priv); + rmStatus = NV_ERR_IN_USE; + goto done; + } + } + + secInfo.gpuOsInfo = priv; + } + + Nv04ControlWithSecInfo(pApi, secInfo); + + if ((pApi->status != NV_OK) && (priv != NULL)) + { + // + // No need to reset `register_or_refcount` as it might be set + // for previous successful calls. We let it clear with FD close. + // + nv_put_file_private(priv); + + secInfo.gpuOsInfo = NULL; + } + + break; + } + + case NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO: + { + NVOS56_PARAMETERS *pApi = data; + void *pOldCpuAddress; + void *pNewCpuAddress; + + NV_CTL_DEVICE_ONLY(nv); + + if (dataSize != sizeof(NVOS56_PARAMETERS)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pOldCpuAddress = NvP64_VALUE(pApi->pOldCpuAddress); + pNewCpuAddress = NvP64_VALUE(pApi->pNewCpuAddress); + + pApi->status = rm_update_device_mapping_info(pApi->hClient, + pApi->hDevice, + pApi->hMemory, + pOldCpuAddress, + pNewCpuAddress); + break; + } + + case NV_ESC_REGISTER_FD: + { + nv_ioctl_register_fd_t *params = data; + void *priv = NULL; + nv_file_private_t *ctl_nvfp; + + if (dataSize != sizeof(nv_ioctl_register_fd_t)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus != NV_OK) + goto done; + + // If there is already a ctl fd registered on this nvfp, fail. + if (nvfp->ctl_nvfp != NULL) + { + // UNLOCK: release API lock + rmapiLockRelease(); + rmStatus = NV_ERR_INVALID_STATE; + goto done; + } + + // + // Note that this call is valid for both "actual" devices and ctrl + // devices. In particular, NV_ESC_ALLOC_OS_EVENT can be used with + // both types of devices. + // But, the ctl_fd passed in should always correspond to a control FD. + // + ctl_nvfp = nv_get_file_private(params->ctl_fd, + NV_TRUE, /* require ctl fd */ + &priv); + if (ctl_nvfp == NULL) + { + // UNLOCK: release API lock + rmapiLockRelease(); + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + // Disallow self-referential links, and disallow links to FDs that + // themselves have a link. + if ((ctl_nvfp == nvfp) || (ctl_nvfp->ctl_nvfp != NULL)) + { + nv_put_file_private(priv); + // UNLOCK: release API lock + rmapiLockRelease(); + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + // Check to avoid cyclic dependency with device refcounting + if (!portAtomicCompareAndSwapU32(&nvfp->register_or_refcount, + NVFP_TYPE_REGISTERED, + NVFP_TYPE_NONE)) + { + nv_put_file_private(priv); + // UNLOCK: release API lock + rmapiLockRelease(); + rmStatus = NV_ERR_IN_USE; + goto done; + } + + // + // nvfp->ctl_nvfp is read outside the lock, so set it atomically. + // Note that once set, this can never be removed until the fd + // associated with nvfp is closed. We hold on to 'priv' until the + // fd is closed, too, to ensure that the fd associated with + // ctl_nvfp remains valid. + // + portAtomicSetSize(&nvfp->ctl_nvfp, ctl_nvfp); + nvfp->ctl_nvfp_priv = priv; + + // UNLOCK: release API lock + rmapiLockRelease(); + + // NOTE: nv_put_file_private(priv) is not called here. It MUST be + // called during cleanup of this nvfp. + rmStatus = NV_OK; + break; + } + + default: + { + NV_PRINTF(LEVEL_ERROR, "unknown NVRM ioctl command: 0x%x\n", cmd); + goto done; + } + } + + rmStatus = NV_OK; +done: + + return rmStatus; +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/exports-stubs.c b/userspace/sdks/nv525/nvalloc/unix/src/exports-stubs.c new file mode 100644 index 0000000..4dac378 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/exports-stubs.c @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include <rmconfig.h> +#include <gpu/subdevice/subdevice.h> +#include <ctrl/ctrl0080/ctrl0080unix.h> +#include <ctrl/ctrl2080/ctrl2080unix.h> + +NV_STATUS NV_API_CALL rm_schedule_gpu_wakeup( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + return NV_OK; +} + +void NV_API_CALL rm_init_dynamic_power_management( + nvidia_stack_t *sp, + nv_state_t *nv, + NvBool bPr3AcpiMethodPresent +) +{ +} + +void NV_API_CALL rm_cleanup_dynamic_power_management( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ +} + +void NV_API_CALL rm_enable_dynamic_power_management( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ +} + +NV_STATUS NV_API_CALL rm_ref_dynamic_power( + nvidia_stack_t *sp, + nv_state_t *nv, + nv_dynamic_power_mode_t mode +) +{ + return NV_OK; +} + +void NV_API_CALL rm_unref_dynamic_power( + nvidia_stack_t *sp, + nv_state_t *nv, + nv_dynamic_power_mode_t mode +) +{ +} + +NV_STATUS NV_API_CALL rm_transition_dynamic_power( + nvidia_stack_t *sp, + nv_state_t *nv, + NvBool bEnter +) +{ + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_power_management( + nvidia_stack_t *sp, + nv_state_t *pNv, + nv_pm_action_t pmAction +) +{ + return NV_OK; +} + +const char* NV_API_CALL rm_get_vidmem_power_status( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + return "?"; +} + +const char* NV_API_CALL rm_get_dynamic_power_management_status( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + return "?"; +} + +const char* NV_API_CALL rm_get_gpu_gcx_support( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvBool bGcxTypeGC6 +) +{ + return "?"; +} + +NV_STATUS +subdeviceCtrlCmdOsUnixGc6BlockerRefCnt_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS *pParams +) +{ + return NV_OK; +} + +NV_STATUS +subdeviceCtrlCmdOsUnixAllowDisallowGcoff_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS *pParams +) +{ + return NV_OK; +} + +NV_STATUS +subdeviceCtrlCmdOsUnixAudioDynamicPower_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS *pParams +) +{ + return NV_OK; +} + +void +RmUpdateGc6ConsoleRefCount +( + nv_state_t *nv, + NvBool bIncrease +) +{ +} + +void +RmInitS0ixPowerManagement +( + nv_state_t *nv +) +{ +} + +void +RmInitDeferredDynamicPowerManagement +( + nv_state_t *nv +) +{ +} + +void +RmDestroyDeferredDynamicPowerManagement +( + nv_state_t *nv +) +{ +} + +void RmHandleDisplayChange +( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ +} + +NV_STATUS +os_ref_dynamic_power +( + nv_state_t *nv, + nv_dynamic_power_mode_t mode +) +{ + return NV_OK; +} + +void +os_unref_dynamic_power +( + nv_state_t *nv, + nv_dynamic_power_mode_t mode +) +{ +} + +NV_STATUS +deviceCtrlCmdOsUnixVTSwitch_IMPL +( + Device *pDevice, + NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS *pParams +) +{ + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_save_low_res_mode( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS RmInitX86EmuState(OBJGPU *pGpu) +{ + return NV_OK; +} + +void RmFreeX86EmuState(OBJGPU *pGpu) +{ +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/gcc_helper.c b/userspace/sdks/nv525/nvalloc/unix/src/gcc_helper.c new file mode 100644 index 0000000..3c1037c --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/gcc_helper.c @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nvtypes.h> +#include <os-interface.h> + +void* memset(void* s, int c, NvUPtr n) +{ + return os_mem_set(s, (NvU8)c, (NvU32)n); +} + +void* memcpy(void* dest, const void* src, NvUPtr n) +{ + return os_mem_copy(dest, src, (NvU32)n); +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/os-hypervisor.c b/userspace/sdks/nv525/nvalloc/unix/src/os-hypervisor.c new file mode 100644 index 0000000..3982c35 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/os-hypervisor.c @@ -0,0 +1,1023 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2014-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * + * @file os-hypervisor.c + * @brief OS specific Hypervisor interfaces for RM + * + */ + +#include "os/os.h" +#include "nv.h" +#include "nv-priv.h" +#include <nvRmReg.h> +#include <virtualization/hypervisor/hypervisor.h> +#include "core/thread_state.h" +#include "core/locks.h" +#include "virtualization/kernel_vgpu_mgr.h" +#include "kernel/gpu/mig_mgr/kernel_mig_manager.h" +#include "kernel/gpu/fifo/kernel_fifo.h" +#include "osapi.h" +#include "virtualization/kernel_hostvgpudeviceapi.h" +#include <objtmr.h> +#include "gpu/bif/kernel_bif.h" +#include "gpu/bus/kern_bus.h" +#include <nv_ref.h> // NV_PMC_BOOT_1_VGPU +#include "nvdevid.h" + +#define NV_VFIO_PCI_BAR1_REGION_INDEX 1 +#define NV_VFIO_PCI_BAR2_REGION_INDEX 2 +#define NV_VFIO_PCI_BAR3_REGION_INDEX 3 + +static NV_STATUS nv_parse_config_params(const char *, const char *, const char, NvU32 *); + +void hypervisorSetHypervVgpuSupported_IMPL(POBJHYPERVISOR pHypervisor) +{ + pHypervisor->bIsHypervVgpuSupported = NV_TRUE; +} + +NvBool hypervisorIsVgxHyper_IMPL(void) +{ + return os_is_vgx_hyper(); +} + +NV_STATUS hypervisorInjectInterrupt_IMPL +( + POBJHYPERVISOR pHypervisor, + VGPU_NS_INTR *pVgpuNsIntr +) +{ + NV_STATUS status = NV_ERR_NOT_SUPPORTED; + + if (pVgpuNsIntr->pVgpuVfioRef) + status = osVgpuInjectInterrupt(pVgpuNsIntr->pVgpuVfioRef); + else + { + if (pVgpuNsIntr->guestMSIAddr && pVgpuNsIntr->guestMSIData) + { + status = os_inject_vgx_msi((NvU16)pVgpuNsIntr->guestDomainId, + pVgpuNsIntr->guestMSIAddr, + pVgpuNsIntr->guestMSIData); + } + } + + return status; +} + +HYPERVISOR_TYPE NV_API_CALL nv_get_hypervisor_type(void) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + POBJHYPERVISOR pHypervisor = SYS_GET_HYPERVISOR(pSys); + return hypervisorGetHypervisorType(pHypervisor); +} + +static NV_STATUS get_available_instances( + NvU32 *avail_instances, + nv_state_t *pNv, + VGPU_TYPE *vgpuTypeInfo, + NvU32 pgpuIndex, + NvU8 devfn +) +{ + NV_STATUS rmStatus = NV_OK; + OBJGPU *pGpu = NULL; + OBJSYS *pSys = SYS_GET_INSTANCE(); + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys); + + *avail_instances = 0; + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + if (pGpu == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__); + rmStatus = NV_ERR_INVALID_STATE; + goto exit; + } + + /* TODO: Needs to have a proper fix this for DriverVM config */ + if (gpuIsSriovEnabled(pGpu) && + !(pHypervisor->getProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED))) + { + NvU8 fnId = devfn - pGpu->sriovState.firstVFOffset; + + if (fnId > 63) + { + NV_ASSERT(0); + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto exit; + } + + if (IS_MIG_ENABLED(pGpu)) + { + if (IS_MIG_IN_USE(pGpu)) { + NvU64 swizzIdInUseMask = 0; + NvU32 partitionFlag = PARTITIONID_INVALID; + KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); + NvU32 id; + + swizzIdInUseMask = kmigmgrGetSwizzIdInUseMask(pGpu, pKernelMIGManager); + + rmStatus = kvgpumgrGetPartitionFlag(vgpuTypeInfo->vgpuTypeId, + &partitionFlag); + if (rmStatus != NV_OK) + { + // Query for a non MIG vgpuType + NV_PRINTF(LEVEL_ERROR, "%s Query for a non MIG vGPU type \n", + __FUNCTION__); + rmStatus = NV_OK; + goto exit; + } + + // Determine valid swizzids not assigned to any vGPU device. + FOR_EACH_INDEX_IN_MASK(64, id, swizzIdInUseMask) + { + KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance; + NvU64 mask = 0; + + rmStatus = kmigmgrGetGPUInstanceInfo(pGpu, pKernelMIGManager, + id, &pKernelMIGGpuInstance); + if (rmStatus != NV_OK) + { + // Didn't find requested GPU instance + NV_PRINTF(LEVEL_ERROR, + "No valid GPU instance with SwizzId - %d found\n", id); + goto exit; + } + + mask = NVBIT64(id); + + if (pKernelMIGGpuInstance->partitionFlag == partitionFlag) + { + // Validate that same ID is not already set and VF is available + if (!(mask & pKernelVgpuMgr->pgpuInfo[pgpuIndex].assignedSwizzIdMask) && + !(pKernelVgpuMgr->pgpuInfo[pgpuIndex].createdVfMask & NVBIT64(fnId))) + { + *avail_instances = 1; + break; + } + } + } + FOR_EACH_INDEX_IN_MASK_END; + } + } + else + { + if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].numCreatedVgpu < vgpuTypeInfo->maxInstance) + { + if (vgpuTypeInfo->gpuInstanceSize) + { + // Query for a MIG vgpuType + NV_PRINTF(LEVEL_ERROR, "%s Query for a MIG vGPU type \n", + __FUNCTION__); + rmStatus = NV_OK; + goto exit; + } + + if (!(pKernelVgpuMgr->pgpuInfo[pgpuIndex].createdVfMask & NVBIT64(fnId))) + { + if (kvgpumgrCheckVgpuTypeCreatable(&pKernelVgpuMgr->pgpuInfo[pgpuIndex], vgpuTypeInfo) == NV_OK) + *avail_instances = 1; + } + } + } + } + else + { + if (kvgpumgrCheckVgpuTypeCreatable(&pKernelVgpuMgr->pgpuInfo[pgpuIndex], vgpuTypeInfo) == NV_OK) + *avail_instances = vgpuTypeInfo->maxInstance - pKernelVgpuMgr->pgpuInfo[pgpuIndex].numCreatedVgpu; + } + +exit: + return rmStatus; +} + +#define MAX_STR_LEN 256 +NV_STATUS NV_API_CALL nv_vgpu_get_type_info( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvU32 vgpuTypeId, + char *buffer, + int type_info, + NvU8 devfn +) +{ + THREAD_STATE_NODE threadState; + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJGPU *pGpu = NULL; + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + NV_STATUS rmStatus = NV_OK; + VGPU_TYPE *vgpuTypeInfo; + NvU32 pgpuIndex, i, avail_instances = 0; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + if (pGpu == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__); + rmStatus = NV_ERR_INVALID_STATE; + goto exit; + } + + if ((rmStatus = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex)) == + NV_OK) + { + for (i = 0; i < MAX_VGPU_TYPES_PER_PGPU; i++) + { + vgpuTypeInfo = pKernelVgpuMgr->pgpuInfo[pgpuIndex].vgpuTypes[i]; + if (vgpuTypeInfo == NULL) + break; + + if (vgpuTypeInfo->vgpuTypeId != vgpuTypeId) + continue; + + switch (type_info) + { + case VGPU_TYPE_NAME: + os_snprintf(buffer, VGPU_STRING_BUFFER_SIZE, "%s\n", + vgpuTypeInfo->vgpuName); + break; + case VGPU_TYPE_DESCRIPTION: + os_snprintf(buffer, MAX_STR_LEN, + "num_heads=%d, frl_config=%d, " + "framebuffer=%dM, max_resolution=%dx%d, max_instance=%d\n", + vgpuTypeInfo->numHeads, vgpuTypeInfo->frlConfig, + vgpuTypeInfo->profileSize >> 20, + vgpuTypeInfo->maxResolutionX, + vgpuTypeInfo->maxResolutionY, + vgpuTypeInfo->maxInstance); + break; + case VGPU_TYPE_INSTANCES: + rmStatus = get_available_instances(&avail_instances, pNv, + vgpuTypeInfo, + pgpuIndex, devfn); + if (rmStatus != NV_OK) + goto exit; + + os_snprintf(buffer, MAX_STR_LEN, "%d\n", avail_instances); + break; + default: + rmStatus = NV_ERR_INVALID_ARGUMENT; + } + break; + } + } + +exit: + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_get_type_ids( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvU32 *numVgpuTypes, + NvU32 *vgpuTypeIds, + NvBool isVirtfn, + NvU8 devfn, + NvBool getCreatableTypes +) +{ + THREAD_STATE_NODE threadState; + OBJSYS *pSys = SYS_GET_INSTANCE(); + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + NV_STATUS rmStatus = NV_OK; + NvU32 pgpuIndex, i, avail_instances = 0; + NvU32 numSupportedVgpuTypes = 0; + VGPU_TYPE *vgpuTypeInfo; + void *fp; + + if (!vgpuTypeIds || !numVgpuTypes) + return NV_ERR_INVALID_ARGUMENT; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + if ((rmStatus = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex)) == + NV_OK) + { + if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].sriovEnabled && !isVirtfn) + { + *numVgpuTypes = 0; + } + else + { + numSupportedVgpuTypes = pKernelVgpuMgr->pgpuInfo[pgpuIndex].numVgpuTypes; + *numVgpuTypes = 0; + + for (i = 0; i < numSupportedVgpuTypes; i++) + { + vgpuTypeInfo = pKernelVgpuMgr->pgpuInfo[pgpuIndex].vgpuTypes[i]; + + if (!getCreatableTypes) + { + // Return all available types + vgpuTypeIds[*numVgpuTypes] = vgpuTypeInfo->vgpuTypeId; + (*numVgpuTypes)++; + continue; + } + + rmStatus = get_available_instances(&avail_instances, pNv, + vgpuTypeInfo, pgpuIndex, + devfn); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Failed to get available instances for vGPU ID: %d, status: 0x%x\n", + vgpuTypeInfo->vgpuTypeId, rmStatus); + continue; + } + + if (avail_instances == 0) + continue; + + vgpuTypeIds[*numVgpuTypes] = vgpuTypeInfo->vgpuTypeId; + (*numVgpuTypes)++; + } + } + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_delete( + nvidia_stack_t *sp, + const NvU8 *pMdevUuid, + NvU16 vgpuId +) +{ + THREAD_STATE_NODE threadState; + void *fp = NULL; + NvU32 rmStatus = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + rmStatus = kvgpumgrDeleteRequestVgpu(pMdevUuid, vgpuId); + // UNLOCK: release API lock + rmapiLockRelease(); + } + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_process_vf_info( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvU8 cmd, + NvU32 domain, + NvU8 bus, + NvU8 slot, + NvU8 function, + NvBool isMdevAttached, + void *vf_pci_info +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + rmStatus = kvgpumgrProcessVfInfo(pNv->gpu_id, cmd, domain, bus, slot, function, isMdevAttached, (vgpu_vf_pci_info *) vf_pci_info); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_create_request( + nvidia_stack_t *sp, + nv_state_t *pNv, + const NvU8 *pMdevUuid, + NvU32 vgpuTypeId, + NvU16 *vgpuId, + NvU32 gpuPciBdf, + NvBool *is_driver_vm +) +{ + THREAD_STATE_NODE threadState; + OBJSYS *pSys = SYS_GET_INSTANCE(); + void *fp = NULL; + OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys); + NV_STATUS rmStatus = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + rmStatus = kvgpumgrCreateRequestVgpu(pNv->gpu_id, pMdevUuid, + vgpuTypeId, vgpuId, gpuPciBdf); + + *is_driver_vm = pHypervisor->getProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_get_bar_info( + nvidia_stack_t *sp, + nv_state_t *pNv, + const NvU8 *pMdevUuid, + NvU64 *size, + NvU32 regionIndex, + void *pVgpuVfioRef +) +{ + REQUEST_VGPU_INFO_NODE *pRequestVgpu = NULL; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK, status; + OBJGPU *pGpu = NULL; + KernelBus *pKernelBus; + KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice; + void *fp = NULL; + NvU32 value = 0; + OBJSYS *pSys = SYS_GET_INSTANCE(); + KernelVgpuMgr * pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit); + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + if (pGpu == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__); + rmStatus = NV_ERR_INVALID_STATE; + goto release_lock; + } + pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + *size = kbusGetPciBarSize(pKernelBus, regionIndex); + + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + kvgpumgrGetHostVgpuDeviceFromMdevUuid(pNv->gpu_id, + pMdevUuid, + &pKernelHostVgpuDevice), release_lock); + + pRequestVgpu = pKernelHostVgpuDevice->pRequestVgpuInfoNode; + if (pRequestVgpu == NULL) + { + rmStatus = NV_ERR_INVALID_POINTER; + goto release_lock; + } + + pKernelHostVgpuDevice->pVgpuVfioRef = pVgpuVfioRef; + + if (regionIndex == NV_VFIO_PCI_BAR1_REGION_INDEX) + { + VGPU_TYPE *vgpuTypeInfo; + NvU32 pgpuIndex = 0; + NvBool bOverrideBar1Size = NV_FALSE; + + // Read BAR1 length from vgpuTypeInfo + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + kvgpumgrGetVgpuTypeInfo(pKernelHostVgpuDevice->vgpuType, &vgpuTypeInfo), release_lock); + + *size = vgpuTypeInfo->bar1Length << 20; + + NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, + kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex), release_lock); + + /* + * check for 'override_bar1_size' param in vgpuExtraParams list first, + * if param is missing there then check it in vgpu_params list + */ + status = nv_parse_config_params((const char*)vgpuTypeInfo->vgpuExtraParams, + "override_bar1_size", ';', &value); + + if (status == NV_OK && value) { + bOverrideBar1Size = NV_TRUE; + } else if (status == NV_ERR_OBJECT_NOT_FOUND) { + status = nv_parse_config_params(pRequestVgpu->configParams, + "override_bar1_size", ',', &value); + if (status == NV_OK && value) + bOverrideBar1Size = NV_TRUE; + } + if (bOverrideBar1Size) { + NvU64 bar1SizeInBytes, guestBar1; + NvU64 gpuBar1LowerLimit = 256 * 1024 * 1024; // bar1 lower limit for override_bar1_length parameter + + bar1SizeInBytes = kbusGetPciBarSize(pKernelBus, NV_VFIO_PCI_BAR1_REGION_INDEX); + if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].sriovEnabled) + { + *size = pGpu->sriovState.vfBarSize[1]; + } + else if (bar1SizeInBytes > gpuBar1LowerLimit) + { + guestBar1 = bar1SizeInBytes / vgpuTypeInfo->maxInstance; + *size = nvPrevPow2_U64(guestBar1); + } + } + } + else if (regionIndex == NV_VFIO_PCI_BAR2_REGION_INDEX || + regionIndex == NV_VFIO_PCI_BAR3_REGION_INDEX) + { + status = nv_parse_config_params(pRequestVgpu->configParams, + "address64", ',', &value); + + if ((status != NV_OK) || ((status == NV_OK) && (value != 0))) + { + if (regionIndex == NV_VFIO_PCI_BAR2_REGION_INDEX) + *size = 0; + else if (regionIndex == NV_VFIO_PCI_BAR3_REGION_INDEX) + *size = kbusGetPciBarSize(pKernelBus, NV_VFIO_PCI_BAR2_REGION_INDEX); + } + } + +release_lock: + // UNLOCK: release API lock + rmapiLockRelease(); + +exit: + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS osVgpuVfioWake( + void *waitQueue +) +{ + vgpu_vfio_info vgpu_info; + + vgpu_info.waitQueue = waitQueue; + + return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VGPU_VFIO_WAKE_WAIT_QUEUE); +} + +NV_STATUS NV_API_CALL nv_vgpu_start( + nvidia_stack_t *sp, + const NvU8 *pMdevUuid, + void *waitQueue, + NvS32 *returnStatus, + NvU8 *vmName, + NvU32 qemuPid +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + rmStatus = kvgpumgrStart(pMdevUuid, waitQueue, returnStatus, + vmName, qemuPid); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +static NV_STATUS nv_parse_config_params( + const char *config_params, + const char *key, + const char delim, + NvU32 *config_value +) +{ + char *ptr, *configParams = rm_remove_spaces(config_params); + char *token, *value, *name; + NvU32 data; + NV_STATUS rmStatus = NV_ERR_OBJECT_NOT_FOUND; + + ptr = configParams; + while ((token = rm_string_token(&ptr, delim)) != NULL) + { + if (!(name = rm_string_token(&token, '=')) || !os_string_length(name)) + continue; + + if (!(value = rm_string_token(&token, '=')) || !os_string_length(value)) + continue; + + data = os_strtoul(value, NULL, 0); + + if (os_string_compare(name, key) == 0) + { + rmStatus = NV_OK; + *config_value = data; + } + } + + // Free the memory allocated by rm_remove_spaces() + os_free_mem(configParams); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_get_sparse_mmap( + nvidia_stack_t *sp , + nv_state_t *pNv, + const NvU8 *pMdevUuid, + NvU64 **offsets, + NvU64 **sizes, + NvU32 *numAreas +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_ERR_INVALID_STATE, status; + OBJGPU *pGpu = NULL; + POBJTMR pTmr = NULL; + KernelFifo *pKernelFifo = NULL; + void *fp = NULL; + REQUEST_VGPU_INFO_NODE *pRequestVgpu = NULL; + KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice; + NvU32 bar0TmrMapSize = 0, bar0FifoMapSize = 0, value = 0; + NvU64 bar0TmrMapOffset = 0, bar0FifoMapOffset = 0; + NvU64 *vfRegionSizes = NULL; + NvU64 *vfRegionOffsets = NULL; + KernelBif *pKernelBif = NULL; + + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + if (pGpu == NULL) + { + rmStatus = NV_ERR_INVALID_STATE; + goto cleanup; + } + pTmr = GPU_GET_TIMER(pGpu); + pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); + pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + *numAreas = 0; + rmStatus = kvgpumgrGetHostVgpuDeviceFromMdevUuid(pNv->gpu_id, pMdevUuid, + &pKernelHostVgpuDevice); + if (rmStatus == NV_OK) + { + if (pKernelHostVgpuDevice->gfid != 0) + { + rmStatus = kbifGetNumVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice, numAreas); + if (rmStatus == NV_OK) + { + os_alloc_mem((void **)&vfRegionOffsets, sizeof(NvU64) * (*numAreas)); + os_alloc_mem((void **)&vfRegionSizes, sizeof (NvU64) * (*numAreas)); + if (vfRegionOffsets && vfRegionSizes) + { + rmStatus = kbifGetVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice, os_page_size, + vfRegionOffsets, vfRegionSizes); + if (rmStatus == NV_OK) + { + *offsets = vfRegionOffsets; + *sizes = vfRegionSizes; + } + } + else + { + if (vfRegionOffsets != NULL) + os_free_mem(vfRegionOffsets); + + if (vfRegionSizes != NULL) + os_free_mem(vfRegionSizes); + + rmStatus = NV_ERR_INSUFFICIENT_RESOURCES; + } + } + } + else + { + pRequestVgpu = pKernelHostVgpuDevice->pRequestVgpuInfoNode; + if (pRequestVgpu == NULL) + { + rmStatus = NV_ERR_INVALID_POINTER; + goto cleanup; + } + + status = nv_parse_config_params(pRequestVgpu->configParams, "direct_gpu_timer_access", ',', &value); + if ((status == NV_OK) && (value != 0)) + { + rmStatus = tmrGetTimerBar0MapInfo_HAL(pGpu, pTmr, + &bar0TmrMapOffset, + &bar0TmrMapSize); + if (rmStatus == NV_OK) + (*numAreas)++; + else + NV_PRINTF(LEVEL_ERROR, + "%s Failed to get NV_PTIMER region \n", + __FUNCTION__); + } + + value = 0; + { + status = kfifoGetUsermodeMapInfo_HAL(pGpu, pKernelFifo, + &bar0FifoMapOffset, + &bar0FifoMapSize); + if (status == NV_OK) + (*numAreas)++; + } + + if (*numAreas != 0) + { + NvU32 i = 0; + NvU64 *tmpOffset, *tmpSize; + os_alloc_mem((void **)offsets, sizeof(NvU64) * (*numAreas)); + os_alloc_mem((void **)sizes, sizeof (NvU64) * (*numAreas)); + + tmpOffset = *offsets; + tmpSize = *sizes; + + if (bar0TmrMapSize != 0) + { + tmpOffset[i] = bar0TmrMapOffset; + tmpSize[i] = bar0TmrMapSize; + i++; + } + + if (bar0FifoMapSize != 0) + { + tmpOffset[i] = bar0FifoMapOffset; + tmpSize[i] = bar0FifoMapSize; + } + } + } + } + +cleanup: + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_vgpu_update_request( + nvidia_stack_t *sp , + const NvU8 *pMdevUuid, + VGPU_DEVICE_STATE deviceState, + NvU64 *offsets, + NvU64 *sizes, + const char *configParams +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_ERR_OBJECT_NOT_FOUND; + void *fp = NULL; + REQUEST_VGPU_INFO_NODE *pRequestVgpu = NULL; + OBJSYS *pSys = SYS_GET_INSTANCE(); + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (offsets != NULL) + os_free_mem(offsets); + + if (sizes != NULL) + os_free_mem(sizes); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + for (pRequestVgpu = listHead(&pKernelVgpuMgr->listRequestVgpuHead); + pRequestVgpu != NULL; + pRequestVgpu = listNext(&pKernelVgpuMgr->listRequestVgpuHead, pRequestVgpu)) + { + if (portMemCmp(pRequestVgpu->mdevUuid, pMdevUuid, VGPU_UUID_SIZE) == 0) + { + + if (configParams != NULL) + portStringCopy(pRequestVgpu->configParams, + sizeof(pRequestVgpu->configParams), + configParams, (portStringLength(configParams) + 1)); + + pRequestVgpu->deviceState = deviceState; + rmStatus = NV_OK; + } + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL nv_gpu_bind_event( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + void *fp = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK) + { + CliAddSystemEvent(NV0000_NOTIFIERS_GPU_BIND_EVENT, 0); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS osVgpuInjectInterrupt(void *vgpuVfioRef) +{ + vgpu_vfio_info vgpu_info; + + vgpu_info.vgpuVfioRef = vgpuVfioRef; + + return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VGPU_VFIO_INJECT_INTERRUPT); +} + +NV_STATUS osVgpuRegisterMdev +( + OS_GPU_INFO *pOsGpuInfo +) +{ + NV_STATUS status = NV_OK; + vgpu_vfio_info vgpu_info; + OBJSYS *pSys = SYS_GET_INSTANCE(); + KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys); + KERNEL_PHYS_GPU_INFO *pPhysGpuInfo; + NvU32 pgpuIndex, i; + OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys); + + status = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pOsGpuInfo->gpu_id, &pgpuIndex); + if (status != NV_OK) + return status; + + pPhysGpuInfo = &(pKernelVgpuMgr->pgpuInfo[pgpuIndex]); + + vgpu_info.numVgpuTypes = pKernelVgpuMgr->pgpuInfo[pgpuIndex].numVgpuTypes; + + status = os_alloc_mem((void **)&vgpu_info.vgpuTypeIds, + ((vgpu_info.numVgpuTypes) * sizeof(NvU32))); + if (status != NV_OK) + return status; + + vgpu_info.nv = pOsGpuInfo; + for (i = 0; i < pPhysGpuInfo->numVgpuTypes; i++) + { + vgpu_info.vgpuTypeIds[i] = pPhysGpuInfo->vgpuTypes[i]->vgpuTypeId; + } + + if ((!pPhysGpuInfo->sriovEnabled) || + (pHypervisor->getProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED))) + { + vgpu_info.is_virtfn = NV_FALSE; + status = os_call_vgpu_vfio((void *)&vgpu_info, CMD_VGPU_VFIO_REGISTER_MDEV); + } + else + { + for (i = 0; i < MAX_VF_COUNT_PER_GPU; i++) + { + if (pPhysGpuInfo->vfPciInfo[i].isNvidiaAttached) + { + vgpu_info.is_virtfn = NV_TRUE; + vgpu_info.domain = pPhysGpuInfo->vfPciInfo[i].domain; + vgpu_info.bus = pPhysGpuInfo->vfPciInfo[i].bus; + vgpu_info.slot = pPhysGpuInfo->vfPciInfo[i].slot; + vgpu_info.function = pPhysGpuInfo->vfPciInfo[i].function; + + status = os_call_vgpu_vfio((void *)&vgpu_info, CMD_VGPU_VFIO_REGISTER_MDEV); + if (status == NV_OK) + { + pPhysGpuInfo->vfPciInfo[i].isMdevAttached = NV_TRUE; + } + } + } + } + + os_free_mem(vgpu_info.vgpuTypeIds); + return status; +} + +NV_STATUS osIsVgpuVfioPresent(void) +{ + vgpu_vfio_info vgpu_info; + + return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VGPU_VFIO_PRESENT); +} + +void initVGXSpecificRegistry(OBJGPU *pGpu) +{ + NvU32 data32; +#if !defined(NVCPU_X86_64) + osWriteRegistryDword(pGpu, NV_REG_STR_RM_BAR2_APERTURE_SIZE_MB, 4); +#endif + osWriteRegistryDword(pGpu, NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR, + NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR_ENABLE); + if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_DUMP_NVLOG, &data32) != NV_OK)) + { + osWriteRegistryDword(pGpu, NV_REG_STR_RM_DUMP_NVLOG, + NV_REG_STR_RM_DUMP_NVLOG_ENABLE); + } + osWriteRegistryDword(pGpu, NV_REG_STR_RM_RC_WATCHDOG, + NV_REG_STR_RM_RC_WATCHDOG_DISABLE); + osWriteRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _READ, _DISABLE) | + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _WRITE, _DISABLE)); +} + + +NV_STATUS rm_is_vgpu_supported_device( + OS_GPU_INFO *pOsGpuInfo, + NvU32 pmc_boot_1 +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + POBJHYPERVISOR pHypervisor = SYS_GET_HYPERVISOR(pSys); + NvBool is_sriov_enabled = FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1); + + // if not running in vGPU mode (guest VM) return NV_OK + if (!(pHypervisor && pHypervisor->bIsHVMGuest && + (FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _PV, pmc_boot_1) || + is_sriov_enabled))) + { + return NV_OK; + } + + return NV_OK; +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/os.c b/userspace/sdks/nv525/nvalloc/unix/src/os.c new file mode 100644 index 0000000..5f65ba4 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/os.c @@ -0,0 +1,5238 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> // NV device driver interface +#include <nv-priv.h> +#include <nv-caps.h> +#include <os/os.h> +#include <nvos.h> +#include <osapi.h> +#include <ctrl/ctrl0000/ctrl0000gpu.h> +#include <ctrl/ctrl0000/ctrl0000unix.h> + +#include <nverror.h> +#include <gpu/device/device.h> + +#include "gpu/gpu.h" +#include <osfuncs.h> +#include <platform/chipset/chipset.h> + +#include "nverror.h" +#include "kernel/gpu/bif/kernel_bif.h" + +#include "gpu/mem_sys/kern_mem_sys.h" + +#include "mem_mgr/io_vaspace.h" +#include <diagnostics/journal.h> +#include "gpu/mem_mgr/mem_desc.h" +#include "core/thread_state.h" +#include <nvacpitypes.h> +#include <platform/acpi_common.h> +#include <core/locks.h> +#include <ctrl/ctrl2080/ctrl2080gpu.h> +#include "virtualization/hypervisor/hypervisor.h" +#include "rmobjexportimport.h" +#include <nvi2c.h> +#include "rmapi/rs_utils.h" +#include "rmapi/client_resource.h" +#include "os/dce_rm_client_ipc.h" +#include "mem_mgr/mem.h" +#include "gpu/mem_mgr/virt_mem_allocator_common.h" + +#include <acpidsmguids.h> +#include <pex.h> +#include "gps.h" +#include "jt.h" + + + + +extern const char *ppOsBugCheckBugcodeStr[]; + + +ct_assert(NV_RM_PAGE_SIZE == RM_PAGE_SIZE); +ct_assert(NV_RM_PAGE_MASK == RM_PAGE_MASK); +ct_assert(NV_RM_PAGE_SHIFT == RM_PAGE_SHIFT); + +typedef struct +{ + NvU32 euid; + NvU32 pid; +} TOKEN_USER, *PTOKEN_USER; + +struct OS_RM_CAPS +{ + NvU32 count; + + // This should be the last element + nv_cap_t **caps; +}; + +NvBool osIsRaisedIRQL() +{ + return (!os_semaphore_may_sleep()); +} + +NvBool osIsISR() +{ + return os_is_isr(); +} + +NV_STATUS osGetDriverBlock +( + OS_GPU_INFO *pOsGpuInfo, + OS_DRIVER_BLOCK *pBlock +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osGetCurrentTick(NvU64 *pTimeInNs) +{ + *pTimeInNs = os_get_current_tick(); + return NV_OK; +} + +NvU64 osGetTickResolution(void) +{ + return os_get_tick_resolution(); +} + +NV_STATUS osGetPerformanceCounter(NvU64 *pTimeInNs) +{ + *pTimeInNs = os_get_current_tick_hr(); + return NV_OK; +} + +NV_STATUS osGetCurrentTime( + NvU32 *pSeconds, + NvU32 *pMicroSeconds +) +{ + return os_get_current_time(pSeconds, pMicroSeconds); +} + +/*! + * @brief Get timestamp for logging. + * + * Everything that logs a time stamp should use this routine for consistency. + * + * The returned value is OS dependent. We want the time stamp to use + * KeQueryPerformanceCounter on Windows so it matches the DirectX timestamps. + * Linux uses microseconds since 1970 (osGetCurrentTime), since matching DirectX + * is not a priority. + * + * osGetTimestampFreq returns the frequency required to decode the time stamps. + * + * @returns system dependent timestamp. + */ +NvU64 osGetTimestamp(void) +{ + NvU32 sec = 0; + NvU32 usec = 0; + osGetCurrentTime(&sec, &usec); + return (NvU64)sec * 1000000 + usec; +} + +/*! + * @brief Get timestamp frequency. + * + * Timestamps are OS dependent. This call returns the frequency + * required to decode them. + * + * @returns Timestamp frequency. For example, 1000000 for MHz. + */ +NvU64 osGetTimestampFreq(void) +{ + return 1000000; +} + +NV_STATUS osDelay(NvU32 milliseconds) +{ + return os_delay(milliseconds); +} + +NV_STATUS osDelayUs(NvU32 microseconds) +{ + return os_delay_us(microseconds); +} + +NV_STATUS osDelayNs(NvU32 nanoseconds) +{ + NvU32 microseconds = NV_MAX(1, (nanoseconds / 1000)); + return os_delay_us(microseconds); +} + +NvU32 osGetCpuFrequency(void) +{ + /* convert os_get_cpu_frequency()'s return value from Hz to MHz */ + return ((NvU32)(os_get_cpu_frequency() / 1000000ULL)); +} + +void* osPciInitHandle( + NvU32 Domain, + NvU8 Bus, + NvU8 Slot, + NvU8 Function, + NvU16 *pVendor, + NvU16 *pDevice +) +{ + // + // Check if the BDF is for a GPU that's already been attached, for which + // we should already have a handle cached. This won't catch devices that + // have been probed but not yet attached, but that shouldn't be a common + // occurrence. + // + // More importantly, having this check here means we don't need to check + // a global list of devices in the kernel interface layer, which could + // have the implication of taking another lock, causing hairy lock + // ordering issues. + // + if (Function == 0) + { + OBJGPU *pGpu = gpumgrGetGpuFromBusInfo(Domain, Bus, Slot); + if (pGpu != NULL) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + if (pVendor) *pVendor = nv->pci_info.vendor_id; + if (pDevice) *pDevice = nv->pci_info.device_id; + return nv->handle; + } + } + + return os_pci_init_handle(Domain, Bus, Slot, Function, pVendor, pDevice); +} + +NvU8 osPciReadByte( + void *pHandle, + NvU32 Offset +) +{ + NvU8 val; + os_pci_read_byte(pHandle, Offset, &val); + return val; +} + +NvU16 osPciReadWord( + void *pHandle, + NvU32 Offset +) +{ + NvU16 val; + os_pci_read_word(pHandle, Offset, &val); + return val; +} + +NvU32 osPciReadDword( + void *pHandle, + NvU32 Offset +) +{ + NvU32 val; + os_pci_read_dword(pHandle, Offset, &val); + return val; +} + +void osPciWriteByte( + void *pHandle, + NvU32 Offset, + NvU8 Value +) +{ + os_pci_write_byte(pHandle, Offset, Value); +} + +void osPciWriteWord( + void *pHandle, + NvU32 Offset, + NvU16 Value +) +{ + os_pci_write_word(pHandle, Offset, Value); +} + +void osPciWriteDword( + void *pHandle, + NvU32 Offset, + NvU32 Value +) +{ + os_pci_write_dword(pHandle, Offset, Value); +} + +void* osMapKernelSpace( + RmPhysAddr Start, + NvU64 Size, + NvU32 Mode, + NvU32 Protect +) +{ + NvU64 offset; + NvU8 *ptr; + + if (0 == Size) + { + NV_ASSERT(Size != 0); + return NULL; + } + + offset = (Start & ~os_page_mask); + Start &= os_page_mask; + Size = ((Size + offset + ~os_page_mask) & os_page_mask); + + ptr = os_map_kernel_space(Start, Size, Mode); + if (ptr != NULL) + return (ptr + offset); + + return NULL; +} + +void osUnmapKernelSpace( + void *pAddress, + NvU64 Size +) +{ + NvU64 offset; + NvUPtr ptr = (NvUPtr)pAddress; + + if (0 == Size) + { + NV_ASSERT(Size != 0); + return; + } + + offset = (ptr & ~os_page_mask); + ptr &= os_page_mask; + Size = ((Size + offset + ~os_page_mask) & os_page_mask); + os_unmap_kernel_space((void *)ptr, Size); +} + +void* osMapIOSpace( + RmPhysAddr Start, + NvU64 Size, + void ** pData, + NvU32 User, + NvU32 Mode, + NvU32 Protect +) +{ + + NvU64 offset; + NvU8 *addr; + + if (0 == Size) + { + NV_ASSERT(Size != 0); + return NULL; + } + + offset = (Start & ~os_page_mask); + Start &= os_page_mask; + Size = ((Size + offset + ~os_page_mask) & os_page_mask); + + if (User) + addr = os_map_user_space(Start, Size, Mode, Protect, pData); + else + addr = os_map_kernel_space(Start, Size, Mode); + if (addr != NULL) + return (addr + offset); + + return addr; +} + +void osUnmapIOSpace( + void *pAddress, + NvU64 Size, + void *pData, + NvU32 User +) +{ + NvU64 offset; + NvUPtr addr = (NvUPtr)pAddress; + + if (0 == Size) + { + NV_ASSERT(Size != 0); + return; + } + + offset = (addr & ~os_page_mask); + addr &= os_page_mask; + Size = ((Size + offset + ~os_page_mask) & os_page_mask); + + if (User) + os_unmap_user_space((void *)addr, Size, pData); + else + os_unmap_kernel_space((void *)addr, Size); +} + +static NV_STATUS setNumaPrivData +( + KernelMemorySystem *pKernelMemorySystem, + nv_state_t *nv, + MEMORY_DESCRIPTOR *pMemDesc +) +{ + NV_STATUS rmStatus = NV_OK; + void *pAllocPrivate = NULL; + NvU64 *addrArray = NULL; + NvU64 numOsPages = pMemDesc->PageCount; + + addrArray = pMemDesc->_pteArray; + + if (NV_RM_PAGE_SIZE < os_page_size) + { + NvU64 numPages; + NvU64 i; + + numPages = pMemDesc->PageCount; + addrArray = portMemAllocNonPaged(numPages * sizeof(NvU64)); + if (addrArray == NULL) + { + return NV_ERR_NO_MEMORY; + } + + portMemCopy((void*)addrArray, + (numPages * sizeof(NvU64)), (void*)pMemDesc->_pteArray, + (numPages * sizeof(NvU64))); + RmDeflateRmToOsPageArray(addrArray, numPages); + numOsPages = NV_RM_PAGES_TO_OS_PAGES(numPages); + + for (i = 0; i < numOsPages; i++) + { + // Update GPA to system physical address + addrArray[i] += pKernelMemorySystem->coherentCpuFbBase; + } + } + + rmStatus = nv_register_phys_pages(nv, addrArray, numOsPages, NV_MEMORY_CACHED, &pAllocPrivate); + if (rmStatus != NV_OK) + { + goto errors; + } + + memdescSetMemData(pMemDesc, pAllocPrivate, NULL); + +errors: + if (NV_RM_PAGE_SIZE < os_page_size) + { + portMemFree(addrArray); + } + + return rmStatus; +} + +NV_STATUS osGetNumMemoryPages +( + MEMORY_DESCRIPTOR *pMemDesc, + NvU32 *pNumPages +) +{ + void *pAllocPrivate = NULL; + + pAllocPrivate = memdescGetMemData(pMemDesc); + if (pAllocPrivate == NULL) + { + NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n"); + return NV_ERR_INVALID_STATE; + } + + return nv_get_num_phys_pages(pAllocPrivate, pNumPages); +} + +NV_STATUS osGetMemoryPages +( + MEMORY_DESCRIPTOR *pMemDesc, + void *pPages, + NvU32 *pNumPages +) +{ + void *pAllocPrivate = NULL; + + pAllocPrivate = memdescGetMemData(pMemDesc); + if (pAllocPrivate == NULL) + { + NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n"); + return NV_ERR_INVALID_STATE; + } + + return nv_get_phys_pages(pAllocPrivate, pPages, pNumPages); +} + +NV_STATUS osMapSystemMemory +( + MEMORY_DESCRIPTOR *pMemDesc, + NvU64 Offset, + NvU64 Length, + NvBool Kernel, + NvU32 Protect, + NvP64 *ppAddress, + NvP64 *ppPrivate +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + + RmPhysAddr userAddress; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NV_STATUS rmStatus = NV_OK; + void *pAllocPrivate = NULL; + void *pAddress; + void *pPrivate = NULL; + NvU64 pageIndex; + NvU32 pageOffset; + + if (pGpu != NULL && + pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) && + memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM) + { + KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); + + rmStatus = setNumaPrivData(pKernelMemorySystem, nv, pMemDesc); + if (rmStatus != NV_OK) + return rmStatus; + } + + *ppAddress = NvP64_NULL; + *ppPrivate = NvP64_NULL; + + if ((Offset + Length) < Length) + return NV_ERR_INVALID_ARGUMENT; + if ((Offset + Length) > pMemDesc->Size) + return NV_ERR_INVALID_ARGUMENT; + + pageIndex = (Offset >> os_page_shift); + pageOffset = (Offset & ~os_page_mask); + + pAllocPrivate = memdescGetMemData(pMemDesc); + if (!pAllocPrivate) + { + NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n"); + return NV_ERR_INVALID_STATE; + } + + if (Kernel) + { + pAddress = nv_alloc_kernel_mapping(nv, pAllocPrivate, + pageIndex, pageOffset, Length, &pPrivate); + if (pAddress == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "failed to create system memory kernel mapping!\n"); + rmStatus = NV_ERR_GENERIC; + } + else + { + *ppAddress = NV_PTR_TO_NvP64(pAddress); + *ppPrivate = NV_PTR_TO_NvP64(pPrivate); + } + } + else + { + rmStatus = nv_alloc_user_mapping(nv, pAllocPrivate, + pageIndex, pageOffset, Length, Protect, &userAddress, + &pPrivate); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "failed to create system memory user mapping!\n"); + } + else + { + *ppAddress = (NvP64)(userAddress); + *ppPrivate = NV_PTR_TO_NvP64(pPrivate); + } + } + + return rmStatus; +} + +void osUnmapSystemMemory +( + MEMORY_DESCRIPTOR *pMemDesc, + NvBool Kernel, + NvU32 ProcessId, + NvP64 pAddress, + NvP64 pPrivate +) +{ + NV_STATUS status; + void *pAllocPrivate = memdescGetMemData(pMemDesc); + OBJGPU *pGpu = pMemDesc->pGpu; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if (Kernel) + { + status = nv_free_kernel_mapping(nv, pAllocPrivate, NvP64_VALUE(pAddress), + NvP64_VALUE(pPrivate)); + } + else + { + status = nv_free_user_mapping(nv, pAllocPrivate, (NvU64)pAddress, + NvP64_VALUE(pPrivate)); + } + + if (pGpu != NULL && + pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) && + memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM && + pAllocPrivate != NULL) + { + nv_unregister_phys_pages(nv, pAllocPrivate); + memdescSetMemData(pMemDesc, NULL, NULL); + } + + NV_ASSERT(status == NV_OK); +} + +void osIoWriteByte( + NvU32 Address, + NvU8 Value +) +{ + os_io_write_byte(Address, Value); +} + +NvU16 osIoReadWord( + NvU32 Address +) +{ + return os_io_read_word(Address); +} + +void osIoWriteWord( + NvU32 Address, + NvU16 Value +) +{ + os_io_write_word(Address, Value); +} + +NvU8 osIoReadByte( + NvU32 Address +) +{ + return os_io_read_byte(Address); +} + +NvBool osIsAdministrator(void) +{ + return os_is_administrator(); +} + +NvBool osAllowPriorityOverride(void) +{ + return os_allow_priority_override(); +} + +NvU32 osGetCurrentProcess(void) +{ + return os_get_current_process(); +} + +void osGetCurrentProcessName(char *ProcName, NvU32 Length) +{ + return os_get_current_process_name(ProcName, Length); +} + +NV_STATUS osGetCurrentThread(OS_THREAD_HANDLE *pThreadId) +{ + NV_STATUS rmStatus; + NvU64 threadId = 0; + + if (pThreadId == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + rmStatus = os_get_current_thread(&threadId); + if (rmStatus == NV_OK) + { + *pThreadId = threadId; + } + else + { + *pThreadId = 0; + } + + return rmStatus; +} + +NV_STATUS osAttachToProcess(void** ppProcessInfo, NvU32 ProcessId) +{ + // + // This function is used by RmUnmapMemory() to attach to the + // process for which a given device memory mapping was + // created, in order to be able to unmap it. On Linux/UNIX + // platforms, we can't "attach" to a random process, but + // since we don't create/destroy user mappings in the RM, we + // don't need to, either. + // + // Report success to the caller to keep RmUnmapMemory() from + // failing, and memory from being leaked as a result. + // + *ppProcessInfo = NULL; + return NV_OK; +} + +void osDetachFromProcess(void* pProcessInfo) +{ + // stub + return; +} + +NvBool osDbgBreakpointEnabled(void) +{ + return NV_TRUE; +} + +NV_STATUS osAcquireRmSema(void *pSema) +{ + return NV_OK; +} + +NV_STATUS osCondAcquireRmSema(void *pSema) +{ + return NV_OK; +} + +NvU32 osReleaseRmSema(void *pSema, OBJGPU *pDpcGpu) +{ + return NV_SEMA_RELEASE_SUCCEED; +} + +void osSpinLoop(void) +{ + // Enable this code to get debug prints from Libos. +} + +NvU64 osGetMaxUserVa(void) +{ + return os_get_max_user_va(); +} + +NV_STATUS osSchedule(void) +{ + return os_schedule(); +} + +NV_STATUS osQueueWorkItemWithFlags( + OBJGPU *pGpu, + OSWorkItemFunction pFunction, + void *pParams, + NvU32 flags +) +{ + nv_work_item_t *pWi; + nv_state_t *nv; + NV_STATUS status; + + pWi = portMemAllocNonPaged(sizeof(nv_work_item_t)); + + if (NULL == pWi) + { + return NV_ERR_NO_MEMORY; + } + + pWi->flags = NV_WORK_ITEM_FLAGS_REQUIRES_GPU; + if (flags & OS_QUEUE_WORKITEM_FLAGS_DONT_FREE_PARAMS) + pWi->flags |= NV_WORK_ITEM_FLAGS_DONT_FREE_DATA; + + if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA) + pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA; + if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW) + pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW; + if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW) + pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW; + if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW) + pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW; + if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW) + pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW; + + if (flags & OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY) + pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY; + + pWi->gpuInstance = gpuGetInstance(pGpu); + pWi->func.pGpuFunction = pFunction; + pWi->pData = pParams; + nv = NV_GET_NV_STATE(pGpu); + + status = os_queue_work_item(nv ? nv->queue : NULL, pWi); + + if (NV_OK != status) + { + portMemFree((void *)pWi); + } + + return status; +} + +NV_STATUS osQueueWorkItem( + OBJGPU *pGpu, + OSWorkItemFunction pFunction, + void *pParams +) +{ + return osQueueWorkItemWithFlags(pGpu, pFunction, pParams, OS_QUEUE_WORKITEM_FLAGS_NONE); +} + +NV_STATUS osQueueSystemWorkItem( + OSSystemWorkItemFunction pFunction, + void *pParams +) +{ + nv_work_item_t *pWi; + NV_STATUS status; + + pWi = portMemAllocNonPaged(sizeof(nv_work_item_t)); + + if (NULL == pWi) + { + return NV_ERR_NO_MEMORY; + } + + pWi->flags = NV_WORK_ITEM_FLAGS_NONE; + pWi->func.pSystemFunction = pFunction; + pWi->pData = pParams; + + status = os_queue_work_item(NULL, pWi); + + if (NV_OK != status) + { + portMemFree((void *)pWi); + } + + return status; +} + +void osQueueMMUFaultHandler(OBJGPU *pGpu) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + nv_schedule_uvm_isr(nv); +} + +static inline nv_dma_device_t* osGetDmaDeviceForMemDesc( + OS_GPU_INFO *pOsGpuInfo, + MEMORY_DESCRIPTOR *pMemDesc +) +{ + return (pOsGpuInfo->niso_dma_dev != NULL) && + memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) ? + pOsGpuInfo->niso_dma_dev : pOsGpuInfo->dma_dev; +} + +NV_STATUS osDmaMapPages( + OS_GPU_INFO *pOsGpuInfo, + MEMORY_DESCRIPTOR *pMemDesc +) +{ + return nv_dma_map_pages( + osGetDmaDeviceForMemDesc(pOsGpuInfo, pMemDesc), + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + memdescGetPteArray(pMemDesc, AT_CPU), + memdescGetContiguity(pMemDesc, AT_CPU), + memdescGetCpuCacheAttrib(pMemDesc), + NULL); +} + +NV_STATUS osDmaUnmapPages( + OS_GPU_INFO *pOsGpuInfo, + MEMORY_DESCRIPTOR *pMemDesc +) +{ + return nv_dma_unmap_pages( + pOsGpuInfo->dma_dev, + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + memdescGetPteArray(pMemDesc, AT_CPU), + NULL); +} + +// +// Set the DMA address size for the given GPU +// +// This is a global device setting and care would need to be taken if it was to +// be modified outside of GPU initialization. At least on Linux other drivers, +// like UVM, might be requesting its own DMA mappings for the same GPU after +// the GPU has been initialized. +// +void osDmaSetAddressSize( + OS_GPU_INFO *pOsGpuInfo, + NvU32 bits +) +{ + nv_set_dma_address_size(pOsGpuInfo, bits); +} + +NV_STATUS osAllocPagesInternal( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJGPU *pGpu = pMemDesc->pGpu; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + void *pMemData = NULL; + NV_STATUS status; + + memdescSetAddress(pMemDesc, NvP64_NULL); + memdescSetMemData(pMemDesc, NULL, NULL); + + NV_ASSERT_OR_RETURN(pMemDesc->PageCount > 0, NV_ERR_INVALID_ARGUMENT); + + if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED)) + { + if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + status = nv_alias_pages( + NV_GET_NV_STATE(pGpu), + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + memdescGetContiguity(pMemDesc, AT_CPU), + memdescGetCpuCacheAttrib(pMemDesc), + memdescGetGuestId(pMemDesc), + memdescGetPteArray(pMemDesc, AT_CPU), + &pMemData); + } + else + { + NvBool unencrypted = 0; + + if (nv && (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_32BIT_ADDRESSABLE))) + nv->force_dma32_alloc = NV_TRUE; + + status = nv_alloc_pages( + NV_GET_NV_STATE(pGpu), + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + memdescGetContiguity(pMemDesc, AT_CPU), + memdescGetCpuCacheAttrib(pMemDesc), + pSys->getProperty(pSys, + PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS), + unencrypted, + memdescGetPteArray(pMemDesc, AT_CPU), + &pMemData); + + if (nv && nv->force_dma32_alloc) + nv->force_dma32_alloc = NV_FALSE; + } + + if (status != NV_OK) + { + return status; + } + + // + // If the OS layer doesn't think in RM page size, we need to inflate the + // PTE array into RM pages. + // + if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + memdescSetMemData(pMemDesc, pMemData, NULL); + + return status; +} + +void osFreePagesInternal( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + NV_STATUS rmStatus; + + if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + rmStatus = nv_free_pages(NV_GET_NV_STATE(pGpu), + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + memdescGetContiguity(pMemDesc, AT_CPU), + memdescGetCpuCacheAttrib(pMemDesc), + memdescGetMemData(pMemDesc)); + NV_ASSERT(rmStatus == NV_OK); +} + +NV_STATUS osLockMem( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + // Not supported on this OS. + DBG_BREAKPOINT(); + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osUnlockMem( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + // Not supported on this OS. + DBG_BREAKPOINT(); + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osMapPciMemoryUser( + OS_GPU_INFO *pOsGpuInfo, + RmPhysAddr busAddress, + NvU64 length, + NvU32 Protect, + NvP64 *pVirtualAddress, + NvP64 *pPriv, + NvU32 modeFlag +) +{ + void *addr; + void *priv = NULL; + + addr = osMapIOSpace(busAddress, length, &priv, NV_TRUE, modeFlag, Protect); + + *pPriv = NV_PTR_TO_NvP64(priv); + *pVirtualAddress = NV_PTR_TO_NvP64(addr); + + return (addr != NULL) ? NV_OK : NV_ERR_GENERIC; +} + +void osUnmapPciMemoryUser( + OS_GPU_INFO *pOsGpuInfo, + NvP64 virtualAddress, + NvU64 length, + NvP64 pPriv +) +{ + void *addr, *priv; + + addr = NvP64_VALUE(virtualAddress); + priv = NvP64_VALUE(pPriv); + + osUnmapIOSpace(addr, length, priv, NV_TRUE); +} + +NV_STATUS osMapPciMemoryKernelOld +( + OBJGPU *pGpu, + RmPhysAddr busAddress, + NvU64 length, + NvU32 Protect, + void **pVirtualAddress, + NvU32 modeFlag +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_kern_mapping_t *mapping; + + if (pVirtualAddress == NULL) + return NV_ERR_GENERIC; + + *pVirtualAddress = os_map_kernel_space(busAddress, length, modeFlag); + if (*pVirtualAddress == NULL) + return NV_ERR_GENERIC; + + mapping = portMemAllocNonPaged(sizeof(nv_kern_mapping_t)); + if (NULL == mapping) + { + os_unmap_kernel_space(*pVirtualAddress, length); + *pVirtualAddress = 0; + return NV_ERR_GENERIC; + } + + mapping->addr = *pVirtualAddress; + mapping->size = length; + mapping->modeFlag = modeFlag; + + mapping->next = nv->kern_mappings; + nv->kern_mappings = mapping; + + return NV_OK; +} + +NV_STATUS osMapPciMemoryKernel64 +( + OBJGPU *pGpu, + RmPhysAddr busAddress, + NvU64 length, + NvU32 Protect, + NvP64 *pVirtualAddress, + NvU32 modeFlag +) +{ + void *tmppVirtualAddress = NvP64_VALUE(pVirtualAddress); + NV_STATUS rc; + + rc = osMapPciMemoryKernelOld(pGpu, + busAddress, + length, + Protect, + &tmppVirtualAddress, + modeFlag); + + *pVirtualAddress = NV_PTR_TO_NvP64(tmppVirtualAddress); + + return rc; +} + +void osUnmapPciMemoryKernelOld +( + OBJGPU *pGpu, + void* virtualAddress +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_kern_mapping_t *mapping, *tmp; + + // this can happen, for example, during a call to RmShutdownAdapter() + // from a failed RmInitAdapter() + if (virtualAddress == NULL) + { + return; + } + + tmp = mapping = nv->kern_mappings; + while (mapping) + { + if (mapping->addr == virtualAddress) + { + if (mapping == nv->kern_mappings) + { + nv->kern_mappings = mapping->next; + } + else + { + tmp->next = mapping->next; + } + + os_unmap_kernel_space(mapping->addr, mapping->size); + + portMemFree(mapping); + return; + } + tmp = mapping; + mapping = mapping->next; + } + + DBG_BREAKPOINT(); +} + +void osUnmapPciMemoryKernel64 +( + OBJGPU *pGpu, + NvP64 virtualAddress +) +{ + osUnmapPciMemoryKernelOld(pGpu, NvP64_VALUE(virtualAddress)); +} + +NV_STATUS osMapGPU( + OBJGPU *pGpu, + RS_PRIV_LEVEL privLevel, + NvU64 offset, + NvU64 length, + NvU32 Protect, + NvP64 *pAddress, + NvP64 *pPriv +) +{ + NV_STATUS rmStatus = NV_OK; + + if (privLevel >= RS_PRIV_LEVEL_KERNEL) + { + if (!portSafeAddU64((NvUPtr)pGpu->deviceMappings[0].gpuNvAddr, offset, (NvU64*)pAddress)) + { + rmStatus = NV_ERR_INVALID_LIMIT; + } + } + else + { + RmPhysAddr busAddress; + if (!portSafeAddU64(pGpu->busInfo.gpuPhysAddr, offset, &busAddress)) + { + rmStatus = NV_ERR_INVALID_LIMIT; + } + else + { + rmStatus = osMapPciMemoryUser(pGpu->pOsGpuInfo, + busAddress, + length, + Protect, + pAddress, + pPriv, + NV_FALSE); + } + } + + return rmStatus; +} + +void osUnmapGPU( + OS_GPU_INFO *pOsGpuInfo, + RS_PRIV_LEVEL privLevel, + NvP64 address, + NvU64 length, + NvP64 priv +) +{ + if (privLevel < RS_PRIV_LEVEL_KERNEL) + { + osUnmapPciMemoryUser(pOsGpuInfo, address, length, priv); + } +} + +NV_STATUS osDeviceClassToDeviceName( + NvU32 deviceInstance, + NvU8 *szName +) +{ + return NV_ERR_GENERIC; +} + +static void postEvent( + nv_event_t *event, + NvU32 hEvent, + NvU32 notifyIndex, + NvU32 info32, + NvU16 info16, + NvBool dataValid +) +{ + nv_state_t *nv = nv_get_ctl_state(); + portSyncSpinlockAcquire(nv->event_spinlock); + if (event->active) + nv_post_event(event, hEvent, notifyIndex, + info32, info16, dataValid); + portSyncSpinlockRelease(nv->event_spinlock); +} + +NvU32 osSetEvent +( + OBJGPU *pGpu, + NvP64 eventID +) +{ + nv_event_t *event = NvP64_VALUE(eventID); + postEvent(event, 0, 0, 0, 0, NV_FALSE); + return 1; +} + +NV_STATUS osNotifyEvent( + OBJGPU *pGpu, + PEVENTNOTIFICATION NotifyEvent, + NvU32 Method, + NvU32 Data, + NV_STATUS Status +) +{ + NV_STATUS rmStatus = NV_OK; + + // notify the event + switch (NotifyEvent->NotifyType) + { + case NV01_EVENT_OS_EVENT: + { + nv_event_t *event = NvP64_VALUE(NotifyEvent->Data); + postEvent(event, + NotifyEvent->hEvent, + NotifyEvent->NotifyIndex, + 0, 0, + NotifyEvent->bEventDataRequired); + break; + } + + // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX. + case NV01_EVENT_KERNEL_CALLBACK: + { + MINIPORT_CALLBACK callBackToMiniport = + (MINIPORT_CALLBACK)NvP64_VALUE(NotifyEvent->Data); + + // perform a direct callback to the miniport + if (callBackToMiniport) + callBackToMiniport(NV_GET_NV_STATE(pGpu)); + break; + } + + case NV01_EVENT_KERNEL_CALLBACK_EX: + { + NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(NotifyEvent->Data); + + // passes two arguments (arg, params) to the kernel callback instead of one (arg). + if (kc && kc->func) + { + kc->func(kc->arg, NULL, NotifyEvent->hEvent, Data, Status); + } + break; + } + + + default: + { + rmStatus = NV_ERR_GENERIC; + break; + } + } + + return rmStatus; + +} // end of osNotifyEvent() + +// Allow CPL Events to be callback or events +NV_STATUS osEventNotification +( + OBJGPU *pGpu, + PEVENTNOTIFICATION pNotifyEvent, + NvU32 notifyIndex, + void * pEventData, + NvU32 eventDataSize +) +{ + return osEventNotificationWithInfo(pGpu, pNotifyEvent, notifyIndex, 0, 0, + pEventData, eventDataSize); +} + +NV_STATUS osEventNotificationWithInfo +( + OBJGPU *pGpu, + PEVENTNOTIFICATION pNotifyEvent, + NvU32 notifyIndex, + NvU32 info32, + NvU16 info16, + void * pEventData, + NvU32 eventDataSize +) +{ + NV_STATUS rmStatus = NV_OK; + + // walk this object's event list and find any matches for this specific notify + for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next) + { + // notifyIndex must match if request isn't for all + if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) && + (pNotifyEvent->NotifyIndex != notifyIndex)) + { + continue; + } + + switch (pNotifyEvent->NotifyType) + { + case NV_EVENT_BUFFER_BIND: + case NV01_EVENT_WIN32_EVENT: + { + nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data); + postEvent(event, + pNotifyEvent->hEvent, + pNotifyEvent->NotifyIndex, + info32, info16, + pNotifyEvent->bEventDataRequired); + break; + } + + // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX. + case NV01_EVENT_KERNEL_CALLBACK: + { + MINIPORT_CALLBACK callBackToMiniport = + (MINIPORT_CALLBACK)NvP64_VALUE(pNotifyEvent->Data); + + // perform a direct callback to the miniport + if (callBackToMiniport) + callBackToMiniport(NV_GET_NV_STATE(pGpu)); + break; + } + + case NV01_EVENT_KERNEL_CALLBACK_EX: + { + NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data); + + if (kc && kc->func) + { + kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK); + } + break; + } + + default: + break; + } + } + + return rmStatus; +} + +// Allow CPL Events to be callback or events +NV_STATUS osObjectEventNotification +( + NvHandle hClient, + NvHandle hObject, + NvU32 hClass, + PEVENTNOTIFICATION pNotifyEvent, + NvU32 notifyIndex, + void *pEventData, + NvU32 eventDataSize +) +{ + NV_STATUS rmStatus = NV_OK; + + NV_PRINTF(LEVEL_INFO, "%s()\n", __FUNCTION__); + // walk this object's event list and find any matches for this specific notify + for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next) + { + // notifyIndex must match if request isn't for all + if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) && + (pNotifyEvent->NotifyIndex != notifyIndex)) + { + continue; + } + + switch (pNotifyEvent->NotifyType) + { + case NV01_EVENT_OS_EVENT: + { + nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data); + postEvent(event, + pNotifyEvent->hEvent, + pNotifyEvent->NotifyIndex, + 0, 0, + pNotifyEvent->bEventDataRequired); + break; + } + + case NV01_EVENT_KERNEL_CALLBACK_EX: + { + NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data); + + if (kc && kc->func) + { + kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK); + } + break; + } + + default: + break; + } + } + + return rmStatus; +} + +NV_STATUS osReferenceObjectCount(void *pEvent) +{ + nv_state_t *nv = nv_get_ctl_state(); + nv_event_t *event = pEvent; + + portSyncSpinlockAcquire(nv->event_spinlock); + ++event->refcount; + portSyncSpinlockRelease(nv->event_spinlock); + return NV_OK; +} + +NV_STATUS osDereferenceObjectCount(void *pOSEvent) +{ + nv_state_t *nv = nv_get_ctl_state(); + nv_event_t *event = pOSEvent; + + portSyncSpinlockAcquire(nv->event_spinlock); + NV_ASSERT(event->refcount > 0); + --event->refcount; + // If event->refcount == 0 but event->active is true, the client + // has not yet freed the OS event. free_os_event will free its + // memory when they do, or else when the client itself is freed. + if (event->refcount == 0 && !event->active) + portMemFree(event); + portSyncSpinlockRelease(nv->event_spinlock); + + return NV_OK; +} + +NV_STATUS osUserHandleToKernelPtr(NvHandle hClient, NvP64 hEvent, NvP64 *pEvent) +{ + nv_state_t *nv = nv_get_ctl_state(); + NvU32 fd = (NvU64)hEvent; + NV_STATUS result; + + portSyncSpinlockAcquire(nv->event_spinlock); + nv_event_t *e = nv->event_list; + while (e != NULL) + { + if (e->fd == fd && e->hParent == hClient) + break; + e = e->next; + } + + if (e != NULL) + { + ++e->refcount; + *pEvent = NV_PTR_TO_NvP64(e); + result = NV_OK; + } + else + result = NV_ERR_OBJECT_NOT_FOUND; + portSyncSpinlockRelease(nv->event_spinlock); + + return result; +} + +NV_STATUS osFlushCpuCache(void) +{ + return os_flush_cpu_cache_all(); +} + +void osFlushCpuWriteCombineBuffer(void) +{ + os_flush_cpu_write_combine_buffer(); +} + + +// +// Evict GPU memory range from the CPU caches. +// +// On some platforms (e.g. P9+V100), the CPU can coherently cache GPU memory +// and RM takes advantage of that. Most everything is handled transparently, +// but there are two exceptions that require explicitly flushing any CPU cache +// lines of GPU memory. These are: +// +// 1) Flushing memory backing ACR regions before they get locked. +// +// Otherwise the cache could get flushed while the regions are locked causing a +// region violation physical fault. See more details in +// acrFlushRegionsFromGpuCoherentCpuCache_IMPL(). +// +// 2) Flushing all of FB before GPU reset (NVLink going down) +// +// Leaving cache entries on the CPU causes fatal errors when the CPU tries +// flushing them later while the link is down. See more details in +// nvlinkStatePostUnload_IMPL(). +// +void osFlushGpuCoherentCpuCacheRange +( + OS_GPU_INFO *pOsGpuInfo, + NvU64 cpuVirtual, + NvU64 size +) +{ + nv_flush_coherent_cpu_cache_range(pOsGpuInfo, cpuVirtual, size); +} + +void osErrorLogV(OBJGPU *pGpu, NvU32 num, const char * pFormat, va_list arglist) +{ + NV_STATUS rmStatus; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((pFormat == NULL) || (*pFormat == '\0')) + { + return; + } + + rmStatus = nv_log_error(nv, num, pFormat, arglist); + NV_ASSERT(rmStatus == NV_OK); +} + +void osErrorLog(OBJGPU *pGpu, NvU32 num, const char* pFormat, ...) +{ + va_list arglist; + va_start(arglist, pFormat); + osErrorLogV(pGpu, num, pFormat, arglist); + va_end(arglist); +} + +NvU32 +osPollHotkeyState +( + OBJGPU *pGpu +) +{ + return 0; +} + +void osDevWriteReg008( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress, + NvV8 thisValue +) +{ + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + return; + } + + NV_PRIV_REG_WR08(pMapping->gpuNvAddr, thisAddress, thisValue); +} + +void osDevWriteReg016( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress, + NvV16 thisValue +) +{ + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + return; + } + + NV_PRIV_REG_WR16(pMapping->gpuNvAddr, thisAddress, thisValue); +} + +void osDevWriteReg032( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress, + NvV32 thisValue +) +{ + NvBool vgpuHandled = NV_FALSE; + + vgpuDevWriteReg032(pGpu, thisAddress, thisValue, &vgpuHandled); + if (vgpuHandled) + { + return; + } + + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + return; + } + + NV_PRIV_REG_WR32(pMapping->gpuNvAddr, thisAddress, thisValue); +} + +NvU8 osDevReadReg008( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress +) +{ + NvU8 retval = 0; + + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + } + else + retval = NV_PRIV_REG_RD08(pMapping->gpuNvAddr, thisAddress); + + return retval; +} + +NvU16 osDevReadReg016( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress +) +{ + NvU16 retval = 0; + + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + } + else + retval = NV_PRIV_REG_RD16(pMapping->gpuNvAddr, thisAddress); + + return retval; +} + +NvU32 osDevReadReg032( + OBJGPU *pGpu, + DEVICE_MAPPING *pMapping, + NvU32 thisAddress +) +{ + NvU32 retval = 0; + NvBool vgpuHandled = NV_FALSE; + + retval = vgpuDevReadReg032(pGpu, thisAddress, &vgpuHandled); + if (vgpuHandled) + { + return retval; + } + + if (thisAddress >= pMapping->gpuNvLength) + { + NV_ASSERT(thisAddress < pMapping->gpuNvLength); + } + else + retval = NV_PRIV_REG_RD32(pMapping->gpuNvAddr, thisAddress); + + return retval; +} + +NV_STATUS osReadRegistryDwordBase( + OBJGPU *pGpu, + const char *regParmStr, + NvU32 *Data +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmReadRegistryDword(nv, regParmStr, Data); +} + +NV_STATUS osWriteRegistryDword( + OBJGPU *pGpu, + const char *regParmStr, + NvU32 Data +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmWriteRegistryDword(nv, regParmStr, Data); +} + +NV_STATUS osReadRegistryBinary( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *Data, + NvU32 *cbLen +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmReadRegistryBinary(nv, regParmStr, Data, cbLen); +} + +NV_STATUS osWriteRegistryBinary( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmWriteRegistryBinary(nv, regParmStr, Data, cbLen); +} + +NV_STATUS osWriteRegistryVolatile( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osReadRegistryVolatile +( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osReadRegistryVolatileSize +( + OBJGPU *pGpu, + const char *regParmStr, + NvU32 *pSize +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS osReadRegistryStringBase( + OBJGPU *pGpu, + const char *regParmStr, + NvU8 *buffer, + NvU32 *pBufferLength +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmReadRegistryString(nv, regParmStr, buffer, pBufferLength); +} + +NV_STATUS osPackageRegistry( + OBJGPU *pGpu, + PACKED_REGISTRY_TABLE *pRegTable, + NvU32 *pSize +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return RmPackageRegistry(nv, pRegTable, pSize); +} + +NvU32 osGetCpuCount() +{ + return os_get_cpu_count(); // Total number of logical CPUs. +} + +NvU32 osGetCurrentProcessorNumber(void) +{ + return os_get_cpu_number(); +} + +void osGetTimeoutParams(OBJGPU *pGpu, NvU32 *pTimeoutUs, NvU32 *pScale, NvU32 *pFlags) +{ + NvU32 gpuMode = gpuGetMode(pGpu); + + NV_ASSERT((NV_GPU_MODE_GRAPHICS_MODE == gpuMode) || + (NV_GPU_MODE_COMPUTE_MODE == gpuMode)); + + if (hypervisorIsVgxHyper()) + { + // + // 1.8 seconds is chosen because it is 90% of the overall hard limit of 2.0 + // seconds, imposed by WDDM driver rules. + // Currently primary use case of VGX is Windows, so setting 1.8 as default + // + *pTimeoutUs = 1.8 * 1000000; + } + else + { + switch (gpuMode) + { + default: + case NV_GPU_MODE_GRAPHICS_MODE: + *pTimeoutUs = 4 * 1000000; + break; + + case NV_GPU_MODE_COMPUTE_MODE: + *pTimeoutUs = 30 * 1000000; + break; + } + } + + *pFlags = GPU_TIMEOUT_FLAGS_OSTIMER; + + *pScale = 1; + if (IS_EMULATION(pGpu) || IS_SIMULATION(pGpu)) + { + *pScale = 60; // 1s -> 1m + } + return; +} + +void osFlushLog() +{ + // Not implemented +} + +NvU32 osGetSimulationMode(void) +{ + return NV_SIM_MODE_HARDWARE; +} + +NV_STATUS +cliresCtrlCmdOsUnixFlushUserCache_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS *pAddressSpaceParams +) +{ + Memory *pMemory; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 start, end; + NvBool bInvalidateOnly; + + NV_CHECK_OK_OR_RETURN(LEVEL_SILENT, + memGetByHandle(RES_GET_CLIENT(pRmCliRes), + pAddressSpaceParams->hObject, + &pMemory)); + + pMemDesc = pMemory->pMemDesc; + + if (memdescGetAddressSpace(pMemDesc) != ADDR_SYSMEM) + { + NV_PRINTF(LEVEL_ERROR, "%s: wrong address space %d\n", + __FUNCTION__, memdescGetAddressSpace(pMemDesc)); + return NV_ERR_INVALID_COMMAND; + } + + if (memdescGetCpuCacheAttrib(pMemDesc) != NV_MEMORY_CACHED) + { + NV_PRINTF(LEVEL_ERROR, "%s: wrong caching type %d\n", + __FUNCTION__, memdescGetCpuCacheAttrib(pMemDesc)); + return NV_ERR_INVALID_COMMAND; + } + + start = pAddressSpaceParams->offset; + end = start + pAddressSpaceParams->length; + + switch(pAddressSpaceParams->cacheOps) + { + case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH_INVALIDATE: + case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH: + bInvalidateOnly = NV_FALSE; + break; + + case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_INVALIDATE: + bInvalidateOnly = NV_TRUE; + break; + + default: + NV_PRINTF(LEVEL_ERROR, "%s: cacheOps not specified\n", __FUNCTION__); + return NV_ERR_INVALID_COMMAND; + } + + if ((end - start) > pMemory->Length) + { + NV_PRINTF(LEVEL_ERROR, + "%s: end address 0x%llx exceeded buffer length: 0x%llx\n", + __FUNCTION__, end, pMemory->Length); + return NV_ERR_INVALID_LIMIT; + } + + if (bInvalidateOnly) + { + // + // XXX: this seems fishy - I'm not sure if invalidating by the kernel + // VA only as nv_dma_cache_invalidate() does here is sufficient for + // this control call. + // pAddressSpaceParams->internalOnly is expected to be the RM client + // VA for this control call; if we wanted to invalidate the user VA we + // could do so using that. + // + // For I/O coherent platforms this won't actually do anything. + // On non-I/O-coherent platforms, there's no need to do a second + // invalidation after the full flush. + // + nv_state_t *nv = NV_GET_NV_STATE(pMemDesc->pGpu); + if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) + { + PIOVAMAPPING pIovaMapping = memdescGetIommuMap(pMemDesc, nv->iovaspace_id); + // + // This should only be called for devices that map memory descriptors + // through the nv-dma library, where the memory descriptor data + // contains all the kernel-specific context we need for the + // invalidation. + // + // (These checks match those in osIovaUnmap() leading up to + // nv_dma_unmap_alloc()). + // + if (pIovaMapping == NULL || + pIovaMapping->pOsData == NULL || + memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED) || + memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + nv_dma_cache_invalidate(nv->dma_dev, pIovaMapping->pOsData); + } + else + { + return NV_ERR_INVALID_ARGUMENT; + } + } + else + { + return os_flush_user_cache(); + } + + return NV_OK; +} + +static NV_STATUS +_initializeExportObjectFd +( + nv_file_private_t *nvfp, + NvHandle hClient, + NvHandle hDevice, + NvU16 maxObjects, + NvU8 *metadata +) +{ + NV_STATUS status; + RsResourceRef *pResourceRef; + Device *pDevice; + NvU32 deviceInstance = NV_MAX_DEVICES; + + if (nvfp->handles != NULL) + { + return NV_ERR_STATE_IN_USE; + } + + if (hDevice != 0) + { + status = serverutilGetResourceRef(hClient, hDevice, &pResourceRef); + if (status != NV_OK) + { + return status; + } + + pDevice = dynamicCast(pResourceRef->pResource, Device); + if (pDevice == NULL) + { + return NV_ERR_INVALID_PARAMETER; + } + + deviceInstance = pDevice->deviceInst; + } + + NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void **)&nvfp->handles, + sizeof(nvfp->handles[0]) * maxObjects)); + + os_mem_set(nvfp->handles, 0, + sizeof(nvfp->handles[0]) * maxObjects); + + nvfp->maxHandles = maxObjects; + nvfp->deviceInstance = deviceInstance; + + if (metadata != NULL) + { + os_mem_copy(nvfp->metadata, metadata, sizeof(nvfp->metadata)); + } + + return NV_OK; +} + +NV_STATUS +cliresCtrlCmdOsUnixExportObjectToFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS *pParams +) +{ + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + RmObjExportHandle hExportHandle = 0; + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + + /* + * This flag is intended to be implemented entirely in the rmapi library in + * userspace, we should never encounter it here. + */ + if (FLD_TEST_DRF(0000_CTRL, _OS_UNIX_EXPORT_OBJECT_TO_FD_FLAGS, + _EMPTY_FD, _TRUE, pParams->flags)) + { + return NV_ERR_INVALID_PARAMETER; + } + + if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM || + pParams->fd == -1) + { + return NV_ERR_INVALID_PARAMETER; + } + + status = RmExportObject(hClient, + pParams->object.data.rmObject.hObject, + &hExportHandle, NULL); + if (status != NV_OK) + { + goto done; + } + NV_ASSERT(hExportHandle != 0); + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + status = _initializeExportObjectFd(nvfp, hClient, + pParams->object.data.rmObject.hDevice, + 1, NULL); + if (status != NV_OK) + { + goto done; + } + + nvfp->handles[0] = hExportHandle; + +done: + + if (status != NV_OK && hExportHandle != 0) + { + RmFreeObjExportHandle(hExportHandle); + } + + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +// This control call has been deprecated. It will be deleted soon. +NV_STATUS +cliresCtrlCmdOsUnixCreateExportObjectFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS *pParams +) +{ + NV_STATUS status; + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + + ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata)); + + if (pParams->maxObjects == 0) + { + return NV_ERR_INVALID_PARAMETER; + } + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice, + pParams->maxObjects, pParams->metadata); + +done: + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixExportObjectsToFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS *pParams +) +{ + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + RmObjExportHandle *pExportHandle; + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + NvU32 i; + NvU32 deviceInstance; + NvU32 result; + NvHandle *exportHandles = NULL; + NvBool bFdSetup = NV_FALSE; + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata)); + + /* Setup export FD if not done */ + if (nvfp->handles == NULL) + { + if (pParams->maxObjects == 0) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice, + pParams->maxObjects, + pParams->metadata); + if (status != NV_OK) + { + goto done; + } + + bFdSetup = NV_TRUE; + } + + if ((nvfp->handles == NULL) || + (pParams->numObjects > + NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS)) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) || + (result > nvfp->maxHandles)) + { + status = NV_ERR_OUT_OF_RANGE; + goto done; + } + + status = os_alloc_mem((void **)&exportHandles, + sizeof(exportHandles[0]) * + pParams->numObjects); + if (status != NV_OK) + { + goto done; + } + + for (i = 0; i < pParams->numObjects; i++) + { + exportHandles[i] = 0; + + if (pParams->objects[i] == 0) + { + continue; + } + + status = RmExportObject(hClient, + pParams->objects[i], + &exportHandles[i], + &deviceInstance); + if (status != NV_OK) + { + goto done; + } + + NV_ASSERT(exportHandles[i] != 0); + + if (deviceInstance != nvfp->deviceInstance) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + } + + for (i = 0; i < pParams->numObjects; i++) + { + pExportHandle = &nvfp->handles[i + pParams->index]; + + // If the handle already exists in this position, free it + if (*pExportHandle != 0) + { + RmFreeObjExportHandle(*pExportHandle); + *pExportHandle = 0; + } + + *pExportHandle = exportHandles[i]; + } + +done: + + if ((status != NV_OK) && (exportHandles != NULL)) + { + for (i = 0; i < pParams->numObjects; i++) + { + if (exportHandles[i] != 0) + { + RmFreeObjExportHandle(exportHandles[i]); + } + } + } + + if (exportHandles != NULL) + { + os_free_mem(exportHandles); + } + + if ((status != NV_OK) && bFdSetup) + { + os_free_mem(nvfp->handles); + nvfp->handles = NULL; + nvfp->maxHandles = 0; + } + + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixImportObjectFromFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS *pParams +) +{ + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + + if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM || + pParams->fd == -1) + { + return NV_ERR_INVALID_PARAMETER; + } + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if ((nvfp->handles == NULL) || (nvfp->handles[0] == 0) || + (nvfp->maxHandles > 1)) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + status = RmImportObject(hClient, + pParams->object.data.rmObject.hParent, + &pParams->object.data.rmObject.hObject, + nvfp->handles[0], NULL); + +done: + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixImportObjectsFromFd_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS *pParams +) +{ + NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes); + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + NvU32 i = 0; + RmObjExportHandle hImportHandle; + NvU32 result; + RM_API *pRmApi; + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if ((nvfp->handles == NULL) || + (pParams->numObjects > + NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS)) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) || + (result > nvfp->maxHandles)) + { + status = NV_ERR_OUT_OF_RANGE; + goto done; + } + + for (i = 0; i < pParams->numObjects; i++) + { + hImportHandle = nvfp->handles[i + pParams->index]; + + /* Nothing to import, just continue */ + if (hImportHandle == 0) + { + pParams->objectTypes[i] = \ + NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_NONE; + continue; + } + + status = RmImportObject(hClient, + pParams->hParent, + &pParams->objects[i], + hImportHandle, + &pParams->objectTypes[i]); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "%s: Unable to import handle (%x, %x, %x)\n", + __FUNCTION__, pParams->hParent, pParams->objects[i], hImportHandle); + goto done; + } + } + +done: + + if (status != NV_OK) + { + pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + while (i > 0) + { + i--; + + if (pParams->objects[i] != 0) + { + pRmApi->Free(pRmApi, hClient, pParams->objects[i]); + } + } + } + + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +NV_STATUS +cliresCtrlCmdOsUnixGetExportObjectInfo_IMPL +( + RmClientResource *pRmCliRes, + NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS *pParams +) +{ + nv_file_private_t *nvfp = NULL; + void *priv = NULL; + NV_STATUS status = NV_OK; + + if (pParams->fd < 0) + { + return NV_ERR_INVALID_PARAMETER; + } + + nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv); + if (nvfp == NULL) + { + NV_ASSERT(priv == NULL); + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + if (nvfp->handles == NULL) + { + status = NV_ERR_INVALID_PARAMETER; + goto done; + } + + pParams->maxObjects = nvfp->maxHandles; + pParams->deviceInstance = nvfp->deviceInstance; + os_mem_copy(pParams->metadata, nvfp->metadata, sizeof(nvfp->metadata)); + +done: + if (priv != NULL) + { + nv_put_file_private(priv); + } + + return status; +} + +/*! + * osAcpiDsm + * + * @brief Handles os specific _DSM method function calls. + * + * Input parameters: + * @param[in] pGpu : OBJGPU pointer + * @param[in] acpiDsmFunction : ACPI DSM function + * @param[in] acpiDsmSubFunction : ACPI DSM subfunction + * @param[in/out] pInOut : in/out buffer, caller should make sure the buffer is large enough. + * @param[in] pSize : when input, size of data that the caller wants to read, in bytes. + * when output, size of valid data in pInOuta in bytes. + */ +NV_STATUS osCallACPI_DSM +( + OBJGPU *pGpu, + ACPI_DSM_FUNCTION acpiDsmFunction, + NvU32 acpiDsmSubFunction, + NvU32 *pInOut, + NvU16 *pSize +) +{ + NV_STATUS status; + NvU8 *pAcpiDsmGuid = NULL; + NvU32 acpiDsmRev; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU16 acpiDsmInArgSize = 4; + NvBool acpiNvpcfDsmFunction = NV_FALSE; + + // do any handling/remapping of guid needed. + status = checkDsmCall(pGpu, + (ACPI_DSM_FUNCTION *) &acpiDsmFunction, + &acpiDsmSubFunction, + pInOut, + pSize); + + // return if subfunction is not supported or we're returning cache data + if (status != NV_WARN_MORE_PROCESSING_REQUIRED) + { + return status; + } + + switch ((NvU32) acpiDsmFunction) + { + case ACPI_DSM_FUNCTION_NBSI: + pAcpiDsmGuid = (NvU8 *) &NBSI_DSM_GUID; + acpiDsmRev = NBSI_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_NVHG: + pAcpiDsmGuid = (NvU8 *) &NVHG_DSM_GUID; + acpiDsmRev = NVHG_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_MXM: + pAcpiDsmGuid = (NvU8 *) &DSM_MXM_GUID; + acpiDsmRev = ACPI_MXM_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_NBCI: + pAcpiDsmGuid = (NvU8 *) &NBCI_DSM_GUID; + acpiDsmRev = NBCI_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_NVOP: + pAcpiDsmGuid = (NvU8 *) &NVOP_DSM_GUID; + acpiDsmRev = NVOP_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_PCFG: + pAcpiDsmGuid = (NvU8 *) &PCFG_DSM_GUID; + acpiDsmRev = PCFG_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_PEX: + pAcpiDsmGuid = (NvU8 *) &PEX_DSM_GUID; + acpiDsmRev = PEX_REVISION_ID; + if (acpiDsmSubFunction == PEX_FUNC_SETLTRLATENCY) + { + acpiDsmInArgSize = (3 + *pSize); + } + break; + case (ACPI_DSM_FUNCTION_JT): + pAcpiDsmGuid = (NvU8 *) &JT_DSM_GUID; + acpiDsmRev = JT_REVISION_ID; + break; + case ACPI_DSM_FUNCTION_NVPCF: + { + pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID; + acpiDsmRev = NVPCF_ACPI_DSM_REVISION_ID; + acpiDsmInArgSize = (*pSize); + acpiNvpcfDsmFunction = NV_TRUE; + break; + } + case ACPI_DSM_FUNCTION_NVPCF_2X: + pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID; + acpiDsmRev = NVPCF_2X_ACPI_DSM_REVISION_ID; + acpiDsmInArgSize = (*pSize); + if (!nv->nvpcf_dsm_in_gpu_scope) + { + acpiNvpcfDsmFunction = NV_TRUE; + } + break; + + default: + return NV_ERR_NOT_SUPPORTED; + break; + } + + status = nv_acpi_dsm_method(nv, + pAcpiDsmGuid, + acpiDsmRev, + acpiNvpcfDsmFunction, + acpiDsmSubFunction, + pInOut, + acpiDsmInArgSize, + NULL, + pInOut, + pSize); + + if (status == NV_OK) + { + if (acpiDsmSubFunction == NV_ACPI_ALL_FUNC_SUPPORT) + { + // if handling get supported functions list... cache it for later calls + cacheDsmSupportedFunction(pGpu, acpiDsmFunction, acpiDsmSubFunction, pInOut, *pSize); + } + } + else if (nvp->b_mobile_config_enabled) + { + NV_PRINTF(LEVEL_ERROR, + "osCallACPI_DSM: Error during 0x%x DSM subfunction 0x%x! status=0x%x\n", + acpiDsmFunction, acpiDsmSubFunction, status); + } + + return status; +} + +NV_STATUS osCallACPI_DOD +( + OBJGPU *pGpu, + NvU32 *pOut, + NvU32 *pSize +) +{ + NV_STATUS rmStatus; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((pOut == NULL) || (pSize == NULL)) + { + return NV_ERR_INVALID_POINTER; + } + + rmStatus = nv_acpi_dod_method(nv, pOut, pSize); + + return rmStatus; +} + +// +// osAcpiDdc +// +// Handles os specific _DDC method function calls. _DDC is to get EDID from SBIOS. +// +NV_STATUS osCallACPI_DDC +( + OBJGPU *pGpu, + NvU32 ulAcpiId, + NvU8 *pOutData, + NvU32 *pOutSize, + NvBool bReadMultiBlock +) +{ + NV_STATUS rmStatus; + + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((pOutData == NULL) || (pOutSize == NULL)) + { + return NV_ERR_INVALID_POINTER; + } + + portMemSet(pOutData, 0, *pOutSize); + + rmStatus = nv_acpi_ddc_method(nv, pOutData, pOutSize, bReadMultiBlock); + + return rmStatus; +} + +// osCallACPI_NVHG_ROM +// Making ACPI Call into SBIOS with ROM method to get display device's ROM data. +// +NV_STATUS osCallACPI_NVHG_ROM +( + OBJGPU *pGpu, + NvU32 *pInData, + NvU32 *pOutData +) +{ + NV_STATUS rmStatus; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((pOutData == NULL) || (pInData == NULL)) + { + return NV_ERR_INVALID_POINTER; + } + + if (pInData[1] > ROM_METHOD_MAX_RETURN_BUFFER_SIZE) + { + return NV_ERR_INVALID_ARGUMENT; + } + + rmStatus = nv_acpi_rom_method(nv, pInData, pOutData); + + return rmStatus; +} + +void osInitSystemStaticConfig(SYS_STATIC_CONFIG *pConfig) +{ + pConfig->bIsNotebook = rm_is_system_notebook(); + pConfig->osType = nv_get_os_type(); + pConfig->osSevStatus = os_sev_status; + pConfig->bOsSevEnabled = os_sev_enabled; +} + +NvU32 osApiLockAcquireConfigureFlags(NvU32 flags) +{ + return flags; +} + +NV_STATUS osGpuLocksQueueRelease(OBJGPU *pGpu, NvU32 dpcGpuLocksRelease) +{ + return NV_SEMA_RELEASE_FAILED; +} + +void osSyncWithRmDestroy() +{ +} + +void osSyncWithGpuDestroy(NvBool bEntry) +{ +} + +void osModifyGpuSwStatePersistence +( + OS_GPU_INFO *pOsGpuInfo, + NvBool bEnable +) +{ + if (bEnable) + { + pOsGpuInfo->flags |= NV_FLAG_PERSISTENT_SW_STATE; + } + else + { + pOsGpuInfo->flags &= ~NV_FLAG_PERSISTENT_SW_STATE; + } +} + +NV_STATUS +osSystemGetBatteryDrain(NvS32 *pChargeRate) +{ + NV_PRINTF(LEVEL_WARNING, "%s: Platform not supported!\n", __FUNCTION__); + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osPexRecoveryCallback +( + OS_GPU_INFO *pOsGpuInfo, + OS_PEX_RECOVERY_STATUS Status +) +{ + NV_ASSERT_FAILED("Not supported"); + return NV_ERR_NOT_SUPPORTED; +} + +// +//osCallACPI_MXDS +// +//Handles OS specific MXDS function call. +// +NV_STATUS osCallACPI_MXDS +( + OBJGPU *pGpu, + NvU32 acpiId, + NvU32 *pInOut +) +{ + NV_STATUS rmStatus; + + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if (pInOut == NULL) + { + return NV_ERR_INVALID_POINTER; + } + + rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDS"); + + return rmStatus; +} + +// +//osCallACPI_MXDM +// +//Handles OS specific MXDM function call. +// +NV_STATUS osCallACPI_MXDM +( + OBJGPU *pGpu, + NvU32 acpiId, + NvU32 *pInOut +) +{ + NV_STATUS rmStatus; + + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if (pInOut == NULL) + { + return NV_ERR_INVALID_POINTER; + } + + rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDM"); + + return rmStatus; +} + +#include "lib/protobuf/prb.h" +#include "lib/protobuf/prb_util.h" +#include "g_nvdebug_pb.h" + +NV_STATUS osGetVersionDump(void * pVoid) +{ + PRB_ENCODER * pPrbEnc = (PRB_ENCODER *)pVoid; + NV_STATUS rmStatus; + os_version_info * pOsVersionInfo = NULL; + const char NV_UNKNOWN_BUILD_VERSION[] = "Unknown build version"; + const char NV_UNKNOWN_BUILD_DATE[] = "Unknown build date"; + + NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void**)&pOsVersionInfo, + sizeof(os_version_info))); + portMemSet(pOsVersionInfo, 0, sizeof(os_version_info)); + + prbEncAddUInt32(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_FAMILY, + NVDEBUG_OS_UNIX); + + rmStatus = os_get_version_info(pOsVersionInfo); + if (rmStatus != NV_OK) + { + goto cleanup; + } + + prbEncAddUInt32(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_OSMAJORVERSION, + pOsVersionInfo->os_major_version); + + prbEncAddUInt32(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_OSMINORVERSION, + pOsVersionInfo->os_minor_version); + + prbEncAddUInt32(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_OSBLDNUM, + pOsVersionInfo->os_build_number); + + if (NULL == pOsVersionInfo->os_build_version_str) + { + pOsVersionInfo->os_build_version_str = NV_UNKNOWN_BUILD_VERSION; + } + + prbEncAddString(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_BUILDVERSION, + pOsVersionInfo->os_build_version_str); + + if (NULL == pOsVersionInfo->os_build_date_plus_str) + { + pOsVersionInfo->os_build_date_plus_str = NV_UNKNOWN_BUILD_DATE; + } + + prbEncAddString(pPrbEnc, + NVDEBUG_SYSTEMINFO_OSINFO_BUILDDATEPLUS, + pOsVersionInfo->os_build_date_plus_str); + +cleanup: + os_free_mem(pOsVersionInfo); + return rmStatus; +} + +NV_STATUS osGetVersion(NvU32 *majorVer, NvU32 *minorVer, NvU32 *buildNum, NvU16 *unusedPatchVersion, NvU16 *unusedProductType) +{ + os_version_info osVersionInfo; + NV_STATUS rmStatus; + + portMemSet(&osVersionInfo, 0, sizeof(osVersionInfo)); + + rmStatus = os_get_version_info(&osVersionInfo); + if (rmStatus == NV_OK) + { + if (majorVer) + *majorVer = osVersionInfo.os_major_version; + if (minorVer) + *minorVer = osVersionInfo.os_minor_version; + if (buildNum) + *buildNum = osVersionInfo.os_build_number; + } + + return rmStatus; +} + +NV_STATUS +osGetSystemCpuLogicalCoreCounts +( + NvU32 *pCpuCoreCount +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osGetSystemCpuC0AndAPerfCounters +( + NvU32 coreIndex, + POS_CPU_CORE_PERF_COUNTERS pCpuPerfData +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +void +osEnableCpuPerformanceCounters +( + OBJOS *pOS +) +{ + NV_ASSERT_FAILED("Not supported"); + return; +} + +NV_STATUS +osCpuDpcObjInit +( + void **ppCpuDpcObj, + OBJGPU *pGpu, + NvU32 coreCount +) +{ + NV_ASSERT_FAILED("Not supported"); + return NV_ERR_NOT_SUPPORTED; +} + +void +osCpuDpcObjQueue +( + void **ppCpuDpcObj, + NvU32 coreCount, + POS_CPU_CORE_PERF_COUNTERS pCpuPerfData +) +{ + NV_ASSERT_FAILED("Not supported"); +} + +void +osCpuDpcObjFree +( + void **ppCpuDpcObj +) +{ + NV_ASSERT_FAILED("Not supported"); +} + +NV_STATUS +osGetCarveoutInfo +( + NvU64 *pAddr, + NvU64 *pSize +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osGetVPRInfo +( + NvU64 *pAddr, + NvU64 *pSize +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osAllocInVPR +( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osGetGenCarveout +( + NvU64 *pAddr, + NvU64 *pSize, + NvU32 id, + NvU64 align +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osI2CClosePorts +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 numPorts +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osI2CTransfer +( + OBJGPU *pGpu, + NvU32 Port, + NvU8 Address, + nv_i2c_msg_t *nv_i2c_msgs, + NvU32 count +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osTegraI2CGetBusState +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 port, + NvS32 *scl, + NvS32 *sda +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osReadI2CBufferDirect +( + OBJGPU *pGpu, + NvU32 Port, + NvU8 Address, + void *pOutputBuffer, + NvU32 OutputSize, + void *pInputBuffer, + NvU32 InputSize +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osWriteI2CBufferDirect +( + OBJGPU *pGpu, + NvU32 Port, + NvU8 Address, + void *pOutputBuffer0, + NvU32 OutputSize0, + void *pOutputBuffer1, + NvU32 OutputSize1 +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NV_STATUS +osGC6PowerControl +( + OBJGPU *pGpu, + NvU32 cmd, + NvU32 *pOut +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +NvBool osTestPcieExtendedConfigAccess(void *handle, NvU32 offset) +{ + OBJGPU *pGpu; + KernelBif *pKernelBif; + NvU32 nvXveId = 0; + NvU32 nvXveVccapHdr = 0; + NvU32 pciStart = 0; + NvU32 pcieStart = 0; + + static NvBool retryAllowed = NV_TRUE; + static NvBool configAccess = NV_FALSE; + + // + // Return early for offset within PCI space + // and does not requires extended config space access + // + if (offset < 0x100) + { + return NV_TRUE; + } + + if (!retryAllowed) + { + return configAccess; + } + + pGpu = gpumgrGetSomeGpu(); + if (pGpu == NULL) + { + return configAccess; + } + + retryAllowed = NV_FALSE; + + pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + if (pKernelBif == NULL || kbifGetBusIntfType_HAL(pKernelBif) != + NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS) + { + return configAccess; + } + + // Now verify PCI and PCIe config start registers. + kbifGetPcieConfigAccessTestRegisters_HAL(pGpu, pKernelBif, &pciStart, &pcieStart); + os_pci_read_dword(handle, pciStart, &nvXveId); + os_pci_read_dword(handle, pcieStart, &nvXveVccapHdr); + + if (NV_OK == kbifVerifyPcieConfigAccessTestRegisters_HAL(pGpu, + pKernelBif, + nvXveId, + nvXveVccapHdr)) + { + configAccess = NV_TRUE; + } + + return configAccess; +} + +/*! + * @brief Map memory into an IOVA space according to the given mapping info. + * + * @param[in] pIovaMapping IOVA mapping info + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osIovaMap +( + PIOVAMAPPING pIovaMapping +) +{ + OBJGPU *pGpu; + nv_state_t *nv, *peer; + NV_STATUS status; + RmPhysAddr base; + NvBool bIsBar0; + PMEMORY_DESCRIPTOR pRootMemDesc; + NvBool bIsFbOffset = NV_FALSE; + NvBool bIsIndirectPeerMapping = NV_FALSE; + NvBool bIsContig; + NV_ADDRESS_SPACE addressSpace; + NvU32 osPageCount; + + if (pIovaMapping == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId); + if (pGpu == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + pRootMemDesc = memdescGetRootMemDesc(pIovaMapping->pPhysMemDesc, NULL); + addressSpace = memdescGetAddressSpace(pIovaMapping->pPhysMemDesc); + if (gpumgrCheckIndirectPeer(pGpu, pRootMemDesc->pGpu) && + (addressSpace == ADDR_FBMEM)) + { + bIsIndirectPeerMapping = NV_TRUE; + } + + if ((addressSpace != ADDR_SYSMEM) && !bIsIndirectPeerMapping) + { + NV_PRINTF(LEVEL_INFO, + "%s passed memory descriptor in an unsupported address space (%s)\n", + __FUNCTION__, + memdescGetApertureString(memdescGetAddressSpace(pIovaMapping->pPhysMemDesc))); + return NV_ERR_NOT_SUPPORTED; + } + + // + // For guest-allocated memory, we don't actually want to do any remapping, + // since the physical address is already the DMA address to be used by the + // GPU. + // + if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED)) + { + return NV_OK; + } + + nv = NV_GET_NV_STATE(pGpu); + + // + // Intercept peer IO type memory. These are contiguous allocations, so no + // need to adjust pages. + // + if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM)) + { + NV_ASSERT(memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU)); + + status = nv_dma_map_mmio(nv->dma_dev, + NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount), + &pIovaMapping->iovaArray[0]); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to map peer IO mem (status = 0x%x)\n", + __FUNCTION__, status); + } + + return status; + } + + // + // We need to check against the "root" GPU, e.g., the GPU that owns this + // allocation. If we're trying to map one of its BARs for a peer, we need + // to handle it differently because it wouldn't have gone through our system + // memory page allocation paths, obviously, and wouldn't have alloc private + // data associated with it. + // + peer = NV_GET_NV_STATE(pRootMemDesc->pGpu); + bIsContig = memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU); + if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig) + { + RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0], + pIovaMapping->pPhysMemDesc->PageCount); + } + + base = memdescGetPhysAddr(pIovaMapping->pPhysMemDesc, AT_CPU, 0); + bIsBar0 = IS_REG_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size); + + bIsFbOffset = IS_FB_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size); + + // + // For indirect peers bIsFbOffset should be NV_TRUE + // TODO:IS_FB_OFFSET macro is currently broken for P9 systems + // Bug 2010857 tracks fixing this + // +#if defined(NVCPU_PPC64LE) + KernelMemorySystem *pRootKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pRootMemDesc->pGpu); + if (bIsIndirectPeerMapping) + { + NvU64 atsBase = base + pRootKernelMemorySystem->coherentCpuFbBase; + if ((atsBase >= pRootKernelMemorySystem->coherentCpuFbBase) && + (atsBase + pIovaMapping->pPhysMemDesc->Size <= + pRootKernelMemorySystem->coherentCpuFbEnd)) + { + bIsFbOffset = NV_TRUE; + } + else + { + NV_ASSERT_OR_RETURN(0, NV_ERR_INVALID_STATE); + } + } +#endif + + void *pPriv = memdescGetMemData(pIovaMapping->pPhysMemDesc); + osPageCount = NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount); + + if (!bIsBar0 && !bIsFbOffset) + { + if (pPriv == NULL) + { + return NV_ERR_INVALID_STATE; + } + } + else if(bIsIndirectPeerMapping) + { + NV_ASSERT(!bIsBar0 && bIsFbOffset); + // + // TODO: Align onlined GPU memory allocation paths with system memory allocation + // That way pMemDesc->pMemData is setup correctly when we try to create mapping + // to onlined memory of indirect peer. After that we can also get rid of some + // extra code in nv_dma_map_alloc. See bug 190324 for details + // + + status = memdescGetNvLinkGpa(pRootMemDesc->pGpu, (bIsContig ? 1 : osPageCount), + &pIovaMapping->iovaArray[0]); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "%s Failed to get SPA\n", __FUNCTION__); + return status; + } + } + + if (!bIsBar0 && (!bIsFbOffset || bIsIndirectPeerMapping)) + { + status = nv_dma_map_alloc( + osGetDmaDeviceForMemDesc(nv, pIovaMapping->pPhysMemDesc), + osPageCount, + &pIovaMapping->iovaArray[0], + bIsContig, &pPriv); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to map allocation (status = 0x%x)\n", + __FUNCTION__, status); + return status; + } + + pIovaMapping->pOsData = pPriv; + } + else if (peer != nv) + { + status = nv_dma_map_peer(nv->dma_dev, peer->dma_dev, bIsBar0 ? 0 : 1, + osPageCount, &pIovaMapping->iovaArray[0]); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_INFO, + "%s: failed to map peer (base = 0x%llx, status = 0x%x)\n", + __FUNCTION__, base, status); + return status; + } + + // + // pOsData must be NULL to distinguish a peer DMA mapping from a + // system memory mapping in osIovaUnmap(), so make sure to set it + // accordingly here. + // + pIovaMapping->pOsData = NULL; + } + else + { + NV_PRINTF(LEVEL_INFO, "cannot map a GPU's BAR to itself\n"); + return NV_ERR_NOT_SUPPORTED; + } + + // + // If the OS layer doesn't think in RM page size, we need to inflate the + // PTE array into RM pages. + // + if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig) + { + RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0], + pIovaMapping->pPhysMemDesc->PageCount); + } + + return NV_OK; +} + +/*! + * @brief Unmap memory from an IOVA space according to the given mapping info. + * + * This mapping info must have been previously mapped by osIovaMap(). + * + * @param[in] pIovaMapping IOVA mapping info + * + */ +void +osIovaUnmap +( + PIOVAMAPPING pIovaMapping +) +{ + OBJGPU *pGpu; + nv_state_t *nv; + void *pPriv; + NV_STATUS status; + + if (pIovaMapping == NULL) + { + return; + } + + pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId); + if (pGpu == NULL) + { + return; + } + + // + // For guest-allocated memory, we never actually remapped the memory, so we + // shouldn't try to unmap it here. + // + if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED)) + { + return; + } + + nv = NV_GET_NV_STATE(pGpu); + + if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM)) + { + nv_dma_unmap_mmio(nv->dma_dev, + NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount), + pIovaMapping->iovaArray[0]); + + return; + } + + // + // TODO: Formalize the interface with the OS layers so we can use a common + // definition of OS_IOVA_MAPPING_DATA. + // + pPriv = (void *)pIovaMapping->pOsData; + + if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0], + pIovaMapping->pPhysMemDesc->PageCount); + } + + if (pPriv != NULL) + { + status = nv_dma_unmap_alloc(nv->dma_dev, + NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount), + &pIovaMapping->iovaArray[0], &pPriv); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to unmap allocation (status = 0x%x)\n", + __FUNCTION__, status); + } + } + else + { + nv_dma_unmap_peer(nv->dma_dev, + NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount), + pIovaMapping->iovaArray[0]); + } + + // + // If the OS layer doesn't think in RM page size, we need to fluff out the + // PTE array into RM pages. + // + if (NV_RM_PAGE_SIZE < os_page_size && + !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU)) + { + RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0], + pIovaMapping->pPhysMemDesc->PageCount); + } + + pIovaMapping->pOsData = NULL; +} + +/*! + * @brief Set the GPU Rail Voltage in Tegra SoC. Currently not supported + * + * @param[in] pGpu GPU object pointer + * @param[in] reqVoltageuV Rail Voltage requested in uV + * @param[out] pSetVoltageuV Rail Voltage set in uV + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osSetGpuRailVoltage +( + OBJGPU *pGpu, + NvU32 reqVoltageuV, + NvU32 *pSetVoltageuV +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Get the GPU Rail Voltage in Tegra SoC. Currently not supported + * + * @param[in] pGpu GPU object pointer + * @param[out] voltageuV Rail Voltage in uV + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osGetGpuRailVoltage +( + OBJGPU *pGpu, + NvU32 *pVoltageuV +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Bring down system in a controlled manner on known error conditions. + * + * @bugCode[in] Error code / reason. + */ +void osBugCheck(NvU32 bugCode) +{ + if (bugCode > OS_BUG_CHECK_BUGCODE_LAST) + { + bugCode = OS_BUG_CHECK_BUGCODE_UNKNOWN; + } + + os_bug_check(bugCode, ppOsBugCheckBugcodeStr[bugCode]); +} + +/*! + * @brief Perform an action at assertion failure. + */ +void osAssertFailed(void) +{ + os_dump_stack(); +} + +/*! + * @brief Get the GPU Chip Info - Speedo and IDDQ values + * + * + * @param[in] pGpu GPU object pointer + * @param[out] pGpuSpeedoHv Pointer to GPU Speedo value at high voltage corner. + * @param[out] pGpuSpeedoLv Pointer to GPU Speedo value at low voltage corner. + * @param[out] pGpuIddq Pointer to GPU Iddq Value + * @param[out] pChipSkuId SKU ID for the chip + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osGetChipInfo +( + OBJGPU *pGpu, + NvU32 *pGpuSpeedoHv, + NvU32 *pGpuSpeedoLv, + NvU32 *pGpuIddq, + NvU32 *pChipSkuId +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/* + * @brief Get the GPU Rail Voltage Info (i.e. Min, Max and StepSize) in Tegra SoC. + * + * @param[in] pGpu GPU object pointer + * @param[out] pMinVoltageuV Minimum Voltage supported on the Rail in Micro Volts + * @param[out] pMaxVoltageuV Maximum Voltage supported on the Rail in Micro Volts + * @param[out] pStepVoltageuV Voltage Step-size supported on the Rail in Micro Volts + * + * @return NV_ERR_NOT_SUPPORTED + */ +NV_STATUS +osGetGpuRailVoltageInfo +( + OBJGPU *pGpu, + NvU32 *pMinVoltageuV, + NvU32 *pMaxVoltageuV, + NvU32 *pStepVoltageuV +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Get the current opaque security token. + * + * For Linux the security token is the effective UID of a process and process ID + * + * Note: This function allocates memory for the token. The onus is on the calling + * function to free the memory associated with the token once its done with it. + * + * @return pointer to the security token. + */ +PSECURITY_TOKEN +osGetSecurityToken() +{ + NV_STATUS rmStatus; + TOKEN_USER *pTokenUser; + + pTokenUser = portMemAllocNonPaged(sizeof(TOKEN_USER)); + if (pTokenUser == NULL) + { + return NULL; + } + rmStatus = os_get_euid(&pTokenUser->euid); + if (rmStatus != NV_OK) + { + portMemFree(pTokenUser); + return NULL; + } + + pTokenUser->pid = os_get_current_process(); + + return (PSECURITY_TOKEN)pTokenUser; +} + +PUID_TOKEN +osGetCurrentUidToken(void) +{ + NV_STATUS rmStatus; + NvU32 *pUidToken; + + pUidToken = portMemAllocNonPaged(sizeof(NvU32)); + if (pUidToken == NULL) + { + return NULL; + } + + rmStatus = os_get_euid(pUidToken); + if (rmStatus != NV_OK) + { + portMemFree(pUidToken); + return NULL; + } + + return (PUID_TOKEN)pUidToken; +} + +/*! + * @brief Interface function to validate the token for the current client + * + * This function takes two tokens as parameters, validates them and checks + * if either the PID or EUID from client database matches the current PID or EUID. + * + * @param[in] pClientSecurityToken security token cached in the client db + * @param[in] pCurrentSecurityToken security token of the current client + * @return NV_OK if the validation is successful + * NV_ERR_INVALID_CLIENT if the tokens do not match + * NV_ERR_INVALID_POINTER if the tokens are invalid + */ +NV_STATUS +osValidateClientTokens +( + PSECURITY_TOKEN pClientSecurityToken, + PSECURITY_TOKEN pCurrentSecurityToken +) +{ + PTOKEN_USER pClientTokenUser = (PTOKEN_USER)pClientSecurityToken; + PTOKEN_USER pCurrentTokenUser = (PTOKEN_USER)pCurrentSecurityToken; + + NV_ASSERT_OR_RETURN((pClientTokenUser != NULL), NV_ERR_INVALID_POINTER); + NV_ASSERT_OR_RETURN((pCurrentTokenUser != NULL), NV_ERR_INVALID_POINTER); + + if ((pClientTokenUser->euid != pCurrentTokenUser->euid) && + (pClientTokenUser->pid != pCurrentTokenUser->pid)) + { + NV_PRINTF(LEVEL_INFO, + "NVRM: %s: Current security token doesn't match the one in the client database. " + "Current EUID: %d, PID: %d; Client DB EUID: %d, PID: %d\n", + __FUNCTION__, pCurrentTokenUser->euid, pCurrentTokenUser->pid, + pClientTokenUser->euid, pClientTokenUser->pid); + return NV_ERR_INVALID_CLIENT; + } + + return NV_OK; +} + +/*! + * @brief Interface function to compare the tokens for two client + * + * This function takes two tokens as parameters, validates them and checks + * if the EUIDs of each token match. + * + * @param[in] pToken1 Token to compare + * @param[in] pToken2 Token to compare + * @return NV_TRUE if the tokens match + * NV_FALSE if the tokens do not match + */ +NvBool +osUidTokensEqual +( + PUID_TOKEN pUidToken1, + PUID_TOKEN pUidToken2 +) +{ + NvU32 * pTokenUser1 = (NvU32*)pUidToken1; + NvU32 * pTokenUser2 = (NvU32*)pUidToken2; + + NV_ASSERT_OR_RETURN((pTokenUser1 != NULL), NV_FALSE); + NV_ASSERT_OR_RETURN((pTokenUser2 != NULL), NV_FALSE); + + if (*pTokenUser1 != *pTokenUser2) + { + return NV_FALSE; + } + + return NV_TRUE; +} + +NvBool +osRemoveGpuSupported +( + void +) +{ + return os_pci_remove_supported(); +} + +/* + * @brief Get the address ranges assigned to local or peer GPUs on a system that + * supports hardware address translation services (ATS) over NVLink/C2C. + * + * @note + * - All address values are in the System Physical Address (SPA) space + * - Targets can either be "Local" (bIsPeer=False) or for a specified "Peer" + * (bIsPeer=True, peerIndex=#) GPU + * - Target address and mask values have a specified bit width, and represent + * the higher order bits above the target address granularity + * + * @param[in] pGpu GPU object pointer + * @param[out] pAddrSysPhys Pointer to hold SPA + * @param[out] pAddrWidth Address range width value pointer + * @param[out] pMask Mask value pointer + * @param[out] pMaskWidth Mask width value pointer + * @param[in] bIsPeer NV_TRUE if this is a peer, local GPU otherwise + * @param[in] peerIndex Peer index + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED + * + * A return value of NV_ERR_NOT_SUPPORTED for the local GPU would + * indicate that the system does not support ATS over NVLink/C2C + */ +NV_STATUS +osGetAtsTargetAddressRange +( + OBJGPU *pGpu, + NvU64 *pAddrSysPhys, + NvU32 *pAddrWidth, + NvU32 *pMask, + NvU32 *pMaskWidth, + NvBool bIsPeer, + NvU32 peerIndex +) +{ +#if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && (defined(NVCPU_PPC64LE) || defined(NVCPU_AARCH64)) + KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + nv_state_t *nv; + const int addrMaskWidth = 0x10; + + if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP)) + return NV_ERR_INVALID_ARGUMENT; + + nv = NV_GET_NV_STATE(pGpu); + + // + // TODO : Bug 1848958 restricts peer device tree parsing. Return early if + // peer values are requested. This should be fixed by passing correct pGpu + // pointer of the peer GPU retrieved using peerIds. + // + if (bIsPeer) + { + const int addrWidth = 0x10; + + *pAddrSysPhys = 0; + *pAddrWidth = addrWidth; + *pMask = 0; + *pMaskWidth = addrMaskWidth; + return NV_OK; + } + else + { + NV_STATUS status = nv_get_device_memory_config(nv, pAddrSysPhys, NULL, + pAddrWidth, NULL); + if (status == NV_OK) + { + *pMask = NVBIT(*pAddrWidth) - 1U; + *pMaskWidth = addrMaskWidth; + } + return status; + } + + return NV_OK; +#endif + return NV_ERR_NOT_SUPPORTED; +} + +/* + * @brief Get the physical address in CPU address map and NUMA node id + * of the GPU memory. + * + * @note + * - The physical address is System Physical Address (SPA) in baremetal/host + * and Intermediate Physical Address(IPA) or Guest Physical Address(GPA) + * inside a VM. + * + * @param[in] pGpu GPU object pointer + * @param[out] pAddrPhys Pointer to hold the physical address of FB in + * CPU address map + * @param[out] pNodeId NUMA nodeID of respective GPU memory + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED + * + */ +NV_STATUS +osGetFbNumaInfo +( + OBJGPU *pGpu, + NvU64 *pAddrPhys, + NvS32 *pNodeId +) +{ +#if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && (defined(NVCPU_PPC64LE) || defined(NVCPU_AARCH64)) + KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + nv_state_t *nv; + + *pNodeId = NV0000_CTRL_NO_NUMA_NODE; + + if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP)) + return NV_ERR_INVALID_ARGUMENT; + + nv = NV_GET_NV_STATE(pGpu); + + NV_STATUS status = nv_get_device_memory_config(nv, NULL, pAddrPhys, NULL, pNodeId); + + return status; +#endif + return NV_ERR_NOT_SUPPORTED; +} + + +/* + * @brief Verif only function to get the chiplib overrides for link connection + * state for all C2C links. + * + * If chiplib overrides exist, each link can either be enabled (1) or disabled (0) + * + * @param[in] pGpu GPU object pointer + * @param[in] maxLinks Size of pLinkConnection array + * @param[out] pLinkConnection array of pLinkConnection values to be populated by MODS + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available) + */ +NV_STATUS +osGetForcedC2CConnection +( + OBJGPU *pGpu, + NvU32 maxLinks, + NvU32 *pLinkConnection +) +{ + int i, ret; + NV_STATUS status; + char path[64]; + OBJOS *pOS; + OBJSYS *pSys; + + NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER); + NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED); + NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT); + + pSys = SYS_GET_INSTANCE(); + pOS = SYS_GET_OS(pSys); + if (pOS == NULL || pOS->osSimEscapeRead == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s: escape reads not supported on platform\n", + __FUNCTION__); + return NV_ERR_NOT_SUPPORTED; + } + + for (i = 0; i < maxLinks; i++) + { + ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|C2C%u", i); + NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1))); + + status = pOS->osSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]); + if (status == NV_OK) + { + NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__, + path, pLinkConnection[i]); + } + else + { + NV_PRINTF(LEVEL_INFO, "%s: osSimEscapeRead for '%s' failed (%u)\n", + __FUNCTION__, path, status); + return NV_ERR_NOT_SUPPORTED; + } + } + return NV_OK; +} + +static NV_STATUS +osGetSmbiosTableInfo +( + const NvU8 *pMappedAddr, + NvU64 *pBaseAddr, + NvU64 *pLength, + NvU64 *pNumSubTypes, + NvU32 *pVersion +) +{ + *pBaseAddr = 0; + *pLength = 0; + *pNumSubTypes = 0; + *pVersion = 0; + + if (portMemCmp(pMappedAddr, "_SM3_", 5) == 0) + { + *pVersion = (pMappedAddr[7] << 8) | pMappedAddr[8]; + portMemCopy(pLength, 4, pMappedAddr + 12, 4); + portMemCopy(pBaseAddr, 8, pMappedAddr + 16, 8); + + *pNumSubTypes = *pLength / 4; + + return NV_OK; + } + + if (portMemCmp(pMappedAddr, "_SM_", 4) == 0) + { + *pVersion = (pMappedAddr[6] << 8) | pMappedAddr[7]; + + pMappedAddr += 16; + + if (portMemCmp(pMappedAddr, "_DMI_", 5) == 0) + { + portMemCopy(pLength, 2, pMappedAddr + 6, 2); + portMemCopy(pBaseAddr, 4, pMappedAddr + 8, 4); + portMemCopy(pNumSubTypes, 2, pMappedAddr + 12, 2); + + if (!*pVersion) + *pVersion = (pMappedAddr[14] & 0xF0) << 4 | + (pMappedAddr[14] & 0x0F); + + return NV_OK; + } + } + + return NV_ERR_INVALID_ADDRESS; +} + + +/* + * @brief Function to export SMBIOS table. Also, maps table in kernel-space. + * + * @param[out] ppBaseVAddr Base virtual address of SMBIOS table. + * @param[out] pLength Size of SMBIOS table. + * @param[out] pNumSubTypes Count of structures (types) embedded in + * the SMBIOS table. + * @param[out] pVersion SMBIOS version + * + * @return NV_OK, NV_ERR_INSUFFICIENT_RESOURCES or NV_ERR_INVALID_ADDRESS + * or errors from OS layer + */ +NV_STATUS +osGetSmbiosTable +( + void **ppBaseVAddr, + NvU64 *pLength, + NvU64 *pNumSubTypes, + NvU32 *pVersion +) +{ + NV_STATUS status = NV_OK; + NvU64 physSmbiosAddr = ~0ull; + void *pMappedAddr = NULL; + NvU64 basePAddr = 0; + + if (!NVCPU_IS_X86_64) + { + return NV_ERR_NOT_SUPPORTED; + } + + status = os_get_smbios_header(&physSmbiosAddr); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_INFO, "%s: Failed query SMBIOS table with error: %x \n", + __FUNCTION__, status); + return status; + } + + NV_ASSERT(physSmbiosAddr != ~0ull); + + pMappedAddr = osMapKernelSpace(physSmbiosAddr, + os_page_size, + NV_MEMORY_CACHED, + NV_PROTECT_READ_WRITE); + if (!pMappedAddr) + { + return NV_ERR_INSUFFICIENT_RESOURCES; + } + + status = osGetSmbiosTableInfo(pMappedAddr, + &basePAddr, + pLength, + pNumSubTypes, + pVersion); + + osUnmapKernelSpace(pMappedAddr, os_page_size); + + if (status != NV_OK) + { + return status; + } + + *ppBaseVAddr = osMapKernelSpace(basePAddr, + *pLength, + NV_MEMORY_CACHED, + NV_PROTECT_READ_WRITE); + if (!*ppBaseVAddr) + { + return NV_ERR_INSUFFICIENT_RESOURCES; + } + + return NV_OK; +} + +/* + * @brief Function to free SMBIOS table mappings + * + * @param[in] pBaseVAddr Base virtual address of SMBIOS table. + * @param[in] length Size of SMBIOS table. + * + */ +void +osPutSmbiosTable +( + void *pBaseVAddr, + NvU64 length +) +{ + osUnmapKernelSpace(pBaseVAddr, length); +} + +NV_STATUS +osGetAcpiRsdpFromUefi +( + NvU32 *pRsdpAddr +) +{ + return os_get_acpi_rsdp_from_uefi(pRsdpAddr); +} + +/* + * @brief Returns NV_TRUE if NvSwitch device is present in the system. + */ +NvBool +osIsNvswitchPresent +( + void +) +{ + return os_is_nvswitch_present(); +} + +/* + * @brief Function to add crashlog buffer entry. + * + * @param[in] pBuffer virt_addr of nvlog buffer + * @param[in] length size of nvlog buffer + */ +void +osAddRecordForCrashLog +( + void *pBuffer, + NvU32 length +) +{ + os_add_record_for_crashLog(pBuffer, length); +} + +/* + * @brief Function to delete crashlog buffer entry. + * + * @param[in] pBuffer virt_addr of nvlog buffer + */ +void +osDeleteRecordForCrashLog +( + void *pBuffer +) +{ + os_delete_record_for_crashLog(pBuffer); +} + +/* + * @brief Queries the sysfs interface to get memblock size + * @param[out] memblock_size Pointer to the memblock_size + */ +NV_STATUS +osNumaMemblockSize +( + NvU64 *memblock_size +) +{ + return os_numa_memblock_size(memblock_size); +} + +NvBool +osNumaOnliningEnabled +( + OS_GPU_INFO *pOsGpuInfo +) +{ + NvS32 numaNodeId = NV0000_CTRL_NO_NUMA_NODE; + + // + // Note that this numaNodeId value fetched from Linux layer might not be + // accurate since it is possible to overwrite it with regkey on some configs + // + if (nv_get_device_memory_config(pOsGpuInfo, NULL, NULL, NULL, + &numaNodeId) != NV_OK) + { + return NV_FALSE; + } + + return (numaNodeId != NV0000_CTRL_NO_NUMA_NODE); +} + +/* + * @brief Function to call NUMA allocation entry. + * + * @param[in] nid NUMA node id + * @param[in] size Allocation size + * @param[in] flag Allocation flags + * @param[out] pAddress Ptr to the allocated physical address + */ +NV_STATUS +osAllocPagesNode +( + NvS32 nid, + NvLength size, + NvU32 flag, + NvU64 *pAddress +) +{ + NV_STATUS status = NV_OK; + NvU32 localFlag = NV_ALLOC_PAGES_NODE_NONE; + + if (pAddress == NULL || nid < 0 || size > NV_U32_MAX) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // Translate the flags + if (flag & OS_ALLOC_PAGES_NODE_SKIP_RECLAIM) + { + localFlag |= NV_ALLOC_PAGES_NODE_SKIP_RECLAIM; + } + + status = os_alloc_pages_node(nid, (NvU32)size, localFlag, pAddress); + return status; +} + +NV_STATUS +osAllocAcquirePage +( + NvU64 pAddress +) +{ + os_get_page(pAddress); + return NV_OK; +} + +NV_STATUS +osAllocReleasePage +( + NvU64 pAddress +) +{ + os_put_page(pAddress); + return NV_OK; +} + +/* + * @brief Function to return refcount on a page + * @param[in] address The physical address of the page + */ +NvU32 +osGetPageRefcount +( + NvU64 pAddress +) +{ + return os_get_page_refcount(pAddress); +} + +/* + * @brief Function to return the number of tail pages if the address is + * referring to a compound page; For non-compound pages, 1 is returned. + * @param[in] address The physical address of the page + */ +NvU32 +osCountTailPages +( + NvU64 pAddress +) +{ + return os_count_tail_pages(pAddress); +} + +/* + * @brief Upon success, gets NPU register address range. + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[out] pBase base (physical) of NPU register address range + * @param[out] pSize size of NPU register address range + */ +NV_STATUS +osGetIbmnpuGenregInfo +( + OS_GPU_INFO *pOsGpuInfo, + NvU64 *pBase, + NvU64 *pSize +) +{ + return nv_get_ibmnpu_genreg_info(pOsGpuInfo, pBase, pSize, NULL); +} + +/* + * @brief Upon success, gets NPU's relaxed ordering mode. + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[out] pMode relaxed ordering mode + */ +NV_STATUS +osGetIbmnpuRelaxedOrderingMode +( + OS_GPU_INFO *pOsGpuInfo, + NvBool *pMode +) +{ + return nv_get_ibmnpu_relaxed_ordering_mode(pOsGpuInfo, pMode); +} + +/* + * @brief Waits for NVLink HW flush on an NPU associated with a GPU. + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + */ +void +osWaitForIbmnpuRsync +( + OS_GPU_INFO *pOsGpuInfo +) +{ + nv_wait_for_ibmnpu_rsync(pOsGpuInfo); +} + +NvU32 +osGetPageSize() +{ + return os_page_size; +} + + + +/* + * @brief Opens a new temporary file for reading and writing + * + * @param[in] ppFile void double pointer + * + * @returns NV_STATUS, NV_OK if success, + NV_ERR_GENERIC, if error + NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osOpenTemporaryFile +( + void **ppFile +) +{ + return os_open_temporary_file(ppFile); +} + +/* + * @brief Closes the specified temporary file + * + * @param[in] pFile Pointer to file + * + * @returns void + */ +void +osCloseFile +( + void *pFile +) +{ + os_close_file(pFile); +} + +/* + * @brief Writes the buffer to the specified file at the given offset + * + * @param[in] pFile Pointer to file (void) + * @param[in] pBuffer Pointer to buffer from which to copy + * @param[in] size Size of the copy + * @parma[in] offset offset in the file + * + * @returns NV_STATUS, NV_OK if success, + NV_ERR_GENERIC, if error + NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osWriteToFile +( + void *pFile, + NvU8 *pBuffer, + NvU64 size, + NvU64 offset +) +{ + return os_write_file(pFile, pBuffer, size, offset); +} + +/* + * @brief Reads from the specified file at the given offset + * + * @param[in] pFile Pointer to file (void *) + * @param[in] pBuffer Pointer to buffer to which the data is copied + * @param[in] size Size of the copy + * @parma[in] offset offset in the file + * + * @returns NV_STATUS, NV_OK if success, + NV_ERR_GENERIC, if error + NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osReadFromFile +( + void *pFile, + NvU8 *pBuffer, + NvU64 size, + NvU64 offset +) +{ + return os_read_file(pFile, pBuffer, size, offset); +} + +/* + * @brief Unregisters caps from the capability framework. + * The function assumes that the caps are allocated and stored in the + * hierarchical order. If they aren't, OS (Linux kernel) would warn and + * leak the caps. + * + * @param[in] pOsRmCaps caps of interest + */ +void +osRmCapUnregister +( + OS_RM_CAPS **ppOsRmCaps +) +{ + OS_RM_CAPS *pOsRmCaps = *ppOsRmCaps; + NvS32 i; + + if (pOsRmCaps == NULL) + { + return; + } + + for (i = pOsRmCaps->count - 1; i >= 0; i--) + { + if (pOsRmCaps->caps[i] != NULL) + { + os_nv_cap_destroy_entry(pOsRmCaps->caps[i]); + } + } + + os_free_mem(pOsRmCaps->caps); + os_free_mem(pOsRmCaps); + + *ppOsRmCaps = NULL; +} + +static NV_STATUS +_allocOsRmCaps +( + OS_RM_CAPS **ppOsRmCaps, + NvU32 count +) +{ + NV_STATUS status; + OS_RM_CAPS *pOsRmCaps; + + *ppOsRmCaps = NULL; + + status = os_alloc_mem((void**)&pOsRmCaps, sizeof(OS_RM_CAPS)); + if (status != NV_OK) + return status; + + pOsRmCaps->count = count; + + status = os_alloc_mem((void**)&pOsRmCaps->caps, sizeof(pOsRmCaps->caps[0]) * count); + if (status != NV_OK) + { + os_free_mem(pOsRmCaps); + return status; + } + + os_mem_set(pOsRmCaps->caps, 0, sizeof(pOsRmCaps->caps[0]) * count); + + *ppOsRmCaps = pOsRmCaps; + return NV_OK; +} + +#define OS_RM_CAP_GPU_DIR 0 +#define OS_RM_CAP_GPU_MIG_DIR 1 +#define OS_RM_CAP_GPU_COUNT 2 + +/* + * @brief Registers OBJGPU with the capability framework. + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[out] ppOsRmCaps GPU OS specific capabilities pointer + */ +NV_STATUS +osRmCapRegisterGpu +( + OS_GPU_INFO *pOsGpuInfo, + OS_RM_CAPS **ppOsRmCaps +) +{ + NvU32 minor = nv_get_dev_minor(pOsGpuInfo); + char name[16]; + NV_STATUS status; + OS_RM_CAPS *pOsRmCaps; + nv_cap_t *parent; + nv_cap_t *cap; + + // Return success on the unsupported platforms. + if (nvidia_caps_root == NULL) + { + return NV_OK; + } + + if (*ppOsRmCaps != NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_GPU_COUNT); + if (status != NV_OK) + return status; + + *ppOsRmCaps = pOsRmCaps; + + os_snprintf(name, sizeof(name), "gpu%u", minor); + name[sizeof(name) - 1] = '\0'; + parent = nvidia_caps_root; + + cap = os_nv_cap_create_dir_entry(parent, name, (OS_RUGO | OS_XUGO)); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup gpu%u directory\n", minor); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_GPU_DIR] = cap; + parent = cap; + + // TODO: Bug 2679591: Add MIG directory only if SMC is enabled. + // For now, always add "mig" directory. + cap = os_nv_cap_create_dir_entry(parent, "mig", (OS_RUGO | OS_XUGO)); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup mig directory\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR] = cap; + + return NV_OK; + +failed: + osRmCapUnregister(ppOsRmCaps); + + return status; +} + +#define OS_RM_CAP_SMC_PART_DIR 0 +#define OS_RM_CAP_SMC_PART_ACCESS_FILE 1 +#define OS_RM_CAP_SMC_PART_COUNT 2 + +/* + * @brief Registers SMC partition (a.k.a. GPU instance) with the capability + * framework + * + * @param[in] pGpuOsRmCaps GPU OS specific capabilities pointer + * @param[out] ppPartitionOsRmCaps OS specific capabilities pointer for SMC partition + * @param[in] swizzId SMC partition swizz ID + */ +NV_STATUS +osRmCapRegisterSmcPartition +( + OS_RM_CAPS *pGpuOsRmCaps, + OS_RM_CAPS **ppPartitionOsRmCaps, + NvU32 swizzId +) +{ + char name[16]; + NV_STATUS status; + nv_cap_t *parent; + nv_cap_t *cap; + OS_RM_CAPS *pOsRmCaps; + + // Return success as there is nothing to do. + if (pGpuOsRmCaps == NULL) + { + return NV_OK; + } + + if (*ppPartitionOsRmCaps != NULL || swizzId >= NV_U32_MAX) + { + return NV_ERR_INVALID_ARGUMENT; + } + + parent = pGpuOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR]; + if (parent == NULL) + { + return NV_ERR_INVALID_STATE; + } + + status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_PART_COUNT); + if (status != NV_OK) + return status; + + *ppPartitionOsRmCaps = pOsRmCaps; + + os_snprintf(name, sizeof(name), "gi%u", swizzId); + name[sizeof(name) - 1] = '\0'; + + cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup gi%u directory\n", + swizzId); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR] = cap; + parent = cap; + + cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n", + swizzId); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_SMC_PART_ACCESS_FILE] = cap; + + return NV_OK; + +failed: + osRmCapUnregister(ppPartitionOsRmCaps); + + return status; +} + +#define OS_RM_CAP_SMC_EXEC_PART_DIR 0 +#define OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE 1 +#define OS_RM_CAP_SMC_EXEC_PART_COUNT 2 + +/* + * @brief Registers SMC execution partition (a.k.a. compute instance) with the + * capability framework + * + * @param[in] pPartitionOsRmCaps OS specific capabilities pointer for SMC partition + * @param[out] ppExecPartitionOsRmCaps OS specific capabilities pointer for SMC execution partition + * @param[in] execPartitionId SMC execution partition ID + */ +NV_STATUS +osRmCapRegisterSmcExecutionPartition +( + OS_RM_CAPS *pPartitionOsRmCaps, + OS_RM_CAPS **ppExecPartitionOsRmCaps, + NvU32 execPartitionId +) +{ + char name[16]; + NV_STATUS status; + nv_cap_t *parent; + nv_cap_t *cap; + OS_RM_CAPS *pOsRmCaps; + + // Return success as there is nothing to do. + if (pPartitionOsRmCaps == NULL) + { + return NV_OK; + } + + if ((*ppExecPartitionOsRmCaps != NULL) || (execPartitionId >= NV_U32_MAX)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + parent = pPartitionOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR]; + if (parent == NULL) + { + return NV_ERR_INVALID_STATE; + } + + status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_EXEC_PART_COUNT); + if (status != NV_OK) + { + return status; + } + + *ppExecPartitionOsRmCaps = pOsRmCaps; + + os_snprintf(name, sizeof(name), "ci%u", execPartitionId); + name[sizeof(name) - 1] = '\0'; + + cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup ci%u directory\n", + execPartitionId); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_DIR] = cap; + parent = cap; + + cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n", + execPartitionId); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + + pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE] = cap; + + return NV_OK; + +failed: + osRmCapUnregister(ppExecPartitionOsRmCaps); + + return status; +} + +/* + * @brief Release the acquired capability + * + * @param[in] dupedCapDescriptor descriptor to be released + */ +void +osRmCapRelease +( + NvU64 dupedCapDescriptor +) +{ + if (dupedCapDescriptor == NV_U64_MAX) + { + return; + } + + os_nv_cap_close_fd((int)dupedCapDescriptor); +} + +#define OS_RM_CAP_SYS_MIG_DIR 0 +#define OS_RM_CAP_SYS_SMC_CONFIG_FILE 1 +#define OS_RM_CAP_SYS_SMC_MONITOR_FILE 2 +#define OS_RM_CAP_SYS_COUNT 3 + +NV_STATUS +osRmCapRegisterSys +( + OS_RM_CAPS **ppOsRmCaps +) +{ + nv_cap_t **ppCaps; + nv_cap_t *parent; + nv_cap_t *cap; + NV_STATUS status; + OS_RM_CAPS *pOsRmCaps; + + if (nvidia_caps_root == NULL) + return NV_ERR_NOT_SUPPORTED; + + status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SYS_COUNT); + if (status != NV_OK) + return status; + + *ppOsRmCaps = pOsRmCaps; + + ppCaps = pOsRmCaps->caps; + + parent = os_nv_cap_create_dir_entry(nvidia_caps_root, "mig", OS_RUGO | OS_XUGO); + if (parent == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create mig directory\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_MIG_DIR] = parent; + + cap = os_nv_cap_create_file_entry(parent, "config", OS_RUSR); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create mig config file\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_SMC_CONFIG_FILE] = cap; + + cap = os_nv_cap_create_file_entry(parent, "monitor", OS_RUGO); + if (cap == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to create mig monitor file\n"); + status = NV_ERR_OPERATING_SYSTEM; + goto failed; + } + ppCaps[OS_RM_CAP_SYS_SMC_MONITOR_FILE] = cap; + + return NV_OK; + +failed: + osRmCapUnregister(ppOsRmCaps); + return status; +} + +/* + * @brief Acquire the requested capability + * + * @param[in] pOsRmCaps opaque pointer to the caps. + * @param[in] rmCap the capability to be acquired. + * @param[in] capDescriptor descriptor to be used for validation + * @param[out] dupedCapDescriptor returns duplicated descriptor if validation + * is successful + * + * Note: On Linux, duplicating fd is helpful to let administrators know about + * the capability users. See https://linux.die.net/man/8/lsof usage. + */ +NV_STATUS +osRmCapAcquire +( + OS_RM_CAPS *pOsRmCaps, + NvU32 rmCap, + NvU64 capDescriptor, + NvU64 *dupedCapDescriptor +) +{ + nv_cap_t *cap; + int fd = (int)capDescriptor; + int duped_fd; + NvU32 index; + NV_STATUS status; + + *dupedCapDescriptor = NV_U64_MAX; + + switch (rmCap) + { + case NV_RM_CAP_SMC_PARTITION_ACCESS: + { + index = OS_RM_CAP_SMC_PART_ACCESS_FILE; + break; + } + case NV_RM_CAP_EXT_FABRIC_MGMT: + { + status = nv_acquire_fabric_mgmt_cap(fd, &duped_fd); + if (status != NV_OK) + { + return status; + } + + goto done; + } + case NV_RM_CAP_SMC_EXEC_PARTITION_ACCESS: + { + index = OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE; + break; + } + case NV_RM_CAP_SYS_SMC_CONFIG: + { + index = OS_RM_CAP_SYS_SMC_CONFIG_FILE; + break; + } + case NV_RM_CAP_SYS_SMC_MONITOR: + { + index = OS_RM_CAP_SYS_SMC_MONITOR_FILE; + break; + } + default: + { + return NV_ERR_INVALID_ARGUMENT; + } + } + + if (pOsRmCaps == NULL) + { + return NV_ERR_NOT_SUPPORTED; + } + + if (index >= pOsRmCaps->count) + { + return NV_ERR_INVALID_ARGUMENT; + } + + cap = pOsRmCaps->caps[index]; + + duped_fd = os_nv_cap_validate_and_dup_fd(cap, fd); + if (duped_fd < 0) + { + return NV_ERR_INSUFFICIENT_PERMISSIONS; + } + +done: + *dupedCapDescriptor = duped_fd; + + return NV_OK; +} + +/* + * @brief Initializes capability descriptor + * + * @param[out] pCapDescriptor descriptor to be used + * + */ +void +osRmCapInitDescriptor +( + NvU64 *pCapDescriptor +) +{ + *pCapDescriptor = NV_U64_MAX; +} + +/* + * @brief Generates random bytes which can be used as a universally unique + * identifier. + * + * @param[out] pBytes Array of random bytes + * @param[in] numBytes Size of the array + */ +NV_STATUS +osGetRandomBytes +( + NvU8 *pBytes, + NvU16 numBytes +) +{ + os_get_random_bytes(pBytes, numBytes); + + return NV_OK; +} + +/* + * @brief Allocate wait queue + * + * @param[out] ppWq Wait queue + */ +NV_STATUS +osAllocWaitQueue +( + OS_WAIT_QUEUE **ppWq +) +{ + return os_alloc_wait_queue(ppWq); +} + +/* + * @brief Free wait queue + * + * @param[in] pWq Wait queue + */ +void +osFreeWaitQueue +( + OS_WAIT_QUEUE *pWq +) +{ + os_free_wait_queue(pWq); +} + +/* + * @brief Put thread to uninterruptible sleep + * + * @param[in] pWq Wait queue + */ +void +osWaitUninterruptible +( + OS_WAIT_QUEUE *pWq +) +{ + os_wait_uninterruptible(pWq); +} + +/* + * @brief Put thread to interruptible sleep + * + * @param[in] pWq Wait queue + */ +void +osWaitInterruptible +( + OS_WAIT_QUEUE *pWq +) +{ + os_wait_interruptible(pWq); +} + +/* + * @brief Wake up thread from uninterruptible sleep + * + * @param[in] pWq Wait queue + */ +void +osWakeUp +( + OS_WAIT_QUEUE *pWq +) +{ + os_wake_up(pWq); +} + +NV_STATUS +osReadPFPciConfigInVF +( + NvU32 addr, + NvU32 *data +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Returns IMP-relevant data collected from other modules + * + * This function is basically a wrapper to call the unix/linux layer. + * + * @param[out] pTegraImpImportData Structure to receive the data + * + * @returns NV_OK if successful, + * NV_ERR_BUFFER_TOO_SMALL if the array in TEGRA_IMP_IMPORT_DATA is + * too small, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * other errors as may be returned by subfunctions. + */ +NV_STATUS +osTegraSocGetImpImportData +( + TEGRA_IMP_IMPORT_DATA *pTegraImpImportData +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Tells BPMP whether or not RFL is valid + * + * Display HW generates an ok_to_switch signal which asserts when mempool + * occupancy is high enough to be able to turn off memory long enough to + * execute a dramclk frequency switch without underflowing display output. + * ok_to_switch drives the RFL ("request for latency") signal in the memory + * unit, and the switch sequencer waits for this signal to go active before + * starting a dramclk switch. However, if the signal is not valid (e.g., if + * display HW or SW has not been initialized yet), the switch sequencer ignores + * the signal. This API tells BPMP whether or not the signal is valid. + * + * @param[in] pOsGpuInfo Per GPU Linux state + * @param[in] bEnable True if RFL will be valid; false if invalid + * + * @returns NV_OK if successful, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * NV_ERR_GENERIC if some other kind of error occurred. + */ +NV_STATUS +osTegraSocEnableDisableRfl +( + OS_GPU_INFO *pOsGpuInfo, + NvBool bEnable +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Allocates a specified amount of ISO memory bandwidth for display + * + * floorBandwidthKBPS is the minimum required (i.e., floor) dramclk frequency + * multiplied by the width of the pipe over which the display data will travel. + * (It is understood that the bandwidth calculated by multiplying the clock + * frequency by the pipe width will not be realistically achievable, due to + * overhead in the memory subsystem. The infrastructure will not actually use + * the bandwidth value, except to reverse the calculation to get the required + * dramclk frequency.) + * + * This function is basically a wrapper to call the unix/linux layer. + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[in] averageBandwidthKBPS Amount of ISO memory bandwidth requested + * @param[in] floorBandwidhtKBPS Min required dramclk freq * pipe width + * + * @returns NV_OK if successful, + * NV_ERR_INSUFFICIENT_RESOURCES if one of the bandwidth values is too + * high, and bandwidth cannot be allocated, + * NV_ERR_NOT_SUPPORTED if the functionality is not available, or + * NV_ERR_GENERIC if some other kind of error occurred. + */ +NV_STATUS +osTegraAllocateDisplayBandwidth +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 averageBandwidthKBPS, + NvU32 floorBandwidthKBPS +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Creates or sets up platform specific nano second resolution timer + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[in] pTmrEvent Pointer to timer event information + * @param[in/out] pTimer pointer to hold high resolution timer object + */ +NV_STATUS +osCreateNanoTimer +( + OS_GPU_INFO *pOsGpuInfo, + void *pTmrEvent, + void **pTimer +) +{ + nv_create_nano_timer(pOsGpuInfo, pTmrEvent, (nv_nano_timer_t **)pTimer); + return NV_OK; +} + +/*! + * @brief Starts platform specific nano second resolution timer + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[in] pTimer pointer to high resolution timer object + * @param[in] timens time in nano seconds + */ +NV_STATUS +osStartNanoTimer +( + OS_GPU_INFO *pOsGpuInfo, + void *pTimer, + NvU64 timeNs +) +{ + nv_start_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer, timeNs); + return NV_OK; +} + +/*! + * @brief Cancels platform specific nano second resolution timer + * + * @param[in] pOsGpuInfo OS specific GPU information pointer + * @param[in] pTimer pointer to timer object + */ +NV_STATUS +osCancelNanoTimer +( + OS_GPU_INFO *pOsGpuInfo, + void *pTimer +) +{ + nv_cancel_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer); + return NV_OK; +} + +/*! + * @brief Destroys & cancels platform specific nano second resolution timer + * + * + * @param[in] pGpu Device of interest + * @param[in] pTimer pointer to timer object + */ +NV_STATUS +osDestroyNanoTimer +( + OS_GPU_INFO *pOsGpuInfo, + void *pTimer +) +{ + nv_destroy_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer); + return NV_OK; +} + +/*! + * @brief Get number of dpAux instances. + * It is wrapper function to call unix/linux layer. + * + * @param[in] pGpu GPU object pointer + * @param[out] pNumIntances Number of dpAux instances. + * + * @returns NV_STATUS, NV_OK if success, + * NV_ERR_GENERIC, if error + * NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ + +NV_STATUS +osGetTegraNumDpAuxInstances +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *pNumIntances +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/* + * @brief Return the priv Data of current IRQ. + * It is wrapper function to call unix/linux layer. + * + * @param[in] pGpu Device of interest + * @param[out] pPrivData privData of current IRQ + * + * @returns NV_STATUS, NV_OK if success, + * NV_ERR_GENERIC, if error + * NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osGetCurrentIrqPrivData +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *pPrivData +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Get the brightness level + * It is wrapper function to call unix/linux layer. + * + * @param[in] pGpu GPU object pointer + * @param[out] brightness Pointer to brightness level + * + * @returns NV_STATUS, NV_OK if success, + * NV_ERR_GENERIC, if error + * NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osGetTegraBrightnessLevel +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *brightness +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Set the brightness level + * It is wrapper function to call unix/linux layer. + * + * @param[in] pGpu GPU object pointer + * @param[out] brightness brightness level + * + * @returns NV_STATUS, NV_OK if success, + * NV_ERR_GENERIC, if error + * NV_ERR_NOT_SUPPORTED, for unsupported platforms + */ +NV_STATUS +osSetTegraBrightnessLevel +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 brightness +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/* @brief Gets syncpoint aperture information + * + * @param[in] OS_GPU_INFO OS specific GPU information pointer + * @param[in] syncpointId + * @param[out] *physAddr + * @param[out] *limit + * @param[out] *offset + */ +NV_STATUS +osGetSyncpointAperture +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 syncpointId, + NvU64 *physAddr, + NvU64 *limit, + NvU32 *offset +) +{ + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Enable PCIe AtomicOp Requester Enable and return + * the completer side capabilities that the requester can send. + * + * @param[in] pOsGpuInfo OS_GPU_INFO OS specific GPU information pointer + * @param[out] pMask mask of supported atomic size, including one or more of: + * OS_PCIE_CAP_MASK_REQ_ATOMICS_32 + * OS_PCIE_CAP_MASK_REQ_ATOMICS_64 + * OS_PCIE_CAP_MASK_REQ_ATOMICS_128 + * + * @returns NV_STATUS, NV_OK if success + * NV_ERR_NOT_SUPPORTED if platform doesn't support this + * feature. + * NV_ERR_GENERIC for any other error + */ + +NV_STATUS +osConfigurePcieReqAtomics +( + OS_GPU_INFO *pOsGpuInfo, + NvU32 *pMask +) +{ + if (pMask) + { + *pMask = 0U; + if (pOsGpuInfo) + { + if (os_enable_pci_req_atomics(pOsGpuInfo->handle, + OS_INTF_PCIE_REQ_ATOMICS_32BIT) == NV_OK) + *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_32; + if (os_enable_pci_req_atomics(pOsGpuInfo->handle, + OS_INTF_PCIE_REQ_ATOMICS_64BIT) == NV_OK) + *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_64; + if (os_enable_pci_req_atomics(pOsGpuInfo->handle, + OS_INTF_PCIE_REQ_ATOMICS_128BIT) == NV_OK) + *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_128; + + if (*pMask != 0) + return NV_OK; + } + } + return NV_ERR_NOT_SUPPORTED; +} + +/*! + * @brief Check GPU is accessible or not + * + * @param[in] pGpu GPU object pointer + * + * @returns NVBool, Returns TRUE if the GPU is accessible, + * FALSE, if error + */ +NvBool +osIsGpuAccessible +( + OBJGPU *pGpu +) +{ + return nv_is_gpu_accessible(NV_GET_NV_STATE(pGpu)); +} + +/*! + * @brief Check whether GPU has received a shutdown notification from the OS + */ +NvBool +osIsGpuShutdown +( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + return nv ? nv->is_shutdown : NV_TRUE; +} + +/*! + * @brief Check GPU OS info matches + * + * @param[in] pGpu GPU object pointer + * + * @returns NVBool, Returns TRUE if matched. + */ +NvBool +osMatchGpuOsInfo +( + OBJGPU *pGpu, + void *pOsInfo +) +{ + return nv_match_gpu_os_info(NV_GET_NV_STATE(pGpu), pOsInfo); +} + +/*! + * @brief Release GPU OS info. + * + * @param[in] pOsInfo GPU OS info pointer + * + * @returns void + */ +void +osReleaseGpuOsInfo +( + void *pOsInfo +) +{ + nv_put_file_private(pOsInfo); +} + +NvBool +osDmabufIsSupported(void) +{ + return os_dma_buf_enabled; +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osapi.c b/userspace/sdks/nv525/nvalloc/unix/src/osapi.c new file mode 100644 index 0000000..dcd8604 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/osapi.c @@ -0,0 +1,5333 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + + +#include <nv_ref.h> +#include <nv.h> +#include <nv-priv.h> +#include <os/os.h> +#include <osapi.h> +#include <class/cl0000.h> +#include <rmosxfac.h> // Declares RmInitRm(). +#include "gpu/gpu.h" +#include <osfuncs.h> +#include <platform/chipset/chipset.h> + +#include <objtmr.h> +#include <gpu/subdevice/subdevice.h> +#include <mem_mgr/mem.h> +#include "kernel/gpu/mem_mgr/mem_mgr.h" + +#include <gpu/mem_sys/kern_mem_sys.h> + +#include <diagnostics/journal.h> +#include <nvrm_registry.h> + +#include <nvUnixVersion.h> +#include <gpu_mgr/gpu_mgr.h> +#include <core/thread_state.h> +#include <platform/acpi_common.h> +#include <core/locks.h> + +#include <mem_mgr/p2p.h> + +#include "rmapi/exports.h" +#include "rmapi/rmapi_utils.h" +#include "rmapi/rs_utils.h" +#include "rmapi/resource_fwd_decls.h" +#include <nv-kernel-rmapi-ops.h> +#include <rmobjexportimport.h> +#include "nv-reg.h" +#include "core/hal_mgr.h" +#include "gpu/device/device.h" + +#include "resserv/rs_server.h" +#include "resserv/rs_client.h" +#include "resserv/rs_resource.h" +#include "gpu/gpu_uuid.h" + +#include "platform/chipset/pci_pbi.h" + +#include "ctrl/ctrl0000/ctrl0000system.h" +#include "ctrl/ctrl0073/ctrl0073dp.h" +#include "ctrl/ctrl0073/ctrl0073system.h" +#include "ctrl/ctrl0073/ctrl0073specific.h" +#include "ctrl/ctrl2080/ctrl2080bios.h" +#include "ctrl/ctrl2080/ctrl2080fb.h" +#include "ctrl/ctrl2080/ctrl2080perf.h" +#include "ctrl/ctrl2080/ctrl2080gpu.h" +#include "ctrl/ctrl402c.h" + +#include "g_nv_name_released.h" // released chip entries from nvChipAutoFlags.h + +#include <virtualization/hypervisor/hypervisor.h> + +#include "gpu/bus/kern_bus.h" + +// +// Helper function which can be called before doing any RM control +// This function: +// +// a. Performs threadStateInit(). +// b. Acquires API lock. +// c. Checks if RMAPI client handle is valid (i.e. RM is initialized) and +// returns early if RMAPI client handle is invalid. +// d. Increments the dynamic power refcount. If GPU is in RTD3 suspended +// state, then it will wake-up the GPU. +// e. Returns the RMAPI interface handle. +// +// This function should be called only when caller doesn't have acquired API +// lock. Caller needs to use RmUnixRmApiEpilogue() after RM control, if +// RmUnixRmApiPrologue() is successful. +// +RM_API *RmUnixRmApiPrologue(nv_state_t *pNv, THREAD_STATE_NODE *pThreadNode, NvU32 module) +{ + threadStateInit(pThreadNode, THREAD_STATE_FLAGS_NONE); + + if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, module)) == NV_OK) + { + if ((pNv->rmapi.hClient != 0) && + (os_ref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE) == NV_OK)) + { + return rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + } + + rmapiLockRelease(); + } + + threadStateFree(pThreadNode, THREAD_STATE_FLAGS_NONE); + + return NULL; +} + +// +// Helper function which can be called after doing RM control, if +// caller has used RmUnixRmApiPrologue() helper function. This function: +// +// a. Decrements the dynamic power refcount. +// b. Release API lock. +// c. Performs threadStateFree(). +// +void RmUnixRmApiEpilogue(nv_state_t *pNv, THREAD_STATE_NODE *pThreadNode) +{ + os_unref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE); + rmapiLockRelease(); + threadStateFree(pThreadNode, THREAD_STATE_FLAGS_NONE); +} + +NvBool RmGpuHasIOSpaceEnabled(nv_state_t * nv) +{ + NvU16 val; + NvBool has_io; + os_pci_read_word(nv->handle, NV_CONFIG_PCI_NV_1, &val); + has_io = FLD_TEST_DRF(_CONFIG, _PCI_NV_1, _IO_SPACE, _ENABLED, val); + return has_io; +} + +// This is a stub function for unix +void osHandleDeferredRecovery( + OBJGPU *pGpu +) +{ + +} + +// This is a stub function for unix +NvBool osIsSwPreInitOnly +( + OS_GPU_INFO *pOsGpuInfo +) +{ + return NV_FALSE; +} + +const NvU8 * RmGetGpuUuidRaw( + nv_state_t *pNv +) +{ + NV_STATUS rmStatus; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + NvU32 gidFlags; + NvBool isApiLockTaken = NV_FALSE; + + if (pNv->nv_uuid_cache.valid) + goto done; + + // + // PBI is not present in simulation and the loop inside + // pciPbiReadUuid takes up considerable amount of time in + // simulation environment during RM load. + // + if (pGpu && IS_SIMULATION(pGpu)) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + } + else + { + rmStatus = pciPbiReadUuid(pNv->handle, pNv->nv_uuid_cache.uuid); + } + + if (rmStatus == NV_OK) + { + rmStatus = gpumgrSetUuid(pNv->gpu_id, pNv->nv_uuid_cache.uuid); + if (rmStatus != NV_OK) + { + return NULL; + } + + pNv->nv_uuid_cache.valid = NV_TRUE; + goto done; + } + else if (rmStatus == NV_ERR_NOT_SUPPORTED) + { + nv_printf(NV_DBG_INFO, + "NVRM: PBI is not supported for GPU " NV_PCI_DEV_FMT "\n", + NV_PCI_DEV_FMT_ARGS(pNv)); + } + + gidFlags = DRF_DEF(2080_GPU_CMD,_GPU_GET_GID_FLAGS,_TYPE,_SHA1) + | DRF_DEF(2080_GPU_CMD,_GPU_GET_GID_FLAGS,_FORMAT,_BINARY); + + if (!rmapiLockIsOwner()) + { + rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU); + if (rmStatus != NV_OK) + { + return NULL; + } + + isApiLockTaken = NV_TRUE; + } + + if (pGpu == NULL) + { + if (isApiLockTaken == NV_TRUE) + { + rmapiLockRelease(); + } + + return NULL; + } + + rmStatus = gpuGetGidInfo(pGpu, NULL, NULL, gidFlags); + if (isApiLockTaken == NV_TRUE) + { + rmapiLockRelease(); + } + + if (rmStatus != NV_OK) + return NULL; + + if (!pGpu->gpuUuid.isInitialized) + return NULL; + + // copy the uuid from the OBJGPU uuid cache + os_mem_copy(pNv->nv_uuid_cache.uuid, pGpu->gpuUuid.uuid, GPU_UUID_LEN); + pNv->nv_uuid_cache.valid = NV_TRUE; + +done: + return pNv->nv_uuid_cache.uuid; +} + +static NV_STATUS RmGpuUuidRawToString( + const NvU8 *pGidData, + char *pBuf, + NvU32 bufLen +) +{ + NvU8 *pGidString; + NvU32 GidStrlen; + NV_STATUS rmStatus; + NvU32 gidFlags; + + gidFlags = DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _ASCII) | + DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1); + + rmStatus = transformGidToUserFriendlyString(pGidData, RM_SHA1_GID_SIZE, + &pGidString, &GidStrlen, + gidFlags); + if (rmStatus != NV_OK) + return rmStatus; + + if (bufLen >= GidStrlen) + portMemCopy(pBuf, bufLen, pGidString, GidStrlen); + else + rmStatus = NV_ERR_BUFFER_TOO_SMALL; + + portMemFree((void *)pGidString); + + return rmStatus; +} + +// This function should be called with the API and GPU locks already acquired. +NV_STATUS +RmLogGpuCrash(OBJGPU *pGpu) +{ + NV_STATUS status = NV_OK; + NvBool bGpuIsLost, bGpuIsConnected; + + if (pGpu == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // + // Re-evaluate whether or not the GPU is accessible. This could be called + // from a recovery context where the OS has re-enabled MMIO for the device. + // This happens during EEH processing on IBM Power + Linux, and marking + // the device as connected again will allow rcdbAddRmGpuDump() to collect + // more GPU state. + // + bGpuIsLost = pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_LOST); + bGpuIsConnected = pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED); + if (!bGpuIsConnected || bGpuIsLost) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 pmcBoot0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0); + if (pmcBoot0 == nvp->pmc_boot_0) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED, NV_TRUE); + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_LOST, NV_FALSE); + } + } + + // + // Log the engine data to the Journal object, to be pulled out later. This + // will return NV_WARN_MORE_PROCESSING_REQUIRED if the dump needed to be + // deferred to a passive IRQL. We still log the crash dump as being created + // in that case since it (should) be created shortly thereafter, and + // there's currently not a good way to print the below notification + // publicly from the core RM when the DPC completes. + // + status = rcdbAddRmGpuDump(pGpu); + if (status != NV_OK && status != NV_WARN_MORE_PROCESSING_REQUIRED) + { + NV_PRINTF(LEVEL_ERROR, + "%s: failed to save GPU crash data\n", __FUNCTION__); + } + else + { + status = NV_OK; + nv_printf(NV_DBG_ERRORS, + "NVRM: A GPU crash dump has been created. If possible, please run\n" + "NVRM: nvidia-bug-report.sh as root to collect this data before\n" + "NVRM: the NVIDIA kernel module is unloaded.\n"); + } + + // Restore the disconnected properties, if they were reset + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED, bGpuIsConnected); + pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_LOST, bGpuIsLost); + + // Restore persistence mode to the way it was prior to the crash + osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, + pGpu->getProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE)); + + return status; +} + +static void free_os_event_under_lock(nv_event_t *event) +{ + event->active = NV_FALSE; + + // If refcount > 0, event will be freed by osDereferenceObjectCount + // when the last associated RM event is freed. + if (event->refcount == 0) + { + portMemFree(event); + } +} + +static void free_os_events( + nv_file_private_t *nvfp, + NvHandle client +) +{ + nv_state_t *nv = nv_get_ctl_state(); + nv_event_t **pprev; + + portSyncSpinlockAcquire(nv->event_spinlock); + + pprev = &nv->event_list; + while (*pprev != NULL) + { + nv_event_t *cur = *pprev; + // + // XXX We must be called from either rm_client_free_os_events() or + // RmFreeUnusedClients() for this to work. + // + if ((cur->hParent == client) || (cur->nvfp == nvfp)) + { + *pprev = cur->next; + free_os_event_under_lock(cur); + } + else + { + pprev = &cur->next; + } + } + + portSyncSpinlockRelease(nv->event_spinlock); +} + +void rm_client_free_os_events( + NvHandle client +) +{ + free_os_events(NULL, client); +} + +void RmFreeUnusedClients( + nv_state_t *nv, + nv_file_private_t *nvfp +) +{ + NvU32 *pClientList; + NvU32 numClients, i; + NV_STATUS status; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + + // + // The 'nvfp' pointer uniquely identifies an open instance in kernel space + // and the kernel interface layer guarantees that we are not called before + // the associated nvfp descriptor is closed. We can thus safely free + // abandoned clients with matching 'nvfp' pointers. + // + status = rmapiGetClientHandlesFromOSInfo(nvfp, &pClientList, &numClients); + if (status != NV_OK) + { + numClients = 0; + } + + for (i = 0; i < numClients; ++i) + { + NV_PRINTF(LEVEL_INFO, "freeing abandoned client 0x%x\n", + pClientList[i]); + + } + + if (numClients != 0) + { + pRmApi->FreeClientList(pRmApi, pClientList, numClients); + + portMemFree(pClientList); + } + + // Clean up any remaining events using this nvfp. + free_os_events(nvfp, 0); +} + +static void RmUnbindLock( + nv_state_t *nv +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if ((pGpu == NULL) || (gpuGetUserClientCount(pGpu) == 0)) + { + nv->flags |= NV_FLAG_UNBIND_LOCK; + } +} + +static NV_STATUS allocate_os_event( + NvHandle hParent, + nv_file_private_t *nvfp, + NvU32 fd +) +{ + nv_state_t *nv = nv_get_ctl_state(); + NvU32 status = NV_OK; + nv_event_t *event; + + nv_event_t *new_event = portMemAllocNonPaged(sizeof(nv_event_t)); + if (new_event == NULL) + { + status = NV_ERR_NO_MEMORY; + goto done; + } + + portSyncSpinlockAcquire(nv->event_spinlock); + for (event = nv->event_list; event; event = event->next) + { + // Only one event may be associated with a given fd. + if (event->hParent == hParent && event->fd == fd) + { + status = NV_ERR_INVALID_ARGUMENT; + portSyncSpinlockRelease(nv->event_spinlock); + goto done; + } + } + + new_event->next = nv->event_list; + nv->event_list = new_event; + portSyncSpinlockRelease(nv->event_spinlock); + +done: + if (status == NV_OK) + { + new_event->hParent = hParent; + new_event->nvfp = nvfp; + new_event->fd = fd; + new_event->active = NV_TRUE; + new_event->refcount = 0; + + NV_PRINTF(LEVEL_INFO, "allocated OS event:\n"); + NV_PRINTF(LEVEL_INFO, " hParent: 0x%x\n", hParent); + NV_PRINTF(LEVEL_INFO, " fd: %d\n", fd); + } + else + { + portMemFree(new_event); + } + + return status; +} + +NV_STATUS RmAllocOsEvent( + NvHandle hParent, + nv_file_private_t *nvfp, + NvU32 fd +) +{ + if (NV_OK != allocate_os_event(hParent, nvfp, fd)) + { + NV_PRINTF(LEVEL_ERROR, "failed to allocate OS event\n"); + return NV_ERR_INSUFFICIENT_RESOURCES; + } + return NV_OK; +} + +static NV_STATUS free_os_event( + NvHandle hParent, + NvU32 fd +) +{ + nv_state_t *nv = nv_get_ctl_state(); + nv_event_t *event, *tmp; + NV_STATUS result; + + portSyncSpinlockAcquire(nv->event_spinlock); + tmp = event = nv->event_list; + while (event) + { + if ((event->fd == fd) && (event->hParent == hParent)) + { + if (event == nv->event_list) + nv->event_list = event->next; + else + tmp->next = event->next; + break; + } + tmp = event; + event = event->next; + } + + if (event != NULL) + { + free_os_event_under_lock(event); + result = NV_OK; + } + else + result = NV_ERR_INVALID_EVENT; + portSyncSpinlockRelease(nv->event_spinlock); + + if (result == NV_OK) + { + NV_PRINTF(LEVEL_INFO, "freed OS event:\n"); + NV_PRINTF(LEVEL_INFO, " hParent: 0x%x\n", hParent); + NV_PRINTF(LEVEL_INFO, " fd: %d\n", fd); + } + else + { + NV_PRINTF(LEVEL_ERROR, "failed to find OS event:\n"); + NV_PRINTF(LEVEL_ERROR, " hParent: 0x%x\n", hParent); + NV_PRINTF(LEVEL_ERROR, " fd: %d\n", fd); + } + + return result; +} + +NV_STATUS RmFreeOsEvent( + NvHandle hParent, + NvU32 fd +) +{ + if (NV_OK != free_os_event(hParent, fd)) + { + return NV_ERR_INVALID_EVENT; + } + return NV_OK; +} + +static void RmExecuteWorkItem( + void *pWorkItem +) +{ + nv_work_item_t *pWi = (nv_work_item_t *)pWorkItem; + NvU32 gpuMask; + NvU32 releaseLocks = 0; + + if (!(pWi->flags & NV_WORK_ITEM_FLAGS_REQUIRES_GPU) && + ((pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW) || + (pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW) || + (pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW) || + (pWi->flags & OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY))) + { + // Requesting one of the GPU locks without providing a GPU instance + NV_ASSERT(0); + goto done; + } + + // Get locks requested by workitem + if (NV_OK != workItemLocksAcquire(pWi->gpuInstance, pWi->flags, + &releaseLocks, &gpuMask)) + { + goto done; + } + + // Some work items may not require a valid GPU instance + if (pWi->flags & NV_WORK_ITEM_FLAGS_REQUIRES_GPU) + { + pWi->func.pGpuFunction(pWi->gpuInstance, pWi->pData); + } + else + { + pWi->func.pSystemFunction(pWi->pData); + } + + // Release any locks taken + workItemLocksRelease(releaseLocks, gpuMask); + +done: + if ((pWi->pData != NULL) && + !(pWi->flags & NV_WORK_ITEM_FLAGS_DONT_FREE_DATA)) + { + portMemFree(pWi->pData); + } + + portMemFree((void *)pWi); +} + +static NV_STATUS RmGetEventData( + nv_file_private_t *nvfp, + NvP64 pEvent, + NvU32 *MoreEvents, + NvBool bUserModeArgs +) +{ + NV_STATUS RmStatus; + NvUnixEvent *pKernelEvent = NULL; + nv_event_t nv_event; + RMAPI_PARAM_COPY paramCopy; + + RmStatus = nv_get_event(nvfp, &nv_event, MoreEvents); + if (RmStatus != NV_OK) + return NV_ERR_OPERATING_SYSTEM; + + // setup for access to client's parameters + RMAPI_PARAM_COPY_INIT(paramCopy, pKernelEvent, pEvent, 1, sizeof(NvUnixEvent)); + RmStatus = rmapiParamsAcquire(¶mCopy, bUserModeArgs); + if (RmStatus != NV_OK) + return NV_ERR_OPERATING_SYSTEM; + + pKernelEvent->hObject = nv_event.hObject; + pKernelEvent->NotifyIndex = nv_event.index; + pKernelEvent->info32 = nv_event.info32; + pKernelEvent->info16 = nv_event.info16; + + // release client buffer access, with copyout as needed + if (rmapiParamsRelease(¶mCopy) != NV_OK) + return NV_ERR_OPERATING_SYSTEM; + + return NV_OK; +} + +static NV_STATUS RmAccessRegistry( + NvHandle hClient, + NvHandle hObject, + NvU32 AccessType, + NvP64 clientDevNodeAddress, + NvU32 DevNodeLength, + NvP64 clientParmStrAddress, + NvU32 ParmStrLength, + NvP64 clientBinaryDataAddress, + NvU32 *pBinaryDataLength, + NvU32 *Data, + NvU32 *Entry +) +{ + NvU32 gpuMask = 0, gpuInstance = 0; + OBJGPU *pGpu; + NvBool isDevice = NV_FALSE; + NV_STATUS RmStatus = NV_ERR_OPERATING_SYSTEM; + RsClient *pClient; + Device *pDevice; + Subdevice *pSubdevice; + + RMAPI_PARAM_COPY devNodeParamCopy; + NvU8 *tmpDevNode = NULL; + NvU32 copyOutDevNodeLength = 0; + + RMAPI_PARAM_COPY parmStrParamCopy; + char *tmpParmStr = NULL; + NvU32 copyOutParmStrLength = 0; + + RMAPI_PARAM_COPY binaryDataParamCopy; + NvU8 *tmpBinaryData = NULL; + NvU32 BinaryDataLength = 0; + NvU32 copyOutBinaryDataLength = 0; + + if (NV_OK != serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClient)) + return NV_ERR_INVALID_CLIENT; + + if (hClient == hObject) + { + pGpu = NULL; + } + else + { + RmStatus = deviceGetByHandle(pClient, hObject, &pDevice); + if (RmStatus != NV_OK) + { + RmStatus = subdeviceGetByHandle(pClient, hObject, &pSubdevice); + if (RmStatus != NV_OK) + goto done; + + RmStatus = rmGpuGroupLockAcquire(pSubdevice->subDeviceInst, + GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU, + &gpuMask); + if (RmStatus != NV_OK) + return RmStatus; + + GPU_RES_SET_THREAD_BC_STATE(pSubdevice); + pGpu = GPU_RES_GET_GPU(pSubdevice); + } + else + { + RmStatus = rmGpuGroupLockAcquire(pDevice->deviceInst, + GPU_LOCK_GRP_DEVICE, + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU, + &gpuMask); + if (RmStatus != NV_OK) + return RmStatus; + + GPU_RES_SET_THREAD_BC_STATE(pDevice); + pGpu = GPU_RES_GET_GPU(pDevice); + isDevice = NV_TRUE; + } + } + + if (pBinaryDataLength) + { + BinaryDataLength = *pBinaryDataLength; + } + + // a passed-in devNode + if (DevNodeLength) + { + // the passed-in DevNodeLength does not account for '\0' + DevNodeLength++; + + if (DevNodeLength > NVOS38_MAX_REGISTRY_STRING_LENGTH) + { + RmStatus = NV_ERR_INVALID_STRING_LENGTH; + goto done; + } + + // get access to client's DevNode + RMAPI_PARAM_COPY_INIT(devNodeParamCopy, tmpDevNode, clientDevNodeAddress, DevNodeLength, 1); + devNodeParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_ZERO_BUFFER; + RmStatus = rmapiParamsAcquire(&devNodeParamCopy, NV_TRUE); + if (RmStatus != NV_OK) + { + RmStatus = NV_ERR_OPERATING_SYSTEM; + goto done; + } + } + + // a passed-in parmStr + if (ParmStrLength) + { + // the passed-in ParmStrLength does not account for '\0' + ParmStrLength++; + + if ((ParmStrLength == 0) || (ParmStrLength > NVOS38_MAX_REGISTRY_STRING_LENGTH)) + { + RmStatus = NV_ERR_INVALID_STRING_LENGTH; + goto done; + } + // get access to client's parmStr + RMAPI_PARAM_COPY_INIT(parmStrParamCopy, tmpParmStr, clientParmStrAddress, ParmStrLength, 1); + RmStatus = rmapiParamsAcquire(&parmStrParamCopy, NV_TRUE); + if (RmStatus != NV_OK) + { + RmStatus = NV_ERR_OPERATING_SYSTEM; + goto done; + } + if (tmpParmStr[ParmStrLength - 1] != '\0') + { + RmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + } + + if ((AccessType == NVOS38_ACCESS_TYPE_READ_BINARY) || + (AccessType == NVOS38_ACCESS_TYPE_WRITE_BINARY)) + { + if ((BinaryDataLength > NVOS38_MAX_REGISTRY_BINARY_LENGTH) || + (BinaryDataLength == 0)) + { + RmStatus = NV_ERR_INVALID_STRING_LENGTH; + goto done; + } + + // get access to client's binaryData + RMAPI_PARAM_COPY_INIT(binaryDataParamCopy, tmpBinaryData, clientBinaryDataAddress, BinaryDataLength, 1); + if (AccessType == NVOS38_ACCESS_TYPE_READ_BINARY) + binaryDataParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYIN; + RmStatus = rmapiParamsAcquire(&binaryDataParamCopy, NV_TRUE); + if (RmStatus != NV_OK) + { + RmStatus = NV_ERR_OPERATING_SYSTEM; + goto done; + } + } + + switch (AccessType) + { + case NVOS38_ACCESS_TYPE_READ_DWORD: + RmStatus = osReadRegistryDword(pGpu, + tmpParmStr, Data); + break; + + case NVOS38_ACCESS_TYPE_WRITE_DWORD: + if (isDevice && osIsAdministrator()) + { + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + RmStatus = osWriteRegistryDword(pGpu, + tmpParmStr, *Data); + + if (RmStatus != NV_OK) + goto done; + } + break; + } + + RmStatus = osWriteRegistryDword(pGpu, + tmpParmStr, *Data); + break; + + case NVOS38_ACCESS_TYPE_READ_BINARY: + RmStatus = osReadRegistryBinary(pGpu, + tmpParmStr, tmpBinaryData, &BinaryDataLength); + + if (RmStatus != NV_OK) + { + goto done; + } + + if (BinaryDataLength) + copyOutBinaryDataLength = BinaryDataLength; + + break; + + case NVOS38_ACCESS_TYPE_WRITE_BINARY: + if (isDevice && osIsAdministrator()) + { + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + RmStatus = osWriteRegistryBinary(pGpu, + tmpParmStr, tmpBinaryData, + BinaryDataLength); + + if (RmStatus != NV_OK) + goto done; + } + break; + } + + RmStatus = osWriteRegistryBinary(pGpu, + tmpParmStr, tmpBinaryData, + BinaryDataLength); + break; + + default: + RmStatus = NV_ERR_INVALID_ACCESS_TYPE; + } + + done: + if (gpuMask != 0) + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + + if (tmpDevNode != NULL) + { + // skip copyout on error + if ((RmStatus != NV_OK) || (copyOutDevNodeLength == 0)) + devNodeParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT; + devNodeParamCopy.paramsSize = copyOutDevNodeLength; + if (NV_OK != rmapiParamsRelease(&devNodeParamCopy)) + if (RmStatus == NV_OK) + RmStatus = NV_ERR_OPERATING_SYSTEM; + } + if (tmpParmStr != NULL) + { + // skip copyout on error + if ((RmStatus != NV_OK) || (copyOutParmStrLength == 0)) + parmStrParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT; + parmStrParamCopy.paramsSize = copyOutParmStrLength; + if (NV_OK != rmapiParamsRelease(&parmStrParamCopy)) + if (RmStatus == NV_OK) + RmStatus = NV_ERR_OPERATING_SYSTEM; + } + if (tmpBinaryData != NULL) + { + // skip copyout on error + if ((RmStatus != NV_OK) || (copyOutBinaryDataLength == 0)) + binaryDataParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT; + binaryDataParamCopy.paramsSize = copyOutBinaryDataLength; + if (NV_OK != rmapiParamsRelease(&binaryDataParamCopy)) + if (RmStatus == NV_OK) + RmStatus = NV_ERR_OPERATING_SYSTEM; + *pBinaryDataLength = copyOutBinaryDataLength; + } + + serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClient); + return RmStatus; +} + +static NV_STATUS RmUpdateDeviceMappingInfo( + NvHandle hClient, + NvHandle hDevice, + NvHandle hMappable, + void *pOldCpuAddress, + void *pNewCpuAddress +) +{ + NV_STATUS status; + RsClient *pClient; + RsResourceRef *pMappableRef; + RsCpuMapping *pCpuMapping; + Device *pDevice; + Subdevice *pSubdevice; + NvU32 gpuMask = 0; + + status = serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClient); + if (status != NV_OK) + return status; + + status = deviceGetByHandle(pClient, hDevice, &pDevice); + if (status != NV_OK) + { + status = subdeviceGetByHandle(pClient, hDevice, &pSubdevice); + if (status != NV_OK) + goto done; + + status = rmGpuGroupLockAcquire(pSubdevice->subDeviceInst, + GPU_LOCK_GRP_SUBDEVICE, + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU, + &gpuMask); + if (status != NV_OK) + goto done; + + GPU_RES_SET_THREAD_BC_STATE(pSubdevice); + } + else + { + status = rmGpuGroupLockAcquire(pDevice->deviceInst, + GPU_LOCK_GRP_DEVICE, + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU, + &gpuMask); + if (status != NV_OK) + goto done; + + GPU_RES_SET_THREAD_BC_STATE(pDevice); + } + + status = clientGetResourceRef(pClient, hMappable, &pMappableRef); + if (status != NV_OK) + goto done; + + if ((objDynamicCastById(pMappableRef->pResource, classId(Memory)) == NULL) && + (objDynamicCastById(pMappableRef->pResource, classId(KernelChannel)) == NULL)) + { + status = NV_ERR_INVALID_OBJECT_HANDLE; + goto done; + } + + status = refFindCpuMappingWithFilter(pMappableRef, + NV_PTR_TO_NvP64(pOldCpuAddress), + serverutilMappingFilterCurrentUserProc, + &pCpuMapping); + if (status != NV_OK) + goto done; + + pCpuMapping->pLinearAddress = NV_PTR_TO_NvP64(pNewCpuAddress); + +done: + + if (gpuMask != 0) + rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); + + serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClient); + return status; +} + +static NV_STATUS RmPerformVersionCheck( + void *pData, + NvU32 dataSize +) +{ + nv_ioctl_rm_api_version_t *pParams; + char clientCh, rmCh; + const char *rmStr = NV_VERSION_STRING; + NvBool relaxed = NV_FALSE; + NvU32 i; + + if (dataSize != sizeof(nv_ioctl_rm_api_version_t)) + return NV_ERR_INVALID_ARGUMENT; + + pParams = pData; + + // + // write the reply value, so that the client knows we recognized + // the request + // + pParams->reply = NV_RM_API_VERSION_REPLY_RECOGNIZED; + + // + // the client requested to override the version check; just return + // success. + // + if (pParams->cmd == NV_RM_API_VERSION_CMD_OVERRIDE) + { + return NV_OK; + } + + // + // the client requested relaxed version checking; we will only + // compare the strings until the first decimal point. + // + if (pParams->cmd == NV_RM_API_VERSION_CMD_RELAXED) + { + relaxed = NV_TRUE; + } + + // + // rmStr (i.e., NV_VERSION_STRING) must be null-terminated and fit within + // NV_RM_API_VERSION_STRING_LENGTH, so that: + // + // (1) If the versions don't match, we can return rmStr in + // pParams->versionString. + // (2) The below loop is guaranteed to not overrun rmStr. + // + if ((os_string_length(rmStr) + 1) > NV_RM_API_VERSION_STRING_LENGTH) + { + return NV_ERR_BUFFER_TOO_SMALL; + } + + for (i = 0; i < NV_RM_API_VERSION_STRING_LENGTH; i++) + { + clientCh = pParams->versionString[i]; + rmCh = rmStr[i]; + + // + // fail if the current character is not the same + // + if (clientCh != rmCh) + { + break; + } + + // + // if relaxed matching was requested, succeed when we find the + // first decimal point + // + if ((relaxed) && (clientCh == '.')) + { + return NV_OK; + } + + // + // we found the end of the strings: succeed + // + if (clientCh == '\0') + { + return NV_OK; + } + } + + // + // the version strings did not match: print an error message and + // copy the RM's version string into pParams->versionString, so + // that the client can report the mismatch; explicitly NULL + // terminate the client's string, since we cannot trust it + // + pParams->versionString[NV_RM_API_VERSION_STRING_LENGTH - 1] = '\0'; + + nv_printf(NV_DBG_ERRORS, + "NVRM: API mismatch: the client has the version %s, but\n" + "NVRM: this kernel module has the version %s. Please\n" + "NVRM: make sure that this kernel module and all NVIDIA driver\n" + "NVRM: components have the same version.\n", + pParams->versionString, NV_VERSION_STRING); + + os_string_copy(pParams->versionString, rmStr); + + return NV_ERR_GENERIC; +} + +// +// Check if the NVPCF _DSM functions are implemented under +// NVPCF scope or GPU device scope. +// As part of RM initialisation this function checks the +// support of NVPCF _DSM function implementation under +// NVPCF scope, in case that fails, clear the cached DSM +// support status and retry the NVPCF _DSM function under +// GPU scope. +// +static void RmCheckNvpcfDsmScope( + OBJGPU *pGpu +) +{ + NvU32 supportedFuncs; + NvU16 dsmDataSize = sizeof(supportedFuncs); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + ACPI_DSM_FUNCTION acpiDsmFunction = ACPI_DSM_FUNCTION_NVPCF_2X; + NvU32 acpiDsmSubFunction = NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_SUPPORTED; + + nv->nvpcf_dsm_in_gpu_scope = NV_FALSE; + + if ((osCallACPI_DSM(pGpu, acpiDsmFunction, acpiDsmSubFunction, + &supportedFuncs, &dsmDataSize) != NV_OK) || + (FLD_TEST_DRF(PCF0100, _CTRL_CONFIG_DSM, + _FUNC_GET_SUPPORTED_IS_SUPPORTED, _NO, supportedFuncs)) || + (dsmDataSize != sizeof(supportedFuncs))) + { + nv->nvpcf_dsm_in_gpu_scope = NV_TRUE; + + // clear cached DSM function status + uncacheDsmFuncStatus(pGpu, acpiDsmFunction, acpiDsmSubFunction); + } +} + +NV_STATUS RmPowerSourceChangeEvent( + nv_state_t *pNv, + NvU32 event_val +) +{ + NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS params = {0}; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + + params.powerStateInfo.powerState = event_val ? NV2080_CTRL_PERF_POWER_SOURCE_BATTERY : + NV2080_CTRL_PERF_POWER_SOURCE_AC; + + return pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_PERF_SET_POWERSTATE, + ¶ms, sizeof(params)); +} + +/*! + * @brief Deal with D-notifier events to apply a performance + * level based on the requested auxiliary power-state. + * Read confluence page "D-Notifiers on Linux" for more details. + * + * @param[in] pGpu OBJGPU pointer. + * @param[in] event_type NvU32 Event type. + */ +static void RmHandleDNotifierEvent( + nv_state_t *pNv, + NvU32 event_type +) +{ + NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + + switch (event_type) + { + case ACPI_NOTIFY_POWER_LEVEL_D1: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P0; + break; + case ACPI_NOTIFY_POWER_LEVEL_D2: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P1; + break; + case ACPI_NOTIFY_POWER_LEVEL_D3: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P2; + break; + case ACPI_NOTIFY_POWER_LEVEL_D4: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P3; + break; + case ACPI_NOTIFY_POWER_LEVEL_D5: + params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P4; + break; + default: + return; + } + + pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_ACPI); + if (pRmApi == NULL) + { + return; + } + + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_PERF_SET_AUX_POWER_STATE, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(pNv, &threadState); + + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: Failed to handle ACPI D-Notifier event, status=0x%x\n", + __FUNCTION__, rmStatus); + } +} + +static NV_STATUS +RmDmabufVerifyMemHandle( + OBJGPU *pGpu, + NvHandle hSrcClient, + NvHandle hMemory, + NvU64 offset, + NvU64 size, + void *pGpuInstanceInfo +) +{ + NV_STATUS status; + RsClient *pClient = NULL; + RsResourceRef *pSrcMemoryRef = NULL; + Memory *pSrcMemory = NULL; + MEMORY_DESCRIPTOR *pMemDesc = NULL; + + NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hSrcClient, &pClient)); + + status = clientGetResourceRef(pClient, hMemory, &pSrcMemoryRef); + if (status != NV_OK) + { + return status; + } + + pSrcMemory = dynamicCast(pSrcMemoryRef->pResource, Memory); + if (pSrcMemory == NULL) + { + return NV_ERR_INVALID_OBJECT; + } + + pMemDesc = pSrcMemory->pMemDesc; + + if (pGpuInstanceInfo != NULL) + { + KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance; + pKernelMIGGpuInstance = (KERNEL_MIG_GPU_INSTANCE *) pGpuInstanceInfo; + + if ((pKernelMIGGpuInstance->pMemoryPartitionHeap != pSrcMemory->pHeap)) + { + return NV_ERR_INVALID_OBJECT_PARENT; + } + } + + // Check if hMemory belongs to the same pGpu + if ((pMemDesc->pGpu != pGpu) && + (pSrcMemory->pGpu != pGpu)) + { + return NV_ERR_INVALID_OBJECT_PARENT; + } + + // Offset and size must be aligned to OS page-size + if (!NV_IS_ALIGNED64(offset, os_page_size) || + !NV_IS_ALIGNED64(size, os_page_size)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // Only supported for vidmem handles + if (memdescGetAddressSpace(pMemDesc) != ADDR_FBMEM) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if ((size == 0) || + (size > memdescGetSize(pMemDesc)) || + (offset > (memdescGetSize(pMemDesc) - size))) + { + return NV_ERR_INVALID_ARGUMENT; + } + + return NV_OK; +} + +static NV_STATUS +RmDmabufGetClientAndDevice( + OBJGPU *pGpu, + NvHandle hClient, + NvHandle *phClient, + NvHandle *phDevice, + NvHandle *phSubdevice, + void **ppGpuInstanceInfo +) +{ + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + + if (IS_MIG_ENABLED(pGpu)) + { + NV_STATUS status; + MIG_INSTANCE_REF ref; + KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); + + status = kmigmgrGetInstanceRefFromClient(pGpu, pKernelMIGManager, + hClient, &ref); + if (status != NV_OK) + { + return status; + } + + status = kmigmgrIncRefCount(ref.pKernelMIGGpuInstance->pShare); + if (status != NV_OK) + { + return status; + } + + *phClient = ref.pKernelMIGGpuInstance->instanceHandles.hClient; + *phDevice = ref.pKernelMIGGpuInstance->instanceHandles.hDevice; + *phSubdevice = ref.pKernelMIGGpuInstance->instanceHandles.hSubdevice; + *ppGpuInstanceInfo = (void *) ref.pKernelMIGGpuInstance; + + return NV_OK; + } + + *phClient = pMemoryManager->hClient; + *phDevice = pMemoryManager->hDevice; + *phSubdevice = pMemoryManager->hSubdevice; + *ppGpuInstanceInfo = NULL; + + return NV_OK; +} + +static void +RmDmabufPutClientAndDevice( + OBJGPU *pGpu, + NvHandle hClient, + NvHandle hDevice, + NvHandle hSubdevice, + void *pGpuInstanceInfo +) +{ + KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance; + + if (pGpuInstanceInfo == NULL) + { + return; + } + + pKernelMIGGpuInstance = (KERNEL_MIG_GPU_INSTANCE *) pGpuInstanceInfo; + + NV_ASSERT_OK(kmigmgrDecRefCount(pKernelMIGGpuInstance->pShare)); +} + +/* + * --------------------------------------------------------------------------- + * + * The routines below are part of the interface between the kernel interface + * layer and the kernel-agnostic portions of the resource manager. + * + * --------------------------------------------------------------------------- + */ + +NvBool NV_API_CALL rm_init_private_state( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NvBool retval; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + retval = RmInitPrivateState(pNv); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return retval; +} + +void NV_API_CALL rm_free_private_state( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + RmFreePrivateState(pNv); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NvBool NV_API_CALL rm_init_adapter( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + NvBool retval = NV_FALSE; + void *fp; + NvBool bEnabled; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_DEVICE_INIT); + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK) + { + if (!((gpumgrQueryGpuDrainState(pNv->gpu_id, &bEnabled, NULL) == NV_OK) + && bEnabled)) + { + if (pNv->flags & NV_FLAG_PERSISTENT_SW_STATE) + { + retval = RmPartiallyInitAdapter(pNv); + } + else + { + retval = RmInitAdapter(pNv); + } + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return retval; +} + +void NV_API_CALL rm_disable_adapter( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_ASSERT_OK(os_flush_work_queue(pNv->queue)); + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + if (pNv->flags & NV_FLAG_PERSISTENT_SW_STATE) + { + RmPartiallyDisableAdapter(pNv); + } + else + { + RmDisableAdapter(pNv); + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_ASSERT_OK(os_flush_work_queue(pNv->queue)); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_shutdown_adapter( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + RmShutdownAdapter(pNv); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_exclude_adapter( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NV_STATUS rmStatus; + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + rmStatus = RmExcludeAdapter(pNv); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_acquire_api_lock( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_release_api_lock( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // UNLOCK: release API lock + rmapiLockRelease(); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_acquire_gpu_lock( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire GPU lock + rmStatus = rmDeviceGpuLocksAcquire(NV_GET_NV_PRIV_PGPU(nv), + GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_OSAPI); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_release_gpu_lock( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // UNLOCK: release GPU lock + rmDeviceGpuLocksRelease(NV_GET_NV_PRIV_PGPU(nv), GPUS_LOCK_FLAGS_NONE, NULL); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_acquire_all_gpus_lock( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire all GPUs lock + rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_release_all_gpus_lock( + nvidia_stack_t *sp +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // UNLOCK: release all GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +/*! + * @brief Handle ACPI_NOTIFY_GPS_STATUS_CHANGE event. + * + * This function is called for GPS when SBIOS trigger + * gps STATUS_CHANGE event, which calls rm control call + * NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL to init the GPS + * data from SBIOS. + */ +static void RmHandleGPSStatusChange +( + nv_state_t *pNv +) +{ +} + +/*! + * @brief Function to handle device specific ACPI events. + * + * @param[in] sp nvidia_stack_t pointer. + * @param[in] nv nv_state_t pointer. + * @param[in] event_type NvU32 Event type. + */ +void NV_API_CALL rm_acpi_notify( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 event_type +) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + + switch (event_type) + { + case ACPI_VIDEO_NOTIFY_PROBE: + { + THREAD_STATE_NODE threadState; + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + RmHandleDisplayChange(sp, nv); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + break; + } + + case ACPI_NOTIFY_GPS_STATUS_CHANGE: + RmHandleGPSStatusChange(nv); + break; + + case ACPI_NOTIFY_POWER_LEVEL_D1: /* fallthrough */ + case ACPI_NOTIFY_POWER_LEVEL_D2: /* fallthrough */ + case ACPI_NOTIFY_POWER_LEVEL_D3: /* fallthrough */ + case ACPI_NOTIFY_POWER_LEVEL_D4: /* fallthrough */ + case ACPI_NOTIFY_POWER_LEVEL_D5: + RmHandleDNotifierEvent(nv, event_type); + break; + + default: + NV_PRINTF(LEVEL_INFO, "No support for 0x%x event\n", event_type); + NV_ASSERT(0); + break; + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +static void nv_align_mmap_offset_length( + nv_usermap_access_params_t *nvuap) +{ + NvU64 page_size = os_page_size; + NvU64 end = nvuap->size + (nvuap->addr & (page_size - 1)); + + nvuap->mmap_start = NV_ALIGN_DOWN(nvuap->addr, page_size); + nvuap->mmap_size = NV_ALIGN_UP(end, page_size); + nvuap->offset = NV_ALIGN_DOWN(nvuap->offset, page_size); +} + +static inline NV_STATUS RmGetArrayMinMax( + NvU64 *array, + NvU64 count, + NvU64 *min, + NvU64 *max +) +{ + NvU64 i; + + if (array == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + *min = array[0]; + *max = array[0]; + + if (count == 1) + return NV_OK; + + for (i = 1; i < count; i++) + { + if (array[i] > *max) + *max = array[i]; + + if (array[i] < *min) + *min = array[i]; + } + + return NV_OK; +} + +static NV_STATUS RmSetUserMapAccessRange( + nv_usermap_access_params_t *nvuap +) +{ + NV_STATUS status = NV_OK; + + if (nvuap->contig) + { + nvuap->access_start = nvuap->mmap_start; + nvuap->access_size = nvuap->mmap_size; + } + else + { + NvU64 highest_address_mapped; + NvU64 lowest_address_mapped; + + status = RmGetArrayMinMax(nvuap->page_array, nvuap->num_pages, + &lowest_address_mapped, + &highest_address_mapped); + if (status != NV_OK) + { + return status; + } + + nvuap->access_start = lowest_address_mapped; + nvuap->access_size = (highest_address_mapped + os_page_size) - lowest_address_mapped; + } + + return status; +} + +static NV_STATUS RmGetAllocPrivate(NvU32, NvU32, NvU64, NvU64, NvU32 *, void **, + NvU64 *); +static NV_STATUS RmValidateMmapRequest(nv_state_t *, NvU64, NvU64, NvU32 *); + +static NV_STATUS RmGetMmapPteArray( + KernelMemorySystem *pKernelMemorySystem, + NvHandle hClient, + NvHandle hMemory, + nv_usermap_access_params_t *nvuap +) +{ + NV_STATUS status = NV_OK; + NvU64 pages = 0; + Memory *pMemory = NULL; + PMEMORY_DESCRIPTOR pMemDesc = NULL; + RsResourceRef *pResourceRef; + NvU64 i; + NvU64 *pteArray; + NvU64 index; + + if (nvuap == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // + // If we're mapping a memory handle, we can get the data from the + // descriptor (required if the allocation being mapped is discontiguous). + // + if (serverutilGetResourceRefWithType(hClient, hMemory, classId(Memory), + &pResourceRef) == NV_OK) + { + pMemory = dynamicCast(pResourceRef->pResource, Memory); + pMemDesc = pMemory->pMemDesc; + nvuap->contig = memdescGetContiguity(pMemDesc, AT_CPU); + } + + // + // In the discontiguous case, the page array needs to be large enough to hold + // the 4K-page-sized addresses that we will then deflate to OS page addresses. + // For the contiguous case, we can create the OS-page-sized addresses + // directly in the array. + // + if (nvuap->contig) + { + pages = nvuap->mmap_size / os_page_size; + } + else + { + pages = nvuap->mmap_size / NV_RM_PAGE_SIZE; + } + + NV_ASSERT_OR_RETURN(pages != 0, NV_ERR_INVALID_ARGUMENT); + + status = os_alloc_mem((void **)&nvuap->page_array, (pages * sizeof(NvU64))); + if (status != NV_OK) + { + return status; + } + + if (!nvuap->contig) + { + pteArray = memdescGetPteArray(pMemDesc, AT_CPU); + index = nvuap->offset / NV_RM_PAGE_SIZE; + + // + // We're guaranteed to have a MEMORY_DESCRIPTOR in the discontiguous + // case. Copy over the addresses now. + // + portMemCopy((void *)nvuap->page_array, + pages * sizeof(NvU64), (void *)&pteArray[index], + pages * sizeof(NvU64)); + + if (NV_RM_PAGE_SIZE < os_page_size) + { + RmDeflateRmToOsPageArray(nvuap->page_array, pages); + pages = NV_RM_PAGES_TO_OS_PAGES(pages); + } + + // + // Convert the GPU physical addresses to system physical addresses, + // if applicable. + // + for (i = 0; i < pages; i++) + { + nvuap->page_array[i] += pKernelMemorySystem->coherentCpuFbBase; + } + } + else + { + // Offset is accounted in mmap_start. + for (nvuap->page_array[0] = nvuap->mmap_start, i = 1; + i < pages; i++) + { + nvuap->page_array[i] = nvuap->page_array[i-1] + os_page_size; + } + } + + nvuap->num_pages = pages; + + return status; +} + +/* Must be called with the API lock and the GPU locks */ +static NV_STATUS RmCreateMmapContextLocked( + NvHandle hClient, + NvHandle hDevice, + NvHandle hMemory, + NvP64 address, + NvU64 size, + NvU64 offset, + NvU32 cachingType, + NvU32 fd +) +{ + NV_STATUS status = NV_OK; + void *pAllocPriv = NULL; + OBJGPU *pGpu = NULL; + KernelMemorySystem *pKernelMemorySystem = NULL; + NvBool bCoherentAtsCpuOffset = NV_FALSE; + nv_state_t *pNv = NULL; + NvU64 addr = (NvU64)address; + NvU32 prot = 0; + NvU64 pageIndex = 0; + nv_usermap_access_params_t *nvuap = NULL; + NvBool bClientMap = (hClient == hDevice); + + if (!bClientMap) + { + if (CliSetGpuContext(hClient, hDevice, &pGpu, NULL) != NV_OK) + { + NvU32 tmp; + if (CliSetSubDeviceContext(hClient, hDevice, &tmp, &pGpu) != NV_OK) + { + // + // If this mapping isn't for a GPU then we don't need to + // create a context for it. + // + return status; + } + } + } + + status = os_alloc_mem((void**)&nvuap, sizeof(nv_usermap_access_params_t)); + if (status != NV_OK) + { + return status; + } + + portMemSet(nvuap, 0, sizeof(nv_usermap_access_params_t)); + nvuap->addr = addr; + nvuap->size = size; + nvuap->offset = offset; + nvuap->caching = cachingType; + + // + // Assume the allocation is contiguous until RmGetMmapPteArray + // determines otherwise. + // + nvuap->contig = NV_TRUE; + nv_align_mmap_offset_length(nvuap); + + if (pGpu != NULL) + { + pNv = NV_GET_NV_STATE(pGpu); + pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); + bCoherentAtsCpuOffset = IS_COHERENT_CPU_ATS_OFFSET(pKernelMemorySystem, addr, size); + } + + // + // If no device is given, or the address isn't in the given device's BARs, + // validate this as a system memory mapping and associate it with the + // control device. + // + if ((pNv == NULL) || + (!IS_REG_OFFSET(pNv, addr, size) && + !IS_FB_OFFSET(pNv, addr, size) && + !bCoherentAtsCpuOffset && + !IS_IMEM_OFFSET(pNv, addr, size))) + { + pNv = nv_get_ctl_state(); + + // + // Validate the mapping request by looking up the underlying sysmem + // allocation. + // + status = RmGetAllocPrivate(hClient, hMemory, addr, size, &prot, &pAllocPriv, + &pageIndex); + + if (status != NV_OK) + { + goto done; + } + } + else + { + // + // Validate the mapping request for ATS and get pteArray + // + if (bCoherentAtsCpuOffset) + { + status = RmGetMmapPteArray(pKernelMemorySystem, hClient, hMemory, nvuap); + if (status != NV_OK) + { + goto done; + } + } + + if (RmSetUserMapAccessRange(nvuap) != NV_OK) + { + goto done; + } + + status = nv_get_usermap_access_params(pNv, nvuap); + if (status != NV_OK) + { + goto done; + } + + // Validate the mapping request for BAR's. + status = RmValidateMmapRequest(pNv, nvuap->access_start, + nvuap->access_size, &prot); + if (status != NV_OK) + { + goto done; + } + } + + status = nv_add_mapping_context_to_file(pNv, nvuap, prot, pAllocPriv, + pageIndex, fd); + +done: + os_free_mem(nvuap); + return status; +} + +// TODO: Bug 1802250: [uvm8] Use an alt stack in all functions in unix/src/osapi.c +NV_STATUS rm_create_mmap_context( + NvHandle hClient, + NvHandle hDevice, + NvHandle hMemory, + NvP64 address, + NvU64 size, + NvU64 offset, + NvU32 cachingType, + NvU32 fd +) +{ + NV_STATUS rmStatus = NV_OK; + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) == NV_OK) + { + RmClient *pClient; + + if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClient)) + return NV_ERR_INVALID_CLIENT; + + if (pClient->ProcID != osGetCurrentProcess()) + { + rmStatus = NV_ERR_INVALID_CLIENT; + } + // LOCK: acquire GPUs lock + else if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) == NV_OK) + { + rmStatus = RmCreateMmapContextLocked(hClient, hDevice, hMemory, + address, size, offset, cachingType, fd); + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + + serverutilReleaseClient(LOCK_ACCESS_READ, pClient); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return rmStatus; +} + +static NV_STATUS RmGetAllocPrivate( + NvU32 hClient, + NvU32 hMemory, + NvU64 offset, + NvU64 length, + NvU32 *pProtection, + void **ppPrivate, + NvU64 *pPageIndex +) +{ + RmClient *pClient; + NV_STATUS rmStatus; + PMEMORY_DESCRIPTOR pMemDesc; + NvU32 pageOffset; + NvU64 pageCount; + NvU64 endingOffset; + RsResourceRef *pResourceRef; + RmResource *pRmResource; + void *pMemData; + NvBool bPeerIoMem; + NvBool bReadOnlyMem; + *pProtection = NV_PROTECT_READ_WRITE; + *ppPrivate = NULL; + + pageOffset = (offset & ~os_page_mask); + offset &= os_page_mask; + + NV_ASSERT_OR_RETURN(rmapiLockIsOwner(), NV_ERR_INVALID_LOCK_STATE); + + if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClient)) + return NV_ERR_INVALID_CLIENT; + + rmStatus = clientGetResourceRef(staticCast(pClient, RsClient), hMemory, &pResourceRef); + if (rmStatus != NV_OK) + goto done; + + pRmResource = dynamicCast(pResourceRef->pResource, RmResource); + if (!pRmResource) + { + rmStatus = NV_ERR_INVALID_OBJECT; + goto done; + } + + rmStatus = rmresGetMemoryMappingDescriptor(pRmResource, &pMemDesc); + if (rmStatus != NV_OK) + goto done; + + bReadOnlyMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY); + bPeerIoMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM); + + if (!(pMemDesc->Allocated || bPeerIoMem)) + { + rmStatus = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + switch (memdescGetAddressSpace(pMemDesc)) + { + case ADDR_SYSMEM: + break; + default: + rmStatus = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + pMemData = memdescGetMemData(pMemDesc); + if (pMemData == NULL) + { + rmStatus = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + rmStatus = os_match_mmap_offset(pMemData, offset, pPageIndex); + if (rmStatus != NV_OK) + goto done; + + if (!portSafeAddU64(pageOffset, length, &endingOffset)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + pageCount = (endingOffset / os_page_size); + + if (!portSafeAddU64(*pPageIndex + ((endingOffset % os_page_size) ? 1 : 0), + pageCount, &pageCount)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + if (pageCount > NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount)) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + goto done; + } + + if (bReadOnlyMem) + *pProtection = NV_PROTECT_READABLE; + + *ppPrivate = pMemData; + +done: + serverutilReleaseClient(LOCK_ACCESS_READ, pClient); + + return rmStatus; +} + +static NV_STATUS RmValidateMmapRequest( + nv_state_t *pNv, + NvU64 offset, + NvU64 length, + NvU32 *pProtection +) +{ + NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS params = { 0 }; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV_STATUS status; + + if (osIsAdministrator()) + { + *pProtection = NV_PROTECT_READ_WRITE; + return NV_OK; + } + + params.addressStart = offset; + params.addressLength = length; + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_GPU_VALIDATE_MEM_MAP_REQUEST, + ¶ms, sizeof(params)); + + if (status == NV_OK) + { + *pProtection = params.protection; + } + + return status; +} + +NV_STATUS rm_get_adapter_status( + nv_state_t *pNv, + NvU32 *pStatus +) +{ + NV_STATUS rmStatus = NV_ERR_OPERATING_SYSTEM; + + // LOCK: acquire API lock + if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI) == NV_OK) + { + rmStatus = RmGetAdapterStatus(pNv, pStatus); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return rmStatus; +} + +NvBool NV_API_CALL rm_init_rm( + nvidia_stack_t *sp +) +{ + NvBool retval; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + retval = RmInitRm(); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return retval; +} + +void NV_API_CALL rm_shutdown_rm( + nvidia_stack_t *sp +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + RmShutdownRm(); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NvBool NV_API_CALL rm_init_event_locks( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + NvBool ret; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pNv->event_spinlock = portSyncSpinlockCreate(portMemAllocatorGetGlobalNonPaged()); + ret = (pNv->event_spinlock != NULL); + + NV_EXIT_RM_RUNTIME(sp,fp); + return ret; +} + +void NV_API_CALL rm_destroy_event_locks( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (pNv && pNv->event_spinlock) + portSyncSpinlockDestroy(pNv->event_spinlock); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_get_vbios_version( + nvidia_stack_t *sp, + nv_state_t *pNv, + char *vbiosString +) +{ + void *fp; + NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS *params; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + const size_t vbiosStringLen = 15; // "xx.xx.xx.xx.xx" + + os_snprintf(vbiosString, vbiosStringLen, "??.??.??.??.??"); + + NV_ENTER_RM_RUNTIME(sp,fp); + + params = portMemAllocNonPaged(sizeof(*params)); + if (params == NULL) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return; + } + + portMemSet(params, 0, sizeof(*params)); + + params->biosInfoList[0].index = NV2080_CTRL_BIOS_INFO_INDEX_REVISION; + params->biosInfoList[1].index = NV2080_CTRL_BIOS_INFO_INDEX_OEM_REVISION; + params->biosInfoListSize = 2; + + pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_VBIOS); + if (pRmApi != NULL) + { + NV_STATUS rmStatus; + + rmStatus = pRmApi->Control(pRmApi, + pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_BIOS_GET_INFO_V2, + params, + sizeof(*params)); + + if (rmStatus == NV_OK) + { + const NvU32 biosRevision = params->biosInfoList[0].data; + const NvU32 biosOEMRevision = params->biosInfoList[1].data; + + os_snprintf(vbiosString, vbiosStringLen, + "%02x.%02x.%02x.%02x.%02x", + (biosRevision & 0xff000000) >> 24, + (biosRevision & 0x00ff0000) >> 16, + (biosRevision & 0x0000ff00) >> 8, + (biosRevision & 0x000000ff) >> 0, + biosOEMRevision); + } + + RmUnixRmApiEpilogue(pNv, &threadState); + } + + portMemFree(params); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_stop_user_channels( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_ERR_INVALID_STATE; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_FIFO); + if (pRmApi != NULL) + { + params.bDisable = NV_TRUE; + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(pNv, &threadState); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_restart_user_channels( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_ERR_INVALID_STATE; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_FIFO); + if (pRmApi != NULL) + { + params.bDisable = NV_FALSE; + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(pNv, &threadState); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +// +// Use this call to check if the chipset is io coherent +// +NvBool NV_API_CALL rm_is_chipset_io_coherent( + nvidia_stack_t *sp) +{ + void *fp; + OBJSYS *pSys; + OBJCL *pCl; + NvBool bIoCoherent = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pSys = SYS_GET_INSTANCE(); + pCl = SYS_GET_CL(pSys); + if (pCl == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "%s: no CL object found, setting io coherent by default\n", + __FUNCTION__); + goto done; + } + + bIoCoherent = pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT); + +done: + NV_EXIT_RM_RUNTIME(sp,fp); + + return bIoCoherent; +} + +NV_STATUS NV_API_CALL rm_ioctl( + nvidia_stack_t *sp, + nv_state_t *pNv, + nv_file_private_t *nvfp, + NvU32 Command, + void *pData, + NvU32 dataSize +) +{ + NV_STATUS rmStatus; + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + rmStatus = RmIoctl(pNv, nvfp, Command, pData, dataSize); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +void NV_API_CALL rm_cleanup_file_private( + nvidia_stack_t *sp, + nv_state_t *pNv, + nv_file_private_t *nvfp +) +{ + THREAD_STATE_NODE threadState; + void *fp; + RM_API *pRmApi = rmapiGetInterface(RMAPI_EXTERNAL); + RM_API_CONTEXT rmApiContext = {0}; + NvU32 i; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + threadStateSetTimeoutOverride(&threadState, 10 * 1000); + + if (rmapiPrologue(pRmApi, &rmApiContext) != NV_OK) + return; + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK) + { + // Unref any object which was exported on this file. + if (nvfp->handles != NULL) + { + for (i = 0; i < nvfp->maxHandles; i++) + { + if (nvfp->handles[i] == 0) + { + continue; + } + + RmFreeObjExportHandle(nvfp->handles[i]); + nvfp->handles[i] = 0; + } + + os_free_mem(nvfp->handles); + nvfp->handles = NULL; + nvfp->maxHandles = 0; + } + + // Free any RM clients associated with this file. + RmFreeUnusedClients(pNv, nvfp); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + rmapiEpilogue(pRmApi, &rmApiContext); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + if (nvfp->ctl_nvfp != NULL) + { + nv_put_file_private(nvfp->ctl_nvfp_priv); + nvfp->ctl_nvfp = NULL; + nvfp->ctl_nvfp_priv = NULL; + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_unbind_lock( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK) + { + RmUnbindLock(pNv); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS rm_alloc_os_event( + NvHandle hClient, + nv_file_private_t *nvfp, + NvU32 fd +) +{ + NV_STATUS RmStatus; + + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_EVENT)) == NV_OK) + { + RmStatus = RmAllocOsEvent(hClient, nvfp, fd); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return RmStatus; +} + +NV_STATUS rm_free_os_event( + NvHandle hClient, + NvU32 fd +) +{ + NV_STATUS RmStatus; + + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_EVENT)) == NV_OK) + { + RmStatus = RmFreeOsEvent(hClient, fd); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return RmStatus; +} + +NV_STATUS rm_get_event_data( + nv_file_private_t *nvfp, + NvP64 pEvent, + NvU32 *MoreEvents +) +{ + NV_STATUS RmStatus; + + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_EVENT)) == NV_OK) + { + RmStatus = RmGetEventData(nvfp, pEvent, MoreEvents, NV_TRUE); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return RmStatus; +} + +NV_STATUS NV_API_CALL rm_read_registry_dword( + nvidia_stack_t *sp, + nv_state_t *nv, + const char *regParmStr, + NvU32 *Data +) +{ + OBJGPU *pGpu = NULL; + NV_STATUS RmStatus; + void *fp; + NvBool isApiLockTaken = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + // + // We can be called from different contexts: + // + // 1) early initialization without device state. + // 2) from outside the RM API (without the lock held) + // + // In context 1)the API lock is not needed and + // in context 2), it needs to be acquired. + // + if (nv != NULL) + { + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) != NV_OK) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return RmStatus; + } + + isApiLockTaken = NV_TRUE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + // Skipping the NULL check as osReadRegistryDword takes care of it. + RmStatus = osReadRegistryDword(pGpu, regParmStr, Data); + + if (isApiLockTaken == NV_TRUE) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return RmStatus; +} + +NV_STATUS NV_API_CALL rm_write_registry_dword( + nvidia_stack_t *sp, + nv_state_t *nv, + const char *regParmStr, + NvU32 Data +) +{ + NV_STATUS RmStatus; + void *fp; + NvBool isApiLockTaken = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (nv != NULL) + { + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return RmStatus; + } + + isApiLockTaken = NV_TRUE; + } + + RmStatus = RmWriteRegistryDword(nv, regParmStr, Data); + + if (isApiLockTaken == NV_TRUE) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return RmStatus; +} + +NV_STATUS NV_API_CALL rm_write_registry_binary( + nvidia_stack_t *sp, + nv_state_t *nv, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + NV_STATUS RmStatus; + void *fp; + NvBool isApiLockTaken = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (nv != NULL) + { + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return RmStatus; + } + + isApiLockTaken = NV_TRUE; + } + + RmStatus = RmWriteRegistryBinary(nv, regParmStr, Data, cbLen); + + if (isApiLockTaken == NV_TRUE) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return RmStatus; +} + +NV_STATUS NV_API_CALL rm_write_registry_string( + nvidia_stack_t *sp, + nv_state_t *nv, + const char *regParmStr, + const char *string, + NvU32 stringLength +) +{ + NV_STATUS rmStatus; + void *fp; + NvBool isApiLockTaken = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + + if (nv != NULL) + { + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; + } + + isApiLockTaken = NV_TRUE; + } + + rmStatus = RmWriteRegistryString(nv, regParmStr, string, (stringLength + 1)); + + if (isApiLockTaken == NV_TRUE) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +static NvBool NV_API_CALL rm_is_space(const char ch) +{ + // + // return true if it is a: + // ' ' : (space - decimal 32.) + // '\t' : (TAB - decimal 9) + // 'LF' : (Line feed, new line - decimal 10) + // 'VT' : (Vertical TAB - decimal 11) + // 'FF' : (Form feed, new page - decimal 12) + // '\r' : (carriage return - decimal 13) + // + return ((ch == ' ') || ((ch >= '\t') && (ch <= '\r'))); +} + +char* NV_API_CALL rm_remove_spaces(const char *in) +{ + unsigned int len = os_string_length(in) + 1; + const char *in_ptr; + char *out, *out_ptr; + + if (os_alloc_mem((void **)&out, len) != NV_OK) + return NULL; + + in_ptr = in; + out_ptr = out; + + while (*in_ptr != '\0') + { + if (!rm_is_space(*in_ptr)) + *out_ptr++ = *in_ptr; + in_ptr++; + } + *out_ptr = '\0'; + + return out; +} + +char* NV_API_CALL rm_string_token(char **strp, const char delim) +{ + char *s, *token; + + if ((strp == NULL) || (*strp == NULL)) + return NULL; + + s = token = *strp; + *strp = NULL; + + for (; *s != '\0'; s++) { + if (*s == delim) { + *s = '\0'; + *strp = ++s; + break; + } + } + + return token; +} + +// Parse string passed in NVRM as module parameter. +void NV_API_CALL rm_parse_option_string(nvidia_stack_t *sp, const char *nvRegistryDwords) +{ + unsigned int i; + nv_parm_t *entry; + char *option_string = NULL; + char *ptr, *token; + char *name, *value; + NvU32 data; + + if (nvRegistryDwords != NULL) + { + if ((option_string = rm_remove_spaces(nvRegistryDwords)) == NULL) + { + return; + } + + ptr = option_string; + + while ((token = rm_string_token(&ptr, ';')) != NULL) + { + if (!(name = rm_string_token(&token, '=')) || !os_string_length(name)) + { + continue; + } + + if (!(value = rm_string_token(&token, '=')) || !os_string_length(value)) + { + continue; + } + + if (rm_string_token(&token, '=') != NULL) + { + continue; + } + + data = os_strtoul(value, NULL, 0); + + for (i = 0; (entry = &nv_parms[i])->name != NULL; i++) + { + if (os_string_compare(entry->name, name) == 0) + break; + } + + if (!entry->name) + rm_write_registry_dword(sp, NULL, name, data); + else + *entry->data = data; + } + + // Free the memory allocated by rm_remove_spaces() + os_free_mem(option_string); + } +} + +NV_STATUS NV_API_CALL rm_run_rc_callback( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + OBJGPU *pGpu; + void *fp; + + /* make sure our timer's not still running when it shouldn't be */ + if (nv == NULL) + return NV_ERR_GENERIC; + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL) + return NV_ERR_GENERIC; + + if (nv->rc_timer_enabled == 0) + return NV_ERR_GENERIC; + + if (!FULL_GPU_SANITY_CHECK(pGpu)) + { + return NV_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + osRun1HzCallbacksNow(pGpu); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return NV_OK; +} + +static NV_STATUS RmRunNanoTimerCallback( + OBJGPU *pGpu, + void *pTmrEvent +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + POBJTMR pTmr = GPU_GET_TIMER(pGpu); + THREAD_STATE_NODE threadState; + NV_STATUS status = NV_OK; + // LOCK: try to acquire GPUs lock + if ((status = rmGpuLocksAcquire(GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_TMR)) != NV_OK) + { + PTMR_EVENT_PVT pEvent = (PTMR_EVENT_PVT) pTmrEvent; + // We failed to acquire the lock; schedule a timer to try again. + return osStartNanoTimer(pGpu->pOsGpuInfo, pEvent->super.pOSTmrCBdata, 1000); + } + + if ((status = osCondAcquireRmSema(pSys->pSema)) != NV_OK) + { + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + return status; + } + + threadStateInitISRAndDeferredIntHandler(&threadState, pGpu, + THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER); + + // Call timer event service + status = tmrEventServiceOSTimerCallback_HAL(pGpu, pTmr, (PTMR_EVENT)pTmrEvent); + + // Out of conflicting thread + threadStateFreeISRAndDeferredIntHandler(&threadState, + pGpu, THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER); + + osReleaseRmSema(pSys->pSema, NULL); + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, pGpu); + + return status; +} + +NV_STATUS NV_API_CALL rm_run_nano_timer_callback +( + nvidia_stack_t *sp, + nv_state_t *nv, + void *pTmrEvent +) +{ + NV_STATUS status; + OBJGPU *pGpu = NULL; + void *fp; + + if (nv == NULL) + return NV_ERR_GENERIC; + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL) + return NV_ERR_GENERIC; + + if (!FULL_GPU_SANITY_CHECK(pGpu)) + { + return NV_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + status = RmRunNanoTimerCallback(pGpu, pTmrEvent); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +void NV_API_CALL rm_execute_work_item( + nvidia_stack_t *sp, + void *pNvWorkItem +) +{ + void *fp; + THREAD_STATE_NODE threadState; + + NV_ENTER_RM_RUNTIME(sp, fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + RmExecuteWorkItem(pNvWorkItem); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp, fp); +} + +const char* NV_API_CALL rm_get_device_name( + NvU16 device, + NvU16 subsystem_vendor, + NvU16 subsystem_device +) +{ + unsigned int i; + const char *tmpName = NULL; + + for (i = 0; i < NV_ARRAY_ELEMENTS(sChipsReleased); i++) + { + // if the device ID doesn't match, go to the next entry + if (device != sChipsReleased[i].devID) + { + continue; + } + + // if the entry has 0 for the subsystem IDs, then the device + // ID match is sufficient, but continue scanning through + // sChipsReleased[] in case there is a subsystem ID match later + // in the table + if (sChipsReleased[i].subSystemVendorID == 0 && + sChipsReleased[i].subSystemID == 0) + { + tmpName = sChipsReleased[i].name; + continue; + } + + if (subsystem_vendor == sChipsReleased[i].subSystemVendorID && + subsystem_device == sChipsReleased[i].subSystemID) + { + tmpName = sChipsReleased[i].name; + break; + } + } + + return (tmpName != NULL) ? tmpName : "Unknown"; +} + +NV_STATUS rm_access_registry( + NvHandle hClient, + NvHandle hObject, + NvU32 AccessType, + NvP64 clientDevNodeAddress, + NvU32 DevNodeLength, + NvP64 clientParmStrAddress, + NvU32 ParmStrLength, + NvP64 clientBinaryDataAddress, + NvU32 *pBinaryDataLength, + NvU32 *Data, + NvU32 *Entry +) +{ + NV_STATUS RmStatus; + NvBool bReadOnly = (AccessType == NVOS38_ACCESS_TYPE_READ_DWORD) || + (AccessType == NVOS38_ACCESS_TYPE_READ_BINARY); + + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(bReadOnly ? RMAPI_LOCK_FLAGS_READ : RMAPI_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_OSAPI)) == NV_OK) + { + RmStatus = RmAccessRegistry(hClient, + hObject, + AccessType, + clientDevNodeAddress, + DevNodeLength, + clientParmStrAddress, + ParmStrLength, + clientBinaryDataAddress, + pBinaryDataLength, + Data, + Entry); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return RmStatus; +} + +NV_STATUS rm_update_device_mapping_info( + NvHandle hClient, + NvHandle hDevice, + NvHandle hMemory, + void *pOldCpuAddress, + void *pNewCpuAddress +) +{ + NV_STATUS RmStatus; + + // LOCK: acquire API lock + if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU)) == NV_OK) + { + RmStatus = RmUpdateDeviceMappingInfo(hClient, + hDevice, + hMemory, + pOldCpuAddress, + pNewCpuAddress); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + return RmStatus; +} + +static void rm_is_device_rm_firmware_capable( + nv_state_t *pNv, + NvU32 pmcBoot42, + NvBool *pbIsFirmwareCapable, + NvBool *pbEnableByDefault +) +{ + NvBool bIsFirmwareCapable = NV_FALSE; + NvBool bEnableByDefault = NV_FALSE; + + if (NV_IS_SOC_DISPLAY_DEVICE(pNv)) + { + bIsFirmwareCapable = NV_TRUE; + } + else + { + bIsFirmwareCapable = gpumgrIsDeviceRmFirmwareCapable(pNv->pci_info.device_id, + pmcBoot42, + &bEnableByDefault); + } + + if (pbIsFirmwareCapable != NULL) + { + *pbIsFirmwareCapable = bIsFirmwareCapable; + } + if (pbEnableByDefault != NULL) + { + *pbEnableByDefault = bEnableByDefault; + } +} + +static NvBool NV_API_CALL rm_is_legacy_device( + NvU16 device_id, + NvU16 subsystem_vendor, + NvU16 subsystem_device, + NvBool print_warning +) +{ + return NV_FALSE; +} + +static NvBool NV_API_CALL rm_is_legacy_arch( + NvU32 pmc_boot_0, + NvU32 pmc_boot_42 +) +{ + NvBool legacy = NV_FALSE; + + return legacy; +} + +NV_STATUS NV_API_CALL rm_is_supported_device( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + OBJSYS *pSys; + POBJHALMGR pHalMgr; + GPUHWREG *reg_mapping; + NvU32 myHalPublicID; + void *fp; + NvU32 pmc_boot_0; + NvU32 pmc_boot_42; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pSys = SYS_GET_INSTANCE(); + pHalMgr = SYS_GET_HALMGR(pSys); + + reg_mapping = osMapKernelSpace(pNv->regs->cpu_address, + os_page_size, + NV_MEMORY_UNCACHED, + NV_PROTECT_READABLE); + + if (reg_mapping == NULL) + { + nv_printf(NV_DBG_ERRORS, "NVRM: failed to map registers!\n"); + rmStatus = NV_ERR_OPERATING_SYSTEM; + goto threadfree; + } + NvU32 pmc_boot_1 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_1); + pmc_boot_0 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_0); + pmc_boot_42 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_42); + + osUnmapKernelSpace(reg_mapping, os_page_size); + + if ((pmc_boot_0 == 0xFFFFFFFF) && (pmc_boot_42 == 0xFFFFFFFF)) + { + nv_printf(NV_DBG_ERRORS, + "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x\n" + "NVRM: (PCI ID: %04x:%04x) installed in this system has\n" + "NVRM: fallen off the bus and is not responding to commands.\n", + pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot, + pNv->pci_info.function, pNv->pci_info.vendor_id, + pNv->pci_info.device_id); + rmStatus = NV_ERR_GPU_IS_LOST; + goto threadfree; + } + + /* + * For legacy architectures, rm_is_legacy_arch() prints "legacy" message. + * We do not want to print "unsupported" message for legacy architectures + * to avoid confusion. Also, the probe should not continue for legacy + * architectures. Hence, we set rmStatus to NV_ERR_NOT_SUPPORTED and + * goto threadfree. + */ + if (rm_is_legacy_arch(pmc_boot_0, pmc_boot_42)) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + goto threadfree; + } + + rmStatus = halmgrGetHalForGpu(pHalMgr, pmc_boot_0, pmc_boot_42, &myHalPublicID); + + if (rmStatus != NV_OK) + { + NvBool bIsFirmwareCapable; + + rm_is_device_rm_firmware_capable(pNv, + pmc_boot_42, + &bIsFirmwareCapable, + NULL); + if (!bIsFirmwareCapable) + { + nv_printf(NV_DBG_ERRORS, + "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n" + "NVRM: installed in this system is not supported by open\n" + "NVRM: nvidia.ko because it does not include the required GPU\n" + "NVRM: System Processor (GSP).\n" + "NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n" + "NVRM: Firmware' sections in the driver README, available on\n" + "NVRM: the Linux graphics driver download page at\n" + "NVRM: www.nvidia.com.\n", + pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot, + pNv->pci_info.function, pNv->pci_info.vendor_id, + pNv->pci_info.device_id, NV_VERSION_STRING); + goto threadfree; + } + goto print_unsupported; + } + + rmStatus = rm_is_vgpu_supported_device(pNv, pmc_boot_1); + + if (rmStatus != NV_OK) + goto print_unsupported; + goto threadfree; + +print_unsupported: + nv_printf(NV_DBG_ERRORS, + "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n" + "NVRM: installed in this system is not supported by the\n" + "NVRM: NVIDIA %s driver release.\n" + "NVRM: Please see 'Appendix A - Supported NVIDIA GPU Products'\n" + "NVRM: in this release's README, available on the operating system\n" + "NVRM: specific graphics driver download page at www.nvidia.com.\n", + pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot, + pNv->pci_info.function, pNv->pci_info.vendor_id, + pNv->pci_info.device_id, NV_VERSION_STRING); + +threadfree: + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NvBool NV_API_CALL rm_is_supported_pci_device( + NvU8 pci_class, + NvU8 pci_subclass, + NvU16 vendor, + NvU16 device, + NvU16 subsystem_vendor, + NvU16 subsystem_device, + NvBool print_legacy_warning +) +{ + const NvU16 nv_pci_vendor_id = 0x10DE; + const NvU16 nv_pci_id_riva_tnt = 0x0020; + const NvU8 nv_pci_class_display = 0x03; + const NvU8 nv_pci_subclass_display_vga = 0x00; + const NvU8 nv_pci_subclass_display_3d = 0x02; + + if (pci_class != nv_pci_class_display) + { + return NV_FALSE; + } + + if ((pci_subclass != nv_pci_subclass_display_vga) && + (pci_subclass != nv_pci_subclass_display_3d)) + { + return NV_FALSE; + } + + if (vendor != nv_pci_vendor_id) + { + return NV_FALSE; + } + + if (device < nv_pci_id_riva_tnt) + { + return NV_FALSE; + } + + if (rm_is_legacy_device( + device, + subsystem_vendor, + subsystem_device, + print_legacy_warning)) + { + return NV_FALSE; + } + + return NV_TRUE; +} + +/* + * Performs the I2C transfers which are related with DP AUX channel + */ +static NV_STATUS RmDpAuxI2CTransfer +( + nv_state_t *pNv, + NvU32 displayId, + NvU8 addr, + NvU32 len, + NvU8 *pData, + NvBool bWrite +) +{ + NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_CTRL_PARAMS *pParams; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV_STATUS status; + + if (len > NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE) + { + NV_PRINTF(LEVEL_ERROR, + "%s: requested I2C transfer length %u is greater than maximum supported length %u\n", + __FUNCTION__, len, NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE); + return NV_ERR_NOT_SUPPORTED; + } + + pParams = portMemAllocNonPaged(sizeof(*pParams)); + if (pParams == NULL) + { + return NV_ERR_NO_MEMORY; + } + + portMemSet(pParams, 0, sizeof(*pParams)); + + pParams->subDeviceInstance = 0; + pParams->displayId = displayId; + pParams->addr = addr; + pParams->size = len; + pParams->bWrite = bWrite; + + if (bWrite) + { + portMemCopy(pParams->data, NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE, + pData, len); + } + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp, + NV0073_CTRL_CMD_DP_AUXCH_I2C_TRANSFER_CTRL, + pParams, sizeof(*pParams)); + + if ((status == NV_OK) && !bWrite) + { + portMemCopy(pData, len, pParams->data, pParams->size); + } + + portMemFree(pParams); + + return status; +} + +/* + * Performs the I2C transfers which are not related with DP AUX channel + */ +static NV_STATUS RmNonDPAuxI2CTransfer +( + nv_state_t *pNv, + NvU8 portId, + nv_i2c_cmd_t type, + NvU8 addr, + NvU8 command, + NvU32 len, + NvU8 *pData +) +{ + NV402C_CTRL_I2C_TRANSACTION_PARAMS *params; + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NV_STATUS rmStatus = NV_OK; + + params = portMemAllocNonPaged(sizeof(*params)); + if (params == NULL) + { + return NV_ERR_NO_MEMORY; + } + + portMemSet(params, 0, sizeof(*params)); + + params->portId = portId; + // precondition our address (our stack requires this) + params->deviceAddress = addr << 1; + + switch (type) + { + case NV_I2C_CMD_WRITE: + params->transData.i2cBlockData.bWrite = NV_TRUE; + /* fall through*/ + + case NV_I2C_CMD_READ: + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_I2C_BLOCK_RW; + params->transData.i2cBlockData.messageLength = len; + params->transData.i2cBlockData.pMessage = pData; + break; + + case NV_I2C_CMD_SMBUS_WRITE: + if (len == 2) + { + params->transData.smbusWordData.bWrite = NV_TRUE; + } + else + { + params->transData.smbusByteData.bWrite = NV_TRUE; + } + /* fall through*/ + + case NV_I2C_CMD_SMBUS_READ: + if (len == 2) + { + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_WORD_RW; + params->transData.smbusWordData.message = pData[0] | ((NvU16)pData[1] << 8); + params->transData.smbusWordData.registerAddress = command; + } + else + { + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BYTE_RW; + params->transData.smbusByteData.message = pData[0]; + params->transData.smbusByteData.registerAddress = command; + } + break; + + case NV_I2C_CMD_SMBUS_BLOCK_WRITE: + params->transData.smbusBlockData.bWrite = NV_TRUE; + /* fall through*/ + + case NV_I2C_CMD_SMBUS_BLOCK_READ: + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BLOCK_RW; + params->transData.smbusBlockData.registerAddress = command; + params->transData.smbusBlockData.messageLength = len; + params->transData.smbusBlockData.pMessage = pData; + break; + + case NV_I2C_CMD_SMBUS_QUICK_WRITE: + params->transData.smbusQuickData.bWrite = NV_TRUE; + /* fall through*/ + + case NV_I2C_CMD_SMBUS_QUICK_READ: + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_QUICK_RW; + break; + + case NV_I2C_CMD_BLOCK_WRITE: + params->transData.i2cBufferData.bWrite = NV_TRUE; + /* fall through */ + + case NV_I2C_CMD_BLOCK_READ: + params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_I2C_BUFFER_RW; + params->transData.i2cBufferData.registerAddress = command; + params->transData.i2cBufferData.messageLength = len; + params->transData.i2cBufferData.pMessage = pData; + break; + + default: + portMemFree(params); + return NV_ERR_INVALID_ARGUMENT; + } + + rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, + pNv->rmapi.hI2C, + NV402C_CTRL_CMD_I2C_TRANSACTION, + params, sizeof(*params)); + + // + // For NV_I2C_CMD_SMBUS_READ, copy the read data to original + // data buffer. + // + if (rmStatus == NV_OK && type == NV_I2C_CMD_SMBUS_READ) + { + if (len == 2) + { + pData[0] = (params->transData.smbusWordData.message & 0xff); + pData[1] = params->transData.smbusWordData.message >> 8; + } + else + { + pData[0] = params->transData.smbusByteData.message; + } + } + + portMemFree(params); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_i2c_transfer( + nvidia_stack_t *sp, + nv_state_t *pNv, + void *pI2cAdapter, + nv_i2c_cmd_t type, + NvU8 addr, + NvU8 command, + NvU32 len, + NvU8 *pData +) +{ + THREAD_STATE_NODE threadState; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + NV_STATUS rmStatus = NV_OK; + OBJGPU *pGpu = NULL; + NvBool unlockApi = NV_FALSE; + NvBool unlockGpu = NV_FALSE; + NvU32 x; + void *fp; + NvU32 numDispId = 0; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (pNvp->flags & NV_INIT_FLAG_PUBLIC_I2C) + { + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK) + goto finish; + + unlockApi = NV_TRUE; + + // LOCK: acquire GPUs lock + if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK) + goto finish; + + unlockGpu = NV_TRUE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + if (!pGpu) + { + rmStatus = NV_ERR_GENERIC; + goto finish; + } + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + if (pNvp->i2c_adapters[x].pOsAdapter == pI2cAdapter) + { + break; + } + } + + if (x == MAX_I2C_ADAPTERS) + { + rmStatus = NV_ERR_GENERIC; + goto finish; + } + + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + NvU32 displayId = pNvp->i2c_adapters[x].displayId[numDispId]; + + if (displayId == INVALID_DISP_ID) + { + continue; + } + + // Handle i2c-over-DpAux adapters separately from regular i2c adapters + if (displayId == 0) + { + rmStatus = RmNonDPAuxI2CTransfer(pNv, pNvp->i2c_adapters[x].port, + type, addr, command, len, pData); + } + else + { + if ((type != NV_I2C_CMD_READ) && (type != NV_I2C_CMD_WRITE)) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + goto semafinish; + } + + rmStatus = RmDpAuxI2CTransfer(pNv, displayId, addr, len, pData, + type == NV_I2C_CMD_WRITE); + } +semafinish: + if (rmStatus == NV_OK) + { + break; + } + } + +finish: + if (unlockGpu) + { + // UNLOCK: release GPU lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + + if (unlockApi) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +static void rm_i2c_add_adapter( + nv_state_t *pNv, + NvU32 port, + NvU32 displayId +) +{ + NvU32 y, free; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + NvU32 numDispId = 0; + + for (y = 0, free = MAX_I2C_ADAPTERS; y < MAX_I2C_ADAPTERS; y++) + { + if (pNvp->i2c_adapters[y].pOsAdapter == NULL) + { + // Only find the first free entry, and ignore the rest + if (free == MAX_I2C_ADAPTERS) + { + free = y; + } + } + else if (pNvp->i2c_adapters[y].port == port) + { + break; + } + } + + if (y < MAX_I2C_ADAPTERS) + { + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + if (pNvp->i2c_adapters[y].displayId[numDispId] == INVALID_DISP_ID) + { + pNvp->i2c_adapters[y].displayId[numDispId] = displayId; + break; + } + else + { + NV_PRINTF(LEVEL_INFO, + "%s: adapter already exists (port=0x%x, displayId=0x%x)\n", + __FUNCTION__, port, + pNvp->i2c_adapters[y].displayId[numDispId]); + } + } + + if (numDispId == MAX_DISP_ID_PER_ADAPTER) + { + NV_PRINTF(LEVEL_ERROR, + "%s: no more free display Id entries in adapter\n", + __FUNCTION__); + } + + return; + } + + if (free == MAX_I2C_ADAPTERS) + { + NV_PRINTF(LEVEL_ERROR, "%s: no more free adapter entries exist\n", + __FUNCTION__); + return; + } + + pNvp->i2c_adapters[free].pOsAdapter = nv_i2c_add_adapter(pNv, port); + pNvp->i2c_adapters[free].port = port; + // When port is added, numDispId will be 0. + pNvp->i2c_adapters[free].displayId[numDispId] = displayId; +} + +void RmI2cAddGpuPorts(nv_state_t * pNv) +{ + NvU32 x = 0; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + NvU32 displayMask; + NV_STATUS status; + NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS systemGetSupportedParams = { 0 }; + + // Make displayId as Invalid. + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + NvU32 numDispId; + + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + pNvp->i2c_adapters[x].displayId[numDispId] = INVALID_DISP_ID; + } + } + + // First, set up the regular i2c adapters - one per i2c port + if (pNv->rmapi.hI2C != 0) + { + NV402C_CTRL_I2C_GET_PORT_INFO_PARAMS i2cPortInfoParams = { 0 }; + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hI2C, + NV402C_CTRL_CMD_I2C_GET_PORT_INFO, + &i2cPortInfoParams, sizeof(i2cPortInfoParams)); + + if (status == NV_OK) + { + for (x = 0; x < NV_ARRAY_ELEMENTS(i2cPortInfoParams.info); x++) + { + // + // Check if this port is implemented and RM I2C framework has + // validated this port. Only limited amount of ports can + // be added to the OS framework. + // + if (FLD_TEST_DRF(402C_CTRL, _I2C_GET_PORT_INFO, _IMPLEMENTED, + _YES, i2cPortInfoParams.info[x]) && + FLD_TEST_DRF(402C_CTRL, _I2C_GET_PORT_INFO, _VALID, + _YES, i2cPortInfoParams.info[x])) + { + rm_i2c_add_adapter(pNv, x, 0); + } + } + } + } + + // + // Now set up the i2c-over-DpAux adapters - one per DP OD + // + // 1. Perform NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS RM control which + // will return the mask for all the display ID's. + // 2. Loop for all the display ID's and do + // NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO RM control call. For each + // output resource, check for the following requirements: + // a. It must be DisplayPort. + // b. It must be internal to the GPU (ie, not on the board) + // c. It must be directly connected to the physical connector (ie, no DP + // 1.2 multistream ODs). + // 3. Perform NV0073_CTRL_CMD_SPECIFIC_GET_I2C_PORTID RM control for + // getting the I2C port data. + // + // With these restrictions, we should only end up with at most one OD + // per DP connector. + // + + if (pNv->rmapi.hDisp == 0) + { + return; + } + + systemGetSupportedParams.subDeviceInstance = 0; + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp, + NV0073_CTRL_CMD_SYSTEM_GET_SUPPORTED, + &systemGetSupportedParams, sizeof(systemGetSupportedParams)); + + if (status != NV_OK) + { + return; + } + + for (displayMask = systemGetSupportedParams.displayMask; + displayMask != 0; + displayMask &= ~LOWESTBIT(displayMask)) + { + NV0073_CTRL_SPECIFIC_OR_GET_INFO_PARAMS orInfoParams = { 0 }; + NvU32 displayId = LOWESTBIT(displayMask); + + orInfoParams.subDeviceInstance = 0; + orInfoParams.displayId = displayId; + + status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp, + NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO, + &orInfoParams, sizeof(orInfoParams)); + + if ((status == NV_OK) && + (orInfoParams.type == NV0073_CTRL_SPECIFIC_OR_TYPE_SOR) && + ((orInfoParams.protocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DP_A) || + (orInfoParams.protocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DP_B)) && + (orInfoParams.location == NV0073_CTRL_SPECIFIC_OR_LOCATION_CHIP) && + (!orInfoParams.bIsDispDynamic)) + { + NV0073_CTRL_SPECIFIC_GET_I2C_PORTID_PARAMS i2cPortIdParams = { 0 }; + + i2cPortIdParams.subDeviceInstance = 0; + i2cPortIdParams.displayId = displayId; + + status = pRmApi->Control(pRmApi, + pNv->rmapi.hClient, + pNv->rmapi.hDisp, + NV0073_CTRL_CMD_SPECIFIC_GET_I2C_PORTID, + &i2cPortIdParams, + sizeof(i2cPortIdParams)); + + if ((status == NV_OK) && + (i2cPortIdParams.ddcPortId != NV0073_CTRL_SPECIFIC_I2C_PORT_NONE)) + { + rm_i2c_add_adapter(pNv, i2cPortIdParams.ddcPortId - 1, displayId); + } + } + } +} + +void NV_API_CALL rm_i2c_remove_adapters( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + // + // Cycle through all adapter entries, and first remove the adapter + // from the list from the kernel, then remove the i2c adapter + // list once that is completed. This should only be used from exit + // module time. Otherwise it could fail to remove some of the + // kernel adapters and subsequent transfer requests would result + // in crashes. + // + NvU32 x = 0; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + NvU32 numDispId; + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + if (pNvp->i2c_adapters[x].pOsAdapter != NULL) + { + nv_i2c_del_adapter(pNv, pNvp->i2c_adapters[x].pOsAdapter); + + pNvp->i2c_adapters[x].pOsAdapter = NULL; + pNvp->i2c_adapters[x].port = 0; + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + pNvp->i2c_adapters[x].displayId[numDispId] = INVALID_DISP_ID; + } + } + } +} + +NvBool NV_API_CALL rm_i2c_is_smbus_capable( + nvidia_stack_t *sp, + nv_state_t *pNv, + void *pI2cAdapter +) +{ + THREAD_STATE_NODE threadState; + nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv); + NV_STATUS rmStatus = NV_OK; + OBJGPU *pGpu = NULL; + NvBool unlock = NV_FALSE; + NvU32 x; + NvBool ret = NV_FALSE; + void *fp; + NvU32 numDispId = 0; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (pNvp->flags & NV_INIT_FLAG_PUBLIC_I2C) + { + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK) + goto semafinish; + + unlock = NV_TRUE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + if (!pGpu) + { + goto semafinish; + } + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + if (pNvp->i2c_adapters[x].pOsAdapter == pI2cAdapter) + { + break; + } + } + + if (x == MAX_I2C_ADAPTERS) + { + goto semafinish; + } + + // we do not support smbus functions on i2c-over-DPAUX + for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++) + { + if (pNvp->i2c_adapters[x].displayId[numDispId] == 0x0) + { + ret = NV_TRUE; + } + } + +semafinish: + if (unlock) + { + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return ret; +} + +NV_STATUS NV_API_CALL rm_perform_version_check( + nvidia_stack_t *sp, + void *pData, + NvU32 dataSize +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + rmStatus = RmPerformVersionCheck(pData, dataSize); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +// +// Handles the Power Source Change event(AC/DC) for Notebooks. +// Notebooks from Maxwell have only one Gpu, so this functions grabs first Gpu +// from GpuMgr and call subdevice RmControl. +// +void NV_API_CALL rm_power_source_change_event( + nvidia_stack_t *sp, + NvU32 event_val +) +{ + THREAD_STATE_NODE threadState; + void *fp; + nv_state_t *nv; + OBJGPU *pGpu = gpumgrGetGpu(0); + NV_STATUS rmStatus = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_EVENT)) == NV_OK) + { + if (pGpu != NULL) + { + nv = NV_GET_NV_STATE(pGpu); + if ((rmStatus = os_ref_dynamic_power(nv, NV_DYNAMIC_PM_FINE)) == + NV_OK) + { + // LOCK: acquire GPU lock + if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_EVENT)) == + NV_OK) + { + rmStatus = RmPowerSourceChangeEvent(nv, event_val); + + // UNLOCK: release GPU lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + os_unref_dynamic_power(nv, NV_DYNAMIC_PM_FINE); + } + // UNLOCK: release API lock + rmapiLockRelease(); + } + } + + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: Failed to handle Power Source change event, status=0x%x\n", + __FUNCTION__, rmStatus); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_p2p_dma_map_pages( + nvidia_stack_t *sp, + nv_dma_device_t *peer, + NvU8 *pGpuUuid, + NvU32 pageSize, + NvU32 pageCount, + NvU64 *pDmaAddresses, + void **ppPriv +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + if (ppPriv == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + *ppPriv = NULL; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + OBJGPU *pGpu = gpumgrGetGpuFromUuid(pGpuUuid, + DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1) | + DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _BINARY)); + if (pGpu == NULL) + { + rmStatus = NV_ERR_INVALID_ARGUMENT; + } + else + { + NvU32 i; + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) + { + NV_ASSERT(pageSize == os_page_size); + + rmStatus = nv_dma_map_alloc(peer, pageCount, pDmaAddresses, + NV_FALSE, ppPriv); + } + else + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + for (i = 0; i < pageCount; i++) + { + // Peer mappings through this API are always via BAR1 + rmStatus = nv_dma_map_peer(peer, nv->dma_dev, 0x1, + pageSize / os_page_size, + &pDmaAddresses[i]); + if ((rmStatus != NV_OK) && (i > 0)) + { + NvU32 j; + for (j = i - 1; j < pageCount; j--) + { + nv_dma_unmap_peer(peer, pageSize / os_page_size, + pDmaAddresses[j]); + } + } + } + } + } + + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_get_gpu_info( + nvidia_stack_t *sp, + NvU64 gpuVirtualAddress, + NvU64 length, + NvU8 **ppGpuUuid, + void **ppGpuInfo +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P); + if (rmStatus == NV_OK) + { + OBJGPU *pGpu; + rmStatus = RmP2PGetGpuByAddress(gpuVirtualAddress, + length, + &pGpu); + if (rmStatus == NV_OK) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + const NvU8 *pGid; + + pGid = RmGetGpuUuidRaw(nv); + if (pGid == NULL) + { + rmStatus = NV_ERR_GPU_UUID_NOT_FOUND; + } + else + { + rmStatus = os_alloc_mem((void **)ppGpuUuid, GPU_UUID_LEN); + if (rmStatus == NV_OK) + { + os_mem_copy(*ppGpuUuid, pGid, GPU_UUID_LEN); + } + } + + *ppGpuInfo = (void *) pGpu; + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent( + nvidia_stack_t *sp, + NvU64 gpuVirtualAddress, + NvU64 length, + void **p2pObject, + NvU64 *pPhysicalAddresses, + NvU32 *pEntries, + void *pPlatformData, + void *pGpuInfo +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + rmStatus = RmP2PGetPagesPersistent(gpuVirtualAddress, + length, + p2pObject, + pPhysicalAddresses, + pEntries, + pPlatformData, + pGpuInfo); + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_get_pages( + nvidia_stack_t *sp, + NvU64 p2pToken, + NvU32 vaSpaceToken, + NvU64 gpuVirtualAddress, + NvU64 length, + NvU64 *pPhysicalAddresses, + NvU32 *pWreqMbH, + NvU32 *pRreqMbH, + NvU32 *pEntries, + NvU8 **ppGpuUuid, + void *pPlatformData +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + OBJGPU *pGpu; + rmStatus = RmP2PGetPagesWithoutCallbackRegistration(p2pToken, + vaSpaceToken, + gpuVirtualAddress, + length, + pPhysicalAddresses, + pWreqMbH, + pRreqMbH, + pEntries, + &pGpu, + pPlatformData); + if (rmStatus == NV_OK) + { + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + const NvU8 *pGid; + + pGid = RmGetGpuUuidRaw(nv); + if (pGid == NULL) + { + NV_ASSERT_OK(RmP2PPutPages(p2pToken, vaSpaceToken, + gpuVirtualAddress, + pPlatformData)); + rmStatus = NV_ERR_GENERIC; + } + else + { + rmStatus = os_alloc_mem((void **)ppGpuUuid, GPU_UUID_LEN); + if (rmStatus == NV_OK) + os_mem_copy(*ppGpuUuid, pGid, GPU_UUID_LEN); + } + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_register_callback( + nvidia_stack_t *sp, + NvU64 p2pToken, + NvU64 gpuVirtualAddress, + NvU64 length, + void *pPlatformData, + void (*pFreeCallback)(void *pData), + void *pData +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + rmStatus = RmP2PRegisterCallback(p2pToken, gpuVirtualAddress, length, + pPlatformData, pFreeCallback, pData); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent( + nvidia_stack_t *sp, + void *p2pObject, + void *pKey +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + rmStatus = RmP2PPutPagesPersistent(p2pObject, pKey); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_p2p_put_pages( + nvidia_stack_t *sp, + NvU64 p2pToken, + NvU32 vaSpaceToken, + NvU64 gpuVirtualAddress, + void *pKey +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK) + { + rmStatus = RmP2PPutPages(p2pToken, + vaSpaceToken, + gpuVirtualAddress, + pKey); + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +char* NV_API_CALL rm_get_gpu_uuid( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + NV_STATUS rmStatus; + const NvU8 *pGid; + char *pGidString; + + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // Allocate space for the ASCII string + rmStatus = os_alloc_mem((void **)&pGidString, GPU_UUID_ASCII_LEN); + if (rmStatus != NV_OK) + { + pGidString = NULL; + goto done; + } + + // Get the raw UUID; note the pGid is cached, so we do not need to free it + pGid = RmGetGpuUuidRaw(nv); + + if (pGid != NULL) + { + // Convert the raw UUID to ASCII + rmStatus = RmGpuUuidRawToString(pGid, pGidString, GPU_UUID_ASCII_LEN); + if (rmStatus != NV_OK) + { + os_free_mem(pGidString); + pGidString = NULL; + } + } + else + { + const char *pTmpString = "GPU-???????\?-???\?-???\?-???\?-????????????"; + + portStringCopy(pGidString, GPU_UUID_ASCII_LEN, pTmpString, + portStringLength(pTmpString) + 1); + } + +done: + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return pGidString; +} + +// +// This function will return the UUID in the binary format +// +const NvU8 * NV_API_CALL rm_get_gpu_uuid_raw( + nvidia_stack_t *sp, + nv_state_t *nv) +{ + THREAD_STATE_NODE threadState; + void *fp; + const NvU8 *pGid; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGid = RmGetGpuUuidRaw(nv); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return pGid; +} + +static void rm_set_firmware_logs( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + NV_STATUS status; + NvU32 data; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + status = RmReadRegistryDword(nv, NV_REG_ENABLE_GPU_FIRMWARE_LOGS, &data); + if (status == NV_OK) + { + if ((data == NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE) +#if defined(DEBUG) || defined(DEVELOP) + || (data == NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG) +#endif + ) + { + nv->enable_firmware_logs = NV_TRUE; + } + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_set_rm_firmware_requested( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + nv->request_firmware = NV_TRUE; + nv->allow_fallback_to_monolithic_rm = NV_FALSE; + + // Check if we want firmware logs + if (nv->request_firmware) + rm_set_firmware_logs(sp, nv); +} + +// +// This function will be called by nv_procfs_read_gpu_info(). +// nv_procfs_read_gpu_info() will not print the 'GPU Firmware:' field at +// all if the 'version' string is empty. +// +// If GSP is enabled (firmware was requested), this function needs to return +// the firmware version or "NA" in case of any errors. +// +// If GSP is not enabled (firmware was not requested), this function needs to +// return the empty string, regardless of error cases. +// +void NV_API_CALL rm_get_firmware_version( + nvidia_stack_t *sp, + nv_state_t *nv, + char *version, + NvLength version_length +) +{ + NV2080_CTRL_GSP_GET_FEATURES_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus = NV_OK; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_GPU); + if (pRmApi != NULL) + { + rmStatus = pRmApi->Control(pRmApi, + nv->rmapi.hClient, + nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_GSP_GET_FEATURES, + ¶ms, + sizeof(params)); + + RmUnixRmApiEpilogue(nv, &threadState); + } + else + { + rmStatus = NV_ERR_INVALID_STATE; + } + + if (rmStatus != NV_OK) + { + if (RMCFG_FEATURE_GSP_CLIENT_RM && nv->request_firmware) + { + const char *pTmpString = "N/A"; + portStringCopy(version, version_length, pTmpString, portStringLength(pTmpString) + 1); + } + NV_PRINTF(LEVEL_INFO, + "%s: Failed to query gpu build versions, status=0x%x\n", + __FUNCTION__, + rmStatus); + goto finish; + } + portMemCopy(version, version_length, params.firmwareVersion, sizeof(params.firmwareVersion)); + +finish: + NV_EXIT_RM_RUNTIME(sp,fp); +} + +// +// disable GPU SW state persistence +// + +void NV_API_CALL rm_disable_gpu_state_persistence(nvidia_stack_t *sp, nv_state_t *nv) +{ + THREAD_STATE_NODE threadState; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu->setProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE, NV_FALSE); + osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, NV_FALSE); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_log_gpu_crash( + nv_stack_t *sp, + nv_state_t *nv +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS status; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if ((status = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DIAG)) == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if ((pGpu != NULL) && + ((status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DIAG)) == NV_OK)) + { + status = RmLogGpuCrash(pGpu); + + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd) +{ + nvidia_kernel_rmapi_ops_t *ops = ops_cmd; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + switch (ops->op) + { + case NV01_FREE: + Nv01FreeKernel(&ops->params.free); + break; + + case NV01_ALLOC_MEMORY: + Nv01AllocMemoryKernel(&ops->params.allocMemory64); + break; + + case NV04_ALLOC: + Nv04AllocKernel(&ops->params.alloc); + break; + + case NV04_VID_HEAP_CONTROL: + Nv04VidHeapControlKernel(ops->params.pVidHeapControl); + break; + + case NV04_MAP_MEMORY: + Nv04MapMemoryKernel(&ops->params.mapMemory); + break; + + case NV04_UNMAP_MEMORY: + Nv04UnmapMemoryKernel(&ops->params.unmapMemory); + break; + + case NV04_ALLOC_CONTEXT_DMA: + Nv04AllocContextDmaKernel(&ops->params.allocContextDma2); + break; + + case NV04_MAP_MEMORY_DMA: + Nv04MapMemoryDmaKernel(&ops->params.mapMemoryDma); + break; + + case NV04_UNMAP_MEMORY_DMA: + Nv04UnmapMemoryDmaKernel(&ops->params.unmapMemoryDma); + break; + + case NV04_BIND_CONTEXT_DMA: + Nv04BindContextDmaKernel(&ops->params.bindContextDma); + break; + + case NV04_CONTROL: + Nv04ControlKernel(&ops->params.control); + break; + + case NV04_DUP_OBJECT: + Nv04DupObjectKernel(&ops->params.dupObject); + break; + + case NV04_SHARE: + Nv04ShareKernel(&ops->params.share); + break; + + case NV04_ADD_VBLANK_CALLBACK: + Nv04AddVblankCallbackKernel(&ops->params.addVblankCallback); + break; + } + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +// +// ACPI method (NVIF/_DSM/WMMX/MXM*/etc.) initialization +// +void RmInitAcpiMethods(OBJOS *pOS, OBJSYS *pSys, OBJGPU *pGpu) +{ + NvU32 handlesPresent; + + if (pSys->getProperty(pSys, PDB_PROP_SYS_NVIF_INIT_DONE)) + return; + + nv_acpi_methods_init(&handlesPresent); + + // Check if NVPCF _DSM functions are implemented under NVPCF or GPU device scope. + RmCheckNvpcfDsmScope(pGpu); + acpiDsmInit(pGpu); + +} + +// +// ACPI method (NVIF/_DSM/WMMX/MXM*/etc.) teardown +// +void RmUnInitAcpiMethods(OBJSYS *pSys) +{ + pSys->setProperty(pSys, PDB_PROP_SYS_NVIF_INIT_DONE, NV_FALSE); + + nv_acpi_methods_uninit(); +} + +// +// Converts an array of OS page address to an array of RM page addresses.This +// assumes that: +// (1) The pteArray is at least pageCount entries large, +// (2) The pageCount is given in RM pages, and +// (3) The OS page entries start at index 0. +// +void RmInflateOsToRmPageArray(RmPhysAddr *pteArray, NvU64 pageCount) +{ + NvUPtr osPageIdx, osPageOffset; + NvU64 i; + + // + // We can do the translation in place by moving backwards, since there + // will always be more RM pages than OS pages + // + for (i = pageCount - 1; i != NV_U64_MAX; i--) + { + osPageIdx = i >> NV_RM_TO_OS_PAGE_SHIFT; + osPageOffset = (i & ((1 << NV_RM_TO_OS_PAGE_SHIFT) - 1)) * + NV_RM_PAGE_SIZE; + pteArray[i] = pteArray[osPageIdx] + osPageOffset; + } +} + +void RmDeflateRmToOsPageArray(RmPhysAddr *pteArray, NvU64 pageCount) +{ + NvU64 i; + + for (i = 0; i < NV_RM_PAGES_TO_OS_PAGES(pageCount); i++) + { + pteArray[i] = pteArray[(i << NV_RM_TO_OS_PAGE_SHIFT)]; + } + + // Zero out the rest of the addresses, which are now invalid + portMemSet(pteArray + i, 0, sizeof(*pteArray) * (pageCount - i)); +} + +NvBool NV_API_CALL +rm_get_device_remove_flag +( + nvidia_stack_t * sp, + NvU32 gpu_id +) +{ + THREAD_STATE_NODE threadState; + void *fp; + NvBool bRemove; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (gpumgrQueryGpuDrainState(gpu_id, NULL, &bRemove) != NV_OK) + { + bRemove = NV_FALSE; + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return bRemove; +} + +NvBool NV_API_CALL +rm_gpu_need_4k_page_isolation +( + nv_state_t *nv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + return nvp->b_4k_page_isolation_required; +} + +NV_STATUS NV_API_CALL rm_get_gpu_numa_info( + nvidia_stack_t *sp, + nv_state_t *nv, + NvS32 *pNid, + NvU64 *pNumaMemAddr, + NvU64 *pNumaMemSize, + NvU64 *pOfflineAddresses, + NvU32 *pOfflineAddressesCount +) +{ + NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS *pParams; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + void *fp; + NV_STATUS status = NV_OK; + + if ((pNid == NULL) || (pNumaMemAddr == NULL) || (pNumaMemSize == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if ((pOfflineAddressesCount != NULL) && + ((pOfflineAddresses == NULL) || + (*pOfflineAddressesCount > NV_ARRAY_ELEMENTS(pParams->numaOfflineAddresses)))) + { + return NV_ERR_INVALID_ARGUMENT; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + pParams = portMemAllocNonPaged(sizeof(*pParams)); + if (pParams == NULL) + { + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_ERR_NO_MEMORY; + } + + portMemSet(pParams, 0, sizeof(*pParams)); + + if (pOfflineAddressesCount != NULL) + { + pParams->numaOfflineAddressesCount = *pOfflineAddressesCount; + } + + pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM); + if (pRmApi == NULL) + { + status = NV_ERR_INVALID_STATE; + goto finish; + } + + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FB_GET_NUMA_INFO, + pParams, sizeof(*pParams)); + + RmUnixRmApiEpilogue(nv, &threadState); + + if (status == NV_OK) + { + NvU32 i; + + *pNid = pParams->numaNodeId; + *pNumaMemAddr = pParams->numaMemAddr; + *pNumaMemSize = pParams->numaMemSize; + *pOfflineAddressesCount = pParams->numaOfflineAddressesCount; + + for (i = 0; i < pParams->numaOfflineAddressesCount; i++) + { + pOfflineAddresses[i] = pParams->numaOfflineAddresses[i]; + } + } + +finish: + portMemFree(pParams); + + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +NV_STATUS NV_API_CALL rm_gpu_numa_online( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + void *fp; + NV_STATUS status = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM); + if (pRmApi == NULL) + { + status = NV_ERR_INVALID_STATE; + goto finish; + } + + params.bOnline = NV_TRUE; + + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(nv, &threadState); + +finish: + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + + +NV_STATUS NV_API_CALL rm_gpu_numa_offline( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS params = { 0 }; + RM_API *pRmApi; + THREAD_STATE_NODE threadState; + void *fp; + NV_STATUS status = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM); + if (pRmApi == NULL) + { + status = NV_ERR_INVALID_STATE; + goto finish; + } + + params.bOnline = NV_FALSE; + + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, + nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS, + ¶ms, sizeof(params)); + + RmUnixRmApiEpilogue(nv, &threadState); + +finish: + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +// +// A device is considered "sequestered" if it has drain state enabled for it. +// The kernel interface layer can use this to check the drain state of a device +// in paths outside of initialization, e.g., when clients attempt to reference +// count the device. +// +NvBool NV_API_CALL rm_is_device_sequestered( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + NvBool bDrain = NV_FALSE; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU) == NV_OK) + { + // + // If gpumgrQueryGpuDrainState succeeds, bDrain will be set as needed. + // If gpumgrQueryGpuDrainState fails, bDrain will stay false; we assume + // that if core RM can't tell us the drain state, it must not be + // attached and the "sequestered" question is not relevant. + // + (void) gpumgrQueryGpuDrainState(pNv->gpu_id, &bDrain, NULL); + + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return bDrain; +} + +void NV_API_CALL rm_check_for_gpu_surprise_removal( + nvidia_stack_t *sp, + nv_state_t *nv +) +{ + THREAD_STATE_NODE threadState; + void *fp; + NV_STATUS rmStatus; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock. + if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU)) == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if ((rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GPU)) == NV_OK) + { + osHandleGpuLost(pGpu); + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + + // UNLOCK: release api lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_set_external_kernel_client_count( + nvidia_stack_t *sp, + nv_state_t *pNv, + NvBool bIncr +) +{ + THREAD_STATE_NODE threadState; + void *fp; + OBJGPU *pGpu; + NV_STATUS rmStatus = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu = NV_GET_NV_PRIV_PGPU(pNv); + + if (pGpu != NULL) + { + rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_GPU); + if (rmStatus == NV_OK) + { + rmStatus = gpuSetExternalKernelClientCount(pGpu, bIncr); + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NvBool rm_get_uefi_console_status( + nv_state_t *nv +) +{ + NvU16 fbWidth, fbHeight, fbDepth, fbPitch; + NvU64 fbSize; + NvU64 fbBaseAddress = 0; + NvBool bConsoleDevice = NV_FALSE; + + // os_get_screen_info() will return dimensions and an address for + // any fbdev driver (e.g., efifb, vesafb, etc). To find if this is a + // UEFI console check the fbBaseAddress: if it was set up by the EFI GOP + // driver, it will point into BAR1 (FB); if it was set up by the VBIOS, + // it will point to BAR2 + 16MB. + os_get_screen_info(&fbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch, + nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address, + nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000); + + fbSize = fbHeight * fbPitch; + + bConsoleDevice = (fbSize != 0); + + return bConsoleDevice; +} + +NvU64 rm_get_uefi_console_size( + nv_state_t *nv, + NvU64 *pFbBaseAddress +) +{ + NvU16 fbWidth, fbHeight, fbDepth, fbPitch; + NvU64 fbSize; + + fbSize = fbWidth = fbHeight = fbDepth = fbPitch = 0; + + // os_get_screen_info() will return dimensions and an address for + // any fbdev driver (e.g., efifb, vesafb, etc). To find if this is a + // UEFI console check the fbBaseAddress: if it was set up by the EFI GOP + // driver, it will point into BAR1 (FB); if it was set up by the VBIOS, + // it will point to BAR2 + 16MB. + os_get_screen_info(pFbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch, + nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address, + nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000); + + fbSize = fbHeight * fbPitch; + + return fbSize; +} + +/* + * IOMMU needs to be present on the server to support SR-IOV vGPU, unless + * we have SR-IOV enabled for remote GPU. + */ + +NvBool NV_API_CALL rm_is_iommu_needed_for_sriov( + nvidia_stack_t *sp, + nv_state_t * nv +) +{ + OBJGPU *pGpu; + NvU32 data; + NvBool ret = NV_TRUE; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (osReadRegistryDword(pGpu, NV_REG_STR_RM_REMOTE_GPU, &data) == NV_OK) + { + if (data == NV_REG_STR_RM_REMOTE_GPU_ENABLE) + ret = NV_FALSE; + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return ret; +} + +NvBool NV_API_CALL rm_disable_iomap_wc(void) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + return pSys->pCl->getProperty(pSys, PDB_PROP_CL_DISABLE_IOMAP_WC) == NV_TRUE; +} + +// +// Verifies the handle, offset and size and dups hMemory. +// Must be called with API lock and GPU lock held. +// +NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hSrcClient, + NvHandle hDstClient, + NvHandle hDevice, + NvHandle hSubdevice, + void *pGpuInstanceInfo, + NvHandle hMemory, + NvU64 offset, + NvU64 size, + NvHandle *phMemoryDuped +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + OBJGPU *pGpu; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + NV_ASSERT(rmapiLockIsOwner()); + + NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu))); + + rmStatus = RmDmabufVerifyMemHandle(pGpu, hSrcClient, hMemory, + offset, size, pGpuInstanceInfo); + if (rmStatus == NV_OK) + { + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + NvHandle hMemoryDuped = 0; + + rmStatus = pRmApi->DupObject(pRmApi, + hDstClient, + hDevice, + &hMemoryDuped, + hSrcClient, + hMemory, + 0); + if (rmStatus == NV_OK) + { + *phMemoryDuped = hMemoryDuped; + } + else if (rmStatus == NV_ERR_INVALID_OBJECT_PARENT) + { + hMemoryDuped = 0; + + // If duping under Device fails, try duping under Subdevice + rmStatus = pRmApi->DupObject(pRmApi, + hDstClient, + hSubdevice, + &hMemoryDuped, + hSrcClient, + hMemory, + 0); + if (rmStatus == NV_OK) + { + *phMemoryDuped = hMemoryDuped; + } + } + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +// +// Frees dup'd hMemory. +// Must be called with API lock and GPU lock held. +// +void NV_API_CALL rm_dma_buf_undup_mem_handle( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hMemory +) +{ + THREAD_STATE_NODE threadState; + RM_API *pRmApi; + OBJGPU *pGpu; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + NV_ASSERT(rmapiLockIsOwner()); + + NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu))); + + pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + + pRmApi->Free(pRmApi, hClient, hMemory); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +// +// Maps a handle to BAR1. +// Must be called with API lock and GPU lock held. +// +NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hMemory, + NvU64 offset, + NvU64 size, + NvU64 *pBar1Va +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + OBJGPU *pGpu; + KernelBus *pKernelBus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + + NV_ASSERT(rmapiLockIsOwner()); + + NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu))); + + rmStatus = kbusMapFbApertureByHandle(pGpu, pKernelBus, hClient, + hMemory, offset, size, pBar1Va); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +// +// Unmaps a handle from BAR1. +// Must be called with API lock and GPU lock held. +// +NV_STATUS NV_API_CALL rm_dma_buf_unmap_mem_handle( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hMemory, + NvU64 size, + NvU64 bar1Va +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + OBJGPU *pGpu; + KernelBus *pKernelBus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + + NV_ASSERT(rmapiLockIsOwner()); + + NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu))); + + rmStatus = kbusUnmapFbApertureByHandle(pGpu, pKernelBus, hClient, + hMemory, bar1Va); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle *phClient, + NvHandle *phDevice, + NvHandle *phSubdevice, + void **ppGpuInstanceInfo +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus == NV_OK) + { + rmStatus = RmDmabufGetClientAndDevice(pGpu, hClient, phClient, phDevice, + phSubdevice, ppGpuInstanceInfo); + + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + + return rmStatus; +} + +void NV_API_CALL rm_dma_buf_put_client_and_device( + nvidia_stack_t *sp, + nv_state_t *nv, + NvHandle hClient, + NvHandle hDevice, + NvHandle hSubdevice, + void *pGpuInstanceInfo +) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + // LOCK: acquire API lock + rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus == NV_OK) + { + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + + rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI); + if (rmStatus == NV_OK) + { + RmDmabufPutClientAndDevice(pGpu, hClient, hDevice, hSubdevice, + pGpuInstanceInfo); + + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + + // UNLOCK: release API lock + rmapiLockRelease(); + } + NV_ASSERT_OK(rmStatus); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +// +// Fetches GSP ucode data for usage during RM Init +// NOTE: Used only on VMWware +// + +NvBool NV_API_CALL rm_is_altstack_in_use(void) +{ +#if defined(__use_altstack__) + return NV_TRUE; +#else + return NV_FALSE; +#endif +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osinit.c b/userspace/sdks/nv525/nvalloc/unix/src/osinit.c new file mode 100644 index 0000000..5080e50 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/osinit.c @@ -0,0 +1,2202 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/************************************************************************************************************** +* +* Description: +* UNIX-general, device-independent initialization code for +* the resource manager. +* +* +**************************************************************************************************************/ + +#include <nv_ref.h> +#include <nv.h> // NV device driver interface +#include <nv-reg.h> +#include <nv-priv.h> +#include <nvos.h> +#include <nvrm_registry.h> +#include <platform/sli/sli.h> +#include <nverror.h> +#include <osapi.h> +#include <core/system.h> +#include <os/os.h> +#include "gpu/gpu.h" +#include <objtmr.h> +#include "gpu/bus/kern_bus.h" +#include "nverror.h" +#include <gpu/bif/kernel_bif.h> +#include <nv-nb-regs.h> +#include <gpu/mem_mgr/mem_mgr.h> +#include <gpu/mem_sys/kern_mem_sys.h> +#include "kernel/gpu/intr/intr.h" +#include <mem_mgr/io_vaspace.h> +#include <gpu/disp/kern_disp.h> +#include <platform/chipset/chipset.h> +#include <kernel/gpu/rc/kernel_rc.h> +#include <kernel/gpu/fifo/kernel_fifo.h> +#include <nv-firmware-chip-family-select.h> +#include <gpu/gsp/kernel_gsp.h> +#include "liblogdecode.h" +#include <gpu/fsp/kern_fsp.h> + +#include <mem_mgr/virt_mem_mgr.h> +#include <virtualization/kernel_vgpu_mgr.h> + +#include <rmosxfac.h> +#include <gpu_mgr/gpu_mgr.h> +#include <core/thread_state.h> +#include <core/locks.h> +#include <rmapi/client.h> +#include <ctrl/ctrl2080/ctrl2080gpu.h> +#include <class/cl00f2.h> + +#include <platform/chipset/pci_pbi.h> + +#include "platform/nbsi/nbsi_read.h" +#include "gpu_mgr/gpu_db.h" +#include <class/cl0080.h> +#include <class/cl0073.h> +#include <class/cl2080.h> +#include <class/cl402c.h> + +#include <gpu/dce_client/dce_client.h> +// RMCONFIG: need definition of REGISTER_ALL_HALS() +#include "g_hal_register.h" + +typedef enum +{ + RM_INIT_OK, + + /* general os errors */ + RM_INIT_REG_SETUP_FAILED = 0x10, + RM_INIT_SYS_ENVIRONMENT_FAILED, + + /* gpu errors */ + RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED = 0x20, + RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED, + RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED, + RM_INIT_GPU_PRE_INIT_FAILED, + RM_INIT_GPU_STATE_INIT_FAILED, + RM_INIT_GPU_LOAD_FAILED, + RM_INIT_GPU_UNIVERSAL_VALIDATION_FAILED, + RM_INIT_GPU_DMA_CONFIGURATION_FAILED, + + /* vbios errors */ + RM_INIT_VBIOS_FAILED = 0x30, + RM_INIT_VBIOS_POST_FAILED, + RM_INIT_VBIOS_X86EMU_FAILED, + + /* scalability errors */ + RM_INIT_SCALABILITY_FAILED = 0x40, + + /* general core rm errors */ + RM_INIT_WATCHDOG_FAILED, + RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED, + RM_GPUDB_REGISTER_FAILED, + + RM_INIT_ALLOC_RMAPI_FAILED, + RM_INIT_GPUINFO_WITH_RMAPI_FAILED, + + /* rm firmware errors */ + RM_INIT_FIRMWARE_POLICY_FAILED = 0x60, + RM_INIT_FIRMWARE_FETCH_FAILED, + RM_INIT_FIRMWARE_INIT_FAILED, + + RM_INIT_MAX_FAILURES +} rm_init_status; + +typedef rm_init_status RM_INIT_STATUS; + +typedef struct { + RM_INIT_STATUS initStatus; + NV_STATUS rmStatus; + NvU32 line; +} UNIX_STATUS; + +#define INIT_UNIX_STATUS { RM_INIT_OK, NV_OK, 0 } +#define RM_INIT_SUCCESS(init) ((init) == RM_INIT_OK) + +#define RM_SET_ERROR(status, err) { (status).initStatus = (err); \ + (status).line = __LINE__; } + + +// +// GPU architectures support DMA addressing up to a certain address width, +// above which all other bits in any given DMA address must not vary +// (e.g., all 0). This value is the minimum of the DMA addressing +// capabilities, in number of physical address bits, for all supported +// GPU architectures. +// +#define NV_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH 36 + +// +// All GPU architectures with GSP support at least 47 physical address bits. +// +#define NV_GSP_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH 47 + +static inline NvU64 nv_encode_pci_info(nv_pci_info_t *pci_info) +{ + return gpuEncodeDomainBusDevice(pci_info->domain, pci_info->bus, pci_info->slot); +} + +static inline NvU32 nv_generate_id_from_pci_info(nv_pci_info_t *pci_info) +{ + return gpuGenerate32BitId(pci_info->domain, pci_info->bus, pci_info->slot); +} + +static inline void nv_os_map_kernel_space(nv_state_t *nv, nv_aperture_t *aperture) +{ + NV_ASSERT(aperture->map == NULL); + + // let's start off assuming a standard device and map the registers + // normally. It is unfortunate to hard-code the register size here, but we don't + // want to fail trying to map all of a multi-devices' register space + aperture->map = osMapKernelSpace(aperture->cpu_address, + aperture->size, + NV_MEMORY_UNCACHED, + NV_PROTECT_READ_WRITE); + aperture->map_u = (nv_phwreg_t)aperture->map; +} + +// local prototypes +static NV_STATUS initCoreLogic(OBJGPU *); +static void initVendorSpecificRegistry(OBJGPU *, NvU16); +static NV_STATUS teardownCoreLogic(OBJOS *, OBJGPU *); +static void initUnixSpecificRegistry(OBJGPU *); + +NvBool osRmInitRm(OBJOS *pOS) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + NvU64 system_memory_size = (NvU64)-1; + + NV_PRINTF(LEVEL_INFO, "init rm\n"); + + if (os_is_efi_enabled()) + { + pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_TRUE); + } + + // have to init this before the debug subsystem, which will + // try to check the value of ResmanDebugLevel + RmInitRegistry(); + + NvU32 data = 0; + if ((osReadRegistryDword(NULL, + NV_REG_ENABLE_PCIE_GEN3, &data) == NV_OK) && + (data != 0)) + { + data = DRF_DEF(_REG_STR, _RM_PCIE, _LINK_SPEED_ALLOW_GEN3, _ENABLE); + + osWriteRegistryDword(NULL, + NV_REG_STR_RM_PCIE_LINK_SPEED, data); + } + + if (!(osReadRegistryDword(NULL, + NV_REG_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, + &data) == NV_OK) || data) + { + pSys->setProperty(pSys, + PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, + NV_TRUE); + } + + if ((osReadRegistryDword(NULL, + NV_REG_ENABLE_STREAM_MEMOPS, + &data) == NV_OK) && (data != 0)) + { + data = DRF_DEF(_REG_STR, _RM_STREAM_MEMOPS, _ENABLE, _YES); + osWriteRegistryDword(NULL, NV_REG_STR_RM_STREAM_MEMOPS, data); + } + + if ((osReadRegistryDword(NULL, + NV_REG_STR_RM_FORCE_BAR_PATH, + &data) == NV_OK) && (data == 1)) + { + // + // Exposing the GPU memory as NUMA node memory requires coherent CPU + // mappings over NVLink. If those mappings are disabled, also disable + // NUMA-onlining of the device memory. + // + osWriteRegistryDword(NULL, + NV_REG_ENABLE_USER_NUMA_MANAGEMENT, 0); + } + + if ((osReadRegistryDword(NULL, + NV_REG_NVLINK_DISABLE, &data) == NV_OK) && + (data == 1)) + { + osWriteRegistryDword(NULL, NV_REG_STR_RM_NVLINK_CONTROL, 0x1); + } + + if ((osReadRegistryDword(NULL, + NV_REG_ENABLE_PCIE_RELAXED_ORDERING_MODE, &data) == NV_OK) && + (data == 1)) + { + osWriteRegistryDword(NULL, + NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING, + NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING_ENABLE); + } + + // init the debug subsystem if necessary + os_dbg_init(); + nvDbgInitRmMsg(NULL); + + // Force nvlog reinit since module params are now available + NVLOG_UPDATE(); + + // Register all supported hals + if (REGISTER_ALL_HALS() != NV_OK) + { + RmDestroyRegistry(NULL); + return NV_FALSE; + } + + system_memory_size = NV_RM_PAGES_PER_OS_PAGE * os_get_num_phys_pages(); + + // if known, relay the number of system memory pages (in terms of RM page + // size) to the RM; this is needed for e.g. TurboCache parts. + if (system_memory_size != (NvU64)-1) + pOS->SystemMemorySize = system_memory_size; + + // Setup any ThreadState defaults + threadStateInitSetupFlags(THREAD_STATE_SETUP_FLAGS_ENABLED | + THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED | + THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED | + THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED); + + return NV_TRUE; +} + +void RmShutdownRm(void) +{ + NV_PRINTF(LEVEL_INFO, "shutdown rm\n"); + + RmDestroyRegistry(NULL); + + // Free objects created with RmInitRm, including the system object + RmDestroyRm(); +} + +// +// osAttachGpu +// +// This routine is used as a callback by the gpumgrAttachGpu +// interface to allow os-dependent code to set up any state +// before engine construction begins. +// +NV_STATUS osAttachGpu( + OBJGPU *pGpu, + void *pOsGpuInfo +) +{ + nv_state_t *nv = (nv_state_t *)pOsGpuInfo; + nv_priv_t *nvp; + + nvp = NV_GET_NV_PRIV(nv); + + nvp->pGpu = pGpu; + + NV_SET_NV_STATE(pGpu, (void *)nv); + + initUnixSpecificRegistry(pGpu); + + // Assign default values to Registry keys for VGX + if (os_is_vgx_hyper()) + { + initVGXSpecificRegistry(pGpu); + } + + return NV_OK; +} + +NV_STATUS osDpcAttachGpu( + OBJGPU *pGpu, + void *pOsGpuInfo +) +{ + return NV_OK; // Nothing to do for unix +} + +void osDpcDetachGpu( + OBJGPU *pGpu +) +{ + return; // Nothing to do for unix +} + +NV_STATUS +osHandleGpuLost +( + OBJGPU *pGpu +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 pmc_boot_0; + + // Determine if we've already run the handler + if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED)) + { + return NV_OK; + } + + pmc_boot_0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0); + if (pmc_boot_0 != nvp->pmc_boot_0) + { + // + // This doesn't support PEX Reset and Recovery yet. + // This will help to prevent accessing registers of a GPU + // which has fallen off the bus. + // + nvErrorLog_va((void *)pGpu, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, + "GPU has fallen off the bus."); + + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU has fallen off the bus.\n"); + + if (pGpu->boardInfo != NULL && pGpu->boardInfo->serialNumber[0] != '\0') + { + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, + "GPU serial number is %s.\n", + pGpu->boardInfo->serialNumber); + } + + gpuSetDisconnectedProperties(pGpu); + + // Trigger the OS's PCI recovery mechanism + if (nv_pci_trigger_recovery(nv) != NV_OK) + { + // + // Initiate a crash dump immediately, since the OS doesn't appear + // to have a mechanism wired up for attempted recovery. + // + (void) RmLogGpuCrash(pGpu); + } + else + { + // + // Make the SW state stick around until the recovery can start, but + // don't change the PDB property: this is only used to report to + // clients whether or not persistence mode is enabled, and we'll + // need it after the recovery callbacks to restore the correct + // persistence mode for the GPU. + // + osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, NV_TRUE); + } + + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + // Set SURPRISE_REMOVAL flag for eGPU to help in device removal. + if ((pCl != NULL) && + pCl->getProperty(pCl, PDB_PROP_CL_IS_EXTERNAL_GPU)) + { + nv->flags |= NV_FLAG_IN_SURPRISE_REMOVAL; + } + DBG_BREAKPOINT(); + } + + return NV_OK; +} + +/* + * Initialize the required GPU information by doing RMAPI control calls + * and store the same in the UNIX specific data structures. + */ +static NV_STATUS +RmInitGpuInfoWithRmApi +( + OBJGPU *pGpu +) +{ + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NV2080_CTRL_GPU_GET_INFO_V2_PARAMS *pGpuInfoParams = { 0 }; + NV_STATUS status; + + // LOCK: acquire GPUs lock + status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); + if (status != NV_OK) + { + return status; + } + + pGpuInfoParams = portMemAllocNonPaged(sizeof(*pGpuInfoParams)); + if (pGpuInfoParams == NULL) + { + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + return NV_ERR_NO_MEMORY; + } + + + portMemSet(pGpuInfoParams, 0, sizeof(*pGpuInfoParams)); + + pGpuInfoParams->gpuInfoListSize = 3; + pGpuInfoParams->gpuInfoList[0].index = NV2080_CTRL_GPU_INFO_INDEX_4K_PAGE_ISOLATION_REQUIRED; + pGpuInfoParams->gpuInfoList[1].index = NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED; + pGpuInfoParams->gpuInfoList[2].index = NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY; + + status = pRmApi->Control(pRmApi, nv->rmapi.hClient, + nv->rmapi.hSubDevice, + NV2080_CTRL_CMD_GPU_GET_INFO_V2, + pGpuInfoParams, sizeof(*pGpuInfoParams)); + + if (status == NV_OK) + { + nvp->b_4k_page_isolation_required = + (pGpuInfoParams->gpuInfoList[0].data == + NV2080_CTRL_GPU_INFO_INDEX_4K_PAGE_ISOLATION_REQUIRED_YES); + nvp->b_mobile_config_enabled = + (pGpuInfoParams->gpuInfoList[1].data == + NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_YES); + nv->dma_buf_supported = + (pGpuInfoParams->gpuInfoList[2].data == + NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES); + } + + portMemFree(pGpuInfoParams); + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + return status; +} + +static void RmSetSocDispDeviceMappings( + GPUATTACHARG *gpuAttachArg, + nv_state_t *nv +) +{ + gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvAddr = (GPUHWREG*) nv->regs->map; + gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvPAddr = nv->regs->cpu_address; + gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvLength = (NvU32) nv->regs->size; +} + +static void RmSetSocDpauxDeviceMappings( + GPUATTACHARG *gpuAttachArg, + nv_state_t *nv +) +{ +} + +static void RmSetSocHdacodecDeviceMappings( + GPUATTACHARG *gpuAttachArg, + nv_state_t *nv +) +{ +} + +static void RmSetSocMipiCalDeviceMappings( + GPUATTACHARG *gpuAttachArg, + nv_state_t *nv +) +{ +} + +static void +osInitNvMapping( + nv_state_t *nv, + NvU32 *pDeviceReference, + UNIX_STATUS *status +) +{ + OBJGPU *pGpu; + OBJSYS *pSys = SYS_GET_INSTANCE(); + GPUATTACHARG *gpuAttachArg; + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvU32 deviceInstance; + NvU32 data = 0; + + NV_PRINTF(LEVEL_INFO, "osInitNvMapping:\n"); + + // allocate the next available gpu device number + status->rmStatus = gpumgrAllocGpuInstance(pDeviceReference); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot get valid gpu instance\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); + return; + } + + // RM_BASIC_LOCK_MODEL: allocate GPU lock + status->rmStatus = rmGpuLockAlloc(*pDeviceReference); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** cannot allocate GPU lock\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(*pDeviceReference); + return; + } + + // attach default single-entry broadcast device for this gpu + status->rmStatus = gpumgrCreateDevice(&deviceInstance, NVBIT(*pDeviceReference), NULL); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot attach bc gpu\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED); + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(*pDeviceReference); + return; + } + + // init attach state + gpuAttachArg = portMemAllocNonPaged(sizeof(GPUATTACHARG)); + if (gpuAttachArg == NULL) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot allocate gpuAttachArg\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(*pDeviceReference); + return; + } + + portMemSet(gpuAttachArg, 0, sizeof(GPUATTACHARG)); + + if (NV_IS_SOC_DISPLAY_DEVICE(nv)) + { + gpuAttachArg->socDeviceArgs.specified = NV_TRUE; + + RmSetSocDispDeviceMappings(gpuAttachArg, nv); + + RmSetSocDpauxDeviceMappings(gpuAttachArg, nv); + + RmSetSocHdacodecDeviceMappings(gpuAttachArg, nv); + + RmSetSocMipiCalDeviceMappings(gpuAttachArg, nv); + + gpuAttachArg->socDeviceArgs.socChipId0 = nv->disp_sw_soc_chip_id; + + gpuAttachArg->socDeviceArgs.iovaspaceId = nv->iovaspace_id; + } + else + { + gpuAttachArg->fbPhysAddr = nv->fb->cpu_address; + gpuAttachArg->fbBaseAddr = (GPUHWREG*) 0; // not mapped + gpuAttachArg->devPhysAddr = nv->regs->cpu_address; + gpuAttachArg->regBaseAddr = (GPUHWREG*) nv->regs->map; + gpuAttachArg->intLine = 0; // don't know yet + gpuAttachArg->instPhysAddr = nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address; + gpuAttachArg->instBaseAddr = (GPUHWREG*) 0; // not mapped + + gpuAttachArg->regLength = nv->regs->size; + gpuAttachArg->fbLength = nv->fb->size; + gpuAttachArg->instLength = nv->bars[NV_GPU_BAR_INDEX_IMEM].size; + + gpuAttachArg->iovaspaceId = nv->iovaspace_id; + } + + // + // we need this to check if we are running on virtual GPU + // in gpuBindHal function later. + // + gpuAttachArg->nvDomainBusDeviceFunc = nv_encode_pci_info(&nv->pci_info); + + gpuAttachArg->bRequestFwClientRm = nv->request_fw_client_rm; + + gpuAttachArg->pOsAttachArg = (void *)nv; + + // use gpu manager to attach gpu + status->rmStatus = gpumgrAttachGpu(*pDeviceReference, gpuAttachArg); + portMemFree(gpuAttachArg); + if (status->rmStatus != NV_OK) + { + gpumgrDestroyDevice(deviceInstance); + RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED); + NV_PRINTF(LEVEL_ERROR, "*** Cannot attach gpu\n"); + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(*pDeviceReference); + return; + } + nvp->flags |= NV_INIT_FLAG_GPUMGR_ATTACH; + + pGpu = gpumgrGetGpu(*pDeviceReference); + + sysInitRegistryOverrides(pSys); + + sysApplyLockingPolicy(pSys); + + pGpu->busInfo.IntLine = nv->interrupt_line; + + // + // Set the DMA address size as soon as we have the HAL to call to + // determine the precise number of physical address bits supported + // by the architecture. DMA allocations should not be made before + // this point. + // + nv_set_dma_address_size(nv, gpuGetPhysAddrWidth_HAL(pGpu, ADDR_SYSMEM)); + + pGpu->dmaStartAddress = (RmPhysAddr)nv_get_dma_start_address(nv); + if (nv->fb != NULL) + { + pGpu->registerAccess.gpuFbAddr = (GPUHWREG*) nv->fb->map; + pGpu->busInfo.gpuPhysFbAddr = nv->fb->cpu_address; + } + + // set default parent gpu + gpumgrSetParentGPU(pGpu, pGpu); + + NV_PRINTF(LEVEL_INFO, "device instance : %d\n", *pDeviceReference); + NV_PRINTF(LEVEL_INFO, "NV regs using linear address : 0x%p\n", + pGpu->deviceMappings[SOC_DEV_MAPPING_DISP].gpuNvAddr); + NV_PRINTF(LEVEL_INFO, + "NV fb using linear address : 0x%p\n", pGpu->registerAccess.gpuFbAddr); + + pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED, NV_TRUE); + pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS, NV_FALSE); + + if (osReadRegistryDword(pGpu, + NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR, &data) == NV_OK) + { + if (data == NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR_ENABLE) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS, NV_TRUE); + } + } + + if (!os_is_vgx_hyper()) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE); + } + + if ((osReadRegistryDword(NULL, + NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS, + &data) == NV_OK) && data) + { + KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + + memmgrSetPmaForcePersistence(pMemoryManager, NV_TRUE); + pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = NV_TRUE; + + nv->preserve_vidmem_allocations = NV_TRUE; + } +} + +void osInitScalabilityOptions +( + OBJGPU *pGpu, + void *pDeviceExtension +) +{ +} + + +static NV_STATUS +osInitScalability( + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + osInitScalabilityOptions(pGpu, NULL); + + // We need PCI-E chipset information very early. + if (pCl != NULL) + { + return clInitPcie(pGpu, pCl); + } + else + { + return NV_ERR_NOT_SUPPORTED; + } +} + +static NV_STATUS +osTeardownScalability( + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + return clTeardownPcie(pGpu, pCl); +} + +static void +populateDeviceAttributes( + OBJGPU *pGpu, + nv_state_t *nv +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + if ((pCl != NULL) && pCl->getProperty(pCl, PDB_PROP_CL_IS_EXTERNAL_GPU)) + { + nv->is_external_gpu = NV_TRUE; + } +} + +static void +RmSetConsolePreservationParams(OBJGPU *pGpu) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NvU64 fbBaseAddress = 0; + NvU64 fbConsoleSize = 0; + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + + // + // PDB_PROP_GPU_PRIMARY_DEVICE should be NV_FALSE for vGPU configuration so + // return early + // + if (os_is_vgx_hyper() || IS_VIRTUAL(pGpu)) + return; + + // + // Check the OS layer for any video memory used by a console + // driver that should be reserved. + // + // If the console driver is using some amount of memory, its + // mapping is presumably either pointing at the start of BAR1 + // (this is the mapping the EFI GOP driver provides for + // efifb), or the upper 16MB of BAR2 (this is the mapping the + // VBIOS provides for, e.g., vesafb). + // + // In the BAR2 case, there is nothing more to do: the upper + // 16MB of BAR2 will remain in physical mode + // (see the documentation for NV_PBUS_BAR2_BLOCK in dev_bus.ref), + // and the console driver can continue to access it. + // + // In the BAR1 case, RM will put all of BAR1 into virtual + // mode, so we need to create a BAR1 mapping for the console + // driver to use. + // + // If the console driver is not using _either_ of those + // mappings, then the console driver will not be able to + // access the memory we reserve. This happens on some + // UEFI systems with multiple GPUs, because the firmware + // initializes the GOP driver on more than one GPU. In + // that case, just skip reserving anything for GPUs where + // neither BAR1 nor BAR2 match the console's base + // address. + // + fbConsoleSize = rm_get_uefi_console_size(nv, &fbBaseAddress); + + if (fbConsoleSize == 0) + { + if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI)) + { + fbConsoleSize = 0x40000; + } + else + { + NV_PRINTF(LEVEL_INFO, "No Frame Buffer Present\n"); + } + } + + if ((fbConsoleSize > 0) && (fbBaseAddress != 0)) + { + KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + pKernelBus->bPreserveBar1ConsoleEnabled = + (fbBaseAddress == nv->fb->cpu_address); + } + + pMemoryManager->Ram.ReservedConsoleDispMemSize = NV_ALIGN_UP(fbConsoleSize, 0x10000); +} + +static NV_STATUS +RmInitDeviceDma( + nv_state_t *nv +) +{ + if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) + { + OBJSYS *pSys = SYS_GET_INSTANCE(); + POBJVMM pVmm = SYS_GET_VMM(pSys); + POBJVASPACE pIOVAS; + NV_STATUS status = vmmCreateVaspace(pVmm, IO_VASPACE_A, + nv->iovaspace_id, 0, 0ULL, ~0ULL, + 0ULL, 0ULL, + NULL, VASPACE_FLAGS_ENABLE_VMM, + &pIOVAS); + if (status != NV_OK) + { + return status; + } + } + + return NV_OK; +} + +static void +RmTeardownDeviceDma( + nv_state_t *nv +) +{ + if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) + { + OBJSYS *pSys = SYS_GET_INSTANCE(); + POBJVMM pVmm = SYS_GET_VMM(pSys); + POBJVASPACE pIOVAS; + + if (NV_OK == vmmGetVaspaceFromId(pVmm, nv->iovaspace_id, IO_VASPACE_A, &pIOVAS)) + { + vmmDestroyVaspace(pVmm, pIOVAS); + } + } +} + +static void +RmInitNvDevice( + NvU32 deviceReference, + UNIX_STATUS *status +) +{ + // set the device context + OBJGPU *pGpu = gpumgrGetGpu(deviceReference); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + NV_PRINTF(LEVEL_INFO, "RmInitNvDevice:\n"); + + NV_PRINTF(LEVEL_INFO, + "device instance : 0x%08x\n", deviceReference); + + // initialize all engines -- calls back osInitMapping() + status->rmStatus = gpumgrStatePreInitGpu(pGpu); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot pre-initialize the device\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_PRE_INIT_FAILED); + return; + } + + os_disable_console_access(); + + status->rmStatus = gpumgrStateInitGpu(pGpu); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize the device\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_STATE_INIT_FAILED); + os_enable_console_access(); + return; + } + nvp->flags |= NV_INIT_FLAG_GPU_STATE; + + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + // + // Make sure bifIsMSIEnabled() gets a chance to look up the OS PCI + // handle for this GPU. This must happen after GPU state + // initialization. + // + if (pKernelBif != NULL) + { + kbifCheckAndRearmMSI(pGpu, pKernelBif); + } + + // Set RM's interrupt enable state to zero here so that interrupts + // won't be enabled during loading + Intr *pIntr = GPU_GET_INTR(pGpu); + if (pIntr != NULL) + { + intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); + } + + status->rmStatus = gpumgrStateLoadGpu(pGpu, GPU_STATE_DEFAULT); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "*** Cannot load state into the device\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_LOAD_FAILED); + os_enable_console_access(); + return; + } + nvp->flags |= NV_INIT_FLAG_GPU_STATE_LOAD; + + os_enable_console_access(); + + status->rmStatus = gpuPerformUniversalValidation_HAL(pGpu); + if (status->rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "*** Failed universal validation\n"); + RM_SET_ERROR(*status, RM_INIT_GPU_UNIVERSAL_VALIDATION_FAILED); + return; + } + + return; +} + +NV_STATUS osInitMapping( + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + NV_PRINTF(LEVEL_INFO, "osInitMapping:\n"); + + // Some PCI BIOSs leave the ROM mapped. This causes problems if it overlays system RAM. Just disable it. + GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_12, 0); + + // make sure our PCI latency timer is sufficient (max it out) + GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_3, + DRF_DEF(_CONFIG, _PCI_NV_3, _LATENCY_TIMER, _248_CLOCKS)); + + if (pCl != NULL) + { + initCoreLogic(pGpu); + } + nvp->flags |= NV_INIT_FLAG_CORE_LOGIC; + + return NV_OK; + +} // end of osInitMapping() + +static NV_STATUS +initCoreLogic( + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + +#if defined(NVCPU_X86_64) + OBJOS *pOS = SYS_GET_OS(pSys); + if (!os_pat_supported()) + pOS->setProperty(pOS, PDB_PROP_OS_PAT_UNSUPPORTED, NV_TRUE); +#endif + + return clInit(pGpu, pCl); +} + +static NV_STATUS +teardownCoreLogic( + OBJOS *pOS, + OBJGPU *pGpu +) +{ + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + + return clTeardown(pGpu, pCl); +} + +static void RmTeardownDpauxRegisters( + nv_state_t *nv +) +{ +} + +static void RmTeardownHdacodecRegisters( + nv_state_t *nv +) +{ +} + +static void RmTeardownMipiCalRegisters( + nv_state_t *nv +) +{ +} + +static NV_STATUS +RmTeardownRegisters( + nv_state_t *nv +) +{ + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Tearing down registers\n"); + + if (nv->regs && nv->regs->map) + { + osUnmapKernelSpace(nv->regs->map, nv->regs->size); + nv->regs->map = 0; + nv->regs->map_u = NULL; + } + + RmTeardownDpauxRegisters(nv); + + RmTeardownHdacodecRegisters(nv); + + RmTeardownMipiCalRegisters(nv); + + return NV_OK; +} + +static NV_STATUS +RmSetupDpauxRegisters( + nv_state_t *nv, + UNIX_STATUS *status +) +{ + + return NV_OK; +} + +static NV_STATUS +RmSetupHdacodecRegisters( + nv_state_t *nv, + UNIX_STATUS *status +) +{ + + return NV_OK; +} + +static NV_STATUS +RmSetupMipiCalRegisters( + nv_state_t *nv, + UNIX_STATUS *status +) +{ + + return NV_OK; +} + +static void +RmSetupRegisters( + nv_state_t *nv, + UNIX_STATUS *status +) +{ + NV_STATUS ret; + + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmSetupRegisters for 0x%x:0x%x\n", + nv->pci_info.vendor_id, nv->pci_info.device_id); + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "pci config info:\n"); + NV_DEV_PRINTF(NV_DBG_SETUP, nv, " registers look like: " NvP64_fmt " " NvP64_fmt, + nv->regs->cpu_address, nv->regs->size); + + if (nv->fb != NULL) + { + NV_DEV_PRINTF(NV_DBG_SETUP, nv, " fb looks like: " NvP64_fmt " " NvP64_fmt, + nv->fb->cpu_address, nv->fb->size); + } + + { + nv_os_map_kernel_space(nv, nv->regs); + } + + if (nv->regs->map == NULL) + { + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "Failed to map regs registers!!\n"); + RM_SET_ERROR(*status, RM_INIT_REG_SETUP_FAILED); + status->rmStatus = NV_ERR_OPERATING_SYSTEM; + return; + } + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Successfully mapped framebuffer and registers\n"); + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "final mappings:\n"); + NV_DEV_PRINTF(NV_DBG_SETUP, nv, " regs: " NvP64_fmt " " NvP64_fmt " 0x%p\n", + nv->regs->cpu_address, nv->regs->size, nv->regs->map); + + ret = RmSetupDpauxRegisters(nv, status); + if (ret != NV_OK) + goto err_unmap_disp_regs; + + ret = RmSetupHdacodecRegisters(nv, status); + if (ret != NV_OK) + { + RmTeardownDpauxRegisters(nv); + goto err_unmap_disp_regs; + } + + ret = RmSetupMipiCalRegisters(nv, status); + if (ret != NV_OK) + { + RmTeardownHdacodecRegisters(nv); + RmTeardownDpauxRegisters(nv); + goto err_unmap_disp_regs; + } + + return; + +err_unmap_disp_regs: + if (nv->regs && nv->regs->map) + { + osUnmapKernelSpace(nv->regs->map, nv->regs->size); + nv->regs->map = 0; + } + + return; +} + +NvBool RmInitPrivateState( + nv_state_t *pNv +) +{ + nv_priv_t *nvp; + NvU32 gpuId; + NvU32 pmc_boot_0 = 0; + NvU32 pmc_boot_42 = 0; + + NV_SET_NV_PRIV(pNv, NULL); + + if (!NV_IS_SOC_DISPLAY_DEVICE(pNv) && !NV_IS_SOC_IGPU_DEVICE(pNv)) + { + pNv->regs->map_u = os_map_kernel_space(pNv->regs->cpu_address, + os_page_size, + NV_MEMORY_UNCACHED); + if (pNv->regs->map_u == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "failed to map GPU registers (DISABLE_INTERRUPTS).\n"); + return NV_FALSE; + } + + pmc_boot_0 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_0); + pmc_boot_42 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_42); + + os_unmap_kernel_space(pNv->regs->map_u, os_page_size); + pNv->regs->map_u = NULL; + } + + if (os_alloc_mem((void **)&nvp, sizeof(*nvp)) != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "failed to allocate private device state.\n"); + return NV_FALSE; + } + + gpuId = nv_generate_id_from_pci_info(&pNv->pci_info); + + if (gpumgrRegisterGpuId(gpuId, nv_encode_pci_info(&pNv->pci_info)) != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "failed to register GPU with GPU manager.\n"); + os_free_mem(nvp); + return NV_FALSE; + } + + pNv->gpu_id = gpuId; + + pNv->iovaspace_id = nv_requires_dma_remap(pNv) ? gpuId : + NV_IOVA_DOMAIN_NONE; + + kvgpumgrAttachGpu(pNv->gpu_id); + + // + // Set up a reasonable default DMA address size, based on the minimum + // possible on currently supported GPUs. + // + nv_set_dma_address_size(pNv, NV_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH); + + os_mem_set(nvp, 0, sizeof(*nvp)); + nvp->status = NV_ERR_INVALID_STATE; + nvp->pmc_boot_0 = pmc_boot_0; + nvp->pmc_boot_42 = pmc_boot_42; + NV_SET_NV_PRIV(pNv, nvp); + + return NV_TRUE; +} + +void RmClearPrivateState( + nv_state_t *pNv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); + NvU32 status; + void *pVbiosCopy = NULL; + void *pRegistryCopy = NULL; + NvU32 vbiosSize; + NvU32 *pEfiDisplayCache; + NvU32 efiDisplayCacheSize; + nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS]; + nv_dynamic_power_t dynamicPowerCopy; + NvU32 x = 0; + NvU32 pmc_boot_0, pmc_boot_42; + + // + // Do not clear private state after GPU resets, it is used while + // recovering the GPU. Only clear the pGpu pointer, which is + // restored during next initialization cycle. + // + if (pNv->flags & NV_FLAG_IN_RECOVERY) + { + nvp->pGpu = NULL; + } + + status = nvp->status; + pVbiosCopy = nvp->pVbiosCopy; + vbiosSize = nvp->vbiosSize; + pRegistryCopy = nvp->pRegistry; + pEfiDisplayCache = nvp->efi.display.pCache; + efiDisplayCacheSize = nvp->efi.display.cacheSize; + dynamicPowerCopy = nvp->dynamic_power; + pmc_boot_0 = nvp->pmc_boot_0; + pmc_boot_42 = nvp->pmc_boot_42; + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + i2c_adapters[x] = nvp->i2c_adapters[x]; + } + + portMemSet(nvp, 0, sizeof(nv_priv_t)); + + nvp->status = status; + nvp->pVbiosCopy = pVbiosCopy; + nvp->vbiosSize = vbiosSize; + nvp->pRegistry = pRegistryCopy; + nvp->efi.display.pCache = pEfiDisplayCache; + nvp->efi.display.cacheSize = efiDisplayCacheSize; + nvp->dynamic_power = dynamicPowerCopy; + nvp->pmc_boot_0 = pmc_boot_0; + nvp->pmc_boot_42 = pmc_boot_42; + + for (x = 0; x < MAX_I2C_ADAPTERS; x++) + { + nvp->i2c_adapters[x] = i2c_adapters[x]; + } + + nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C; +} + +void RmFreePrivateState( + nv_state_t *pNv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); + + gpumgrUnregisterGpuId(pNv->gpu_id); + + kvgpumgrDetachGpu(pNv->gpu_id); + + RmDestroyRegistry(pNv); + + if (nvp != NULL) + { + portMemFree(nvp->pVbiosCopy); + portMemFree(nvp->efi.display.pCache); + os_free_mem(nvp); + } + + NV_SET_NV_PRIV(pNv, NULL); +} + +NvBool RmPartiallyInitAdapter( + nv_state_t *nv +) +{ + NV_PRINTF(LEVEL_INFO, "%s: %04x:%02x:%02x.0\n", __FUNCTION__, + nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot); + + nv_start_rc_timer(nv); + + return NV_TRUE; +} + +static NV_STATUS +RmInitX86Emu( + OBJGPU *pGpu +) +{ + NV_STATUS status = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + PORT_UNREFERENCED_VARIABLE(nv); + +#if NVCPU_IS_X86_64 + status = RmInitX86EmuState(pGpu); +#else + // We don't expect a "primary VGA" adapter on non-amd64 platforms + NV_ASSERT(!NV_PRIMARY_VGA(nv)); +#endif + + return status; +} + +static NV_STATUS RmRegisterGpudb( + OBJGPU *pGpu +) +{ + NV_STATUS rmStatus; + const NvU8 *pGid; + nv_state_t *pNv = NV_GET_NV_STATE(pGpu); + + pGid = RmGetGpuUuidRaw(pNv); + if (pGid == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to get UUID\n"); + return NV_ERR_OPERATING_SYSTEM; + } + + rmStatus = gpudbRegisterGpu(pGid, &pGpu->gpuClData.upstreamPort.addr, + pGpu->busInfo.nvDomainBusDeviceFunc); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Failed to register GPU with GPU data base\n"); + } + + return rmStatus; +} + +static void RmUnixFreeRmApi( + nv_state_t *nv +) +{ + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + if (nv->rmapi.hClient != 0) + { + pRmApi->Free(pRmApi, nv->rmapi.hClient, nv->rmapi.hClient); + } + + portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi)); +} + +static NvBool RmUnixAllocRmApi( + nv_state_t *nv, + NvU32 deviceId +) +{ + NV0080_ALLOC_PARAMETERS deviceParams = { 0 }; + NV2080_ALLOC_PARAMETERS subDeviceParams = { 0 }; + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi)); + + if (pRmApi->AllocWithHandle( + pRmApi, + NV01_NULL_OBJECT, + NV01_NULL_OBJECT, + NV01_NULL_OBJECT, + NV01_ROOT, + &nv->rmapi.hClient) != NV_OK) + { + goto fail; + } + + // + // Any call to rmapiDelPendingDevices() will internally delete the UNIX OS + // layer RMAPI handles. Set this flag to preserve these handles. These + // handles will be freed explicitly by RmUnixFreeRmApi(). + // + if (!rmclientSetClientFlagsByHandle(nv->rmapi.hClient, + RMAPI_CLIENT_FLAG_RM_INTERNAL_CLIENT)) + { + goto fail; + } + + deviceParams.deviceId = deviceId; + + if (pRmApi->Alloc( + pRmApi, + nv->rmapi.hClient, + nv->rmapi.hClient, + &nv->rmapi.hDevice, + NV01_DEVICE_0, + &deviceParams) != NV_OK) + { + goto fail; + } + + subDeviceParams.subDeviceId = 0; + + if (pRmApi->Alloc( + pRmApi, + nv->rmapi.hClient, + nv->rmapi.hDevice, + &nv->rmapi.hSubDevice, + NV20_SUBDEVICE_0, + &subDeviceParams) != NV_OK) + { + goto fail; + } + + // + // The NV40_I2C allocation expected to fail, if it is disabled + // with RM config. + // + if (pRmApi->Alloc( + pRmApi, + nv->rmapi.hClient, + nv->rmapi.hSubDevice, + &nv->rmapi.hI2C, + NV40_I2C, + NULL) != NV_OK) + { + nv->rmapi.hI2C = 0; + } + + // + // The NV04_DISPLAY_COMMON allocation expected to fail for displayless + // system. nv->rmapi.hDisp value needs to be checked before doing display + // related control calls. + // + if (pRmApi->Alloc( + pRmApi, + nv->rmapi.hClient, + nv->rmapi.hDevice, + &nv->rmapi.hDisp, + NV04_DISPLAY_COMMON, + NULL) != NV_OK) + { + nv->rmapi.hDisp = 0; + } + + return NV_TRUE; + +fail: + RmUnixFreeRmApi(nv); + return NV_FALSE; +} + +NvBool RmInitAdapter( + nv_state_t *nv +) +{ + NvU32 devicereference = 0; + UNIX_STATUS status = INIT_UNIX_STATUS; + nv_priv_t *nvp; + NvBool retVal = NV_FALSE; + OBJSYS *pSys; + OBJGPU *pGpu = NULL; + OBJOS *pOS; + KernelDisplay *pKernelDisplay; + const void *gspFwHandle = NULL; + const void *gspFwLogHandle = NULL; + + GSP_FIRMWARE gspFw = {0}; + PORT_UNREFERENCED_VARIABLE(gspFw); + + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter\n"); + + nv->flags &= ~NV_FLAG_PASSTHRU; + + RmSetupRegisters(nv, &status); + if (! RM_INIT_SUCCESS(status.initStatus) ) + goto failed; + + nvp = NV_GET_NV_PRIV(nv); + nvp->status = NV_ERR_OPERATING_SYSTEM; + + status.rmStatus = RmInitDeviceDma(nv); + if (status.rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Cannot configure the device for DMA\n"); + RM_SET_ERROR(status, RM_INIT_GPU_DMA_CONFIGURATION_FAILED); + goto shutdown; + } + + nvp->flags |= NV_INIT_FLAG_DMA; + + pSys = SYS_GET_INSTANCE(); + + // + // WAR: If the below UEFI property is set, display RM will attempt to read + // the state cache during RM init in order to retrieve a snapshot of the + // display state that the UEFI driver has already programmed. On Orin + // (T234D), the UEFI boot flow is being enabled on Linux, but our UEFI + // driver doesn't have any display support right now. As such, our UEFI + // driver won't allocate any of the display channels, which means that RM + // will attempt to read the state cache for uninitialized channels. WAR this + // issue by un-setting the below UEFI property for now. + // + // JIRA task TDS-5094 tracks adding display support to the UEFI driver. + // + if (NV_IS_SOC_DISPLAY_DEVICE(nv)) { + pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_FALSE); + } + + // + // Get firmware from the OS, if requested, and decide if RM will run as a + // firmware client. + // + if (nv->request_firmware) + { + NvU32 gpuArch = (DRF_VAL(_PMC, _BOOT_42, _ARCHITECTURE, nvp->pmc_boot_42) << + GPU_ARCH_SHIFT); + NvU32 gpuImpl = DRF_VAL(_PMC, _BOOT_42, _IMPLEMENTATION, nvp->pmc_boot_42); + + nv_firmware_chip_family_t chipFamily = nv_firmware_get_chip_family(gpuArch, gpuImpl); + + nv_set_dma_address_size(nv, NV_GSP_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH); + + gspFwHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP, + chipFamily, + &gspFw.pBuf, + &gspFw.size); + if (gspFwHandle == NULL && + !nv->allow_fallback_to_monolithic_rm) + { + RM_SET_ERROR(status, RM_INIT_FIRMWARE_FETCH_FAILED); + goto shutdown; + } + else if (gspFwHandle != NULL) + { +#if LIBOS_LOG_DECODE_ENABLE + if (nv->enable_firmware_logs) + { + gspFwLogHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP_LOG, + chipFamily, + &gspFw.pLogElf, + &gspFw.logElfSize); + if (gspFwLogHandle == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to load gsp_log_*.bin, no GSP-RM logs will be printed (non-fatal)\n"); + } + } +#endif + + nv->request_fw_client_rm = NV_TRUE; + } + else + { + nv->request_fw_client_rm = NV_FALSE; + } + } + + // initialize the RM device register mapping + osInitNvMapping(nv, &devicereference, &status); + if (! RM_INIT_SUCCESS(status.initStatus) ) + { + switch (status.rmStatus) + { + case NV_ERR_NOT_SUPPORTED: + nvp->status = NV_ERR_NOT_SUPPORTED; + break; + } + NV_PRINTF(LEVEL_ERROR, + "osInitNvMapping failed, bailing out of RmInitAdapter\n"); + goto shutdown; + } + + // + // now we can have a pdev for the first time... + // + pGpu = gpumgrGetGpu(devicereference); + + pOS = SYS_GET_OS(pSys); + + status.rmStatus = osInitScalability(pGpu); + if (status.rmStatus == NV_OK) + { + nvp->flags |= NV_INIT_FLAG_SCALABILITY; + } + else if (status.rmStatus != NV_ERR_NOT_SUPPORTED) + { + RM_SET_ERROR(status, RM_INIT_SCALABILITY_FAILED); + goto shutdown; + } + + KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu); + if ((pKernelFsp != NULL) && !IS_GSP_CLIENT(pGpu) && !IS_VIRTUAL(pGpu)) + { + status.rmStatus = kfspSendBootCommands_HAL(pGpu, pKernelFsp); + if (status.rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "FSP boot command failed.\n"); + goto shutdown; + } + } + + RmSetConsolePreservationParams(pGpu); + + RmInitAcpiMethods(pOS, pSys, pGpu); + + // + // If GSP fw RM support is enabled then start the GSP microcode + // (including the task running the full instance of the RM) and + // exchange the necessary initial RPC messages before continuing + // with GPU initialization here. + // + if (IS_GSP_CLIENT(pGpu)) + { + status.rmStatus = kgspInitRm(pGpu, GPU_GET_KERNEL_GSP(pGpu), &gspFw); + if (status.rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Cannot initialize GSP firmware RM\n"); + RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); + goto shutdown; + } + } + else if (nv->request_fw_client_rm) + { + // We were expecting to enable GSP-RM but something went wrong. + if (!nv->allow_fallback_to_monolithic_rm) + { + RM_SET_ERROR(status, RM_INIT_FIRMWARE_POLICY_FAILED); + goto shutdown; + } + else + { + NV_PRINTF(LEVEL_NOTICE, "Falling back to monolithic RM\n"); + } + } + + pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + if (pKernelDisplay != NULL) + { + kdispSetWarPurgeSatellitesOnCoreFree(pKernelDisplay, NV_TRUE); + } + + if (IS_PASSTHRU(pGpu)) + nv->flags |= NV_FLAG_PASSTHRU; + + populateDeviceAttributes(pGpu, nv); + + initVendorSpecificRegistry(pGpu, nv->pci_info.device_id); + if (!IS_VIRTUAL(pGpu) && !IS_GSP_CLIENT(pGpu)) + { + initNbsiTable(pGpu); + } + + // finally, initialize the device + RmInitNvDevice(devicereference, &status); + if (! RM_INIT_SUCCESS(status.initStatus) ) + { + NV_PRINTF(LEVEL_ERROR, + "RmInitNvDevice failed, bailing out of RmInitAdapter\n"); + switch (status.rmStatus) + { + case NV_ERR_INSUFFICIENT_POWER: + nvp->status = NV_ERR_INSUFFICIENT_POWER; + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, + "GPU does not have the necessary power cables connected.\n"); + break; + } + goto shutdown; + } + + // LOCK: acquire GPUs lock + status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, + RM_LOCK_MODULES_INIT); + if (status.rmStatus != NV_OK) + { + goto shutdown; + } + + status.rmStatus = osVerifySystemEnvironment(pGpu); + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_INIT_SYS_ENVIRONMENT_FAILED); + switch (status.rmStatus) + { + case NV_ERR_IRQ_NOT_FIRING: + nvp->status = NV_ERR_IRQ_NOT_FIRING; + break; + } + NV_PRINTF(LEVEL_ERROR, "RmVerifySystemEnvironment failed, bailing!\n"); + goto shutdown; + } + + Intr *pIntr = GPU_GET_INTR(pGpu); + if (pIntr != NULL) + { + intrSetIntrEn(pIntr, INTERRUPT_TYPE_HARDWARE); + } + + KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); + // initialize the watchdog (disabled by default) + status.rmStatus = pKernelRc != NULL ? krcWatchdogInit_HAL(pGpu, pKernelRc) : + NV_ERR_NOT_SUPPORTED; + if (status.rmStatus == NV_OK) + { + krcWatchdogDisable(pKernelRc); + nvp->flags |= NV_INIT_FLAG_FIFO_WATCHDOG; + } + else if (status.rmStatus == NV_ERR_NOT_SUPPORTED) + { + NV_PRINTF(LEVEL_INFO, + "krcWatchdogInit returned _NOT_SUPPORTED. For Kepler GPUs in PGOB mode, this is normal\n"); + } + else + { + RM_SET_ERROR(status, RM_INIT_WATCHDOG_FAILED); + NV_PRINTF(LEVEL_ERROR, + "krcWatchdogInit failed, bailing out of RmInitAdapter\n"); + goto shutdown; + } + nv_start_rc_timer(nv); + + nvp->status = NV_OK; + + if (!RmUnixAllocRmApi(nv, devicereference)) { + RM_SET_ERROR(status, RM_INIT_ALLOC_RMAPI_FAILED); + status.rmStatus = NV_ERR_GENERIC; + goto shutdown; + } + + status.rmStatus = RmInitGpuInfoWithRmApi(pGpu); + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_INIT_GPUINFO_WITH_RMAPI_FAILED); + goto shutdown; + } + + status.rmStatus = RmInitX86Emu(pGpu); + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_INIT_VBIOS_X86EMU_FAILED); + NV_PRINTF(LEVEL_ERROR, + "RmInitX86Emu failed, bailing out of RmInitAdapter\n"); + goto shutdown; + } + + // i2c only on master device?? + RmI2cAddGpuPorts(nv); + nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C; + + // This fifo hal call will fail on pre-fermi gpus. In that case, userd + // info will remain 0 and skipped by fb mmap code + KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); + NvU64 udaddr = 0; + NvU32 udsize = 0; + if (pKernelFifo != NULL) + { + status.rmStatus = kfifoGetUserdBar1MapInfo_HAL(pGpu, pKernelFifo, &udaddr, &udsize); + if (status.rmStatus == NV_OK) + { + nv->ud.cpu_address = (nv->fb->cpu_address + udaddr); + nv->ud.size = udsize; + } + else if (status.rmStatus != NV_ERR_NOT_SUPPORTED) + { + RM_SET_ERROR(status, RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED); + NV_PRINTF(LEVEL_ERROR, + "kfifoGetUserdBar1MapInfo failed, bailing out of RmInitAdapter\n"); + goto shutdown; + } + } + + nv->flags &= ~NV_FLAG_IN_RECOVERY; + + pOS->setProperty(pOS, PDB_PROP_OS_SYSTEM_EVENTS_SUPPORTED, NV_TRUE); + + RmInitS0ixPowerManagement(nv); + RmInitDeferredDynamicPowerManagement(nv); + + if (!NV_IS_SOC_DISPLAY_DEVICE(nv) && !NV_IS_SOC_IGPU_DEVICE(nv)) + { + status.rmStatus = RmRegisterGpudb(pGpu); + if (status.rmStatus != NV_OK) + { + RM_SET_ERROR(status, RM_GPUDB_REGISTER_FAILED); + goto shutdown; + } + } + + if (nvp->b_mobile_config_enabled) + { + NvU32 ac_plugged = 0; + if (nv_acpi_get_powersource(&ac_plugged) == NV_OK) + { + // LOCK: acquire GPU lock + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_NONE) == NV_OK) + { + // + // As we have already acquired the API Lock here, we are calling + // RmPowerSourceChangeEvent directly instead of rm_power_source_change_event. + // + RmPowerSourceChangeEvent(nv, !ac_plugged); + + // UNLOCK: release GPU lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + } + } + + { + // OpenRM support for features beyond what is used on Data Center GPUs + // is still fairly immature, so for now require users to opt into use of + // OpenRM with a special registry key, if not on a Data Center GPU. + const GspStaticConfigInfo *pSCI = GPU_GET_GSP_STATIC_INFO(pGpu); + + if (pSCI->computeBranding != COMPUTE_BRANDING_TYPE_TESLA && + ((pGpu->idInfo.PCIDeviceID >> 16) & 0xffff) != NV_PCI_DEVID_DEVICE_PG189_SKU600) + { + NvU32 data = NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DEFAULT; + RmReadRegistryDword(nv, NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS, &data); + + if (data == NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DISABLE) + { + if (!nv->printed_openrm_enable_unsupported_gpus_error) + { + nv_printf(NV_DBG_ERRORS, + "NVRM: Open nvidia.ko is only ready for use on Data Center GPUs.\n"); + nv_printf(NV_DBG_ERRORS, + "NVRM: To force use of Open nvidia.ko on other GPUs, see the\n"); + nv_printf(NV_DBG_ERRORS, + "NVRM: 'OpenRmEnableUnsupportedGpus' kernel module parameter described\n"); + nv_printf(NV_DBG_ERRORS, + "NVRM: in the README.\n"); + nv->printed_openrm_enable_unsupported_gpus_error = NV_TRUE; + } + RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); + goto shutdown; + } + } + } + + NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter succeeded!\n"); + + retVal = NV_TRUE; + goto done; + + shutdown: + nv->flags &= ~NV_FLAG_IN_RECOVERY; + + // call ShutdownAdapter to undo anything we've done above + RmShutdownAdapter(nv); + + failed: + NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "RmInitAdapter failed! (0x%x:0x%x:%d)\n", + status.initStatus, status.rmStatus, status.line); + +done: + nv_put_firmware(gspFwHandle); + nv_put_firmware(gspFwLogHandle); + + return retVal; +} + +void RmShutdownAdapter( + nv_state_t *nv +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + NV_STATUS rmStatus; + + if ((pGpu != NULL) && (nvp->flags & NV_INIT_FLAG_GPUMGR_ATTACH)) + { + NvU32 gpuInstance = gpuGetInstance(pGpu); + NvU32 deviceInstance = gpuGetDeviceInstance(pGpu); + OBJSYS *pSys = SYS_GET_INSTANCE(); + + RmUnixFreeRmApi(nv); + + nv->ud.cpu_address = 0; + nv->ud.size = 0; + + // LOCK: acquire GPUs lock + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + RmDestroyDeferredDynamicPowerManagement(nv); + + freeNbsiTable(pGpu); + + gpuFreeEventHandle(pGpu); + + OBJCL *pCl = SYS_GET_CL(pSys); + OBJOS *pOS = GPU_GET_OS(pGpu); + if (pCl != NULL) + { + if (nvp->flags & NV_INIT_FLAG_CORE_LOGIC) + { + teardownCoreLogic(pOS, pGpu); + } + else if (nvp->flags & NV_INIT_FLAG_SCALABILITY) + { + osTeardownScalability(pGpu); + } + } + + rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance)); + rmapiDelPendingDevices(NVBIT(gpuInstance)); + + os_disable_console_access(); + + if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) + { + rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT); + NV_ASSERT(rmStatus == NV_OK); + } + + if (nvp->flags & NV_INIT_FLAG_GPU_STATE) + { + rmStatus = gpuStateDestroy(pGpu); + NV_ASSERT(rmStatus == NV_OK); + } + + os_enable_console_access(); + + //if (nvp->flags & NV_INIT_FLAG_HAL) + // destroyHal(pDev); + +#if NVCPU_IS_X86_64 + RmFreeX86EmuState(pGpu); +#endif + + gpumgrDetachGpu(gpuInstance); + gpumgrDestroyDevice(deviceInstance); + + if (nvp->flags & NV_INIT_FLAG_DMA) + { + RmTeardownDeviceDma(nv); + } + + RmClearPrivateState(nv); + + RmUnInitAcpiMethods(pSys); + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + + // RM_BASIC_LOCK_MODEL: free GPU lock + rmGpuLockFree(deviceInstance); + } + } + else + { + RmClearPrivateState(nv); + } + + RmTeardownRegisters(nv); +} + +void RmPartiallyDisableAdapter( + nv_state_t *nv +) +{ + NV_PRINTF(LEVEL_INFO, "%s: RM is in SW Persistence mode\n", __FUNCTION__); + + nv_stop_rc_timer(nv); +} + +void RmDisableAdapter( + nv_state_t *nv +) +{ + NV_STATUS rmStatus; + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); + NvU32 gpuMask; + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + // + // Normally, we re-enable interrupts when we release the lock + // after RmDisableAdapter(), before disabling the bottom-half ISR. + // If an interrupt came in in that time window, but the bottom half + // wasn't scheduled to run in time, the semaphore acquired by the + // top-half would never be released afterwards (since the bottom-half + // would never run). This call to intrSetIntrEn() ensures that this won't + // happen, by making sure interrupts remain disabled on the GPU in question after + // RmDisableAdapter(). + // + Intr *pIntr = GPU_GET_INTR(pGpu); + if (pIntr != NULL) + { + intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SECONDARY_BUS_RESET) || + pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_FULLCHIP_RESET)) + { + pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_TIMEOUT_RECOVERY, NV_TRUE); + nv->flags |= NV_FLAG_IN_RECOVERY; + } + + // + // Free the client allocated resources. + // + // This needs to happen prior to tearing down SLI state when SLI is enabled. + // + // Note this doesn't free RM internal resource allocations. Those are + // freed during (gpumgrUpdateSLIConfig->...->)gpuStateUnload. + // + // We need to free resources for all GPUs linked in a group as + // gpumgrUpdateSLIConfig will teardown GPU state for the entire set. + // + gpuMask = gpumgrGetGpuMask(pGpu); + + rmapiSetDelPendingClientResourcesFromGpuMask(gpuMask); + rmapiDelPendingDevices(gpuMask); + + // LOCK: acquire GPUs lock + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) + { + nv_stop_rc_timer(nv); + + OBJSYS *pSys = SYS_GET_INSTANCE(); + OBJCL *pCl = SYS_GET_CL(pSys); + OBJOS *pOS = SYS_GET_OS(pSys); + if (pCl != NULL) + { + teardownCoreLogic(pOS, pGpu); + nvp->flags &= ~NV_INIT_FLAG_CORE_LOGIC; + } + + if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG) + { + krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu)); + nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG; + } + + os_disable_console_access(); + + if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) + { + rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT); + NV_ASSERT(rmStatus == NV_OK); + nvp->flags &= ~NV_INIT_FLAG_GPU_STATE_LOAD; + } + + os_enable_console_access(); + + // UNLOCK: release GPUs lock + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } +} + +NV_STATUS RmGetAdapterStatus( + nv_state_t *pNv, + NvU32 *pStatus +) +{ + // + // This status is determined in RmInitAdapter(); the glue layer + // requests it when the adapter failed to initialize to learn + // more about the error condition. This is currently limited to + // osVerifySystemEnvironment() failures. + // + nv_priv_t *nvp; + + nvp = NV_GET_NV_PRIV(pNv); + if (nvp == NULL) + { + return NV_ERR_INVALID_STATE; + } + + *pStatus = nvp->status; + return NV_OK; +} + +static void initVendorSpecificRegistry( + OBJGPU *pGpu, + NvU16 device_id +) +{ + NV_STATUS rmStatus; + NvU32 i; + NvU32 subsystem_id; + NvU32 subsystem_vendor_id; + NvU32 subsystem_device_id; + NvU32 vendor_id = 0; + + if (!pGpu) + return; + + rmStatus = GPU_BUS_CFG_RD32(pGpu, + NV_CONFIG_PCI_NV_11, &subsystem_id); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s: Cannot read NV_CONFIG_PCI_NV_11\n", __FUNCTION__); + return; + } + + subsystem_vendor_id = (subsystem_id & 0xffff); + subsystem_device_id = (subsystem_id >> 16); + + for (i = 0; (nb_id_table[i].subsystem_vendor_id) != 0; i++) + { + if ((nb_id_table[i].subsystem_vendor_id == subsystem_vendor_id) && + (nb_id_table[i].subsystem_device_id == subsystem_device_id) && + (nb_id_table[i].gpu_device_id == device_id)) + { + vendor_id = subsystem_vendor_id; + break; + } + } + + if (vendor_id != 0) + { + for (i = 0; nb_reg_table[i].vendor_id != 0; i++) + { + if (nb_reg_table[i].vendor_id == vendor_id) + { + osWriteRegistryDword(pGpu, nb_reg_table[i].name, + nb_reg_table[i].data); + } + } + } +} + +static void initUnixSpecificRegistry( + OBJGPU *pGpu +) +{ + // By default, enable GPU reset on Unix + osWriteRegistryDword(pGpu, "RMSecBusResetEnable", 1); + osWriteRegistryDword(pGpu, "RMForcePcieConfigSave", 1); + +} + +void +osRemoveGpu( + NvU32 domain, + NvU8 bus, + NvU8 device +) +{ + void *handle; + + handle = os_pci_init_handle(domain, bus, device, 0, NULL, NULL); + if (handle != NULL) + { + os_pci_remove(handle); + } +} + +/* + * Check to see if this board supports GPU exclusion. + */ +static NvBool RmIsExcludingAllowed( + nv_state_t *pNv +) +{ + NvU32 feature = 0; + NvU32 brand; + + // DGX-2/HGX-2 systems pre-date the PBI call + if (pNv->pci_info.device_id == 0x1db8) + return NV_TRUE; + + if (pciPbiGetFeature(pNv->handle, &feature) != NV_OK) + return NV_FALSE; + + brand = DRF_VAL(_PBI, _EXECUTE_ROUTINE, _GET_FEATURE_EXCLUSION, feature); + + return (brand == NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION_ALLOWED); +} + +NV_STATUS RmExcludeAdapter( + nv_state_t *nv +) +{ + NV_STATUS rmStatus; + + if (!RmIsExcludingAllowed(nv)) + return NV_ERR_NOT_SUPPORTED; + + rmStatus = gpumgrExcludeGpuId(nv->gpu_id); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "failed to exclude GPU: 0x%x\n", rmStatus); + return rmStatus; + } + + /* + * GPU exclusion only happens during initialization, and therefore there + * is no need to stop activity on the GPU + */ + + return rmStatus; +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osmemdesc.c b/userspace/sdks/nv525/nvalloc/unix/src/osmemdesc.c new file mode 100644 index 0000000..ad49cf7 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/osmemdesc.c @@ -0,0 +1,1030 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2012-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/******************* OS Memory Descriptor APIS *****************************\ +* * +* This contains routines to create and destroy OS memory descriptor * +* * +****************************************************************************/ + +#include <nv.h> // NV device driver interface +#include <os/os.h> +#include <osapi.h> +#include <nvos.h> +#include <rmapi/client.h> +#include <rmapi/rs_utils.h> +#include <gpu_mgr/gpu_mgr.h> + +#include "gpu/bif/kernel_bif.h" + +static NV_STATUS osCreateOsDescriptorFromPageArray(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void **); +static void osDestroyOsDescriptorPageArray(PMEMORY_DESCRIPTOR); + +static NV_STATUS osCreateOsDescriptorFromIoMemory(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static void osDestroyOsDescriptorFromIoMemory(PMEMORY_DESCRIPTOR); + +static NV_STATUS osCreateOsDescriptorFromPhysAddr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static void osDestroyOsDescriptorFromPhysAddr(PMEMORY_DESCRIPTOR); + +static NV_STATUS osCreateOsDescriptorFromFileHandle(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static NV_STATUS osCreateOsDescriptorFromDmaBufPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static void osDestroyOsDescriptorFromDmaBuf(PMEMORY_DESCRIPTOR); +static NV_STATUS osCreateOsDescriptorFromSgtPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**); +static void osDestroyOsDescriptorFromSgt(PMEMORY_DESCRIPTOR); + +static NV_STATUS osCheckGpuBarsOverlapAddrRange(NvRangeU64 addrRange); + +NV_STATUS +osCreateMemFromOsDescriptor +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + NvU32 descriptorType, + RS_PRIV_LEVEL privilegeLevel +) +{ + RmClient* pClient; + NV_STATUS rmStatus; + void *pPrivate; + + if ((pDescriptor == NvP64_NULL) || + (*pLimit == 0) || + (serverutilGetClientUnderLock(hClient, &pClient) != NV_OK)) + { + return NV_ERR_INVALID_PARAM_STRUCT; + } + + // + // For the sake of simplicity, unmatched RM and OS page + // sizes are not currently supported in this path, except for + // PPC64LE and aarch64. + // + // Also, the nvmap handle is sent which can be any random number so + // the virtual address alignment sanity check can't be done here. + // + if (!NVCPU_IS_PPC64LE && + !NVCPU_IS_AARCH64 && + (NV_RM_PAGE_SIZE != os_page_size)) + { + return NV_ERR_NOT_SUPPORTED; + } + + // + // The two checks below use cached privilege because they + // concern the privilege level of the client, and not the + // privilege level of the calling context which may be + // overridden to KERNEL at some internal callsites. + // + + // + // The RM cannot obtain a table of physical addresses + // for a kernel virtual address range on all of + // the supported UNIX platforms. Since this path is + // not really compelling for kernel allocations on any + // of those platforms, it is not supported. + // For UVM, they could have pre-allocated sysmem to register + // with RM so we put in an exception for that case. + // + if ((rmclientGetCachedPrivilege(pClient) >= RS_PRIV_LEVEL_KERNEL) && + (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR) && + (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE) && + (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) && + (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR)) + { + return NV_ERR_NOT_SUPPORTED; + } + + // + // NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR and + // NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR can only be utilized by kernel space + // rm-clients. + // + if ((rmclientGetCachedPrivilege(pClient) < RS_PRIV_LEVEL_KERNEL) && + ((descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) || + (descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR))) + { + return NV_ERR_NOT_SUPPORTED; + } + + switch (descriptorType) + { + case NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS: + rmStatus = NV_ERR_NOT_SUPPORTED; + break; + case NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR: + if (privilegeLevel < RS_PRIV_LEVEL_KERNEL) + { + rmStatus = NV_ERR_NOT_SUPPORTED; + break; + } + rmStatus = osCreateOsDescriptorFromPhysAddr(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY: + rmStatus = osCreateOsDescriptorFromIoMemory(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY: + rmStatus = osCreateOsDescriptorFromPageArray(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE: + rmStatus = osCreateOsDescriptorFromFileHandle(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR: + rmStatus = osCreateOsDescriptorFromDmaBufPtr(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + case NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR: + rmStatus = osCreateOsDescriptorFromSgtPtr(pGpu, pDescriptor, + hClient, flags, pLimit, ppMemDesc, &pPrivate); + break; + default: + rmStatus = NV_ERR_INVALID_ARGUMENT; + break; + } + + return rmStatus; +} + +static NV_STATUS +osCreateMemdescFromPages +( + OBJGPU *pGpu, + NvU64 size, + NvU32 flags, + NvU32 cacheType, + MEMORY_DESCRIPTOR **ppMemDesc, + void *pImportPriv, + void **ppPrivate +) +{ + NV_STATUS rmStatus; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 memdescFlags = MEMDESC_FLAGS_NONE; + NvU32 gpuCachedFlags; + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO; + } + + rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0, + NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM, + cacheType, memdescFlags); + if (rmStatus != NV_OK) + { + return rmStatus; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags)) + gpuCachedFlags = NV_MEMORY_CACHED; + else + gpuCachedFlags = NV_MEMORY_UNCACHED; + + pMemDesc = *ppMemDesc; + rmStatus = nv_register_user_pages(NV_GET_NV_STATE(pGpu), + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + memdescGetPteArray(pMemDesc, AT_CPU), pImportPriv, + ppPrivate); + if (rmStatus != NV_OK) + { + memdescDestroy(pMemDesc); + return rmStatus; + } + + memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags); + memdescSetAddress(pMemDesc, NvP64_NULL); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE); + + // + // If the OS layer doesn't think in RM page size, we need to inflate the + // PTE array into RM pages. + // + if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + // + // memdescMapIommu() requires the OS-private data to be set on the memory + // descriptor, but we don't want to wire up the teardown callback just yet: + // that callback needs to unpin the pages, but that will already be done + // as part of failure handling further up the stack if memdescMapIommu() + // fails. So we only set up the priv-data cleanup callback once we're sure + // this call will succeed. + // + memdescSetMemData(pMemDesc, *ppPrivate, NULL); + + rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); + if (rmStatus != NV_OK) + { + if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + nv_unregister_user_pages(NV_GET_NV_STATE(pGpu), + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + NULL /* import_priv */, ppPrivate); + memdescDestroy(pMemDesc); + return rmStatus; + } + + return NV_OK; +} + +static NV_STATUS +osCreateOsDescriptorFromPageArray +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus; + + *ppPrivate = NvP64_VALUE(pDescriptor); + + // + // Since the only type of memory permitted in this path + // is anonymous user memory, certain restrictions + // apply for the allocation flags: + // + // 1) anonymous memory is write-back cacheable, hence + // the _COHERENCY flag must match. + // + // 2) the RM has no control over the location of the + // associated pages in memory and thus cannot + // honor requests for contiguous memory. + // + // These restrictions are enforced here to avoid subtle + // bugs later on. + // + if ((!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, flags) && + !FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, flags)) || + FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags)) + { + return NV_ERR_INVALID_FLAGS; + } + + rmStatus = osCreateMemdescFromPages(pGpu, (*pLimit + 1), flags, + NV_MEMORY_CACHED, ppMemDesc, + NULL /* pImportPriv */, ppPrivate); + if (rmStatus != NV_OK) + { + return rmStatus; + } + + // All is well - wire up the cleanup callback now + memdescSetMemData(*ppMemDesc, memdescGetMemData(*ppMemDesc), + osDestroyOsDescriptorPageArray); + + return NV_OK; +} + +/*! + * @brief Checks if the given address range overlaps with the BARs for any of + * the GPUs. + */ +static NV_STATUS +osCheckGpuBarsOverlapAddrRange +( + NvRangeU64 addrRange +) +{ + NvRangeU64 gpuPhysAddrRange; + NvRangeU64 gpuPhysFbAddrRange; + NvRangeU64 gpuPhysInstAddrRange; + NvU32 gpuInstance; + OBJGPU *pGpu; + NvU32 gpuMask; + NV_STATUS rmStatus; + + rmStatus = gpumgrGetGpuAttachInfo(NULL, &gpuMask); + NV_ASSERT_OR_RETURN(rmStatus == NV_OK, rmStatus); + + gpuInstance = 0; + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + NV_INIT_RANGE(gpuPhysFbAddrRange, gpumgrGetGpuPhysFbAddr(pGpu), + gpumgrGetGpuPhysFbAddr(pGpu) + pGpu->fbLength -1); + + NV_INIT_RANGE(gpuPhysAddrRange, pGpu->busInfo.gpuPhysAddr, + pGpu->busInfo.gpuPhysAddr + pGpu->deviceMappings[0].gpuNvLength -1); + + NV_INIT_RANGE(gpuPhysInstAddrRange, pGpu->busInfo.gpuPhysInstAddr, + pGpu->busInfo.gpuPhysInstAddr + pGpu->instLength -1); + + if (NV_IS_OVERLAPPING_RANGE(gpuPhysFbAddrRange, addrRange) || + NV_IS_OVERLAPPING_RANGE(gpuPhysAddrRange, addrRange) || + NV_IS_OVERLAPPING_RANGE(gpuPhysInstAddrRange, addrRange)) + { + return NV_ERR_INVALID_ADDRESS; + } + } + + return NV_OK; +} + +static NV_STATUS +osCreateOsDescriptorFromIoMemory +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus; + NvU32 gpuCachedFlags; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 *pPteArray; + NvRangeU64 physAddrRange; + NvU64 *base = 0; + NvBool bAllowMmap; + + // + // Unlike the page array path, this one deals exclusively + // with I/O memory, which is expected to be contiguous + // physically, and which may only be accessed with uncached + // transactions. + // + if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags) || + !FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags)) + { + return NV_ERR_INVALID_FLAGS; + } + + // + // _PEER_MAP_OVERRIDE flag is controlled by the RM and not the client. + // + // RM will set the _PEER_MAP_OVERRIDE_REQUIRED flag itself for IO memory + // memory imported with RmVidHeapControl. + // + if (FLD_TEST_DRF(OS02, _FLAGS, _PEER_MAP_OVERRIDE, _REQUIRED, flags)) + { + // + // Don't allow MMIO mappings for unprivileged users + // This is a temporary WAR for bug 1630288 "[PeerSync] threat related + // to GPU" + // + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + if (!pKernelBif->peerMappingOverride && !osIsAdministrator()) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): permission denied, allowPeermapping=%d\n", + __FUNCTION__, pKernelBif->peerMappingOverride); + return NV_ERR_INSUFFICIENT_PERMISSIONS; + } + } + + bAllowMmap = !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NEVER_MAP, flags); + + base = (void *)(NvUPtr)pDescriptor; + + // + // There is an architectural deadlock scenario involved when full-duplex P2P + // enabled over BAR1. See #3 in the description of bug 1571948 which explains + // the classic deadlock. So, make sure to error out usermode's memory + // registration if a memory range falls within any of the available GPU's + // BAR window. + // + physAddrRange.min = *base; + physAddrRange.max = *base + *pLimit; + + rmStatus = osCheckGpuBarsOverlapAddrRange(physAddrRange); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): phys range 0x%016llx-0x%016llx overlaps with GPU BARs", + __FUNCTION__, physAddrRange.min, physAddrRange.max); + return rmStatus; + } + + rmStatus = memdescCreate(ppMemDesc, pGpu, (*pLimit + 1), 0, + NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM, + NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): error %d while attempting to create the MMIO mapping\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + pMemDesc = *ppMemDesc; + + if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags)) + gpuCachedFlags = NV_MEMORY_CACHED; + else + gpuCachedFlags = NV_MEMORY_UNCACHED; + + memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags); + memdescSetAddress(pMemDesc, NvP64_NULL); + memdescSetMemData(pMemDesc, NULL, NULL); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM, NV_TRUE); + + pPteArray = memdescGetPteArray(pMemDesc, AT_CPU); + pPteArray[0] = *base; + + *ppPrivate = NULL; + + if (bAllowMmap) + { + rmStatus = nv_register_peer_io_mem(NV_GET_NV_STATE(pGpu), pPteArray, + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + ppPrivate); + if (rmStatus != NV_OK) + { + memdescDestroy(pMemDesc); + return rmStatus; + } + } + + memdescSetMemData(pMemDesc, *ppPrivate, NULL); + + // + // memdescMapIommu() requires the OS-private data to be set on the memory + // descriptor, but we don't want to wire up the teardown callback just yet: + // that callback needs to unpin the pages, but that will already be done + // as part of failure handling further up the stack if memdescMapIommu() + // fails. So we only set up the priv-data cleanup callback once we're sure + // this call will succeed. + // + rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); + if (rmStatus != NV_OK) + { + if (*ppPrivate != NULL) + { + nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), *ppPrivate); + } + memdescDestroy(pMemDesc); + return rmStatus; + } + + // All is well - wire up the cleanup callback now + memdescSetMemData(pMemDesc, memdescGetMemData(pMemDesc), + osDestroyOsDescriptorFromIoMemory); + + return NV_OK; +} + +static NV_STATUS +osCreateOsDescriptorFromPhysAddr +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 *pPteArray; + NvU64 base = 0; + NvU32 cache_type = NV_MEMORY_CACHED; + NvU64 memdescFlags = MEMDESC_FLAGS_NONE; + NvU64 *pPhys_addrs; + NvU64 num_os_pages; + NvU32 idx; + // Currently only work with contiguous sysmem allocations + if (!FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags)) + { + return NV_ERR_INVALID_FLAGS; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_TYPE_SYNCPOINT, _APERTURE, flags)) + { + // Syncpoint memory is uncached, DMA mapping needs to skip CPU sync. + cache_type = NV_MEMORY_UNCACHED; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO; + } + + base = (NvU64)pDescriptor; + rmStatus = memdescCreate(ppMemDesc, pGpu, (*pLimit + 1), 0, + NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM, + cache_type, memdescFlags); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): error %d while creating memdesc for kernel memory\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + pMemDesc = *ppMemDesc; + + memdescSetAddress(pMemDesc, NvP64_NULL); + memdescSetMemData(pMemDesc, NULL, NULL); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE); + + pPteArray = memdescGetPteArray(pMemDesc, AT_CPU); + pPteArray[0] = base; + + num_os_pages = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount); + pPhys_addrs = portMemAllocNonPaged(sizeof(NvU64) * num_os_pages); + if (pPhys_addrs == NULL) + return NV_ERR_NO_MEMORY; + + for (idx = 0; idx < num_os_pages; idx++) + { + pPhys_addrs[idx] = base + (idx * os_page_size); + } + + *ppPrivate = NULL; + rmStatus = nv_register_phys_pages(NV_GET_NV_STATE(pGpu), pPhys_addrs, + num_os_pages, + memdescGetCpuCacheAttrib(pMemDesc), + ppPrivate); + if (rmStatus != NV_OK) + { + memdescDestroy(pMemDesc); + goto cleanup; + } + + memdescSetMemData(pMemDesc, *ppPrivate, + osDestroyOsDescriptorFromPhysAddr); + +cleanup: + portMemFree(pPhys_addrs); + + return rmStatus; +} + +static NV_STATUS +_createMemdescFromDmaBufSgtHelper +( + OBJGPU *pGpu, + NvU32 flags, + void *pImportPriv, + struct sg_table *pImportSgt, + NvU32 size, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate, + MEM_DATA_RELEASE_CALL_BACK *pMemDataReleaseCallback +) +{ + NV_STATUS rmStatus = NV_OK; + NvU32 cacheType = NV_MEMORY_UNCACHED; + MEMORY_DESCRIPTOR *pMemDesc; + NvU64 memdescFlags = MEMDESC_FLAGS_NONE; + NvU32 gpuCachedFlags; + + NV_ASSERT((pMemDataReleaseCallback == osDestroyOsDescriptorFromDmaBuf) || + (pMemDataReleaseCallback == osDestroyOsDescriptorFromSgt)); + + if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_COMBINE, flags)) + { + cacheType = NV_MEMORY_WRITECOMBINED; + } + else if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags)) + { + cacheType = NV_MEMORY_CACHED; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags)) + { + memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO; + } + + rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0, + NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM, + cacheType, memdescFlags); + if (rmStatus != NV_OK) + { + return rmStatus; + } + + if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags)) + { + gpuCachedFlags = NV_MEMORY_CACHED; + } + else + { + gpuCachedFlags = NV_MEMORY_UNCACHED; + } + + pMemDesc = *ppMemDesc; + + memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags); + memdescSetAddress(pMemDesc, NvP64_NULL); + memdescSetMemData(pMemDesc, NULL, NULL); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE); + + *ppPrivate = NULL; + rmStatus = nv_register_sgt(NV_GET_NV_STATE(pGpu), + memdescGetPteArray(pMemDesc, AT_CPU), + NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount), + memdescGetCpuCacheAttrib(pMemDesc), + ppPrivate, + pImportSgt, + pImportPriv); + if (rmStatus != NV_OK) + { + memdescDestroy(pMemDesc); + return rmStatus; + } + + // + // If the OS layer doesn't think in RM page size, we need to inflate the + // PTE array into RM pages. + // + if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + memdescSetMemData(*ppMemDesc, *ppPrivate, NULL); + + // + // memdescMapIommu() requires the OS-private data to be set on the memory + // descriptor, but we don't want to wire up the teardown callback just yet: + // that callback does teardown that will already be done as part of failure + // handling further up the stack if memdescMapIommu() fails. So we only + // setup the priv-data cleanup callback once we're sure this call will + // succeed. + // + rmStatus = memdescMapIommu(*ppMemDesc, pGpu->busInfo.iovaspaceId); + if (rmStatus != NV_OK) + { + if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + if (*ppPrivate != NULL) + { + nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt, + (void **) &pImportPriv, *ppPrivate); + } + memdescDestroy(pMemDesc); + return rmStatus; + } + + // All is well - wire up the cleanup callback now + memdescSetMemData(*ppMemDesc, *ppPrivate, pMemDataReleaseCallback); + + return rmStatus; +} + +static NV_STATUS +_createMemdescFromDmaBuf +( + OBJGPU *pGpu, + NvU32 flags, + nv_dma_buf_t *pImportPriv, + void *pUserPages, + struct sg_table *pImportSgt, + NvU32 size, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = + _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPriv, pImportSgt, + size, ppMemDesc, ppPrivate, + osDestroyOsDescriptorFromDmaBuf); + + NV_ASSERT(pUserPages == NULL); + + if (rmStatus != NV_OK) + { + nv_dma_release_dma_buf(NULL, pImportPriv); + } + + return rmStatus; +} + +static NV_STATUS +_createMemdescFromSgt +( + OBJGPU *pGpu, + NvU32 flags, + struct drm_gem_object *pImportPrivGem, + struct sg_table *pImportSgt, + NvU32 size, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = + _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPrivGem, + pImportSgt, size, ppMemDesc, + ppPrivate, + osDestroyOsDescriptorFromSgt); + if (rmStatus != NV_OK) + { + nv_dma_release_sgt(pImportSgt, pImportPrivGem); + } + + return rmStatus; +} + +static NV_STATUS +osCreateOsDescriptorFromFileHandle +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NvU32 size = 0; + void *pUserPages = NULL; + nv_dma_buf_t *pImportPriv = NULL; + struct sg_table *pImportSgt = NULL; + NvS32 fd; + + fd = (NvS32)((NvU64)pDescriptor); + if ((NvU64)fd != (NvU64)pDescriptor) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): fd must fit within a signed 32-bit integer!\n", + __FUNCTION__); + return NV_ERR_INVALID_ARGUMENT; + } + + rmStatus = nv_dma_import_from_fd(nv->dma_dev, fd, &size, + &pUserPages, &pImportSgt, &pImportPriv); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): Error (%d) while trying to import fd!\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv, + pUserPages, pImportSgt, + size, ppMemDesc, ppPrivate); +} + +static NV_STATUS +osCreateOsDescriptorFromSgtPtr +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS *params = + (NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS*)((NvUPtr) pDescriptor); + + struct sg_table *sgt = params->sgt; + struct drm_gem_object *gem = params->gem; + + rmStatus = nv_dma_import_sgt(nv->dma_dev, sgt, gem); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): Error (%d) while trying to import sgt!\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + return _createMemdescFromSgt(pGpu, flags, gem, sgt, + (*pLimit + 1), ppMemDesc, ppPrivate); +} + +static NV_STATUS +osCreateOsDescriptorFromDmaBufPtr +( + OBJGPU *pGpu, + NvP64 pDescriptor, + NvHandle hClient, + NvU32 flags, + NvU64 *pLimit, + MEMORY_DESCRIPTOR **ppMemDesc, + void **ppPrivate +) +{ + NV_STATUS rmStatus = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NvU32 size = 0; + void *pUserPages = NULL; + nv_dma_buf_t *pImportPriv = NULL; + struct sg_table *pImportSgt = NULL; + void *dmaBuf = (void*)((NvUPtr)pDescriptor); + + rmStatus = nv_dma_import_dma_buf(nv->dma_dev, dmaBuf, &size, + &pUserPages, &pImportSgt, &pImportPriv); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "%s(): Error (%d) while trying to import dma_buf!\n", + __FUNCTION__, rmStatus); + return rmStatus; + } + + return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv, + pUserPages, pImportSgt, + size, ppMemDesc, ppPrivate); +} + +static void +osDestroyOsDescriptorFromPhysAddr +( + PMEMORY_DESCRIPTOR pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + void *pPrivate; + + pPrivate = memdescGetMemData(pMemDesc); + NV_ASSERT(pPrivate != NULL); + + nv_unregister_phys_pages(NV_GET_NV_STATE(pGpu), pPrivate); +} + +static void +osDestroyOsDescriptorFromIoMemory +( + PMEMORY_DESCRIPTOR pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + void *pPrivate = memdescGetMemData(pMemDesc); + + if (pPrivate == NULL) + { + return; + } + + nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), pPrivate); +} + +static void +osDestroyOsDescriptorPageArray +( + MEMORY_DESCRIPTOR *pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + NvU64 osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount); + NV_STATUS status; + void *pPrivate; + + pPrivate = memdescGetMemData(pMemDesc); + + NV_ASSERT(pPrivate != NULL); + + // + // TODO: Bug 1811006: Notably skip any IOMMU mapping management as the + // pMemDesc->pGpu might have been torn down already and the pGpu passed in + // doesn't necessarily have IOMMU mappings. For now just allow memdescDestroy() + // to clean up whatever is there (this may not work correctly either if any + // of the IOMMU mappings have outlasted their VASPACEs). This should + // be cleaned up once the fix for bug 1811006 is known. + // + + if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + nv_unregister_user_pages(NV_GET_NV_STATE(pGpu), osPageCount, + NULL /* import_priv */, &pPrivate); + + if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_FOREIGN_PAGE) == NV_FALSE) + { + status = os_unlock_user_pages(osPageCount, pPrivate); + NV_ASSERT(status == NV_OK); + } + else + { + os_free_mem(pPrivate); + } +} + +static void +osDestroyOsDescriptorFromDmaBuf +( + PMEMORY_DESCRIPTOR pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + void *pPrivate = memdescGetMemData(pMemDesc); + + struct sg_table *pImportSgt; + void *pImportPriv; + + /* + * Unmap IOMMU now or we will get a kernel crash when it is unmapped after + * pImportSgt is freed. + */ + memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); + + if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt, + &pImportPriv, pPrivate); + + /* + * pImportSgt doesn't need to be passed to nv_dma_release_dma_buf() because + * the DMA-BUF associated with pImportPriv already has a reference to the + * SGT. + */ + + nv_dma_release_dma_buf(NULL, pImportPriv); +} + +static void +osDestroyOsDescriptorFromSgt +( + PMEMORY_DESCRIPTOR pMemDesc +) +{ + OBJGPU *pGpu = pMemDesc->pGpu; + void *pPrivate = memdescGetMemData(pMemDesc); + + struct sg_table *pImportSgt; + struct drm_gem_object *pImportPrivGem; + + NV_ASSERT(pPrivate != NULL); + + /* + * Unmap IOMMU now or we will get a kernel crash when it is unmapped after + * pImportSgt is freed. + */ + memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId); + + if ((NV_RM_PAGE_SIZE < os_page_size) && + !memdescGetContiguity(pMemDesc, AT_CPU)) + { + RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU), + pMemDesc->PageCount); + } + + nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt, + (void **) &pImportPrivGem, pPrivate); + + nv_dma_release_sgt(pImportSgt, pImportPrivGem); +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osnvlink.c b/userspace/sdks/nv525/nvalloc/unix/src/osnvlink.c new file mode 100644 index 0000000..2d597a1 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/osnvlink.c @@ -0,0 +1,698 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include "kernel/gpu/nvlink/kernel_nvlink.h" +#include "osapi.h" + +#if defined(INCLUDE_NVLINK_LIB) +#include "nvlink.h" +// +// The functions in this file are a workaround for a significant design flaw +// where RM callbacks for the nvlink library are built with the altstack +// enabled, but the nvlink library is not built with altstack support. Whenever +// the library calls a callback, the stack switching needs to be accounted for +// or else we will observe corruption of data structures in the nvlink library +// as data is pushed onto what the callback thinks is the stack. See bug +// 1710300. +// +// This bug has also exposed other problems, such as the complete lack of +// locking awareness by these callbacks (e.g., assumption that the RMAPI and +// GPU locks are always held on entry, which is not a legitimate assumption). +// For now, we ignore that just to unblock testing. +// +extern NvlStatus knvlinkCoreAddLinkCallback(struct nvlink_link *); +extern NvlStatus knvlinkCoreRemoveLinkCallback(struct nvlink_link *); +extern NvlStatus knvlinkCoreLockLinkCallback(struct nvlink_link *); +extern void knvlinkCoreUnlockLinkCallback(struct nvlink_link *); +extern NvlStatus knvlinkCoreQueueLinkChangeCallback(struct nvlink_link_change *); +extern NvlStatus knvlinkCoreSetDlLinkModeCallback(struct nvlink_link *, NvU64, NvU32); +extern NvlStatus knvlinkCoreGetDlLinkModeCallback(struct nvlink_link *, NvU64 *); +extern NvlStatus knvlinkCoreSetTlLinkModeCallback(struct nvlink_link *, NvU64, NvU32); +extern NvlStatus knvlinkCoreGetTlLinkModeCallback(struct nvlink_link *, NvU64 *); +extern NvlStatus knvlinkCoreGetTxSublinkModeCallback(struct nvlink_link *, NvU64 *, NvU32 *); +extern NvlStatus knvlinkCoreSetTxSublinkModeCallback(struct nvlink_link *, NvU64, NvU32); +extern NvlStatus knvlinkCoreGetRxSublinkModeCallback(struct nvlink_link *, NvU64 *, NvU32 *); +extern NvlStatus knvlinkCoreSetRxSublinkModeCallback(struct nvlink_link *, NvU64, NvU32); +extern NvlStatus knvlinkCoreReadDiscoveryTokenCallback(struct nvlink_link *, NvU64 *); +extern NvlStatus knvlinkCoreWriteDiscoveryTokenCallback(struct nvlink_link *, NvU64); +extern void knvlinkCoreTrainingCompleteCallback(struct nvlink_link *); +extern void knvlinkCoreGetUphyLoadCallback(struct nvlink_link *, NvBool*); + +/*! + * @brief Helper to allocate an alternate stack from within core RM. + * + * This needs to be an NV_API_CALL (built to use the original stack instead + * of the altstack) since it is called before we switch to using the altstack. + */ +static NV_STATUS NV_API_CALL osNvlinkAllocAltStack(nvidia_stack_t **pSp) +{ + NV_STATUS status = NV_OK; + nvidia_stack_t *sp = NULL; +#if defined(NVCPU_X86_64) && defined(__use_altstack__) + status = os_alloc_mem((void **)&sp, sizeof(nvidia_stack_t)); + if (status == NV_OK) + { + sp->size = sizeof(sp->stack); + sp->top = sp->stack + sp->size; + } +#endif + *pSp = sp; + return status; +} + +/*! + * @brief Helper to free an alternate stack from within core RM. + * + * This needs to be an NV_API_CALL (built to use the original stack instead + * of the altstack) since it is called after we've switched back to using the + * original stack. + */ +static void NV_API_CALL osNvlinkFreeAltStack(nvidia_stack_t *sp) +{ +#if defined(NVCPU_X86_64) && defined(__use_altstack__) + os_free_mem(sp); +#endif +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_add_link +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp; + + if (NV_OK != osNvlinkAllocAltStack(&sp)) + { + return NVL_ERR_GENERIC; + } + + NV_ENTER_RM_RUNTIME(sp, fp); + + status = knvlinkCoreAddLinkCallback(link); + + NV_EXIT_RM_RUNTIME(sp, fp); + + if (status == NVL_SUCCESS) + { + pLink->pOsInfo = sp; + } + else + { + osNvlinkFreeAltStack(sp); + } + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_remove_link +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + pLink->pOsInfo = NULL; + + NV_ENTER_RM_RUNTIME(sp, fp); + + status = knvlinkCoreRemoveLinkCallback(link); + + NV_EXIT_RM_RUNTIME(sp, fp); + + osNvlinkFreeAltStack(sp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_lock_link +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreLockLinkCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static void NV_API_CALL rm_nvlink_ops_unlock_link +( + struct nvlink_link *link +) +{ + void *fp; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + knvlinkCoreUnlockLinkCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_queue_link_change +( + struct nvlink_link_change *link_change +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link_change->master->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreQueueLinkChangeCallback(link_change); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_dl_link_mode +( + struct nvlink_link *link, + NvU64 mode, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetDlLinkModeCallback(link, mode, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_dl_link_mode +( + struct nvlink_link *link, + NvU64 *mode +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetDlLinkModeCallback(link, mode); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_tl_link_mode +( + struct nvlink_link *link, + NvU64 mode, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetTlLinkModeCallback(link, mode, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_tl_link_mode +( + struct nvlink_link *link, + NvU64 *mode +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetTlLinkModeCallback(link, mode); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_tx_mode +( + struct nvlink_link *link, + NvU64 mode, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetTxSublinkModeCallback(link, mode, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_tx_mode +( + struct nvlink_link *link, + NvU64 *mode, + NvU32 *subMode +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetTxSublinkModeCallback(link, mode, subMode); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_rx_mode +( + struct nvlink_link *link, + NvU64 mode, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetRxSublinkModeCallback(link, mode, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_rx_mode +( + struct nvlink_link *link, + NvU64 *mode, + NvU32 *subMode +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetRxSublinkModeCallback(link, mode, subMode); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_rx_detect +( + struct nvlink_link *link, + NvU32 flags +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreSetRxSublinkDetectCallback(link, flags); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_rx_detect +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreGetRxSublinkDetectCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static void NV_API_CALL rm_nvlink_get_uphy_load +( + struct nvlink_link *link, + NvBool *bUnlocked +) +{ + void *fp; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + knvlinkCoreGetUphyLoadCallback(link, bUnlocked); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_read_link_discovery_token +( + struct nvlink_link *link, + NvU64 *token +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreReadDiscoveryTokenCallback(link, token); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_write_link_discovery_token +( + struct nvlink_link *link, + NvU64 token +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreWriteDiscoveryTokenCallback(link, token); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + + return status; +} + +static void NV_API_CALL rm_nvlink_ops_training_complete +( + struct nvlink_link *link +) +{ + void *fp; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK *pLink = link->link_info; + nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + knvlinkCoreTrainingCompleteCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); +} + +static NvlStatus NV_API_CALL rm_nvlink_ops_ali_training +( + struct nvlink_link *link +) +{ + void *fp; + NvlStatus status; + THREAD_STATE_NODE threadState = {0}; + KNVLINK_RM_LINK * pLink = link->link_info; + nvidia_stack_t * sp = (nvidia_stack_t *)pLink->pOsInfo; + + NV_ENTER_RM_RUNTIME(sp, fp); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + status = knvlinkCoreAliTrainingCallback(link); + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + + NV_EXIT_RM_RUNTIME(sp, fp); + return status; +} + +#endif /* defined(INCLUDE_NVLINK_LIB) */ + +const struct nvlink_link_handlers* osGetNvlinkLinkCallbacks(void) +{ +#if defined(INCLUDE_NVLINK_LIB) + static const struct nvlink_link_handlers rm_nvlink_link_ops = + { + .add = rm_nvlink_ops_add_link, + .remove = rm_nvlink_ops_remove_link, + .lock = rm_nvlink_ops_lock_link, + .unlock = rm_nvlink_ops_unlock_link, + .queue_link_change = rm_nvlink_ops_queue_link_change, + .set_dl_link_mode = rm_nvlink_ops_set_dl_link_mode, + .get_dl_link_mode = rm_nvlink_ops_get_dl_link_mode, + .set_tl_link_mode = rm_nvlink_ops_set_tl_link_mode, + .get_tl_link_mode = rm_nvlink_ops_get_tl_link_mode, + .set_tx_mode = rm_nvlink_ops_set_link_tx_mode, + .get_tx_mode = rm_nvlink_ops_get_link_tx_mode, + .set_rx_mode = rm_nvlink_ops_set_link_rx_mode, + .get_rx_mode = rm_nvlink_ops_get_link_rx_mode, + .set_rx_detect = rm_nvlink_ops_set_link_rx_detect, + .get_rx_detect = rm_nvlink_ops_get_link_rx_detect, + .write_discovery_token = rm_nvlink_ops_write_link_discovery_token, + .read_discovery_token = rm_nvlink_ops_read_link_discovery_token, + .training_complete = rm_nvlink_ops_training_complete, + .get_uphy_load = rm_nvlink_get_uphy_load, + .ali_training = rm_nvlink_ops_ali_training, + }; + + return &rm_nvlink_link_ops; +#else + return NULL; +#endif +} + +/* + * @brief Verif only function to get the chiplib overrides for link connection + * state for all NVLINKs. + * + * If chiplib overrides exist, each link can either be enabled (1) or disabled (0) + * + * @param[in] pGpu GPU object pointer + * @param[in] maxLinks Size of pLinkConnection array + * @param[out] pLinkConnection array of pLinkConnection values to be populated by MODS + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available) + */ +NV_STATUS +osGetForcedNVLinkConnection +( + OBJGPU *pGpu, + NvU32 maxLinks, + NvU32 *pLinkConnection +) +{ + int i, ret; + NV_STATUS status; + char path[64]; + OBJSYS *pSys; + OBJOS *pOS; + + NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER); + NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED); + NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT); + + pSys = SYS_GET_INSTANCE(); + pOS = SYS_GET_OS(pSys); + if (pOS == NULL || pOS->osSimEscapeRead == NULL) + { + NV_PRINTF(LEVEL_ERROR, "%s: escape reads not supported on platform\n", + __FUNCTION__); + return NV_ERR_NOT_SUPPORTED; + } + + for (i = 0; i < maxLinks; i++) + { + ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|NVLink%u", i); + NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1))); + + status = pOS->osSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]); + if (status == NV_OK) + { + NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__, + path, pLinkConnection[i]); + } + else + { + NV_PRINTF(LEVEL_INFO, "%s: osSimEscapeRead for '%s' failed (%u)\n", + __FUNCTION__, path, status); + return NV_ERR_NOT_SUPPORTED; + } + } + return NV_OK; +} + +/* + * @brief Get Platform suggested NVLink linerate + * + * NVLink will use this function to get the platform suggested linerate + * if available in FRU or device tree. + * + * @param[in] pGpu GPU object pointer + * @param[out] NvU32 * Suggested datarate + * + * @return NV_OK or NV_ERR_NOT_SUPPORTED (platform linerate data not available) + */ +NV_STATUS +osGetPlatformNvlinkLinerate +( + OBJGPU *pGpu, + NvU32 *lineRate +) +{ +#if defined(NVCPU_PPC64LE) + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); + + if (!pKernelNvlink) + return NV_ERR_INVALID_ARGUMENT; + + return nv_get_nvlink_line_rate(nv, lineRate); +#else + //TODO : FRU based method to be filled out by Bug 200285656 + //*lineRate = 0; + //return NV_OK; + return NV_ERR_NOT_SUPPORTED; +#endif +} + +void +osSetNVLinkSysmemLinkState +( + OBJGPU *pGpu, + NvBool enabled +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + NV_ASSERT(enabled); + if (enabled) + nv_dma_enable_nvlink(nv->dma_dev); +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osunix.c b/userspace/sdks/nv525/nvalloc/unix/src/osunix.c new file mode 100644 index 0000000..7d70602 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/osunix.c @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +/***************************** HW State Routines ***************************\ +* * +* Fills in os specific function pointers for the Unix OS object. * +* * +\***************************************************************************/ + +#include <osfuncs.h> +#include <os/os.h> + +static void initOSSpecificFunctionPointers(OBJOS *); +static void initMiscOSFunctionPointers(OBJOS *); +static void initUnixOSFunctionPointers(OBJOS *); +static void initOSSpecificProperties(OBJOS *); + +void +osInitObjOS(OBJOS *pOS) +{ + initOSSpecificFunctionPointers(pOS); + initOSSpecificProperties(pOS); +} + +static void +initOSSpecificFunctionPointers(OBJOS *pOS) +{ + initMiscOSFunctionPointers(pOS); + initUnixOSFunctionPointers(pOS); +} + +static void +initMiscOSFunctionPointers(OBJOS *pOS) +{ + pOS->osQueueWorkItem = osQueueWorkItem; + pOS->osQueueWorkItemWithFlags = osQueueWorkItemWithFlags; + pOS->osQueueSystemWorkItem = osQueueSystemWorkItem; +} + +static void +initUnixOSFunctionPointers(OBJOS *pOS) +{ +#if defined(NVCPU_X86_64) + pOS->osNv_rdcr4 = nv_rdcr4; + pOS->osNv_cpuid = nv_cpuid; +#endif + + pOS->osCallACPI_DSM = osCallACPI_DSM; + pOS->osCallACPI_DDC = osCallACPI_DDC; + pOS->osCallACPI_NVHG_ROM = osCallACPI_NVHG_ROM; + pOS->osCallACPI_DOD = osCallACPI_DOD; + pOS->osCallACPI_MXDM = osCallACPI_MXDM; + pOS->osCallACPI_MXDS = osCallACPI_MXDS; + + pOS->osDbgBreakpointEnabled = osDbgBreakpointEnabled; +} + +static void +initOSSpecificProperties +( + OBJOS *pOS +) +{ + pOS->setProperty(pOS, PDB_PROP_OS_ONDEMAND_VBLANK_CONTROL_ENABLE_DEFAULT, NV_TRUE); + pOS->setProperty(pOS, PDB_PROP_OS_CACHED_MEMORY_MAPPINGS_FOR_ACPI_TABLE, NV_TRUE); + pOS->setProperty(pOS, PDB_PROP_OS_LIMIT_GPU_RESET, NV_TRUE); +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/registry.c b/userspace/sdks/nv525/nvalloc/unix/src/registry.c new file mode 100644 index 0000000..522b937 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/registry.c @@ -0,0 +1,705 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include <nv-priv.h> +#include <nvos.h> + +#if defined(DEBUG_REGISTRY) +#define DBG_REG_PRINTF(a, ...) \ + NV_PRINTF(LEVEL_INFO, a, ##__VA_ARGS__) +#else +#define DBG_REG_PRINTF(a, ...) +#endif + +static NvS32 stringCaseCompare( + const char *string1, + const char *string2 +) +{ + NvU8 c1, c2; + + do + { + c1 = *string1, c2 = *string2; + if (c1 >= 'A' && c1 <= 'Z') + c1 += ('a' - 'A'); + if (c2 >= 'A' && c2 <= 'Z') + c2 += ('a' - 'A'); + string1++, string2++; + } + while ((c1 == c2) && (c1 != '\0')); + + return (c1 - c2); +} + +static nv_reg_entry_t *the_registry = NULL; + +static nv_reg_entry_t* regCreateNewRegistryKey( + nv_state_t *nv, + const char *regParmStr +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_reg_entry_t *new_reg = NULL; + char *new_ParmStr = NULL; + NvU32 parm_size; + + if (regParmStr == NULL) + { + DBG_BREAKPOINT(); + return NULL; + } + + new_reg = portMemAllocNonPaged(sizeof(nv_reg_entry_t)); + if (NULL == new_reg) + { + NV_PRINTF(LEVEL_ERROR, "failed to grow registry\n"); + return NULL; + } + + portMemSet(new_reg, 0, sizeof(nv_reg_entry_t)); + + if (regParmStr != NULL) + { + parm_size = (portStringLength(regParmStr) + 1); + new_ParmStr = portMemAllocNonPaged(parm_size); + if (NULL == new_ParmStr) + { + NV_PRINTF(LEVEL_ERROR, "failed to allocate registry param string\n"); + portMemFree(new_reg); + return NULL; + } + + NV_ASSERT(parm_size <= NVOS38_MAX_REGISTRY_STRING_LENGTH); + + if (portMemCopy(new_ParmStr, parm_size, regParmStr, parm_size) == NULL) + { + NV_PRINTF(LEVEL_ERROR, "failed to copy registry param string\n"); + portMemFree(new_ParmStr); + portMemFree(new_reg); + return NULL; + } + } + + new_reg->regParmStr = new_ParmStr; + new_reg->type = NV_REGISTRY_ENTRY_TYPE_UNKNOWN; + + if (nvp != NULL) + { + new_reg->next = nvp->pRegistry; + nvp->pRegistry = new_reg; + DBG_REG_PRINTF("local registry now at 0x%p\n", nvp->pRegistry); + } + else + { + new_reg->next = the_registry; + the_registry = new_reg; + DBG_REG_PRINTF("global registry now at 0x%p\n", the_registry); + } + + return new_reg; +} + +static NV_STATUS regFreeEntry(nv_reg_entry_t *tmp) +{ + portMemFree(tmp->regParmStr); + tmp->regParmStr = NULL; + { + portMemFree(tmp->pdata); + tmp->pdata = NULL; + tmp->len = 0; + } + portMemFree(tmp); + + return NV_OK; +} + +static nv_reg_entry_t* regFindRegistryEntry( + nv_state_t *nv, + const char *regParmStr, + NvU32 type, + NvBool *bGlobalEntry +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_reg_entry_t *tmp; + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + if (nvp != NULL) + { + tmp = nvp->pRegistry; + DBG_REG_PRINTF(" local registry at 0x%p\n", tmp); + + while ((tmp != NULL) && (tmp->regParmStr != NULL)) + { + DBG_REG_PRINTF(" Testing against %s\n", + tmp->regParmStr); + if ((stringCaseCompare(tmp->regParmStr, regParmStr) == 0) && + (type == tmp->type)) + { + DBG_REG_PRINTF(" found a match!\n"); + if (bGlobalEntry) + *bGlobalEntry = NV_FALSE; + return tmp; + } + tmp = tmp->next; + } + } + + tmp = the_registry; + DBG_REG_PRINTF(" global registry at 0x%p\n", tmp); + + while ((tmp != NULL) && (tmp->regParmStr != NULL)) + { + DBG_REG_PRINTF(" Testing against %s\n", + tmp->regParmStr); + if ((stringCaseCompare(tmp->regParmStr, regParmStr) == 0) && + (type == tmp->type)) + { + DBG_REG_PRINTF(" found a match!\n"); + if (bGlobalEntry) + *bGlobalEntry = NV_TRUE; + return tmp; + } + tmp = tmp->next; + } + + DBG_REG_PRINTF(" no match\n"); + return NULL; +} + +NV_STATUS RmWriteRegistryDword( + nv_state_t *nv, + const char *regParmStr, + NvU32 Data +) +{ + nv_reg_entry_t *tmp; + NvBool bGlobalEntry; + + if (regParmStr == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s -> 0x%x\n", __FUNCTION__, regParmStr, Data); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_DWORD, &bGlobalEntry); + + // If we found an entry and we were looking for a global entry and + // found a global, or we were looking for a per-GPU entry and found a + // per-GPU entry + if (tmp != NULL && + ((nv == NULL && bGlobalEntry) || + (nv != NULL && !bGlobalEntry))) + { + tmp->data = Data; + + if (stringCaseCompare(regParmStr, "ResmanDebugLevel") == 0) + { + os_dbg_set_level(Data); + } + + return NV_OK; + } + + tmp = regCreateNewRegistryKey(nv, regParmStr); + if (tmp == NULL) + return NV_ERR_GENERIC; + + tmp->type = NV_REGISTRY_ENTRY_TYPE_DWORD; + tmp->data = Data; + + return NV_OK; +} + +NV_STATUS RmReadRegistryDword( + nv_state_t *nv, + const char *regParmStr, + NvU32 *Data +) +{ + nv_reg_entry_t *tmp; + + if ((regParmStr == NULL) || (Data == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_DWORD, NULL); + if (tmp == NULL) + { + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_BINARY, NULL); + if ((tmp != NULL) && (tmp->len >= sizeof(NvU32))) + { + *Data = *(NvU32 *)tmp->pdata; + } + else + { + DBG_REG_PRINTF(" not found\n"); + return NV_ERR_GENERIC; + } + } + else + { + *Data = tmp->data; + } + + DBG_REG_PRINTF(" found in the_registry: 0x%x\n", *Data); + + return NV_OK; +} + +NV_STATUS RmReadRegistryBinary( + nv_state_t *nv, + const char *regParmStr, + NvU8 *Data, + NvU32 *cbLen +) +{ + nv_reg_entry_t *tmp; + NV_STATUS status; + + if ((regParmStr == NULL) || (Data == NULL) || (cbLen == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_BINARY, NULL); + if (tmp == NULL) + { + DBG_REG_PRINTF(" not found\n"); + return NV_ERR_GENERIC; + } + + DBG_REG_PRINTF(" found\n"); + + if (*cbLen >= tmp->len) + { + portMemCopy((NvU8 *)Data, *cbLen, (NvU8 *)tmp->pdata, tmp->len); + *cbLen = tmp->len; + status = NV_OK; + } + else + { + NV_PRINTF(LEVEL_ERROR, + "buffer (length: %u) is too small (data length: %u)\n", + *cbLen, tmp->len); + status = NV_ERR_GENERIC; + } + + return status; +} + +NV_STATUS RmWriteRegistryBinary( + nv_state_t *nv, + const char *regParmStr, + NvU8 *Data, + NvU32 cbLen +) +{ + nv_reg_entry_t *tmp; + NvBool bGlobalEntry; + + if ((regParmStr == NULL) || (Data == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_BINARY, &bGlobalEntry); + + // If we found an entry and we were looking for a global entry and + // found a global, or we were looking for a per-GPU entry and found a + // per-GPU entry + if (tmp != NULL && + ((nv == NULL && bGlobalEntry) || + (nv != NULL && !bGlobalEntry))) + { + if (tmp->pdata != NULL) + { + portMemFree(tmp->pdata); + tmp->pdata = NULL; + tmp->len = 0; + } + } + else + { + tmp = regCreateNewRegistryKey(nv, regParmStr); + if (tmp == NULL) + { + NV_PRINTF(LEVEL_ERROR, "failed to create binary registry entry\n"); + return NV_ERR_GENERIC; + } + } + + tmp->pdata = portMemAllocNonPaged(cbLen); + if (NULL == tmp->pdata) + { + NV_PRINTF(LEVEL_ERROR, "failed to write binary registry entry\n"); + return NV_ERR_GENERIC; + } + + tmp->type = NV_REGISTRY_ENTRY_TYPE_BINARY; + tmp->len = cbLen; + portMemCopy((NvU8 *)tmp->pdata, tmp->len, (NvU8 *)Data, cbLen); + + return NV_OK; +} + +NV_STATUS RmWriteRegistryString( + nv_state_t *nv, + const char *regParmStr, + const char *buffer, + NvU32 bufferLength +) +{ + nv_reg_entry_t *tmp; + NvBool bGlobalEntry; + + if ((regParmStr == NULL) || (buffer == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_STRING, &bGlobalEntry); + + // If we found an entry and we were looking for a global entry and + // found a global, or we were looking for a per-GPU entry and found a + // per-GPU entry + if (tmp != NULL && + ((nv == NULL && bGlobalEntry) || + (nv != NULL && !bGlobalEntry))) + { + if (tmp->pdata != NULL) + { + portMemFree(tmp->pdata); + tmp->len = 0; + tmp->pdata = NULL; + } + } + else + { + tmp = regCreateNewRegistryKey(nv, regParmStr); + if (tmp == NULL) + { + NV_PRINTF(LEVEL_ERROR, + "failed to allocate a string registry entry!\n"); + return NV_ERR_INSUFFICIENT_RESOURCES; + } + } + + tmp->pdata = portMemAllocNonPaged(bufferLength); + if (tmp->pdata == NULL) + { + NV_PRINTF(LEVEL_ERROR, "failed to write a string registry entry!\n"); + return NV_ERR_NO_MEMORY; + } + + tmp->type = NV_REGISTRY_ENTRY_TYPE_STRING; + tmp->len = bufferLength; + portMemCopy((void *)tmp->pdata, tmp->len, buffer, (bufferLength - 1)); + tmp->pdata[bufferLength-1] = '\0'; + + return NV_OK; +} + +NV_STATUS RmReadRegistryString( + nv_state_t *nv, + const char *regParmStr, + NvU8 *buffer, + NvU32 *pBufferLength +) +{ + NvU32 bufferLength; + nv_reg_entry_t *tmp; + + if ((regParmStr == NULL) || (buffer == NULL) || (pBufferLength == NULL)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr); + + bufferLength = *pBufferLength; + *pBufferLength = 0; + *buffer = '\0'; + + tmp = regFindRegistryEntry(nv, regParmStr, + NV_REGISTRY_ENTRY_TYPE_STRING, NULL); + if (tmp == NULL) + { + return NV_ERR_GENERIC; + } + + if (bufferLength >= tmp->len) + { + portMemCopy((void *)buffer, bufferLength, (void *)tmp->pdata, tmp->len); + *pBufferLength = tmp->len; + } + else + { + NV_PRINTF(LEVEL_ERROR, + "buffer (length: %u) is too small (data length: %u)\n", + bufferLength, tmp->len); + return NV_ERR_BUFFER_TOO_SMALL; + } + + return NV_OK; +} + +NV_STATUS RmInitRegistry(void) +{ + NV_STATUS rmStatus; + + rmStatus = os_registry_init(); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "failed to initialize the OS registry!\n"); + } + + return rmStatus; +} + +NV_STATUS RmDestroyRegistry(nv_state_t *nv) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_reg_entry_t *tmp; + + if (nvp != NULL) + { + tmp = nvp->pRegistry; + nvp->pRegistry = NULL; + } + else + { + tmp = the_registry; + the_registry = NULL; + } + + while (tmp != NULL) + { + nv_reg_entry_t *entry = tmp; + tmp = tmp->next; + regFreeEntry(entry); + } + + return NV_OK; +} + +static void regCountEntriesAndSize( + NvU32 *pNumEntries, // Pointer to number of entries + NvU32 *pSize, // Pointer to total size + nv_reg_entry_t *pRegEntry // Pointer local or global registry +) +{ + // + // Note that *pNumEntries and *pSize are not initialized here. This is so + // we can accumulate totals of both global and local registries. + // + NvU32 numEntries = *pNumEntries; + NvU32 size = *pSize; + + while ((pRegEntry != NULL) && (pRegEntry->regParmStr != NULL)) + { + size += portStringLength(pRegEntry->regParmStr) + 1 + pRegEntry->len; + numEntries++; + pRegEntry = pRegEntry->next; + } + + *pNumEntries = numEntries; + *pSize = size; +} + +static NV_STATUS regCopyEntriesToPackedBuffer( + PACKED_REGISTRY_TABLE *pRegTable, // Pointer to packed record + nv_reg_entry_t *pRegEntry, // Pointer local or global registry + NvU32 *pEntryIndex, // Pointer to next index + NvU32 *pDataOffset // Pointer to offset of next data byte. +) +{ + NvU8 *pByte = (NvU8 *)pRegTable; // Byte version of record pointer. + NV_STATUS nvStatus = NV_OK; + NvU32 entryIndex = *pEntryIndex; + NvU32 dataOffset = *pDataOffset; + + // Walk the records and copy the data. + while ((pRegEntry != NULL) && (pRegEntry->regParmStr != NULL)) + { + PACKED_REGISTRY_ENTRY *pEntry = &pRegTable->entries[entryIndex]; + NvU32 slen = portStringLength(pRegEntry->regParmStr) + 1; + + // Sanity check the data offset and index against counted totals. + if ((dataOffset + slen + pRegEntry->len > pRegTable->size) || + (entryIndex >= pRegTable->numEntries)) + { + // Something has changed since we counted them? + NV_PRINTF(LEVEL_ERROR, "Registry entry record is full\n"); + nvStatus = NV_ERR_INVALID_STATE; + break; + } + + // Copy registry entry name to data blob. + pEntry->nameOffset = dataOffset; + portMemCopy(&pByte[dataOffset], slen, pRegEntry->regParmStr, slen); + dataOffset += slen; + + switch (pRegEntry->type) + { + case NV_REGISTRY_ENTRY_TYPE_DWORD: + pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_DWORD; + pEntry->length = sizeof(NvU32); + pEntry->data = pRegEntry->data; + break; + + case NV_REGISTRY_ENTRY_TYPE_BINARY: + case NV_REGISTRY_ENTRY_TYPE_STRING: + pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_STRING; + if (pRegEntry->type == NV_REGISTRY_ENTRY_TYPE_BINARY) + pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_BINARY; + + pEntry->length = pRegEntry->len; + pEntry->data = dataOffset; + portMemCopy(&pByte[dataOffset], pEntry->length, + pRegEntry->pdata, pRegEntry->len); + dataOffset += pRegEntry->len; + break; + + default: + // We should never get here. + pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_UNKNOWN; + pEntry->length = 0; + pEntry->data = 0; + DBG_BREAKPOINT(); + break; + } + + pRegEntry = pRegEntry->next; + entryIndex++; + } + + *pEntryIndex = entryIndex; + *pDataOffset = dataOffset; + return nvStatus; +} + +// Package registry entries +NV_STATUS RmPackageRegistry( + nv_state_t *nv, + PACKED_REGISTRY_TABLE *pRegTable, + NvU32 *pSize +) +{ + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + nv_reg_entry_t *pLocalRegistry = NULL; + NV_STATUS nvStatus = NV_OK; + NvU32 totalSize; + NvU32 numEntries; + + if (pSize == NULL) + return NV_ERR_INVALID_ARGUMENT; + + // Use the local (per-device) registry if we have one. + if (nvp != NULL) + pLocalRegistry = nvp->pRegistry; + + numEntries = 0; + totalSize = NV_OFFSETOF(PACKED_REGISTRY_TABLE, entries); + + // Count the number of global entries and total size. + regCountEntriesAndSize(&numEntries, &totalSize, the_registry); + + // Count the number of local entries and total size. + regCountEntriesAndSize(&numEntries, &totalSize, pLocalRegistry); + + // Add table record size into total size. + totalSize += sizeof(PACKED_REGISTRY_ENTRY) * numEntries; + + // + // If this function is called to only compute total size of registry table, + // then we are done here. + // + if (pRegTable == NULL) + { + *pSize = totalSize; + return NV_OK; + } + + // Return warning if there are no registry entries. + if (numEntries == 0) + return NV_WARN_NOTHING_TO_DO; + + if (totalSize > *pSize) + { + NV_PRINTF(LEVEL_ERROR, "Registry entries overflow RPC record\n"); + return NV_ERR_BUFFER_TOO_SMALL; + } + + // Fill in our new structure with the first pass (counting) values. + pRegTable->size = totalSize; + *pSize = totalSize; + pRegTable->numEntries = numEntries; + + // Offset of first byte after the registry entry table. + totalSize = NV_OFFSETOF(PACKED_REGISTRY_TABLE, entries) + + (sizeof(PACKED_REGISTRY_ENTRY) * numEntries); + + // Starting index in the registry entry table. + numEntries = 0; + + // Walk the global registry and copy the data. + nvStatus = regCopyEntriesToPackedBuffer(pRegTable, + the_registry, &numEntries, &totalSize); + + // Walk the local registry and copy the data. + if (nvStatus == NV_OK) + { + nvStatus = regCopyEntriesToPackedBuffer(pRegTable, + pLocalRegistry, &numEntries, &totalSize); + } + + // Sanity check second pass against first pass. + if ((numEntries != pRegTable->numEntries) || (totalSize != pRegTable->size)) + { + NV_PRINTF(LEVEL_ERROR, "First/second pass mismatch\n"); + nvStatus = NV_ERR_INVALID_STATE; + } + + return nvStatus; +} + diff --git a/userspace/sdks/nv525/nvalloc/unix/src/rm-gpu-ops.c b/userspace/sdks/nv525/nvalloc/unix/src/rm-gpu-ops.c new file mode 100644 index 0000000..d542393 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/rm-gpu-ops.c @@ -0,0 +1,813 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include <os/os.h> +#include <osapi.h> +#include <core/thread_state.h> +#include "rmapi/nv_gpu_ops.h" +#include "gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h" + +NV_STATUS NV_API_CALL rm_gpu_ops_create_session( + nvidia_stack_t *sp, + struct gpuSession **session) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsCreateSession(session); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_destroy_session ( + nvidia_stack_t *sp, gpuSessionHandle session) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDestroySession(session); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_device_create ( + nvidia_stack_t *sp, + nvgpuSessionHandle_t session, + const gpuInfo *pGpuInfo, + const NvProcessorUuid *gpuUuid, + nvgpuDeviceHandle_t *device, + NvBool bCreateSmcPartition) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDeviceCreate(session, pGpuInfo, gpuUuid, device, bCreateSmcPartition); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_device_destroy ( + nvidia_stack_t *sp, + gpuDeviceHandle device) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDeviceDestroy(device); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_address_space_create ( + nvidia_stack_t *sp, + gpuDeviceHandle device, + NvU64 vaBase, + NvU64 vaSize, + gpuAddressSpaceHandle *vaSpace, + gpuAddressSpaceInfo *vaSpaceInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsAddressSpaceCreate(device, vaBase, vaSize, vaSpace, + vaSpaceInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_dup_address_space( + nvidia_stack_t *sp, + gpuDeviceHandle device, + NvHandle hUserClient, + NvHandle hUserVASpace, + gpuAddressSpaceHandle *dupedVaspace, + gpuAddressSpaceInfo *vaSpaceInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDupAddressSpace(device, hUserClient, hUserVASpace, + dupedVaspace, vaSpaceInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_address_space_destroy(nvidia_stack_t *sp, + gpuAddressSpaceHandle vaspace) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsAddressSpaceDestroy(vaspace); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_alloc_fb( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, + NvLength size, NvU64 *gpuOffset, gpuAllocInfo *allocInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsMemoryAllocFb(vaspace, size, gpuOffset, allocInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_p2p_caps(nvidia_stack_t *sp, + gpuDeviceHandle device1, + gpuDeviceHandle device2, + getP2PCapsParams *pP2pCapsParams) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetP2PCaps(device1, device2, pP2pCapsParams); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_alloc_sys( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, + NvLength size, NvU64 *gpuOffset, gpuAllocInfo *allocInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsMemoryAllocSys(vaspace, size, gpuOffset, allocInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_register_callbacks( + nvidia_stack_t *sp, + void *pPma, + pmaEvictPagesCb_t evictPages, + pmaEvictRangeCb_t evictRange, + void *callbackData) +{ + THREAD_STATE_NODE threadState; + NV_STATUS rmStatus; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + rmStatus = pmaRegisterEvictionCb(pPma, evictPages, evictRange, callbackData); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +void NV_API_CALL rm_gpu_ops_pma_unregister_callbacks( + nvidia_stack_t *sp, + void *pPma) +{ + THREAD_STATE_NODE threadState; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + pmaUnregisterEvictionCb(pPma); + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_pma_object( + nvidia_stack_t *sp, + gpuDeviceHandle device, + void **pPma, + const nvgpuPmaStatistics_t *pPmaPubStats) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetPmaObject(device, pPma, + (const UvmPmaStatistics **)pPmaPubStats); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_alloc_pages( + nvidia_stack_t *sp, void *pPma, + NvLength pageCount, NvU32 pageSize, + nvgpuPmaAllocationOptions_t pPmaAllocOptions, + NvU64 *pPages) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPmaAllocPages(pPma, pageCount, pageSize, + pPmaAllocOptions, pPages); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_pin_pages( + nvidia_stack_t *sp, void *pPma, + NvU64 *pPages, NvLength pageCount, NvU32 pageSize, NvU32 flags) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPmaPinPages(pPma, pPages, pageCount, pageSize, flags); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_unpin_pages( + nvidia_stack_t *sp, void *pPma, + NvU64 *pPages, NvLength pageCount, NvU32 pageSize) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPmaUnpinPages(pPma, pPages, pageCount, pageSize); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_map( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, + NvU64 gpuOffset, NvLength length, void **cpuPtr, NvU32 pageSize) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsMemoryCpuMap(vaspace, gpuOffset, length, cpuPtr, + pageSize); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_ummap( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, void* cpuPtr) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsMemoryCpuUnMap(vaspace, cpuPtr); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_channel_allocate(nvidia_stack_t *sp, + gpuAddressSpaceHandle vaspace, + const gpuChannelAllocParams *allocParams, + gpuChannelHandle *channel, + gpuChannelInfo *channelInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsChannelAllocate(vaspace, allocParams, channel, + channelInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_channel_destroy(nvidia_stack_t * sp, + nvgpuChannelHandle_t channel) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsChannelDestroy(channel); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_pma_free_pages(nvidia_stack_t *sp, + void *pPma, NvU64 *pPages, NvLength pageCount, NvU32 pageSize, NvU32 flags) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsPmaFreePages(pPma, pPages, pageCount, pageSize, flags); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_memory_free( + nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, NvU64 gpuOffset) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsMemoryFree(vaspace, gpuOffset); + NV_EXIT_RM_RUNTIME(sp,fp); + return NV_OK; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_query_caps(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuCaps * caps) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsQueryCaps(device, caps); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_query_ces_caps(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuCesCaps *caps) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsQueryCesCaps(device, caps); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_gpu_info(nvidia_stack_t *sp, + const NvProcessorUuid *pUuid, + const gpuClientInfo *pGpuClientInfo, + gpuInfo *pGpuInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetGpuInfo(pUuid, pGpuClientInfo, pGpuInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_service_device_interrupts_rm(nvidia_stack_t *sp, + gpuDeviceHandle device) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsServiceDeviceInterruptsRM(device); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_set_page_directory (nvidia_stack_t *sp, + gpuAddressSpaceHandle vaSpace, + NvU64 physAddress, unsigned numEntries, + NvBool bVidMemAperture, NvU32 pasid) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsSetPageDirectory(vaSpace, physAddress, numEntries, + bVidMemAperture, pasid); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_unset_page_directory (nvidia_stack_t *sp, + gpuAddressSpaceHandle vaSpace) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsUnsetPageDirectory(vaSpace); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_dup_allocation(nvidia_stack_t *sp, + gpuAddressSpaceHandle srcVaSpace, + NvU64 srcAddress, + gpuAddressSpaceHandle dstVaSpace, + NvU64 dstVaAlignment, + NvU64 *dstAddress) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDupAllocation(srcVaSpace, srcAddress, dstVaSpace, dstVaAlignment, dstAddress); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_dup_memory (nvidia_stack_t *sp, + gpuDeviceHandle device, + NvHandle hClient, + NvHandle hPhysMemory, + NvHandle *hDupMemory, + nvgpuMemoryInfo_t gpuMemoryInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDupMemory(device, hClient, hPhysMemory, hDupMemory, gpuMemoryInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_free_duped_handle (nvidia_stack_t *sp, + gpuDeviceHandle device, + NvHandle hPhysHandle) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsFreeDupedHandle(device, hPhysHandle); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_fb_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuFbInfo * fbInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetFbInfo(device, fbInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_ecc_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuEccInfo * eccInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetEccInfo(device, eccInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +// +// Please see the comments for nvUvmInterfaceOwnPageFaultIntr(), in +// nv_uvm_interface.h, for the recommended way to use this routine. +// +// How it works: +// +// The rmGpuLocksAcquire call generally saves the current GPU interrupt +// state, then disables interrupt generation for one (or all) GPUs. +// Likewise, the rmGpuLocksRelease call restores (re-enables) those +// interrupts to their previous state. However, the rmGpuLocksRelease +// call does NOT restore interrupts that RM does not own. +// +// This is rather hard to find in the code, so: very approximately, the +// following sequence happens: rmGpuLocksRelease, osEnableInterrupts, +// intrRestoreNonStall_HAL, intrEncodeIntrEn_HAL, and that last one skips +// over any interrupts that RM does not own. +// +// This means that things are a bit asymmetric, because this routine +// actually changes that ownership in between the rmGpuLocksAcquire and +// rmGpuLocksRelease calls. So: +// +// -- If you call this routine with bOwnInterrupts == NV_TRUE (UVM is +// taking ownership from the RM), then rmGpuLocksAcquire disables all +// GPU interrupts. Then the ownership is taken away from RM, so the +// rmGpuLocksRelease call leaves the replayable page fault interrupts +// disabled. It is then up to UVM (the caller) to enable replayable +// page fault interrupts when it is ready. +// +// -- If you call this routine with bOwnInterrupts == NV_FALSE (UVM is +// returning ownership to the RM), then rmGpuLocksAcquire disables +// all GPU interrupts that RM owns. Then the ownership is returned to +// RM, so the rmGpuLocksRelease call re-enables replayable page fault +// interrupts. So, that implies that you need to disable replayable page +// fault interrupts before calling this routine, in order to hand +// over a GPU to RM that is not generating interrupts, until RM is +// ready to handle the interrupts. +// +NV_STATUS NV_API_CALL rm_gpu_ops_own_page_fault_intr(nvidia_stack_t *sp, + struct gpuDevice *device, + NvBool bOwnInterrupts) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsOwnPageFaultIntr(device, bOwnInterrupts); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_init_fault_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuFaultInfo *pFaultInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsInitFaultInfo(device, pFaultInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_destroy_fault_info (nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuFaultInfo *pFaultInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDestroyFaultInfo(device, pFaultInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +// Functions +// +// - rm_gpu_ops_has_pending_non_replayable_faults +// - rm_gpu_ops_get_non_replayable_faults +// +// Cannot take the GPU/RM lock because it is called during fault servicing. +// This could produce deadlocks if the UVM bottom half gets stuck behind a +// stalling interrupt that cannot be serviced if UVM is holding the lock. +// +// However, these functions can be safely called with no locks because it is +// just accessing the given client shadow fault buffer, which is implemented +// using a lock-free queue. There is a different client shadow fault buffer +// per GPU: RM top-half producer, UVM top/bottom-half consumer. + +NV_STATUS NV_API_CALL rm_gpu_ops_has_pending_non_replayable_faults(nvidia_stack_t *sp, + gpuFaultInfo *pFaultInfo, + NvBool *hasPendingFaults) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsHasPendingNonReplayableFaults(pFaultInfo, hasPendingFaults); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_get_non_replayable_faults(nvidia_stack_t *sp, + gpuFaultInfo *pFaultInfo, + void *faultBuffer, + NvU32 *numFaults) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsGetNonReplayableFaults(pFaultInfo, faultBuffer, numFaults); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_init_access_cntr_info(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessCntrInfo *accessCntrInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsInitAccessCntrInfo(device, accessCntrInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_destroy_access_cntr_info(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessCntrInfo *accessCntrInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDestroyAccessCntrInfo(device, accessCntrInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_enable_access_cntr(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessCntrInfo *accessCntrInfo, + gpuAccessCntrConfig *accessCntrConfig) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsEnableAccessCntr(device, accessCntrInfo, accessCntrConfig); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL rm_gpu_ops_disable_access_cntr(nvidia_stack_t *sp, + gpuDeviceHandle device, + gpuAccessCntrInfo *accessCntrInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsDisableAccessCntr(device, accessCntrInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_p2p_object_create(nvidia_stack_t *sp, + gpuDeviceHandle device1, + gpuDeviceHandle device2, + NvHandle *hP2pObject) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsP2pObjectCreate(device1, device2, hP2pObject); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_p2p_object_destroy(nvidia_stack_t *sp, + nvgpuSessionHandle_t session, + NvHandle hP2pObject) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + nvGpuOpsP2pObjectDestroy(session, hP2pObject); + NV_EXIT_RM_RUNTIME(sp, fp); +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_get_external_alloc_ptes(nvidia_stack_t* sp, + nvgpuAddressSpaceHandle_t vaSpace, + NvHandle hDupedMemory, + NvU64 offset, + NvU64 size, + nvgpuExternalMappingInfo_t gpuExternalMappingInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsGetExternalAllocPtes(vaSpace, hDupedMemory, offset, size, + gpuExternalMappingInfo); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_retain_channel(nvidia_stack_t* sp, + nvgpuAddressSpaceHandle_t vaSpace, + NvHandle hClient, + NvHandle hChannel, + void **retainedChannel, + nvgpuChannelInstanceInfo_t channelInstanceInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsRetainChannel(vaSpace, hClient, hChannel, + (gpuRetainedChannel **)retainedChannel, + channelInstanceInfo); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_bind_channel_resources(nvidia_stack_t* sp, + void *retainedChannel, + nvgpuChannelResourceBindParams_t channelResourceBindParams) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsBindChannelResources(retainedChannel, + channelResourceBindParams); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_release_channel(nvidia_stack_t *sp, void *retainedChannel) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + nvGpuOpsReleaseChannel(retainedChannel); + NV_EXIT_RM_RUNTIME(sp, fp); +} + +void NV_API_CALL +rm_gpu_ops_stop_channel(nvidia_stack_t * sp, + void *retainedChannel, + NvBool bImmediate) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsStopChannel(retainedChannel, bImmediate); + NV_EXIT_RM_RUNTIME(sp, fp); +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_get_channel_resource_ptes(nvidia_stack_t* sp, + nvgpuAddressSpaceHandle_t vaSpace, + NvP64 resourceDescriptor, + NvU64 offset, + NvU64 size, + nvgpuExternalMappingInfo_t gpuExternalMappingInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp, fp); + rmStatus = nvGpuOpsGetChannelResourcePtes(vaSpace, resourceDescriptor, + offset, size, + gpuExternalMappingInfo); + NV_EXIT_RM_RUNTIME(sp, fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_report_non_replayable_fault(nvidia_stack_t *sp, + nvgpuDeviceHandle_t device, + const void *pFaultPacket) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsReportNonReplayableFault(device, pFaultPacket); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_paging_channel_allocate(nvidia_stack_t *sp, + gpuDeviceHandle device, + const gpuPagingChannelAllocParams *allocParams, + gpuPagingChannelHandle *channel, + gpuPagingChannelInfo *channelInfo) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPagingChannelAllocate(device, allocParams, channel, + channelInfo); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_paging_channel_destroy(nvidia_stack_t *sp, + gpuPagingChannelHandle channel) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsPagingChannelDestroy(channel); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_paging_channels_map(nvidia_stack_t *sp, + gpuAddressSpaceHandle srcVaSpace, + NvU64 srcAddress, + gpuDeviceHandle device, + NvU64 *dstAddress) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPagingChannelsMap(srcVaSpace, srcAddress, device, dstAddress); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + +void NV_API_CALL +rm_gpu_ops_paging_channels_unmap(nvidia_stack_t *sp, + gpuAddressSpaceHandle srcVaSpace, + NvU64 srcAddress, + gpuDeviceHandle device) +{ + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + nvGpuOpsPagingChannelsUnmap(srcVaSpace, srcAddress, device); + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL +rm_gpu_ops_paging_channel_push_stream(nvidia_stack_t *sp, + gpuPagingChannelHandle channel, + char *methodStream, + NvU32 methodStreamSize) +{ + NV_STATUS rmStatus; + void *fp; + NV_ENTER_RM_RUNTIME(sp,fp); + rmStatus = nvGpuOpsPagingChannelPushStream(channel, methodStream, methodStreamSize); + NV_EXIT_RM_RUNTIME(sp,fp); + return rmStatus; +} + diff --git a/userspace/sdks/nv525/nvalloc/unix/src/rmobjexportimport.c b/userspace/sdks/nv525/nvalloc/unix/src/rmobjexportimport.c new file mode 100644 index 0000000..dcc8d94 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/rmobjexportimport.c @@ -0,0 +1,634 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/*! + * @file + * + * @brief Provides RmExportObject, RmImportObject, RmFreeObjExportHandle and + * RmGetExportObjectInfo interfaces : + * + * These interfaces allow rm clients to export their objects into + * a unique RmObjExportHandle which another rm client could + * import, even if the source rm client gets destroyed. + * + * RM's device instance may get destroyed asynchronously, in which + * case exported objects residing on that device instance also get + * destroyed. This means it is not possible to import it back, but the + * RmObjExportHandle into which the object had been exported still + * remains valid but no other object could get it. + * + * There are not init/fini routines, it is the responsibility of the + * rest of RM's eco-system to make sure that all RmObjExportHandles get + * freed during driver unload. + * + * The api lock is expected to be held before calling into + * rmobjexportimport.c; do not hold gpu or any other lock. + */ + +#include "rmobjexportimport.h" +#include "nvlimits.h" +#include "gpu/device/device.h" + +#include "containers/map.h" +#include "rmapi/rmapi.h" +#include "rmapi/rs_utils.h" + +#include "class/cl0080.h" +#include "class/cl2080.h" +#include <ctrl/ctrl0000/ctrl0000unix.h> +#include <ctrl/ctrl0000/ctrl0000client.h> + +// +// A reference to an RmObjExportHandle +// generated by function RmGenerateObjExportHandle(). +// +typedef struct +{ + NvU32 deviceInstance; +} RmObjExportHandleRef; +MAKE_MAP(RmObjExportHandleMap, RmObjExportHandleRef); + +// +// Memory allocator +// +PORT_MEM_ALLOCATOR *pMemAllocator; + +// +// Map RmObjExportHandle -> RmObjExportHandleRef +// +RmObjExportHandleMap objExportHandleMap; + +// +// Rm client to use to dup an object exported to RmObjExportHandle. The minimal +// requirement for duping is to have a device object allocated. This rm client +// is simply like any other external rm client and has no any special handling. +// +// We keep this rm client just like any other external rm client: if +// gpu(s)/device gets powered-down/uninitialized, rm objects allocated by +// external rm clients and located on that gpu(s)/device gets freed (the +// os-layer does that). In that way, code in this file doesn't need to worry +// about freeing exported objects located on that gpu(s)/device. +// +NvHandle hObjExportRmClient; + +// +// Tracker for device and subdevice handles. For now only one subdevice +// (instance 0) is supported per device. +// +typedef struct +{ + NvHandle hRmDevice; + NvHandle hRmSubDevice; +} RmObjExportDevice; + +RmObjExportDevice objExportDevice[NV_MAX_DEVICES]; + +// +// Usage reference counter for static object in this file like rm client used to +// dup an exported object, memory allocator, map etc. +// +NvU64 objExportImportRefCount; + +// +// Static functions for internal use to code in this file. +// +static NV_STATUS RmRefObjExportImport (void); +static void RmUnrefObjExportImport (void); + +static RmObjExportHandle RmGenerateObjExportHandle (NvU32 deviceInstance); +static NV_STATUS RmUnrefObjExportHandle (RmObjExportHandle hObject); + +// +// Free the RmObjExportHandle. +// +static NV_STATUS RmUnrefObjExportHandle(RmObjExportHandle hObject) +{ + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + RmObjExportHandleRef *pHandleRef = + mapFind(&objExportHandleMap, hObject); + + if (pHandleRef == NULL) + { + return NV_ERR_OBJECT_NOT_FOUND; + } + + if (pRmApi->Free(pRmApi, + hObjExportRmClient, + (NvHandle)mapKey(&objExportHandleMap, pHandleRef)) != NV_OK) + { + NV_PRINTF(LEVEL_WARNING, + "Exported object trying to free was zombie in %s\n", + __FUNCTION__); + } + + mapRemove(&objExportHandleMap, pHandleRef); + + return NV_OK; +} + +// +// Generate unique RmObjExportHandle. +// +static RmObjExportHandle RmGenerateObjExportHandle(NvU32 deviceInstance) +{ + // + // The object export handle belongs to range of 0 to + // (MAX_OBJ_EXPORT_HANDLES - 1). + // + // Handle 0 is considered as invalid object handle, this function generates + // handle from range of 1 to (MAX_OBJ_EXPORT_HANDLES - 1). + // + #define MAX_OBJ_EXPORT_HANDLES 0x80000 + + static NvHandle hObjExportHandleNext = 1; + + RmObjExportHandle hStartHandle = hObjExportHandleNext; + RmObjExportHandle hObject = 0; + + do + { + RmObjExportHandleRef *pHandleRef; + + hObject = hObjExportHandleNext++; + /* Reset hObjExportHandleNext to next valid handle */ + if (hObjExportHandleNext == MAX_OBJ_EXPORT_HANDLES) { + hObjExportHandleNext = 1; + } + + pHandleRef = mapFind(&objExportHandleMap, hObject); + + if (hObject != hObjExportRmClient && pHandleRef == NULL) + { + break; + } + else + { + hObject = 0; + } + + } while(hObjExportHandleNext != hStartHandle); + + if (hObject != 0) + { + RmObjExportHandleRef *pHandleRef = + mapInsertNew(&objExportHandleMap, hObject); + + if (pHandleRef != NULL) + { + pHandleRef->deviceInstance = deviceInstance; + } + else + { + hObject = 0; + } + } + + return hObject; +} + +// +// Validate that the given hObject is not one of our internally used handles. +// +// Note that mapFind(&objExportHandleMap, hObject) could still fail; that is the +// caller's responsibility. +// +static NvBool RmValidateHandleAgainstInternalHandles(RmObjExportHandle hObject) +{ + NvU32 i; + + // + // No external RmObjExportHandle could be valid if hObjExportRmClient has + // not been allocated yet, or if it is equal to any of the handles used + // internally by code in this file. + // + if (objExportImportRefCount == 0 || hObjExportRmClient == 0 || + hObject == hObjExportRmClient) + { + return NV_FALSE; + } + + for (i = 0; i < NV_ARRAY_ELEMENTS(objExportDevice); i++) + { + if (objExportDevice[i].hRmDevice != 0 && + (hObject == objExportDevice[i].hRmDevice || + hObject == objExportDevice[i].hRmSubDevice)) + { + return NV_FALSE; + } + } + + return NV_TRUE; +} + +// +// Increment reference count of static objects internally +// used by code in this file. +// +static NV_STATUS RmRefObjExportImport(void) +{ + NV_STATUS rmStatus = NV_OK; + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + if ((objExportImportRefCount++) != 0) + { + NV_ASSERT(hObjExportRmClient != 0); + NV_ASSERT(pMemAllocator != NULL); + return NV_OK; + } + + rmStatus = pRmApi->AllocWithHandle(pRmApi, + NV01_NULL_OBJECT, + NV01_NULL_OBJECT, + NV01_NULL_OBJECT, + NV01_ROOT, + &hObjExportRmClient); + if (rmStatus != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Unable to alloc root in %s\n", __FUNCTION__); + goto failed; + } + + pMemAllocator = portMemAllocatorCreateNonPaged(); + + if (pMemAllocator == NULL) + { + NV_PRINTF(LEVEL_ERROR, "Failed to alloc memory allocator in %s\n", + __FUNCTION__); + goto failed; + } + + mapInit(&objExportHandleMap, pMemAllocator); + + return NV_OK; + +failed: + + RmUnrefObjExportImport(); + + return rmStatus; +} + +// +// Decrement reference count of static objects internally used by code in this +// file, and free them if reference count reaches to zero. +// +static void RmUnrefObjExportImport(void) +{ + RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); + + if ((--objExportImportRefCount) != 0) + { + return; + } + + if (pMemAllocator != NULL) + { + NvU32 i; + + for (i = 0; i < NV_ARRAY_ELEMENTS(objExportDevice); i++) + { + if (objExportDevice[i].hRmDevice != 0) + { + RmUnrefObjExportHandle(objExportDevice[i].hRmSubDevice); + objExportDevice[i].hRmSubDevice = 0; + RmUnrefObjExportHandle(objExportDevice[i].hRmDevice); + objExportDevice[i].hRmDevice = 0; + } + } + + mapDestroy(&objExportHandleMap); + + portMemAllocatorRelease(pMemAllocator); + pMemAllocator = NULL; + } + + if (hObjExportRmClient != 0) + { + NV_STATUS rmStatus = pRmApi->Free(pRmApi, + hObjExportRmClient, + hObjExportRmClient); + + NV_ASSERT(rmStatus == NV_OK); + hObjExportRmClient = 0; + } +} + +NV_STATUS RmExportObject(NvHandle hSrcClient, NvHandle hSrcObject, + RmObjExportHandle *pDstObject, NvU32 *pDeviceInstance) +{ + RmObjExportHandle hDstObject; + NvU32 deviceInstance = NV_MAX_DEVICES; + NvHandle hTmpObject; + NvBool bClientAsDstParent = NV_FALSE; + NV_STATUS status; + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + if (pDstObject == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + // + // Find the device instance on which the rm object exists. + // + hTmpObject = hSrcObject; + do + { + RsResourceRef *pResourceRef; + + status = serverutilGetResourceRef(hSrcClient, hTmpObject, &pResourceRef); + if (status != NV_OK) + return status; + + Device *pDevice = dynamicCast(pResourceRef->pResource, Device); + if (pDevice != NULL) + { + deviceInstance = pDevice->deviceInst; + break; + } + + hTmpObject = pResourceRef->pParentRef ? pResourceRef->pParentRef->hResource : 0; + } while (hTmpObject != 0); + + // If a memory object is not parented by a device, use client as a parent. + if ((hTmpObject == 0) || (deviceInstance >= NV_MAX_DEVICES)) + { + bClientAsDstParent = NV_TRUE; + } + + status = RmRefObjExportImport(); + + if (status != NV_OK) + { + return status; + } + + if (!bClientAsDstParent && + ((objExportDevice[deviceInstance].hRmDevice == 0) || + serverutilValidateNewResourceHandle(hObjExportRmClient, + objExportDevice[deviceInstance].hRmDevice))) + { + // + // Device object has not been created or it got destroyed in the + // teardown path of device instance destruction; allocate a fresh device + // object. + // + NV0080_ALLOC_PARAMETERS params; + NV2080_ALLOC_PARAMETERS subdevParams; + + if (objExportDevice[deviceInstance].hRmDevice == 0) + { + NV_ASSERT(objExportDevice[deviceInstance].hRmSubDevice == 0); + + objExportDevice[deviceInstance].hRmDevice = + RmGenerateObjExportHandle(deviceInstance); + objExportDevice[deviceInstance].hRmSubDevice = + RmGenerateObjExportHandle(deviceInstance); + + if (objExportDevice[deviceInstance].hRmDevice == 0 || + objExportDevice[deviceInstance].hRmSubDevice == 0) + { + NV_PRINTF(LEVEL_ERROR, "Failed to allocate object handles in %s\n", + __FUNCTION__); + + status = NV_ERR_NO_MEMORY; + goto done; + } + } + + portMemSet(¶ms, 0, sizeof(NV0080_ALLOC_PARAMETERS)); + + params.deviceId = deviceInstance; + + status = pRmApi->AllocWithHandle(pRmApi, + hObjExportRmClient, + hObjExportRmClient, + objExportDevice[deviceInstance].hRmDevice, + NV01_DEVICE_0, + ¶ms); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Unable to alloc device in %s\n", + __FUNCTION__); + goto done; + } + + portMemSet(&subdevParams, 0, sizeof(NV2080_ALLOC_PARAMETERS)); + + subdevParams.subDeviceId = 0; + + status = pRmApi->AllocWithHandle(pRmApi, + hObjExportRmClient, + objExportDevice[deviceInstance].hRmDevice, + objExportDevice[deviceInstance].hRmSubDevice, + NV20_SUBDEVICE_0, + &subdevParams); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, "Unable to alloc subdevice in %s\n", + __FUNCTION__); + + (void) pRmApi->Free(pRmApi, hObjExportRmClient, + objExportDevice[deviceInstance].hRmDevice); + goto done; + } + } + + hDstObject = RmGenerateObjExportHandle(deviceInstance); + + if (hDstObject == 0) + { + NV_PRINTF(LEVEL_ERROR, "Failed to allocate object handle in %s\n", + __FUNCTION__); + status = NV_ERR_NO_MEMORY; + goto done; + } + + // If duping under device handle fails, try subdevice handle. + status = pRmApi->DupObject(pRmApi, + hObjExportRmClient, + bClientAsDstParent ? hObjExportRmClient : + objExportDevice[deviceInstance].hRmDevice, + &hDstObject, + hSrcClient, + hSrcObject, + 0 /* flags */); + if (status != NV_OK) + { + if (!bClientAsDstParent && (status == NV_ERR_INVALID_OBJECT_PARENT)) + { + NV_PRINTF(LEVEL_INFO, + "pRmApi->DupObject(Dev, failed due to invalid parent in %s." + " Now attempting DupObject with Subdev handle.\n", + __FUNCTION__); + + status = pRmApi->DupObject(pRmApi, + hObjExportRmClient, + objExportDevice[deviceInstance].hRmSubDevice, + &hDstObject, + hSrcClient, + hSrcObject, + 0 /* flags */); + if (status != NV_OK) + { + RmUnrefObjExportHandle(hDstObject); + + NV_PRINTF(LEVEL_ERROR, + "pRmApi->DupObject(Subdev, failed with error code 0x%x in %s\n", + status, __FUNCTION__); + goto done; + } + } + else + { + RmUnrefObjExportHandle(hDstObject); + + NV_PRINTF(LEVEL_ERROR, + "pRmApi->DupObject(Dev, failed with error code 0x%x in %s\n", + status, __FUNCTION__); + goto done; + } + } + + if (pDeviceInstance != NULL) + { + *pDeviceInstance = deviceInstance; + } + + *pDstObject = hDstObject; + +done: + if (status != NV_OK) + { + RmUnrefObjExportImport(); + } + + return status; +} + +void RmFreeObjExportHandle(RmObjExportHandle hObject) +{ + if (!RmValidateHandleAgainstInternalHandles(hObject)) + { + NV_PRINTF(LEVEL_ERROR, "Invalid handle to exported object in %s\n", + __FUNCTION__); + return; + } + + RmUnrefObjExportHandle(hObject); + + RmUnrefObjExportImport(); +} + +NV_STATUS RmImportObject(NvHandle hDstClient, NvHandle hDstParent, + NvHandle *phDstObject, RmObjExportHandle hSrcObject, + NvU8 *pObjectType) +{ + NV_STATUS status; + NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS params; + RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); + + if (!RmValidateHandleAgainstInternalHandles(hSrcObject)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if (mapFind(&objExportHandleMap, hSrcObject) == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + if (pObjectType != NULL) + { + params.hObject = hSrcObject; + params.mapFlags = 0; + params.addrSpaceType = \ + NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_INVALID; + + status = pRmApi->Control(pRmApi, hObjExportRmClient, hObjExportRmClient, + NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE, + ¶ms, sizeof(params)); + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "GET_ADDR_SPACE_TYPE failed with error code 0x%x in %s\n", + status, __FUNCTION__); + return status; + } + + switch (params.addrSpaceType) + { + case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_SYSMEM: + *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_SYSMEM; + break; + case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_VIDMEM: + *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_VIDMEM; + break; + case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC: + *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC; + break; +#if defined(NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC) && \ + defined(NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC) + case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC: + *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC; + break; +#endif + default: + NV_ASSERT_OK_OR_RETURN(NV_ERR_INVALID_ARGUMENT); + } + } + + status = pRmApi->DupObject(pRmApi, hDstClient, hDstParent, phDstObject, + hObjExportRmClient, hSrcObject, + 0 /* flags */); + + if (status != NV_OK) + { + NV_PRINTF(LEVEL_ERROR, + "pRmApi->DupObject(pRmApi, failed with error code 0x%x in %s\n", + status, __FUNCTION__); + return status; + } + + return NV_OK; +} + +NV_STATUS RmGetExportObjectInfo(RmObjExportHandle hSrcObject, NvU32 *deviceInstance) +{ + RmObjExportHandleRef *pHandleRef = NULL; + + if (!RmValidateHandleAgainstInternalHandles(hSrcObject)) + { + return NV_ERR_INVALID_ARGUMENT; + } + + pHandleRef = mapFind(&objExportHandleMap, hSrcObject); + if (pHandleRef == NULL) + { + return NV_ERR_OBJECT_NOT_FOUND; + } + + *deviceInstance = pHandleRef->deviceInstance; + return NV_OK; +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/subdevice_ctrl_os_unix.c b/userspace/sdks/nv525/nvalloc/unix/src/subdevice_ctrl_os_unix.c new file mode 100644 index 0000000..6989c7c --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/subdevice_ctrl_os_unix.c @@ -0,0 +1,100 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> // NV device driver interface +#include <nv-priv.h> +#include <os/os.h> +#include <nvos.h> +#include <osapi.h> +#include "gpu/gpu.h" +#include "gpu/gpu_resource.h" +#include "gpu/subdevice/subdevice.h" +#include <osfuncs.h> +#include <diagnostics/journal.h> +#include "gpu/mem_mgr/mem_desc.h" +#include "mem_mgr/mem.h" +#include <nvpcie.h> +#include <core/locks.h> +#include "rmapi/rs_utils.h" +#include "rmapi/client_resource.h" +#include <class/cl0000.h> +#include <class/cl90cd.h> +#include <class/cl0005.h> // NV01_EVENT +#include <class/cl003e.h> // NV01_MEMORY_SYSTEM +#include <class/cl844c.h> // G84_PERFBUFFER +#include <ctrl/ctrl0000/ctrl0000gpu.h> +#include <ctrl/ctrl0000/ctrl0000unix.h> +#include <ctrl/ctrl2080/ctrl2080gpu.h> +#include <ctrl/ctrl2080/ctrl2080unix.h> + +/*! + * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS + * RmControl request. It will check if the GPU video memory will be + * persistent during system suspend/resume cycle. + * + * @param[in] pSubdevice + * @param[in,out] pParams + * + * @return + * NV_OK Success + */ +NV_STATUS +subdeviceCtrlCmdOsUnixVidmemPersistenceStatus_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS *pParams +) +{ + OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + + pParams->bVidmemPersistent = !gpuIsVidmemPreservationBrokenBug3172217(pGpu) && + (nv->preserve_vidmem_allocations || + nvp->s0ix_pm_enabled); + + return NV_OK; +} + +/*! + * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS + * RmControl request. It sets restore TGP flag which is used + * to restore TGP limits when client is killed. + * + * @param[in] pSubdevice + * @param[in] pParams + * + * @return + * NV_OK Success + */ +NV_STATUS +subdeviceCtrlCmdOsUnixUpdateTgpStatus_IMPL +( + Subdevice *pSubdevice, + NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS *pParams +) +{ + pSubdevice->bUpdateTGP = pParams->bUpdateTGP; + + return NV_OK; +} diff --git a/userspace/sdks/nv525/nvalloc/unix/src/unix_console.c b/userspace/sdks/nv525/nvalloc/unix/src/unix_console.c new file mode 100644 index 0000000..8b1c055 --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/unix_console.c @@ -0,0 +1,118 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <core/locks.h> +#include <ctrl/ctrl0080/ctrl0080unix.h> +#include <gpu/device/device.h> +#include <gpu/gpu.h> +#include <nv-priv.h> +#include <nv.h> +#include <osapi.h> +#include <gpu/mem_mgr/mem_mgr.h> + +NV_STATUS deviceCtrlCmdOsUnixVTGetFBInfo_IMPL +( + Device *pDevice, + NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS *pParams +) +{ + OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); + NvBool bContinue = NV_TRUE; + + if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FB) == NV_OK) + { + // See if the console is on one of the subdevices of this device. + portMemSet(pParams, 0, sizeof(*pParams)); + + SLI_LOOP_START(SLI_LOOP_FLAGS_NONE) + + MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + + if ((memmgrGetReservedConsoleMemDesc(pGpu, pMemoryManager) != NULL) && bContinue) + { + NvU64 baseAddr; + + // There should only be one. + NV_ASSERT(pParams->width == 0); + + pParams->subDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu); + + // Console is either mapped to BAR1 or BAR2 + 16 MB + os_get_screen_info(&baseAddr, &pParams->width, + &pParams->height, &pParams->depth, + &pParams->pitch, + nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address, + nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000); + + if (baseAddr != 0) + { + bContinue = NV_FALSE; + } + } + + SLI_LOOP_END + + rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); + } + else + { + NV_PRINTF(LEVEL_INFO,"%s: Failed to acquire GPU lock", __FUNCTION__); + } + + return NV_OK; +} + + +void +dispdeviceFillVgaSavedDisplayState +( + OBJGPU *pGpu, + NvU64 vgaAddr, + NvU8 vgaMemType, + NvBool vgaValid, + NvU64 workspaceAddr, + NvU8 workspaceMemType, + NvBool workspaceValid, + NvBool baseValid, + NvBool workspaceBaseValid +) +{ + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvBool use_vbios = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv); + + if (use_vbios) + { + nvp->vga.base.addr = vgaAddr; + nvp->vga.base.memTarget = vgaMemType; + nvp->vga.base.valid = vgaValid; + nvp->vga.baseValid = baseValid; + + nvp->vga.workspaceBase.addr = workspaceAddr; + nvp->vga.workspaceBase.memTarget = workspaceMemType; + nvp->vga.workspaceBase.valid = workspaceValid; + nvp->vga.workspaceBaseValid = workspaceBaseValid; + } +} + diff --git a/userspace/sdks/nv525/nvalloc/unix/src/unix_intr.c b/userspace/sdks/nv525/nvalloc/unix/src/unix_intr.c new file mode 100644 index 0000000..d60a30d --- /dev/null +++ b/userspace/sdks/nv525/nvalloc/unix/src/unix_intr.c @@ -0,0 +1,700 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include <nv.h> +#include <nv-priv.h> +#include <osapi.h> +#include <core/thread_state.h> +#include <core/locks.h> +#include <gpu/gpu.h> +#include "kernel/gpu/intr/intr.h" +#include "gpu/bif/kernel_bif.h" +#include "gpu/mmu/kern_gmmu.h" +#include "gpu/disp/kern_disp.h" +#include <nv_sriov_defines.h> +#include "objtmr.h" + + +static NvBool osInterruptPending( + OBJGPU *pGpu, + NvBool *serviced, + THREAD_STATE_NODE *pThreadState +) +{ + POBJDISP pDisp; + KernelDisplay *pKernelDisplay; + NvBool pending, sema_release; + THREAD_STATE_NODE threadState; + NvU32 gpuMask, gpuInstance; + Intr *pIntr = NULL; + MC_ENGINE_BITVECTOR intr0Pending; + MC_ENGINE_BITVECTOR intr1Pending; + + *serviced = NV_FALSE; + pending = NV_FALSE; + sema_release = NV_TRUE; + OBJGPU *pDeviceLockGpu = pGpu; + NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer + PORT_MEM_ALLOCATOR *pIsrAllocator; + + // + // GPU interrupt servicing ("top half") + // + // Top-level processing of GPU interrupts is performed using the + // steps below; although the code is straight forward, there + // are a few points to be aware of: + // + // 1) The GPUs lock is acquired for two reasons: to allow + // looping over GPUs atomically in SLI and to sanity + // check the PCI configuration space of any initialized + // GPUs. If the acquisition fails, the early return + // is acceptable since GPU interrupts are disabled while + // the lock is held; note that returning success + // in this case could interfere with the processing + // of third-party device interrupts if the IRQ is shared. + // Due to the above, some interrupts may be reported as + // unhandled if invocations of the ISR registered with + // the kernel are not serialized. This is bad, but + // ignored by currently supported kernels, provided most + // interrupts are handled. + // + // 2) Since acquisition of the lock disables interrupts + // on all initialized GPUs, NV_PMC_INTR_EN_0 can not be + // relied up on to determine whether interrupts are + // expected from a given GPU. The code below is therefore + // forced to rely on software state. NV_PMC_INTR_EN_0 + // is read only as a sanity check to guard against + // invalid GPU state (lack of PCI memory access, etc.). + // + // 3) High priority interrupts (VBLANK, etc.), are serviced in + // this function, service of all other interrupts is + // deferred until a bottom half. If a bottom half needs + // to be scheduled, release of the GPUs lock is + // likewise deferred until completion of the bottom half. + // + // 4) To reduce the risk of starvation, an effort is made to + // consolidate processing of interrupts pending on + // all GPUs sharing a given IRQ. + // + // 5) Care is taken to ensure that the consolidated interrupt + // processing is performed in the context of a GPU + // that has interrupts pending. Else if additional ISR + // processing via a bottom-half is required, this + // bottom-half ISR might race against the GPU's shut-down + // path. + // + + pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); + tlsIsrInit(pIsrAllocator); + + // For SWRL granular locking process the countdown timer interrupt. + if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_SWRL_GRANULAR_LOCKING)) + { + threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + + gpuMask = gpumgrGetGpuMask(pDeviceLockGpu); + + gpuInstance = 0; + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + pIntr = GPU_GET_INTR(pGpu); + + if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)) + { + // If interrupt enable is garbage the GPU is probably in a bad state + if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX) + { + continue; + } + + intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState); + POBJTMR pTmr = GPU_GET_TIMER(pGpu); + *serviced = tmrServiceSwrlWrapper(pGpu, pTmr, &intr0Pending, &threadState); + } + } + + threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + } + + // LOCK: try to acquire GPUs lock + if (rmDeviceGpuLocksAcquire(pDeviceLockGpu, GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_ISR) == NV_OK) + { + threadStateInitISRAndDeferredIntHandler(&threadState, + pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR); + + gpuMask = gpumgrGetGpuMask(pDeviceLockGpu); + + gpuInstance = 0; + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + pIntr = GPU_GET_INTR(pGpu); + pDisp = GPU_GET_DISP(pGpu); + pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); + + if ((pDisp != NULL) && pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + } + else if ((pIntr != NULL) && INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)) + { + // If interrupt enable is garbage the GPU is probably in a bad state + if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX) + continue; + + intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState); + if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP)) + { + if (pKernelDisplay != NULL) + { + kdispServiceVblank_HAL(pGpu, pKernelDisplay, 0, + (VBLANK_STATE_PROCESS_LOW_LATENCY | + VBLANK_STATE_PROCESS_CALLED_FROM_ISR), + &threadState); + *serviced = NV_TRUE; + intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState); + } + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) && + !pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS)) + { + pIntr = GPU_GET_INTR(pGpu); + if (pIntr != NULL) + { + NvBool bCtxswLog = NV_FALSE; + intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState); + intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog); + } + } + + if (!bitVectorTestAllCleared(&intr0Pending) || + !bitVectorTestAllCleared(&intr1Pending)) + { + pending = NV_TRUE; + sema_release = NV_FALSE; + } + } + } + threadStateFreeISRAndDeferredIntHandler(&threadState, + pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR); + + if (sema_release) + { + NV_ASSERT(!pending); + + // UNLOCK: release GPUs lock + rmDeviceGpuLocksRelease(pDeviceLockGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } + else + { + rmDeviceGpuLockSetOwner(pDeviceLockGpu, GPUS_LOCK_OWNER_PENDING_DPC_REFRESH); + } + } + + if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) && + pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS)) + { + threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + + gpuMask = gpumgrGetGpuMask(pDeviceLockGpu); + gpuInstance = 0; + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + pIntr = GPU_GET_INTR(pGpu); + if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))) + { + NvBool bCtxswLog = NV_FALSE; + intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState); + intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog); + if (!bitVectorTestAllCleared(&intr1Pending)) + { + intrServiceNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState); + *serviced = NV_TRUE; + } + } + } + threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + } + + tlsIsrDestroy(pIsrAllocator); + portMemAllocatorRelease(pIsrAllocator); + + return pending; +} + +NV_STATUS osIsr( + OBJGPU *pGpu +) +{ + NV_STATUS status = NV_OK; + nv_state_t *nv = NV_GET_NV_STATE(pGpu); + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + NvBool pending = NV_FALSE; + NvBool serviced = NV_FALSE; + Intr *pIntr; + + if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) + { + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */); + } + else + { + pIntr = GPU_GET_INTR(pGpu); + if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)) + { + KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); + pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */); + kbifCheckAndRearmMSI(pGpu, pKernelBif); + } + } + } + + if (!pending && (IS_VIRTUAL(pGpu) || !serviced)) + status = NV_ERR_NO_INTR_PENDING; + else if (pending) + status = NV_WARN_MORE_PROCESSING_REQUIRED; + + return status; +} + +/* + * Helper function to determine when the RM SEMA/GPUS LOCK should toggle + * interrupts. Based on the state of the GPU - we must add cases here as we + * discover them. + * + * Noteworthy special cases: + * + * - Suspend/resume: the GPU could still be suspended and not accessible + * on the bus, while passive-level threads need to grab the GPUs + * lock, or other GPUs are being resumed and triggering interrupts. + * + * - SLI state transitions: interrupts are disabled manually prior to + * removing GPUs from the lock mask leading up to SLI link/unlink + * operations on UNIX, but since the GPUs lock is not held by design in + * these paths, it needs to be ensured that GPUs lock acquisitions + * occurring aynchronously do not re-enable interrupts on any of the + * GPUs undergoing the SLI state transition. + * + * @param[in] pGpu OBJGPU pointer + * + * @return NV_TRUE if the RM SEMA/GPUS LOCK should toggle interrupts, NV_FALSE + * otherwise. + */ +NvBool osLockShouldToggleInterrupts(OBJGPU *pGpu) +{ + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + return NV_TRUE; + + return (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH) && + gpuIsStateLoaded(pGpu) && + !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SLI_LINK_CODEPATH)); +} + +void osEnableInterrupts(OBJGPU *pGpu) +{ + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + // enable irq through os call + nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_TRUE); + return; + } + else + { + Intr *pIntr = GPU_GET_INTR(pGpu); + NvU32 intrEn; + + if (!pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING)) + NV_ASSERT(intrGetIntrEnFromHw_HAL(pGpu, pIntr, NULL) == INTERRUPT_TYPE_DISABLED); + + intrEn = intrGetIntrEn(pIntr); + intrSetIntrEnInHw_HAL(pGpu, pIntr, intrEn, NULL); + + if (pIntr != NULL) + { + intrSetStall_HAL(pGpu, pIntr, intrEn, NULL); + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED)) + { + if (pIntr != NULL) + { + intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL); + } + } + + } +} + +void osDisableInterrupts( + OBJGPU *pGpu, + NvBool bIsr +) +{ + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + // disable irq through os call + nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_FALSE); + return; + } + else + { + Intr *pIntr = GPU_GET_INTR(pGpu); + NvU32 new_intr_en_0 = INTERRUPT_TYPE_DISABLED; + + intrSetIntrEnInHw_HAL(pGpu, pIntr, new_intr_en_0, NULL); + + if (pIntr != NULL) + { + intrSetStall_HAL(pGpu, pIntr, new_intr_en_0, NULL); + } + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED)) + { + if (pIntr != NULL) + { + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS)) + { + intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL); + } + else + { + intrRestoreNonStall_HAL(pGpu, pIntr, new_intr_en_0, NULL); + } + } + } + } +} + +static void RmIsrBottomHalf( + nv_state_t *pNv +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + THREAD_STATE_NODE threadState; + OS_THREAD_HANDLE threadId; + NvU32 gpuMask, gpuInstance; + OBJGPU *pDeviceLockGpu = pGpu; + Intr *pIntr = NULL; + POBJDISP pDisp = NULL; + NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer + PORT_MEM_ALLOCATOR *pIsrAllocator; + + pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); + tlsIsrInit(pIsrAllocator); + + // + // The owning thread changes as the ISR acquires the GPUs lock, + // but the bottom half releases it. Refresh the ThreadId owner to be + // correct here for the bottom half context. + // + osGetCurrentThread(&threadId); + rmDeviceGpuLockSetOwner(pDeviceLockGpu, threadId); + + gpuMask = gpumgrGetGpuMask(pGpu); + + gpuInstance = 0; + while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) + { + + threadStateInitISRAndDeferredIntHandler(&threadState, + pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER); + + pIntr = GPU_GET_INTR(pGpu); + pDisp = GPU_GET_DISP(pGpu); + + // + // Call disp service incase of SOC Display, + // TODO : with multi interrupt handling based on irq aux interrupts are serviced by dpAuxService + // See JIRA task TDS-4253. + // + if ((pDisp != NULL) && pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) + { + } + else if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))) + { + intrServiceStall_HAL(pGpu, pIntr); + + if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) && + !pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS)) + { + MC_ENGINE_BITVECTOR intrPending; + intrServiceNonStall_HAL(pGpu, pIntr, &intrPending, &threadState); + } + } + + threadStateFreeISRAndDeferredIntHandler(&threadState, + pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER); + } + + // UNLOCK: release GPUs lock + rmDeviceGpuLocksRelease(pDeviceLockGpu, GPUS_LOCK_FLAGS_NONE, NULL); + + tlsIsrDestroy(pIsrAllocator); + portMemAllocatorRelease(pIsrAllocator); +} + +static void RmIsrBottomHalfUnlocked( + nv_state_t *pNv +) +{ + OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); + Intr *pIntr; + THREAD_STATE_NODE threadState; + + // In the GSP client scenario, the fatal fault interrupt is not shared + // by UVM and CPU-RM. Instead, it is handled entirely by GSP-RM. We + // therefore do not expect this function to be called. But if it is, bail + // without attempting to service interrupts. + if (IS_GSP_CLIENT(pGpu)) + { + return; + } + + // Grab GPU lock here as this kthread-item was enqueued without grabbing GPU lock + if (rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DPC) == NV_OK) + { + if (FULL_GPU_SANITY_CHECK(pGpu)) + { + pIntr = GPU_GET_INTR(pGpu); + + threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); + + if (intrGetIntrEn(pIntr) != INTERRUPT_TYPE_DISABLED) + { + MC_ENGINE_BITVECTOR intrPending; + intrGetPendingStall_HAL(pGpu, pIntr, &intrPending, &threadState); + intrServiceNonStallBottomHalf(pGpu, pIntr, &intrPending, &threadState); + } + + threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); + } + + rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); + } +} + +NvBool NV_API_CALL rm_isr( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 *NeedBottomHalf +) +{ + NV_STATUS status; + nv_priv_t *nvp = NV_GET_NV_PRIV(nv); + OBJGPU *pGpu; + NvBool retval; + void *fp; + + if ((nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) == 0) + { + return NV_FALSE; + } + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL) + { + return NV_FALSE; + } + + NV_ENTER_RM_RUNTIME(sp,fp); + + // call actual isr function here + status = isrWrapper(pGpu->testIntr, pGpu); + + switch (status) + { + case NV_OK: + *NeedBottomHalf = NV_FALSE; + retval = NV_TRUE; + break; + case NV_WARN_MORE_PROCESSING_REQUIRED: + *NeedBottomHalf = NV_TRUE; + retval = NV_TRUE; + break; + case NV_ERR_NO_INTR_PENDING: + default: + *NeedBottomHalf = NV_FALSE; + retval = NV_FALSE; + break; + } + + NV_EXIT_RM_RUNTIME(sp,fp); + + return retval; +} + +void NV_API_CALL rm_isr_bh( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + RmIsrBottomHalf(pNv); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +void NV_API_CALL rm_isr_bh_unlocked( + nvidia_stack_t *sp, + nv_state_t *pNv +) +{ + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + RmIsrBottomHalfUnlocked(pNv); + + NV_EXIT_RM_RUNTIME(sp,fp); +} + +NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 *faultsCopied +) +{ + NV_STATUS status = NV_OK; + OBJGPU *pGpu; + void *fp; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL || faultsCopied == NULL) + { + status = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + // Non-replayable faults are copied to the client shadow buffer by GSP-RM. + if (IS_GSP_CLIENT(pGpu)) + { + status = NV_ERR_NOT_SUPPORTED; + goto done; + } + +done: + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +// +// Use this call when MMU faults needs to be copied +// outisde of RM lock. +// +NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults_unlocked( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 *faultsCopied +) +{ + OBJGPU *pGpu; + void *fp; + NV_STATUS status = NV_OK; + + NV_ENTER_RM_RUNTIME(sp,fp); + + pGpu = NV_GET_NV_PRIV_PGPU(nv); + if (pGpu == NULL || faultsCopied == NULL) + { + status = NV_ERR_OBJECT_NOT_FOUND; + goto done; + } + + // Non-replayable faults are copied to the client shadow buffer by GSP-RM. + if (IS_GSP_CLIENT(pGpu)) + { + status = NV_ERR_NOT_SUPPORTED; + goto done; + } + +done: + NV_EXIT_RM_RUNTIME(sp,fp); + + return status; +} + +// +// Wrapper to handle calls to copy mmu faults +// +NV_STATUS rm_gpu_handle_mmu_faults( + nvidia_stack_t *sp, + nv_state_t *nv, + NvU32 *faultsCopied +) +{ + NvU32 status = NV_OK; + + *faultsCopied = 0; + + OBJGPU *pGpu; + pGpu = NV_GET_NV_PRIV_PGPU(nv); + + if (pGpu == NULL) + { + return NV_ERR_OBJECT_NOT_FOUND; + } + + if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && !IS_VIRTUAL(pGpu)) + { + THREAD_STATE_NODE threadState; + + KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); + Intr *pIntr = GPU_GET_INTR(pGpu); + + NvU32 hw_put = 0; + NvU32 hw_get = 0; + + threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + + kgmmuReadFaultBufferPutPtr_HAL(pGpu, pKernelGmmu, NON_REPLAYABLE_FAULT_BUFFER, + &hw_put, &threadState); + + kgmmuReadFaultBufferGetPtr_HAL(pGpu, pKernelGmmu, NON_REPLAYABLE_FAULT_BUFFER, + &hw_get, &threadState); + + if(hw_get != hw_put) + { + // We have to clear the top level interrupt bit here since otherwise + // the bottom half will attempt to service the interrupt on the CPU + // side before GSP recieves the notification and services it + kgmmuClearNonReplayableFaultIntr(pGpu, pKernelGmmu, &threadState); + status = intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_NON_REPLAYABLE_FAULT_HANDLE); + + } + + threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); + } + else + { + status = rm_gpu_copy_mmu_faults_unlocked(sp, nv, faultsCopied); + } + return status; +} + |