summaryrefslogtreecommitdiffstats
path: root/userspace/sdks/nv525/nvalloc/unix/src
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-09-13 18:36:21 -0400
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-09-13 18:36:21 -0400
commit00e54c13c7c9030c4d5f7151467f2d85a3730e29 (patch)
tree17a45239df8d2e4912ff3697ed06e85a10b969de /userspace/sdks/nv525/nvalloc/unix/src
downloadopen-iov-00e54c13c7c9030c4d5f7151467f2d85a3730e29.tar.gz
open-iov-00e54c13c7c9030c4d5f7151467f2d85a3730e29.zip
Initial commit: Open-IOV vGPU-IO complete stack
- Firmware decompilation: GSP 570.144W (28.5MB ELF, RV64IMAC), 132 decompiled C files - Kernel modules: nvidia.ko (34MB, OpenRM), open-iov-vfio.ko (2.8MB, VFIO mdev), gsp-vgpu.ko (1.8MB, guest driver) - Userspace: libiov.so/libvgpu.so (76KB), example_vgpu (28KB), rpc_emulator.c - PCI config space emulation (256B with PM, MSI, MSI-X, PCIe caps) - 12 RPC types fully mapped from firmware decompilation - BAR trap, WPR, scheduling, notification, migration emulation - Helper scripts: load_vgpu_stack, create_mdev, launch_qemu_vgpu, test_vgpu_stack, setup_guest_driver, unload_vgpu_stack - Architecture documentation - Wiki: 10 technical documents - Git LFS for large binaries (.ko, .bin, .iso, .img, .o, ghidra repos) - nvidia-open-gpu-kernel-modules as submodule
Diffstat (limited to 'userspace/sdks/nv525/nvalloc/unix/src')
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_cpuid.c52
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_rdcr.c44
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/escape.c938
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/exports-stubs.c235
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/gcc_helper.c35
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/os-hypervisor.c1023
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/os.c5238
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/osapi.c5333
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/osinit.c2202
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/osmemdesc.c1030
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/osnvlink.c698
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/osunix.c88
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/registry.c705
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/rm-gpu-ops.c813
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/rmobjexportimport.c634
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/subdevice_ctrl_os_unix.c100
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/unix_console.c118
-rw-r--r--userspace/sdks/nv525/nvalloc/unix/src/unix_intr.c700
18 files changed, 19986 insertions, 0 deletions
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_cpuid.c b/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_cpuid.c
new file mode 100644
index 0000000..0ebb1c8
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_cpuid.c
@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <os/os.h>
+
+/* inline assembler routines for UNIX platforms */
+
+#if defined(NVCPU_X86_64)
+
+NvS32 nv_cpuid(
+ OBJOS *pOS,
+ NvS32 op,
+ NvS32 subop,
+ NvU32 *eax,
+ NvU32 *ebx,
+ NvU32 *ecx,
+ NvU32 *edx
+)
+{
+ asm volatile (" cpuid \n"
+ : "=a" (*eax),
+ "=b" (*ebx),
+ "=c" (*ecx),
+ "=d" (*edx)
+ : "a" (op),
+ "c" (subop)
+ : "cc");
+
+ return 1;
+}
+
+#endif
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_rdcr.c b/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_rdcr.c
new file mode 100644
index 0000000..56e6bfd
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/asm/x86/nv_rdcr.c
@@ -0,0 +1,44 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <os/os.h>
+
+/* inline assembler routines for UNIX platforms */
+
+#if defined(NVCPU_X86_64)
+
+NvU32 nv_rdcr4(OBJOS *pOS)
+{
+ NvU64 val;
+ asm volatile ("movq %%cr4,%0" : "=r" (val));
+ return (NvU32)val;
+}
+
+NvU64 nv_rdcr3(OBJOS *pOS)
+{
+ NvU64 val;
+ asm volatile ("movq %%cr3,%0" : "=r" (val));
+ return val;
+}
+
+#endif
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/escape.c b/userspace/sdks/nv525/nvalloc/unix/src/escape.c
new file mode 100644
index 0000000..c7b451a
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/escape.c
@@ -0,0 +1,938 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+
+//***************************** Module Header **********************************
+//
+// This code is linked into the resource manager proper. It receives the
+// ioctl from the resource manager's customer, unbundles the args and
+// calls the correct resman routines.
+//
+//******************************************************************************
+
+#include <core/prelude.h>
+#include <core/locks.h>
+#include <nv.h>
+#include <nv_escape.h>
+#include <osapi.h>
+#include <rmapi/exports.h>
+#include <nv-unix-nvos-params-wrappers.h>
+
+#include <nvos.h>
+#include <class/cl0000.h> // NV01_ROOT
+#include <class/cl0001.h> // NV01_ROOT_NON_PRIV
+#include <class/cl0005.h> // NV01_EVENT
+#include <class/cl003e.h> // NV01_MEMORY_SYSTEM
+#include <class/cl0071.h> // NV01_MEMORY_SYSTEM_OS_DESCRIPTOR
+
+#include <ctrl/ctrl00fd.h>
+
+#define NV_CTL_DEVICE_ONLY(nv) \
+{ \
+ if (((nv)->flags & NV_FLAG_CONTROL) == 0) \
+ { \
+ rmStatus = NV_ERR_INVALID_ARGUMENT; \
+ goto done; \
+ } \
+}
+
+#define NV_ACTUAL_DEVICE_ONLY(nv) \
+{ \
+ if (((nv)->flags & NV_FLAG_CONTROL) != 0) \
+ { \
+ rmStatus = NV_ERR_INVALID_ARGUMENT; \
+ goto done; \
+ } \
+}
+
+static NvBool RmIsDeviceRefNeeded(NVOS54_PARAMETERS *pApi)
+{
+ switch(pApi->cmd)
+ {
+ case NV00FD_CTRL_CMD_ATTACH_MEM:
+ return NV_TRUE;
+ default:
+ return NV_FALSE;
+ }
+}
+
+static NV_STATUS RmGetDeviceFd(NVOS54_PARAMETERS *pApi, NvS32 *pFd)
+{
+ RMAPI_PARAM_COPY paramCopy;
+ void *pKernelParams;
+ NvU32 paramSize;
+ NV_STATUS status;
+
+ *pFd = -1;
+
+ switch(pApi->cmd)
+ {
+ case NV00FD_CTRL_CMD_ATTACH_MEM:
+ paramSize = sizeof(NV00FD_CTRL_ATTACH_MEM_PARAMS);
+ break;
+ default:
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ RMAPI_PARAM_COPY_INIT(paramCopy, pKernelParams, pApi->params, paramSize, 1);
+
+ status = rmapiParamsAcquire(&paramCopy, NV_TRUE);
+ if (status != NV_OK)
+ return status;
+
+ switch(pApi->cmd)
+ {
+ case NV00FD_CTRL_CMD_ATTACH_MEM:
+ *pFd = (NvS32)((NV00FD_CTRL_ATTACH_MEM_PARAMS *)pKernelParams)->devDescriptor;
+ break;
+ default:
+ NV_ASSERT(0);
+ break;
+ }
+
+ NV_ASSERT(rmapiParamsRelease(&paramCopy) == NV_OK);
+
+ return status;
+}
+
+// Only return errors through pApi->status
+static void RmCreateOsDescriptor(NVOS32_PARAMETERS *pApi, API_SECURITY_INFO secInfo)
+{
+ NV_STATUS rmStatus;
+ NvBool writable;
+ NvU32 flags = 0;
+ NvU64 allocSize, pageCount, *pPteArray = NULL;
+ void *pDescriptor, *pPageArray = NULL;
+
+ pDescriptor = NvP64_VALUE(pApi->data.AllocOsDesc.descriptor);
+ if (((NvUPtr)pDescriptor & ~os_page_mask) != 0)
+ {
+ rmStatus = NV_ERR_NOT_SUPPORTED;
+ goto done;
+ }
+
+ // Check to prevent an NvU64 overflow
+ if ((pApi->data.AllocOsDesc.limit + 1) == 0)
+ {
+ rmStatus = NV_ERR_INVALID_LIMIT;
+ goto done;
+ }
+
+ allocSize = (pApi->data.AllocOsDesc.limit + 1);
+ pageCount = (1 + ((allocSize - 1) / os_page_size));
+
+ writable = FLD_TEST_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_WRITE, pApi->data.AllocOsDesc.attr2);
+
+ flags = FLD_SET_DRF_NUM(_LOCK_USER_PAGES, _FLAGS, _WRITE, writable, flags);
+ rmStatus = os_lock_user_pages(pDescriptor, pageCount, &pPageArray, flags);
+ if (rmStatus == NV_OK)
+ {
+ pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPageArray;
+ pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY;
+ }
+ else if (rmStatus == NV_ERR_INVALID_ADDRESS)
+ {
+ rmStatus = os_lookup_user_io_memory(pDescriptor, pageCount,
+ &pPteArray, &pPageArray);
+ if (rmStatus == NV_OK)
+ {
+ if (pPageArray != NULL)
+ {
+ pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPageArray;
+ pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY;
+ }
+ else if (pPteArray != NULL)
+ {
+ pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPteArray;
+ pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY;
+ }
+ else
+ {
+ NV_ASSERT_FAILED("unknown memory import type");
+ rmStatus = NV_ERR_NOT_SUPPORTED;
+ }
+ }
+ }
+ if (rmStatus != NV_OK)
+ goto done;
+
+ Nv04VidHeapControlWithSecInfo(pApi, secInfo);
+
+ if (pApi->status != NV_OK)
+ {
+ switch (pApi->data.AllocOsDesc.descriptorType)
+ {
+ default:
+ break;
+ case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY:
+ os_unlock_user_pages(pageCount, pPageArray);
+ break;
+ }
+ }
+
+done:
+ if (rmStatus != NV_OK)
+ pApi->status = rmStatus;
+}
+
+// Only return errors through pApi->status
+static void RmAllocOsDescriptor(NVOS02_PARAMETERS *pApi, API_SECURITY_INFO secInfo)
+{
+ NV_STATUS rmStatus = NV_OK;
+ NvU32 flags, attr, attr2;
+ NVOS32_PARAMETERS *pVidHeapParams;
+
+ if (!FLD_TEST_DRF(OS02, _FLAGS, _LOCATION, _PCI, pApi->flags) ||
+ !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NO_MAP, pApi->flags))
+ {
+ rmStatus = NV_ERR_INVALID_FLAGS;
+ goto done;
+ }
+
+ attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI);
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, pApi->flags) ||
+ FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, pApi->flags))
+ {
+ attr = FLD_SET_DRF(OS32, _ATTR, _COHERENCY, _WRITE_BACK, attr);
+ }
+ else if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, pApi->flags))
+ attr = FLD_SET_DRF(OS32, _ATTR, _COHERENCY, _UNCACHED, attr);
+ else {
+ rmStatus = NV_ERR_INVALID_FLAGS;
+ goto done;
+ }
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, pApi->flags))
+ attr = FLD_SET_DRF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS, attr);
+ else
+ attr = FLD_SET_DRF(OS32, _ATTR, _PHYSICALITY, _NONCONTIGUOUS, attr);
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, pApi->flags))
+ attr2 = DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _YES);
+ else
+ attr2 = DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _NO);
+
+ pVidHeapParams = portMemAllocNonPaged(sizeof(NVOS32_PARAMETERS));
+ if (pVidHeapParams == NULL)
+ {
+ rmStatus = NV_ERR_NO_MEMORY;
+ goto done;
+ }
+ portMemSet(pVidHeapParams, 0, sizeof(NVOS32_PARAMETERS));
+
+ pVidHeapParams->hRoot = pApi->hRoot;
+ pVidHeapParams->hObjectParent = pApi->hObjectParent;
+ pVidHeapParams->function = NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR;
+
+ flags = (NVOS32_ALLOC_FLAGS_MEMORY_HANDLE_PROVIDED |
+ NVOS32_ALLOC_FLAGS_MAP_NOT_REQUIRED);
+
+ if (DRF_VAL(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, pApi->flags))
+ attr2 = FLD_SET_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_ONLY, attr2);
+
+ // Currently CPU-RO memory implies GPU-RO as well
+ if (DRF_VAL(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, pApi->flags) ||
+ DRF_VAL(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, pApi->flags))
+ attr2 = FLD_SET_DRF(OS32, _ATTR2, _PROTECTION_DEVICE, _READ_ONLY, attr2);
+
+ pVidHeapParams->data.AllocOsDesc.hMemory = pApi->hObjectNew;
+ pVidHeapParams->data.AllocOsDesc.flags = flags;
+ pVidHeapParams->data.AllocOsDesc.attr = attr;
+ pVidHeapParams->data.AllocOsDesc.attr2 = attr2;
+ pVidHeapParams->data.AllocOsDesc.descriptor = pApi->pMemory;
+ pVidHeapParams->data.AllocOsDesc.limit = pApi->limit;
+ pVidHeapParams->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS;
+
+ RmCreateOsDescriptor(pVidHeapParams, secInfo);
+
+ pApi->status = pVidHeapParams->status;
+
+ portMemFree(pVidHeapParams);
+
+done:
+ if (rmStatus != NV_OK)
+ pApi->status = rmStatus;
+}
+
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hRoot) == NV_OFFSETOF(NVOS64_PARAMETERS, hRoot));
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hObjectParent) == NV_OFFSETOF(NVOS64_PARAMETERS, hObjectParent));
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hObjectNew) == NV_OFFSETOF(NVOS64_PARAMETERS, hObjectNew));
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hClass) == NV_OFFSETOF(NVOS64_PARAMETERS, hClass));
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, pAllocParms) == NV_OFFSETOF(NVOS64_PARAMETERS, pAllocParms));
+
+NV_STATUS RmIoctl(
+ nv_state_t *nv,
+ nv_file_private_t *nvfp,
+ NvU32 cmd,
+ void *data,
+ NvU32 dataSize
+)
+{
+ NV_STATUS rmStatus = NV_ERR_GENERIC;
+ API_SECURITY_INFO secInfo = { };
+
+ secInfo.privLevel = osIsAdministrator() ? RS_PRIV_LEVEL_USER_ROOT : RS_PRIV_LEVEL_USER;
+ secInfo.paramLocation = PARAM_LOCATION_USER;
+ secInfo.pProcessToken = NULL;
+ secInfo.gpuOsInfo = NULL;
+ secInfo.clientOSInfo = nvfp->ctl_nvfp;
+ if (secInfo.clientOSInfo == NULL)
+ secInfo.clientOSInfo = nvfp;
+
+ switch (cmd)
+ {
+ case NV_ESC_RM_ALLOC_MEMORY:
+ {
+ nv_ioctl_nvos02_parameters_with_fd *pApi;
+ NVOS02_PARAMETERS *pParms;
+
+ pApi = data;
+ pParms = &pApi->params;
+
+ NV_ACTUAL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(nv_ioctl_nvos02_parameters_with_fd))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ if (pParms->hClass == NV01_MEMORY_SYSTEM_OS_DESCRIPTOR)
+ RmAllocOsDescriptor(pParms, secInfo);
+ else
+ {
+ NvU32 flags = pParms->flags;
+
+ Nv01AllocMemoryWithSecInfo(pParms, secInfo);
+
+ //
+ // If the system memory is going to be mapped immediately,
+ // create the mmap context for it now.
+ //
+ if ((pParms->hClass == NV01_MEMORY_SYSTEM) &&
+ (!FLD_TEST_DRF(OS02, _FLAGS, _ALLOC, _NONE, flags)) &&
+ (!FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NO_MAP, flags)) &&
+ (pParms->status == NV_OK))
+ {
+ if (rm_create_mmap_context(pParms->hRoot,
+ pParms->hObjectParent, pParms->hObjectNew,
+ pParms->pMemory, pParms->limit + 1, 0,
+ NV_MEMORY_DEFAULT,
+ pApi->fd) != NV_OK)
+ {
+ NV_PRINTF(LEVEL_WARNING,
+ "could not create mmap context for %p\n",
+ NvP64_VALUE(pParms->pMemory));
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+ }
+ }
+
+ break;
+ }
+
+ case NV_ESC_RM_ALLOC_OBJECT:
+ {
+ NVOS05_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS05_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv01AllocObjectWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_ALLOC:
+ {
+ NVOS21_PARAMETERS *pApi = data;
+ NVOS64_PARAMETERS *pApiAccess = data;
+ NvBool bAccessApi = (dataSize == sizeof(NVOS64_PARAMETERS));
+
+ if ((dataSize != sizeof(NVOS21_PARAMETERS)) &&
+ (dataSize != sizeof(NVOS64_PARAMETERS)))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ switch (pApi->hClass)
+ {
+ case NV01_ROOT:
+ case NV01_ROOT_CLIENT:
+ case NV01_ROOT_NON_PRIV:
+ {
+ NV_CTL_DEVICE_ONLY(nv);
+
+ // Force userspace client allocations to be the _CLIENT class.
+ pApi->hClass = NV01_ROOT_CLIENT;
+ break;
+ }
+ case NV01_EVENT:
+ case NV01_EVENT_OS_EVENT:
+ case NV01_EVENT_KERNEL_CALLBACK:
+ case NV01_EVENT_KERNEL_CALLBACK_EX:
+ {
+ break;
+ }
+ default:
+ {
+ NV_CTL_DEVICE_ONLY(nv);
+ break;
+ }
+ }
+
+ if (!bAccessApi)
+ {
+ Nv04AllocWithSecInfo(pApi, secInfo);
+ }
+ else
+ {
+ Nv04AllocWithAccessSecInfo(pApiAccess, secInfo);
+ }
+
+ break;
+ }
+
+ case NV_ESC_RM_FREE:
+ {
+ NVOS00_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS00_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv01FreeWithSecInfo(pApi, secInfo);
+
+ if (pApi->status == NV_OK &&
+ pApi->hObjectOld == pApi->hRoot)
+ {
+ rm_client_free_os_events(pApi->hRoot);
+ }
+
+ break;
+ }
+
+ case NV_ESC_RM_VID_HEAP_CONTROL:
+ {
+ NVOS32_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS32_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ if (pApi->function == NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR)
+ RmCreateOsDescriptor(pApi, secInfo);
+ else
+ Nv04VidHeapControlWithSecInfo(pApi, secInfo);
+
+ break;
+ }
+
+ case NV_ESC_RM_I2C_ACCESS:
+ {
+ NVOS_I2C_ACCESS_PARAMS *pApi = data;
+
+ NV_ACTUAL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS_I2C_ACCESS_PARAMS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04I2CAccessWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_IDLE_CHANNELS:
+ {
+ NVOS30_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS30_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04IdleChannelsWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_MAP_MEMORY:
+ {
+ nv_ioctl_nvos33_parameters_with_fd *pApi;
+ NVOS33_PARAMETERS *pParms;
+
+ pApi = data;
+ pParms = &pApi->params;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(nv_ioctl_nvos33_parameters_with_fd))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ // Don't allow userspace to override the caching type
+ pParms->flags = FLD_SET_DRF(OS33, _FLAGS, _CACHING_TYPE, _DEFAULT, pParms->flags);
+ Nv04MapMemoryWithSecInfo(pParms, secInfo);
+
+ if (pParms->status == NV_OK)
+ {
+ pParms->status = rm_create_mmap_context(pParms->hClient,
+ pParms->hDevice, pParms->hMemory,
+ pParms->pLinearAddress, pParms->length,
+ pParms->offset,
+ DRF_VAL(OS33, _FLAGS, _CACHING_TYPE, pParms->flags),
+ pApi->fd);
+ if (pParms->status != NV_OK)
+ {
+ NVOS34_PARAMETERS params;
+ portMemSet(&params, 0, sizeof(NVOS34_PARAMETERS));
+ params.hClient = pParms->hClient;
+ params.hDevice = pParms->hDevice;
+ params.hMemory = pParms->hMemory;
+ params.pLinearAddress = pParms->pLinearAddress;
+ params.flags = pParms->flags;
+ Nv04UnmapMemoryWithSecInfo(&params, secInfo);
+ }
+ }
+ break;
+ }
+
+ case NV_ESC_RM_UNMAP_MEMORY:
+ {
+ NVOS34_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS34_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04UnmapMemoryWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_ACCESS_REGISTRY:
+ {
+ NVOS38_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS38_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ pApi->status = rm_access_registry(pApi->hClient,
+ pApi->hObject,
+ pApi->AccessType,
+ pApi->pDevNode,
+ pApi->DevNodeLength,
+ pApi->pParmStr,
+ pApi->ParmStrLength,
+ pApi->pBinaryData,
+ &pApi->BinaryDataLength,
+ &pApi->Data,
+ &pApi->Entry);
+ break;
+ }
+
+ case NV_ESC_RM_ALLOC_CONTEXT_DMA2:
+ {
+ NVOS39_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS39_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04AllocContextDmaWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_BIND_CONTEXT_DMA:
+ {
+ NVOS49_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS49_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04BindContextDmaWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_MAP_MEMORY_DMA:
+ {
+ NVOS46_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS46_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04MapMemoryDmaWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_UNMAP_MEMORY_DMA:
+ {
+ NVOS47_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS47_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04UnmapMemoryDmaWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_DUP_OBJECT:
+ {
+ NVOS55_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS55_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04DupObjectWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_RM_SHARE:
+ {
+ NVOS57_PARAMETERS *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS57_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ Nv04ShareWithSecInfo(pApi, secInfo);
+ break;
+ }
+
+ case NV_ESC_ALLOC_OS_EVENT:
+ {
+ nv_ioctl_alloc_os_event_t *pApi = data;
+
+ if (dataSize != sizeof(nv_ioctl_alloc_os_event_t))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ pApi->Status = rm_alloc_os_event(pApi->hClient,
+ nvfp,
+ pApi->fd);
+ break;
+ }
+
+ case NV_ESC_FREE_OS_EVENT:
+ {
+ nv_ioctl_free_os_event_t *pApi = data;
+
+ if (dataSize != sizeof(nv_ioctl_free_os_event_t))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ pApi->Status = rm_free_os_event(pApi->hClient, pApi->fd);
+ break;
+ }
+
+ case NV_ESC_RM_GET_EVENT_DATA:
+ {
+ NVOS41_PARAMETERS *pApi = data;
+
+ if (dataSize != sizeof(NVOS41_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ pApi->status = rm_get_event_data(nvfp,
+ pApi->pEvent,
+ &pApi->MoreEvents);
+ break;
+ }
+
+ case NV_ESC_STATUS_CODE:
+ {
+ nv_state_t *pNv;
+ nv_ioctl_status_code_t *pApi = data;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(nv_ioctl_status_code_t))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ pNv = nv_get_adapter_state(pApi->domain, pApi->bus, pApi->slot);
+ if (pNv == NULL)
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ rmStatus = rm_get_adapter_status(pNv, &pApi->status);
+
+ if (rmStatus != NV_OK)
+ goto done;
+
+ break;
+ }
+
+ case NV_ESC_RM_CONTROL:
+ {
+ NVOS54_PARAMETERS *pApi = data;
+ void *priv = NULL;
+ nv_file_private_t *dev_nvfp = NULL;
+ NvS32 fd;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS54_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ if (RmIsDeviceRefNeeded(pApi))
+ {
+ rmStatus = RmGetDeviceFd(pApi, &fd);
+ if (rmStatus != NV_OK)
+ {
+ goto done;
+ }
+
+ dev_nvfp = nv_get_file_private(fd, NV_FALSE, &priv);
+ if (dev_nvfp == NULL)
+ {
+ rmStatus = NV_ERR_INVALID_DEVICE;
+ goto done;
+ }
+
+ // Check to avoid cyclic dependency with NV_ESC_REGISTER_FD
+ if (!portAtomicCompareAndSwapU32(&dev_nvfp->register_or_refcount,
+ NVFP_TYPE_REFCOUNTED,
+ NVFP_TYPE_NONE))
+ {
+ // Is this already refcounted...
+ if (dev_nvfp->register_or_refcount != NVFP_TYPE_REFCOUNTED)
+ {
+ nv_put_file_private(priv);
+ rmStatus = NV_ERR_IN_USE;
+ goto done;
+ }
+ }
+
+ secInfo.gpuOsInfo = priv;
+ }
+
+ Nv04ControlWithSecInfo(pApi, secInfo);
+
+ if ((pApi->status != NV_OK) && (priv != NULL))
+ {
+ //
+ // No need to reset `register_or_refcount` as it might be set
+ // for previous successful calls. We let it clear with FD close.
+ //
+ nv_put_file_private(priv);
+
+ secInfo.gpuOsInfo = NULL;
+ }
+
+ break;
+ }
+
+ case NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO:
+ {
+ NVOS56_PARAMETERS *pApi = data;
+ void *pOldCpuAddress;
+ void *pNewCpuAddress;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (dataSize != sizeof(NVOS56_PARAMETERS))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ pOldCpuAddress = NvP64_VALUE(pApi->pOldCpuAddress);
+ pNewCpuAddress = NvP64_VALUE(pApi->pNewCpuAddress);
+
+ pApi->status = rm_update_device_mapping_info(pApi->hClient,
+ pApi->hDevice,
+ pApi->hMemory,
+ pOldCpuAddress,
+ pNewCpuAddress);
+ break;
+ }
+
+ case NV_ESC_REGISTER_FD:
+ {
+ nv_ioctl_register_fd_t *params = data;
+ void *priv = NULL;
+ nv_file_private_t *ctl_nvfp;
+
+ if (dataSize != sizeof(nv_ioctl_register_fd_t))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ // LOCK: acquire API lock
+ rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+ if (rmStatus != NV_OK)
+ goto done;
+
+ // If there is already a ctl fd registered on this nvfp, fail.
+ if (nvfp->ctl_nvfp != NULL)
+ {
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ rmStatus = NV_ERR_INVALID_STATE;
+ goto done;
+ }
+
+ //
+ // Note that this call is valid for both "actual" devices and ctrl
+ // devices. In particular, NV_ESC_ALLOC_OS_EVENT can be used with
+ // both types of devices.
+ // But, the ctl_fd passed in should always correspond to a control FD.
+ //
+ ctl_nvfp = nv_get_file_private(params->ctl_fd,
+ NV_TRUE, /* require ctl fd */
+ &priv);
+ if (ctl_nvfp == NULL)
+ {
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ // Disallow self-referential links, and disallow links to FDs that
+ // themselves have a link.
+ if ((ctl_nvfp == nvfp) || (ctl_nvfp->ctl_nvfp != NULL))
+ {
+ nv_put_file_private(priv);
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ // Check to avoid cyclic dependency with device refcounting
+ if (!portAtomicCompareAndSwapU32(&nvfp->register_or_refcount,
+ NVFP_TYPE_REGISTERED,
+ NVFP_TYPE_NONE))
+ {
+ nv_put_file_private(priv);
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ rmStatus = NV_ERR_IN_USE;
+ goto done;
+ }
+
+ //
+ // nvfp->ctl_nvfp is read outside the lock, so set it atomically.
+ // Note that once set, this can never be removed until the fd
+ // associated with nvfp is closed. We hold on to 'priv' until the
+ // fd is closed, too, to ensure that the fd associated with
+ // ctl_nvfp remains valid.
+ //
+ portAtomicSetSize(&nvfp->ctl_nvfp, ctl_nvfp);
+ nvfp->ctl_nvfp_priv = priv;
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+
+ // NOTE: nv_put_file_private(priv) is not called here. It MUST be
+ // called during cleanup of this nvfp.
+ rmStatus = NV_OK;
+ break;
+ }
+
+ default:
+ {
+ NV_PRINTF(LEVEL_ERROR, "unknown NVRM ioctl command: 0x%x\n", cmd);
+ goto done;
+ }
+ }
+
+ rmStatus = NV_OK;
+done:
+
+ return rmStatus;
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/exports-stubs.c b/userspace/sdks/nv525/nvalloc/unix/src/exports-stubs.c
new file mode 100644
index 0000000..4dac378
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/exports-stubs.c
@@ -0,0 +1,235 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include <rmconfig.h>
+#include <gpu/subdevice/subdevice.h>
+#include <ctrl/ctrl0080/ctrl0080unix.h>
+#include <ctrl/ctrl2080/ctrl2080unix.h>
+
+NV_STATUS NV_API_CALL rm_schedule_gpu_wakeup(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ return NV_OK;
+}
+
+void NV_API_CALL rm_init_dynamic_power_management(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvBool bPr3AcpiMethodPresent
+)
+{
+}
+
+void NV_API_CALL rm_cleanup_dynamic_power_management(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+}
+
+void NV_API_CALL rm_enable_dynamic_power_management(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+}
+
+NV_STATUS NV_API_CALL rm_ref_dynamic_power(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ nv_dynamic_power_mode_t mode
+)
+{
+ return NV_OK;
+}
+
+void NV_API_CALL rm_unref_dynamic_power(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ nv_dynamic_power_mode_t mode
+)
+{
+}
+
+NV_STATUS NV_API_CALL rm_transition_dynamic_power(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvBool bEnter
+)
+{
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_power_management(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ nv_pm_action_t pmAction
+)
+{
+ return NV_OK;
+}
+
+const char* NV_API_CALL rm_get_vidmem_power_status(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ return "?";
+}
+
+const char* NV_API_CALL rm_get_dynamic_power_management_status(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ return "?";
+}
+
+const char* NV_API_CALL rm_get_gpu_gcx_support(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ NvBool bGcxTypeGC6
+)
+{
+ return "?";
+}
+
+NV_STATUS
+subdeviceCtrlCmdOsUnixGc6BlockerRefCnt_IMPL
+(
+ Subdevice *pSubdevice,
+ NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS *pParams
+)
+{
+ return NV_OK;
+}
+
+NV_STATUS
+subdeviceCtrlCmdOsUnixAllowDisallowGcoff_IMPL
+(
+ Subdevice *pSubdevice,
+ NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS *pParams
+)
+{
+ return NV_OK;
+}
+
+NV_STATUS
+subdeviceCtrlCmdOsUnixAudioDynamicPower_IMPL
+(
+ Subdevice *pSubdevice,
+ NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS *pParams
+)
+{
+ return NV_OK;
+}
+
+void
+RmUpdateGc6ConsoleRefCount
+(
+ nv_state_t *nv,
+ NvBool bIncrease
+)
+{
+}
+
+void
+RmInitS0ixPowerManagement
+(
+ nv_state_t *nv
+)
+{
+}
+
+void
+RmInitDeferredDynamicPowerManagement
+(
+ nv_state_t *nv
+)
+{
+}
+
+void
+RmDestroyDeferredDynamicPowerManagement
+(
+ nv_state_t *nv
+)
+{
+}
+
+void RmHandleDisplayChange
+(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+}
+
+NV_STATUS
+os_ref_dynamic_power
+(
+ nv_state_t *nv,
+ nv_dynamic_power_mode_t mode
+)
+{
+ return NV_OK;
+}
+
+void
+os_unref_dynamic_power
+(
+ nv_state_t *nv,
+ nv_dynamic_power_mode_t mode
+)
+{
+}
+
+NV_STATUS
+deviceCtrlCmdOsUnixVTSwitch_IMPL
+(
+ Device *pDevice,
+ NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS *pParams
+)
+{
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_save_low_res_mode(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS RmInitX86EmuState(OBJGPU *pGpu)
+{
+ return NV_OK;
+}
+
+void RmFreeX86EmuState(OBJGPU *pGpu)
+{
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/gcc_helper.c b/userspace/sdks/nv525/nvalloc/unix/src/gcc_helper.c
new file mode 100644
index 0000000..3c1037c
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/gcc_helper.c
@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvtypes.h>
+#include <os-interface.h>
+
+void* memset(void* s, int c, NvUPtr n)
+{
+ return os_mem_set(s, (NvU8)c, (NvU32)n);
+}
+
+void* memcpy(void* dest, const void* src, NvUPtr n)
+{
+ return os_mem_copy(dest, src, (NvU32)n);
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/os-hypervisor.c b/userspace/sdks/nv525/nvalloc/unix/src/os-hypervisor.c
new file mode 100644
index 0000000..3982c35
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/os-hypervisor.c
@@ -0,0 +1,1023 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2014-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ *
+ * @file os-hypervisor.c
+ * @brief OS specific Hypervisor interfaces for RM
+ *
+ */
+
+#include "os/os.h"
+#include "nv.h"
+#include "nv-priv.h"
+#include <nvRmReg.h>
+#include <virtualization/hypervisor/hypervisor.h>
+#include "core/thread_state.h"
+#include "core/locks.h"
+#include "virtualization/kernel_vgpu_mgr.h"
+#include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
+#include "kernel/gpu/fifo/kernel_fifo.h"
+#include "osapi.h"
+#include "virtualization/kernel_hostvgpudeviceapi.h"
+#include <objtmr.h>
+#include "gpu/bif/kernel_bif.h"
+#include "gpu/bus/kern_bus.h"
+#include <nv_ref.h> // NV_PMC_BOOT_1_VGPU
+#include "nvdevid.h"
+
+#define NV_VFIO_PCI_BAR1_REGION_INDEX 1
+#define NV_VFIO_PCI_BAR2_REGION_INDEX 2
+#define NV_VFIO_PCI_BAR3_REGION_INDEX 3
+
+static NV_STATUS nv_parse_config_params(const char *, const char *, const char, NvU32 *);
+
+void hypervisorSetHypervVgpuSupported_IMPL(POBJHYPERVISOR pHypervisor)
+{
+ pHypervisor->bIsHypervVgpuSupported = NV_TRUE;
+}
+
+NvBool hypervisorIsVgxHyper_IMPL(void)
+{
+ return os_is_vgx_hyper();
+}
+
+NV_STATUS hypervisorInjectInterrupt_IMPL
+(
+ POBJHYPERVISOR pHypervisor,
+ VGPU_NS_INTR *pVgpuNsIntr
+)
+{
+ NV_STATUS status = NV_ERR_NOT_SUPPORTED;
+
+ if (pVgpuNsIntr->pVgpuVfioRef)
+ status = osVgpuInjectInterrupt(pVgpuNsIntr->pVgpuVfioRef);
+ else
+ {
+ if (pVgpuNsIntr->guestMSIAddr && pVgpuNsIntr->guestMSIData)
+ {
+ status = os_inject_vgx_msi((NvU16)pVgpuNsIntr->guestDomainId,
+ pVgpuNsIntr->guestMSIAddr,
+ pVgpuNsIntr->guestMSIData);
+ }
+ }
+
+ return status;
+}
+
+HYPERVISOR_TYPE NV_API_CALL nv_get_hypervisor_type(void)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ POBJHYPERVISOR pHypervisor = SYS_GET_HYPERVISOR(pSys);
+ return hypervisorGetHypervisorType(pHypervisor);
+}
+
+static NV_STATUS get_available_instances(
+ NvU32 *avail_instances,
+ nv_state_t *pNv,
+ VGPU_TYPE *vgpuTypeInfo,
+ NvU32 pgpuIndex,
+ NvU8 devfn
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+ OBJGPU *pGpu = NULL;
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+ OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
+
+ *avail_instances = 0;
+
+ pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+ if (pGpu == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__);
+ rmStatus = NV_ERR_INVALID_STATE;
+ goto exit;
+ }
+
+ /* TODO: Needs to have a proper fix this for DriverVM config */
+ if (gpuIsSriovEnabled(pGpu) &&
+ !(pHypervisor->getProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED)))
+ {
+ NvU8 fnId = devfn - pGpu->sriovState.firstVFOffset;
+
+ if (fnId > 63)
+ {
+ NV_ASSERT(0);
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto exit;
+ }
+
+ if (IS_MIG_ENABLED(pGpu))
+ {
+ if (IS_MIG_IN_USE(pGpu)) {
+ NvU64 swizzIdInUseMask = 0;
+ NvU32 partitionFlag = PARTITIONID_INVALID;
+ KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
+ NvU32 id;
+
+ swizzIdInUseMask = kmigmgrGetSwizzIdInUseMask(pGpu, pKernelMIGManager);
+
+ rmStatus = kvgpumgrGetPartitionFlag(vgpuTypeInfo->vgpuTypeId,
+ &partitionFlag);
+ if (rmStatus != NV_OK)
+ {
+ // Query for a non MIG vgpuType
+ NV_PRINTF(LEVEL_ERROR, "%s Query for a non MIG vGPU type \n",
+ __FUNCTION__);
+ rmStatus = NV_OK;
+ goto exit;
+ }
+
+ // Determine valid swizzids not assigned to any vGPU device.
+ FOR_EACH_INDEX_IN_MASK(64, id, swizzIdInUseMask)
+ {
+ KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance;
+ NvU64 mask = 0;
+
+ rmStatus = kmigmgrGetGPUInstanceInfo(pGpu, pKernelMIGManager,
+ id, &pKernelMIGGpuInstance);
+ if (rmStatus != NV_OK)
+ {
+ // Didn't find requested GPU instance
+ NV_PRINTF(LEVEL_ERROR,
+ "No valid GPU instance with SwizzId - %d found\n", id);
+ goto exit;
+ }
+
+ mask = NVBIT64(id);
+
+ if (pKernelMIGGpuInstance->partitionFlag == partitionFlag)
+ {
+ // Validate that same ID is not already set and VF is available
+ if (!(mask & pKernelVgpuMgr->pgpuInfo[pgpuIndex].assignedSwizzIdMask) &&
+ !(pKernelVgpuMgr->pgpuInfo[pgpuIndex].createdVfMask & NVBIT64(fnId)))
+ {
+ *avail_instances = 1;
+ break;
+ }
+ }
+ }
+ FOR_EACH_INDEX_IN_MASK_END;
+ }
+ }
+ else
+ {
+ if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].numCreatedVgpu < vgpuTypeInfo->maxInstance)
+ {
+ if (vgpuTypeInfo->gpuInstanceSize)
+ {
+ // Query for a MIG vgpuType
+ NV_PRINTF(LEVEL_ERROR, "%s Query for a MIG vGPU type \n",
+ __FUNCTION__);
+ rmStatus = NV_OK;
+ goto exit;
+ }
+
+ if (!(pKernelVgpuMgr->pgpuInfo[pgpuIndex].createdVfMask & NVBIT64(fnId)))
+ {
+ if (kvgpumgrCheckVgpuTypeCreatable(&pKernelVgpuMgr->pgpuInfo[pgpuIndex], vgpuTypeInfo) == NV_OK)
+ *avail_instances = 1;
+ }
+ }
+ }
+ }
+ else
+ {
+ if (kvgpumgrCheckVgpuTypeCreatable(&pKernelVgpuMgr->pgpuInfo[pgpuIndex], vgpuTypeInfo) == NV_OK)
+ *avail_instances = vgpuTypeInfo->maxInstance - pKernelVgpuMgr->pgpuInfo[pgpuIndex].numCreatedVgpu;
+ }
+
+exit:
+ return rmStatus;
+}
+
+#define MAX_STR_LEN 256
+NV_STATUS NV_API_CALL nv_vgpu_get_type_info(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ NvU32 vgpuTypeId,
+ char *buffer,
+ int type_info,
+ NvU8 devfn
+)
+{
+ THREAD_STATE_NODE threadState;
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJGPU *pGpu = NULL;
+ KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+ NV_STATUS rmStatus = NV_OK;
+ VGPU_TYPE *vgpuTypeInfo;
+ NvU32 pgpuIndex, i, avail_instances = 0;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+ if (pGpu == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__);
+ rmStatus = NV_ERR_INVALID_STATE;
+ goto exit;
+ }
+
+ if ((rmStatus = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex)) ==
+ NV_OK)
+ {
+ for (i = 0; i < MAX_VGPU_TYPES_PER_PGPU; i++)
+ {
+ vgpuTypeInfo = pKernelVgpuMgr->pgpuInfo[pgpuIndex].vgpuTypes[i];
+ if (vgpuTypeInfo == NULL)
+ break;
+
+ if (vgpuTypeInfo->vgpuTypeId != vgpuTypeId)
+ continue;
+
+ switch (type_info)
+ {
+ case VGPU_TYPE_NAME:
+ os_snprintf(buffer, VGPU_STRING_BUFFER_SIZE, "%s\n",
+ vgpuTypeInfo->vgpuName);
+ break;
+ case VGPU_TYPE_DESCRIPTION:
+ os_snprintf(buffer, MAX_STR_LEN,
+ "num_heads=%d, frl_config=%d, "
+ "framebuffer=%dM, max_resolution=%dx%d, max_instance=%d\n",
+ vgpuTypeInfo->numHeads, vgpuTypeInfo->frlConfig,
+ vgpuTypeInfo->profileSize >> 20,
+ vgpuTypeInfo->maxResolutionX,
+ vgpuTypeInfo->maxResolutionY,
+ vgpuTypeInfo->maxInstance);
+ break;
+ case VGPU_TYPE_INSTANCES:
+ rmStatus = get_available_instances(&avail_instances, pNv,
+ vgpuTypeInfo,
+ pgpuIndex, devfn);
+ if (rmStatus != NV_OK)
+ goto exit;
+
+ os_snprintf(buffer, MAX_STR_LEN, "%d\n", avail_instances);
+ break;
+ default:
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ }
+ break;
+ }
+ }
+
+exit:
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_get_type_ids(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ NvU32 *numVgpuTypes,
+ NvU32 *vgpuTypeIds,
+ NvBool isVirtfn,
+ NvU8 devfn,
+ NvBool getCreatableTypes
+)
+{
+ THREAD_STATE_NODE threadState;
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+ NV_STATUS rmStatus = NV_OK;
+ NvU32 pgpuIndex, i, avail_instances = 0;
+ NvU32 numSupportedVgpuTypes = 0;
+ VGPU_TYPE *vgpuTypeInfo;
+ void *fp;
+
+ if (!vgpuTypeIds || !numVgpuTypes)
+ return NV_ERR_INVALID_ARGUMENT;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ if ((rmStatus = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex)) ==
+ NV_OK)
+ {
+ if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].sriovEnabled && !isVirtfn)
+ {
+ *numVgpuTypes = 0;
+ }
+ else
+ {
+ numSupportedVgpuTypes = pKernelVgpuMgr->pgpuInfo[pgpuIndex].numVgpuTypes;
+ *numVgpuTypes = 0;
+
+ for (i = 0; i < numSupportedVgpuTypes; i++)
+ {
+ vgpuTypeInfo = pKernelVgpuMgr->pgpuInfo[pgpuIndex].vgpuTypes[i];
+
+ if (!getCreatableTypes)
+ {
+ // Return all available types
+ vgpuTypeIds[*numVgpuTypes] = vgpuTypeInfo->vgpuTypeId;
+ (*numVgpuTypes)++;
+ continue;
+ }
+
+ rmStatus = get_available_instances(&avail_instances, pNv,
+ vgpuTypeInfo, pgpuIndex,
+ devfn);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to get available instances for vGPU ID: %d, status: 0x%x\n",
+ vgpuTypeInfo->vgpuTypeId, rmStatus);
+ continue;
+ }
+
+ if (avail_instances == 0)
+ continue;
+
+ vgpuTypeIds[*numVgpuTypes] = vgpuTypeInfo->vgpuTypeId;
+ (*numVgpuTypes)++;
+ }
+ }
+ }
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_delete(
+ nvidia_stack_t *sp,
+ const NvU8 *pMdevUuid,
+ NvU16 vgpuId
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp = NULL;
+ NvU32 rmStatus = NV_OK;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ rmStatus = kvgpumgrDeleteRequestVgpu(pMdevUuid, vgpuId);
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_process_vf_info(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ NvU8 cmd,
+ NvU32 domain,
+ NvU8 bus,
+ NvU8 slot,
+ NvU8 function,
+ NvBool isMdevAttached,
+ void *vf_pci_info
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_OK;
+ void *fp = NULL;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ rmStatus = kvgpumgrProcessVfInfo(pNv->gpu_id, cmd, domain, bus, slot, function, isMdevAttached, (vgpu_vf_pci_info *) vf_pci_info);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_create_request(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ const NvU8 *pMdevUuid,
+ NvU32 vgpuTypeId,
+ NvU16 *vgpuId,
+ NvU32 gpuPciBdf,
+ NvBool *is_driver_vm
+)
+{
+ THREAD_STATE_NODE threadState;
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ void *fp = NULL;
+ OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
+ NV_STATUS rmStatus = NV_OK;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ rmStatus = kvgpumgrCreateRequestVgpu(pNv->gpu_id, pMdevUuid,
+ vgpuTypeId, vgpuId, gpuPciBdf);
+
+ *is_driver_vm = pHypervisor->getProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_get_bar_info(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ const NvU8 *pMdevUuid,
+ NvU64 *size,
+ NvU32 regionIndex,
+ void *pVgpuVfioRef
+)
+{
+ REQUEST_VGPU_INFO_NODE *pRequestVgpu = NULL;
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_OK, status;
+ OBJGPU *pGpu = NULL;
+ KernelBus *pKernelBus;
+ KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice;
+ void *fp = NULL;
+ NvU32 value = 0;
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ KernelVgpuMgr * pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+ if (pGpu == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__);
+ rmStatus = NV_ERR_INVALID_STATE;
+ goto release_lock;
+ }
+ pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+ *size = kbusGetPciBarSize(pKernelBus, regionIndex);
+
+ NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+ kvgpumgrGetHostVgpuDeviceFromMdevUuid(pNv->gpu_id,
+ pMdevUuid,
+ &pKernelHostVgpuDevice), release_lock);
+
+ pRequestVgpu = pKernelHostVgpuDevice->pRequestVgpuInfoNode;
+ if (pRequestVgpu == NULL)
+ {
+ rmStatus = NV_ERR_INVALID_POINTER;
+ goto release_lock;
+ }
+
+ pKernelHostVgpuDevice->pVgpuVfioRef = pVgpuVfioRef;
+
+ if (regionIndex == NV_VFIO_PCI_BAR1_REGION_INDEX)
+ {
+ VGPU_TYPE *vgpuTypeInfo;
+ NvU32 pgpuIndex = 0;
+ NvBool bOverrideBar1Size = NV_FALSE;
+
+ // Read BAR1 length from vgpuTypeInfo
+ NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+ kvgpumgrGetVgpuTypeInfo(pKernelHostVgpuDevice->vgpuType, &vgpuTypeInfo), release_lock);
+
+ *size = vgpuTypeInfo->bar1Length << 20;
+
+ NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+ kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex), release_lock);
+
+ /*
+ * check for 'override_bar1_size' param in vgpuExtraParams list first,
+ * if param is missing there then check it in vgpu_params list
+ */
+ status = nv_parse_config_params((const char*)vgpuTypeInfo->vgpuExtraParams,
+ "override_bar1_size", ';', &value);
+
+ if (status == NV_OK && value) {
+ bOverrideBar1Size = NV_TRUE;
+ } else if (status == NV_ERR_OBJECT_NOT_FOUND) {
+ status = nv_parse_config_params(pRequestVgpu->configParams,
+ "override_bar1_size", ',', &value);
+ if (status == NV_OK && value)
+ bOverrideBar1Size = NV_TRUE;
+ }
+ if (bOverrideBar1Size) {
+ NvU64 bar1SizeInBytes, guestBar1;
+ NvU64 gpuBar1LowerLimit = 256 * 1024 * 1024; // bar1 lower limit for override_bar1_length parameter
+
+ bar1SizeInBytes = kbusGetPciBarSize(pKernelBus, NV_VFIO_PCI_BAR1_REGION_INDEX);
+ if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].sriovEnabled)
+ {
+ *size = pGpu->sriovState.vfBarSize[1];
+ }
+ else if (bar1SizeInBytes > gpuBar1LowerLimit)
+ {
+ guestBar1 = bar1SizeInBytes / vgpuTypeInfo->maxInstance;
+ *size = nvPrevPow2_U64(guestBar1);
+ }
+ }
+ }
+ else if (regionIndex == NV_VFIO_PCI_BAR2_REGION_INDEX ||
+ regionIndex == NV_VFIO_PCI_BAR3_REGION_INDEX)
+ {
+ status = nv_parse_config_params(pRequestVgpu->configParams,
+ "address64", ',', &value);
+
+ if ((status != NV_OK) || ((status == NV_OK) && (value != 0)))
+ {
+ if (regionIndex == NV_VFIO_PCI_BAR2_REGION_INDEX)
+ *size = 0;
+ else if (regionIndex == NV_VFIO_PCI_BAR3_REGION_INDEX)
+ *size = kbusGetPciBarSize(pKernelBus, NV_VFIO_PCI_BAR2_REGION_INDEX);
+ }
+ }
+
+release_lock:
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+
+exit:
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS osVgpuVfioWake(
+ void *waitQueue
+)
+{
+ vgpu_vfio_info vgpu_info;
+
+ vgpu_info.waitQueue = waitQueue;
+
+ return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VGPU_VFIO_WAKE_WAIT_QUEUE);
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_start(
+ nvidia_stack_t *sp,
+ const NvU8 *pMdevUuid,
+ void *waitQueue,
+ NvS32 *returnStatus,
+ NvU8 *vmName,
+ NvU32 qemuPid
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_OK;
+ void *fp = NULL;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ rmStatus = kvgpumgrStart(pMdevUuid, waitQueue, returnStatus,
+ vmName, qemuPid);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+static NV_STATUS nv_parse_config_params(
+ const char *config_params,
+ const char *key,
+ const char delim,
+ NvU32 *config_value
+)
+{
+ char *ptr, *configParams = rm_remove_spaces(config_params);
+ char *token, *value, *name;
+ NvU32 data;
+ NV_STATUS rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+
+ ptr = configParams;
+ while ((token = rm_string_token(&ptr, delim)) != NULL)
+ {
+ if (!(name = rm_string_token(&token, '=')) || !os_string_length(name))
+ continue;
+
+ if (!(value = rm_string_token(&token, '=')) || !os_string_length(value))
+ continue;
+
+ data = os_strtoul(value, NULL, 0);
+
+ if (os_string_compare(name, key) == 0)
+ {
+ rmStatus = NV_OK;
+ *config_value = data;
+ }
+ }
+
+ // Free the memory allocated by rm_remove_spaces()
+ os_free_mem(configParams);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_get_sparse_mmap(
+ nvidia_stack_t *sp ,
+ nv_state_t *pNv,
+ const NvU8 *pMdevUuid,
+ NvU64 **offsets,
+ NvU64 **sizes,
+ NvU32 *numAreas
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_ERR_INVALID_STATE, status;
+ OBJGPU *pGpu = NULL;
+ POBJTMR pTmr = NULL;
+ KernelFifo *pKernelFifo = NULL;
+ void *fp = NULL;
+ REQUEST_VGPU_INFO_NODE *pRequestVgpu = NULL;
+ KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice;
+ NvU32 bar0TmrMapSize = 0, bar0FifoMapSize = 0, value = 0;
+ NvU64 bar0TmrMapOffset = 0, bar0FifoMapOffset = 0;
+ NvU64 *vfRegionSizes = NULL;
+ NvU64 *vfRegionOffsets = NULL;
+ KernelBif *pKernelBif = NULL;
+
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+ if (pGpu == NULL)
+ {
+ rmStatus = NV_ERR_INVALID_STATE;
+ goto cleanup;
+ }
+ pTmr = GPU_GET_TIMER(pGpu);
+ pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
+ pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+ *numAreas = 0;
+ rmStatus = kvgpumgrGetHostVgpuDeviceFromMdevUuid(pNv->gpu_id, pMdevUuid,
+ &pKernelHostVgpuDevice);
+ if (rmStatus == NV_OK)
+ {
+ if (pKernelHostVgpuDevice->gfid != 0)
+ {
+ rmStatus = kbifGetNumVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice, numAreas);
+ if (rmStatus == NV_OK)
+ {
+ os_alloc_mem((void **)&vfRegionOffsets, sizeof(NvU64) * (*numAreas));
+ os_alloc_mem((void **)&vfRegionSizes, sizeof (NvU64) * (*numAreas));
+ if (vfRegionOffsets && vfRegionSizes)
+ {
+ rmStatus = kbifGetVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice, os_page_size,
+ vfRegionOffsets, vfRegionSizes);
+ if (rmStatus == NV_OK)
+ {
+ *offsets = vfRegionOffsets;
+ *sizes = vfRegionSizes;
+ }
+ }
+ else
+ {
+ if (vfRegionOffsets != NULL)
+ os_free_mem(vfRegionOffsets);
+
+ if (vfRegionSizes != NULL)
+ os_free_mem(vfRegionSizes);
+
+ rmStatus = NV_ERR_INSUFFICIENT_RESOURCES;
+ }
+ }
+ }
+ else
+ {
+ pRequestVgpu = pKernelHostVgpuDevice->pRequestVgpuInfoNode;
+ if (pRequestVgpu == NULL)
+ {
+ rmStatus = NV_ERR_INVALID_POINTER;
+ goto cleanup;
+ }
+
+ status = nv_parse_config_params(pRequestVgpu->configParams, "direct_gpu_timer_access", ',', &value);
+ if ((status == NV_OK) && (value != 0))
+ {
+ rmStatus = tmrGetTimerBar0MapInfo_HAL(pGpu, pTmr,
+ &bar0TmrMapOffset,
+ &bar0TmrMapSize);
+ if (rmStatus == NV_OK)
+ (*numAreas)++;
+ else
+ NV_PRINTF(LEVEL_ERROR,
+ "%s Failed to get NV_PTIMER region \n",
+ __FUNCTION__);
+ }
+
+ value = 0;
+ {
+ status = kfifoGetUsermodeMapInfo_HAL(pGpu, pKernelFifo,
+ &bar0FifoMapOffset,
+ &bar0FifoMapSize);
+ if (status == NV_OK)
+ (*numAreas)++;
+ }
+
+ if (*numAreas != 0)
+ {
+ NvU32 i = 0;
+ NvU64 *tmpOffset, *tmpSize;
+ os_alloc_mem((void **)offsets, sizeof(NvU64) * (*numAreas));
+ os_alloc_mem((void **)sizes, sizeof (NvU64) * (*numAreas));
+
+ tmpOffset = *offsets;
+ tmpSize = *sizes;
+
+ if (bar0TmrMapSize != 0)
+ {
+ tmpOffset[i] = bar0TmrMapOffset;
+ tmpSize[i] = bar0TmrMapSize;
+ i++;
+ }
+
+ if (bar0FifoMapSize != 0)
+ {
+ tmpOffset[i] = bar0FifoMapOffset;
+ tmpSize[i] = bar0FifoMapSize;
+ }
+ }
+ }
+ }
+
+cleanup:
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_update_request(
+ nvidia_stack_t *sp ,
+ const NvU8 *pMdevUuid,
+ VGPU_DEVICE_STATE deviceState,
+ NvU64 *offsets,
+ NvU64 *sizes,
+ const char *configParams
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+ void *fp = NULL;
+ REQUEST_VGPU_INFO_NODE *pRequestVgpu = NULL;
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if (offsets != NULL)
+ os_free_mem(offsets);
+
+ if (sizes != NULL)
+ os_free_mem(sizes);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ for (pRequestVgpu = listHead(&pKernelVgpuMgr->listRequestVgpuHead);
+ pRequestVgpu != NULL;
+ pRequestVgpu = listNext(&pKernelVgpuMgr->listRequestVgpuHead, pRequestVgpu))
+ {
+ if (portMemCmp(pRequestVgpu->mdevUuid, pMdevUuid, VGPU_UUID_SIZE) == 0)
+ {
+
+ if (configParams != NULL)
+ portStringCopy(pRequestVgpu->configParams,
+ sizeof(pRequestVgpu->configParams),
+ configParams, (portStringLength(configParams) + 1));
+
+ pRequestVgpu->deviceState = deviceState;
+ rmStatus = NV_OK;
+ }
+ }
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_gpu_bind_event(
+ nvidia_stack_t *sp
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_OK;
+ void *fp = NULL;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+ {
+ CliAddSystemEvent(NV0000_NOTIFIERS_GPU_BIND_EVENT, 0);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS osVgpuInjectInterrupt(void *vgpuVfioRef)
+{
+ vgpu_vfio_info vgpu_info;
+
+ vgpu_info.vgpuVfioRef = vgpuVfioRef;
+
+ return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VGPU_VFIO_INJECT_INTERRUPT);
+}
+
+NV_STATUS osVgpuRegisterMdev
+(
+ OS_GPU_INFO *pOsGpuInfo
+)
+{
+ NV_STATUS status = NV_OK;
+ vgpu_vfio_info vgpu_info;
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+ KERNEL_PHYS_GPU_INFO *pPhysGpuInfo;
+ NvU32 pgpuIndex, i;
+ OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
+
+ status = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pOsGpuInfo->gpu_id, &pgpuIndex);
+ if (status != NV_OK)
+ return status;
+
+ pPhysGpuInfo = &(pKernelVgpuMgr->pgpuInfo[pgpuIndex]);
+
+ vgpu_info.numVgpuTypes = pKernelVgpuMgr->pgpuInfo[pgpuIndex].numVgpuTypes;
+
+ status = os_alloc_mem((void **)&vgpu_info.vgpuTypeIds,
+ ((vgpu_info.numVgpuTypes) * sizeof(NvU32)));
+ if (status != NV_OK)
+ return status;
+
+ vgpu_info.nv = pOsGpuInfo;
+ for (i = 0; i < pPhysGpuInfo->numVgpuTypes; i++)
+ {
+ vgpu_info.vgpuTypeIds[i] = pPhysGpuInfo->vgpuTypes[i]->vgpuTypeId;
+ }
+
+ if ((!pPhysGpuInfo->sriovEnabled) ||
+ (pHypervisor->getProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED)))
+ {
+ vgpu_info.is_virtfn = NV_FALSE;
+ status = os_call_vgpu_vfio((void *)&vgpu_info, CMD_VGPU_VFIO_REGISTER_MDEV);
+ }
+ else
+ {
+ for (i = 0; i < MAX_VF_COUNT_PER_GPU; i++)
+ {
+ if (pPhysGpuInfo->vfPciInfo[i].isNvidiaAttached)
+ {
+ vgpu_info.is_virtfn = NV_TRUE;
+ vgpu_info.domain = pPhysGpuInfo->vfPciInfo[i].domain;
+ vgpu_info.bus = pPhysGpuInfo->vfPciInfo[i].bus;
+ vgpu_info.slot = pPhysGpuInfo->vfPciInfo[i].slot;
+ vgpu_info.function = pPhysGpuInfo->vfPciInfo[i].function;
+
+ status = os_call_vgpu_vfio((void *)&vgpu_info, CMD_VGPU_VFIO_REGISTER_MDEV);
+ if (status == NV_OK)
+ {
+ pPhysGpuInfo->vfPciInfo[i].isMdevAttached = NV_TRUE;
+ }
+ }
+ }
+ }
+
+ os_free_mem(vgpu_info.vgpuTypeIds);
+ return status;
+}
+
+NV_STATUS osIsVgpuVfioPresent(void)
+{
+ vgpu_vfio_info vgpu_info;
+
+ return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VGPU_VFIO_PRESENT);
+}
+
+void initVGXSpecificRegistry(OBJGPU *pGpu)
+{
+ NvU32 data32;
+#if !defined(NVCPU_X86_64)
+ osWriteRegistryDword(pGpu, NV_REG_STR_RM_BAR2_APERTURE_SIZE_MB, 4);
+#endif
+ osWriteRegistryDword(pGpu, NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR,
+ NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR_ENABLE);
+ if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_DUMP_NVLOG, &data32) != NV_OK))
+ {
+ osWriteRegistryDword(pGpu, NV_REG_STR_RM_DUMP_NVLOG,
+ NV_REG_STR_RM_DUMP_NVLOG_ENABLE);
+ }
+ osWriteRegistryDword(pGpu, NV_REG_STR_RM_RC_WATCHDOG,
+ NV_REG_STR_RM_RC_WATCHDOG_DISABLE);
+ osWriteRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P,
+ DRF_DEF(_REG_STR, _CL_FORCE_P2P, _READ, _DISABLE) |
+ DRF_DEF(_REG_STR, _CL_FORCE_P2P, _WRITE, _DISABLE));
+}
+
+
+NV_STATUS rm_is_vgpu_supported_device(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 pmc_boot_1
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ POBJHYPERVISOR pHypervisor = SYS_GET_HYPERVISOR(pSys);
+ NvBool is_sriov_enabled = FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1);
+
+ // if not running in vGPU mode (guest VM) return NV_OK
+ if (!(pHypervisor && pHypervisor->bIsHVMGuest &&
+ (FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _PV, pmc_boot_1) ||
+ is_sriov_enabled)))
+ {
+ return NV_OK;
+ }
+
+ return NV_OK;
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/os.c b/userspace/sdks/nv525/nvalloc/unix/src/os.c
new file mode 100644
index 0000000..5f65ba4
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/os.c
@@ -0,0 +1,5238 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h> // NV device driver interface
+#include <nv-priv.h>
+#include <nv-caps.h>
+#include <os/os.h>
+#include <nvos.h>
+#include <osapi.h>
+#include <ctrl/ctrl0000/ctrl0000gpu.h>
+#include <ctrl/ctrl0000/ctrl0000unix.h>
+
+#include <nverror.h>
+#include <gpu/device/device.h>
+
+#include "gpu/gpu.h"
+#include <osfuncs.h>
+#include <platform/chipset/chipset.h>
+
+#include "nverror.h"
+#include "kernel/gpu/bif/kernel_bif.h"
+
+#include "gpu/mem_sys/kern_mem_sys.h"
+
+#include "mem_mgr/io_vaspace.h"
+#include <diagnostics/journal.h>
+#include "gpu/mem_mgr/mem_desc.h"
+#include "core/thread_state.h"
+#include <nvacpitypes.h>
+#include <platform/acpi_common.h>
+#include <core/locks.h>
+#include <ctrl/ctrl2080/ctrl2080gpu.h>
+#include "virtualization/hypervisor/hypervisor.h"
+#include "rmobjexportimport.h"
+#include <nvi2c.h>
+#include "rmapi/rs_utils.h"
+#include "rmapi/client_resource.h"
+#include "os/dce_rm_client_ipc.h"
+#include "mem_mgr/mem.h"
+#include "gpu/mem_mgr/virt_mem_allocator_common.h"
+
+#include <acpidsmguids.h>
+#include <pex.h>
+#include "gps.h"
+#include "jt.h"
+
+
+
+
+extern const char *ppOsBugCheckBugcodeStr[];
+
+
+ct_assert(NV_RM_PAGE_SIZE == RM_PAGE_SIZE);
+ct_assert(NV_RM_PAGE_MASK == RM_PAGE_MASK);
+ct_assert(NV_RM_PAGE_SHIFT == RM_PAGE_SHIFT);
+
+typedef struct
+{
+ NvU32 euid;
+ NvU32 pid;
+} TOKEN_USER, *PTOKEN_USER;
+
+struct OS_RM_CAPS
+{
+ NvU32 count;
+
+ // This should be the last element
+ nv_cap_t **caps;
+};
+
+NvBool osIsRaisedIRQL()
+{
+ return (!os_semaphore_may_sleep());
+}
+
+NvBool osIsISR()
+{
+ return os_is_isr();
+}
+
+NV_STATUS osGetDriverBlock
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ OS_DRIVER_BLOCK *pBlock
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osGetCurrentTick(NvU64 *pTimeInNs)
+{
+ *pTimeInNs = os_get_current_tick();
+ return NV_OK;
+}
+
+NvU64 osGetTickResolution(void)
+{
+ return os_get_tick_resolution();
+}
+
+NV_STATUS osGetPerformanceCounter(NvU64 *pTimeInNs)
+{
+ *pTimeInNs = os_get_current_tick_hr();
+ return NV_OK;
+}
+
+NV_STATUS osGetCurrentTime(
+ NvU32 *pSeconds,
+ NvU32 *pMicroSeconds
+)
+{
+ return os_get_current_time(pSeconds, pMicroSeconds);
+}
+
+/*!
+ * @brief Get timestamp for logging.
+ *
+ * Everything that logs a time stamp should use this routine for consistency.
+ *
+ * The returned value is OS dependent. We want the time stamp to use
+ * KeQueryPerformanceCounter on Windows so it matches the DirectX timestamps.
+ * Linux uses microseconds since 1970 (osGetCurrentTime), since matching DirectX
+ * is not a priority.
+ *
+ * osGetTimestampFreq returns the frequency required to decode the time stamps.
+ *
+ * @returns system dependent timestamp.
+ */
+NvU64 osGetTimestamp(void)
+{
+ NvU32 sec = 0;
+ NvU32 usec = 0;
+ osGetCurrentTime(&sec, &usec);
+ return (NvU64)sec * 1000000 + usec;
+}
+
+/*!
+ * @brief Get timestamp frequency.
+ *
+ * Timestamps are OS dependent. This call returns the frequency
+ * required to decode them.
+ *
+ * @returns Timestamp frequency. For example, 1000000 for MHz.
+ */
+NvU64 osGetTimestampFreq(void)
+{
+ return 1000000;
+}
+
+NV_STATUS osDelay(NvU32 milliseconds)
+{
+ return os_delay(milliseconds);
+}
+
+NV_STATUS osDelayUs(NvU32 microseconds)
+{
+ return os_delay_us(microseconds);
+}
+
+NV_STATUS osDelayNs(NvU32 nanoseconds)
+{
+ NvU32 microseconds = NV_MAX(1, (nanoseconds / 1000));
+ return os_delay_us(microseconds);
+}
+
+NvU32 osGetCpuFrequency(void)
+{
+ /* convert os_get_cpu_frequency()'s return value from Hz to MHz */
+ return ((NvU32)(os_get_cpu_frequency() / 1000000ULL));
+}
+
+void* osPciInitHandle(
+ NvU32 Domain,
+ NvU8 Bus,
+ NvU8 Slot,
+ NvU8 Function,
+ NvU16 *pVendor,
+ NvU16 *pDevice
+)
+{
+ //
+ // Check if the BDF is for a GPU that's already been attached, for which
+ // we should already have a handle cached. This won't catch devices that
+ // have been probed but not yet attached, but that shouldn't be a common
+ // occurrence.
+ //
+ // More importantly, having this check here means we don't need to check
+ // a global list of devices in the kernel interface layer, which could
+ // have the implication of taking another lock, causing hairy lock
+ // ordering issues.
+ //
+ if (Function == 0)
+ {
+ OBJGPU *pGpu = gpumgrGetGpuFromBusInfo(Domain, Bus, Slot);
+ if (pGpu != NULL)
+ {
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ if (pVendor) *pVendor = nv->pci_info.vendor_id;
+ if (pDevice) *pDevice = nv->pci_info.device_id;
+ return nv->handle;
+ }
+ }
+
+ return os_pci_init_handle(Domain, Bus, Slot, Function, pVendor, pDevice);
+}
+
+NvU8 osPciReadByte(
+ void *pHandle,
+ NvU32 Offset
+)
+{
+ NvU8 val;
+ os_pci_read_byte(pHandle, Offset, &val);
+ return val;
+}
+
+NvU16 osPciReadWord(
+ void *pHandle,
+ NvU32 Offset
+)
+{
+ NvU16 val;
+ os_pci_read_word(pHandle, Offset, &val);
+ return val;
+}
+
+NvU32 osPciReadDword(
+ void *pHandle,
+ NvU32 Offset
+)
+{
+ NvU32 val;
+ os_pci_read_dword(pHandle, Offset, &val);
+ return val;
+}
+
+void osPciWriteByte(
+ void *pHandle,
+ NvU32 Offset,
+ NvU8 Value
+)
+{
+ os_pci_write_byte(pHandle, Offset, Value);
+}
+
+void osPciWriteWord(
+ void *pHandle,
+ NvU32 Offset,
+ NvU16 Value
+)
+{
+ os_pci_write_word(pHandle, Offset, Value);
+}
+
+void osPciWriteDword(
+ void *pHandle,
+ NvU32 Offset,
+ NvU32 Value
+)
+{
+ os_pci_write_dword(pHandle, Offset, Value);
+}
+
+void* osMapKernelSpace(
+ RmPhysAddr Start,
+ NvU64 Size,
+ NvU32 Mode,
+ NvU32 Protect
+)
+{
+ NvU64 offset;
+ NvU8 *ptr;
+
+ if (0 == Size)
+ {
+ NV_ASSERT(Size != 0);
+ return NULL;
+ }
+
+ offset = (Start & ~os_page_mask);
+ Start &= os_page_mask;
+ Size = ((Size + offset + ~os_page_mask) & os_page_mask);
+
+ ptr = os_map_kernel_space(Start, Size, Mode);
+ if (ptr != NULL)
+ return (ptr + offset);
+
+ return NULL;
+}
+
+void osUnmapKernelSpace(
+ void *pAddress,
+ NvU64 Size
+)
+{
+ NvU64 offset;
+ NvUPtr ptr = (NvUPtr)pAddress;
+
+ if (0 == Size)
+ {
+ NV_ASSERT(Size != 0);
+ return;
+ }
+
+ offset = (ptr & ~os_page_mask);
+ ptr &= os_page_mask;
+ Size = ((Size + offset + ~os_page_mask) & os_page_mask);
+ os_unmap_kernel_space((void *)ptr, Size);
+}
+
+void* osMapIOSpace(
+ RmPhysAddr Start,
+ NvU64 Size,
+ void ** pData,
+ NvU32 User,
+ NvU32 Mode,
+ NvU32 Protect
+)
+{
+
+ NvU64 offset;
+ NvU8 *addr;
+
+ if (0 == Size)
+ {
+ NV_ASSERT(Size != 0);
+ return NULL;
+ }
+
+ offset = (Start & ~os_page_mask);
+ Start &= os_page_mask;
+ Size = ((Size + offset + ~os_page_mask) & os_page_mask);
+
+ if (User)
+ addr = os_map_user_space(Start, Size, Mode, Protect, pData);
+ else
+ addr = os_map_kernel_space(Start, Size, Mode);
+ if (addr != NULL)
+ return (addr + offset);
+
+ return addr;
+}
+
+void osUnmapIOSpace(
+ void *pAddress,
+ NvU64 Size,
+ void *pData,
+ NvU32 User
+)
+{
+ NvU64 offset;
+ NvUPtr addr = (NvUPtr)pAddress;
+
+ if (0 == Size)
+ {
+ NV_ASSERT(Size != 0);
+ return;
+ }
+
+ offset = (addr & ~os_page_mask);
+ addr &= os_page_mask;
+ Size = ((Size + offset + ~os_page_mask) & os_page_mask);
+
+ if (User)
+ os_unmap_user_space((void *)addr, Size, pData);
+ else
+ os_unmap_kernel_space((void *)addr, Size);
+}
+
+static NV_STATUS setNumaPrivData
+(
+ KernelMemorySystem *pKernelMemorySystem,
+ nv_state_t *nv,
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+ void *pAllocPrivate = NULL;
+ NvU64 *addrArray = NULL;
+ NvU64 numOsPages = pMemDesc->PageCount;
+
+ addrArray = pMemDesc->_pteArray;
+
+ if (NV_RM_PAGE_SIZE < os_page_size)
+ {
+ NvU64 numPages;
+ NvU64 i;
+
+ numPages = pMemDesc->PageCount;
+ addrArray = portMemAllocNonPaged(numPages * sizeof(NvU64));
+ if (addrArray == NULL)
+ {
+ return NV_ERR_NO_MEMORY;
+ }
+
+ portMemCopy((void*)addrArray,
+ (numPages * sizeof(NvU64)), (void*)pMemDesc->_pteArray,
+ (numPages * sizeof(NvU64)));
+ RmDeflateRmToOsPageArray(addrArray, numPages);
+ numOsPages = NV_RM_PAGES_TO_OS_PAGES(numPages);
+
+ for (i = 0; i < numOsPages; i++)
+ {
+ // Update GPA to system physical address
+ addrArray[i] += pKernelMemorySystem->coherentCpuFbBase;
+ }
+ }
+
+ rmStatus = nv_register_phys_pages(nv, addrArray, numOsPages, NV_MEMORY_CACHED, &pAllocPrivate);
+ if (rmStatus != NV_OK)
+ {
+ goto errors;
+ }
+
+ memdescSetMemData(pMemDesc, pAllocPrivate, NULL);
+
+errors:
+ if (NV_RM_PAGE_SIZE < os_page_size)
+ {
+ portMemFree(addrArray);
+ }
+
+ return rmStatus;
+}
+
+NV_STATUS osGetNumMemoryPages
+(
+ MEMORY_DESCRIPTOR *pMemDesc,
+ NvU32 *pNumPages
+)
+{
+ void *pAllocPrivate = NULL;
+
+ pAllocPrivate = memdescGetMemData(pMemDesc);
+ if (pAllocPrivate == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
+ return NV_ERR_INVALID_STATE;
+ }
+
+ return nv_get_num_phys_pages(pAllocPrivate, pNumPages);
+}
+
+NV_STATUS osGetMemoryPages
+(
+ MEMORY_DESCRIPTOR *pMemDesc,
+ void *pPages,
+ NvU32 *pNumPages
+)
+{
+ void *pAllocPrivate = NULL;
+
+ pAllocPrivate = memdescGetMemData(pMemDesc);
+ if (pAllocPrivate == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
+ return NV_ERR_INVALID_STATE;
+ }
+
+ return nv_get_phys_pages(pAllocPrivate, pPages, pNumPages);
+}
+
+NV_STATUS osMapSystemMemory
+(
+ MEMORY_DESCRIPTOR *pMemDesc,
+ NvU64 Offset,
+ NvU64 Length,
+ NvBool Kernel,
+ NvU32 Protect,
+ NvP64 *ppAddress,
+ NvP64 *ppPrivate
+)
+{
+ OBJGPU *pGpu = pMemDesc->pGpu;
+
+ RmPhysAddr userAddress;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ NV_STATUS rmStatus = NV_OK;
+ void *pAllocPrivate = NULL;
+ void *pAddress;
+ void *pPrivate = NULL;
+ NvU64 pageIndex;
+ NvU32 pageOffset;
+
+ if (pGpu != NULL &&
+ pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) &&
+ memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)
+ {
+ KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
+
+ rmStatus = setNumaPrivData(pKernelMemorySystem, nv, pMemDesc);
+ if (rmStatus != NV_OK)
+ return rmStatus;
+ }
+
+ *ppAddress = NvP64_NULL;
+ *ppPrivate = NvP64_NULL;
+
+ if ((Offset + Length) < Length)
+ return NV_ERR_INVALID_ARGUMENT;
+ if ((Offset + Length) > pMemDesc->Size)
+ return NV_ERR_INVALID_ARGUMENT;
+
+ pageIndex = (Offset >> os_page_shift);
+ pageOffset = (Offset & ~os_page_mask);
+
+ pAllocPrivate = memdescGetMemData(pMemDesc);
+ if (!pAllocPrivate)
+ {
+ NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
+ return NV_ERR_INVALID_STATE;
+ }
+
+ if (Kernel)
+ {
+ pAddress = nv_alloc_kernel_mapping(nv, pAllocPrivate,
+ pageIndex, pageOffset, Length, &pPrivate);
+ if (pAddress == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "failed to create system memory kernel mapping!\n");
+ rmStatus = NV_ERR_GENERIC;
+ }
+ else
+ {
+ *ppAddress = NV_PTR_TO_NvP64(pAddress);
+ *ppPrivate = NV_PTR_TO_NvP64(pPrivate);
+ }
+ }
+ else
+ {
+ rmStatus = nv_alloc_user_mapping(nv, pAllocPrivate,
+ pageIndex, pageOffset, Length, Protect, &userAddress,
+ &pPrivate);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "failed to create system memory user mapping!\n");
+ }
+ else
+ {
+ *ppAddress = (NvP64)(userAddress);
+ *ppPrivate = NV_PTR_TO_NvP64(pPrivate);
+ }
+ }
+
+ return rmStatus;
+}
+
+void osUnmapSystemMemory
+(
+ MEMORY_DESCRIPTOR *pMemDesc,
+ NvBool Kernel,
+ NvU32 ProcessId,
+ NvP64 pAddress,
+ NvP64 pPrivate
+)
+{
+ NV_STATUS status;
+ void *pAllocPrivate = memdescGetMemData(pMemDesc);
+ OBJGPU *pGpu = pMemDesc->pGpu;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ if (Kernel)
+ {
+ status = nv_free_kernel_mapping(nv, pAllocPrivate, NvP64_VALUE(pAddress),
+ NvP64_VALUE(pPrivate));
+ }
+ else
+ {
+ status = nv_free_user_mapping(nv, pAllocPrivate, (NvU64)pAddress,
+ NvP64_VALUE(pPrivate));
+ }
+
+ if (pGpu != NULL &&
+ pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) &&
+ memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM &&
+ pAllocPrivate != NULL)
+ {
+ nv_unregister_phys_pages(nv, pAllocPrivate);
+ memdescSetMemData(pMemDesc, NULL, NULL);
+ }
+
+ NV_ASSERT(status == NV_OK);
+}
+
+void osIoWriteByte(
+ NvU32 Address,
+ NvU8 Value
+)
+{
+ os_io_write_byte(Address, Value);
+}
+
+NvU16 osIoReadWord(
+ NvU32 Address
+)
+{
+ return os_io_read_word(Address);
+}
+
+void osIoWriteWord(
+ NvU32 Address,
+ NvU16 Value
+)
+{
+ os_io_write_word(Address, Value);
+}
+
+NvU8 osIoReadByte(
+ NvU32 Address
+)
+{
+ return os_io_read_byte(Address);
+}
+
+NvBool osIsAdministrator(void)
+{
+ return os_is_administrator();
+}
+
+NvBool osAllowPriorityOverride(void)
+{
+ return os_allow_priority_override();
+}
+
+NvU32 osGetCurrentProcess(void)
+{
+ return os_get_current_process();
+}
+
+void osGetCurrentProcessName(char *ProcName, NvU32 Length)
+{
+ return os_get_current_process_name(ProcName, Length);
+}
+
+NV_STATUS osGetCurrentThread(OS_THREAD_HANDLE *pThreadId)
+{
+ NV_STATUS rmStatus;
+ NvU64 threadId = 0;
+
+ if (pThreadId == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ rmStatus = os_get_current_thread(&threadId);
+ if (rmStatus == NV_OK)
+ {
+ *pThreadId = threadId;
+ }
+ else
+ {
+ *pThreadId = 0;
+ }
+
+ return rmStatus;
+}
+
+NV_STATUS osAttachToProcess(void** ppProcessInfo, NvU32 ProcessId)
+{
+ //
+ // This function is used by RmUnmapMemory() to attach to the
+ // process for which a given device memory mapping was
+ // created, in order to be able to unmap it. On Linux/UNIX
+ // platforms, we can't "attach" to a random process, but
+ // since we don't create/destroy user mappings in the RM, we
+ // don't need to, either.
+ //
+ // Report success to the caller to keep RmUnmapMemory() from
+ // failing, and memory from being leaked as a result.
+ //
+ *ppProcessInfo = NULL;
+ return NV_OK;
+}
+
+void osDetachFromProcess(void* pProcessInfo)
+{
+ // stub
+ return;
+}
+
+NvBool osDbgBreakpointEnabled(void)
+{
+ return NV_TRUE;
+}
+
+NV_STATUS osAcquireRmSema(void *pSema)
+{
+ return NV_OK;
+}
+
+NV_STATUS osCondAcquireRmSema(void *pSema)
+{
+ return NV_OK;
+}
+
+NvU32 osReleaseRmSema(void *pSema, OBJGPU *pDpcGpu)
+{
+ return NV_SEMA_RELEASE_SUCCEED;
+}
+
+void osSpinLoop(void)
+{
+ // Enable this code to get debug prints from Libos.
+}
+
+NvU64 osGetMaxUserVa(void)
+{
+ return os_get_max_user_va();
+}
+
+NV_STATUS osSchedule(void)
+{
+ return os_schedule();
+}
+
+NV_STATUS osQueueWorkItemWithFlags(
+ OBJGPU *pGpu,
+ OSWorkItemFunction pFunction,
+ void *pParams,
+ NvU32 flags
+)
+{
+ nv_work_item_t *pWi;
+ nv_state_t *nv;
+ NV_STATUS status;
+
+ pWi = portMemAllocNonPaged(sizeof(nv_work_item_t));
+
+ if (NULL == pWi)
+ {
+ return NV_ERR_NO_MEMORY;
+ }
+
+ pWi->flags = NV_WORK_ITEM_FLAGS_REQUIRES_GPU;
+ if (flags & OS_QUEUE_WORKITEM_FLAGS_DONT_FREE_PARAMS)
+ pWi->flags |= NV_WORK_ITEM_FLAGS_DONT_FREE_DATA;
+
+ if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA)
+ pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA;
+ if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW)
+ pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW;
+ if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW)
+ pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW;
+ if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW)
+ pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW;
+ if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW)
+ pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW;
+
+ if (flags & OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY)
+ pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY;
+
+ pWi->gpuInstance = gpuGetInstance(pGpu);
+ pWi->func.pGpuFunction = pFunction;
+ pWi->pData = pParams;
+ nv = NV_GET_NV_STATE(pGpu);
+
+ status = os_queue_work_item(nv ? nv->queue : NULL, pWi);
+
+ if (NV_OK != status)
+ {
+ portMemFree((void *)pWi);
+ }
+
+ return status;
+}
+
+NV_STATUS osQueueWorkItem(
+ OBJGPU *pGpu,
+ OSWorkItemFunction pFunction,
+ void *pParams
+)
+{
+ return osQueueWorkItemWithFlags(pGpu, pFunction, pParams, OS_QUEUE_WORKITEM_FLAGS_NONE);
+}
+
+NV_STATUS osQueueSystemWorkItem(
+ OSSystemWorkItemFunction pFunction,
+ void *pParams
+)
+{
+ nv_work_item_t *pWi;
+ NV_STATUS status;
+
+ pWi = portMemAllocNonPaged(sizeof(nv_work_item_t));
+
+ if (NULL == pWi)
+ {
+ return NV_ERR_NO_MEMORY;
+ }
+
+ pWi->flags = NV_WORK_ITEM_FLAGS_NONE;
+ pWi->func.pSystemFunction = pFunction;
+ pWi->pData = pParams;
+
+ status = os_queue_work_item(NULL, pWi);
+
+ if (NV_OK != status)
+ {
+ portMemFree((void *)pWi);
+ }
+
+ return status;
+}
+
+void osQueueMMUFaultHandler(OBJGPU *pGpu)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ nv_schedule_uvm_isr(nv);
+}
+
+static inline nv_dma_device_t* osGetDmaDeviceForMemDesc(
+ OS_GPU_INFO *pOsGpuInfo,
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ return (pOsGpuInfo->niso_dma_dev != NULL) &&
+ memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) ?
+ pOsGpuInfo->niso_dma_dev : pOsGpuInfo->dma_dev;
+}
+
+NV_STATUS osDmaMapPages(
+ OS_GPU_INFO *pOsGpuInfo,
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ return nv_dma_map_pages(
+ osGetDmaDeviceForMemDesc(pOsGpuInfo, pMemDesc),
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ memdescGetPteArray(pMemDesc, AT_CPU),
+ memdescGetContiguity(pMemDesc, AT_CPU),
+ memdescGetCpuCacheAttrib(pMemDesc),
+ NULL);
+}
+
+NV_STATUS osDmaUnmapPages(
+ OS_GPU_INFO *pOsGpuInfo,
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ return nv_dma_unmap_pages(
+ pOsGpuInfo->dma_dev,
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ memdescGetPteArray(pMemDesc, AT_CPU),
+ NULL);
+}
+
+//
+// Set the DMA address size for the given GPU
+//
+// This is a global device setting and care would need to be taken if it was to
+// be modified outside of GPU initialization. At least on Linux other drivers,
+// like UVM, might be requesting its own DMA mappings for the same GPU after
+// the GPU has been initialized.
+//
+void osDmaSetAddressSize(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 bits
+)
+{
+ nv_set_dma_address_size(pOsGpuInfo, bits);
+}
+
+NV_STATUS osAllocPagesInternal(
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJGPU *pGpu = pMemDesc->pGpu;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ void *pMemData = NULL;
+ NV_STATUS status;
+
+ memdescSetAddress(pMemDesc, NvP64_NULL);
+ memdescSetMemData(pMemDesc, NULL, NULL);
+
+ NV_ASSERT_OR_RETURN(pMemDesc->PageCount > 0, NV_ERR_INVALID_ARGUMENT);
+
+ if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
+ {
+ if (NV_RM_PAGE_SIZE < os_page_size &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ status = nv_alias_pages(
+ NV_GET_NV_STATE(pGpu),
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ memdescGetContiguity(pMemDesc, AT_CPU),
+ memdescGetCpuCacheAttrib(pMemDesc),
+ memdescGetGuestId(pMemDesc),
+ memdescGetPteArray(pMemDesc, AT_CPU),
+ &pMemData);
+ }
+ else
+ {
+ NvBool unencrypted = 0;
+
+ if (nv && (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_32BIT_ADDRESSABLE)))
+ nv->force_dma32_alloc = NV_TRUE;
+
+ status = nv_alloc_pages(
+ NV_GET_NV_STATE(pGpu),
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ memdescGetContiguity(pMemDesc, AT_CPU),
+ memdescGetCpuCacheAttrib(pMemDesc),
+ pSys->getProperty(pSys,
+ PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS),
+ unencrypted,
+ memdescGetPteArray(pMemDesc, AT_CPU),
+ &pMemData);
+
+ if (nv && nv->force_dma32_alloc)
+ nv->force_dma32_alloc = NV_FALSE;
+ }
+
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ //
+ // If the OS layer doesn't think in RM page size, we need to inflate the
+ // PTE array into RM pages.
+ //
+ if (NV_RM_PAGE_SIZE < os_page_size &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ memdescSetMemData(pMemDesc, pMemData, NULL);
+
+ return status;
+}
+
+void osFreePagesInternal(
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ OBJGPU *pGpu = pMemDesc->pGpu;
+ NV_STATUS rmStatus;
+
+ if (NV_RM_PAGE_SIZE < os_page_size &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ rmStatus = nv_free_pages(NV_GET_NV_STATE(pGpu),
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ memdescGetContiguity(pMemDesc, AT_CPU),
+ memdescGetCpuCacheAttrib(pMemDesc),
+ memdescGetMemData(pMemDesc));
+ NV_ASSERT(rmStatus == NV_OK);
+}
+
+NV_STATUS osLockMem(
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ // Not supported on this OS.
+ DBG_BREAKPOINT();
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osUnlockMem(
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ // Not supported on this OS.
+ DBG_BREAKPOINT();
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osMapPciMemoryUser(
+ OS_GPU_INFO *pOsGpuInfo,
+ RmPhysAddr busAddress,
+ NvU64 length,
+ NvU32 Protect,
+ NvP64 *pVirtualAddress,
+ NvP64 *pPriv,
+ NvU32 modeFlag
+)
+{
+ void *addr;
+ void *priv = NULL;
+
+ addr = osMapIOSpace(busAddress, length, &priv, NV_TRUE, modeFlag, Protect);
+
+ *pPriv = NV_PTR_TO_NvP64(priv);
+ *pVirtualAddress = NV_PTR_TO_NvP64(addr);
+
+ return (addr != NULL) ? NV_OK : NV_ERR_GENERIC;
+}
+
+void osUnmapPciMemoryUser(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvP64 virtualAddress,
+ NvU64 length,
+ NvP64 pPriv
+)
+{
+ void *addr, *priv;
+
+ addr = NvP64_VALUE(virtualAddress);
+ priv = NvP64_VALUE(pPriv);
+
+ osUnmapIOSpace(addr, length, priv, NV_TRUE);
+}
+
+NV_STATUS osMapPciMemoryKernelOld
+(
+ OBJGPU *pGpu,
+ RmPhysAddr busAddress,
+ NvU64 length,
+ NvU32 Protect,
+ void **pVirtualAddress,
+ NvU32 modeFlag
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_kern_mapping_t *mapping;
+
+ if (pVirtualAddress == NULL)
+ return NV_ERR_GENERIC;
+
+ *pVirtualAddress = os_map_kernel_space(busAddress, length, modeFlag);
+ if (*pVirtualAddress == NULL)
+ return NV_ERR_GENERIC;
+
+ mapping = portMemAllocNonPaged(sizeof(nv_kern_mapping_t));
+ if (NULL == mapping)
+ {
+ os_unmap_kernel_space(*pVirtualAddress, length);
+ *pVirtualAddress = 0;
+ return NV_ERR_GENERIC;
+ }
+
+ mapping->addr = *pVirtualAddress;
+ mapping->size = length;
+ mapping->modeFlag = modeFlag;
+
+ mapping->next = nv->kern_mappings;
+ nv->kern_mappings = mapping;
+
+ return NV_OK;
+}
+
+NV_STATUS osMapPciMemoryKernel64
+(
+ OBJGPU *pGpu,
+ RmPhysAddr busAddress,
+ NvU64 length,
+ NvU32 Protect,
+ NvP64 *pVirtualAddress,
+ NvU32 modeFlag
+)
+{
+ void *tmppVirtualAddress = NvP64_VALUE(pVirtualAddress);
+ NV_STATUS rc;
+
+ rc = osMapPciMemoryKernelOld(pGpu,
+ busAddress,
+ length,
+ Protect,
+ &tmppVirtualAddress,
+ modeFlag);
+
+ *pVirtualAddress = NV_PTR_TO_NvP64(tmppVirtualAddress);
+
+ return rc;
+}
+
+void osUnmapPciMemoryKernelOld
+(
+ OBJGPU *pGpu,
+ void* virtualAddress
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_kern_mapping_t *mapping, *tmp;
+
+ // this can happen, for example, during a call to RmShutdownAdapter()
+ // from a failed RmInitAdapter()
+ if (virtualAddress == NULL)
+ {
+ return;
+ }
+
+ tmp = mapping = nv->kern_mappings;
+ while (mapping)
+ {
+ if (mapping->addr == virtualAddress)
+ {
+ if (mapping == nv->kern_mappings)
+ {
+ nv->kern_mappings = mapping->next;
+ }
+ else
+ {
+ tmp->next = mapping->next;
+ }
+
+ os_unmap_kernel_space(mapping->addr, mapping->size);
+
+ portMemFree(mapping);
+ return;
+ }
+ tmp = mapping;
+ mapping = mapping->next;
+ }
+
+ DBG_BREAKPOINT();
+}
+
+void osUnmapPciMemoryKernel64
+(
+ OBJGPU *pGpu,
+ NvP64 virtualAddress
+)
+{
+ osUnmapPciMemoryKernelOld(pGpu, NvP64_VALUE(virtualAddress));
+}
+
+NV_STATUS osMapGPU(
+ OBJGPU *pGpu,
+ RS_PRIV_LEVEL privLevel,
+ NvU64 offset,
+ NvU64 length,
+ NvU32 Protect,
+ NvP64 *pAddress,
+ NvP64 *pPriv
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+
+ if (privLevel >= RS_PRIV_LEVEL_KERNEL)
+ {
+ if (!portSafeAddU64((NvUPtr)pGpu->deviceMappings[0].gpuNvAddr, offset, (NvU64*)pAddress))
+ {
+ rmStatus = NV_ERR_INVALID_LIMIT;
+ }
+ }
+ else
+ {
+ RmPhysAddr busAddress;
+ if (!portSafeAddU64(pGpu->busInfo.gpuPhysAddr, offset, &busAddress))
+ {
+ rmStatus = NV_ERR_INVALID_LIMIT;
+ }
+ else
+ {
+ rmStatus = osMapPciMemoryUser(pGpu->pOsGpuInfo,
+ busAddress,
+ length,
+ Protect,
+ pAddress,
+ pPriv,
+ NV_FALSE);
+ }
+ }
+
+ return rmStatus;
+}
+
+void osUnmapGPU(
+ OS_GPU_INFO *pOsGpuInfo,
+ RS_PRIV_LEVEL privLevel,
+ NvP64 address,
+ NvU64 length,
+ NvP64 priv
+)
+{
+ if (privLevel < RS_PRIV_LEVEL_KERNEL)
+ {
+ osUnmapPciMemoryUser(pOsGpuInfo, address, length, priv);
+ }
+}
+
+NV_STATUS osDeviceClassToDeviceName(
+ NvU32 deviceInstance,
+ NvU8 *szName
+)
+{
+ return NV_ERR_GENERIC;
+}
+
+static void postEvent(
+ nv_event_t *event,
+ NvU32 hEvent,
+ NvU32 notifyIndex,
+ NvU32 info32,
+ NvU16 info16,
+ NvBool dataValid
+)
+{
+ nv_state_t *nv = nv_get_ctl_state();
+ portSyncSpinlockAcquire(nv->event_spinlock);
+ if (event->active)
+ nv_post_event(event, hEvent, notifyIndex,
+ info32, info16, dataValid);
+ portSyncSpinlockRelease(nv->event_spinlock);
+}
+
+NvU32 osSetEvent
+(
+ OBJGPU *pGpu,
+ NvP64 eventID
+)
+{
+ nv_event_t *event = NvP64_VALUE(eventID);
+ postEvent(event, 0, 0, 0, 0, NV_FALSE);
+ return 1;
+}
+
+NV_STATUS osNotifyEvent(
+ OBJGPU *pGpu,
+ PEVENTNOTIFICATION NotifyEvent,
+ NvU32 Method,
+ NvU32 Data,
+ NV_STATUS Status
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+
+ // notify the event
+ switch (NotifyEvent->NotifyType)
+ {
+ case NV01_EVENT_OS_EVENT:
+ {
+ nv_event_t *event = NvP64_VALUE(NotifyEvent->Data);
+ postEvent(event,
+ NotifyEvent->hEvent,
+ NotifyEvent->NotifyIndex,
+ 0, 0,
+ NotifyEvent->bEventDataRequired);
+ break;
+ }
+
+ // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX.
+ case NV01_EVENT_KERNEL_CALLBACK:
+ {
+ MINIPORT_CALLBACK callBackToMiniport =
+ (MINIPORT_CALLBACK)NvP64_VALUE(NotifyEvent->Data);
+
+ // perform a direct callback to the miniport
+ if (callBackToMiniport)
+ callBackToMiniport(NV_GET_NV_STATE(pGpu));
+ break;
+ }
+
+ case NV01_EVENT_KERNEL_CALLBACK_EX:
+ {
+ NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(NotifyEvent->Data);
+
+ // passes two arguments (arg, params) to the kernel callback instead of one (arg).
+ if (kc && kc->func)
+ {
+ kc->func(kc->arg, NULL, NotifyEvent->hEvent, Data, Status);
+ }
+ break;
+ }
+
+
+ default:
+ {
+ rmStatus = NV_ERR_GENERIC;
+ break;
+ }
+ }
+
+ return rmStatus;
+
+} // end of osNotifyEvent()
+
+// Allow CPL Events to be callback or events
+NV_STATUS osEventNotification
+(
+ OBJGPU *pGpu,
+ PEVENTNOTIFICATION pNotifyEvent,
+ NvU32 notifyIndex,
+ void * pEventData,
+ NvU32 eventDataSize
+)
+{
+ return osEventNotificationWithInfo(pGpu, pNotifyEvent, notifyIndex, 0, 0,
+ pEventData, eventDataSize);
+}
+
+NV_STATUS osEventNotificationWithInfo
+(
+ OBJGPU *pGpu,
+ PEVENTNOTIFICATION pNotifyEvent,
+ NvU32 notifyIndex,
+ NvU32 info32,
+ NvU16 info16,
+ void * pEventData,
+ NvU32 eventDataSize
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+
+ // walk this object's event list and find any matches for this specific notify
+ for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next)
+ {
+ // notifyIndex must match if request isn't for all
+ if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) &&
+ (pNotifyEvent->NotifyIndex != notifyIndex))
+ {
+ continue;
+ }
+
+ switch (pNotifyEvent->NotifyType)
+ {
+ case NV_EVENT_BUFFER_BIND:
+ case NV01_EVENT_WIN32_EVENT:
+ {
+ nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data);
+ postEvent(event,
+ pNotifyEvent->hEvent,
+ pNotifyEvent->NotifyIndex,
+ info32, info16,
+ pNotifyEvent->bEventDataRequired);
+ break;
+ }
+
+ // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX.
+ case NV01_EVENT_KERNEL_CALLBACK:
+ {
+ MINIPORT_CALLBACK callBackToMiniport =
+ (MINIPORT_CALLBACK)NvP64_VALUE(pNotifyEvent->Data);
+
+ // perform a direct callback to the miniport
+ if (callBackToMiniport)
+ callBackToMiniport(NV_GET_NV_STATE(pGpu));
+ break;
+ }
+
+ case NV01_EVENT_KERNEL_CALLBACK_EX:
+ {
+ NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data);
+
+ if (kc && kc->func)
+ {
+ kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK);
+ }
+ break;
+ }
+
+ default:
+ break;
+ }
+ }
+
+ return rmStatus;
+}
+
+// Allow CPL Events to be callback or events
+NV_STATUS osObjectEventNotification
+(
+ NvHandle hClient,
+ NvHandle hObject,
+ NvU32 hClass,
+ PEVENTNOTIFICATION pNotifyEvent,
+ NvU32 notifyIndex,
+ void *pEventData,
+ NvU32 eventDataSize
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+
+ NV_PRINTF(LEVEL_INFO, "%s()\n", __FUNCTION__);
+ // walk this object's event list and find any matches for this specific notify
+ for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next)
+ {
+ // notifyIndex must match if request isn't for all
+ if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) &&
+ (pNotifyEvent->NotifyIndex != notifyIndex))
+ {
+ continue;
+ }
+
+ switch (pNotifyEvent->NotifyType)
+ {
+ case NV01_EVENT_OS_EVENT:
+ {
+ nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data);
+ postEvent(event,
+ pNotifyEvent->hEvent,
+ pNotifyEvent->NotifyIndex,
+ 0, 0,
+ pNotifyEvent->bEventDataRequired);
+ break;
+ }
+
+ case NV01_EVENT_KERNEL_CALLBACK_EX:
+ {
+ NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data);
+
+ if (kc && kc->func)
+ {
+ kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK);
+ }
+ break;
+ }
+
+ default:
+ break;
+ }
+ }
+
+ return rmStatus;
+}
+
+NV_STATUS osReferenceObjectCount(void *pEvent)
+{
+ nv_state_t *nv = nv_get_ctl_state();
+ nv_event_t *event = pEvent;
+
+ portSyncSpinlockAcquire(nv->event_spinlock);
+ ++event->refcount;
+ portSyncSpinlockRelease(nv->event_spinlock);
+ return NV_OK;
+}
+
+NV_STATUS osDereferenceObjectCount(void *pOSEvent)
+{
+ nv_state_t *nv = nv_get_ctl_state();
+ nv_event_t *event = pOSEvent;
+
+ portSyncSpinlockAcquire(nv->event_spinlock);
+ NV_ASSERT(event->refcount > 0);
+ --event->refcount;
+ // If event->refcount == 0 but event->active is true, the client
+ // has not yet freed the OS event. free_os_event will free its
+ // memory when they do, or else when the client itself is freed.
+ if (event->refcount == 0 && !event->active)
+ portMemFree(event);
+ portSyncSpinlockRelease(nv->event_spinlock);
+
+ return NV_OK;
+}
+
+NV_STATUS osUserHandleToKernelPtr(NvHandle hClient, NvP64 hEvent, NvP64 *pEvent)
+{
+ nv_state_t *nv = nv_get_ctl_state();
+ NvU32 fd = (NvU64)hEvent;
+ NV_STATUS result;
+
+ portSyncSpinlockAcquire(nv->event_spinlock);
+ nv_event_t *e = nv->event_list;
+ while (e != NULL)
+ {
+ if (e->fd == fd && e->hParent == hClient)
+ break;
+ e = e->next;
+ }
+
+ if (e != NULL)
+ {
+ ++e->refcount;
+ *pEvent = NV_PTR_TO_NvP64(e);
+ result = NV_OK;
+ }
+ else
+ result = NV_ERR_OBJECT_NOT_FOUND;
+ portSyncSpinlockRelease(nv->event_spinlock);
+
+ return result;
+}
+
+NV_STATUS osFlushCpuCache(void)
+{
+ return os_flush_cpu_cache_all();
+}
+
+void osFlushCpuWriteCombineBuffer(void)
+{
+ os_flush_cpu_write_combine_buffer();
+}
+
+
+//
+// Evict GPU memory range from the CPU caches.
+//
+// On some platforms (e.g. P9+V100), the CPU can coherently cache GPU memory
+// and RM takes advantage of that. Most everything is handled transparently,
+// but there are two exceptions that require explicitly flushing any CPU cache
+// lines of GPU memory. These are:
+//
+// 1) Flushing memory backing ACR regions before they get locked.
+//
+// Otherwise the cache could get flushed while the regions are locked causing a
+// region violation physical fault. See more details in
+// acrFlushRegionsFromGpuCoherentCpuCache_IMPL().
+//
+// 2) Flushing all of FB before GPU reset (NVLink going down)
+//
+// Leaving cache entries on the CPU causes fatal errors when the CPU tries
+// flushing them later while the link is down. See more details in
+// nvlinkStatePostUnload_IMPL().
+//
+void osFlushGpuCoherentCpuCacheRange
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU64 cpuVirtual,
+ NvU64 size
+)
+{
+ nv_flush_coherent_cpu_cache_range(pOsGpuInfo, cpuVirtual, size);
+}
+
+void osErrorLogV(OBJGPU *pGpu, NvU32 num, const char * pFormat, va_list arglist)
+{
+ NV_STATUS rmStatus;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ if ((pFormat == NULL) || (*pFormat == '\0'))
+ {
+ return;
+ }
+
+ rmStatus = nv_log_error(nv, num, pFormat, arglist);
+ NV_ASSERT(rmStatus == NV_OK);
+}
+
+void osErrorLog(OBJGPU *pGpu, NvU32 num, const char* pFormat, ...)
+{
+ va_list arglist;
+ va_start(arglist, pFormat);
+ osErrorLogV(pGpu, num, pFormat, arglist);
+ va_end(arglist);
+}
+
+NvU32
+osPollHotkeyState
+(
+ OBJGPU *pGpu
+)
+{
+ return 0;
+}
+
+void osDevWriteReg008(
+ OBJGPU *pGpu,
+ DEVICE_MAPPING *pMapping,
+ NvU32 thisAddress,
+ NvV8 thisValue
+)
+{
+ if (thisAddress >= pMapping->gpuNvLength)
+ {
+ NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+ return;
+ }
+
+ NV_PRIV_REG_WR08(pMapping->gpuNvAddr, thisAddress, thisValue);
+}
+
+void osDevWriteReg016(
+ OBJGPU *pGpu,
+ DEVICE_MAPPING *pMapping,
+ NvU32 thisAddress,
+ NvV16 thisValue
+)
+{
+ if (thisAddress >= pMapping->gpuNvLength)
+ {
+ NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+ return;
+ }
+
+ NV_PRIV_REG_WR16(pMapping->gpuNvAddr, thisAddress, thisValue);
+}
+
+void osDevWriteReg032(
+ OBJGPU *pGpu,
+ DEVICE_MAPPING *pMapping,
+ NvU32 thisAddress,
+ NvV32 thisValue
+)
+{
+ NvBool vgpuHandled = NV_FALSE;
+
+ vgpuDevWriteReg032(pGpu, thisAddress, thisValue, &vgpuHandled);
+ if (vgpuHandled)
+ {
+ return;
+ }
+
+ if (thisAddress >= pMapping->gpuNvLength)
+ {
+ NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+ return;
+ }
+
+ NV_PRIV_REG_WR32(pMapping->gpuNvAddr, thisAddress, thisValue);
+}
+
+NvU8 osDevReadReg008(
+ OBJGPU *pGpu,
+ DEVICE_MAPPING *pMapping,
+ NvU32 thisAddress
+)
+{
+ NvU8 retval = 0;
+
+ if (thisAddress >= pMapping->gpuNvLength)
+ {
+ NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+ }
+ else
+ retval = NV_PRIV_REG_RD08(pMapping->gpuNvAddr, thisAddress);
+
+ return retval;
+}
+
+NvU16 osDevReadReg016(
+ OBJGPU *pGpu,
+ DEVICE_MAPPING *pMapping,
+ NvU32 thisAddress
+)
+{
+ NvU16 retval = 0;
+
+ if (thisAddress >= pMapping->gpuNvLength)
+ {
+ NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+ }
+ else
+ retval = NV_PRIV_REG_RD16(pMapping->gpuNvAddr, thisAddress);
+
+ return retval;
+}
+
+NvU32 osDevReadReg032(
+ OBJGPU *pGpu,
+ DEVICE_MAPPING *pMapping,
+ NvU32 thisAddress
+)
+{
+ NvU32 retval = 0;
+ NvBool vgpuHandled = NV_FALSE;
+
+ retval = vgpuDevReadReg032(pGpu, thisAddress, &vgpuHandled);
+ if (vgpuHandled)
+ {
+ return retval;
+ }
+
+ if (thisAddress >= pMapping->gpuNvLength)
+ {
+ NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+ }
+ else
+ retval = NV_PRIV_REG_RD32(pMapping->gpuNvAddr, thisAddress);
+
+ return retval;
+}
+
+NV_STATUS osReadRegistryDwordBase(
+ OBJGPU *pGpu,
+ const char *regParmStr,
+ NvU32 *Data
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ return RmReadRegistryDword(nv, regParmStr, Data);
+}
+
+NV_STATUS osWriteRegistryDword(
+ OBJGPU *pGpu,
+ const char *regParmStr,
+ NvU32 Data
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ return RmWriteRegistryDword(nv, regParmStr, Data);
+}
+
+NV_STATUS osReadRegistryBinary(
+ OBJGPU *pGpu,
+ const char *regParmStr,
+ NvU8 *Data,
+ NvU32 *cbLen
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ return RmReadRegistryBinary(nv, regParmStr, Data, cbLen);
+}
+
+NV_STATUS osWriteRegistryBinary(
+ OBJGPU *pGpu,
+ const char *regParmStr,
+ NvU8 *Data,
+ NvU32 cbLen
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ return RmWriteRegistryBinary(nv, regParmStr, Data, cbLen);
+}
+
+NV_STATUS osWriteRegistryVolatile(
+ OBJGPU *pGpu,
+ const char *regParmStr,
+ NvU8 *Data,
+ NvU32 cbLen
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osReadRegistryVolatile
+(
+ OBJGPU *pGpu,
+ const char *regParmStr,
+ NvU8 *Data,
+ NvU32 cbLen
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osReadRegistryVolatileSize
+(
+ OBJGPU *pGpu,
+ const char *regParmStr,
+ NvU32 *pSize
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osReadRegistryStringBase(
+ OBJGPU *pGpu,
+ const char *regParmStr,
+ NvU8 *buffer,
+ NvU32 *pBufferLength
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ return RmReadRegistryString(nv, regParmStr, buffer, pBufferLength);
+}
+
+NV_STATUS osPackageRegistry(
+ OBJGPU *pGpu,
+ PACKED_REGISTRY_TABLE *pRegTable,
+ NvU32 *pSize
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ return RmPackageRegistry(nv, pRegTable, pSize);
+}
+
+NvU32 osGetCpuCount()
+{
+ return os_get_cpu_count(); // Total number of logical CPUs.
+}
+
+NvU32 osGetCurrentProcessorNumber(void)
+{
+ return os_get_cpu_number();
+}
+
+void osGetTimeoutParams(OBJGPU *pGpu, NvU32 *pTimeoutUs, NvU32 *pScale, NvU32 *pFlags)
+{
+ NvU32 gpuMode = gpuGetMode(pGpu);
+
+ NV_ASSERT((NV_GPU_MODE_GRAPHICS_MODE == gpuMode) ||
+ (NV_GPU_MODE_COMPUTE_MODE == gpuMode));
+
+ if (hypervisorIsVgxHyper())
+ {
+ //
+ // 1.8 seconds is chosen because it is 90% of the overall hard limit of 2.0
+ // seconds, imposed by WDDM driver rules.
+ // Currently primary use case of VGX is Windows, so setting 1.8 as default
+ //
+ *pTimeoutUs = 1.8 * 1000000;
+ }
+ else
+ {
+ switch (gpuMode)
+ {
+ default:
+ case NV_GPU_MODE_GRAPHICS_MODE:
+ *pTimeoutUs = 4 * 1000000;
+ break;
+
+ case NV_GPU_MODE_COMPUTE_MODE:
+ *pTimeoutUs = 30 * 1000000;
+ break;
+ }
+ }
+
+ *pFlags = GPU_TIMEOUT_FLAGS_OSTIMER;
+
+ *pScale = 1;
+ if (IS_EMULATION(pGpu) || IS_SIMULATION(pGpu))
+ {
+ *pScale = 60; // 1s -> 1m
+ }
+ return;
+}
+
+void osFlushLog()
+{
+ // Not implemented
+}
+
+NvU32 osGetSimulationMode(void)
+{
+ return NV_SIM_MODE_HARDWARE;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixFlushUserCache_IMPL
+(
+ RmClientResource *pRmCliRes,
+ NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS *pAddressSpaceParams
+)
+{
+ Memory *pMemory;
+ MEMORY_DESCRIPTOR *pMemDesc;
+ NvU64 start, end;
+ NvBool bInvalidateOnly;
+
+ NV_CHECK_OK_OR_RETURN(LEVEL_SILENT,
+ memGetByHandle(RES_GET_CLIENT(pRmCliRes),
+ pAddressSpaceParams->hObject,
+ &pMemory));
+
+ pMemDesc = pMemory->pMemDesc;
+
+ if (memdescGetAddressSpace(pMemDesc) != ADDR_SYSMEM)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s: wrong address space %d\n",
+ __FUNCTION__, memdescGetAddressSpace(pMemDesc));
+ return NV_ERR_INVALID_COMMAND;
+ }
+
+ if (memdescGetCpuCacheAttrib(pMemDesc) != NV_MEMORY_CACHED)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s: wrong caching type %d\n",
+ __FUNCTION__, memdescGetCpuCacheAttrib(pMemDesc));
+ return NV_ERR_INVALID_COMMAND;
+ }
+
+ start = pAddressSpaceParams->offset;
+ end = start + pAddressSpaceParams->length;
+
+ switch(pAddressSpaceParams->cacheOps)
+ {
+ case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH_INVALIDATE:
+ case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH:
+ bInvalidateOnly = NV_FALSE;
+ break;
+
+ case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_INVALIDATE:
+ bInvalidateOnly = NV_TRUE;
+ break;
+
+ default:
+ NV_PRINTF(LEVEL_ERROR, "%s: cacheOps not specified\n", __FUNCTION__);
+ return NV_ERR_INVALID_COMMAND;
+ }
+
+ if ((end - start) > pMemory->Length)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: end address 0x%llx exceeded buffer length: 0x%llx\n",
+ __FUNCTION__, end, pMemory->Length);
+ return NV_ERR_INVALID_LIMIT;
+ }
+
+ if (bInvalidateOnly)
+ {
+ //
+ // XXX: this seems fishy - I'm not sure if invalidating by the kernel
+ // VA only as nv_dma_cache_invalidate() does here is sufficient for
+ // this control call.
+ // pAddressSpaceParams->internalOnly is expected to be the RM client
+ // VA for this control call; if we wanted to invalidate the user VA we
+ // could do so using that.
+ //
+ // For I/O coherent platforms this won't actually do anything.
+ // On non-I/O-coherent platforms, there's no need to do a second
+ // invalidation after the full flush.
+ //
+ nv_state_t *nv = NV_GET_NV_STATE(pMemDesc->pGpu);
+ if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
+ {
+ PIOVAMAPPING pIovaMapping = memdescGetIommuMap(pMemDesc, nv->iovaspace_id);
+ //
+ // This should only be called for devices that map memory descriptors
+ // through the nv-dma library, where the memory descriptor data
+ // contains all the kernel-specific context we need for the
+ // invalidation.
+ //
+ // (These checks match those in osIovaUnmap() leading up to
+ // nv_dma_unmap_alloc()).
+ //
+ if (pIovaMapping == NULL ||
+ pIovaMapping->pOsData == NULL ||
+ memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED) ||
+ memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ nv_dma_cache_invalidate(nv->dma_dev, pIovaMapping->pOsData);
+ }
+ else
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+ }
+ else
+ {
+ return os_flush_user_cache();
+ }
+
+ return NV_OK;
+}
+
+static NV_STATUS
+_initializeExportObjectFd
+(
+ nv_file_private_t *nvfp,
+ NvHandle hClient,
+ NvHandle hDevice,
+ NvU16 maxObjects,
+ NvU8 *metadata
+)
+{
+ NV_STATUS status;
+ RsResourceRef *pResourceRef;
+ Device *pDevice;
+ NvU32 deviceInstance = NV_MAX_DEVICES;
+
+ if (nvfp->handles != NULL)
+ {
+ return NV_ERR_STATE_IN_USE;
+ }
+
+ if (hDevice != 0)
+ {
+ status = serverutilGetResourceRef(hClient, hDevice, &pResourceRef);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ pDevice = dynamicCast(pResourceRef->pResource, Device);
+ if (pDevice == NULL)
+ {
+ return NV_ERR_INVALID_PARAMETER;
+ }
+
+ deviceInstance = pDevice->deviceInst;
+ }
+
+ NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void **)&nvfp->handles,
+ sizeof(nvfp->handles[0]) * maxObjects));
+
+ os_mem_set(nvfp->handles, 0,
+ sizeof(nvfp->handles[0]) * maxObjects);
+
+ nvfp->maxHandles = maxObjects;
+ nvfp->deviceInstance = deviceInstance;
+
+ if (metadata != NULL)
+ {
+ os_mem_copy(nvfp->metadata, metadata, sizeof(nvfp->metadata));
+ }
+
+ return NV_OK;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixExportObjectToFd_IMPL
+(
+ RmClientResource *pRmCliRes,
+ NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS *pParams
+)
+{
+ NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+ RmObjExportHandle hExportHandle = 0;
+ nv_file_private_t *nvfp = NULL;
+ void *priv = NULL;
+ NV_STATUS status = NV_OK;
+
+ /*
+ * This flag is intended to be implemented entirely in the rmapi library in
+ * userspace, we should never encounter it here.
+ */
+ if (FLD_TEST_DRF(0000_CTRL, _OS_UNIX_EXPORT_OBJECT_TO_FD_FLAGS,
+ _EMPTY_FD, _TRUE, pParams->flags))
+ {
+ return NV_ERR_INVALID_PARAMETER;
+ }
+
+ if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM ||
+ pParams->fd == -1)
+ {
+ return NV_ERR_INVALID_PARAMETER;
+ }
+
+ status = RmExportObject(hClient,
+ pParams->object.data.rmObject.hObject,
+ &hExportHandle, NULL);
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+ NV_ASSERT(hExportHandle != 0);
+
+ nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+ if (nvfp == NULL)
+ {
+ NV_ASSERT(priv == NULL);
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ status = _initializeExportObjectFd(nvfp, hClient,
+ pParams->object.data.rmObject.hDevice,
+ 1, NULL);
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+
+ nvfp->handles[0] = hExportHandle;
+
+done:
+
+ if (status != NV_OK && hExportHandle != 0)
+ {
+ RmFreeObjExportHandle(hExportHandle);
+ }
+
+ if (priv != NULL)
+ {
+ nv_put_file_private(priv);
+ }
+
+ return status;
+}
+
+// This control call has been deprecated. It will be deleted soon.
+NV_STATUS
+cliresCtrlCmdOsUnixCreateExportObjectFd_IMPL
+(
+ RmClientResource *pRmCliRes,
+ NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS *pParams
+)
+{
+ NV_STATUS status;
+ NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+ nv_file_private_t *nvfp = NULL;
+ void *priv = NULL;
+
+ ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata));
+
+ if (pParams->maxObjects == 0)
+ {
+ return NV_ERR_INVALID_PARAMETER;
+ }
+
+ nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+ if (nvfp == NULL)
+ {
+ NV_ASSERT(priv == NULL);
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice,
+ pParams->maxObjects, pParams->metadata);
+
+done:
+ if (priv != NULL)
+ {
+ nv_put_file_private(priv);
+ }
+
+ return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixExportObjectsToFd_IMPL
+(
+ RmClientResource *pRmCliRes,
+ NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS *pParams
+)
+{
+ NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+ RmObjExportHandle *pExportHandle;
+ nv_file_private_t *nvfp = NULL;
+ void *priv = NULL;
+ NV_STATUS status = NV_OK;
+ NvU32 i;
+ NvU32 deviceInstance;
+ NvU32 result;
+ NvHandle *exportHandles = NULL;
+ NvBool bFdSetup = NV_FALSE;
+
+ nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+ if (nvfp == NULL)
+ {
+ NV_ASSERT(priv == NULL);
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata));
+
+ /* Setup export FD if not done */
+ if (nvfp->handles == NULL)
+ {
+ if (pParams->maxObjects == 0)
+ {
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice,
+ pParams->maxObjects,
+ pParams->metadata);
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+
+ bFdSetup = NV_TRUE;
+ }
+
+ if ((nvfp->handles == NULL) ||
+ (pParams->numObjects >
+ NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS))
+ {
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) ||
+ (result > nvfp->maxHandles))
+ {
+ status = NV_ERR_OUT_OF_RANGE;
+ goto done;
+ }
+
+ status = os_alloc_mem((void **)&exportHandles,
+ sizeof(exportHandles[0]) *
+ pParams->numObjects);
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+
+ for (i = 0; i < pParams->numObjects; i++)
+ {
+ exportHandles[i] = 0;
+
+ if (pParams->objects[i] == 0)
+ {
+ continue;
+ }
+
+ status = RmExportObject(hClient,
+ pParams->objects[i],
+ &exportHandles[i],
+ &deviceInstance);
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+
+ NV_ASSERT(exportHandles[i] != 0);
+
+ if (deviceInstance != nvfp->deviceInstance)
+ {
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+ }
+
+ for (i = 0; i < pParams->numObjects; i++)
+ {
+ pExportHandle = &nvfp->handles[i + pParams->index];
+
+ // If the handle already exists in this position, free it
+ if (*pExportHandle != 0)
+ {
+ RmFreeObjExportHandle(*pExportHandle);
+ *pExportHandle = 0;
+ }
+
+ *pExportHandle = exportHandles[i];
+ }
+
+done:
+
+ if ((status != NV_OK) && (exportHandles != NULL))
+ {
+ for (i = 0; i < pParams->numObjects; i++)
+ {
+ if (exportHandles[i] != 0)
+ {
+ RmFreeObjExportHandle(exportHandles[i]);
+ }
+ }
+ }
+
+ if (exportHandles != NULL)
+ {
+ os_free_mem(exportHandles);
+ }
+
+ if ((status != NV_OK) && bFdSetup)
+ {
+ os_free_mem(nvfp->handles);
+ nvfp->handles = NULL;
+ nvfp->maxHandles = 0;
+ }
+
+ if (priv != NULL)
+ {
+ nv_put_file_private(priv);
+ }
+
+ return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixImportObjectFromFd_IMPL
+(
+ RmClientResource *pRmCliRes,
+ NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS *pParams
+)
+{
+ NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+ nv_file_private_t *nvfp = NULL;
+ void *priv = NULL;
+ NV_STATUS status = NV_OK;
+
+ if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM ||
+ pParams->fd == -1)
+ {
+ return NV_ERR_INVALID_PARAMETER;
+ }
+
+ nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+ if (nvfp == NULL)
+ {
+ NV_ASSERT(priv == NULL);
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ if ((nvfp->handles == NULL) || (nvfp->handles[0] == 0) ||
+ (nvfp->maxHandles > 1))
+ {
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ status = RmImportObject(hClient,
+ pParams->object.data.rmObject.hParent,
+ &pParams->object.data.rmObject.hObject,
+ nvfp->handles[0], NULL);
+
+done:
+ if (priv != NULL)
+ {
+ nv_put_file_private(priv);
+ }
+
+ return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixImportObjectsFromFd_IMPL
+(
+ RmClientResource *pRmCliRes,
+ NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS *pParams
+)
+{
+ NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+ nv_file_private_t *nvfp = NULL;
+ void *priv = NULL;
+ NV_STATUS status = NV_OK;
+ NvU32 i = 0;
+ RmObjExportHandle hImportHandle;
+ NvU32 result;
+ RM_API *pRmApi;
+
+ nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+ if (nvfp == NULL)
+ {
+ NV_ASSERT(priv == NULL);
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ if ((nvfp->handles == NULL) ||
+ (pParams->numObjects >
+ NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS))
+ {
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) ||
+ (result > nvfp->maxHandles))
+ {
+ status = NV_ERR_OUT_OF_RANGE;
+ goto done;
+ }
+
+ for (i = 0; i < pParams->numObjects; i++)
+ {
+ hImportHandle = nvfp->handles[i + pParams->index];
+
+ /* Nothing to import, just continue */
+ if (hImportHandle == 0)
+ {
+ pParams->objectTypes[i] = \
+ NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_NONE;
+ continue;
+ }
+
+ status = RmImportObject(hClient,
+ pParams->hParent,
+ &pParams->objects[i],
+ hImportHandle,
+ &pParams->objectTypes[i]);
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s: Unable to import handle (%x, %x, %x)\n",
+ __FUNCTION__, pParams->hParent, pParams->objects[i], hImportHandle);
+ goto done;
+ }
+ }
+
+done:
+
+ if (status != NV_OK)
+ {
+ pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+ while (i > 0)
+ {
+ i--;
+
+ if (pParams->objects[i] != 0)
+ {
+ pRmApi->Free(pRmApi, hClient, pParams->objects[i]);
+ }
+ }
+ }
+
+ if (priv != NULL)
+ {
+ nv_put_file_private(priv);
+ }
+
+ return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixGetExportObjectInfo_IMPL
+(
+ RmClientResource *pRmCliRes,
+ NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS *pParams
+)
+{
+ nv_file_private_t *nvfp = NULL;
+ void *priv = NULL;
+ NV_STATUS status = NV_OK;
+
+ if (pParams->fd < 0)
+ {
+ return NV_ERR_INVALID_PARAMETER;
+ }
+
+ nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+ if (nvfp == NULL)
+ {
+ NV_ASSERT(priv == NULL);
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ if (nvfp->handles == NULL)
+ {
+ status = NV_ERR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ pParams->maxObjects = nvfp->maxHandles;
+ pParams->deviceInstance = nvfp->deviceInstance;
+ os_mem_copy(pParams->metadata, nvfp->metadata, sizeof(nvfp->metadata));
+
+done:
+ if (priv != NULL)
+ {
+ nv_put_file_private(priv);
+ }
+
+ return status;
+}
+
+/*!
+ * osAcpiDsm
+ *
+ * @brief Handles os specific _DSM method function calls.
+ *
+ * Input parameters:
+ * @param[in] pGpu : OBJGPU pointer
+ * @param[in] acpiDsmFunction : ACPI DSM function
+ * @param[in] acpiDsmSubFunction : ACPI DSM subfunction
+ * @param[in/out] pInOut : in/out buffer, caller should make sure the buffer is large enough.
+ * @param[in] pSize : when input, size of data that the caller wants to read, in bytes.
+ * when output, size of valid data in pInOuta in bytes.
+ */
+NV_STATUS osCallACPI_DSM
+(
+ OBJGPU *pGpu,
+ ACPI_DSM_FUNCTION acpiDsmFunction,
+ NvU32 acpiDsmSubFunction,
+ NvU32 *pInOut,
+ NvU16 *pSize
+)
+{
+ NV_STATUS status;
+ NvU8 *pAcpiDsmGuid = NULL;
+ NvU32 acpiDsmRev;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ NvU16 acpiDsmInArgSize = 4;
+ NvBool acpiNvpcfDsmFunction = NV_FALSE;
+
+ // do any handling/remapping of guid needed.
+ status = checkDsmCall(pGpu,
+ (ACPI_DSM_FUNCTION *) &acpiDsmFunction,
+ &acpiDsmSubFunction,
+ pInOut,
+ pSize);
+
+ // return if subfunction is not supported or we're returning cache data
+ if (status != NV_WARN_MORE_PROCESSING_REQUIRED)
+ {
+ return status;
+ }
+
+ switch ((NvU32) acpiDsmFunction)
+ {
+ case ACPI_DSM_FUNCTION_NBSI:
+ pAcpiDsmGuid = (NvU8 *) &NBSI_DSM_GUID;
+ acpiDsmRev = NBSI_REVISION_ID;
+ break;
+ case ACPI_DSM_FUNCTION_NVHG:
+ pAcpiDsmGuid = (NvU8 *) &NVHG_DSM_GUID;
+ acpiDsmRev = NVHG_REVISION_ID;
+ break;
+ case ACPI_DSM_FUNCTION_MXM:
+ pAcpiDsmGuid = (NvU8 *) &DSM_MXM_GUID;
+ acpiDsmRev = ACPI_MXM_REVISION_ID;
+ break;
+ case ACPI_DSM_FUNCTION_NBCI:
+ pAcpiDsmGuid = (NvU8 *) &NBCI_DSM_GUID;
+ acpiDsmRev = NBCI_REVISION_ID;
+ break;
+ case ACPI_DSM_FUNCTION_NVOP:
+ pAcpiDsmGuid = (NvU8 *) &NVOP_DSM_GUID;
+ acpiDsmRev = NVOP_REVISION_ID;
+ break;
+ case ACPI_DSM_FUNCTION_PCFG:
+ pAcpiDsmGuid = (NvU8 *) &PCFG_DSM_GUID;
+ acpiDsmRev = PCFG_REVISION_ID;
+ break;
+ case ACPI_DSM_FUNCTION_PEX:
+ pAcpiDsmGuid = (NvU8 *) &PEX_DSM_GUID;
+ acpiDsmRev = PEX_REVISION_ID;
+ if (acpiDsmSubFunction == PEX_FUNC_SETLTRLATENCY)
+ {
+ acpiDsmInArgSize = (3 + *pSize);
+ }
+ break;
+ case (ACPI_DSM_FUNCTION_JT):
+ pAcpiDsmGuid = (NvU8 *) &JT_DSM_GUID;
+ acpiDsmRev = JT_REVISION_ID;
+ break;
+ case ACPI_DSM_FUNCTION_NVPCF:
+ {
+ pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID;
+ acpiDsmRev = NVPCF_ACPI_DSM_REVISION_ID;
+ acpiDsmInArgSize = (*pSize);
+ acpiNvpcfDsmFunction = NV_TRUE;
+ break;
+ }
+ case ACPI_DSM_FUNCTION_NVPCF_2X:
+ pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID;
+ acpiDsmRev = NVPCF_2X_ACPI_DSM_REVISION_ID;
+ acpiDsmInArgSize = (*pSize);
+ if (!nv->nvpcf_dsm_in_gpu_scope)
+ {
+ acpiNvpcfDsmFunction = NV_TRUE;
+ }
+ break;
+
+ default:
+ return NV_ERR_NOT_SUPPORTED;
+ break;
+ }
+
+ status = nv_acpi_dsm_method(nv,
+ pAcpiDsmGuid,
+ acpiDsmRev,
+ acpiNvpcfDsmFunction,
+ acpiDsmSubFunction,
+ pInOut,
+ acpiDsmInArgSize,
+ NULL,
+ pInOut,
+ pSize);
+
+ if (status == NV_OK)
+ {
+ if (acpiDsmSubFunction == NV_ACPI_ALL_FUNC_SUPPORT)
+ {
+ // if handling get supported functions list... cache it for later calls
+ cacheDsmSupportedFunction(pGpu, acpiDsmFunction, acpiDsmSubFunction, pInOut, *pSize);
+ }
+ }
+ else if (nvp->b_mobile_config_enabled)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "osCallACPI_DSM: Error during 0x%x DSM subfunction 0x%x! status=0x%x\n",
+ acpiDsmFunction, acpiDsmSubFunction, status);
+ }
+
+ return status;
+}
+
+NV_STATUS osCallACPI_DOD
+(
+ OBJGPU *pGpu,
+ NvU32 *pOut,
+ NvU32 *pSize
+)
+{
+ NV_STATUS rmStatus;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ if ((pOut == NULL) || (pSize == NULL))
+ {
+ return NV_ERR_INVALID_POINTER;
+ }
+
+ rmStatus = nv_acpi_dod_method(nv, pOut, pSize);
+
+ return rmStatus;
+}
+
+//
+// osAcpiDdc
+//
+// Handles os specific _DDC method function calls. _DDC is to get EDID from SBIOS.
+//
+NV_STATUS osCallACPI_DDC
+(
+ OBJGPU *pGpu,
+ NvU32 ulAcpiId,
+ NvU8 *pOutData,
+ NvU32 *pOutSize,
+ NvBool bReadMultiBlock
+)
+{
+ NV_STATUS rmStatus;
+
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ if ((pOutData == NULL) || (pOutSize == NULL))
+ {
+ return NV_ERR_INVALID_POINTER;
+ }
+
+ portMemSet(pOutData, 0, *pOutSize);
+
+ rmStatus = nv_acpi_ddc_method(nv, pOutData, pOutSize, bReadMultiBlock);
+
+ return rmStatus;
+}
+
+// osCallACPI_NVHG_ROM
+// Making ACPI Call into SBIOS with ROM method to get display device's ROM data.
+//
+NV_STATUS osCallACPI_NVHG_ROM
+(
+ OBJGPU *pGpu,
+ NvU32 *pInData,
+ NvU32 *pOutData
+)
+{
+ NV_STATUS rmStatus;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ if ((pOutData == NULL) || (pInData == NULL))
+ {
+ return NV_ERR_INVALID_POINTER;
+ }
+
+ if (pInData[1] > ROM_METHOD_MAX_RETURN_BUFFER_SIZE)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ rmStatus = nv_acpi_rom_method(nv, pInData, pOutData);
+
+ return rmStatus;
+}
+
+void osInitSystemStaticConfig(SYS_STATIC_CONFIG *pConfig)
+{
+ pConfig->bIsNotebook = rm_is_system_notebook();
+ pConfig->osType = nv_get_os_type();
+ pConfig->osSevStatus = os_sev_status;
+ pConfig->bOsSevEnabled = os_sev_enabled;
+}
+
+NvU32 osApiLockAcquireConfigureFlags(NvU32 flags)
+{
+ return flags;
+}
+
+NV_STATUS osGpuLocksQueueRelease(OBJGPU *pGpu, NvU32 dpcGpuLocksRelease)
+{
+ return NV_SEMA_RELEASE_FAILED;
+}
+
+void osSyncWithRmDestroy()
+{
+}
+
+void osSyncWithGpuDestroy(NvBool bEntry)
+{
+}
+
+void osModifyGpuSwStatePersistence
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvBool bEnable
+)
+{
+ if (bEnable)
+ {
+ pOsGpuInfo->flags |= NV_FLAG_PERSISTENT_SW_STATE;
+ }
+ else
+ {
+ pOsGpuInfo->flags &= ~NV_FLAG_PERSISTENT_SW_STATE;
+ }
+}
+
+NV_STATUS
+osSystemGetBatteryDrain(NvS32 *pChargeRate)
+{
+ NV_PRINTF(LEVEL_WARNING, "%s: Platform not supported!\n", __FUNCTION__);
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osPexRecoveryCallback
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ OS_PEX_RECOVERY_STATUS Status
+)
+{
+ NV_ASSERT_FAILED("Not supported");
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+//
+//osCallACPI_MXDS
+//
+//Handles OS specific MXDS function call.
+//
+NV_STATUS osCallACPI_MXDS
+(
+ OBJGPU *pGpu,
+ NvU32 acpiId,
+ NvU32 *pInOut
+)
+{
+ NV_STATUS rmStatus;
+
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ if (pInOut == NULL)
+ {
+ return NV_ERR_INVALID_POINTER;
+ }
+
+ rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDS");
+
+ return rmStatus;
+}
+
+//
+//osCallACPI_MXDM
+//
+//Handles OS specific MXDM function call.
+//
+NV_STATUS osCallACPI_MXDM
+(
+ OBJGPU *pGpu,
+ NvU32 acpiId,
+ NvU32 *pInOut
+)
+{
+ NV_STATUS rmStatus;
+
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ if (pInOut == NULL)
+ {
+ return NV_ERR_INVALID_POINTER;
+ }
+
+ rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDM");
+
+ return rmStatus;
+}
+
+#include "lib/protobuf/prb.h"
+#include "lib/protobuf/prb_util.h"
+#include "g_nvdebug_pb.h"
+
+NV_STATUS osGetVersionDump(void * pVoid)
+{
+ PRB_ENCODER * pPrbEnc = (PRB_ENCODER *)pVoid;
+ NV_STATUS rmStatus;
+ os_version_info * pOsVersionInfo = NULL;
+ const char NV_UNKNOWN_BUILD_VERSION[] = "Unknown build version";
+ const char NV_UNKNOWN_BUILD_DATE[] = "Unknown build date";
+
+ NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void**)&pOsVersionInfo,
+ sizeof(os_version_info)));
+ portMemSet(pOsVersionInfo, 0, sizeof(os_version_info));
+
+ prbEncAddUInt32(pPrbEnc,
+ NVDEBUG_SYSTEMINFO_OSINFO_FAMILY,
+ NVDEBUG_OS_UNIX);
+
+ rmStatus = os_get_version_info(pOsVersionInfo);
+ if (rmStatus != NV_OK)
+ {
+ goto cleanup;
+ }
+
+ prbEncAddUInt32(pPrbEnc,
+ NVDEBUG_SYSTEMINFO_OSINFO_OSMAJORVERSION,
+ pOsVersionInfo->os_major_version);
+
+ prbEncAddUInt32(pPrbEnc,
+ NVDEBUG_SYSTEMINFO_OSINFO_OSMINORVERSION,
+ pOsVersionInfo->os_minor_version);
+
+ prbEncAddUInt32(pPrbEnc,
+ NVDEBUG_SYSTEMINFO_OSINFO_OSBLDNUM,
+ pOsVersionInfo->os_build_number);
+
+ if (NULL == pOsVersionInfo->os_build_version_str)
+ {
+ pOsVersionInfo->os_build_version_str = NV_UNKNOWN_BUILD_VERSION;
+ }
+
+ prbEncAddString(pPrbEnc,
+ NVDEBUG_SYSTEMINFO_OSINFO_BUILDVERSION,
+ pOsVersionInfo->os_build_version_str);
+
+ if (NULL == pOsVersionInfo->os_build_date_plus_str)
+ {
+ pOsVersionInfo->os_build_date_plus_str = NV_UNKNOWN_BUILD_DATE;
+ }
+
+ prbEncAddString(pPrbEnc,
+ NVDEBUG_SYSTEMINFO_OSINFO_BUILDDATEPLUS,
+ pOsVersionInfo->os_build_date_plus_str);
+
+cleanup:
+ os_free_mem(pOsVersionInfo);
+ return rmStatus;
+}
+
+NV_STATUS osGetVersion(NvU32 *majorVer, NvU32 *minorVer, NvU32 *buildNum, NvU16 *unusedPatchVersion, NvU16 *unusedProductType)
+{
+ os_version_info osVersionInfo;
+ NV_STATUS rmStatus;
+
+ portMemSet(&osVersionInfo, 0, sizeof(osVersionInfo));
+
+ rmStatus = os_get_version_info(&osVersionInfo);
+ if (rmStatus == NV_OK)
+ {
+ if (majorVer)
+ *majorVer = osVersionInfo.os_major_version;
+ if (minorVer)
+ *minorVer = osVersionInfo.os_minor_version;
+ if (buildNum)
+ *buildNum = osVersionInfo.os_build_number;
+ }
+
+ return rmStatus;
+}
+
+NV_STATUS
+osGetSystemCpuLogicalCoreCounts
+(
+ NvU32 *pCpuCoreCount
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osGetSystemCpuC0AndAPerfCounters
+(
+ NvU32 coreIndex,
+ POS_CPU_CORE_PERF_COUNTERS pCpuPerfData
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+void
+osEnableCpuPerformanceCounters
+(
+ OBJOS *pOS
+)
+{
+ NV_ASSERT_FAILED("Not supported");
+ return;
+}
+
+NV_STATUS
+osCpuDpcObjInit
+(
+ void **ppCpuDpcObj,
+ OBJGPU *pGpu,
+ NvU32 coreCount
+)
+{
+ NV_ASSERT_FAILED("Not supported");
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+void
+osCpuDpcObjQueue
+(
+ void **ppCpuDpcObj,
+ NvU32 coreCount,
+ POS_CPU_CORE_PERF_COUNTERS pCpuPerfData
+)
+{
+ NV_ASSERT_FAILED("Not supported");
+}
+
+void
+osCpuDpcObjFree
+(
+ void **ppCpuDpcObj
+)
+{
+ NV_ASSERT_FAILED("Not supported");
+}
+
+NV_STATUS
+osGetCarveoutInfo
+(
+ NvU64 *pAddr,
+ NvU64 *pSize
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osGetVPRInfo
+(
+ NvU64 *pAddr,
+ NvU64 *pSize
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osAllocInVPR
+(
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osGetGenCarveout
+(
+ NvU64 *pAddr,
+ NvU64 *pSize,
+ NvU32 id,
+ NvU64 align
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osI2CClosePorts
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 numPorts
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osI2CTransfer
+(
+ OBJGPU *pGpu,
+ NvU32 Port,
+ NvU8 Address,
+ nv_i2c_msg_t *nv_i2c_msgs,
+ NvU32 count
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osTegraI2CGetBusState
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 port,
+ NvS32 *scl,
+ NvS32 *sda
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osReadI2CBufferDirect
+(
+ OBJGPU *pGpu,
+ NvU32 Port,
+ NvU8 Address,
+ void *pOutputBuffer,
+ NvU32 OutputSize,
+ void *pInputBuffer,
+ NvU32 InputSize
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osWriteI2CBufferDirect
+(
+ OBJGPU *pGpu,
+ NvU32 Port,
+ NvU8 Address,
+ void *pOutputBuffer0,
+ NvU32 OutputSize0,
+ void *pOutputBuffer1,
+ NvU32 OutputSize1
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osGC6PowerControl
+(
+ OBJGPU *pGpu,
+ NvU32 cmd,
+ NvU32 *pOut
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+NvBool osTestPcieExtendedConfigAccess(void *handle, NvU32 offset)
+{
+ OBJGPU *pGpu;
+ KernelBif *pKernelBif;
+ NvU32 nvXveId = 0;
+ NvU32 nvXveVccapHdr = 0;
+ NvU32 pciStart = 0;
+ NvU32 pcieStart = 0;
+
+ static NvBool retryAllowed = NV_TRUE;
+ static NvBool configAccess = NV_FALSE;
+
+ //
+ // Return early for offset within PCI space
+ // and does not requires extended config space access
+ //
+ if (offset < 0x100)
+ {
+ return NV_TRUE;
+ }
+
+ if (!retryAllowed)
+ {
+ return configAccess;
+ }
+
+ pGpu = gpumgrGetSomeGpu();
+ if (pGpu == NULL)
+ {
+ return configAccess;
+ }
+
+ retryAllowed = NV_FALSE;
+
+ pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+ if (pKernelBif == NULL || kbifGetBusIntfType_HAL(pKernelBif) !=
+ NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS)
+ {
+ return configAccess;
+ }
+
+ // Now verify PCI and PCIe config start registers.
+ kbifGetPcieConfigAccessTestRegisters_HAL(pGpu, pKernelBif, &pciStart, &pcieStart);
+ os_pci_read_dword(handle, pciStart, &nvXveId);
+ os_pci_read_dword(handle, pcieStart, &nvXveVccapHdr);
+
+ if (NV_OK == kbifVerifyPcieConfigAccessTestRegisters_HAL(pGpu,
+ pKernelBif,
+ nvXveId,
+ nvXveVccapHdr))
+ {
+ configAccess = NV_TRUE;
+ }
+
+ return configAccess;
+}
+
+/*!
+ * @brief Map memory into an IOVA space according to the given mapping info.
+ *
+ * @param[in] pIovaMapping IOVA mapping info
+ *
+ * @return NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osIovaMap
+(
+ PIOVAMAPPING pIovaMapping
+)
+{
+ OBJGPU *pGpu;
+ nv_state_t *nv, *peer;
+ NV_STATUS status;
+ RmPhysAddr base;
+ NvBool bIsBar0;
+ PMEMORY_DESCRIPTOR pRootMemDesc;
+ NvBool bIsFbOffset = NV_FALSE;
+ NvBool bIsIndirectPeerMapping = NV_FALSE;
+ NvBool bIsContig;
+ NV_ADDRESS_SPACE addressSpace;
+ NvU32 osPageCount;
+
+ if (pIovaMapping == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId);
+ if (pGpu == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ pRootMemDesc = memdescGetRootMemDesc(pIovaMapping->pPhysMemDesc, NULL);
+ addressSpace = memdescGetAddressSpace(pIovaMapping->pPhysMemDesc);
+ if (gpumgrCheckIndirectPeer(pGpu, pRootMemDesc->pGpu) &&
+ (addressSpace == ADDR_FBMEM))
+ {
+ bIsIndirectPeerMapping = NV_TRUE;
+ }
+
+ if ((addressSpace != ADDR_SYSMEM) && !bIsIndirectPeerMapping)
+ {
+ NV_PRINTF(LEVEL_INFO,
+ "%s passed memory descriptor in an unsupported address space (%s)\n",
+ __FUNCTION__,
+ memdescGetApertureString(memdescGetAddressSpace(pIovaMapping->pPhysMemDesc)));
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ //
+ // For guest-allocated memory, we don't actually want to do any remapping,
+ // since the physical address is already the DMA address to be used by the
+ // GPU.
+ //
+ if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
+ {
+ return NV_OK;
+ }
+
+ nv = NV_GET_NV_STATE(pGpu);
+
+ //
+ // Intercept peer IO type memory. These are contiguous allocations, so no
+ // need to adjust pages.
+ //
+ if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
+ {
+ NV_ASSERT(memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU));
+
+ status = nv_dma_map_mmio(nv->dma_dev,
+ NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount),
+ &pIovaMapping->iovaArray[0]);
+
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: failed to map peer IO mem (status = 0x%x)\n",
+ __FUNCTION__, status);
+ }
+
+ return status;
+ }
+
+ //
+ // We need to check against the "root" GPU, e.g., the GPU that owns this
+ // allocation. If we're trying to map one of its BARs for a peer, we need
+ // to handle it differently because it wouldn't have gone through our system
+ // memory page allocation paths, obviously, and wouldn't have alloc private
+ // data associated with it.
+ //
+ peer = NV_GET_NV_STATE(pRootMemDesc->pGpu);
+ bIsContig = memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU);
+ if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig)
+ {
+ RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0],
+ pIovaMapping->pPhysMemDesc->PageCount);
+ }
+
+ base = memdescGetPhysAddr(pIovaMapping->pPhysMemDesc, AT_CPU, 0);
+ bIsBar0 = IS_REG_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size);
+
+ bIsFbOffset = IS_FB_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size);
+
+ //
+ // For indirect peers bIsFbOffset should be NV_TRUE
+ // TODO:IS_FB_OFFSET macro is currently broken for P9 systems
+ // Bug 2010857 tracks fixing this
+ //
+#if defined(NVCPU_PPC64LE)
+ KernelMemorySystem *pRootKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pRootMemDesc->pGpu);
+ if (bIsIndirectPeerMapping)
+ {
+ NvU64 atsBase = base + pRootKernelMemorySystem->coherentCpuFbBase;
+ if ((atsBase >= pRootKernelMemorySystem->coherentCpuFbBase) &&
+ (atsBase + pIovaMapping->pPhysMemDesc->Size <=
+ pRootKernelMemorySystem->coherentCpuFbEnd))
+ {
+ bIsFbOffset = NV_TRUE;
+ }
+ else
+ {
+ NV_ASSERT_OR_RETURN(0, NV_ERR_INVALID_STATE);
+ }
+ }
+#endif
+
+ void *pPriv = memdescGetMemData(pIovaMapping->pPhysMemDesc);
+ osPageCount = NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount);
+
+ if (!bIsBar0 && !bIsFbOffset)
+ {
+ if (pPriv == NULL)
+ {
+ return NV_ERR_INVALID_STATE;
+ }
+ }
+ else if(bIsIndirectPeerMapping)
+ {
+ NV_ASSERT(!bIsBar0 && bIsFbOffset);
+ //
+ // TODO: Align onlined GPU memory allocation paths with system memory allocation
+ // That way pMemDesc->pMemData is setup correctly when we try to create mapping
+ // to onlined memory of indirect peer. After that we can also get rid of some
+ // extra code in nv_dma_map_alloc. See bug 190324 for details
+ //
+
+ status = memdescGetNvLinkGpa(pRootMemDesc->pGpu, (bIsContig ? 1 : osPageCount),
+ &pIovaMapping->iovaArray[0]);
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s Failed to get SPA\n", __FUNCTION__);
+ return status;
+ }
+ }
+
+ if (!bIsBar0 && (!bIsFbOffset || bIsIndirectPeerMapping))
+ {
+ status = nv_dma_map_alloc(
+ osGetDmaDeviceForMemDesc(nv, pIovaMapping->pPhysMemDesc),
+ osPageCount,
+ &pIovaMapping->iovaArray[0],
+ bIsContig, &pPriv);
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: failed to map allocation (status = 0x%x)\n",
+ __FUNCTION__, status);
+ return status;
+ }
+
+ pIovaMapping->pOsData = pPriv;
+ }
+ else if (peer != nv)
+ {
+ status = nv_dma_map_peer(nv->dma_dev, peer->dma_dev, bIsBar0 ? 0 : 1,
+ osPageCount, &pIovaMapping->iovaArray[0]);
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_INFO,
+ "%s: failed to map peer (base = 0x%llx, status = 0x%x)\n",
+ __FUNCTION__, base, status);
+ return status;
+ }
+
+ //
+ // pOsData must be NULL to distinguish a peer DMA mapping from a
+ // system memory mapping in osIovaUnmap(), so make sure to set it
+ // accordingly here.
+ //
+ pIovaMapping->pOsData = NULL;
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_INFO, "cannot map a GPU's BAR to itself\n");
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ //
+ // If the OS layer doesn't think in RM page size, we need to inflate the
+ // PTE array into RM pages.
+ //
+ if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig)
+ {
+ RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0],
+ pIovaMapping->pPhysMemDesc->PageCount);
+ }
+
+ return NV_OK;
+}
+
+/*!
+ * @brief Unmap memory from an IOVA space according to the given mapping info.
+ *
+ * This mapping info must have been previously mapped by osIovaMap().
+ *
+ * @param[in] pIovaMapping IOVA mapping info
+ *
+ */
+void
+osIovaUnmap
+(
+ PIOVAMAPPING pIovaMapping
+)
+{
+ OBJGPU *pGpu;
+ nv_state_t *nv;
+ void *pPriv;
+ NV_STATUS status;
+
+ if (pIovaMapping == NULL)
+ {
+ return;
+ }
+
+ pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId);
+ if (pGpu == NULL)
+ {
+ return;
+ }
+
+ //
+ // For guest-allocated memory, we never actually remapped the memory, so we
+ // shouldn't try to unmap it here.
+ //
+ if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
+ {
+ return;
+ }
+
+ nv = NV_GET_NV_STATE(pGpu);
+
+ if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
+ {
+ nv_dma_unmap_mmio(nv->dma_dev,
+ NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount),
+ pIovaMapping->iovaArray[0]);
+
+ return;
+ }
+
+ //
+ // TODO: Formalize the interface with the OS layers so we can use a common
+ // definition of OS_IOVA_MAPPING_DATA.
+ //
+ pPriv = (void *)pIovaMapping->pOsData;
+
+ if (NV_RM_PAGE_SIZE < os_page_size &&
+ !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU))
+ {
+ RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0],
+ pIovaMapping->pPhysMemDesc->PageCount);
+ }
+
+ if (pPriv != NULL)
+ {
+ status = nv_dma_unmap_alloc(nv->dma_dev,
+ NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount),
+ &pIovaMapping->iovaArray[0], &pPriv);
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: failed to unmap allocation (status = 0x%x)\n",
+ __FUNCTION__, status);
+ }
+ }
+ else
+ {
+ nv_dma_unmap_peer(nv->dma_dev,
+ NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount),
+ pIovaMapping->iovaArray[0]);
+ }
+
+ //
+ // If the OS layer doesn't think in RM page size, we need to fluff out the
+ // PTE array into RM pages.
+ //
+ if (NV_RM_PAGE_SIZE < os_page_size &&
+ !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU))
+ {
+ RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0],
+ pIovaMapping->pPhysMemDesc->PageCount);
+ }
+
+ pIovaMapping->pOsData = NULL;
+}
+
+/*!
+ * @brief Set the GPU Rail Voltage in Tegra SoC. Currently not supported
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[in] reqVoltageuV Rail Voltage requested in uV
+ * @param[out] pSetVoltageuV Rail Voltage set in uV
+ *
+ * @return NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osSetGpuRailVoltage
+(
+ OBJGPU *pGpu,
+ NvU32 reqVoltageuV,
+ NvU32 *pSetVoltageuV
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Get the GPU Rail Voltage in Tegra SoC. Currently not supported
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] voltageuV Rail Voltage in uV
+ *
+ * @return NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osGetGpuRailVoltage
+(
+ OBJGPU *pGpu,
+ NvU32 *pVoltageuV
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Bring down system in a controlled manner on known error conditions.
+ *
+ * @bugCode[in] Error code / reason.
+ */
+void osBugCheck(NvU32 bugCode)
+{
+ if (bugCode > OS_BUG_CHECK_BUGCODE_LAST)
+ {
+ bugCode = OS_BUG_CHECK_BUGCODE_UNKNOWN;
+ }
+
+ os_bug_check(bugCode, ppOsBugCheckBugcodeStr[bugCode]);
+}
+
+/*!
+ * @brief Perform an action at assertion failure.
+ */
+void osAssertFailed(void)
+{
+ os_dump_stack();
+}
+
+/*!
+ * @brief Get the GPU Chip Info - Speedo and IDDQ values
+ *
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] pGpuSpeedoHv Pointer to GPU Speedo value at high voltage corner.
+ * @param[out] pGpuSpeedoLv Pointer to GPU Speedo value at low voltage corner.
+ * @param[out] pGpuIddq Pointer to GPU Iddq Value
+ * @param[out] pChipSkuId SKU ID for the chip
+ *
+ * @return NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osGetChipInfo
+(
+ OBJGPU *pGpu,
+ NvU32 *pGpuSpeedoHv,
+ NvU32 *pGpuSpeedoLv,
+ NvU32 *pGpuIddq,
+ NvU32 *pChipSkuId
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*
+ * @brief Get the GPU Rail Voltage Info (i.e. Min, Max and StepSize) in Tegra SoC.
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] pMinVoltageuV Minimum Voltage supported on the Rail in Micro Volts
+ * @param[out] pMaxVoltageuV Maximum Voltage supported on the Rail in Micro Volts
+ * @param[out] pStepVoltageuV Voltage Step-size supported on the Rail in Micro Volts
+ *
+ * @return NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osGetGpuRailVoltageInfo
+(
+ OBJGPU *pGpu,
+ NvU32 *pMinVoltageuV,
+ NvU32 *pMaxVoltageuV,
+ NvU32 *pStepVoltageuV
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Get the current opaque security token.
+ *
+ * For Linux the security token is the effective UID of a process and process ID
+ *
+ * Note: This function allocates memory for the token. The onus is on the calling
+ * function to free the memory associated with the token once its done with it.
+ *
+ * @return pointer to the security token.
+ */
+PSECURITY_TOKEN
+osGetSecurityToken()
+{
+ NV_STATUS rmStatus;
+ TOKEN_USER *pTokenUser;
+
+ pTokenUser = portMemAllocNonPaged(sizeof(TOKEN_USER));
+ if (pTokenUser == NULL)
+ {
+ return NULL;
+ }
+ rmStatus = os_get_euid(&pTokenUser->euid);
+ if (rmStatus != NV_OK)
+ {
+ portMemFree(pTokenUser);
+ return NULL;
+ }
+
+ pTokenUser->pid = os_get_current_process();
+
+ return (PSECURITY_TOKEN)pTokenUser;
+}
+
+PUID_TOKEN
+osGetCurrentUidToken(void)
+{
+ NV_STATUS rmStatus;
+ NvU32 *pUidToken;
+
+ pUidToken = portMemAllocNonPaged(sizeof(NvU32));
+ if (pUidToken == NULL)
+ {
+ return NULL;
+ }
+
+ rmStatus = os_get_euid(pUidToken);
+ if (rmStatus != NV_OK)
+ {
+ portMemFree(pUidToken);
+ return NULL;
+ }
+
+ return (PUID_TOKEN)pUidToken;
+}
+
+/*!
+ * @brief Interface function to validate the token for the current client
+ *
+ * This function takes two tokens as parameters, validates them and checks
+ * if either the PID or EUID from client database matches the current PID or EUID.
+ *
+ * @param[in] pClientSecurityToken security token cached in the client db
+ * @param[in] pCurrentSecurityToken security token of the current client
+ * @return NV_OK if the validation is successful
+ * NV_ERR_INVALID_CLIENT if the tokens do not match
+ * NV_ERR_INVALID_POINTER if the tokens are invalid
+ */
+NV_STATUS
+osValidateClientTokens
+(
+ PSECURITY_TOKEN pClientSecurityToken,
+ PSECURITY_TOKEN pCurrentSecurityToken
+)
+{
+ PTOKEN_USER pClientTokenUser = (PTOKEN_USER)pClientSecurityToken;
+ PTOKEN_USER pCurrentTokenUser = (PTOKEN_USER)pCurrentSecurityToken;
+
+ NV_ASSERT_OR_RETURN((pClientTokenUser != NULL), NV_ERR_INVALID_POINTER);
+ NV_ASSERT_OR_RETURN((pCurrentTokenUser != NULL), NV_ERR_INVALID_POINTER);
+
+ if ((pClientTokenUser->euid != pCurrentTokenUser->euid) &&
+ (pClientTokenUser->pid != pCurrentTokenUser->pid))
+ {
+ NV_PRINTF(LEVEL_INFO,
+ "NVRM: %s: Current security token doesn't match the one in the client database. "
+ "Current EUID: %d, PID: %d; Client DB EUID: %d, PID: %d\n",
+ __FUNCTION__, pCurrentTokenUser->euid, pCurrentTokenUser->pid,
+ pClientTokenUser->euid, pClientTokenUser->pid);
+ return NV_ERR_INVALID_CLIENT;
+ }
+
+ return NV_OK;
+}
+
+/*!
+ * @brief Interface function to compare the tokens for two client
+ *
+ * This function takes two tokens as parameters, validates them and checks
+ * if the EUIDs of each token match.
+ *
+ * @param[in] pToken1 Token to compare
+ * @param[in] pToken2 Token to compare
+ * @return NV_TRUE if the tokens match
+ * NV_FALSE if the tokens do not match
+ */
+NvBool
+osUidTokensEqual
+(
+ PUID_TOKEN pUidToken1,
+ PUID_TOKEN pUidToken2
+)
+{
+ NvU32 * pTokenUser1 = (NvU32*)pUidToken1;
+ NvU32 * pTokenUser2 = (NvU32*)pUidToken2;
+
+ NV_ASSERT_OR_RETURN((pTokenUser1 != NULL), NV_FALSE);
+ NV_ASSERT_OR_RETURN((pTokenUser2 != NULL), NV_FALSE);
+
+ if (*pTokenUser1 != *pTokenUser2)
+ {
+ return NV_FALSE;
+ }
+
+ return NV_TRUE;
+}
+
+NvBool
+osRemoveGpuSupported
+(
+ void
+)
+{
+ return os_pci_remove_supported();
+}
+
+/*
+ * @brief Get the address ranges assigned to local or peer GPUs on a system that
+ * supports hardware address translation services (ATS) over NVLink/C2C.
+ *
+ * @note
+ * - All address values are in the System Physical Address (SPA) space
+ * - Targets can either be "Local" (bIsPeer=False) or for a specified "Peer"
+ * (bIsPeer=True, peerIndex=#) GPU
+ * - Target address and mask values have a specified bit width, and represent
+ * the higher order bits above the target address granularity
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] pAddrSysPhys Pointer to hold SPA
+ * @param[out] pAddrWidth Address range width value pointer
+ * @param[out] pMask Mask value pointer
+ * @param[out] pMaskWidth Mask width value pointer
+ * @param[in] bIsPeer NV_TRUE if this is a peer, local GPU otherwise
+ * @param[in] peerIndex Peer index
+ *
+ * @return NV_OK or NV_ERR_NOT_SUPPORTED
+ *
+ * A return value of NV_ERR_NOT_SUPPORTED for the local GPU would
+ * indicate that the system does not support ATS over NVLink/C2C
+ */
+NV_STATUS
+osGetAtsTargetAddressRange
+(
+ OBJGPU *pGpu,
+ NvU64 *pAddrSysPhys,
+ NvU32 *pAddrWidth,
+ NvU32 *pMask,
+ NvU32 *pMaskWidth,
+ NvBool bIsPeer,
+ NvU32 peerIndex
+)
+{
+#if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && (defined(NVCPU_PPC64LE) || defined(NVCPU_AARCH64))
+ KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
+ KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+ nv_state_t *nv;
+ const int addrMaskWidth = 0x10;
+
+ if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP))
+ return NV_ERR_INVALID_ARGUMENT;
+
+ nv = NV_GET_NV_STATE(pGpu);
+
+ //
+ // TODO : Bug 1848958 restricts peer device tree parsing. Return early if
+ // peer values are requested. This should be fixed by passing correct pGpu
+ // pointer of the peer GPU retrieved using peerIds.
+ //
+ if (bIsPeer)
+ {
+ const int addrWidth = 0x10;
+
+ *pAddrSysPhys = 0;
+ *pAddrWidth = addrWidth;
+ *pMask = 0;
+ *pMaskWidth = addrMaskWidth;
+ return NV_OK;
+ }
+ else
+ {
+ NV_STATUS status = nv_get_device_memory_config(nv, pAddrSysPhys, NULL,
+ pAddrWidth, NULL);
+ if (status == NV_OK)
+ {
+ *pMask = NVBIT(*pAddrWidth) - 1U;
+ *pMaskWidth = addrMaskWidth;
+ }
+ return status;
+ }
+
+ return NV_OK;
+#endif
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*
+ * @brief Get the physical address in CPU address map and NUMA node id
+ * of the GPU memory.
+ *
+ * @note
+ * - The physical address is System Physical Address (SPA) in baremetal/host
+ * and Intermediate Physical Address(IPA) or Guest Physical Address(GPA)
+ * inside a VM.
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] pAddrPhys Pointer to hold the physical address of FB in
+ * CPU address map
+ * @param[out] pNodeId NUMA nodeID of respective GPU memory
+ *
+ * @return NV_OK or NV_ERR_NOT_SUPPORTED
+ *
+ */
+NV_STATUS
+osGetFbNumaInfo
+(
+ OBJGPU *pGpu,
+ NvU64 *pAddrPhys,
+ NvS32 *pNodeId
+)
+{
+#if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && (defined(NVCPU_PPC64LE) || defined(NVCPU_AARCH64))
+ KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
+ KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+ nv_state_t *nv;
+
+ *pNodeId = NV0000_CTRL_NO_NUMA_NODE;
+
+ if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP))
+ return NV_ERR_INVALID_ARGUMENT;
+
+ nv = NV_GET_NV_STATE(pGpu);
+
+ NV_STATUS status = nv_get_device_memory_config(nv, NULL, pAddrPhys, NULL, pNodeId);
+
+ return status;
+#endif
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+
+/*
+ * @brief Verif only function to get the chiplib overrides for link connection
+ * state for all C2C links.
+ *
+ * If chiplib overrides exist, each link can either be enabled (1) or disabled (0)
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[in] maxLinks Size of pLinkConnection array
+ * @param[out] pLinkConnection array of pLinkConnection values to be populated by MODS
+ *
+ * @return NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available)
+ */
+NV_STATUS
+osGetForcedC2CConnection
+(
+ OBJGPU *pGpu,
+ NvU32 maxLinks,
+ NvU32 *pLinkConnection
+)
+{
+ int i, ret;
+ NV_STATUS status;
+ char path[64];
+ OBJOS *pOS;
+ OBJSYS *pSys;
+
+ NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER);
+ NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED);
+ NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT);
+
+ pSys = SYS_GET_INSTANCE();
+ pOS = SYS_GET_OS(pSys);
+ if (pOS == NULL || pOS->osSimEscapeRead == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s: escape reads not supported on platform\n",
+ __FUNCTION__);
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ for (i = 0; i < maxLinks; i++)
+ {
+ ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|C2C%u", i);
+ NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1)));
+
+ status = pOS->osSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]);
+ if (status == NV_OK)
+ {
+ NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__,
+ path, pLinkConnection[i]);
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_INFO, "%s: osSimEscapeRead for '%s' failed (%u)\n",
+ __FUNCTION__, path, status);
+ return NV_ERR_NOT_SUPPORTED;
+ }
+ }
+ return NV_OK;
+}
+
+static NV_STATUS
+osGetSmbiosTableInfo
+(
+ const NvU8 *pMappedAddr,
+ NvU64 *pBaseAddr,
+ NvU64 *pLength,
+ NvU64 *pNumSubTypes,
+ NvU32 *pVersion
+)
+{
+ *pBaseAddr = 0;
+ *pLength = 0;
+ *pNumSubTypes = 0;
+ *pVersion = 0;
+
+ if (portMemCmp(pMappedAddr, "_SM3_", 5) == 0)
+ {
+ *pVersion = (pMappedAddr[7] << 8) | pMappedAddr[8];
+ portMemCopy(pLength, 4, pMappedAddr + 12, 4);
+ portMemCopy(pBaseAddr, 8, pMappedAddr + 16, 8);
+
+ *pNumSubTypes = *pLength / 4;
+
+ return NV_OK;
+ }
+
+ if (portMemCmp(pMappedAddr, "_SM_", 4) == 0)
+ {
+ *pVersion = (pMappedAddr[6] << 8) | pMappedAddr[7];
+
+ pMappedAddr += 16;
+
+ if (portMemCmp(pMappedAddr, "_DMI_", 5) == 0)
+ {
+ portMemCopy(pLength, 2, pMappedAddr + 6, 2);
+ portMemCopy(pBaseAddr, 4, pMappedAddr + 8, 4);
+ portMemCopy(pNumSubTypes, 2, pMappedAddr + 12, 2);
+
+ if (!*pVersion)
+ *pVersion = (pMappedAddr[14] & 0xF0) << 4 |
+ (pMappedAddr[14] & 0x0F);
+
+ return NV_OK;
+ }
+ }
+
+ return NV_ERR_INVALID_ADDRESS;
+}
+
+
+/*
+ * @brief Function to export SMBIOS table. Also, maps table in kernel-space.
+ *
+ * @param[out] ppBaseVAddr Base virtual address of SMBIOS table.
+ * @param[out] pLength Size of SMBIOS table.
+ * @param[out] pNumSubTypes Count of structures (types) embedded in
+ * the SMBIOS table.
+ * @param[out] pVersion SMBIOS version
+ *
+ * @return NV_OK, NV_ERR_INSUFFICIENT_RESOURCES or NV_ERR_INVALID_ADDRESS
+ * or errors from OS layer
+ */
+NV_STATUS
+osGetSmbiosTable
+(
+ void **ppBaseVAddr,
+ NvU64 *pLength,
+ NvU64 *pNumSubTypes,
+ NvU32 *pVersion
+)
+{
+ NV_STATUS status = NV_OK;
+ NvU64 physSmbiosAddr = ~0ull;
+ void *pMappedAddr = NULL;
+ NvU64 basePAddr = 0;
+
+ if (!NVCPU_IS_X86_64)
+ {
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ status = os_get_smbios_header(&physSmbiosAddr);
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_INFO, "%s: Failed query SMBIOS table with error: %x \n",
+ __FUNCTION__, status);
+ return status;
+ }
+
+ NV_ASSERT(physSmbiosAddr != ~0ull);
+
+ pMappedAddr = osMapKernelSpace(physSmbiosAddr,
+ os_page_size,
+ NV_MEMORY_CACHED,
+ NV_PROTECT_READ_WRITE);
+ if (!pMappedAddr)
+ {
+ return NV_ERR_INSUFFICIENT_RESOURCES;
+ }
+
+ status = osGetSmbiosTableInfo(pMappedAddr,
+ &basePAddr,
+ pLength,
+ pNumSubTypes,
+ pVersion);
+
+ osUnmapKernelSpace(pMappedAddr, os_page_size);
+
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ *ppBaseVAddr = osMapKernelSpace(basePAddr,
+ *pLength,
+ NV_MEMORY_CACHED,
+ NV_PROTECT_READ_WRITE);
+ if (!*ppBaseVAddr)
+ {
+ return NV_ERR_INSUFFICIENT_RESOURCES;
+ }
+
+ return NV_OK;
+}
+
+/*
+ * @brief Function to free SMBIOS table mappings
+ *
+ * @param[in] pBaseVAddr Base virtual address of SMBIOS table.
+ * @param[in] length Size of SMBIOS table.
+ *
+ */
+void
+osPutSmbiosTable
+(
+ void *pBaseVAddr,
+ NvU64 length
+)
+{
+ osUnmapKernelSpace(pBaseVAddr, length);
+}
+
+NV_STATUS
+osGetAcpiRsdpFromUefi
+(
+ NvU32 *pRsdpAddr
+)
+{
+ return os_get_acpi_rsdp_from_uefi(pRsdpAddr);
+}
+
+/*
+ * @brief Returns NV_TRUE if NvSwitch device is present in the system.
+ */
+NvBool
+osIsNvswitchPresent
+(
+ void
+)
+{
+ return os_is_nvswitch_present();
+}
+
+/*
+ * @brief Function to add crashlog buffer entry.
+ *
+ * @param[in] pBuffer virt_addr of nvlog buffer
+ * @param[in] length size of nvlog buffer
+ */
+void
+osAddRecordForCrashLog
+(
+ void *pBuffer,
+ NvU32 length
+)
+{
+ os_add_record_for_crashLog(pBuffer, length);
+}
+
+/*
+ * @brief Function to delete crashlog buffer entry.
+ *
+ * @param[in] pBuffer virt_addr of nvlog buffer
+ */
+void
+osDeleteRecordForCrashLog
+(
+ void *pBuffer
+)
+{
+ os_delete_record_for_crashLog(pBuffer);
+}
+
+/*
+ * @brief Queries the sysfs interface to get memblock size
+ * @param[out] memblock_size Pointer to the memblock_size
+ */
+NV_STATUS
+osNumaMemblockSize
+(
+ NvU64 *memblock_size
+)
+{
+ return os_numa_memblock_size(memblock_size);
+}
+
+NvBool
+osNumaOnliningEnabled
+(
+ OS_GPU_INFO *pOsGpuInfo
+)
+{
+ NvS32 numaNodeId = NV0000_CTRL_NO_NUMA_NODE;
+
+ //
+ // Note that this numaNodeId value fetched from Linux layer might not be
+ // accurate since it is possible to overwrite it with regkey on some configs
+ //
+ if (nv_get_device_memory_config(pOsGpuInfo, NULL, NULL, NULL,
+ &numaNodeId) != NV_OK)
+ {
+ return NV_FALSE;
+ }
+
+ return (numaNodeId != NV0000_CTRL_NO_NUMA_NODE);
+}
+
+/*
+ * @brief Function to call NUMA allocation entry.
+ *
+ * @param[in] nid NUMA node id
+ * @param[in] size Allocation size
+ * @param[in] flag Allocation flags
+ * @param[out] pAddress Ptr to the allocated physical address
+ */
+NV_STATUS
+osAllocPagesNode
+(
+ NvS32 nid,
+ NvLength size,
+ NvU32 flag,
+ NvU64 *pAddress
+)
+{
+ NV_STATUS status = NV_OK;
+ NvU32 localFlag = NV_ALLOC_PAGES_NODE_NONE;
+
+ if (pAddress == NULL || nid < 0 || size > NV_U32_MAX)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ // Translate the flags
+ if (flag & OS_ALLOC_PAGES_NODE_SKIP_RECLAIM)
+ {
+ localFlag |= NV_ALLOC_PAGES_NODE_SKIP_RECLAIM;
+ }
+
+ status = os_alloc_pages_node(nid, (NvU32)size, localFlag, pAddress);
+ return status;
+}
+
+NV_STATUS
+osAllocAcquirePage
+(
+ NvU64 pAddress
+)
+{
+ os_get_page(pAddress);
+ return NV_OK;
+}
+
+NV_STATUS
+osAllocReleasePage
+(
+ NvU64 pAddress
+)
+{
+ os_put_page(pAddress);
+ return NV_OK;
+}
+
+/*
+ * @brief Function to return refcount on a page
+ * @param[in] address The physical address of the page
+ */
+NvU32
+osGetPageRefcount
+(
+ NvU64 pAddress
+)
+{
+ return os_get_page_refcount(pAddress);
+}
+
+/*
+ * @brief Function to return the number of tail pages if the address is
+ * referring to a compound page; For non-compound pages, 1 is returned.
+ * @param[in] address The physical address of the page
+ */
+NvU32
+osCountTailPages
+(
+ NvU64 pAddress
+)
+{
+ return os_count_tail_pages(pAddress);
+}
+
+/*
+ * @brief Upon success, gets NPU register address range.
+ *
+ * @param[in] pOsGpuInfo OS specific GPU information pointer
+ * @param[out] pBase base (physical) of NPU register address range
+ * @param[out] pSize size of NPU register address range
+ */
+NV_STATUS
+osGetIbmnpuGenregInfo
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU64 *pBase,
+ NvU64 *pSize
+)
+{
+ return nv_get_ibmnpu_genreg_info(pOsGpuInfo, pBase, pSize, NULL);
+}
+
+/*
+ * @brief Upon success, gets NPU's relaxed ordering mode.
+ *
+ * @param[in] pOsGpuInfo OS specific GPU information pointer
+ * @param[out] pMode relaxed ordering mode
+ */
+NV_STATUS
+osGetIbmnpuRelaxedOrderingMode
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvBool *pMode
+)
+{
+ return nv_get_ibmnpu_relaxed_ordering_mode(pOsGpuInfo, pMode);
+}
+
+/*
+ * @brief Waits for NVLink HW flush on an NPU associated with a GPU.
+ *
+ * @param[in] pOsGpuInfo OS specific GPU information pointer
+ */
+void
+osWaitForIbmnpuRsync
+(
+ OS_GPU_INFO *pOsGpuInfo
+)
+{
+ nv_wait_for_ibmnpu_rsync(pOsGpuInfo);
+}
+
+NvU32
+osGetPageSize()
+{
+ return os_page_size;
+}
+
+
+
+/*
+ * @brief Opens a new temporary file for reading and writing
+ *
+ * @param[in] ppFile void double pointer
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ NV_ERR_GENERIC, if error
+ NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osOpenTemporaryFile
+(
+ void **ppFile
+)
+{
+ return os_open_temporary_file(ppFile);
+}
+
+/*
+ * @brief Closes the specified temporary file
+ *
+ * @param[in] pFile Pointer to file
+ *
+ * @returns void
+ */
+void
+osCloseFile
+(
+ void *pFile
+)
+{
+ os_close_file(pFile);
+}
+
+/*
+ * @brief Writes the buffer to the specified file at the given offset
+ *
+ * @param[in] pFile Pointer to file (void)
+ * @param[in] pBuffer Pointer to buffer from which to copy
+ * @param[in] size Size of the copy
+ * @parma[in] offset offset in the file
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ NV_ERR_GENERIC, if error
+ NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osWriteToFile
+(
+ void *pFile,
+ NvU8 *pBuffer,
+ NvU64 size,
+ NvU64 offset
+)
+{
+ return os_write_file(pFile, pBuffer, size, offset);
+}
+
+/*
+ * @brief Reads from the specified file at the given offset
+ *
+ * @param[in] pFile Pointer to file (void *)
+ * @param[in] pBuffer Pointer to buffer to which the data is copied
+ * @param[in] size Size of the copy
+ * @parma[in] offset offset in the file
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ NV_ERR_GENERIC, if error
+ NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osReadFromFile
+(
+ void *pFile,
+ NvU8 *pBuffer,
+ NvU64 size,
+ NvU64 offset
+)
+{
+ return os_read_file(pFile, pBuffer, size, offset);
+}
+
+/*
+ * @brief Unregisters caps from the capability framework.
+ * The function assumes that the caps are allocated and stored in the
+ * hierarchical order. If they aren't, OS (Linux kernel) would warn and
+ * leak the caps.
+ *
+ * @param[in] pOsRmCaps caps of interest
+ */
+void
+osRmCapUnregister
+(
+ OS_RM_CAPS **ppOsRmCaps
+)
+{
+ OS_RM_CAPS *pOsRmCaps = *ppOsRmCaps;
+ NvS32 i;
+
+ if (pOsRmCaps == NULL)
+ {
+ return;
+ }
+
+ for (i = pOsRmCaps->count - 1; i >= 0; i--)
+ {
+ if (pOsRmCaps->caps[i] != NULL)
+ {
+ os_nv_cap_destroy_entry(pOsRmCaps->caps[i]);
+ }
+ }
+
+ os_free_mem(pOsRmCaps->caps);
+ os_free_mem(pOsRmCaps);
+
+ *ppOsRmCaps = NULL;
+}
+
+static NV_STATUS
+_allocOsRmCaps
+(
+ OS_RM_CAPS **ppOsRmCaps,
+ NvU32 count
+)
+{
+ NV_STATUS status;
+ OS_RM_CAPS *pOsRmCaps;
+
+ *ppOsRmCaps = NULL;
+
+ status = os_alloc_mem((void**)&pOsRmCaps, sizeof(OS_RM_CAPS));
+ if (status != NV_OK)
+ return status;
+
+ pOsRmCaps->count = count;
+
+ status = os_alloc_mem((void**)&pOsRmCaps->caps, sizeof(pOsRmCaps->caps[0]) * count);
+ if (status != NV_OK)
+ {
+ os_free_mem(pOsRmCaps);
+ return status;
+ }
+
+ os_mem_set(pOsRmCaps->caps, 0, sizeof(pOsRmCaps->caps[0]) * count);
+
+ *ppOsRmCaps = pOsRmCaps;
+ return NV_OK;
+}
+
+#define OS_RM_CAP_GPU_DIR 0
+#define OS_RM_CAP_GPU_MIG_DIR 1
+#define OS_RM_CAP_GPU_COUNT 2
+
+/*
+ * @brief Registers OBJGPU with the capability framework.
+ *
+ * @param[in] pOsGpuInfo OS specific GPU information pointer
+ * @param[out] ppOsRmCaps GPU OS specific capabilities pointer
+ */
+NV_STATUS
+osRmCapRegisterGpu
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ OS_RM_CAPS **ppOsRmCaps
+)
+{
+ NvU32 minor = nv_get_dev_minor(pOsGpuInfo);
+ char name[16];
+ NV_STATUS status;
+ OS_RM_CAPS *pOsRmCaps;
+ nv_cap_t *parent;
+ nv_cap_t *cap;
+
+ // Return success on the unsupported platforms.
+ if (nvidia_caps_root == NULL)
+ {
+ return NV_OK;
+ }
+
+ if (*ppOsRmCaps != NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_GPU_COUNT);
+ if (status != NV_OK)
+ return status;
+
+ *ppOsRmCaps = pOsRmCaps;
+
+ os_snprintf(name, sizeof(name), "gpu%u", minor);
+ name[sizeof(name) - 1] = '\0';
+ parent = nvidia_caps_root;
+
+ cap = os_nv_cap_create_dir_entry(parent, name, (OS_RUGO | OS_XUGO));
+ if (cap == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to setup gpu%u directory\n", minor);
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+
+ pOsRmCaps->caps[OS_RM_CAP_GPU_DIR] = cap;
+ parent = cap;
+
+ // TODO: Bug 2679591: Add MIG directory only if SMC is enabled.
+ // For now, always add "mig" directory.
+ cap = os_nv_cap_create_dir_entry(parent, "mig", (OS_RUGO | OS_XUGO));
+ if (cap == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to setup mig directory\n");
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+
+ pOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR] = cap;
+
+ return NV_OK;
+
+failed:
+ osRmCapUnregister(ppOsRmCaps);
+
+ return status;
+}
+
+#define OS_RM_CAP_SMC_PART_DIR 0
+#define OS_RM_CAP_SMC_PART_ACCESS_FILE 1
+#define OS_RM_CAP_SMC_PART_COUNT 2
+
+/*
+ * @brief Registers SMC partition (a.k.a. GPU instance) with the capability
+ * framework
+ *
+ * @param[in] pGpuOsRmCaps GPU OS specific capabilities pointer
+ * @param[out] ppPartitionOsRmCaps OS specific capabilities pointer for SMC partition
+ * @param[in] swizzId SMC partition swizz ID
+ */
+NV_STATUS
+osRmCapRegisterSmcPartition
+(
+ OS_RM_CAPS *pGpuOsRmCaps,
+ OS_RM_CAPS **ppPartitionOsRmCaps,
+ NvU32 swizzId
+)
+{
+ char name[16];
+ NV_STATUS status;
+ nv_cap_t *parent;
+ nv_cap_t *cap;
+ OS_RM_CAPS *pOsRmCaps;
+
+ // Return success as there is nothing to do.
+ if (pGpuOsRmCaps == NULL)
+ {
+ return NV_OK;
+ }
+
+ if (*ppPartitionOsRmCaps != NULL || swizzId >= NV_U32_MAX)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ parent = pGpuOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR];
+ if (parent == NULL)
+ {
+ return NV_ERR_INVALID_STATE;
+ }
+
+ status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_PART_COUNT);
+ if (status != NV_OK)
+ return status;
+
+ *ppPartitionOsRmCaps = pOsRmCaps;
+
+ os_snprintf(name, sizeof(name), "gi%u", swizzId);
+ name[sizeof(name) - 1] = '\0';
+
+ cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO);
+ if (cap == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to setup gi%u directory\n",
+ swizzId);
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+
+ pOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR] = cap;
+ parent = cap;
+
+ cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO);
+ if (cap == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n",
+ swizzId);
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+
+ pOsRmCaps->caps[OS_RM_CAP_SMC_PART_ACCESS_FILE] = cap;
+
+ return NV_OK;
+
+failed:
+ osRmCapUnregister(ppPartitionOsRmCaps);
+
+ return status;
+}
+
+#define OS_RM_CAP_SMC_EXEC_PART_DIR 0
+#define OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE 1
+#define OS_RM_CAP_SMC_EXEC_PART_COUNT 2
+
+/*
+ * @brief Registers SMC execution partition (a.k.a. compute instance) with the
+ * capability framework
+ *
+ * @param[in] pPartitionOsRmCaps OS specific capabilities pointer for SMC partition
+ * @param[out] ppExecPartitionOsRmCaps OS specific capabilities pointer for SMC execution partition
+ * @param[in] execPartitionId SMC execution partition ID
+ */
+NV_STATUS
+osRmCapRegisterSmcExecutionPartition
+(
+ OS_RM_CAPS *pPartitionOsRmCaps,
+ OS_RM_CAPS **ppExecPartitionOsRmCaps,
+ NvU32 execPartitionId
+)
+{
+ char name[16];
+ NV_STATUS status;
+ nv_cap_t *parent;
+ nv_cap_t *cap;
+ OS_RM_CAPS *pOsRmCaps;
+
+ // Return success as there is nothing to do.
+ if (pPartitionOsRmCaps == NULL)
+ {
+ return NV_OK;
+ }
+
+ if ((*ppExecPartitionOsRmCaps != NULL) || (execPartitionId >= NV_U32_MAX))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ parent = pPartitionOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR];
+ if (parent == NULL)
+ {
+ return NV_ERR_INVALID_STATE;
+ }
+
+ status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_EXEC_PART_COUNT);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ *ppExecPartitionOsRmCaps = pOsRmCaps;
+
+ os_snprintf(name, sizeof(name), "ci%u", execPartitionId);
+ name[sizeof(name) - 1] = '\0';
+
+ cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO);
+ if (cap == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to setup ci%u directory\n",
+ execPartitionId);
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+
+ pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_DIR] = cap;
+ parent = cap;
+
+ cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO);
+ if (cap == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n",
+ execPartitionId);
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+
+ pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE] = cap;
+
+ return NV_OK;
+
+failed:
+ osRmCapUnregister(ppExecPartitionOsRmCaps);
+
+ return status;
+}
+
+/*
+ * @brief Release the acquired capability
+ *
+ * @param[in] dupedCapDescriptor descriptor to be released
+ */
+void
+osRmCapRelease
+(
+ NvU64 dupedCapDescriptor
+)
+{
+ if (dupedCapDescriptor == NV_U64_MAX)
+ {
+ return;
+ }
+
+ os_nv_cap_close_fd((int)dupedCapDescriptor);
+}
+
+#define OS_RM_CAP_SYS_MIG_DIR 0
+#define OS_RM_CAP_SYS_SMC_CONFIG_FILE 1
+#define OS_RM_CAP_SYS_SMC_MONITOR_FILE 2
+#define OS_RM_CAP_SYS_COUNT 3
+
+NV_STATUS
+osRmCapRegisterSys
+(
+ OS_RM_CAPS **ppOsRmCaps
+)
+{
+ nv_cap_t **ppCaps;
+ nv_cap_t *parent;
+ nv_cap_t *cap;
+ NV_STATUS status;
+ OS_RM_CAPS *pOsRmCaps;
+
+ if (nvidia_caps_root == NULL)
+ return NV_ERR_NOT_SUPPORTED;
+
+ status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SYS_COUNT);
+ if (status != NV_OK)
+ return status;
+
+ *ppOsRmCaps = pOsRmCaps;
+
+ ppCaps = pOsRmCaps->caps;
+
+ parent = os_nv_cap_create_dir_entry(nvidia_caps_root, "mig", OS_RUGO | OS_XUGO);
+ if (parent == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to create mig directory\n");
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+ ppCaps[OS_RM_CAP_SYS_MIG_DIR] = parent;
+
+ cap = os_nv_cap_create_file_entry(parent, "config", OS_RUSR);
+ if (cap == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to create mig config file\n");
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+ ppCaps[OS_RM_CAP_SYS_SMC_CONFIG_FILE] = cap;
+
+ cap = os_nv_cap_create_file_entry(parent, "monitor", OS_RUGO);
+ if (cap == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to create mig monitor file\n");
+ status = NV_ERR_OPERATING_SYSTEM;
+ goto failed;
+ }
+ ppCaps[OS_RM_CAP_SYS_SMC_MONITOR_FILE] = cap;
+
+ return NV_OK;
+
+failed:
+ osRmCapUnregister(ppOsRmCaps);
+ return status;
+}
+
+/*
+ * @brief Acquire the requested capability
+ *
+ * @param[in] pOsRmCaps opaque pointer to the caps.
+ * @param[in] rmCap the capability to be acquired.
+ * @param[in] capDescriptor descriptor to be used for validation
+ * @param[out] dupedCapDescriptor returns duplicated descriptor if validation
+ * is successful
+ *
+ * Note: On Linux, duplicating fd is helpful to let administrators know about
+ * the capability users. See https://linux.die.net/man/8/lsof usage.
+ */
+NV_STATUS
+osRmCapAcquire
+(
+ OS_RM_CAPS *pOsRmCaps,
+ NvU32 rmCap,
+ NvU64 capDescriptor,
+ NvU64 *dupedCapDescriptor
+)
+{
+ nv_cap_t *cap;
+ int fd = (int)capDescriptor;
+ int duped_fd;
+ NvU32 index;
+ NV_STATUS status;
+
+ *dupedCapDescriptor = NV_U64_MAX;
+
+ switch (rmCap)
+ {
+ case NV_RM_CAP_SMC_PARTITION_ACCESS:
+ {
+ index = OS_RM_CAP_SMC_PART_ACCESS_FILE;
+ break;
+ }
+ case NV_RM_CAP_EXT_FABRIC_MGMT:
+ {
+ status = nv_acquire_fabric_mgmt_cap(fd, &duped_fd);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ goto done;
+ }
+ case NV_RM_CAP_SMC_EXEC_PARTITION_ACCESS:
+ {
+ index = OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE;
+ break;
+ }
+ case NV_RM_CAP_SYS_SMC_CONFIG:
+ {
+ index = OS_RM_CAP_SYS_SMC_CONFIG_FILE;
+ break;
+ }
+ case NV_RM_CAP_SYS_SMC_MONITOR:
+ {
+ index = OS_RM_CAP_SYS_SMC_MONITOR_FILE;
+ break;
+ }
+ default:
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+ }
+
+ if (pOsRmCaps == NULL)
+ {
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ if (index >= pOsRmCaps->count)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ cap = pOsRmCaps->caps[index];
+
+ duped_fd = os_nv_cap_validate_and_dup_fd(cap, fd);
+ if (duped_fd < 0)
+ {
+ return NV_ERR_INSUFFICIENT_PERMISSIONS;
+ }
+
+done:
+ *dupedCapDescriptor = duped_fd;
+
+ return NV_OK;
+}
+
+/*
+ * @brief Initializes capability descriptor
+ *
+ * @param[out] pCapDescriptor descriptor to be used
+ *
+ */
+void
+osRmCapInitDescriptor
+(
+ NvU64 *pCapDescriptor
+)
+{
+ *pCapDescriptor = NV_U64_MAX;
+}
+
+/*
+ * @brief Generates random bytes which can be used as a universally unique
+ * identifier.
+ *
+ * @param[out] pBytes Array of random bytes
+ * @param[in] numBytes Size of the array
+ */
+NV_STATUS
+osGetRandomBytes
+(
+ NvU8 *pBytes,
+ NvU16 numBytes
+)
+{
+ os_get_random_bytes(pBytes, numBytes);
+
+ return NV_OK;
+}
+
+/*
+ * @brief Allocate wait queue
+ *
+ * @param[out] ppWq Wait queue
+ */
+NV_STATUS
+osAllocWaitQueue
+(
+ OS_WAIT_QUEUE **ppWq
+)
+{
+ return os_alloc_wait_queue(ppWq);
+}
+
+/*
+ * @brief Free wait queue
+ *
+ * @param[in] pWq Wait queue
+ */
+void
+osFreeWaitQueue
+(
+ OS_WAIT_QUEUE *pWq
+)
+{
+ os_free_wait_queue(pWq);
+}
+
+/*
+ * @brief Put thread to uninterruptible sleep
+ *
+ * @param[in] pWq Wait queue
+ */
+void
+osWaitUninterruptible
+(
+ OS_WAIT_QUEUE *pWq
+)
+{
+ os_wait_uninterruptible(pWq);
+}
+
+/*
+ * @brief Put thread to interruptible sleep
+ *
+ * @param[in] pWq Wait queue
+ */
+void
+osWaitInterruptible
+(
+ OS_WAIT_QUEUE *pWq
+)
+{
+ os_wait_interruptible(pWq);
+}
+
+/*
+ * @brief Wake up thread from uninterruptible sleep
+ *
+ * @param[in] pWq Wait queue
+ */
+void
+osWakeUp
+(
+ OS_WAIT_QUEUE *pWq
+)
+{
+ os_wake_up(pWq);
+}
+
+NV_STATUS
+osReadPFPciConfigInVF
+(
+ NvU32 addr,
+ NvU32 *data
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Returns IMP-relevant data collected from other modules
+ *
+ * This function is basically a wrapper to call the unix/linux layer.
+ *
+ * @param[out] pTegraImpImportData Structure to receive the data
+ *
+ * @returns NV_OK if successful,
+ * NV_ERR_BUFFER_TOO_SMALL if the array in TEGRA_IMP_IMPORT_DATA is
+ * too small,
+ * NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ * other errors as may be returned by subfunctions.
+ */
+NV_STATUS
+osTegraSocGetImpImportData
+(
+ TEGRA_IMP_IMPORT_DATA *pTegraImpImportData
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Tells BPMP whether or not RFL is valid
+ *
+ * Display HW generates an ok_to_switch signal which asserts when mempool
+ * occupancy is high enough to be able to turn off memory long enough to
+ * execute a dramclk frequency switch without underflowing display output.
+ * ok_to_switch drives the RFL ("request for latency") signal in the memory
+ * unit, and the switch sequencer waits for this signal to go active before
+ * starting a dramclk switch. However, if the signal is not valid (e.g., if
+ * display HW or SW has not been initialized yet), the switch sequencer ignores
+ * the signal. This API tells BPMP whether or not the signal is valid.
+ *
+ * @param[in] pOsGpuInfo Per GPU Linux state
+ * @param[in] bEnable True if RFL will be valid; false if invalid
+ *
+ * @returns NV_OK if successful,
+ * NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ * NV_ERR_GENERIC if some other kind of error occurred.
+ */
+NV_STATUS
+osTegraSocEnableDisableRfl
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvBool bEnable
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Allocates a specified amount of ISO memory bandwidth for display
+ *
+ * floorBandwidthKBPS is the minimum required (i.e., floor) dramclk frequency
+ * multiplied by the width of the pipe over which the display data will travel.
+ * (It is understood that the bandwidth calculated by multiplying the clock
+ * frequency by the pipe width will not be realistically achievable, due to
+ * overhead in the memory subsystem. The infrastructure will not actually use
+ * the bandwidth value, except to reverse the calculation to get the required
+ * dramclk frequency.)
+ *
+ * This function is basically a wrapper to call the unix/linux layer.
+ *
+ * @param[in] pOsGpuInfo OS specific GPU information pointer
+ * @param[in] averageBandwidthKBPS Amount of ISO memory bandwidth requested
+ * @param[in] floorBandwidhtKBPS Min required dramclk freq * pipe width
+ *
+ * @returns NV_OK if successful,
+ * NV_ERR_INSUFFICIENT_RESOURCES if one of the bandwidth values is too
+ * high, and bandwidth cannot be allocated,
+ * NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ * NV_ERR_GENERIC if some other kind of error occurred.
+ */
+NV_STATUS
+osTegraAllocateDisplayBandwidth
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 averageBandwidthKBPS,
+ NvU32 floorBandwidthKBPS
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Creates or sets up platform specific nano second resolution timer
+ *
+ * @param[in] pOsGpuInfo OS specific GPU information pointer
+ * @param[in] pTmrEvent Pointer to timer event information
+ * @param[in/out] pTimer pointer to hold high resolution timer object
+ */
+NV_STATUS
+osCreateNanoTimer
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ void *pTmrEvent,
+ void **pTimer
+)
+{
+ nv_create_nano_timer(pOsGpuInfo, pTmrEvent, (nv_nano_timer_t **)pTimer);
+ return NV_OK;
+}
+
+/*!
+ * @brief Starts platform specific nano second resolution timer
+ *
+ * @param[in] pOsGpuInfo OS specific GPU information pointer
+ * @param[in] pTimer pointer to high resolution timer object
+ * @param[in] timens time in nano seconds
+ */
+NV_STATUS
+osStartNanoTimer
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ void *pTimer,
+ NvU64 timeNs
+)
+{
+ nv_start_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer, timeNs);
+ return NV_OK;
+}
+
+/*!
+ * @brief Cancels platform specific nano second resolution timer
+ *
+ * @param[in] pOsGpuInfo OS specific GPU information pointer
+ * @param[in] pTimer pointer to timer object
+ */
+NV_STATUS
+osCancelNanoTimer
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ void *pTimer
+)
+{
+ nv_cancel_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer);
+ return NV_OK;
+}
+
+/*!
+ * @brief Destroys & cancels platform specific nano second resolution timer
+ *
+ *
+ * @param[in] pGpu Device of interest
+ * @param[in] pTimer pointer to timer object
+ */
+NV_STATUS
+osDestroyNanoTimer
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ void *pTimer
+)
+{
+ nv_destroy_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer);
+ return NV_OK;
+}
+
+/*!
+ * @brief Get number of dpAux instances.
+ * It is wrapper function to call unix/linux layer.
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] pNumIntances Number of dpAux instances.
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ * NV_ERR_GENERIC, if error
+ * NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+
+NV_STATUS
+osGetTegraNumDpAuxInstances
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 *pNumIntances
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*
+ * @brief Return the priv Data of current IRQ.
+ * It is wrapper function to call unix/linux layer.
+ *
+ * @param[in] pGpu Device of interest
+ * @param[out] pPrivData privData of current IRQ
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ * NV_ERR_GENERIC, if error
+ * NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osGetCurrentIrqPrivData
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 *pPrivData
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Get the brightness level
+ * It is wrapper function to call unix/linux layer.
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] brightness Pointer to brightness level
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ * NV_ERR_GENERIC, if error
+ * NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osGetTegraBrightnessLevel
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 *brightness
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Set the brightness level
+ * It is wrapper function to call unix/linux layer.
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] brightness brightness level
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ * NV_ERR_GENERIC, if error
+ * NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osSetTegraBrightnessLevel
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 brightness
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/* @brief Gets syncpoint aperture information
+ *
+ * @param[in] OS_GPU_INFO OS specific GPU information pointer
+ * @param[in] syncpointId
+ * @param[out] *physAddr
+ * @param[out] *limit
+ * @param[out] *offset
+ */
+NV_STATUS
+osGetSyncpointAperture
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 syncpointId,
+ NvU64 *physAddr,
+ NvU64 *limit,
+ NvU32 *offset
+)
+{
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Enable PCIe AtomicOp Requester Enable and return
+ * the completer side capabilities that the requester can send.
+ *
+ * @param[in] pOsGpuInfo OS_GPU_INFO OS specific GPU information pointer
+ * @param[out] pMask mask of supported atomic size, including one or more of:
+ * OS_PCIE_CAP_MASK_REQ_ATOMICS_32
+ * OS_PCIE_CAP_MASK_REQ_ATOMICS_64
+ * OS_PCIE_CAP_MASK_REQ_ATOMICS_128
+ *
+ * @returns NV_STATUS, NV_OK if success
+ * NV_ERR_NOT_SUPPORTED if platform doesn't support this
+ * feature.
+ * NV_ERR_GENERIC for any other error
+ */
+
+NV_STATUS
+osConfigurePcieReqAtomics
+(
+ OS_GPU_INFO *pOsGpuInfo,
+ NvU32 *pMask
+)
+{
+ if (pMask)
+ {
+ *pMask = 0U;
+ if (pOsGpuInfo)
+ {
+ if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
+ OS_INTF_PCIE_REQ_ATOMICS_32BIT) == NV_OK)
+ *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_32;
+ if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
+ OS_INTF_PCIE_REQ_ATOMICS_64BIT) == NV_OK)
+ *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_64;
+ if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
+ OS_INTF_PCIE_REQ_ATOMICS_128BIT) == NV_OK)
+ *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_128;
+
+ if (*pMask != 0)
+ return NV_OK;
+ }
+ }
+ return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Check GPU is accessible or not
+ *
+ * @param[in] pGpu GPU object pointer
+ *
+ * @returns NVBool, Returns TRUE if the GPU is accessible,
+ * FALSE, if error
+ */
+NvBool
+osIsGpuAccessible
+(
+ OBJGPU *pGpu
+)
+{
+ return nv_is_gpu_accessible(NV_GET_NV_STATE(pGpu));
+}
+
+/*!
+ * @brief Check whether GPU has received a shutdown notification from the OS
+ */
+NvBool
+osIsGpuShutdown
+(
+ OBJGPU *pGpu
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ return nv ? nv->is_shutdown : NV_TRUE;
+}
+
+/*!
+ * @brief Check GPU OS info matches
+ *
+ * @param[in] pGpu GPU object pointer
+ *
+ * @returns NVBool, Returns TRUE if matched.
+ */
+NvBool
+osMatchGpuOsInfo
+(
+ OBJGPU *pGpu,
+ void *pOsInfo
+)
+{
+ return nv_match_gpu_os_info(NV_GET_NV_STATE(pGpu), pOsInfo);
+}
+
+/*!
+ * @brief Release GPU OS info.
+ *
+ * @param[in] pOsInfo GPU OS info pointer
+ *
+ * @returns void
+ */
+void
+osReleaseGpuOsInfo
+(
+ void *pOsInfo
+)
+{
+ nv_put_file_private(pOsInfo);
+}
+
+NvBool
+osDmabufIsSupported(void)
+{
+ return os_dma_buf_enabled;
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osapi.c b/userspace/sdks/nv525/nvalloc/unix/src/osapi.c
new file mode 100644
index 0000000..dcd8604
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/osapi.c
@@ -0,0 +1,5333 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+
+#include <nv_ref.h>
+#include <nv.h>
+#include <nv-priv.h>
+#include <os/os.h>
+#include <osapi.h>
+#include <class/cl0000.h>
+#include <rmosxfac.h> // Declares RmInitRm().
+#include "gpu/gpu.h"
+#include <osfuncs.h>
+#include <platform/chipset/chipset.h>
+
+#include <objtmr.h>
+#include <gpu/subdevice/subdevice.h>
+#include <mem_mgr/mem.h>
+#include "kernel/gpu/mem_mgr/mem_mgr.h"
+
+#include <gpu/mem_sys/kern_mem_sys.h>
+
+#include <diagnostics/journal.h>
+#include <nvrm_registry.h>
+
+#include <nvUnixVersion.h>
+#include <gpu_mgr/gpu_mgr.h>
+#include <core/thread_state.h>
+#include <platform/acpi_common.h>
+#include <core/locks.h>
+
+#include <mem_mgr/p2p.h>
+
+#include "rmapi/exports.h"
+#include "rmapi/rmapi_utils.h"
+#include "rmapi/rs_utils.h"
+#include "rmapi/resource_fwd_decls.h"
+#include <nv-kernel-rmapi-ops.h>
+#include <rmobjexportimport.h>
+#include "nv-reg.h"
+#include "core/hal_mgr.h"
+#include "gpu/device/device.h"
+
+#include "resserv/rs_server.h"
+#include "resserv/rs_client.h"
+#include "resserv/rs_resource.h"
+#include "gpu/gpu_uuid.h"
+
+#include "platform/chipset/pci_pbi.h"
+
+#include "ctrl/ctrl0000/ctrl0000system.h"
+#include "ctrl/ctrl0073/ctrl0073dp.h"
+#include "ctrl/ctrl0073/ctrl0073system.h"
+#include "ctrl/ctrl0073/ctrl0073specific.h"
+#include "ctrl/ctrl2080/ctrl2080bios.h"
+#include "ctrl/ctrl2080/ctrl2080fb.h"
+#include "ctrl/ctrl2080/ctrl2080perf.h"
+#include "ctrl/ctrl2080/ctrl2080gpu.h"
+#include "ctrl/ctrl402c.h"
+
+#include "g_nv_name_released.h" // released chip entries from nvChipAutoFlags.h
+
+#include <virtualization/hypervisor/hypervisor.h>
+
+#include "gpu/bus/kern_bus.h"
+
+//
+// Helper function which can be called before doing any RM control
+// This function:
+//
+// a. Performs threadStateInit().
+// b. Acquires API lock.
+// c. Checks if RMAPI client handle is valid (i.e. RM is initialized) and
+// returns early if RMAPI client handle is invalid.
+// d. Increments the dynamic power refcount. If GPU is in RTD3 suspended
+// state, then it will wake-up the GPU.
+// e. Returns the RMAPI interface handle.
+//
+// This function should be called only when caller doesn't have acquired API
+// lock. Caller needs to use RmUnixRmApiEpilogue() after RM control, if
+// RmUnixRmApiPrologue() is successful.
+//
+RM_API *RmUnixRmApiPrologue(nv_state_t *pNv, THREAD_STATE_NODE *pThreadNode, NvU32 module)
+{
+ threadStateInit(pThreadNode, THREAD_STATE_FLAGS_NONE);
+
+ if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, module)) == NV_OK)
+ {
+ if ((pNv->rmapi.hClient != 0) &&
+ (os_ref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE) == NV_OK))
+ {
+ return rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+ }
+
+ rmapiLockRelease();
+ }
+
+ threadStateFree(pThreadNode, THREAD_STATE_FLAGS_NONE);
+
+ return NULL;
+}
+
+//
+// Helper function which can be called after doing RM control, if
+// caller has used RmUnixRmApiPrologue() helper function. This function:
+//
+// a. Decrements the dynamic power refcount.
+// b. Release API lock.
+// c. Performs threadStateFree().
+//
+void RmUnixRmApiEpilogue(nv_state_t *pNv, THREAD_STATE_NODE *pThreadNode)
+{
+ os_unref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE);
+ rmapiLockRelease();
+ threadStateFree(pThreadNode, THREAD_STATE_FLAGS_NONE);
+}
+
+NvBool RmGpuHasIOSpaceEnabled(nv_state_t * nv)
+{
+ NvU16 val;
+ NvBool has_io;
+ os_pci_read_word(nv->handle, NV_CONFIG_PCI_NV_1, &val);
+ has_io = FLD_TEST_DRF(_CONFIG, _PCI_NV_1, _IO_SPACE, _ENABLED, val);
+ return has_io;
+}
+
+// This is a stub function for unix
+void osHandleDeferredRecovery(
+ OBJGPU *pGpu
+)
+{
+
+}
+
+// This is a stub function for unix
+NvBool osIsSwPreInitOnly
+(
+ OS_GPU_INFO *pOsGpuInfo
+)
+{
+ return NV_FALSE;
+}
+
+const NvU8 * RmGetGpuUuidRaw(
+ nv_state_t *pNv
+)
+{
+ NV_STATUS rmStatus;
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+ NvU32 gidFlags;
+ NvBool isApiLockTaken = NV_FALSE;
+
+ if (pNv->nv_uuid_cache.valid)
+ goto done;
+
+ //
+ // PBI is not present in simulation and the loop inside
+ // pciPbiReadUuid takes up considerable amount of time in
+ // simulation environment during RM load.
+ //
+ if (pGpu && IS_SIMULATION(pGpu))
+ {
+ rmStatus = NV_ERR_NOT_SUPPORTED;
+ }
+ else
+ {
+ rmStatus = pciPbiReadUuid(pNv->handle, pNv->nv_uuid_cache.uuid);
+ }
+
+ if (rmStatus == NV_OK)
+ {
+ rmStatus = gpumgrSetUuid(pNv->gpu_id, pNv->nv_uuid_cache.uuid);
+ if (rmStatus != NV_OK)
+ {
+ return NULL;
+ }
+
+ pNv->nv_uuid_cache.valid = NV_TRUE;
+ goto done;
+ }
+ else if (rmStatus == NV_ERR_NOT_SUPPORTED)
+ {
+ nv_printf(NV_DBG_INFO,
+ "NVRM: PBI is not supported for GPU " NV_PCI_DEV_FMT "\n",
+ NV_PCI_DEV_FMT_ARGS(pNv));
+ }
+
+ gidFlags = DRF_DEF(2080_GPU_CMD,_GPU_GET_GID_FLAGS,_TYPE,_SHA1)
+ | DRF_DEF(2080_GPU_CMD,_GPU_GET_GID_FLAGS,_FORMAT,_BINARY);
+
+ if (!rmapiLockIsOwner())
+ {
+ rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU);
+ if (rmStatus != NV_OK)
+ {
+ return NULL;
+ }
+
+ isApiLockTaken = NV_TRUE;
+ }
+
+ if (pGpu == NULL)
+ {
+ if (isApiLockTaken == NV_TRUE)
+ {
+ rmapiLockRelease();
+ }
+
+ return NULL;
+ }
+
+ rmStatus = gpuGetGidInfo(pGpu, NULL, NULL, gidFlags);
+ if (isApiLockTaken == NV_TRUE)
+ {
+ rmapiLockRelease();
+ }
+
+ if (rmStatus != NV_OK)
+ return NULL;
+
+ if (!pGpu->gpuUuid.isInitialized)
+ return NULL;
+
+ // copy the uuid from the OBJGPU uuid cache
+ os_mem_copy(pNv->nv_uuid_cache.uuid, pGpu->gpuUuid.uuid, GPU_UUID_LEN);
+ pNv->nv_uuid_cache.valid = NV_TRUE;
+
+done:
+ return pNv->nv_uuid_cache.uuid;
+}
+
+static NV_STATUS RmGpuUuidRawToString(
+ const NvU8 *pGidData,
+ char *pBuf,
+ NvU32 bufLen
+)
+{
+ NvU8 *pGidString;
+ NvU32 GidStrlen;
+ NV_STATUS rmStatus;
+ NvU32 gidFlags;
+
+ gidFlags = DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _ASCII) |
+ DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1);
+
+ rmStatus = transformGidToUserFriendlyString(pGidData, RM_SHA1_GID_SIZE,
+ &pGidString, &GidStrlen,
+ gidFlags);
+ if (rmStatus != NV_OK)
+ return rmStatus;
+
+ if (bufLen >= GidStrlen)
+ portMemCopy(pBuf, bufLen, pGidString, GidStrlen);
+ else
+ rmStatus = NV_ERR_BUFFER_TOO_SMALL;
+
+ portMemFree((void *)pGidString);
+
+ return rmStatus;
+}
+
+// This function should be called with the API and GPU locks already acquired.
+NV_STATUS
+RmLogGpuCrash(OBJGPU *pGpu)
+{
+ NV_STATUS status = NV_OK;
+ NvBool bGpuIsLost, bGpuIsConnected;
+
+ if (pGpu == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ //
+ // Re-evaluate whether or not the GPU is accessible. This could be called
+ // from a recovery context where the OS has re-enabled MMIO for the device.
+ // This happens during EEH processing on IBM Power + Linux, and marking
+ // the device as connected again will allow rcdbAddRmGpuDump() to collect
+ // more GPU state.
+ //
+ bGpuIsLost = pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_LOST);
+ bGpuIsConnected = pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED);
+ if (!bGpuIsConnected || bGpuIsLost)
+ {
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ NvU32 pmcBoot0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0);
+ if (pmcBoot0 == nvp->pmc_boot_0)
+ {
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED, NV_TRUE);
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_LOST, NV_FALSE);
+ }
+ }
+
+ //
+ // Log the engine data to the Journal object, to be pulled out later. This
+ // will return NV_WARN_MORE_PROCESSING_REQUIRED if the dump needed to be
+ // deferred to a passive IRQL. We still log the crash dump as being created
+ // in that case since it (should) be created shortly thereafter, and
+ // there's currently not a good way to print the below notification
+ // publicly from the core RM when the DPC completes.
+ //
+ status = rcdbAddRmGpuDump(pGpu);
+ if (status != NV_OK && status != NV_WARN_MORE_PROCESSING_REQUIRED)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: failed to save GPU crash data\n", __FUNCTION__);
+ }
+ else
+ {
+ status = NV_OK;
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: A GPU crash dump has been created. If possible, please run\n"
+ "NVRM: nvidia-bug-report.sh as root to collect this data before\n"
+ "NVRM: the NVIDIA kernel module is unloaded.\n");
+ }
+
+ // Restore the disconnected properties, if they were reset
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED, bGpuIsConnected);
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_LOST, bGpuIsLost);
+
+ // Restore persistence mode to the way it was prior to the crash
+ osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo,
+ pGpu->getProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE));
+
+ return status;
+}
+
+static void free_os_event_under_lock(nv_event_t *event)
+{
+ event->active = NV_FALSE;
+
+ // If refcount > 0, event will be freed by osDereferenceObjectCount
+ // when the last associated RM event is freed.
+ if (event->refcount == 0)
+ {
+ portMemFree(event);
+ }
+}
+
+static void free_os_events(
+ nv_file_private_t *nvfp,
+ NvHandle client
+)
+{
+ nv_state_t *nv = nv_get_ctl_state();
+ nv_event_t **pprev;
+
+ portSyncSpinlockAcquire(nv->event_spinlock);
+
+ pprev = &nv->event_list;
+ while (*pprev != NULL)
+ {
+ nv_event_t *cur = *pprev;
+ //
+ // XXX We must be called from either rm_client_free_os_events() or
+ // RmFreeUnusedClients() for this to work.
+ //
+ if ((cur->hParent == client) || (cur->nvfp == nvfp))
+ {
+ *pprev = cur->next;
+ free_os_event_under_lock(cur);
+ }
+ else
+ {
+ pprev = &cur->next;
+ }
+ }
+
+ portSyncSpinlockRelease(nv->event_spinlock);
+}
+
+void rm_client_free_os_events(
+ NvHandle client
+)
+{
+ free_os_events(NULL, client);
+}
+
+void RmFreeUnusedClients(
+ nv_state_t *nv,
+ nv_file_private_t *nvfp
+)
+{
+ NvU32 *pClientList;
+ NvU32 numClients, i;
+ NV_STATUS status;
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+
+ //
+ // The 'nvfp' pointer uniquely identifies an open instance in kernel space
+ // and the kernel interface layer guarantees that we are not called before
+ // the associated nvfp descriptor is closed. We can thus safely free
+ // abandoned clients with matching 'nvfp' pointers.
+ //
+ status = rmapiGetClientHandlesFromOSInfo(nvfp, &pClientList, &numClients);
+ if (status != NV_OK)
+ {
+ numClients = 0;
+ }
+
+ for (i = 0; i < numClients; ++i)
+ {
+ NV_PRINTF(LEVEL_INFO, "freeing abandoned client 0x%x\n",
+ pClientList[i]);
+
+ }
+
+ if (numClients != 0)
+ {
+ pRmApi->FreeClientList(pRmApi, pClientList, numClients);
+
+ portMemFree(pClientList);
+ }
+
+ // Clean up any remaining events using this nvfp.
+ free_os_events(nvfp, 0);
+}
+
+static void RmUnbindLock(
+ nv_state_t *nv
+)
+{
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ if ((pGpu == NULL) || (gpuGetUserClientCount(pGpu) == 0))
+ {
+ nv->flags |= NV_FLAG_UNBIND_LOCK;
+ }
+}
+
+static NV_STATUS allocate_os_event(
+ NvHandle hParent,
+ nv_file_private_t *nvfp,
+ NvU32 fd
+)
+{
+ nv_state_t *nv = nv_get_ctl_state();
+ NvU32 status = NV_OK;
+ nv_event_t *event;
+
+ nv_event_t *new_event = portMemAllocNonPaged(sizeof(nv_event_t));
+ if (new_event == NULL)
+ {
+ status = NV_ERR_NO_MEMORY;
+ goto done;
+ }
+
+ portSyncSpinlockAcquire(nv->event_spinlock);
+ for (event = nv->event_list; event; event = event->next)
+ {
+ // Only one event may be associated with a given fd.
+ if (event->hParent == hParent && event->fd == fd)
+ {
+ status = NV_ERR_INVALID_ARGUMENT;
+ portSyncSpinlockRelease(nv->event_spinlock);
+ goto done;
+ }
+ }
+
+ new_event->next = nv->event_list;
+ nv->event_list = new_event;
+ portSyncSpinlockRelease(nv->event_spinlock);
+
+done:
+ if (status == NV_OK)
+ {
+ new_event->hParent = hParent;
+ new_event->nvfp = nvfp;
+ new_event->fd = fd;
+ new_event->active = NV_TRUE;
+ new_event->refcount = 0;
+
+ NV_PRINTF(LEVEL_INFO, "allocated OS event:\n");
+ NV_PRINTF(LEVEL_INFO, " hParent: 0x%x\n", hParent);
+ NV_PRINTF(LEVEL_INFO, " fd: %d\n", fd);
+ }
+ else
+ {
+ portMemFree(new_event);
+ }
+
+ return status;
+}
+
+NV_STATUS RmAllocOsEvent(
+ NvHandle hParent,
+ nv_file_private_t *nvfp,
+ NvU32 fd
+)
+{
+ if (NV_OK != allocate_os_event(hParent, nvfp, fd))
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to allocate OS event\n");
+ return NV_ERR_INSUFFICIENT_RESOURCES;
+ }
+ return NV_OK;
+}
+
+static NV_STATUS free_os_event(
+ NvHandle hParent,
+ NvU32 fd
+)
+{
+ nv_state_t *nv = nv_get_ctl_state();
+ nv_event_t *event, *tmp;
+ NV_STATUS result;
+
+ portSyncSpinlockAcquire(nv->event_spinlock);
+ tmp = event = nv->event_list;
+ while (event)
+ {
+ if ((event->fd == fd) && (event->hParent == hParent))
+ {
+ if (event == nv->event_list)
+ nv->event_list = event->next;
+ else
+ tmp->next = event->next;
+ break;
+ }
+ tmp = event;
+ event = event->next;
+ }
+
+ if (event != NULL)
+ {
+ free_os_event_under_lock(event);
+ result = NV_OK;
+ }
+ else
+ result = NV_ERR_INVALID_EVENT;
+ portSyncSpinlockRelease(nv->event_spinlock);
+
+ if (result == NV_OK)
+ {
+ NV_PRINTF(LEVEL_INFO, "freed OS event:\n");
+ NV_PRINTF(LEVEL_INFO, " hParent: 0x%x\n", hParent);
+ NV_PRINTF(LEVEL_INFO, " fd: %d\n", fd);
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to find OS event:\n");
+ NV_PRINTF(LEVEL_ERROR, " hParent: 0x%x\n", hParent);
+ NV_PRINTF(LEVEL_ERROR, " fd: %d\n", fd);
+ }
+
+ return result;
+}
+
+NV_STATUS RmFreeOsEvent(
+ NvHandle hParent,
+ NvU32 fd
+)
+{
+ if (NV_OK != free_os_event(hParent, fd))
+ {
+ return NV_ERR_INVALID_EVENT;
+ }
+ return NV_OK;
+}
+
+static void RmExecuteWorkItem(
+ void *pWorkItem
+)
+{
+ nv_work_item_t *pWi = (nv_work_item_t *)pWorkItem;
+ NvU32 gpuMask;
+ NvU32 releaseLocks = 0;
+
+ if (!(pWi->flags & NV_WORK_ITEM_FLAGS_REQUIRES_GPU) &&
+ ((pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW) ||
+ (pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW) ||
+ (pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW) ||
+ (pWi->flags & OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY)))
+ {
+ // Requesting one of the GPU locks without providing a GPU instance
+ NV_ASSERT(0);
+ goto done;
+ }
+
+ // Get locks requested by workitem
+ if (NV_OK != workItemLocksAcquire(pWi->gpuInstance, pWi->flags,
+ &releaseLocks, &gpuMask))
+ {
+ goto done;
+ }
+
+ // Some work items may not require a valid GPU instance
+ if (pWi->flags & NV_WORK_ITEM_FLAGS_REQUIRES_GPU)
+ {
+ pWi->func.pGpuFunction(pWi->gpuInstance, pWi->pData);
+ }
+ else
+ {
+ pWi->func.pSystemFunction(pWi->pData);
+ }
+
+ // Release any locks taken
+ workItemLocksRelease(releaseLocks, gpuMask);
+
+done:
+ if ((pWi->pData != NULL) &&
+ !(pWi->flags & NV_WORK_ITEM_FLAGS_DONT_FREE_DATA))
+ {
+ portMemFree(pWi->pData);
+ }
+
+ portMemFree((void *)pWi);
+}
+
+static NV_STATUS RmGetEventData(
+ nv_file_private_t *nvfp,
+ NvP64 pEvent,
+ NvU32 *MoreEvents,
+ NvBool bUserModeArgs
+)
+{
+ NV_STATUS RmStatus;
+ NvUnixEvent *pKernelEvent = NULL;
+ nv_event_t nv_event;
+ RMAPI_PARAM_COPY paramCopy;
+
+ RmStatus = nv_get_event(nvfp, &nv_event, MoreEvents);
+ if (RmStatus != NV_OK)
+ return NV_ERR_OPERATING_SYSTEM;
+
+ // setup for access to client's parameters
+ RMAPI_PARAM_COPY_INIT(paramCopy, pKernelEvent, pEvent, 1, sizeof(NvUnixEvent));
+ RmStatus = rmapiParamsAcquire(&paramCopy, bUserModeArgs);
+ if (RmStatus != NV_OK)
+ return NV_ERR_OPERATING_SYSTEM;
+
+ pKernelEvent->hObject = nv_event.hObject;
+ pKernelEvent->NotifyIndex = nv_event.index;
+ pKernelEvent->info32 = nv_event.info32;
+ pKernelEvent->info16 = nv_event.info16;
+
+ // release client buffer access, with copyout as needed
+ if (rmapiParamsRelease(&paramCopy) != NV_OK)
+ return NV_ERR_OPERATING_SYSTEM;
+
+ return NV_OK;
+}
+
+static NV_STATUS RmAccessRegistry(
+ NvHandle hClient,
+ NvHandle hObject,
+ NvU32 AccessType,
+ NvP64 clientDevNodeAddress,
+ NvU32 DevNodeLength,
+ NvP64 clientParmStrAddress,
+ NvU32 ParmStrLength,
+ NvP64 clientBinaryDataAddress,
+ NvU32 *pBinaryDataLength,
+ NvU32 *Data,
+ NvU32 *Entry
+)
+{
+ NvU32 gpuMask = 0, gpuInstance = 0;
+ OBJGPU *pGpu;
+ NvBool isDevice = NV_FALSE;
+ NV_STATUS RmStatus = NV_ERR_OPERATING_SYSTEM;
+ RsClient *pClient;
+ Device *pDevice;
+ Subdevice *pSubdevice;
+
+ RMAPI_PARAM_COPY devNodeParamCopy;
+ NvU8 *tmpDevNode = NULL;
+ NvU32 copyOutDevNodeLength = 0;
+
+ RMAPI_PARAM_COPY parmStrParamCopy;
+ char *tmpParmStr = NULL;
+ NvU32 copyOutParmStrLength = 0;
+
+ RMAPI_PARAM_COPY binaryDataParamCopy;
+ NvU8 *tmpBinaryData = NULL;
+ NvU32 BinaryDataLength = 0;
+ NvU32 copyOutBinaryDataLength = 0;
+
+ if (NV_OK != serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClient))
+ return NV_ERR_INVALID_CLIENT;
+
+ if (hClient == hObject)
+ {
+ pGpu = NULL;
+ }
+ else
+ {
+ RmStatus = deviceGetByHandle(pClient, hObject, &pDevice);
+ if (RmStatus != NV_OK)
+ {
+ RmStatus = subdeviceGetByHandle(pClient, hObject, &pSubdevice);
+ if (RmStatus != NV_OK)
+ goto done;
+
+ RmStatus = rmGpuGroupLockAcquire(pSubdevice->subDeviceInst,
+ GPU_LOCK_GRP_SUBDEVICE,
+ GPUS_LOCK_FLAGS_NONE,
+ RM_LOCK_MODULES_GPU,
+ &gpuMask);
+ if (RmStatus != NV_OK)
+ return RmStatus;
+
+ GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
+ pGpu = GPU_RES_GET_GPU(pSubdevice);
+ }
+ else
+ {
+ RmStatus = rmGpuGroupLockAcquire(pDevice->deviceInst,
+ GPU_LOCK_GRP_DEVICE,
+ GPUS_LOCK_FLAGS_NONE,
+ RM_LOCK_MODULES_GPU,
+ &gpuMask);
+ if (RmStatus != NV_OK)
+ return RmStatus;
+
+ GPU_RES_SET_THREAD_BC_STATE(pDevice);
+ pGpu = GPU_RES_GET_GPU(pDevice);
+ isDevice = NV_TRUE;
+ }
+ }
+
+ if (pBinaryDataLength)
+ {
+ BinaryDataLength = *pBinaryDataLength;
+ }
+
+ // a passed-in devNode
+ if (DevNodeLength)
+ {
+ // the passed-in DevNodeLength does not account for '\0'
+ DevNodeLength++;
+
+ if (DevNodeLength > NVOS38_MAX_REGISTRY_STRING_LENGTH)
+ {
+ RmStatus = NV_ERR_INVALID_STRING_LENGTH;
+ goto done;
+ }
+
+ // get access to client's DevNode
+ RMAPI_PARAM_COPY_INIT(devNodeParamCopy, tmpDevNode, clientDevNodeAddress, DevNodeLength, 1);
+ devNodeParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_ZERO_BUFFER;
+ RmStatus = rmapiParamsAcquire(&devNodeParamCopy, NV_TRUE);
+ if (RmStatus != NV_OK)
+ {
+ RmStatus = NV_ERR_OPERATING_SYSTEM;
+ goto done;
+ }
+ }
+
+ // a passed-in parmStr
+ if (ParmStrLength)
+ {
+ // the passed-in ParmStrLength does not account for '\0'
+ ParmStrLength++;
+
+ if ((ParmStrLength == 0) || (ParmStrLength > NVOS38_MAX_REGISTRY_STRING_LENGTH))
+ {
+ RmStatus = NV_ERR_INVALID_STRING_LENGTH;
+ goto done;
+ }
+ // get access to client's parmStr
+ RMAPI_PARAM_COPY_INIT(parmStrParamCopy, tmpParmStr, clientParmStrAddress, ParmStrLength, 1);
+ RmStatus = rmapiParamsAcquire(&parmStrParamCopy, NV_TRUE);
+ if (RmStatus != NV_OK)
+ {
+ RmStatus = NV_ERR_OPERATING_SYSTEM;
+ goto done;
+ }
+ if (tmpParmStr[ParmStrLength - 1] != '\0')
+ {
+ RmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+ }
+
+ if ((AccessType == NVOS38_ACCESS_TYPE_READ_BINARY) ||
+ (AccessType == NVOS38_ACCESS_TYPE_WRITE_BINARY))
+ {
+ if ((BinaryDataLength > NVOS38_MAX_REGISTRY_BINARY_LENGTH) ||
+ (BinaryDataLength == 0))
+ {
+ RmStatus = NV_ERR_INVALID_STRING_LENGTH;
+ goto done;
+ }
+
+ // get access to client's binaryData
+ RMAPI_PARAM_COPY_INIT(binaryDataParamCopy, tmpBinaryData, clientBinaryDataAddress, BinaryDataLength, 1);
+ if (AccessType == NVOS38_ACCESS_TYPE_READ_BINARY)
+ binaryDataParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYIN;
+ RmStatus = rmapiParamsAcquire(&binaryDataParamCopy, NV_TRUE);
+ if (RmStatus != NV_OK)
+ {
+ RmStatus = NV_ERR_OPERATING_SYSTEM;
+ goto done;
+ }
+ }
+
+ switch (AccessType)
+ {
+ case NVOS38_ACCESS_TYPE_READ_DWORD:
+ RmStatus = osReadRegistryDword(pGpu,
+ tmpParmStr, Data);
+ break;
+
+ case NVOS38_ACCESS_TYPE_WRITE_DWORD:
+ if (isDevice && osIsAdministrator())
+ {
+ while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+ {
+ RmStatus = osWriteRegistryDword(pGpu,
+ tmpParmStr, *Data);
+
+ if (RmStatus != NV_OK)
+ goto done;
+ }
+ break;
+ }
+
+ RmStatus = osWriteRegistryDword(pGpu,
+ tmpParmStr, *Data);
+ break;
+
+ case NVOS38_ACCESS_TYPE_READ_BINARY:
+ RmStatus = osReadRegistryBinary(pGpu,
+ tmpParmStr, tmpBinaryData, &BinaryDataLength);
+
+ if (RmStatus != NV_OK)
+ {
+ goto done;
+ }
+
+ if (BinaryDataLength)
+ copyOutBinaryDataLength = BinaryDataLength;
+
+ break;
+
+ case NVOS38_ACCESS_TYPE_WRITE_BINARY:
+ if (isDevice && osIsAdministrator())
+ {
+ while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+ {
+ RmStatus = osWriteRegistryBinary(pGpu,
+ tmpParmStr, tmpBinaryData,
+ BinaryDataLength);
+
+ if (RmStatus != NV_OK)
+ goto done;
+ }
+ break;
+ }
+
+ RmStatus = osWriteRegistryBinary(pGpu,
+ tmpParmStr, tmpBinaryData,
+ BinaryDataLength);
+ break;
+
+ default:
+ RmStatus = NV_ERR_INVALID_ACCESS_TYPE;
+ }
+
+ done:
+ if (gpuMask != 0)
+ rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+
+ if (tmpDevNode != NULL)
+ {
+ // skip copyout on error
+ if ((RmStatus != NV_OK) || (copyOutDevNodeLength == 0))
+ devNodeParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
+ devNodeParamCopy.paramsSize = copyOutDevNodeLength;
+ if (NV_OK != rmapiParamsRelease(&devNodeParamCopy))
+ if (RmStatus == NV_OK)
+ RmStatus = NV_ERR_OPERATING_SYSTEM;
+ }
+ if (tmpParmStr != NULL)
+ {
+ // skip copyout on error
+ if ((RmStatus != NV_OK) || (copyOutParmStrLength == 0))
+ parmStrParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
+ parmStrParamCopy.paramsSize = copyOutParmStrLength;
+ if (NV_OK != rmapiParamsRelease(&parmStrParamCopy))
+ if (RmStatus == NV_OK)
+ RmStatus = NV_ERR_OPERATING_SYSTEM;
+ }
+ if (tmpBinaryData != NULL)
+ {
+ // skip copyout on error
+ if ((RmStatus != NV_OK) || (copyOutBinaryDataLength == 0))
+ binaryDataParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
+ binaryDataParamCopy.paramsSize = copyOutBinaryDataLength;
+ if (NV_OK != rmapiParamsRelease(&binaryDataParamCopy))
+ if (RmStatus == NV_OK)
+ RmStatus = NV_ERR_OPERATING_SYSTEM;
+ *pBinaryDataLength = copyOutBinaryDataLength;
+ }
+
+ serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClient);
+ return RmStatus;
+}
+
+static NV_STATUS RmUpdateDeviceMappingInfo(
+ NvHandle hClient,
+ NvHandle hDevice,
+ NvHandle hMappable,
+ void *pOldCpuAddress,
+ void *pNewCpuAddress
+)
+{
+ NV_STATUS status;
+ RsClient *pClient;
+ RsResourceRef *pMappableRef;
+ RsCpuMapping *pCpuMapping;
+ Device *pDevice;
+ Subdevice *pSubdevice;
+ NvU32 gpuMask = 0;
+
+ status = serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClient);
+ if (status != NV_OK)
+ return status;
+
+ status = deviceGetByHandle(pClient, hDevice, &pDevice);
+ if (status != NV_OK)
+ {
+ status = subdeviceGetByHandle(pClient, hDevice, &pSubdevice);
+ if (status != NV_OK)
+ goto done;
+
+ status = rmGpuGroupLockAcquire(pSubdevice->subDeviceInst,
+ GPU_LOCK_GRP_SUBDEVICE,
+ GPUS_LOCK_FLAGS_NONE,
+ RM_LOCK_MODULES_GPU,
+ &gpuMask);
+ if (status != NV_OK)
+ goto done;
+
+ GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
+ }
+ else
+ {
+ status = rmGpuGroupLockAcquire(pDevice->deviceInst,
+ GPU_LOCK_GRP_DEVICE,
+ GPUS_LOCK_FLAGS_NONE,
+ RM_LOCK_MODULES_GPU,
+ &gpuMask);
+ if (status != NV_OK)
+ goto done;
+
+ GPU_RES_SET_THREAD_BC_STATE(pDevice);
+ }
+
+ status = clientGetResourceRef(pClient, hMappable, &pMappableRef);
+ if (status != NV_OK)
+ goto done;
+
+ if ((objDynamicCastById(pMappableRef->pResource, classId(Memory)) == NULL) &&
+ (objDynamicCastById(pMappableRef->pResource, classId(KernelChannel)) == NULL))
+ {
+ status = NV_ERR_INVALID_OBJECT_HANDLE;
+ goto done;
+ }
+
+ status = refFindCpuMappingWithFilter(pMappableRef,
+ NV_PTR_TO_NvP64(pOldCpuAddress),
+ serverutilMappingFilterCurrentUserProc,
+ &pCpuMapping);
+ if (status != NV_OK)
+ goto done;
+
+ pCpuMapping->pLinearAddress = NV_PTR_TO_NvP64(pNewCpuAddress);
+
+done:
+
+ if (gpuMask != 0)
+ rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+
+ serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClient);
+ return status;
+}
+
+static NV_STATUS RmPerformVersionCheck(
+ void *pData,
+ NvU32 dataSize
+)
+{
+ nv_ioctl_rm_api_version_t *pParams;
+ char clientCh, rmCh;
+ const char *rmStr = NV_VERSION_STRING;
+ NvBool relaxed = NV_FALSE;
+ NvU32 i;
+
+ if (dataSize != sizeof(nv_ioctl_rm_api_version_t))
+ return NV_ERR_INVALID_ARGUMENT;
+
+ pParams = pData;
+
+ //
+ // write the reply value, so that the client knows we recognized
+ // the request
+ //
+ pParams->reply = NV_RM_API_VERSION_REPLY_RECOGNIZED;
+
+ //
+ // the client requested to override the version check; just return
+ // success.
+ //
+ if (pParams->cmd == NV_RM_API_VERSION_CMD_OVERRIDE)
+ {
+ return NV_OK;
+ }
+
+ //
+ // the client requested relaxed version checking; we will only
+ // compare the strings until the first decimal point.
+ //
+ if (pParams->cmd == NV_RM_API_VERSION_CMD_RELAXED)
+ {
+ relaxed = NV_TRUE;
+ }
+
+ //
+ // rmStr (i.e., NV_VERSION_STRING) must be null-terminated and fit within
+ // NV_RM_API_VERSION_STRING_LENGTH, so that:
+ //
+ // (1) If the versions don't match, we can return rmStr in
+ // pParams->versionString.
+ // (2) The below loop is guaranteed to not overrun rmStr.
+ //
+ if ((os_string_length(rmStr) + 1) > NV_RM_API_VERSION_STRING_LENGTH)
+ {
+ return NV_ERR_BUFFER_TOO_SMALL;
+ }
+
+ for (i = 0; i < NV_RM_API_VERSION_STRING_LENGTH; i++)
+ {
+ clientCh = pParams->versionString[i];
+ rmCh = rmStr[i];
+
+ //
+ // fail if the current character is not the same
+ //
+ if (clientCh != rmCh)
+ {
+ break;
+ }
+
+ //
+ // if relaxed matching was requested, succeed when we find the
+ // first decimal point
+ //
+ if ((relaxed) && (clientCh == '.'))
+ {
+ return NV_OK;
+ }
+
+ //
+ // we found the end of the strings: succeed
+ //
+ if (clientCh == '\0')
+ {
+ return NV_OK;
+ }
+ }
+
+ //
+ // the version strings did not match: print an error message and
+ // copy the RM's version string into pParams->versionString, so
+ // that the client can report the mismatch; explicitly NULL
+ // terminate the client's string, since we cannot trust it
+ //
+ pParams->versionString[NV_RM_API_VERSION_STRING_LENGTH - 1] = '\0';
+
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: API mismatch: the client has the version %s, but\n"
+ "NVRM: this kernel module has the version %s. Please\n"
+ "NVRM: make sure that this kernel module and all NVIDIA driver\n"
+ "NVRM: components have the same version.\n",
+ pParams->versionString, NV_VERSION_STRING);
+
+ os_string_copy(pParams->versionString, rmStr);
+
+ return NV_ERR_GENERIC;
+}
+
+//
+// Check if the NVPCF _DSM functions are implemented under
+// NVPCF scope or GPU device scope.
+// As part of RM initialisation this function checks the
+// support of NVPCF _DSM function implementation under
+// NVPCF scope, in case that fails, clear the cached DSM
+// support status and retry the NVPCF _DSM function under
+// GPU scope.
+//
+static void RmCheckNvpcfDsmScope(
+ OBJGPU *pGpu
+)
+{
+ NvU32 supportedFuncs;
+ NvU16 dsmDataSize = sizeof(supportedFuncs);
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ ACPI_DSM_FUNCTION acpiDsmFunction = ACPI_DSM_FUNCTION_NVPCF_2X;
+ NvU32 acpiDsmSubFunction = NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_SUPPORTED;
+
+ nv->nvpcf_dsm_in_gpu_scope = NV_FALSE;
+
+ if ((osCallACPI_DSM(pGpu, acpiDsmFunction, acpiDsmSubFunction,
+ &supportedFuncs, &dsmDataSize) != NV_OK) ||
+ (FLD_TEST_DRF(PCF0100, _CTRL_CONFIG_DSM,
+ _FUNC_GET_SUPPORTED_IS_SUPPORTED, _NO, supportedFuncs)) ||
+ (dsmDataSize != sizeof(supportedFuncs)))
+ {
+ nv->nvpcf_dsm_in_gpu_scope = NV_TRUE;
+
+ // clear cached DSM function status
+ uncacheDsmFuncStatus(pGpu, acpiDsmFunction, acpiDsmSubFunction);
+ }
+}
+
+NV_STATUS RmPowerSourceChangeEvent(
+ nv_state_t *pNv,
+ NvU32 event_val
+)
+{
+ NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS params = {0};
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+
+ params.powerStateInfo.powerState = event_val ? NV2080_CTRL_PERF_POWER_SOURCE_BATTERY :
+ NV2080_CTRL_PERF_POWER_SOURCE_AC;
+
+ return pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+ pNv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_PERF_SET_POWERSTATE,
+ &params, sizeof(params));
+}
+
+/*!
+ * @brief Deal with D-notifier events to apply a performance
+ * level based on the requested auxiliary power-state.
+ * Read confluence page "D-Notifiers on Linux" for more details.
+ *
+ * @param[in] pGpu OBJGPU pointer.
+ * @param[in] event_type NvU32 Event type.
+ */
+static void RmHandleDNotifierEvent(
+ nv_state_t *pNv,
+ NvU32 event_type
+)
+{
+ NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS params = { 0 };
+ RM_API *pRmApi;
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_OK;
+
+ switch (event_type)
+ {
+ case ACPI_NOTIFY_POWER_LEVEL_D1:
+ params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P0;
+ break;
+ case ACPI_NOTIFY_POWER_LEVEL_D2:
+ params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P1;
+ break;
+ case ACPI_NOTIFY_POWER_LEVEL_D3:
+ params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P2;
+ break;
+ case ACPI_NOTIFY_POWER_LEVEL_D4:
+ params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P3;
+ break;
+ case ACPI_NOTIFY_POWER_LEVEL_D5:
+ params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P4;
+ break;
+ default:
+ return;
+ }
+
+ pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_ACPI);
+ if (pRmApi == NULL)
+ {
+ return;
+ }
+
+ rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+ pNv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_PERF_SET_AUX_POWER_STATE,
+ &params, sizeof(params));
+
+ RmUnixRmApiEpilogue(pNv, &threadState);
+
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: Failed to handle ACPI D-Notifier event, status=0x%x\n",
+ __FUNCTION__, rmStatus);
+ }
+}
+
+static NV_STATUS
+RmDmabufVerifyMemHandle(
+ OBJGPU *pGpu,
+ NvHandle hSrcClient,
+ NvHandle hMemory,
+ NvU64 offset,
+ NvU64 size,
+ void *pGpuInstanceInfo
+)
+{
+ NV_STATUS status;
+ RsClient *pClient = NULL;
+ RsResourceRef *pSrcMemoryRef = NULL;
+ Memory *pSrcMemory = NULL;
+ MEMORY_DESCRIPTOR *pMemDesc = NULL;
+
+ NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hSrcClient, &pClient));
+
+ status = clientGetResourceRef(pClient, hMemory, &pSrcMemoryRef);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ pSrcMemory = dynamicCast(pSrcMemoryRef->pResource, Memory);
+ if (pSrcMemory == NULL)
+ {
+ return NV_ERR_INVALID_OBJECT;
+ }
+
+ pMemDesc = pSrcMemory->pMemDesc;
+
+ if (pGpuInstanceInfo != NULL)
+ {
+ KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance;
+ pKernelMIGGpuInstance = (KERNEL_MIG_GPU_INSTANCE *) pGpuInstanceInfo;
+
+ if ((pKernelMIGGpuInstance->pMemoryPartitionHeap != pSrcMemory->pHeap))
+ {
+ return NV_ERR_INVALID_OBJECT_PARENT;
+ }
+ }
+
+ // Check if hMemory belongs to the same pGpu
+ if ((pMemDesc->pGpu != pGpu) &&
+ (pSrcMemory->pGpu != pGpu))
+ {
+ return NV_ERR_INVALID_OBJECT_PARENT;
+ }
+
+ // Offset and size must be aligned to OS page-size
+ if (!NV_IS_ALIGNED64(offset, os_page_size) ||
+ !NV_IS_ALIGNED64(size, os_page_size))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ // Only supported for vidmem handles
+ if (memdescGetAddressSpace(pMemDesc) != ADDR_FBMEM)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ if ((size == 0) ||
+ (size > memdescGetSize(pMemDesc)) ||
+ (offset > (memdescGetSize(pMemDesc) - size)))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ return NV_OK;
+}
+
+static NV_STATUS
+RmDmabufGetClientAndDevice(
+ OBJGPU *pGpu,
+ NvHandle hClient,
+ NvHandle *phClient,
+ NvHandle *phDevice,
+ NvHandle *phSubdevice,
+ void **ppGpuInstanceInfo
+)
+{
+ MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+
+ if (IS_MIG_ENABLED(pGpu))
+ {
+ NV_STATUS status;
+ MIG_INSTANCE_REF ref;
+ KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
+
+ status = kmigmgrGetInstanceRefFromClient(pGpu, pKernelMIGManager,
+ hClient, &ref);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ status = kmigmgrIncRefCount(ref.pKernelMIGGpuInstance->pShare);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ *phClient = ref.pKernelMIGGpuInstance->instanceHandles.hClient;
+ *phDevice = ref.pKernelMIGGpuInstance->instanceHandles.hDevice;
+ *phSubdevice = ref.pKernelMIGGpuInstance->instanceHandles.hSubdevice;
+ *ppGpuInstanceInfo = (void *) ref.pKernelMIGGpuInstance;
+
+ return NV_OK;
+ }
+
+ *phClient = pMemoryManager->hClient;
+ *phDevice = pMemoryManager->hDevice;
+ *phSubdevice = pMemoryManager->hSubdevice;
+ *ppGpuInstanceInfo = NULL;
+
+ return NV_OK;
+}
+
+static void
+RmDmabufPutClientAndDevice(
+ OBJGPU *pGpu,
+ NvHandle hClient,
+ NvHandle hDevice,
+ NvHandle hSubdevice,
+ void *pGpuInstanceInfo
+)
+{
+ KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance;
+
+ if (pGpuInstanceInfo == NULL)
+ {
+ return;
+ }
+
+ pKernelMIGGpuInstance = (KERNEL_MIG_GPU_INSTANCE *) pGpuInstanceInfo;
+
+ NV_ASSERT_OK(kmigmgrDecRefCount(pKernelMIGGpuInstance->pShare));
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ *
+ * The routines below are part of the interface between the kernel interface
+ * layer and the kernel-agnostic portions of the resource manager.
+ *
+ * ---------------------------------------------------------------------------
+ */
+
+NvBool NV_API_CALL rm_init_private_state(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ NvBool retval;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ retval = RmInitPrivateState(pNv);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return retval;
+}
+
+void NV_API_CALL rm_free_private_state(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ RmFreePrivateState(pNv);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NvBool NV_API_CALL rm_init_adapter(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ THREAD_STATE_NODE threadState;
+ NvBool retval = NV_FALSE;
+ void *fp;
+ NvBool bEnabled;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_DEVICE_INIT);
+
+ // LOCK: acquire API lock
+ if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK)
+ {
+ if (!((gpumgrQueryGpuDrainState(pNv->gpu_id, &bEnabled, NULL) == NV_OK)
+ && bEnabled))
+ {
+ if (pNv->flags & NV_FLAG_PERSISTENT_SW_STATE)
+ {
+ retval = RmPartiallyInitAdapter(pNv);
+ }
+ else
+ {
+ retval = RmInitAdapter(pNv);
+ }
+ }
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return retval;
+}
+
+void NV_API_CALL rm_disable_adapter(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_ASSERT_OK(os_flush_work_queue(pNv->queue));
+
+ // LOCK: acquire API lock
+ if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+ {
+ if (pNv->flags & NV_FLAG_PERSISTENT_SW_STATE)
+ {
+ RmPartiallyDisableAdapter(pNv);
+ }
+ else
+ {
+ RmDisableAdapter(pNv);
+ }
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ NV_ASSERT_OK(os_flush_work_queue(pNv->queue));
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_shutdown_adapter(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+ {
+ RmShutdownAdapter(pNv);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_exclude_adapter(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ NV_STATUS rmStatus;
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ rmStatus = RmExcludeAdapter(pNv);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_acquire_api_lock(
+ nvidia_stack_t *sp
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_release_api_lock(
+ nvidia_stack_t *sp
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_acquire_gpu_lock(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire GPU lock
+ rmStatus = rmDeviceGpuLocksAcquire(NV_GET_NV_PRIV_PGPU(nv),
+ GPUS_LOCK_FLAGS_NONE,
+ RM_LOCK_MODULES_OSAPI);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_release_gpu_lock(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // UNLOCK: release GPU lock
+ rmDeviceGpuLocksRelease(NV_GET_NV_PRIV_PGPU(nv), GPUS_LOCK_FLAGS_NONE, NULL);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_acquire_all_gpus_lock(
+ nvidia_stack_t *sp
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire all GPUs lock
+ rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_release_all_gpus_lock(
+ nvidia_stack_t *sp
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // UNLOCK: release all GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return NV_OK;
+}
+
+/*!
+ * @brief Handle ACPI_NOTIFY_GPS_STATUS_CHANGE event.
+ *
+ * This function is called for GPS when SBIOS trigger
+ * gps STATUS_CHANGE event, which calls rm control call
+ * NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL to init the GPS
+ * data from SBIOS.
+ */
+static void RmHandleGPSStatusChange
+(
+ nv_state_t *pNv
+)
+{
+}
+
+/*!
+ * @brief Function to handle device specific ACPI events.
+ *
+ * @param[in] sp nvidia_stack_t pointer.
+ * @param[in] nv nv_state_t pointer.
+ * @param[in] event_type NvU32 Event type.
+ */
+void NV_API_CALL rm_acpi_notify(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvU32 event_type
+)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ switch (event_type)
+ {
+ case ACPI_VIDEO_NOTIFY_PROBE:
+ {
+ THREAD_STATE_NODE threadState;
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ RmHandleDisplayChange(sp, nv);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ break;
+ }
+
+ case ACPI_NOTIFY_GPS_STATUS_CHANGE:
+ RmHandleGPSStatusChange(nv);
+ break;
+
+ case ACPI_NOTIFY_POWER_LEVEL_D1: /* fallthrough */
+ case ACPI_NOTIFY_POWER_LEVEL_D2: /* fallthrough */
+ case ACPI_NOTIFY_POWER_LEVEL_D3: /* fallthrough */
+ case ACPI_NOTIFY_POWER_LEVEL_D4: /* fallthrough */
+ case ACPI_NOTIFY_POWER_LEVEL_D5:
+ RmHandleDNotifierEvent(nv, event_type);
+ break;
+
+ default:
+ NV_PRINTF(LEVEL_INFO, "No support for 0x%x event\n", event_type);
+ NV_ASSERT(0);
+ break;
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+static void nv_align_mmap_offset_length(
+ nv_usermap_access_params_t *nvuap)
+{
+ NvU64 page_size = os_page_size;
+ NvU64 end = nvuap->size + (nvuap->addr & (page_size - 1));
+
+ nvuap->mmap_start = NV_ALIGN_DOWN(nvuap->addr, page_size);
+ nvuap->mmap_size = NV_ALIGN_UP(end, page_size);
+ nvuap->offset = NV_ALIGN_DOWN(nvuap->offset, page_size);
+}
+
+static inline NV_STATUS RmGetArrayMinMax(
+ NvU64 *array,
+ NvU64 count,
+ NvU64 *min,
+ NvU64 *max
+)
+{
+ NvU64 i;
+
+ if (array == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ *min = array[0];
+ *max = array[0];
+
+ if (count == 1)
+ return NV_OK;
+
+ for (i = 1; i < count; i++)
+ {
+ if (array[i] > *max)
+ *max = array[i];
+
+ if (array[i] < *min)
+ *min = array[i];
+ }
+
+ return NV_OK;
+}
+
+static NV_STATUS RmSetUserMapAccessRange(
+ nv_usermap_access_params_t *nvuap
+)
+{
+ NV_STATUS status = NV_OK;
+
+ if (nvuap->contig)
+ {
+ nvuap->access_start = nvuap->mmap_start;
+ nvuap->access_size = nvuap->mmap_size;
+ }
+ else
+ {
+ NvU64 highest_address_mapped;
+ NvU64 lowest_address_mapped;
+
+ status = RmGetArrayMinMax(nvuap->page_array, nvuap->num_pages,
+ &lowest_address_mapped,
+ &highest_address_mapped);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ nvuap->access_start = lowest_address_mapped;
+ nvuap->access_size = (highest_address_mapped + os_page_size) - lowest_address_mapped;
+ }
+
+ return status;
+}
+
+static NV_STATUS RmGetAllocPrivate(NvU32, NvU32, NvU64, NvU64, NvU32 *, void **,
+ NvU64 *);
+static NV_STATUS RmValidateMmapRequest(nv_state_t *, NvU64, NvU64, NvU32 *);
+
+static NV_STATUS RmGetMmapPteArray(
+ KernelMemorySystem *pKernelMemorySystem,
+ NvHandle hClient,
+ NvHandle hMemory,
+ nv_usermap_access_params_t *nvuap
+)
+{
+ NV_STATUS status = NV_OK;
+ NvU64 pages = 0;
+ Memory *pMemory = NULL;
+ PMEMORY_DESCRIPTOR pMemDesc = NULL;
+ RsResourceRef *pResourceRef;
+ NvU64 i;
+ NvU64 *pteArray;
+ NvU64 index;
+
+ if (nvuap == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ //
+ // If we're mapping a memory handle, we can get the data from the
+ // descriptor (required if the allocation being mapped is discontiguous).
+ //
+ if (serverutilGetResourceRefWithType(hClient, hMemory, classId(Memory),
+ &pResourceRef) == NV_OK)
+ {
+ pMemory = dynamicCast(pResourceRef->pResource, Memory);
+ pMemDesc = pMemory->pMemDesc;
+ nvuap->contig = memdescGetContiguity(pMemDesc, AT_CPU);
+ }
+
+ //
+ // In the discontiguous case, the page array needs to be large enough to hold
+ // the 4K-page-sized addresses that we will then deflate to OS page addresses.
+ // For the contiguous case, we can create the OS-page-sized addresses
+ // directly in the array.
+ //
+ if (nvuap->contig)
+ {
+ pages = nvuap->mmap_size / os_page_size;
+ }
+ else
+ {
+ pages = nvuap->mmap_size / NV_RM_PAGE_SIZE;
+ }
+
+ NV_ASSERT_OR_RETURN(pages != 0, NV_ERR_INVALID_ARGUMENT);
+
+ status = os_alloc_mem((void **)&nvuap->page_array, (pages * sizeof(NvU64)));
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ if (!nvuap->contig)
+ {
+ pteArray = memdescGetPteArray(pMemDesc, AT_CPU);
+ index = nvuap->offset / NV_RM_PAGE_SIZE;
+
+ //
+ // We're guaranteed to have a MEMORY_DESCRIPTOR in the discontiguous
+ // case. Copy over the addresses now.
+ //
+ portMemCopy((void *)nvuap->page_array,
+ pages * sizeof(NvU64), (void *)&pteArray[index],
+ pages * sizeof(NvU64));
+
+ if (NV_RM_PAGE_SIZE < os_page_size)
+ {
+ RmDeflateRmToOsPageArray(nvuap->page_array, pages);
+ pages = NV_RM_PAGES_TO_OS_PAGES(pages);
+ }
+
+ //
+ // Convert the GPU physical addresses to system physical addresses,
+ // if applicable.
+ //
+ for (i = 0; i < pages; i++)
+ {
+ nvuap->page_array[i] += pKernelMemorySystem->coherentCpuFbBase;
+ }
+ }
+ else
+ {
+ // Offset is accounted in mmap_start.
+ for (nvuap->page_array[0] = nvuap->mmap_start, i = 1;
+ i < pages; i++)
+ {
+ nvuap->page_array[i] = nvuap->page_array[i-1] + os_page_size;
+ }
+ }
+
+ nvuap->num_pages = pages;
+
+ return status;
+}
+
+/* Must be called with the API lock and the GPU locks */
+static NV_STATUS RmCreateMmapContextLocked(
+ NvHandle hClient,
+ NvHandle hDevice,
+ NvHandle hMemory,
+ NvP64 address,
+ NvU64 size,
+ NvU64 offset,
+ NvU32 cachingType,
+ NvU32 fd
+)
+{
+ NV_STATUS status = NV_OK;
+ void *pAllocPriv = NULL;
+ OBJGPU *pGpu = NULL;
+ KernelMemorySystem *pKernelMemorySystem = NULL;
+ NvBool bCoherentAtsCpuOffset = NV_FALSE;
+ nv_state_t *pNv = NULL;
+ NvU64 addr = (NvU64)address;
+ NvU32 prot = 0;
+ NvU64 pageIndex = 0;
+ nv_usermap_access_params_t *nvuap = NULL;
+ NvBool bClientMap = (hClient == hDevice);
+
+ if (!bClientMap)
+ {
+ if (CliSetGpuContext(hClient, hDevice, &pGpu, NULL) != NV_OK)
+ {
+ NvU32 tmp;
+ if (CliSetSubDeviceContext(hClient, hDevice, &tmp, &pGpu) != NV_OK)
+ {
+ //
+ // If this mapping isn't for a GPU then we don't need to
+ // create a context for it.
+ //
+ return status;
+ }
+ }
+ }
+
+ status = os_alloc_mem((void**)&nvuap, sizeof(nv_usermap_access_params_t));
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ portMemSet(nvuap, 0, sizeof(nv_usermap_access_params_t));
+ nvuap->addr = addr;
+ nvuap->size = size;
+ nvuap->offset = offset;
+ nvuap->caching = cachingType;
+
+ //
+ // Assume the allocation is contiguous until RmGetMmapPteArray
+ // determines otherwise.
+ //
+ nvuap->contig = NV_TRUE;
+ nv_align_mmap_offset_length(nvuap);
+
+ if (pGpu != NULL)
+ {
+ pNv = NV_GET_NV_STATE(pGpu);
+ pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
+ bCoherentAtsCpuOffset = IS_COHERENT_CPU_ATS_OFFSET(pKernelMemorySystem, addr, size);
+ }
+
+ //
+ // If no device is given, or the address isn't in the given device's BARs,
+ // validate this as a system memory mapping and associate it with the
+ // control device.
+ //
+ if ((pNv == NULL) ||
+ (!IS_REG_OFFSET(pNv, addr, size) &&
+ !IS_FB_OFFSET(pNv, addr, size) &&
+ !bCoherentAtsCpuOffset &&
+ !IS_IMEM_OFFSET(pNv, addr, size)))
+ {
+ pNv = nv_get_ctl_state();
+
+ //
+ // Validate the mapping request by looking up the underlying sysmem
+ // allocation.
+ //
+ status = RmGetAllocPrivate(hClient, hMemory, addr, size, &prot, &pAllocPriv,
+ &pageIndex);
+
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+ }
+ else
+ {
+ //
+ // Validate the mapping request for ATS and get pteArray
+ //
+ if (bCoherentAtsCpuOffset)
+ {
+ status = RmGetMmapPteArray(pKernelMemorySystem, hClient, hMemory, nvuap);
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+ }
+
+ if (RmSetUserMapAccessRange(nvuap) != NV_OK)
+ {
+ goto done;
+ }
+
+ status = nv_get_usermap_access_params(pNv, nvuap);
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+
+ // Validate the mapping request for BAR's.
+ status = RmValidateMmapRequest(pNv, nvuap->access_start,
+ nvuap->access_size, &prot);
+ if (status != NV_OK)
+ {
+ goto done;
+ }
+ }
+
+ status = nv_add_mapping_context_to_file(pNv, nvuap, prot, pAllocPriv,
+ pageIndex, fd);
+
+done:
+ os_free_mem(nvuap);
+ return status;
+}
+
+// TODO: Bug 1802250: [uvm8] Use an alt stack in all functions in unix/src/osapi.c
+NV_STATUS rm_create_mmap_context(
+ NvHandle hClient,
+ NvHandle hDevice,
+ NvHandle hMemory,
+ NvP64 address,
+ NvU64 size,
+ NvU64 offset,
+ NvU32 cachingType,
+ NvU32 fd
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) == NV_OK)
+ {
+ RmClient *pClient;
+
+ if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClient))
+ return NV_ERR_INVALID_CLIENT;
+
+ if (pClient->ProcID != osGetCurrentProcess())
+ {
+ rmStatus = NV_ERR_INVALID_CLIENT;
+ }
+ // LOCK: acquire GPUs lock
+ else if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) == NV_OK)
+ {
+ rmStatus = RmCreateMmapContextLocked(hClient, hDevice, hMemory,
+ address, size, offset, cachingType, fd);
+ // UNLOCK: release GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+
+ serverutilReleaseClient(LOCK_ACCESS_READ, pClient);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ return rmStatus;
+}
+
+static NV_STATUS RmGetAllocPrivate(
+ NvU32 hClient,
+ NvU32 hMemory,
+ NvU64 offset,
+ NvU64 length,
+ NvU32 *pProtection,
+ void **ppPrivate,
+ NvU64 *pPageIndex
+)
+{
+ RmClient *pClient;
+ NV_STATUS rmStatus;
+ PMEMORY_DESCRIPTOR pMemDesc;
+ NvU32 pageOffset;
+ NvU64 pageCount;
+ NvU64 endingOffset;
+ RsResourceRef *pResourceRef;
+ RmResource *pRmResource;
+ void *pMemData;
+ NvBool bPeerIoMem;
+ NvBool bReadOnlyMem;
+ *pProtection = NV_PROTECT_READ_WRITE;
+ *ppPrivate = NULL;
+
+ pageOffset = (offset & ~os_page_mask);
+ offset &= os_page_mask;
+
+ NV_ASSERT_OR_RETURN(rmapiLockIsOwner(), NV_ERR_INVALID_LOCK_STATE);
+
+ if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClient))
+ return NV_ERR_INVALID_CLIENT;
+
+ rmStatus = clientGetResourceRef(staticCast(pClient, RsClient), hMemory, &pResourceRef);
+ if (rmStatus != NV_OK)
+ goto done;
+
+ pRmResource = dynamicCast(pResourceRef->pResource, RmResource);
+ if (!pRmResource)
+ {
+ rmStatus = NV_ERR_INVALID_OBJECT;
+ goto done;
+ }
+
+ rmStatus = rmresGetMemoryMappingDescriptor(pRmResource, &pMemDesc);
+ if (rmStatus != NV_OK)
+ goto done;
+
+ bReadOnlyMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY);
+ bPeerIoMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM);
+
+ if (!(pMemDesc->Allocated || bPeerIoMem))
+ {
+ rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+ goto done;
+ }
+
+ switch (memdescGetAddressSpace(pMemDesc))
+ {
+ case ADDR_SYSMEM:
+ break;
+ default:
+ rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+ goto done;
+ }
+
+ pMemData = memdescGetMemData(pMemDesc);
+ if (pMemData == NULL)
+ {
+ rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+ goto done;
+ }
+
+ rmStatus = os_match_mmap_offset(pMemData, offset, pPageIndex);
+ if (rmStatus != NV_OK)
+ goto done;
+
+ if (!portSafeAddU64(pageOffset, length, &endingOffset))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ pageCount = (endingOffset / os_page_size);
+
+ if (!portSafeAddU64(*pPageIndex + ((endingOffset % os_page_size) ? 1 : 0),
+ pageCount, &pageCount))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ if (pageCount > NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount))
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ if (bReadOnlyMem)
+ *pProtection = NV_PROTECT_READABLE;
+
+ *ppPrivate = pMemData;
+
+done:
+ serverutilReleaseClient(LOCK_ACCESS_READ, pClient);
+
+ return rmStatus;
+}
+
+static NV_STATUS RmValidateMmapRequest(
+ nv_state_t *pNv,
+ NvU64 offset,
+ NvU64 length,
+ NvU32 *pProtection
+)
+{
+ NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS params = { 0 };
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+ NV_STATUS status;
+
+ if (osIsAdministrator())
+ {
+ *pProtection = NV_PROTECT_READ_WRITE;
+ return NV_OK;
+ }
+
+ params.addressStart = offset;
+ params.addressLength = length;
+
+ status = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+ pNv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_GPU_VALIDATE_MEM_MAP_REQUEST,
+ &params, sizeof(params));
+
+ if (status == NV_OK)
+ {
+ *pProtection = params.protection;
+ }
+
+ return status;
+}
+
+NV_STATUS rm_get_adapter_status(
+ nv_state_t *pNv,
+ NvU32 *pStatus
+)
+{
+ NV_STATUS rmStatus = NV_ERR_OPERATING_SYSTEM;
+
+ // LOCK: acquire API lock
+ if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI) == NV_OK)
+ {
+ rmStatus = RmGetAdapterStatus(pNv, pStatus);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ return rmStatus;
+}
+
+NvBool NV_API_CALL rm_init_rm(
+ nvidia_stack_t *sp
+)
+{
+ NvBool retval;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ retval = RmInitRm();
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return retval;
+}
+
+void NV_API_CALL rm_shutdown_rm(
+ nvidia_stack_t *sp
+)
+{
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ RmShutdownRm();
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NvBool NV_API_CALL rm_init_event_locks(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ void *fp;
+ NvBool ret;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pNv->event_spinlock = portSyncSpinlockCreate(portMemAllocatorGetGlobalNonPaged());
+ ret = (pNv->event_spinlock != NULL);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return ret;
+}
+
+void NV_API_CALL rm_destroy_event_locks(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ if (pNv && pNv->event_spinlock)
+ portSyncSpinlockDestroy(pNv->event_spinlock);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_get_vbios_version(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ char *vbiosString
+)
+{
+ void *fp;
+ NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS *params;
+ RM_API *pRmApi;
+ THREAD_STATE_NODE threadState;
+ const size_t vbiosStringLen = 15; // "xx.xx.xx.xx.xx"
+
+ os_snprintf(vbiosString, vbiosStringLen, "??.??.??.??.??");
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ params = portMemAllocNonPaged(sizeof(*params));
+ if (params == NULL)
+ {
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return;
+ }
+
+ portMemSet(params, 0, sizeof(*params));
+
+ params->biosInfoList[0].index = NV2080_CTRL_BIOS_INFO_INDEX_REVISION;
+ params->biosInfoList[1].index = NV2080_CTRL_BIOS_INFO_INDEX_OEM_REVISION;
+ params->biosInfoListSize = 2;
+
+ pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_VBIOS);
+ if (pRmApi != NULL)
+ {
+ NV_STATUS rmStatus;
+
+ rmStatus = pRmApi->Control(pRmApi,
+ pNv->rmapi.hClient,
+ pNv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_BIOS_GET_INFO_V2,
+ params,
+ sizeof(*params));
+
+ if (rmStatus == NV_OK)
+ {
+ const NvU32 biosRevision = params->biosInfoList[0].data;
+ const NvU32 biosOEMRevision = params->biosInfoList[1].data;
+
+ os_snprintf(vbiosString, vbiosStringLen,
+ "%02x.%02x.%02x.%02x.%02x",
+ (biosRevision & 0xff000000) >> 24,
+ (biosRevision & 0x00ff0000) >> 16,
+ (biosRevision & 0x0000ff00) >> 8,
+ (biosRevision & 0x000000ff) >> 0,
+ biosOEMRevision);
+ }
+
+ RmUnixRmApiEpilogue(pNv, &threadState);
+ }
+
+ portMemFree(params);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_stop_user_channels(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS params = { 0 };
+ RM_API *pRmApi;
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_ERR_INVALID_STATE;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_FIFO);
+ if (pRmApi != NULL)
+ {
+ params.bDisable = NV_TRUE;
+ rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+ pNv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS,
+ &params, sizeof(params));
+
+ RmUnixRmApiEpilogue(pNv, &threadState);
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_restart_user_channels(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS params = { 0 };
+ RM_API *pRmApi;
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_ERR_INVALID_STATE;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_FIFO);
+ if (pRmApi != NULL)
+ {
+ params.bDisable = NV_FALSE;
+ rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS,
+ &params, sizeof(params));
+
+ RmUnixRmApiEpilogue(pNv, &threadState);
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+//
+// Use this call to check if the chipset is io coherent
+//
+NvBool NV_API_CALL rm_is_chipset_io_coherent(
+ nvidia_stack_t *sp)
+{
+ void *fp;
+ OBJSYS *pSys;
+ OBJCL *pCl;
+ NvBool bIoCoherent = NV_FALSE;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pSys = SYS_GET_INSTANCE();
+ pCl = SYS_GET_CL(pSys);
+ if (pCl == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: no CL object found, setting io coherent by default\n",
+ __FUNCTION__);
+ goto done;
+ }
+
+ bIoCoherent = pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT);
+
+done:
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return bIoCoherent;
+}
+
+NV_STATUS NV_API_CALL rm_ioctl(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ nv_file_private_t *nvfp,
+ NvU32 Command,
+ void *pData,
+ NvU32 dataSize
+)
+{
+ NV_STATUS rmStatus;
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ rmStatus = RmIoctl(pNv, nvfp, Command, pData, dataSize);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+void NV_API_CALL rm_cleanup_file_private(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ nv_file_private_t *nvfp
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_EXTERNAL);
+ RM_API_CONTEXT rmApiContext = {0};
+ NvU32 i;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ threadStateSetTimeoutOverride(&threadState, 10 * 1000);
+
+ if (rmapiPrologue(pRmApi, &rmApiContext) != NV_OK)
+ return;
+
+ // LOCK: acquire API lock
+ if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK)
+ {
+ // Unref any object which was exported on this file.
+ if (nvfp->handles != NULL)
+ {
+ for (i = 0; i < nvfp->maxHandles; i++)
+ {
+ if (nvfp->handles[i] == 0)
+ {
+ continue;
+ }
+
+ RmFreeObjExportHandle(nvfp->handles[i]);
+ nvfp->handles[i] = 0;
+ }
+
+ os_free_mem(nvfp->handles);
+ nvfp->handles = NULL;
+ nvfp->maxHandles = 0;
+ }
+
+ // Free any RM clients associated with this file.
+ RmFreeUnusedClients(pNv, nvfp);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ rmapiEpilogue(pRmApi, &rmApiContext);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if (nvfp->ctl_nvfp != NULL)
+ {
+ nv_put_file_private(nvfp->ctl_nvfp_priv);
+ nvfp->ctl_nvfp = NULL;
+ nvfp->ctl_nvfp_priv = NULL;
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_unbind_lock(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK)
+ {
+ RmUnbindLock(pNv);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS rm_alloc_os_event(
+ NvHandle hClient,
+ nv_file_private_t *nvfp,
+ NvU32 fd
+)
+{
+ NV_STATUS RmStatus;
+
+ // LOCK: acquire API lock
+ if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_EVENT)) == NV_OK)
+ {
+ RmStatus = RmAllocOsEvent(hClient, nvfp, fd);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ return RmStatus;
+}
+
+NV_STATUS rm_free_os_event(
+ NvHandle hClient,
+ NvU32 fd
+)
+{
+ NV_STATUS RmStatus;
+
+ // LOCK: acquire API lock
+ if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_EVENT)) == NV_OK)
+ {
+ RmStatus = RmFreeOsEvent(hClient, fd);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ return RmStatus;
+}
+
+NV_STATUS rm_get_event_data(
+ nv_file_private_t *nvfp,
+ NvP64 pEvent,
+ NvU32 *MoreEvents
+)
+{
+ NV_STATUS RmStatus;
+
+ // LOCK: acquire API lock
+ if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_EVENT)) == NV_OK)
+ {
+ RmStatus = RmGetEventData(nvfp, pEvent, MoreEvents, NV_TRUE);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ return RmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_read_registry_dword(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU32 *Data
+)
+{
+ OBJGPU *pGpu = NULL;
+ NV_STATUS RmStatus;
+ void *fp;
+ NvBool isApiLockTaken = NV_FALSE;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ //
+ // We can be called from different contexts:
+ //
+ // 1) early initialization without device state.
+ // 2) from outside the RM API (without the lock held)
+ //
+ // In context 1)the API lock is not needed and
+ // in context 2), it needs to be acquired.
+ //
+ if (nv != NULL)
+ {
+ // LOCK: acquire API lock
+ if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) != NV_OK)
+ {
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return RmStatus;
+ }
+
+ isApiLockTaken = NV_TRUE;
+ }
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ // Skipping the NULL check as osReadRegistryDword takes care of it.
+ RmStatus = osReadRegistryDword(pGpu, regParmStr, Data);
+
+ if (isApiLockTaken == NV_TRUE)
+ {
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return RmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_write_registry_dword(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU32 Data
+)
+{
+ NV_STATUS RmStatus;
+ void *fp;
+ NvBool isApiLockTaken = NV_FALSE;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ if (nv != NULL)
+ {
+ // LOCK: acquire API lock
+ if ((RmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK)
+ {
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return RmStatus;
+ }
+
+ isApiLockTaken = NV_TRUE;
+ }
+
+ RmStatus = RmWriteRegistryDword(nv, regParmStr, Data);
+
+ if (isApiLockTaken == NV_TRUE)
+ {
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return RmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_write_registry_binary(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU8 *Data,
+ NvU32 cbLen
+)
+{
+ NV_STATUS RmStatus;
+ void *fp;
+ NvBool isApiLockTaken = NV_FALSE;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ if (nv != NULL)
+ {
+ // LOCK: acquire API lock
+ if ((RmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK)
+ {
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return RmStatus;
+ }
+
+ isApiLockTaken = NV_TRUE;
+ }
+
+ RmStatus = RmWriteRegistryBinary(nv, regParmStr, Data, cbLen);
+
+ if (isApiLockTaken == NV_TRUE)
+ {
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return RmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_write_registry_string(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ const char *regParmStr,
+ const char *string,
+ NvU32 stringLength
+)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NvBool isApiLockTaken = NV_FALSE;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ if (nv != NULL)
+ {
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK)
+ {
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+ }
+
+ isApiLockTaken = NV_TRUE;
+ }
+
+ rmStatus = RmWriteRegistryString(nv, regParmStr, string, (stringLength + 1));
+
+ if (isApiLockTaken == NV_TRUE)
+ {
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+static NvBool NV_API_CALL rm_is_space(const char ch)
+{
+ //
+ // return true if it is a:
+ // ' ' : (space - decimal 32.)
+ // '\t' : (TAB - decimal 9)
+ // 'LF' : (Line feed, new line - decimal 10)
+ // 'VT' : (Vertical TAB - decimal 11)
+ // 'FF' : (Form feed, new page - decimal 12)
+ // '\r' : (carriage return - decimal 13)
+ //
+ return ((ch == ' ') || ((ch >= '\t') && (ch <= '\r')));
+}
+
+char* NV_API_CALL rm_remove_spaces(const char *in)
+{
+ unsigned int len = os_string_length(in) + 1;
+ const char *in_ptr;
+ char *out, *out_ptr;
+
+ if (os_alloc_mem((void **)&out, len) != NV_OK)
+ return NULL;
+
+ in_ptr = in;
+ out_ptr = out;
+
+ while (*in_ptr != '\0')
+ {
+ if (!rm_is_space(*in_ptr))
+ *out_ptr++ = *in_ptr;
+ in_ptr++;
+ }
+ *out_ptr = '\0';
+
+ return out;
+}
+
+char* NV_API_CALL rm_string_token(char **strp, const char delim)
+{
+ char *s, *token;
+
+ if ((strp == NULL) || (*strp == NULL))
+ return NULL;
+
+ s = token = *strp;
+ *strp = NULL;
+
+ for (; *s != '\0'; s++) {
+ if (*s == delim) {
+ *s = '\0';
+ *strp = ++s;
+ break;
+ }
+ }
+
+ return token;
+}
+
+// Parse string passed in NVRM as module parameter.
+void NV_API_CALL rm_parse_option_string(nvidia_stack_t *sp, const char *nvRegistryDwords)
+{
+ unsigned int i;
+ nv_parm_t *entry;
+ char *option_string = NULL;
+ char *ptr, *token;
+ char *name, *value;
+ NvU32 data;
+
+ if (nvRegistryDwords != NULL)
+ {
+ if ((option_string = rm_remove_spaces(nvRegistryDwords)) == NULL)
+ {
+ return;
+ }
+
+ ptr = option_string;
+
+ while ((token = rm_string_token(&ptr, ';')) != NULL)
+ {
+ if (!(name = rm_string_token(&token, '=')) || !os_string_length(name))
+ {
+ continue;
+ }
+
+ if (!(value = rm_string_token(&token, '=')) || !os_string_length(value))
+ {
+ continue;
+ }
+
+ if (rm_string_token(&token, '=') != NULL)
+ {
+ continue;
+ }
+
+ data = os_strtoul(value, NULL, 0);
+
+ for (i = 0; (entry = &nv_parms[i])->name != NULL; i++)
+ {
+ if (os_string_compare(entry->name, name) == 0)
+ break;
+ }
+
+ if (!entry->name)
+ rm_write_registry_dword(sp, NULL, name, data);
+ else
+ *entry->data = data;
+ }
+
+ // Free the memory allocated by rm_remove_spaces()
+ os_free_mem(option_string);
+ }
+}
+
+NV_STATUS NV_API_CALL rm_run_rc_callback(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ OBJGPU *pGpu;
+ void *fp;
+
+ /* make sure our timer's not still running when it shouldn't be */
+ if (nv == NULL)
+ return NV_ERR_GENERIC;
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ if (pGpu == NULL)
+ return NV_ERR_GENERIC;
+
+ if (nv->rc_timer_enabled == 0)
+ return NV_ERR_GENERIC;
+
+ if (!FULL_GPU_SANITY_CHECK(pGpu))
+ {
+ return NV_ERR_GENERIC;
+ }
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ osRun1HzCallbacksNow(pGpu);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return NV_OK;
+}
+
+static NV_STATUS RmRunNanoTimerCallback(
+ OBJGPU *pGpu,
+ void *pTmrEvent
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ POBJTMR pTmr = GPU_GET_TIMER(pGpu);
+ THREAD_STATE_NODE threadState;
+ NV_STATUS status = NV_OK;
+ // LOCK: try to acquire GPUs lock
+ if ((status = rmGpuLocksAcquire(GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_TMR)) != NV_OK)
+ {
+ PTMR_EVENT_PVT pEvent = (PTMR_EVENT_PVT) pTmrEvent;
+ // We failed to acquire the lock; schedule a timer to try again.
+ return osStartNanoTimer(pGpu->pOsGpuInfo, pEvent->super.pOSTmrCBdata, 1000);
+ }
+
+ if ((status = osCondAcquireRmSema(pSys->pSema)) != NV_OK)
+ {
+ // UNLOCK: release GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+ return status;
+ }
+
+ threadStateInitISRAndDeferredIntHandler(&threadState, pGpu,
+ THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER);
+
+ // Call timer event service
+ status = tmrEventServiceOSTimerCallback_HAL(pGpu, pTmr, (PTMR_EVENT)pTmrEvent);
+
+ // Out of conflicting thread
+ threadStateFreeISRAndDeferredIntHandler(&threadState,
+ pGpu, THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER);
+
+ osReleaseRmSema(pSys->pSema, NULL);
+ // UNLOCK: release GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, pGpu);
+
+ return status;
+}
+
+NV_STATUS NV_API_CALL rm_run_nano_timer_callback
+(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ void *pTmrEvent
+)
+{
+ NV_STATUS status;
+ OBJGPU *pGpu = NULL;
+ void *fp;
+
+ if (nv == NULL)
+ return NV_ERR_GENERIC;
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ if (pGpu == NULL)
+ return NV_ERR_GENERIC;
+
+ if (!FULL_GPU_SANITY_CHECK(pGpu))
+ {
+ return NV_ERR_GENERIC;
+ }
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ status = RmRunNanoTimerCallback(pGpu, pTmrEvent);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return status;
+}
+
+void NV_API_CALL rm_execute_work_item(
+ nvidia_stack_t *sp,
+ void *pNvWorkItem
+)
+{
+ void *fp;
+ THREAD_STATE_NODE threadState;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ RmExecuteWorkItem(pNvWorkItem);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+const char* NV_API_CALL rm_get_device_name(
+ NvU16 device,
+ NvU16 subsystem_vendor,
+ NvU16 subsystem_device
+)
+{
+ unsigned int i;
+ const char *tmpName = NULL;
+
+ for (i = 0; i < NV_ARRAY_ELEMENTS(sChipsReleased); i++)
+ {
+ // if the device ID doesn't match, go to the next entry
+ if (device != sChipsReleased[i].devID)
+ {
+ continue;
+ }
+
+ // if the entry has 0 for the subsystem IDs, then the device
+ // ID match is sufficient, but continue scanning through
+ // sChipsReleased[] in case there is a subsystem ID match later
+ // in the table
+ if (sChipsReleased[i].subSystemVendorID == 0 &&
+ sChipsReleased[i].subSystemID == 0)
+ {
+ tmpName = sChipsReleased[i].name;
+ continue;
+ }
+
+ if (subsystem_vendor == sChipsReleased[i].subSystemVendorID &&
+ subsystem_device == sChipsReleased[i].subSystemID)
+ {
+ tmpName = sChipsReleased[i].name;
+ break;
+ }
+ }
+
+ return (tmpName != NULL) ? tmpName : "Unknown";
+}
+
+NV_STATUS rm_access_registry(
+ NvHandle hClient,
+ NvHandle hObject,
+ NvU32 AccessType,
+ NvP64 clientDevNodeAddress,
+ NvU32 DevNodeLength,
+ NvP64 clientParmStrAddress,
+ NvU32 ParmStrLength,
+ NvP64 clientBinaryDataAddress,
+ NvU32 *pBinaryDataLength,
+ NvU32 *Data,
+ NvU32 *Entry
+)
+{
+ NV_STATUS RmStatus;
+ NvBool bReadOnly = (AccessType == NVOS38_ACCESS_TYPE_READ_DWORD) ||
+ (AccessType == NVOS38_ACCESS_TYPE_READ_BINARY);
+
+ // LOCK: acquire API lock
+ if ((RmStatus = rmapiLockAcquire(bReadOnly ? RMAPI_LOCK_FLAGS_READ : RMAPI_LOCK_FLAGS_NONE,
+ RM_LOCK_MODULES_OSAPI)) == NV_OK)
+ {
+ RmStatus = RmAccessRegistry(hClient,
+ hObject,
+ AccessType,
+ clientDevNodeAddress,
+ DevNodeLength,
+ clientParmStrAddress,
+ ParmStrLength,
+ clientBinaryDataAddress,
+ pBinaryDataLength,
+ Data,
+ Entry);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ return RmStatus;
+}
+
+NV_STATUS rm_update_device_mapping_info(
+ NvHandle hClient,
+ NvHandle hDevice,
+ NvHandle hMemory,
+ void *pOldCpuAddress,
+ void *pNewCpuAddress
+)
+{
+ NV_STATUS RmStatus;
+
+ // LOCK: acquire API lock
+ if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU)) == NV_OK)
+ {
+ RmStatus = RmUpdateDeviceMappingInfo(hClient,
+ hDevice,
+ hMemory,
+ pOldCpuAddress,
+ pNewCpuAddress);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ return RmStatus;
+}
+
+static void rm_is_device_rm_firmware_capable(
+ nv_state_t *pNv,
+ NvU32 pmcBoot42,
+ NvBool *pbIsFirmwareCapable,
+ NvBool *pbEnableByDefault
+)
+{
+ NvBool bIsFirmwareCapable = NV_FALSE;
+ NvBool bEnableByDefault = NV_FALSE;
+
+ if (NV_IS_SOC_DISPLAY_DEVICE(pNv))
+ {
+ bIsFirmwareCapable = NV_TRUE;
+ }
+ else
+ {
+ bIsFirmwareCapable = gpumgrIsDeviceRmFirmwareCapable(pNv->pci_info.device_id,
+ pmcBoot42,
+ &bEnableByDefault);
+ }
+
+ if (pbIsFirmwareCapable != NULL)
+ {
+ *pbIsFirmwareCapable = bIsFirmwareCapable;
+ }
+ if (pbEnableByDefault != NULL)
+ {
+ *pbEnableByDefault = bEnableByDefault;
+ }
+}
+
+static NvBool NV_API_CALL rm_is_legacy_device(
+ NvU16 device_id,
+ NvU16 subsystem_vendor,
+ NvU16 subsystem_device,
+ NvBool print_warning
+)
+{
+ return NV_FALSE;
+}
+
+static NvBool NV_API_CALL rm_is_legacy_arch(
+ NvU32 pmc_boot_0,
+ NvU32 pmc_boot_42
+)
+{
+ NvBool legacy = NV_FALSE;
+
+ return legacy;
+}
+
+NV_STATUS NV_API_CALL rm_is_supported_device(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ OBJSYS *pSys;
+ POBJHALMGR pHalMgr;
+ GPUHWREG *reg_mapping;
+ NvU32 myHalPublicID;
+ void *fp;
+ NvU32 pmc_boot_0;
+ NvU32 pmc_boot_42;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pSys = SYS_GET_INSTANCE();
+ pHalMgr = SYS_GET_HALMGR(pSys);
+
+ reg_mapping = osMapKernelSpace(pNv->regs->cpu_address,
+ os_page_size,
+ NV_MEMORY_UNCACHED,
+ NV_PROTECT_READABLE);
+
+ if (reg_mapping == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to map registers!\n");
+ rmStatus = NV_ERR_OPERATING_SYSTEM;
+ goto threadfree;
+ }
+ NvU32 pmc_boot_1 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_1);
+ pmc_boot_0 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_0);
+ pmc_boot_42 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_42);
+
+ osUnmapKernelSpace(reg_mapping, os_page_size);
+
+ if ((pmc_boot_0 == 0xFFFFFFFF) && (pmc_boot_42 == 0xFFFFFFFF))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x\n"
+ "NVRM: (PCI ID: %04x:%04x) installed in this system has\n"
+ "NVRM: fallen off the bus and is not responding to commands.\n",
+ pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
+ pNv->pci_info.function, pNv->pci_info.vendor_id,
+ pNv->pci_info.device_id);
+ rmStatus = NV_ERR_GPU_IS_LOST;
+ goto threadfree;
+ }
+
+ /*
+ * For legacy architectures, rm_is_legacy_arch() prints "legacy" message.
+ * We do not want to print "unsupported" message for legacy architectures
+ * to avoid confusion. Also, the probe should not continue for legacy
+ * architectures. Hence, we set rmStatus to NV_ERR_NOT_SUPPORTED and
+ * goto threadfree.
+ */
+ if (rm_is_legacy_arch(pmc_boot_0, pmc_boot_42))
+ {
+ rmStatus = NV_ERR_NOT_SUPPORTED;
+ goto threadfree;
+ }
+
+ rmStatus = halmgrGetHalForGpu(pHalMgr, pmc_boot_0, pmc_boot_42, &myHalPublicID);
+
+ if (rmStatus != NV_OK)
+ {
+ NvBool bIsFirmwareCapable;
+
+ rm_is_device_rm_firmware_capable(pNv,
+ pmc_boot_42,
+ &bIsFirmwareCapable,
+ NULL);
+ if (!bIsFirmwareCapable)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
+ "NVRM: installed in this system is not supported by open\n"
+ "NVRM: nvidia.ko because it does not include the required GPU\n"
+ "NVRM: System Processor (GSP).\n"
+ "NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n"
+ "NVRM: Firmware' sections in the driver README, available on\n"
+ "NVRM: the Linux graphics driver download page at\n"
+ "NVRM: www.nvidia.com.\n",
+ pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
+ pNv->pci_info.function, pNv->pci_info.vendor_id,
+ pNv->pci_info.device_id, NV_VERSION_STRING);
+ goto threadfree;
+ }
+ goto print_unsupported;
+ }
+
+ rmStatus = rm_is_vgpu_supported_device(pNv, pmc_boot_1);
+
+ if (rmStatus != NV_OK)
+ goto print_unsupported;
+ goto threadfree;
+
+print_unsupported:
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
+ "NVRM: installed in this system is not supported by the\n"
+ "NVRM: NVIDIA %s driver release.\n"
+ "NVRM: Please see 'Appendix A - Supported NVIDIA GPU Products'\n"
+ "NVRM: in this release's README, available on the operating system\n"
+ "NVRM: specific graphics driver download page at www.nvidia.com.\n",
+ pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
+ pNv->pci_info.function, pNv->pci_info.vendor_id,
+ pNv->pci_info.device_id, NV_VERSION_STRING);
+
+threadfree:
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NvBool NV_API_CALL rm_is_supported_pci_device(
+ NvU8 pci_class,
+ NvU8 pci_subclass,
+ NvU16 vendor,
+ NvU16 device,
+ NvU16 subsystem_vendor,
+ NvU16 subsystem_device,
+ NvBool print_legacy_warning
+)
+{
+ const NvU16 nv_pci_vendor_id = 0x10DE;
+ const NvU16 nv_pci_id_riva_tnt = 0x0020;
+ const NvU8 nv_pci_class_display = 0x03;
+ const NvU8 nv_pci_subclass_display_vga = 0x00;
+ const NvU8 nv_pci_subclass_display_3d = 0x02;
+
+ if (pci_class != nv_pci_class_display)
+ {
+ return NV_FALSE;
+ }
+
+ if ((pci_subclass != nv_pci_subclass_display_vga) &&
+ (pci_subclass != nv_pci_subclass_display_3d))
+ {
+ return NV_FALSE;
+ }
+
+ if (vendor != nv_pci_vendor_id)
+ {
+ return NV_FALSE;
+ }
+
+ if (device < nv_pci_id_riva_tnt)
+ {
+ return NV_FALSE;
+ }
+
+ if (rm_is_legacy_device(
+ device,
+ subsystem_vendor,
+ subsystem_device,
+ print_legacy_warning))
+ {
+ return NV_FALSE;
+ }
+
+ return NV_TRUE;
+}
+
+/*
+ * Performs the I2C transfers which are related with DP AUX channel
+ */
+static NV_STATUS RmDpAuxI2CTransfer
+(
+ nv_state_t *pNv,
+ NvU32 displayId,
+ NvU8 addr,
+ NvU32 len,
+ NvU8 *pData,
+ NvBool bWrite
+)
+{
+ NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_CTRL_PARAMS *pParams;
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+ NV_STATUS status;
+
+ if (len > NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: requested I2C transfer length %u is greater than maximum supported length %u\n",
+ __FUNCTION__, len, NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE);
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ pParams = portMemAllocNonPaged(sizeof(*pParams));
+ if (pParams == NULL)
+ {
+ return NV_ERR_NO_MEMORY;
+ }
+
+ portMemSet(pParams, 0, sizeof(*pParams));
+
+ pParams->subDeviceInstance = 0;
+ pParams->displayId = displayId;
+ pParams->addr = addr;
+ pParams->size = len;
+ pParams->bWrite = bWrite;
+
+ if (bWrite)
+ {
+ portMemCopy(pParams->data, NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE,
+ pData, len);
+ }
+
+ status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp,
+ NV0073_CTRL_CMD_DP_AUXCH_I2C_TRANSFER_CTRL,
+ pParams, sizeof(*pParams));
+
+ if ((status == NV_OK) && !bWrite)
+ {
+ portMemCopy(pData, len, pParams->data, pParams->size);
+ }
+
+ portMemFree(pParams);
+
+ return status;
+}
+
+/*
+ * Performs the I2C transfers which are not related with DP AUX channel
+ */
+static NV_STATUS RmNonDPAuxI2CTransfer
+(
+ nv_state_t *pNv,
+ NvU8 portId,
+ nv_i2c_cmd_t type,
+ NvU8 addr,
+ NvU8 command,
+ NvU32 len,
+ NvU8 *pData
+)
+{
+ NV402C_CTRL_I2C_TRANSACTION_PARAMS *params;
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+ NV_STATUS rmStatus = NV_OK;
+
+ params = portMemAllocNonPaged(sizeof(*params));
+ if (params == NULL)
+ {
+ return NV_ERR_NO_MEMORY;
+ }
+
+ portMemSet(params, 0, sizeof(*params));
+
+ params->portId = portId;
+ // precondition our address (our stack requires this)
+ params->deviceAddress = addr << 1;
+
+ switch (type)
+ {
+ case NV_I2C_CMD_WRITE:
+ params->transData.i2cBlockData.bWrite = NV_TRUE;
+ /* fall through*/
+
+ case NV_I2C_CMD_READ:
+ params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_I2C_BLOCK_RW;
+ params->transData.i2cBlockData.messageLength = len;
+ params->transData.i2cBlockData.pMessage = pData;
+ break;
+
+ case NV_I2C_CMD_SMBUS_WRITE:
+ if (len == 2)
+ {
+ params->transData.smbusWordData.bWrite = NV_TRUE;
+ }
+ else
+ {
+ params->transData.smbusByteData.bWrite = NV_TRUE;
+ }
+ /* fall through*/
+
+ case NV_I2C_CMD_SMBUS_READ:
+ if (len == 2)
+ {
+ params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_WORD_RW;
+ params->transData.smbusWordData.message = pData[0] | ((NvU16)pData[1] << 8);
+ params->transData.smbusWordData.registerAddress = command;
+ }
+ else
+ {
+ params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BYTE_RW;
+ params->transData.smbusByteData.message = pData[0];
+ params->transData.smbusByteData.registerAddress = command;
+ }
+ break;
+
+ case NV_I2C_CMD_SMBUS_BLOCK_WRITE:
+ params->transData.smbusBlockData.bWrite = NV_TRUE;
+ /* fall through*/
+
+ case NV_I2C_CMD_SMBUS_BLOCK_READ:
+ params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BLOCK_RW;
+ params->transData.smbusBlockData.registerAddress = command;
+ params->transData.smbusBlockData.messageLength = len;
+ params->transData.smbusBlockData.pMessage = pData;
+ break;
+
+ case NV_I2C_CMD_SMBUS_QUICK_WRITE:
+ params->transData.smbusQuickData.bWrite = NV_TRUE;
+ /* fall through*/
+
+ case NV_I2C_CMD_SMBUS_QUICK_READ:
+ params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_QUICK_RW;
+ break;
+
+ case NV_I2C_CMD_BLOCK_WRITE:
+ params->transData.i2cBufferData.bWrite = NV_TRUE;
+ /* fall through */
+
+ case NV_I2C_CMD_BLOCK_READ:
+ params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_I2C_BUFFER_RW;
+ params->transData.i2cBufferData.registerAddress = command;
+ params->transData.i2cBufferData.messageLength = len;
+ params->transData.i2cBufferData.pMessage = pData;
+ break;
+
+ default:
+ portMemFree(params);
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+ pNv->rmapi.hI2C,
+ NV402C_CTRL_CMD_I2C_TRANSACTION,
+ params, sizeof(*params));
+
+ //
+ // For NV_I2C_CMD_SMBUS_READ, copy the read data to original
+ // data buffer.
+ //
+ if (rmStatus == NV_OK && type == NV_I2C_CMD_SMBUS_READ)
+ {
+ if (len == 2)
+ {
+ pData[0] = (params->transData.smbusWordData.message & 0xff);
+ pData[1] = params->transData.smbusWordData.message >> 8;
+ }
+ else
+ {
+ pData[0] = params->transData.smbusByteData.message;
+ }
+ }
+
+ portMemFree(params);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_i2c_transfer(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ void *pI2cAdapter,
+ nv_i2c_cmd_t type,
+ NvU8 addr,
+ NvU8 command,
+ NvU32 len,
+ NvU8 *pData
+)
+{
+ THREAD_STATE_NODE threadState;
+ nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+ NV_STATUS rmStatus = NV_OK;
+ OBJGPU *pGpu = NULL;
+ NvBool unlockApi = NV_FALSE;
+ NvBool unlockGpu = NV_FALSE;
+ NvU32 x;
+ void *fp;
+ NvU32 numDispId = 0;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if (pNvp->flags & NV_INIT_FLAG_PUBLIC_I2C)
+ {
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK)
+ goto finish;
+
+ unlockApi = NV_TRUE;
+
+ // LOCK: acquire GPUs lock
+ if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK)
+ goto finish;
+
+ unlockGpu = NV_TRUE;
+ }
+
+ pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+ if (!pGpu)
+ {
+ rmStatus = NV_ERR_GENERIC;
+ goto finish;
+ }
+
+ for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+ {
+ if (pNvp->i2c_adapters[x].pOsAdapter == pI2cAdapter)
+ {
+ break;
+ }
+ }
+
+ if (x == MAX_I2C_ADAPTERS)
+ {
+ rmStatus = NV_ERR_GENERIC;
+ goto finish;
+ }
+
+ for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+ {
+ NvU32 displayId = pNvp->i2c_adapters[x].displayId[numDispId];
+
+ if (displayId == INVALID_DISP_ID)
+ {
+ continue;
+ }
+
+ // Handle i2c-over-DpAux adapters separately from regular i2c adapters
+ if (displayId == 0)
+ {
+ rmStatus = RmNonDPAuxI2CTransfer(pNv, pNvp->i2c_adapters[x].port,
+ type, addr, command, len, pData);
+ }
+ else
+ {
+ if ((type != NV_I2C_CMD_READ) && (type != NV_I2C_CMD_WRITE))
+ {
+ rmStatus = NV_ERR_NOT_SUPPORTED;
+ goto semafinish;
+ }
+
+ rmStatus = RmDpAuxI2CTransfer(pNv, displayId, addr, len, pData,
+ type == NV_I2C_CMD_WRITE);
+ }
+semafinish:
+ if (rmStatus == NV_OK)
+ {
+ break;
+ }
+ }
+
+finish:
+ if (unlockGpu)
+ {
+ // UNLOCK: release GPU lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+
+ if (unlockApi)
+ {
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+static void rm_i2c_add_adapter(
+ nv_state_t *pNv,
+ NvU32 port,
+ NvU32 displayId
+)
+{
+ NvU32 y, free;
+ nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+ NvU32 numDispId = 0;
+
+ for (y = 0, free = MAX_I2C_ADAPTERS; y < MAX_I2C_ADAPTERS; y++)
+ {
+ if (pNvp->i2c_adapters[y].pOsAdapter == NULL)
+ {
+ // Only find the first free entry, and ignore the rest
+ if (free == MAX_I2C_ADAPTERS)
+ {
+ free = y;
+ }
+ }
+ else if (pNvp->i2c_adapters[y].port == port)
+ {
+ break;
+ }
+ }
+
+ if (y < MAX_I2C_ADAPTERS)
+ {
+ for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+ {
+ if (pNvp->i2c_adapters[y].displayId[numDispId] == INVALID_DISP_ID)
+ {
+ pNvp->i2c_adapters[y].displayId[numDispId] = displayId;
+ break;
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_INFO,
+ "%s: adapter already exists (port=0x%x, displayId=0x%x)\n",
+ __FUNCTION__, port,
+ pNvp->i2c_adapters[y].displayId[numDispId]);
+ }
+ }
+
+ if (numDispId == MAX_DISP_ID_PER_ADAPTER)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: no more free display Id entries in adapter\n",
+ __FUNCTION__);
+ }
+
+ return;
+ }
+
+ if (free == MAX_I2C_ADAPTERS)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s: no more free adapter entries exist\n",
+ __FUNCTION__);
+ return;
+ }
+
+ pNvp->i2c_adapters[free].pOsAdapter = nv_i2c_add_adapter(pNv, port);
+ pNvp->i2c_adapters[free].port = port;
+ // When port is added, numDispId will be 0.
+ pNvp->i2c_adapters[free].displayId[numDispId] = displayId;
+}
+
+void RmI2cAddGpuPorts(nv_state_t * pNv)
+{
+ NvU32 x = 0;
+ nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+ NvU32 displayMask;
+ NV_STATUS status;
+ NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS systemGetSupportedParams = { 0 };
+
+ // Make displayId as Invalid.
+ for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+ {
+ NvU32 numDispId;
+
+ for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+ {
+ pNvp->i2c_adapters[x].displayId[numDispId] = INVALID_DISP_ID;
+ }
+ }
+
+ // First, set up the regular i2c adapters - one per i2c port
+ if (pNv->rmapi.hI2C != 0)
+ {
+ NV402C_CTRL_I2C_GET_PORT_INFO_PARAMS i2cPortInfoParams = { 0 };
+
+ status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hI2C,
+ NV402C_CTRL_CMD_I2C_GET_PORT_INFO,
+ &i2cPortInfoParams, sizeof(i2cPortInfoParams));
+
+ if (status == NV_OK)
+ {
+ for (x = 0; x < NV_ARRAY_ELEMENTS(i2cPortInfoParams.info); x++)
+ {
+ //
+ // Check if this port is implemented and RM I2C framework has
+ // validated this port. Only limited amount of ports can
+ // be added to the OS framework.
+ //
+ if (FLD_TEST_DRF(402C_CTRL, _I2C_GET_PORT_INFO, _IMPLEMENTED,
+ _YES, i2cPortInfoParams.info[x]) &&
+ FLD_TEST_DRF(402C_CTRL, _I2C_GET_PORT_INFO, _VALID,
+ _YES, i2cPortInfoParams.info[x]))
+ {
+ rm_i2c_add_adapter(pNv, x, 0);
+ }
+ }
+ }
+ }
+
+ //
+ // Now set up the i2c-over-DpAux adapters - one per DP OD
+ //
+ // 1. Perform NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS RM control which
+ // will return the mask for all the display ID's.
+ // 2. Loop for all the display ID's and do
+ // NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO RM control call. For each
+ // output resource, check for the following requirements:
+ // a. It must be DisplayPort.
+ // b. It must be internal to the GPU (ie, not on the board)
+ // c. It must be directly connected to the physical connector (ie, no DP
+ // 1.2 multistream ODs).
+ // 3. Perform NV0073_CTRL_CMD_SPECIFIC_GET_I2C_PORTID RM control for
+ // getting the I2C port data.
+ //
+ // With these restrictions, we should only end up with at most one OD
+ // per DP connector.
+ //
+
+ if (pNv->rmapi.hDisp == 0)
+ {
+ return;
+ }
+
+ systemGetSupportedParams.subDeviceInstance = 0;
+ status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp,
+ NV0073_CTRL_CMD_SYSTEM_GET_SUPPORTED,
+ &systemGetSupportedParams, sizeof(systemGetSupportedParams));
+
+ if (status != NV_OK)
+ {
+ return;
+ }
+
+ for (displayMask = systemGetSupportedParams.displayMask;
+ displayMask != 0;
+ displayMask &= ~LOWESTBIT(displayMask))
+ {
+ NV0073_CTRL_SPECIFIC_OR_GET_INFO_PARAMS orInfoParams = { 0 };
+ NvU32 displayId = LOWESTBIT(displayMask);
+
+ orInfoParams.subDeviceInstance = 0;
+ orInfoParams.displayId = displayId;
+
+ status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp,
+ NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO,
+ &orInfoParams, sizeof(orInfoParams));
+
+ if ((status == NV_OK) &&
+ (orInfoParams.type == NV0073_CTRL_SPECIFIC_OR_TYPE_SOR) &&
+ ((orInfoParams.protocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DP_A) ||
+ (orInfoParams.protocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DP_B)) &&
+ (orInfoParams.location == NV0073_CTRL_SPECIFIC_OR_LOCATION_CHIP) &&
+ (!orInfoParams.bIsDispDynamic))
+ {
+ NV0073_CTRL_SPECIFIC_GET_I2C_PORTID_PARAMS i2cPortIdParams = { 0 };
+
+ i2cPortIdParams.subDeviceInstance = 0;
+ i2cPortIdParams.displayId = displayId;
+
+ status = pRmApi->Control(pRmApi,
+ pNv->rmapi.hClient,
+ pNv->rmapi.hDisp,
+ NV0073_CTRL_CMD_SPECIFIC_GET_I2C_PORTID,
+ &i2cPortIdParams,
+ sizeof(i2cPortIdParams));
+
+ if ((status == NV_OK) &&
+ (i2cPortIdParams.ddcPortId != NV0073_CTRL_SPECIFIC_I2C_PORT_NONE))
+ {
+ rm_i2c_add_adapter(pNv, i2cPortIdParams.ddcPortId - 1, displayId);
+ }
+ }
+ }
+}
+
+void NV_API_CALL rm_i2c_remove_adapters(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ //
+ // Cycle through all adapter entries, and first remove the adapter
+ // from the list from the kernel, then remove the i2c adapter
+ // list once that is completed. This should only be used from exit
+ // module time. Otherwise it could fail to remove some of the
+ // kernel adapters and subsequent transfer requests would result
+ // in crashes.
+ //
+ NvU32 x = 0;
+ nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+ NvU32 numDispId;
+
+ for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+ {
+ if (pNvp->i2c_adapters[x].pOsAdapter != NULL)
+ {
+ nv_i2c_del_adapter(pNv, pNvp->i2c_adapters[x].pOsAdapter);
+
+ pNvp->i2c_adapters[x].pOsAdapter = NULL;
+ pNvp->i2c_adapters[x].port = 0;
+ for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+ {
+ pNvp->i2c_adapters[x].displayId[numDispId] = INVALID_DISP_ID;
+ }
+ }
+ }
+}
+
+NvBool NV_API_CALL rm_i2c_is_smbus_capable(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ void *pI2cAdapter
+)
+{
+ THREAD_STATE_NODE threadState;
+ nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+ NV_STATUS rmStatus = NV_OK;
+ OBJGPU *pGpu = NULL;
+ NvBool unlock = NV_FALSE;
+ NvU32 x;
+ NvBool ret = NV_FALSE;
+ void *fp;
+ NvU32 numDispId = 0;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if (pNvp->flags & NV_INIT_FLAG_PUBLIC_I2C)
+ {
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK)
+ goto semafinish;
+
+ unlock = NV_TRUE;
+ }
+
+ pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+ if (!pGpu)
+ {
+ goto semafinish;
+ }
+
+ for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+ {
+ if (pNvp->i2c_adapters[x].pOsAdapter == pI2cAdapter)
+ {
+ break;
+ }
+ }
+
+ if (x == MAX_I2C_ADAPTERS)
+ {
+ goto semafinish;
+ }
+
+ // we do not support smbus functions on i2c-over-DPAUX
+ for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+ {
+ if (pNvp->i2c_adapters[x].displayId[numDispId] == 0x0)
+ {
+ ret = NV_TRUE;
+ }
+ }
+
+semafinish:
+ if (unlock)
+ {
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return ret;
+}
+
+NV_STATUS NV_API_CALL rm_perform_version_check(
+ nvidia_stack_t *sp,
+ void *pData,
+ NvU32 dataSize
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ rmStatus = RmPerformVersionCheck(pData, dataSize);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+//
+// Handles the Power Source Change event(AC/DC) for Notebooks.
+// Notebooks from Maxwell have only one Gpu, so this functions grabs first Gpu
+// from GpuMgr and call subdevice RmControl.
+//
+void NV_API_CALL rm_power_source_change_event(
+ nvidia_stack_t *sp,
+ NvU32 event_val
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ nv_state_t *nv;
+ OBJGPU *pGpu = gpumgrGetGpu(0);
+ NV_STATUS rmStatus = NV_OK;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_EVENT)) == NV_OK)
+ {
+ if (pGpu != NULL)
+ {
+ nv = NV_GET_NV_STATE(pGpu);
+ if ((rmStatus = os_ref_dynamic_power(nv, NV_DYNAMIC_PM_FINE)) ==
+ NV_OK)
+ {
+ // LOCK: acquire GPU lock
+ if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_EVENT)) ==
+ NV_OK)
+ {
+ rmStatus = RmPowerSourceChangeEvent(nv, event_val);
+
+ // UNLOCK: release GPU lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+ os_unref_dynamic_power(nv, NV_DYNAMIC_PM_FINE);
+ }
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+ }
+
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: Failed to handle Power Source change event, status=0x%x\n",
+ __FUNCTION__, rmStatus);
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_p2p_dma_map_pages(
+ nvidia_stack_t *sp,
+ nv_dma_device_t *peer,
+ NvU8 *pGpuUuid,
+ NvU32 pageSize,
+ NvU32 pageCount,
+ NvU64 *pDmaAddresses,
+ void **ppPriv
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ if (ppPriv == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ *ppPriv = NULL;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+ {
+ OBJGPU *pGpu = gpumgrGetGpuFromUuid(pGpuUuid,
+ DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1) |
+ DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _BINARY));
+ if (pGpu == NULL)
+ {
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ }
+ else
+ {
+ NvU32 i;
+
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING))
+ {
+ NV_ASSERT(pageSize == os_page_size);
+
+ rmStatus = nv_dma_map_alloc(peer, pageCount, pDmaAddresses,
+ NV_FALSE, ppPriv);
+ }
+ else
+ {
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ for (i = 0; i < pageCount; i++)
+ {
+ // Peer mappings through this API are always via BAR1
+ rmStatus = nv_dma_map_peer(peer, nv->dma_dev, 0x1,
+ pageSize / os_page_size,
+ &pDmaAddresses[i]);
+ if ((rmStatus != NV_OK) && (i > 0))
+ {
+ NvU32 j;
+ for (j = i - 1; j < pageCount; j--)
+ {
+ nv_dma_unmap_peer(peer, pageSize / os_page_size,
+ pDmaAddresses[j]);
+ }
+ }
+ }
+ }
+ }
+
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_get_gpu_info(
+ nvidia_stack_t *sp,
+ NvU64 gpuVirtualAddress,
+ NvU64 length,
+ NvU8 **ppGpuUuid,
+ void **ppGpuInfo
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P);
+ if (rmStatus == NV_OK)
+ {
+ OBJGPU *pGpu;
+ rmStatus = RmP2PGetGpuByAddress(gpuVirtualAddress,
+ length,
+ &pGpu);
+ if (rmStatus == NV_OK)
+ {
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ const NvU8 *pGid;
+
+ pGid = RmGetGpuUuidRaw(nv);
+ if (pGid == NULL)
+ {
+ rmStatus = NV_ERR_GPU_UUID_NOT_FOUND;
+ }
+ else
+ {
+ rmStatus = os_alloc_mem((void **)ppGpuUuid, GPU_UUID_LEN);
+ if (rmStatus == NV_OK)
+ {
+ os_mem_copy(*ppGpuUuid, pGid, GPU_UUID_LEN);
+ }
+ }
+
+ *ppGpuInfo = (void *) pGpu;
+ }
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent(
+ nvidia_stack_t *sp,
+ NvU64 gpuVirtualAddress,
+ NvU64 length,
+ void **p2pObject,
+ NvU64 *pPhysicalAddresses,
+ NvU32 *pEntries,
+ void *pPlatformData,
+ void *pGpuInfo
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+ {
+ rmStatus = RmP2PGetPagesPersistent(gpuVirtualAddress,
+ length,
+ p2pObject,
+ pPhysicalAddresses,
+ pEntries,
+ pPlatformData,
+ pGpuInfo);
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_get_pages(
+ nvidia_stack_t *sp,
+ NvU64 p2pToken,
+ NvU32 vaSpaceToken,
+ NvU64 gpuVirtualAddress,
+ NvU64 length,
+ NvU64 *pPhysicalAddresses,
+ NvU32 *pWreqMbH,
+ NvU32 *pRreqMbH,
+ NvU32 *pEntries,
+ NvU8 **ppGpuUuid,
+ void *pPlatformData
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+ {
+ OBJGPU *pGpu;
+ rmStatus = RmP2PGetPagesWithoutCallbackRegistration(p2pToken,
+ vaSpaceToken,
+ gpuVirtualAddress,
+ length,
+ pPhysicalAddresses,
+ pWreqMbH,
+ pRreqMbH,
+ pEntries,
+ &pGpu,
+ pPlatformData);
+ if (rmStatus == NV_OK)
+ {
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ const NvU8 *pGid;
+
+ pGid = RmGetGpuUuidRaw(nv);
+ if (pGid == NULL)
+ {
+ NV_ASSERT_OK(RmP2PPutPages(p2pToken, vaSpaceToken,
+ gpuVirtualAddress,
+ pPlatformData));
+ rmStatus = NV_ERR_GENERIC;
+ }
+ else
+ {
+ rmStatus = os_alloc_mem((void **)ppGpuUuid, GPU_UUID_LEN);
+ if (rmStatus == NV_OK)
+ os_mem_copy(*ppGpuUuid, pGid, GPU_UUID_LEN);
+ }
+ }
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_register_callback(
+ nvidia_stack_t *sp,
+ NvU64 p2pToken,
+ NvU64 gpuVirtualAddress,
+ NvU64 length,
+ void *pPlatformData,
+ void (*pFreeCallback)(void *pData),
+ void *pData
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+ {
+ rmStatus = RmP2PRegisterCallback(p2pToken, gpuVirtualAddress, length,
+ pPlatformData, pFreeCallback, pData);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(
+ nvidia_stack_t *sp,
+ void *p2pObject,
+ void *pKey
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+ {
+ rmStatus = RmP2PPutPagesPersistent(p2pObject, pKey);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_put_pages(
+ nvidia_stack_t *sp,
+ NvU64 p2pToken,
+ NvU32 vaSpaceToken,
+ NvU64 gpuVirtualAddress,
+ void *pKey
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+ {
+ rmStatus = RmP2PPutPages(p2pToken,
+ vaSpaceToken,
+ gpuVirtualAddress,
+ pKey);
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+char* NV_API_CALL rm_get_gpu_uuid(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ NV_STATUS rmStatus;
+ const NvU8 *pGid;
+ char *pGidString;
+
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // Allocate space for the ASCII string
+ rmStatus = os_alloc_mem((void **)&pGidString, GPU_UUID_ASCII_LEN);
+ if (rmStatus != NV_OK)
+ {
+ pGidString = NULL;
+ goto done;
+ }
+
+ // Get the raw UUID; note the pGid is cached, so we do not need to free it
+ pGid = RmGetGpuUuidRaw(nv);
+
+ if (pGid != NULL)
+ {
+ // Convert the raw UUID to ASCII
+ rmStatus = RmGpuUuidRawToString(pGid, pGidString, GPU_UUID_ASCII_LEN);
+ if (rmStatus != NV_OK)
+ {
+ os_free_mem(pGidString);
+ pGidString = NULL;
+ }
+ }
+ else
+ {
+ const char *pTmpString = "GPU-???????\?-???\?-???\?-???\?-????????????";
+
+ portStringCopy(pGidString, GPU_UUID_ASCII_LEN, pTmpString,
+ portStringLength(pTmpString) + 1);
+ }
+
+done:
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return pGidString;
+}
+
+//
+// This function will return the UUID in the binary format
+//
+const NvU8 * NV_API_CALL rm_get_gpu_uuid_raw(
+ nvidia_stack_t *sp,
+ nv_state_t *nv)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ const NvU8 *pGid;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pGid = RmGetGpuUuidRaw(nv);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return pGid;
+}
+
+static void rm_set_firmware_logs(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ NV_STATUS status;
+ NvU32 data;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ status = RmReadRegistryDword(nv, NV_REG_ENABLE_GPU_FIRMWARE_LOGS, &data);
+ if (status == NV_OK)
+ {
+ if ((data == NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE)
+#if defined(DEBUG) || defined(DEVELOP)
+ || (data == NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG)
+#endif
+ )
+ {
+ nv->enable_firmware_logs = NV_TRUE;
+ }
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_set_rm_firmware_requested(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ nv->request_firmware = NV_TRUE;
+ nv->allow_fallback_to_monolithic_rm = NV_FALSE;
+
+ // Check if we want firmware logs
+ if (nv->request_firmware)
+ rm_set_firmware_logs(sp, nv);
+}
+
+//
+// This function will be called by nv_procfs_read_gpu_info().
+// nv_procfs_read_gpu_info() will not print the 'GPU Firmware:' field at
+// all if the 'version' string is empty.
+//
+// If GSP is enabled (firmware was requested), this function needs to return
+// the firmware version or "NA" in case of any errors.
+//
+// If GSP is not enabled (firmware was not requested), this function needs to
+// return the empty string, regardless of error cases.
+//
+void NV_API_CALL rm_get_firmware_version(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ char *version,
+ NvLength version_length
+)
+{
+ NV2080_CTRL_GSP_GET_FEATURES_PARAMS params = { 0 };
+ RM_API *pRmApi;
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus = NV_OK;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_GPU);
+ if (pRmApi != NULL)
+ {
+ rmStatus = pRmApi->Control(pRmApi,
+ nv->rmapi.hClient,
+ nv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_GSP_GET_FEATURES,
+ &params,
+ sizeof(params));
+
+ RmUnixRmApiEpilogue(nv, &threadState);
+ }
+ else
+ {
+ rmStatus = NV_ERR_INVALID_STATE;
+ }
+
+ if (rmStatus != NV_OK)
+ {
+ if (RMCFG_FEATURE_GSP_CLIENT_RM && nv->request_firmware)
+ {
+ const char *pTmpString = "N/A";
+ portStringCopy(version, version_length, pTmpString, portStringLength(pTmpString) + 1);
+ }
+ NV_PRINTF(LEVEL_INFO,
+ "%s: Failed to query gpu build versions, status=0x%x\n",
+ __FUNCTION__,
+ rmStatus);
+ goto finish;
+ }
+ portMemCopy(version, version_length, params.firmwareVersion, sizeof(params.firmwareVersion));
+
+finish:
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+//
+// disable GPU SW state persistence
+//
+
+void NV_API_CALL rm_disable_gpu_state_persistence(nvidia_stack_t *sp, nv_state_t *nv)
+{
+ THREAD_STATE_NODE threadState;
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE, NV_FALSE);
+ osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, NV_FALSE);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_log_gpu_crash(
+ nv_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS status;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if ((status = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DIAG)) == NV_OK)
+ {
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ if ((pGpu != NULL) &&
+ ((status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DIAG)) == NV_OK))
+ {
+ status = RmLogGpuCrash(pGpu);
+
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return status;
+}
+
+void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd)
+{
+ nvidia_kernel_rmapi_ops_t *ops = ops_cmd;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ switch (ops->op)
+ {
+ case NV01_FREE:
+ Nv01FreeKernel(&ops->params.free);
+ break;
+
+ case NV01_ALLOC_MEMORY:
+ Nv01AllocMemoryKernel(&ops->params.allocMemory64);
+ break;
+
+ case NV04_ALLOC:
+ Nv04AllocKernel(&ops->params.alloc);
+ break;
+
+ case NV04_VID_HEAP_CONTROL:
+ Nv04VidHeapControlKernel(ops->params.pVidHeapControl);
+ break;
+
+ case NV04_MAP_MEMORY:
+ Nv04MapMemoryKernel(&ops->params.mapMemory);
+ break;
+
+ case NV04_UNMAP_MEMORY:
+ Nv04UnmapMemoryKernel(&ops->params.unmapMemory);
+ break;
+
+ case NV04_ALLOC_CONTEXT_DMA:
+ Nv04AllocContextDmaKernel(&ops->params.allocContextDma2);
+ break;
+
+ case NV04_MAP_MEMORY_DMA:
+ Nv04MapMemoryDmaKernel(&ops->params.mapMemoryDma);
+ break;
+
+ case NV04_UNMAP_MEMORY_DMA:
+ Nv04UnmapMemoryDmaKernel(&ops->params.unmapMemoryDma);
+ break;
+
+ case NV04_BIND_CONTEXT_DMA:
+ Nv04BindContextDmaKernel(&ops->params.bindContextDma);
+ break;
+
+ case NV04_CONTROL:
+ Nv04ControlKernel(&ops->params.control);
+ break;
+
+ case NV04_DUP_OBJECT:
+ Nv04DupObjectKernel(&ops->params.dupObject);
+ break;
+
+ case NV04_SHARE:
+ Nv04ShareKernel(&ops->params.share);
+ break;
+
+ case NV04_ADD_VBLANK_CALLBACK:
+ Nv04AddVblankCallbackKernel(&ops->params.addVblankCallback);
+ break;
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+//
+// ACPI method (NVIF/_DSM/WMMX/MXM*/etc.) initialization
+//
+void RmInitAcpiMethods(OBJOS *pOS, OBJSYS *pSys, OBJGPU *pGpu)
+{
+ NvU32 handlesPresent;
+
+ if (pSys->getProperty(pSys, PDB_PROP_SYS_NVIF_INIT_DONE))
+ return;
+
+ nv_acpi_methods_init(&handlesPresent);
+
+ // Check if NVPCF _DSM functions are implemented under NVPCF or GPU device scope.
+ RmCheckNvpcfDsmScope(pGpu);
+ acpiDsmInit(pGpu);
+
+}
+
+//
+// ACPI method (NVIF/_DSM/WMMX/MXM*/etc.) teardown
+//
+void RmUnInitAcpiMethods(OBJSYS *pSys)
+{
+ pSys->setProperty(pSys, PDB_PROP_SYS_NVIF_INIT_DONE, NV_FALSE);
+
+ nv_acpi_methods_uninit();
+}
+
+//
+// Converts an array of OS page address to an array of RM page addresses.This
+// assumes that:
+// (1) The pteArray is at least pageCount entries large,
+// (2) The pageCount is given in RM pages, and
+// (3) The OS page entries start at index 0.
+//
+void RmInflateOsToRmPageArray(RmPhysAddr *pteArray, NvU64 pageCount)
+{
+ NvUPtr osPageIdx, osPageOffset;
+ NvU64 i;
+
+ //
+ // We can do the translation in place by moving backwards, since there
+ // will always be more RM pages than OS pages
+ //
+ for (i = pageCount - 1; i != NV_U64_MAX; i--)
+ {
+ osPageIdx = i >> NV_RM_TO_OS_PAGE_SHIFT;
+ osPageOffset = (i & ((1 << NV_RM_TO_OS_PAGE_SHIFT) - 1)) *
+ NV_RM_PAGE_SIZE;
+ pteArray[i] = pteArray[osPageIdx] + osPageOffset;
+ }
+}
+
+void RmDeflateRmToOsPageArray(RmPhysAddr *pteArray, NvU64 pageCount)
+{
+ NvU64 i;
+
+ for (i = 0; i < NV_RM_PAGES_TO_OS_PAGES(pageCount); i++)
+ {
+ pteArray[i] = pteArray[(i << NV_RM_TO_OS_PAGE_SHIFT)];
+ }
+
+ // Zero out the rest of the addresses, which are now invalid
+ portMemSet(pteArray + i, 0, sizeof(*pteArray) * (pageCount - i));
+}
+
+NvBool NV_API_CALL
+rm_get_device_remove_flag
+(
+ nvidia_stack_t * sp,
+ NvU32 gpu_id
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ NvBool bRemove;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if (gpumgrQueryGpuDrainState(gpu_id, NULL, &bRemove) != NV_OK)
+ {
+ bRemove = NV_FALSE;
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return bRemove;
+}
+
+NvBool NV_API_CALL
+rm_gpu_need_4k_page_isolation
+(
+ nv_state_t *nv
+)
+{
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+ return nvp->b_4k_page_isolation_required;
+}
+
+NV_STATUS NV_API_CALL rm_get_gpu_numa_info(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvS32 *pNid,
+ NvU64 *pNumaMemAddr,
+ NvU64 *pNumaMemSize,
+ NvU64 *pOfflineAddresses,
+ NvU32 *pOfflineAddressesCount
+)
+{
+ NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS *pParams;
+ RM_API *pRmApi;
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ NV_STATUS status = NV_OK;
+
+ if ((pNid == NULL) || (pNumaMemAddr == NULL) || (pNumaMemSize == NULL))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ if ((pOfflineAddressesCount != NULL) &&
+ ((pOfflineAddresses == NULL) ||
+ (*pOfflineAddressesCount > NV_ARRAY_ELEMENTS(pParams->numaOfflineAddresses))))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pParams = portMemAllocNonPaged(sizeof(*pParams));
+ if (pParams == NULL)
+ {
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return NV_ERR_NO_MEMORY;
+ }
+
+ portMemSet(pParams, 0, sizeof(*pParams));
+
+ if (pOfflineAddressesCount != NULL)
+ {
+ pParams->numaOfflineAddressesCount = *pOfflineAddressesCount;
+ }
+
+ pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM);
+ if (pRmApi == NULL)
+ {
+ status = NV_ERR_INVALID_STATE;
+ goto finish;
+ }
+
+ status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_FB_GET_NUMA_INFO,
+ pParams, sizeof(*pParams));
+
+ RmUnixRmApiEpilogue(nv, &threadState);
+
+ if (status == NV_OK)
+ {
+ NvU32 i;
+
+ *pNid = pParams->numaNodeId;
+ *pNumaMemAddr = pParams->numaMemAddr;
+ *pNumaMemSize = pParams->numaMemSize;
+ *pOfflineAddressesCount = pParams->numaOfflineAddressesCount;
+
+ for (i = 0; i < pParams->numaOfflineAddressesCount; i++)
+ {
+ pOfflineAddresses[i] = pParams->numaOfflineAddresses[i];
+ }
+ }
+
+finish:
+ portMemFree(pParams);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return status;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_numa_online(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS params = { 0 };
+ RM_API *pRmApi;
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ NV_STATUS status = NV_OK;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM);
+ if (pRmApi == NULL)
+ {
+ status = NV_ERR_INVALID_STATE;
+ goto finish;
+ }
+
+ params.bOnline = NV_TRUE;
+
+ status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS,
+ &params, sizeof(params));
+
+ RmUnixRmApiEpilogue(nv, &threadState);
+
+finish:
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return status;
+}
+
+
+NV_STATUS NV_API_CALL rm_gpu_numa_offline(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS params = { 0 };
+ RM_API *pRmApi;
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ NV_STATUS status = NV_OK;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM);
+ if (pRmApi == NULL)
+ {
+ status = NV_ERR_INVALID_STATE;
+ goto finish;
+ }
+
+ params.bOnline = NV_FALSE;
+
+ status = pRmApi->Control(pRmApi, nv->rmapi.hClient,
+ nv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS,
+ &params, sizeof(params));
+
+ RmUnixRmApiEpilogue(nv, &threadState);
+
+finish:
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return status;
+}
+
+//
+// A device is considered "sequestered" if it has drain state enabled for it.
+// The kernel interface layer can use this to check the drain state of a device
+// in paths outside of initialization, e.g., when clients attempt to reference
+// count the device.
+//
+NvBool NV_API_CALL rm_is_device_sequestered(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ NvBool bDrain = NV_FALSE;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU) == NV_OK)
+ {
+ //
+ // If gpumgrQueryGpuDrainState succeeds, bDrain will be set as needed.
+ // If gpumgrQueryGpuDrainState fails, bDrain will stay false; we assume
+ // that if core RM can't tell us the drain state, it must not be
+ // attached and the "sequestered" question is not relevant.
+ //
+ (void) gpumgrQueryGpuDrainState(pNv->gpu_id, &bDrain, NULL);
+
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return bDrain;
+}
+
+void NV_API_CALL rm_check_for_gpu_surprise_removal(
+ nvidia_stack_t *sp,
+ nv_state_t *nv
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ NV_STATUS rmStatus;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock.
+ if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU)) == NV_OK)
+ {
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ if ((rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GPU)) == NV_OK)
+ {
+ osHandleGpuLost(pGpu);
+ rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+
+ // UNLOCK: release api lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_set_external_kernel_client_count(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv,
+ NvBool bIncr
+)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+ OBJGPU *pGpu;
+ NV_STATUS rmStatus = NV_OK;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+ if (pGpu != NULL)
+ {
+ rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE,
+ RM_LOCK_MODULES_GPU);
+ if (rmStatus == NV_OK)
+ {
+ rmStatus = gpuSetExternalKernelClientCount(pGpu, bIncr);
+ rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NvBool rm_get_uefi_console_status(
+ nv_state_t *nv
+)
+{
+ NvU16 fbWidth, fbHeight, fbDepth, fbPitch;
+ NvU64 fbSize;
+ NvU64 fbBaseAddress = 0;
+ NvBool bConsoleDevice = NV_FALSE;
+
+ // os_get_screen_info() will return dimensions and an address for
+ // any fbdev driver (e.g., efifb, vesafb, etc). To find if this is a
+ // UEFI console check the fbBaseAddress: if it was set up by the EFI GOP
+ // driver, it will point into BAR1 (FB); if it was set up by the VBIOS,
+ // it will point to BAR2 + 16MB.
+ os_get_screen_info(&fbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch,
+ nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address,
+ nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000);
+
+ fbSize = fbHeight * fbPitch;
+
+ bConsoleDevice = (fbSize != 0);
+
+ return bConsoleDevice;
+}
+
+NvU64 rm_get_uefi_console_size(
+ nv_state_t *nv,
+ NvU64 *pFbBaseAddress
+)
+{
+ NvU16 fbWidth, fbHeight, fbDepth, fbPitch;
+ NvU64 fbSize;
+
+ fbSize = fbWidth = fbHeight = fbDepth = fbPitch = 0;
+
+ // os_get_screen_info() will return dimensions and an address for
+ // any fbdev driver (e.g., efifb, vesafb, etc). To find if this is a
+ // UEFI console check the fbBaseAddress: if it was set up by the EFI GOP
+ // driver, it will point into BAR1 (FB); if it was set up by the VBIOS,
+ // it will point to BAR2 + 16MB.
+ os_get_screen_info(pFbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch,
+ nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address,
+ nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000);
+
+ fbSize = fbHeight * fbPitch;
+
+ return fbSize;
+}
+
+/*
+ * IOMMU needs to be present on the server to support SR-IOV vGPU, unless
+ * we have SR-IOV enabled for remote GPU.
+ */
+
+NvBool NV_API_CALL rm_is_iommu_needed_for_sriov(
+ nvidia_stack_t *sp,
+ nv_state_t * nv
+)
+{
+ OBJGPU *pGpu;
+ NvU32 data;
+ NvBool ret = NV_TRUE;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ if (osReadRegistryDword(pGpu, NV_REG_STR_RM_REMOTE_GPU, &data) == NV_OK)
+ {
+ if (data == NV_REG_STR_RM_REMOTE_GPU_ENABLE)
+ ret = NV_FALSE;
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return ret;
+}
+
+NvBool NV_API_CALL rm_disable_iomap_wc(void)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ return pSys->pCl->getProperty(pSys, PDB_PROP_CL_DISABLE_IOMAP_WC) == NV_TRUE;
+}
+
+//
+// Verifies the handle, offset and size and dups hMemory.
+// Must be called with API lock and GPU lock held.
+//
+NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvHandle hSrcClient,
+ NvHandle hDstClient,
+ NvHandle hDevice,
+ NvHandle hSubdevice,
+ void *pGpuInstanceInfo,
+ NvHandle hMemory,
+ NvU64 offset,
+ NvU64 size,
+ NvHandle *phMemoryDuped
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ OBJGPU *pGpu;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ NV_ASSERT(rmapiLockIsOwner());
+
+ NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
+
+ rmStatus = RmDmabufVerifyMemHandle(pGpu, hSrcClient, hMemory,
+ offset, size, pGpuInstanceInfo);
+ if (rmStatus == NV_OK)
+ {
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+ NvHandle hMemoryDuped = 0;
+
+ rmStatus = pRmApi->DupObject(pRmApi,
+ hDstClient,
+ hDevice,
+ &hMemoryDuped,
+ hSrcClient,
+ hMemory,
+ 0);
+ if (rmStatus == NV_OK)
+ {
+ *phMemoryDuped = hMemoryDuped;
+ }
+ else if (rmStatus == NV_ERR_INVALID_OBJECT_PARENT)
+ {
+ hMemoryDuped = 0;
+
+ // If duping under Device fails, try duping under Subdevice
+ rmStatus = pRmApi->DupObject(pRmApi,
+ hDstClient,
+ hSubdevice,
+ &hMemoryDuped,
+ hSrcClient,
+ hMemory,
+ 0);
+ if (rmStatus == NV_OK)
+ {
+ *phMemoryDuped = hMemoryDuped;
+ }
+ }
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+//
+// Frees dup'd hMemory.
+// Must be called with API lock and GPU lock held.
+//
+void NV_API_CALL rm_dma_buf_undup_mem_handle(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvHandle hClient,
+ NvHandle hMemory
+)
+{
+ THREAD_STATE_NODE threadState;
+ RM_API *pRmApi;
+ OBJGPU *pGpu;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ NV_ASSERT(rmapiLockIsOwner());
+
+ NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
+
+ pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+
+ pRmApi->Free(pRmApi, hClient, hMemory);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+//
+// Maps a handle to BAR1.
+// Must be called with API lock and GPU lock held.
+//
+NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvHandle hClient,
+ NvHandle hMemory,
+ NvU64 offset,
+ NvU64 size,
+ NvU64 *pBar1Va
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ OBJGPU *pGpu;
+ KernelBus *pKernelBus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+
+ NV_ASSERT(rmapiLockIsOwner());
+
+ NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
+
+ rmStatus = kbusMapFbApertureByHandle(pGpu, pKernelBus, hClient,
+ hMemory, offset, size, pBar1Va);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+//
+// Unmaps a handle from BAR1.
+// Must be called with API lock and GPU lock held.
+//
+NV_STATUS NV_API_CALL rm_dma_buf_unmap_mem_handle(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvHandle hClient,
+ NvHandle hMemory,
+ NvU64 size,
+ NvU64 bar1Va
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ OBJGPU *pGpu;
+ KernelBus *pKernelBus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+
+ NV_ASSERT(rmapiLockIsOwner());
+
+ NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
+
+ rmStatus = kbusUnmapFbApertureByHandle(pGpu, pKernelBus, hClient,
+ hMemory, bar1Va);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvHandle hClient,
+ NvHandle *phClient,
+ NvHandle *phDevice,
+ NvHandle *phSubdevice,
+ void **ppGpuInstanceInfo
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+ if (rmStatus == NV_OK)
+ {
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+ if (rmStatus == NV_OK)
+ {
+ rmStatus = RmDmabufGetClientAndDevice(pGpu, hClient, phClient, phDevice,
+ phSubdevice, ppGpuInstanceInfo);
+
+ rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return rmStatus;
+}
+
+void NV_API_CALL rm_dma_buf_put_client_and_device(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvHandle hClient,
+ NvHandle hDevice,
+ NvHandle hSubdevice,
+ void *pGpuInstanceInfo
+)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ // LOCK: acquire API lock
+ rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+ if (rmStatus == NV_OK)
+ {
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+ if (rmStatus == NV_OK)
+ {
+ RmDmabufPutClientAndDevice(pGpu, hClient, hDevice, hSubdevice,
+ pGpuInstanceInfo);
+
+ rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+
+ // UNLOCK: release API lock
+ rmapiLockRelease();
+ }
+ NV_ASSERT_OK(rmStatus);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+//
+// Fetches GSP ucode data for usage during RM Init
+// NOTE: Used only on VMWware
+//
+
+NvBool NV_API_CALL rm_is_altstack_in_use(void)
+{
+#if defined(__use_altstack__)
+ return NV_TRUE;
+#else
+ return NV_FALSE;
+#endif
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osinit.c b/userspace/sdks/nv525/nvalloc/unix/src/osinit.c
new file mode 100644
index 0000000..5080e50
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/osinit.c
@@ -0,0 +1,2202 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**************************************************************************************************************
+*
+* Description:
+* UNIX-general, device-independent initialization code for
+* the resource manager.
+*
+*
+**************************************************************************************************************/
+
+#include <nv_ref.h>
+#include <nv.h> // NV device driver interface
+#include <nv-reg.h>
+#include <nv-priv.h>
+#include <nvos.h>
+#include <nvrm_registry.h>
+#include <platform/sli/sli.h>
+#include <nverror.h>
+#include <osapi.h>
+#include <core/system.h>
+#include <os/os.h>
+#include "gpu/gpu.h"
+#include <objtmr.h>
+#include "gpu/bus/kern_bus.h"
+#include "nverror.h"
+#include <gpu/bif/kernel_bif.h>
+#include <nv-nb-regs.h>
+#include <gpu/mem_mgr/mem_mgr.h>
+#include <gpu/mem_sys/kern_mem_sys.h>
+#include "kernel/gpu/intr/intr.h"
+#include <mem_mgr/io_vaspace.h>
+#include <gpu/disp/kern_disp.h>
+#include <platform/chipset/chipset.h>
+#include <kernel/gpu/rc/kernel_rc.h>
+#include <kernel/gpu/fifo/kernel_fifo.h>
+#include <nv-firmware-chip-family-select.h>
+#include <gpu/gsp/kernel_gsp.h>
+#include "liblogdecode.h"
+#include <gpu/fsp/kern_fsp.h>
+
+#include <mem_mgr/virt_mem_mgr.h>
+#include <virtualization/kernel_vgpu_mgr.h>
+
+#include <rmosxfac.h>
+#include <gpu_mgr/gpu_mgr.h>
+#include <core/thread_state.h>
+#include <core/locks.h>
+#include <rmapi/client.h>
+#include <ctrl/ctrl2080/ctrl2080gpu.h>
+#include <class/cl00f2.h>
+
+#include <platform/chipset/pci_pbi.h>
+
+#include "platform/nbsi/nbsi_read.h"
+#include "gpu_mgr/gpu_db.h"
+#include <class/cl0080.h>
+#include <class/cl0073.h>
+#include <class/cl2080.h>
+#include <class/cl402c.h>
+
+#include <gpu/dce_client/dce_client.h>
+// RMCONFIG: need definition of REGISTER_ALL_HALS()
+#include "g_hal_register.h"
+
+typedef enum
+{
+ RM_INIT_OK,
+
+ /* general os errors */
+ RM_INIT_REG_SETUP_FAILED = 0x10,
+ RM_INIT_SYS_ENVIRONMENT_FAILED,
+
+ /* gpu errors */
+ RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED = 0x20,
+ RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED,
+ RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED,
+ RM_INIT_GPU_PRE_INIT_FAILED,
+ RM_INIT_GPU_STATE_INIT_FAILED,
+ RM_INIT_GPU_LOAD_FAILED,
+ RM_INIT_GPU_UNIVERSAL_VALIDATION_FAILED,
+ RM_INIT_GPU_DMA_CONFIGURATION_FAILED,
+
+ /* vbios errors */
+ RM_INIT_VBIOS_FAILED = 0x30,
+ RM_INIT_VBIOS_POST_FAILED,
+ RM_INIT_VBIOS_X86EMU_FAILED,
+
+ /* scalability errors */
+ RM_INIT_SCALABILITY_FAILED = 0x40,
+
+ /* general core rm errors */
+ RM_INIT_WATCHDOG_FAILED,
+ RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED,
+ RM_GPUDB_REGISTER_FAILED,
+
+ RM_INIT_ALLOC_RMAPI_FAILED,
+ RM_INIT_GPUINFO_WITH_RMAPI_FAILED,
+
+ /* rm firmware errors */
+ RM_INIT_FIRMWARE_POLICY_FAILED = 0x60,
+ RM_INIT_FIRMWARE_FETCH_FAILED,
+ RM_INIT_FIRMWARE_INIT_FAILED,
+
+ RM_INIT_MAX_FAILURES
+} rm_init_status;
+
+typedef rm_init_status RM_INIT_STATUS;
+
+typedef struct {
+ RM_INIT_STATUS initStatus;
+ NV_STATUS rmStatus;
+ NvU32 line;
+} UNIX_STATUS;
+
+#define INIT_UNIX_STATUS { RM_INIT_OK, NV_OK, 0 }
+#define RM_INIT_SUCCESS(init) ((init) == RM_INIT_OK)
+
+#define RM_SET_ERROR(status, err) { (status).initStatus = (err); \
+ (status).line = __LINE__; }
+
+
+//
+// GPU architectures support DMA addressing up to a certain address width,
+// above which all other bits in any given DMA address must not vary
+// (e.g., all 0). This value is the minimum of the DMA addressing
+// capabilities, in number of physical address bits, for all supported
+// GPU architectures.
+//
+#define NV_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH 36
+
+//
+// All GPU architectures with GSP support at least 47 physical address bits.
+//
+#define NV_GSP_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH 47
+
+static inline NvU64 nv_encode_pci_info(nv_pci_info_t *pci_info)
+{
+ return gpuEncodeDomainBusDevice(pci_info->domain, pci_info->bus, pci_info->slot);
+}
+
+static inline NvU32 nv_generate_id_from_pci_info(nv_pci_info_t *pci_info)
+{
+ return gpuGenerate32BitId(pci_info->domain, pci_info->bus, pci_info->slot);
+}
+
+static inline void nv_os_map_kernel_space(nv_state_t *nv, nv_aperture_t *aperture)
+{
+ NV_ASSERT(aperture->map == NULL);
+
+ // let's start off assuming a standard device and map the registers
+ // normally. It is unfortunate to hard-code the register size here, but we don't
+ // want to fail trying to map all of a multi-devices' register space
+ aperture->map = osMapKernelSpace(aperture->cpu_address,
+ aperture->size,
+ NV_MEMORY_UNCACHED,
+ NV_PROTECT_READ_WRITE);
+ aperture->map_u = (nv_phwreg_t)aperture->map;
+}
+
+// local prototypes
+static NV_STATUS initCoreLogic(OBJGPU *);
+static void initVendorSpecificRegistry(OBJGPU *, NvU16);
+static NV_STATUS teardownCoreLogic(OBJOS *, OBJGPU *);
+static void initUnixSpecificRegistry(OBJGPU *);
+
+NvBool osRmInitRm(OBJOS *pOS)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ NvU64 system_memory_size = (NvU64)-1;
+
+ NV_PRINTF(LEVEL_INFO, "init rm\n");
+
+ if (os_is_efi_enabled())
+ {
+ pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_TRUE);
+ }
+
+ // have to init this before the debug subsystem, which will
+ // try to check the value of ResmanDebugLevel
+ RmInitRegistry();
+
+ NvU32 data = 0;
+ if ((osReadRegistryDword(NULL,
+ NV_REG_ENABLE_PCIE_GEN3, &data) == NV_OK) &&
+ (data != 0))
+ {
+ data = DRF_DEF(_REG_STR, _RM_PCIE, _LINK_SPEED_ALLOW_GEN3, _ENABLE);
+
+ osWriteRegistryDword(NULL,
+ NV_REG_STR_RM_PCIE_LINK_SPEED, data);
+ }
+
+ if (!(osReadRegistryDword(NULL,
+ NV_REG_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS,
+ &data) == NV_OK) || data)
+ {
+ pSys->setProperty(pSys,
+ PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS,
+ NV_TRUE);
+ }
+
+ if ((osReadRegistryDword(NULL,
+ NV_REG_ENABLE_STREAM_MEMOPS,
+ &data) == NV_OK) && (data != 0))
+ {
+ data = DRF_DEF(_REG_STR, _RM_STREAM_MEMOPS, _ENABLE, _YES);
+ osWriteRegistryDword(NULL, NV_REG_STR_RM_STREAM_MEMOPS, data);
+ }
+
+ if ((osReadRegistryDword(NULL,
+ NV_REG_STR_RM_FORCE_BAR_PATH,
+ &data) == NV_OK) && (data == 1))
+ {
+ //
+ // Exposing the GPU memory as NUMA node memory requires coherent CPU
+ // mappings over NVLink. If those mappings are disabled, also disable
+ // NUMA-onlining of the device memory.
+ //
+ osWriteRegistryDword(NULL,
+ NV_REG_ENABLE_USER_NUMA_MANAGEMENT, 0);
+ }
+
+ if ((osReadRegistryDword(NULL,
+ NV_REG_NVLINK_DISABLE, &data) == NV_OK) &&
+ (data == 1))
+ {
+ osWriteRegistryDword(NULL, NV_REG_STR_RM_NVLINK_CONTROL, 0x1);
+ }
+
+ if ((osReadRegistryDword(NULL,
+ NV_REG_ENABLE_PCIE_RELAXED_ORDERING_MODE, &data) == NV_OK) &&
+ (data == 1))
+ {
+ osWriteRegistryDword(NULL,
+ NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING,
+ NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING_ENABLE);
+ }
+
+ // init the debug subsystem if necessary
+ os_dbg_init();
+ nvDbgInitRmMsg(NULL);
+
+ // Force nvlog reinit since module params are now available
+ NVLOG_UPDATE();
+
+ // Register all supported hals
+ if (REGISTER_ALL_HALS() != NV_OK)
+ {
+ RmDestroyRegistry(NULL);
+ return NV_FALSE;
+ }
+
+ system_memory_size = NV_RM_PAGES_PER_OS_PAGE * os_get_num_phys_pages();
+
+ // if known, relay the number of system memory pages (in terms of RM page
+ // size) to the RM; this is needed for e.g. TurboCache parts.
+ if (system_memory_size != (NvU64)-1)
+ pOS->SystemMemorySize = system_memory_size;
+
+ // Setup any ThreadState defaults
+ threadStateInitSetupFlags(THREAD_STATE_SETUP_FLAGS_ENABLED |
+ THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED |
+ THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED |
+ THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED);
+
+ return NV_TRUE;
+}
+
+void RmShutdownRm(void)
+{
+ NV_PRINTF(LEVEL_INFO, "shutdown rm\n");
+
+ RmDestroyRegistry(NULL);
+
+ // Free objects created with RmInitRm, including the system object
+ RmDestroyRm();
+}
+
+//
+// osAttachGpu
+//
+// This routine is used as a callback by the gpumgrAttachGpu
+// interface to allow os-dependent code to set up any state
+// before engine construction begins.
+//
+NV_STATUS osAttachGpu(
+ OBJGPU *pGpu,
+ void *pOsGpuInfo
+)
+{
+ nv_state_t *nv = (nv_state_t *)pOsGpuInfo;
+ nv_priv_t *nvp;
+
+ nvp = NV_GET_NV_PRIV(nv);
+
+ nvp->pGpu = pGpu;
+
+ NV_SET_NV_STATE(pGpu, (void *)nv);
+
+ initUnixSpecificRegistry(pGpu);
+
+ // Assign default values to Registry keys for VGX
+ if (os_is_vgx_hyper())
+ {
+ initVGXSpecificRegistry(pGpu);
+ }
+
+ return NV_OK;
+}
+
+NV_STATUS osDpcAttachGpu(
+ OBJGPU *pGpu,
+ void *pOsGpuInfo
+)
+{
+ return NV_OK; // Nothing to do for unix
+}
+
+void osDpcDetachGpu(
+ OBJGPU *pGpu
+)
+{
+ return; // Nothing to do for unix
+}
+
+NV_STATUS
+osHandleGpuLost
+(
+ OBJGPU *pGpu
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ NvU32 pmc_boot_0;
+
+ // Determine if we've already run the handler
+ if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED))
+ {
+ return NV_OK;
+ }
+
+ pmc_boot_0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0);
+ if (pmc_boot_0 != nvp->pmc_boot_0)
+ {
+ //
+ // This doesn't support PEX Reset and Recovery yet.
+ // This will help to prevent accessing registers of a GPU
+ // which has fallen off the bus.
+ //
+ nvErrorLog_va((void *)pGpu, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS,
+ "GPU has fallen off the bus.");
+
+ NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU has fallen off the bus.\n");
+
+ if (pGpu->boardInfo != NULL && pGpu->boardInfo->serialNumber[0] != '\0')
+ {
+ NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
+ "GPU serial number is %s.\n",
+ pGpu->boardInfo->serialNumber);
+ }
+
+ gpuSetDisconnectedProperties(pGpu);
+
+ // Trigger the OS's PCI recovery mechanism
+ if (nv_pci_trigger_recovery(nv) != NV_OK)
+ {
+ //
+ // Initiate a crash dump immediately, since the OS doesn't appear
+ // to have a mechanism wired up for attempted recovery.
+ //
+ (void) RmLogGpuCrash(pGpu);
+ }
+ else
+ {
+ //
+ // Make the SW state stick around until the recovery can start, but
+ // don't change the PDB property: this is only used to report to
+ // clients whether or not persistence mode is enabled, and we'll
+ // need it after the recovery callbacks to restore the correct
+ // persistence mode for the GPU.
+ //
+ osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, NV_TRUE);
+ }
+
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJCL *pCl = SYS_GET_CL(pSys);
+ // Set SURPRISE_REMOVAL flag for eGPU to help in device removal.
+ if ((pCl != NULL) &&
+ pCl->getProperty(pCl, PDB_PROP_CL_IS_EXTERNAL_GPU))
+ {
+ nv->flags |= NV_FLAG_IN_SURPRISE_REMOVAL;
+ }
+ DBG_BREAKPOINT();
+ }
+
+ return NV_OK;
+}
+
+/*
+ * Initialize the required GPU information by doing RMAPI control calls
+ * and store the same in the UNIX specific data structures.
+ */
+static NV_STATUS
+RmInitGpuInfoWithRmApi
+(
+ OBJGPU *pGpu
+)
+{
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ NV2080_CTRL_GPU_GET_INFO_V2_PARAMS *pGpuInfoParams = { 0 };
+ NV_STATUS status;
+
+ // LOCK: acquire GPUs lock
+ status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ pGpuInfoParams = portMemAllocNonPaged(sizeof(*pGpuInfoParams));
+ if (pGpuInfoParams == NULL)
+ {
+ // UNLOCK: release GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+ return NV_ERR_NO_MEMORY;
+ }
+
+
+ portMemSet(pGpuInfoParams, 0, sizeof(*pGpuInfoParams));
+
+ pGpuInfoParams->gpuInfoListSize = 3;
+ pGpuInfoParams->gpuInfoList[0].index = NV2080_CTRL_GPU_INFO_INDEX_4K_PAGE_ISOLATION_REQUIRED;
+ pGpuInfoParams->gpuInfoList[1].index = NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED;
+ pGpuInfoParams->gpuInfoList[2].index = NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY;
+
+ status = pRmApi->Control(pRmApi, nv->rmapi.hClient,
+ nv->rmapi.hSubDevice,
+ NV2080_CTRL_CMD_GPU_GET_INFO_V2,
+ pGpuInfoParams, sizeof(*pGpuInfoParams));
+
+ if (status == NV_OK)
+ {
+ nvp->b_4k_page_isolation_required =
+ (pGpuInfoParams->gpuInfoList[0].data ==
+ NV2080_CTRL_GPU_INFO_INDEX_4K_PAGE_ISOLATION_REQUIRED_YES);
+ nvp->b_mobile_config_enabled =
+ (pGpuInfoParams->gpuInfoList[1].data ==
+ NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_YES);
+ nv->dma_buf_supported =
+ (pGpuInfoParams->gpuInfoList[2].data ==
+ NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES);
+ }
+
+ portMemFree(pGpuInfoParams);
+
+ // UNLOCK: release GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+ return status;
+}
+
+static void RmSetSocDispDeviceMappings(
+ GPUATTACHARG *gpuAttachArg,
+ nv_state_t *nv
+)
+{
+ gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvAddr = (GPUHWREG*) nv->regs->map;
+ gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvPAddr = nv->regs->cpu_address;
+ gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvLength = (NvU32) nv->regs->size;
+}
+
+static void RmSetSocDpauxDeviceMappings(
+ GPUATTACHARG *gpuAttachArg,
+ nv_state_t *nv
+)
+{
+}
+
+static void RmSetSocHdacodecDeviceMappings(
+ GPUATTACHARG *gpuAttachArg,
+ nv_state_t *nv
+)
+{
+}
+
+static void RmSetSocMipiCalDeviceMappings(
+ GPUATTACHARG *gpuAttachArg,
+ nv_state_t *nv
+)
+{
+}
+
+static void
+osInitNvMapping(
+ nv_state_t *nv,
+ NvU32 *pDeviceReference,
+ UNIX_STATUS *status
+)
+{
+ OBJGPU *pGpu;
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ GPUATTACHARG *gpuAttachArg;
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ NvU32 deviceInstance;
+ NvU32 data = 0;
+
+ NV_PRINTF(LEVEL_INFO, "osInitNvMapping:\n");
+
+ // allocate the next available gpu device number
+ status->rmStatus = gpumgrAllocGpuInstance(pDeviceReference);
+ if (status->rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "*** Cannot get valid gpu instance\n");
+ RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED);
+ return;
+ }
+
+ // RM_BASIC_LOCK_MODEL: allocate GPU lock
+ status->rmStatus = rmGpuLockAlloc(*pDeviceReference);
+ if (status->rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "*** cannot allocate GPU lock\n");
+ RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED);
+ // RM_BASIC_LOCK_MODEL: free GPU lock
+ rmGpuLockFree(*pDeviceReference);
+ return;
+ }
+
+ // attach default single-entry broadcast device for this gpu
+ status->rmStatus = gpumgrCreateDevice(&deviceInstance, NVBIT(*pDeviceReference), NULL);
+ if (status->rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "*** Cannot attach bc gpu\n");
+ RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED);
+ // RM_BASIC_LOCK_MODEL: free GPU lock
+ rmGpuLockFree(*pDeviceReference);
+ return;
+ }
+
+ // init attach state
+ gpuAttachArg = portMemAllocNonPaged(sizeof(GPUATTACHARG));
+ if (gpuAttachArg == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "*** Cannot allocate gpuAttachArg\n");
+ RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED);
+ // RM_BASIC_LOCK_MODEL: free GPU lock
+ rmGpuLockFree(*pDeviceReference);
+ return;
+ }
+
+ portMemSet(gpuAttachArg, 0, sizeof(GPUATTACHARG));
+
+ if (NV_IS_SOC_DISPLAY_DEVICE(nv))
+ {
+ gpuAttachArg->socDeviceArgs.specified = NV_TRUE;
+
+ RmSetSocDispDeviceMappings(gpuAttachArg, nv);
+
+ RmSetSocDpauxDeviceMappings(gpuAttachArg, nv);
+
+ RmSetSocHdacodecDeviceMappings(gpuAttachArg, nv);
+
+ RmSetSocMipiCalDeviceMappings(gpuAttachArg, nv);
+
+ gpuAttachArg->socDeviceArgs.socChipId0 = nv->disp_sw_soc_chip_id;
+
+ gpuAttachArg->socDeviceArgs.iovaspaceId = nv->iovaspace_id;
+ }
+ else
+ {
+ gpuAttachArg->fbPhysAddr = nv->fb->cpu_address;
+ gpuAttachArg->fbBaseAddr = (GPUHWREG*) 0; // not mapped
+ gpuAttachArg->devPhysAddr = nv->regs->cpu_address;
+ gpuAttachArg->regBaseAddr = (GPUHWREG*) nv->regs->map;
+ gpuAttachArg->intLine = 0; // don't know yet
+ gpuAttachArg->instPhysAddr = nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address;
+ gpuAttachArg->instBaseAddr = (GPUHWREG*) 0; // not mapped
+
+ gpuAttachArg->regLength = nv->regs->size;
+ gpuAttachArg->fbLength = nv->fb->size;
+ gpuAttachArg->instLength = nv->bars[NV_GPU_BAR_INDEX_IMEM].size;
+
+ gpuAttachArg->iovaspaceId = nv->iovaspace_id;
+ }
+
+ //
+ // we need this to check if we are running on virtual GPU
+ // in gpuBindHal function later.
+ //
+ gpuAttachArg->nvDomainBusDeviceFunc = nv_encode_pci_info(&nv->pci_info);
+
+ gpuAttachArg->bRequestFwClientRm = nv->request_fw_client_rm;
+
+ gpuAttachArg->pOsAttachArg = (void *)nv;
+
+ // use gpu manager to attach gpu
+ status->rmStatus = gpumgrAttachGpu(*pDeviceReference, gpuAttachArg);
+ portMemFree(gpuAttachArg);
+ if (status->rmStatus != NV_OK)
+ {
+ gpumgrDestroyDevice(deviceInstance);
+ RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED);
+ NV_PRINTF(LEVEL_ERROR, "*** Cannot attach gpu\n");
+ // RM_BASIC_LOCK_MODEL: free GPU lock
+ rmGpuLockFree(*pDeviceReference);
+ return;
+ }
+ nvp->flags |= NV_INIT_FLAG_GPUMGR_ATTACH;
+
+ pGpu = gpumgrGetGpu(*pDeviceReference);
+
+ sysInitRegistryOverrides(pSys);
+
+ sysApplyLockingPolicy(pSys);
+
+ pGpu->busInfo.IntLine = nv->interrupt_line;
+
+ //
+ // Set the DMA address size as soon as we have the HAL to call to
+ // determine the precise number of physical address bits supported
+ // by the architecture. DMA allocations should not be made before
+ // this point.
+ //
+ nv_set_dma_address_size(nv, gpuGetPhysAddrWidth_HAL(pGpu, ADDR_SYSMEM));
+
+ pGpu->dmaStartAddress = (RmPhysAddr)nv_get_dma_start_address(nv);
+ if (nv->fb != NULL)
+ {
+ pGpu->registerAccess.gpuFbAddr = (GPUHWREG*) nv->fb->map;
+ pGpu->busInfo.gpuPhysFbAddr = nv->fb->cpu_address;
+ }
+
+ // set default parent gpu
+ gpumgrSetParentGPU(pGpu, pGpu);
+
+ NV_PRINTF(LEVEL_INFO, "device instance : %d\n", *pDeviceReference);
+ NV_PRINTF(LEVEL_INFO, "NV regs using linear address : 0x%p\n",
+ pGpu->deviceMappings[SOC_DEV_MAPPING_DISP].gpuNvAddr);
+ NV_PRINTF(LEVEL_INFO,
+ "NV fb using linear address : 0x%p\n", pGpu->registerAccess.gpuFbAddr);
+
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED, NV_TRUE);
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS, NV_FALSE);
+
+ if (osReadRegistryDword(pGpu,
+ NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR, &data) == NV_OK)
+ {
+ if (data == NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR_ENABLE)
+ {
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS, NV_TRUE);
+ }
+ }
+
+ if (!os_is_vgx_hyper())
+ {
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE);
+ }
+
+ if ((osReadRegistryDword(NULL,
+ NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS,
+ &data) == NV_OK) && data)
+ {
+ KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
+ MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+
+ memmgrSetPmaForcePersistence(pMemoryManager, NV_TRUE);
+ pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = NV_TRUE;
+
+ nv->preserve_vidmem_allocations = NV_TRUE;
+ }
+}
+
+void osInitScalabilityOptions
+(
+ OBJGPU *pGpu,
+ void *pDeviceExtension
+)
+{
+}
+
+
+static NV_STATUS
+osInitScalability(
+ OBJGPU *pGpu
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJCL *pCl = SYS_GET_CL(pSys);
+
+ osInitScalabilityOptions(pGpu, NULL);
+
+ // We need PCI-E chipset information very early.
+ if (pCl != NULL)
+ {
+ return clInitPcie(pGpu, pCl);
+ }
+ else
+ {
+ return NV_ERR_NOT_SUPPORTED;
+ }
+}
+
+static NV_STATUS
+osTeardownScalability(
+ OBJGPU *pGpu
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJCL *pCl = SYS_GET_CL(pSys);
+
+ return clTeardownPcie(pGpu, pCl);
+}
+
+static void
+populateDeviceAttributes(
+ OBJGPU *pGpu,
+ nv_state_t *nv
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJCL *pCl = SYS_GET_CL(pSys);
+
+ if ((pCl != NULL) && pCl->getProperty(pCl, PDB_PROP_CL_IS_EXTERNAL_GPU))
+ {
+ nv->is_external_gpu = NV_TRUE;
+ }
+}
+
+static void
+RmSetConsolePreservationParams(OBJGPU *pGpu)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ NvU64 fbBaseAddress = 0;
+ NvU64 fbConsoleSize = 0;
+ MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+
+ //
+ // PDB_PROP_GPU_PRIMARY_DEVICE should be NV_FALSE for vGPU configuration so
+ // return early
+ //
+ if (os_is_vgx_hyper() || IS_VIRTUAL(pGpu))
+ return;
+
+ //
+ // Check the OS layer for any video memory used by a console
+ // driver that should be reserved.
+ //
+ // If the console driver is using some amount of memory, its
+ // mapping is presumably either pointing at the start of BAR1
+ // (this is the mapping the EFI GOP driver provides for
+ // efifb), or the upper 16MB of BAR2 (this is the mapping the
+ // VBIOS provides for, e.g., vesafb).
+ //
+ // In the BAR2 case, there is nothing more to do: the upper
+ // 16MB of BAR2 will remain in physical mode
+ // (see the documentation for NV_PBUS_BAR2_BLOCK in dev_bus.ref),
+ // and the console driver can continue to access it.
+ //
+ // In the BAR1 case, RM will put all of BAR1 into virtual
+ // mode, so we need to create a BAR1 mapping for the console
+ // driver to use.
+ //
+ // If the console driver is not using _either_ of those
+ // mappings, then the console driver will not be able to
+ // access the memory we reserve. This happens on some
+ // UEFI systems with multiple GPUs, because the firmware
+ // initializes the GOP driver on more than one GPU. In
+ // that case, just skip reserving anything for GPUs where
+ // neither BAR1 nor BAR2 match the console's base
+ // address.
+ //
+ fbConsoleSize = rm_get_uefi_console_size(nv, &fbBaseAddress);
+
+ if (fbConsoleSize == 0)
+ {
+ if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI))
+ {
+ fbConsoleSize = 0x40000;
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_INFO, "No Frame Buffer Present\n");
+ }
+ }
+
+ if ((fbConsoleSize > 0) && (fbBaseAddress != 0))
+ {
+ KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+ pKernelBus->bPreserveBar1ConsoleEnabled =
+ (fbBaseAddress == nv->fb->cpu_address);
+ }
+
+ pMemoryManager->Ram.ReservedConsoleDispMemSize = NV_ALIGN_UP(fbConsoleSize, 0x10000);
+}
+
+static NV_STATUS
+RmInitDeviceDma(
+ nv_state_t *nv
+)
+{
+ if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
+ {
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ POBJVMM pVmm = SYS_GET_VMM(pSys);
+ POBJVASPACE pIOVAS;
+ NV_STATUS status = vmmCreateVaspace(pVmm, IO_VASPACE_A,
+ nv->iovaspace_id, 0, 0ULL, ~0ULL,
+ 0ULL, 0ULL,
+ NULL, VASPACE_FLAGS_ENABLE_VMM,
+ &pIOVAS);
+ if (status != NV_OK)
+ {
+ return status;
+ }
+ }
+
+ return NV_OK;
+}
+
+static void
+RmTeardownDeviceDma(
+ nv_state_t *nv
+)
+{
+ if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
+ {
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ POBJVMM pVmm = SYS_GET_VMM(pSys);
+ POBJVASPACE pIOVAS;
+
+ if (NV_OK == vmmGetVaspaceFromId(pVmm, nv->iovaspace_id, IO_VASPACE_A, &pIOVAS))
+ {
+ vmmDestroyVaspace(pVmm, pIOVAS);
+ }
+ }
+}
+
+static void
+RmInitNvDevice(
+ NvU32 deviceReference,
+ UNIX_STATUS *status
+)
+{
+ // set the device context
+ OBJGPU *pGpu = gpumgrGetGpu(deviceReference);
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+ NV_PRINTF(LEVEL_INFO, "RmInitNvDevice:\n");
+
+ NV_PRINTF(LEVEL_INFO,
+ "device instance : 0x%08x\n", deviceReference);
+
+ // initialize all engines -- calls back osInitMapping()
+ status->rmStatus = gpumgrStatePreInitGpu(pGpu);
+ if (status->rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "*** Cannot pre-initialize the device\n");
+ RM_SET_ERROR(*status, RM_INIT_GPU_PRE_INIT_FAILED);
+ return;
+ }
+
+ os_disable_console_access();
+
+ status->rmStatus = gpumgrStateInitGpu(pGpu);
+ if (status->rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize the device\n");
+ RM_SET_ERROR(*status, RM_INIT_GPU_STATE_INIT_FAILED);
+ os_enable_console_access();
+ return;
+ }
+ nvp->flags |= NV_INIT_FLAG_GPU_STATE;
+
+ KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+ //
+ // Make sure bifIsMSIEnabled() gets a chance to look up the OS PCI
+ // handle for this GPU. This must happen after GPU state
+ // initialization.
+ //
+ if (pKernelBif != NULL)
+ {
+ kbifCheckAndRearmMSI(pGpu, pKernelBif);
+ }
+
+ // Set RM's interrupt enable state to zero here so that interrupts
+ // won't be enabled during loading
+ Intr *pIntr = GPU_GET_INTR(pGpu);
+ if (pIntr != NULL)
+ {
+ intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED);
+ }
+
+ status->rmStatus = gpumgrStateLoadGpu(pGpu, GPU_STATE_DEFAULT);
+ if (status->rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "*** Cannot load state into the device\n");
+ RM_SET_ERROR(*status, RM_INIT_GPU_LOAD_FAILED);
+ os_enable_console_access();
+ return;
+ }
+ nvp->flags |= NV_INIT_FLAG_GPU_STATE_LOAD;
+
+ os_enable_console_access();
+
+ status->rmStatus = gpuPerformUniversalValidation_HAL(pGpu);
+ if (status->rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "*** Failed universal validation\n");
+ RM_SET_ERROR(*status, RM_INIT_GPU_UNIVERSAL_VALIDATION_FAILED);
+ return;
+ }
+
+ return;
+}
+
+NV_STATUS osInitMapping(
+ OBJGPU *pGpu
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJCL *pCl = SYS_GET_CL(pSys);
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+ NV_PRINTF(LEVEL_INFO, "osInitMapping:\n");
+
+ // Some PCI BIOSs leave the ROM mapped. This causes problems if it overlays system RAM. Just disable it.
+ GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_12, 0);
+
+ // make sure our PCI latency timer is sufficient (max it out)
+ GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_3,
+ DRF_DEF(_CONFIG, _PCI_NV_3, _LATENCY_TIMER, _248_CLOCKS));
+
+ if (pCl != NULL)
+ {
+ initCoreLogic(pGpu);
+ }
+ nvp->flags |= NV_INIT_FLAG_CORE_LOGIC;
+
+ return NV_OK;
+
+} // end of osInitMapping()
+
+static NV_STATUS
+initCoreLogic(
+ OBJGPU *pGpu
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJCL *pCl = SYS_GET_CL(pSys);
+
+#if defined(NVCPU_X86_64)
+ OBJOS *pOS = SYS_GET_OS(pSys);
+ if (!os_pat_supported())
+ pOS->setProperty(pOS, PDB_PROP_OS_PAT_UNSUPPORTED, NV_TRUE);
+#endif
+
+ return clInit(pGpu, pCl);
+}
+
+static NV_STATUS
+teardownCoreLogic(
+ OBJOS *pOS,
+ OBJGPU *pGpu
+)
+{
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJCL *pCl = SYS_GET_CL(pSys);
+
+ return clTeardown(pGpu, pCl);
+}
+
+static void RmTeardownDpauxRegisters(
+ nv_state_t *nv
+)
+{
+}
+
+static void RmTeardownHdacodecRegisters(
+ nv_state_t *nv
+)
+{
+}
+
+static void RmTeardownMipiCalRegisters(
+ nv_state_t *nv
+)
+{
+}
+
+static NV_STATUS
+RmTeardownRegisters(
+ nv_state_t *nv
+)
+{
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Tearing down registers\n");
+
+ if (nv->regs && nv->regs->map)
+ {
+ osUnmapKernelSpace(nv->regs->map, nv->regs->size);
+ nv->regs->map = 0;
+ nv->regs->map_u = NULL;
+ }
+
+ RmTeardownDpauxRegisters(nv);
+
+ RmTeardownHdacodecRegisters(nv);
+
+ RmTeardownMipiCalRegisters(nv);
+
+ return NV_OK;
+}
+
+static NV_STATUS
+RmSetupDpauxRegisters(
+ nv_state_t *nv,
+ UNIX_STATUS *status
+)
+{
+
+ return NV_OK;
+}
+
+static NV_STATUS
+RmSetupHdacodecRegisters(
+ nv_state_t *nv,
+ UNIX_STATUS *status
+)
+{
+
+ return NV_OK;
+}
+
+static NV_STATUS
+RmSetupMipiCalRegisters(
+ nv_state_t *nv,
+ UNIX_STATUS *status
+)
+{
+
+ return NV_OK;
+}
+
+static void
+RmSetupRegisters(
+ nv_state_t *nv,
+ UNIX_STATUS *status
+)
+{
+ NV_STATUS ret;
+
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmSetupRegisters for 0x%x:0x%x\n",
+ nv->pci_info.vendor_id, nv->pci_info.device_id);
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, "pci config info:\n");
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, " registers look like: " NvP64_fmt " " NvP64_fmt,
+ nv->regs->cpu_address, nv->regs->size);
+
+ if (nv->fb != NULL)
+ {
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, " fb looks like: " NvP64_fmt " " NvP64_fmt,
+ nv->fb->cpu_address, nv->fb->size);
+ }
+
+ {
+ nv_os_map_kernel_space(nv, nv->regs);
+ }
+
+ if (nv->regs->map == NULL)
+ {
+ NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "Failed to map regs registers!!\n");
+ RM_SET_ERROR(*status, RM_INIT_REG_SETUP_FAILED);
+ status->rmStatus = NV_ERR_OPERATING_SYSTEM;
+ return;
+ }
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Successfully mapped framebuffer and registers\n");
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, "final mappings:\n");
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, " regs: " NvP64_fmt " " NvP64_fmt " 0x%p\n",
+ nv->regs->cpu_address, nv->regs->size, nv->regs->map);
+
+ ret = RmSetupDpauxRegisters(nv, status);
+ if (ret != NV_OK)
+ goto err_unmap_disp_regs;
+
+ ret = RmSetupHdacodecRegisters(nv, status);
+ if (ret != NV_OK)
+ {
+ RmTeardownDpauxRegisters(nv);
+ goto err_unmap_disp_regs;
+ }
+
+ ret = RmSetupMipiCalRegisters(nv, status);
+ if (ret != NV_OK)
+ {
+ RmTeardownHdacodecRegisters(nv);
+ RmTeardownDpauxRegisters(nv);
+ goto err_unmap_disp_regs;
+ }
+
+ return;
+
+err_unmap_disp_regs:
+ if (nv->regs && nv->regs->map)
+ {
+ osUnmapKernelSpace(nv->regs->map, nv->regs->size);
+ nv->regs->map = 0;
+ }
+
+ return;
+}
+
+NvBool RmInitPrivateState(
+ nv_state_t *pNv
+)
+{
+ nv_priv_t *nvp;
+ NvU32 gpuId;
+ NvU32 pmc_boot_0 = 0;
+ NvU32 pmc_boot_42 = 0;
+
+ NV_SET_NV_PRIV(pNv, NULL);
+
+ if (!NV_IS_SOC_DISPLAY_DEVICE(pNv) && !NV_IS_SOC_IGPU_DEVICE(pNv))
+ {
+ pNv->regs->map_u = os_map_kernel_space(pNv->regs->cpu_address,
+ os_page_size,
+ NV_MEMORY_UNCACHED);
+ if (pNv->regs->map_u == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "failed to map GPU registers (DISABLE_INTERRUPTS).\n");
+ return NV_FALSE;
+ }
+
+ pmc_boot_0 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_0);
+ pmc_boot_42 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_42);
+
+ os_unmap_kernel_space(pNv->regs->map_u, os_page_size);
+ pNv->regs->map_u = NULL;
+ }
+
+ if (os_alloc_mem((void **)&nvp, sizeof(*nvp)) != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "failed to allocate private device state.\n");
+ return NV_FALSE;
+ }
+
+ gpuId = nv_generate_id_from_pci_info(&pNv->pci_info);
+
+ if (gpumgrRegisterGpuId(gpuId, nv_encode_pci_info(&pNv->pci_info)) != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "failed to register GPU with GPU manager.\n");
+ os_free_mem(nvp);
+ return NV_FALSE;
+ }
+
+ pNv->gpu_id = gpuId;
+
+ pNv->iovaspace_id = nv_requires_dma_remap(pNv) ? gpuId :
+ NV_IOVA_DOMAIN_NONE;
+
+ kvgpumgrAttachGpu(pNv->gpu_id);
+
+ //
+ // Set up a reasonable default DMA address size, based on the minimum
+ // possible on currently supported GPUs.
+ //
+ nv_set_dma_address_size(pNv, NV_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH);
+
+ os_mem_set(nvp, 0, sizeof(*nvp));
+ nvp->status = NV_ERR_INVALID_STATE;
+ nvp->pmc_boot_0 = pmc_boot_0;
+ nvp->pmc_boot_42 = pmc_boot_42;
+ NV_SET_NV_PRIV(pNv, nvp);
+
+ return NV_TRUE;
+}
+
+void RmClearPrivateState(
+ nv_state_t *pNv
+)
+{
+ nv_priv_t *nvp = NV_GET_NV_PRIV(pNv);
+ NvU32 status;
+ void *pVbiosCopy = NULL;
+ void *pRegistryCopy = NULL;
+ NvU32 vbiosSize;
+ NvU32 *pEfiDisplayCache;
+ NvU32 efiDisplayCacheSize;
+ nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS];
+ nv_dynamic_power_t dynamicPowerCopy;
+ NvU32 x = 0;
+ NvU32 pmc_boot_0, pmc_boot_42;
+
+ //
+ // Do not clear private state after GPU resets, it is used while
+ // recovering the GPU. Only clear the pGpu pointer, which is
+ // restored during next initialization cycle.
+ //
+ if (pNv->flags & NV_FLAG_IN_RECOVERY)
+ {
+ nvp->pGpu = NULL;
+ }
+
+ status = nvp->status;
+ pVbiosCopy = nvp->pVbiosCopy;
+ vbiosSize = nvp->vbiosSize;
+ pRegistryCopy = nvp->pRegistry;
+ pEfiDisplayCache = nvp->efi.display.pCache;
+ efiDisplayCacheSize = nvp->efi.display.cacheSize;
+ dynamicPowerCopy = nvp->dynamic_power;
+ pmc_boot_0 = nvp->pmc_boot_0;
+ pmc_boot_42 = nvp->pmc_boot_42;
+
+ for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+ {
+ i2c_adapters[x] = nvp->i2c_adapters[x];
+ }
+
+ portMemSet(nvp, 0, sizeof(nv_priv_t));
+
+ nvp->status = status;
+ nvp->pVbiosCopy = pVbiosCopy;
+ nvp->vbiosSize = vbiosSize;
+ nvp->pRegistry = pRegistryCopy;
+ nvp->efi.display.pCache = pEfiDisplayCache;
+ nvp->efi.display.cacheSize = efiDisplayCacheSize;
+ nvp->dynamic_power = dynamicPowerCopy;
+ nvp->pmc_boot_0 = pmc_boot_0;
+ nvp->pmc_boot_42 = pmc_boot_42;
+
+ for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+ {
+ nvp->i2c_adapters[x] = i2c_adapters[x];
+ }
+
+ nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C;
+}
+
+void RmFreePrivateState(
+ nv_state_t *pNv
+)
+{
+ nv_priv_t *nvp = NV_GET_NV_PRIV(pNv);
+
+ gpumgrUnregisterGpuId(pNv->gpu_id);
+
+ kvgpumgrDetachGpu(pNv->gpu_id);
+
+ RmDestroyRegistry(pNv);
+
+ if (nvp != NULL)
+ {
+ portMemFree(nvp->pVbiosCopy);
+ portMemFree(nvp->efi.display.pCache);
+ os_free_mem(nvp);
+ }
+
+ NV_SET_NV_PRIV(pNv, NULL);
+}
+
+NvBool RmPartiallyInitAdapter(
+ nv_state_t *nv
+)
+{
+ NV_PRINTF(LEVEL_INFO, "%s: %04x:%02x:%02x.0\n", __FUNCTION__,
+ nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot);
+
+ nv_start_rc_timer(nv);
+
+ return NV_TRUE;
+}
+
+static NV_STATUS
+RmInitX86Emu(
+ OBJGPU *pGpu
+)
+{
+ NV_STATUS status = NV_OK;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ PORT_UNREFERENCED_VARIABLE(nv);
+
+#if NVCPU_IS_X86_64
+ status = RmInitX86EmuState(pGpu);
+#else
+ // We don't expect a "primary VGA" adapter on non-amd64 platforms
+ NV_ASSERT(!NV_PRIMARY_VGA(nv));
+#endif
+
+ return status;
+}
+
+static NV_STATUS RmRegisterGpudb(
+ OBJGPU *pGpu
+)
+{
+ NV_STATUS rmStatus;
+ const NvU8 *pGid;
+ nv_state_t *pNv = NV_GET_NV_STATE(pGpu);
+
+ pGid = RmGetGpuUuidRaw(pNv);
+ if (pGid == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to get UUID\n");
+ return NV_ERR_OPERATING_SYSTEM;
+ }
+
+ rmStatus = gpudbRegisterGpu(pGid, &pGpu->gpuClData.upstreamPort.addr,
+ pGpu->busInfo.nvDomainBusDeviceFunc);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to register GPU with GPU data base\n");
+ }
+
+ return rmStatus;
+}
+
+static void RmUnixFreeRmApi(
+ nv_state_t *nv
+)
+{
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+ if (nv->rmapi.hClient != 0)
+ {
+ pRmApi->Free(pRmApi, nv->rmapi.hClient, nv->rmapi.hClient);
+ }
+
+ portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi));
+}
+
+static NvBool RmUnixAllocRmApi(
+ nv_state_t *nv,
+ NvU32 deviceId
+)
+{
+ NV0080_ALLOC_PARAMETERS deviceParams = { 0 };
+ NV2080_ALLOC_PARAMETERS subDeviceParams = { 0 };
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+ portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi));
+
+ if (pRmApi->AllocWithHandle(
+ pRmApi,
+ NV01_NULL_OBJECT,
+ NV01_NULL_OBJECT,
+ NV01_NULL_OBJECT,
+ NV01_ROOT,
+ &nv->rmapi.hClient) != NV_OK)
+ {
+ goto fail;
+ }
+
+ //
+ // Any call to rmapiDelPendingDevices() will internally delete the UNIX OS
+ // layer RMAPI handles. Set this flag to preserve these handles. These
+ // handles will be freed explicitly by RmUnixFreeRmApi().
+ //
+ if (!rmclientSetClientFlagsByHandle(nv->rmapi.hClient,
+ RMAPI_CLIENT_FLAG_RM_INTERNAL_CLIENT))
+ {
+ goto fail;
+ }
+
+ deviceParams.deviceId = deviceId;
+
+ if (pRmApi->Alloc(
+ pRmApi,
+ nv->rmapi.hClient,
+ nv->rmapi.hClient,
+ &nv->rmapi.hDevice,
+ NV01_DEVICE_0,
+ &deviceParams) != NV_OK)
+ {
+ goto fail;
+ }
+
+ subDeviceParams.subDeviceId = 0;
+
+ if (pRmApi->Alloc(
+ pRmApi,
+ nv->rmapi.hClient,
+ nv->rmapi.hDevice,
+ &nv->rmapi.hSubDevice,
+ NV20_SUBDEVICE_0,
+ &subDeviceParams) != NV_OK)
+ {
+ goto fail;
+ }
+
+ //
+ // The NV40_I2C allocation expected to fail, if it is disabled
+ // with RM config.
+ //
+ if (pRmApi->Alloc(
+ pRmApi,
+ nv->rmapi.hClient,
+ nv->rmapi.hSubDevice,
+ &nv->rmapi.hI2C,
+ NV40_I2C,
+ NULL) != NV_OK)
+ {
+ nv->rmapi.hI2C = 0;
+ }
+
+ //
+ // The NV04_DISPLAY_COMMON allocation expected to fail for displayless
+ // system. nv->rmapi.hDisp value needs to be checked before doing display
+ // related control calls.
+ //
+ if (pRmApi->Alloc(
+ pRmApi,
+ nv->rmapi.hClient,
+ nv->rmapi.hDevice,
+ &nv->rmapi.hDisp,
+ NV04_DISPLAY_COMMON,
+ NULL) != NV_OK)
+ {
+ nv->rmapi.hDisp = 0;
+ }
+
+ return NV_TRUE;
+
+fail:
+ RmUnixFreeRmApi(nv);
+ return NV_FALSE;
+}
+
+NvBool RmInitAdapter(
+ nv_state_t *nv
+)
+{
+ NvU32 devicereference = 0;
+ UNIX_STATUS status = INIT_UNIX_STATUS;
+ nv_priv_t *nvp;
+ NvBool retVal = NV_FALSE;
+ OBJSYS *pSys;
+ OBJGPU *pGpu = NULL;
+ OBJOS *pOS;
+ KernelDisplay *pKernelDisplay;
+ const void *gspFwHandle = NULL;
+ const void *gspFwLogHandle = NULL;
+
+ GSP_FIRMWARE gspFw = {0};
+ PORT_UNREFERENCED_VARIABLE(gspFw);
+
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter\n");
+
+ nv->flags &= ~NV_FLAG_PASSTHRU;
+
+ RmSetupRegisters(nv, &status);
+ if (! RM_INIT_SUCCESS(status.initStatus) )
+ goto failed;
+
+ nvp = NV_GET_NV_PRIV(nv);
+ nvp->status = NV_ERR_OPERATING_SYSTEM;
+
+ status.rmStatus = RmInitDeviceDma(nv);
+ if (status.rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Cannot configure the device for DMA\n");
+ RM_SET_ERROR(status, RM_INIT_GPU_DMA_CONFIGURATION_FAILED);
+ goto shutdown;
+ }
+
+ nvp->flags |= NV_INIT_FLAG_DMA;
+
+ pSys = SYS_GET_INSTANCE();
+
+ //
+ // WAR: If the below UEFI property is set, display RM will attempt to read
+ // the state cache during RM init in order to retrieve a snapshot of the
+ // display state that the UEFI driver has already programmed. On Orin
+ // (T234D), the UEFI boot flow is being enabled on Linux, but our UEFI
+ // driver doesn't have any display support right now. As such, our UEFI
+ // driver won't allocate any of the display channels, which means that RM
+ // will attempt to read the state cache for uninitialized channels. WAR this
+ // issue by un-setting the below UEFI property for now.
+ //
+ // JIRA task TDS-5094 tracks adding display support to the UEFI driver.
+ //
+ if (NV_IS_SOC_DISPLAY_DEVICE(nv)) {
+ pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_FALSE);
+ }
+
+ //
+ // Get firmware from the OS, if requested, and decide if RM will run as a
+ // firmware client.
+ //
+ if (nv->request_firmware)
+ {
+ NvU32 gpuArch = (DRF_VAL(_PMC, _BOOT_42, _ARCHITECTURE, nvp->pmc_boot_42) <<
+ GPU_ARCH_SHIFT);
+ NvU32 gpuImpl = DRF_VAL(_PMC, _BOOT_42, _IMPLEMENTATION, nvp->pmc_boot_42);
+
+ nv_firmware_chip_family_t chipFamily = nv_firmware_get_chip_family(gpuArch, gpuImpl);
+
+ nv_set_dma_address_size(nv, NV_GSP_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH);
+
+ gspFwHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP,
+ chipFamily,
+ &gspFw.pBuf,
+ &gspFw.size);
+ if (gspFwHandle == NULL &&
+ !nv->allow_fallback_to_monolithic_rm)
+ {
+ RM_SET_ERROR(status, RM_INIT_FIRMWARE_FETCH_FAILED);
+ goto shutdown;
+ }
+ else if (gspFwHandle != NULL)
+ {
+#if LIBOS_LOG_DECODE_ENABLE
+ if (nv->enable_firmware_logs)
+ {
+ gspFwLogHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP_LOG,
+ chipFamily,
+ &gspFw.pLogElf,
+ &gspFw.logElfSize);
+ if (gspFwLogHandle == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to load gsp_log_*.bin, no GSP-RM logs will be printed (non-fatal)\n");
+ }
+ }
+#endif
+
+ nv->request_fw_client_rm = NV_TRUE;
+ }
+ else
+ {
+ nv->request_fw_client_rm = NV_FALSE;
+ }
+ }
+
+ // initialize the RM device register mapping
+ osInitNvMapping(nv, &devicereference, &status);
+ if (! RM_INIT_SUCCESS(status.initStatus) )
+ {
+ switch (status.rmStatus)
+ {
+ case NV_ERR_NOT_SUPPORTED:
+ nvp->status = NV_ERR_NOT_SUPPORTED;
+ break;
+ }
+ NV_PRINTF(LEVEL_ERROR,
+ "osInitNvMapping failed, bailing out of RmInitAdapter\n");
+ goto shutdown;
+ }
+
+ //
+ // now we can have a pdev for the first time...
+ //
+ pGpu = gpumgrGetGpu(devicereference);
+
+ pOS = SYS_GET_OS(pSys);
+
+ status.rmStatus = osInitScalability(pGpu);
+ if (status.rmStatus == NV_OK)
+ {
+ nvp->flags |= NV_INIT_FLAG_SCALABILITY;
+ }
+ else if (status.rmStatus != NV_ERR_NOT_SUPPORTED)
+ {
+ RM_SET_ERROR(status, RM_INIT_SCALABILITY_FAILED);
+ goto shutdown;
+ }
+
+ KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu);
+ if ((pKernelFsp != NULL) && !IS_GSP_CLIENT(pGpu) && !IS_VIRTUAL(pGpu))
+ {
+ status.rmStatus = kfspSendBootCommands_HAL(pGpu, pKernelFsp);
+ if (status.rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "FSP boot command failed.\n");
+ goto shutdown;
+ }
+ }
+
+ RmSetConsolePreservationParams(pGpu);
+
+ RmInitAcpiMethods(pOS, pSys, pGpu);
+
+ //
+ // If GSP fw RM support is enabled then start the GSP microcode
+ // (including the task running the full instance of the RM) and
+ // exchange the necessary initial RPC messages before continuing
+ // with GPU initialization here.
+ //
+ if (IS_GSP_CLIENT(pGpu))
+ {
+ status.rmStatus = kgspInitRm(pGpu, GPU_GET_KERNEL_GSP(pGpu), &gspFw);
+ if (status.rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Cannot initialize GSP firmware RM\n");
+ RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED);
+ goto shutdown;
+ }
+ }
+ else if (nv->request_fw_client_rm)
+ {
+ // We were expecting to enable GSP-RM but something went wrong.
+ if (!nv->allow_fallback_to_monolithic_rm)
+ {
+ RM_SET_ERROR(status, RM_INIT_FIRMWARE_POLICY_FAILED);
+ goto shutdown;
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_NOTICE, "Falling back to monolithic RM\n");
+ }
+ }
+
+ pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+ if (pKernelDisplay != NULL)
+ {
+ kdispSetWarPurgeSatellitesOnCoreFree(pKernelDisplay, NV_TRUE);
+ }
+
+ if (IS_PASSTHRU(pGpu))
+ nv->flags |= NV_FLAG_PASSTHRU;
+
+ populateDeviceAttributes(pGpu, nv);
+
+ initVendorSpecificRegistry(pGpu, nv->pci_info.device_id);
+ if (!IS_VIRTUAL(pGpu) && !IS_GSP_CLIENT(pGpu))
+ {
+ initNbsiTable(pGpu);
+ }
+
+ // finally, initialize the device
+ RmInitNvDevice(devicereference, &status);
+ if (! RM_INIT_SUCCESS(status.initStatus) )
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "RmInitNvDevice failed, bailing out of RmInitAdapter\n");
+ switch (status.rmStatus)
+ {
+ case NV_ERR_INSUFFICIENT_POWER:
+ nvp->status = NV_ERR_INSUFFICIENT_POWER;
+ NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
+ "GPU does not have the necessary power cables connected.\n");
+ break;
+ }
+ goto shutdown;
+ }
+
+ // LOCK: acquire GPUs lock
+ status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE,
+ RM_LOCK_MODULES_INIT);
+ if (status.rmStatus != NV_OK)
+ {
+ goto shutdown;
+ }
+
+ status.rmStatus = osVerifySystemEnvironment(pGpu);
+
+ // UNLOCK: release GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+ if (status.rmStatus != NV_OK)
+ {
+ RM_SET_ERROR(status, RM_INIT_SYS_ENVIRONMENT_FAILED);
+ switch (status.rmStatus)
+ {
+ case NV_ERR_IRQ_NOT_FIRING:
+ nvp->status = NV_ERR_IRQ_NOT_FIRING;
+ break;
+ }
+ NV_PRINTF(LEVEL_ERROR, "RmVerifySystemEnvironment failed, bailing!\n");
+ goto shutdown;
+ }
+
+ Intr *pIntr = GPU_GET_INTR(pGpu);
+ if (pIntr != NULL)
+ {
+ intrSetIntrEn(pIntr, INTERRUPT_TYPE_HARDWARE);
+ }
+
+ KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu);
+ // initialize the watchdog (disabled by default)
+ status.rmStatus = pKernelRc != NULL ? krcWatchdogInit_HAL(pGpu, pKernelRc) :
+ NV_ERR_NOT_SUPPORTED;
+ if (status.rmStatus == NV_OK)
+ {
+ krcWatchdogDisable(pKernelRc);
+ nvp->flags |= NV_INIT_FLAG_FIFO_WATCHDOG;
+ }
+ else if (status.rmStatus == NV_ERR_NOT_SUPPORTED)
+ {
+ NV_PRINTF(LEVEL_INFO,
+ "krcWatchdogInit returned _NOT_SUPPORTED. For Kepler GPUs in PGOB mode, this is normal\n");
+ }
+ else
+ {
+ RM_SET_ERROR(status, RM_INIT_WATCHDOG_FAILED);
+ NV_PRINTF(LEVEL_ERROR,
+ "krcWatchdogInit failed, bailing out of RmInitAdapter\n");
+ goto shutdown;
+ }
+ nv_start_rc_timer(nv);
+
+ nvp->status = NV_OK;
+
+ if (!RmUnixAllocRmApi(nv, devicereference)) {
+ RM_SET_ERROR(status, RM_INIT_ALLOC_RMAPI_FAILED);
+ status.rmStatus = NV_ERR_GENERIC;
+ goto shutdown;
+ }
+
+ status.rmStatus = RmInitGpuInfoWithRmApi(pGpu);
+ if (status.rmStatus != NV_OK)
+ {
+ RM_SET_ERROR(status, RM_INIT_GPUINFO_WITH_RMAPI_FAILED);
+ goto shutdown;
+ }
+
+ status.rmStatus = RmInitX86Emu(pGpu);
+ if (status.rmStatus != NV_OK)
+ {
+ RM_SET_ERROR(status, RM_INIT_VBIOS_X86EMU_FAILED);
+ NV_PRINTF(LEVEL_ERROR,
+ "RmInitX86Emu failed, bailing out of RmInitAdapter\n");
+ goto shutdown;
+ }
+
+ // i2c only on master device??
+ RmI2cAddGpuPorts(nv);
+ nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C;
+
+ // This fifo hal call will fail on pre-fermi gpus. In that case, userd
+ // info will remain 0 and skipped by fb mmap code
+ KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
+ NvU64 udaddr = 0;
+ NvU32 udsize = 0;
+ if (pKernelFifo != NULL)
+ {
+ status.rmStatus = kfifoGetUserdBar1MapInfo_HAL(pGpu, pKernelFifo, &udaddr, &udsize);
+ if (status.rmStatus == NV_OK)
+ {
+ nv->ud.cpu_address = (nv->fb->cpu_address + udaddr);
+ nv->ud.size = udsize;
+ }
+ else if (status.rmStatus != NV_ERR_NOT_SUPPORTED)
+ {
+ RM_SET_ERROR(status, RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED);
+ NV_PRINTF(LEVEL_ERROR,
+ "kfifoGetUserdBar1MapInfo failed, bailing out of RmInitAdapter\n");
+ goto shutdown;
+ }
+ }
+
+ nv->flags &= ~NV_FLAG_IN_RECOVERY;
+
+ pOS->setProperty(pOS, PDB_PROP_OS_SYSTEM_EVENTS_SUPPORTED, NV_TRUE);
+
+ RmInitS0ixPowerManagement(nv);
+ RmInitDeferredDynamicPowerManagement(nv);
+
+ if (!NV_IS_SOC_DISPLAY_DEVICE(nv) && !NV_IS_SOC_IGPU_DEVICE(nv))
+ {
+ status.rmStatus = RmRegisterGpudb(pGpu);
+ if (status.rmStatus != NV_OK)
+ {
+ RM_SET_ERROR(status, RM_GPUDB_REGISTER_FAILED);
+ goto shutdown;
+ }
+ }
+
+ if (nvp->b_mobile_config_enabled)
+ {
+ NvU32 ac_plugged = 0;
+ if (nv_acpi_get_powersource(&ac_plugged) == NV_OK)
+ {
+ // LOCK: acquire GPU lock
+ if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_NONE) == NV_OK)
+ {
+ //
+ // As we have already acquired the API Lock here, we are calling
+ // RmPowerSourceChangeEvent directly instead of rm_power_source_change_event.
+ //
+ RmPowerSourceChangeEvent(nv, !ac_plugged);
+
+ // UNLOCK: release GPU lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+ }
+ }
+
+ {
+ // OpenRM support for features beyond what is used on Data Center GPUs
+ // is still fairly immature, so for now require users to opt into use of
+ // OpenRM with a special registry key, if not on a Data Center GPU.
+ const GspStaticConfigInfo *pSCI = GPU_GET_GSP_STATIC_INFO(pGpu);
+
+ if (pSCI->computeBranding != COMPUTE_BRANDING_TYPE_TESLA &&
+ ((pGpu->idInfo.PCIDeviceID >> 16) & 0xffff) != NV_PCI_DEVID_DEVICE_PG189_SKU600)
+ {
+ NvU32 data = NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DEFAULT;
+ RmReadRegistryDword(nv, NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS, &data);
+
+ if (data == NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DISABLE)
+ {
+ if (!nv->printed_openrm_enable_unsupported_gpus_error)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: Open nvidia.ko is only ready for use on Data Center GPUs.\n");
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: To force use of Open nvidia.ko on other GPUs, see the\n");
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: 'OpenRmEnableUnsupportedGpus' kernel module parameter described\n");
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: in the README.\n");
+ nv->printed_openrm_enable_unsupported_gpus_error = NV_TRUE;
+ }
+ RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED);
+ goto shutdown;
+ }
+ }
+ }
+
+ NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter succeeded!\n");
+
+ retVal = NV_TRUE;
+ goto done;
+
+ shutdown:
+ nv->flags &= ~NV_FLAG_IN_RECOVERY;
+
+ // call ShutdownAdapter to undo anything we've done above
+ RmShutdownAdapter(nv);
+
+ failed:
+ NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "RmInitAdapter failed! (0x%x:0x%x:%d)\n",
+ status.initStatus, status.rmStatus, status.line);
+
+done:
+ nv_put_firmware(gspFwHandle);
+ nv_put_firmware(gspFwLogHandle);
+
+ return retVal;
+}
+
+void RmShutdownAdapter(
+ nv_state_t *nv
+)
+{
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ NV_STATUS rmStatus;
+
+ if ((pGpu != NULL) && (nvp->flags & NV_INIT_FLAG_GPUMGR_ATTACH))
+ {
+ NvU32 gpuInstance = gpuGetInstance(pGpu);
+ NvU32 deviceInstance = gpuGetDeviceInstance(pGpu);
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+
+ RmUnixFreeRmApi(nv);
+
+ nv->ud.cpu_address = 0;
+ nv->ud.size = 0;
+
+ // LOCK: acquire GPUs lock
+ if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+ {
+ RmDestroyDeferredDynamicPowerManagement(nv);
+
+ freeNbsiTable(pGpu);
+
+ gpuFreeEventHandle(pGpu);
+
+ OBJCL *pCl = SYS_GET_CL(pSys);
+ OBJOS *pOS = GPU_GET_OS(pGpu);
+ if (pCl != NULL)
+ {
+ if (nvp->flags & NV_INIT_FLAG_CORE_LOGIC)
+ {
+ teardownCoreLogic(pOS, pGpu);
+ }
+ else if (nvp->flags & NV_INIT_FLAG_SCALABILITY)
+ {
+ osTeardownScalability(pGpu);
+ }
+ }
+
+ rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance));
+ rmapiDelPendingDevices(NVBIT(gpuInstance));
+
+ os_disable_console_access();
+
+ if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
+ {
+ rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT);
+ NV_ASSERT(rmStatus == NV_OK);
+ }
+
+ if (nvp->flags & NV_INIT_FLAG_GPU_STATE)
+ {
+ rmStatus = gpuStateDestroy(pGpu);
+ NV_ASSERT(rmStatus == NV_OK);
+ }
+
+ os_enable_console_access();
+
+ //if (nvp->flags & NV_INIT_FLAG_HAL)
+ // destroyHal(pDev);
+
+#if NVCPU_IS_X86_64
+ RmFreeX86EmuState(pGpu);
+#endif
+
+ gpumgrDetachGpu(gpuInstance);
+ gpumgrDestroyDevice(deviceInstance);
+
+ if (nvp->flags & NV_INIT_FLAG_DMA)
+ {
+ RmTeardownDeviceDma(nv);
+ }
+
+ RmClearPrivateState(nv);
+
+ RmUnInitAcpiMethods(pSys);
+
+ // UNLOCK: release GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+ // RM_BASIC_LOCK_MODEL: free GPU lock
+ rmGpuLockFree(deviceInstance);
+ }
+ }
+ else
+ {
+ RmClearPrivateState(nv);
+ }
+
+ RmTeardownRegisters(nv);
+}
+
+void RmPartiallyDisableAdapter(
+ nv_state_t *nv
+)
+{
+ NV_PRINTF(LEVEL_INFO, "%s: RM is in SW Persistence mode\n", __FUNCTION__);
+
+ nv_stop_rc_timer(nv);
+}
+
+void RmDisableAdapter(
+ nv_state_t *nv
+)
+{
+ NV_STATUS rmStatus;
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ NvU32 gpuMask;
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+ //
+ // Normally, we re-enable interrupts when we release the lock
+ // after RmDisableAdapter(), before disabling the bottom-half ISR.
+ // If an interrupt came in in that time window, but the bottom half
+ // wasn't scheduled to run in time, the semaphore acquired by the
+ // top-half would never be released afterwards (since the bottom-half
+ // would never run). This call to intrSetIntrEn() ensures that this won't
+ // happen, by making sure interrupts remain disabled on the GPU in question after
+ // RmDisableAdapter().
+ //
+ Intr *pIntr = GPU_GET_INTR(pGpu);
+ if (pIntr != NULL)
+ {
+ intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED);
+ }
+
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SECONDARY_BUS_RESET) ||
+ pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_FULLCHIP_RESET))
+ {
+ pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_TIMEOUT_RECOVERY, NV_TRUE);
+ nv->flags |= NV_FLAG_IN_RECOVERY;
+ }
+
+ //
+ // Free the client allocated resources.
+ //
+ // This needs to happen prior to tearing down SLI state when SLI is enabled.
+ //
+ // Note this doesn't free RM internal resource allocations. Those are
+ // freed during (gpumgrUpdateSLIConfig->...->)gpuStateUnload.
+ //
+ // We need to free resources for all GPUs linked in a group as
+ // gpumgrUpdateSLIConfig will teardown GPU state for the entire set.
+ //
+ gpuMask = gpumgrGetGpuMask(pGpu);
+
+ rmapiSetDelPendingClientResourcesFromGpuMask(gpuMask);
+ rmapiDelPendingDevices(gpuMask);
+
+ // LOCK: acquire GPUs lock
+ if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+ {
+ nv_stop_rc_timer(nv);
+
+ OBJSYS *pSys = SYS_GET_INSTANCE();
+ OBJCL *pCl = SYS_GET_CL(pSys);
+ OBJOS *pOS = SYS_GET_OS(pSys);
+ if (pCl != NULL)
+ {
+ teardownCoreLogic(pOS, pGpu);
+ nvp->flags &= ~NV_INIT_FLAG_CORE_LOGIC;
+ }
+
+ if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG)
+ {
+ krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu));
+ nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG;
+ }
+
+ os_disable_console_access();
+
+ if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
+ {
+ rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT);
+ NV_ASSERT(rmStatus == NV_OK);
+ nvp->flags &= ~NV_INIT_FLAG_GPU_STATE_LOAD;
+ }
+
+ os_enable_console_access();
+
+ // UNLOCK: release GPUs lock
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+}
+
+NV_STATUS RmGetAdapterStatus(
+ nv_state_t *pNv,
+ NvU32 *pStatus
+)
+{
+ //
+ // This status is determined in RmInitAdapter(); the glue layer
+ // requests it when the adapter failed to initialize to learn
+ // more about the error condition. This is currently limited to
+ // osVerifySystemEnvironment() failures.
+ //
+ nv_priv_t *nvp;
+
+ nvp = NV_GET_NV_PRIV(pNv);
+ if (nvp == NULL)
+ {
+ return NV_ERR_INVALID_STATE;
+ }
+
+ *pStatus = nvp->status;
+ return NV_OK;
+}
+
+static void initVendorSpecificRegistry(
+ OBJGPU *pGpu,
+ NvU16 device_id
+)
+{
+ NV_STATUS rmStatus;
+ NvU32 i;
+ NvU32 subsystem_id;
+ NvU32 subsystem_vendor_id;
+ NvU32 subsystem_device_id;
+ NvU32 vendor_id = 0;
+
+ if (!pGpu)
+ return;
+
+ rmStatus = GPU_BUS_CFG_RD32(pGpu,
+ NV_CONFIG_PCI_NV_11, &subsystem_id);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s: Cannot read NV_CONFIG_PCI_NV_11\n", __FUNCTION__);
+ return;
+ }
+
+ subsystem_vendor_id = (subsystem_id & 0xffff);
+ subsystem_device_id = (subsystem_id >> 16);
+
+ for (i = 0; (nb_id_table[i].subsystem_vendor_id) != 0; i++)
+ {
+ if ((nb_id_table[i].subsystem_vendor_id == subsystem_vendor_id) &&
+ (nb_id_table[i].subsystem_device_id == subsystem_device_id) &&
+ (nb_id_table[i].gpu_device_id == device_id))
+ {
+ vendor_id = subsystem_vendor_id;
+ break;
+ }
+ }
+
+ if (vendor_id != 0)
+ {
+ for (i = 0; nb_reg_table[i].vendor_id != 0; i++)
+ {
+ if (nb_reg_table[i].vendor_id == vendor_id)
+ {
+ osWriteRegistryDword(pGpu, nb_reg_table[i].name,
+ nb_reg_table[i].data);
+ }
+ }
+ }
+}
+
+static void initUnixSpecificRegistry(
+ OBJGPU *pGpu
+)
+{
+ // By default, enable GPU reset on Unix
+ osWriteRegistryDword(pGpu, "RMSecBusResetEnable", 1);
+ osWriteRegistryDword(pGpu, "RMForcePcieConfigSave", 1);
+
+}
+
+void
+osRemoveGpu(
+ NvU32 domain,
+ NvU8 bus,
+ NvU8 device
+)
+{
+ void *handle;
+
+ handle = os_pci_init_handle(domain, bus, device, 0, NULL, NULL);
+ if (handle != NULL)
+ {
+ os_pci_remove(handle);
+ }
+}
+
+/*
+ * Check to see if this board supports GPU exclusion.
+ */
+static NvBool RmIsExcludingAllowed(
+ nv_state_t *pNv
+)
+{
+ NvU32 feature = 0;
+ NvU32 brand;
+
+ // DGX-2/HGX-2 systems pre-date the PBI call
+ if (pNv->pci_info.device_id == 0x1db8)
+ return NV_TRUE;
+
+ if (pciPbiGetFeature(pNv->handle, &feature) != NV_OK)
+ return NV_FALSE;
+
+ brand = DRF_VAL(_PBI, _EXECUTE_ROUTINE, _GET_FEATURE_EXCLUSION, feature);
+
+ return (brand == NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION_ALLOWED);
+}
+
+NV_STATUS RmExcludeAdapter(
+ nv_state_t *nv
+)
+{
+ NV_STATUS rmStatus;
+
+ if (!RmIsExcludingAllowed(nv))
+ return NV_ERR_NOT_SUPPORTED;
+
+ rmStatus = gpumgrExcludeGpuId(nv->gpu_id);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to exclude GPU: 0x%x\n", rmStatus);
+ return rmStatus;
+ }
+
+ /*
+ * GPU exclusion only happens during initialization, and therefore there
+ * is no need to stop activity on the GPU
+ */
+
+ return rmStatus;
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osmemdesc.c b/userspace/sdks/nv525/nvalloc/unix/src/osmemdesc.c
new file mode 100644
index 0000000..ad49cf7
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/osmemdesc.c
@@ -0,0 +1,1030 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2012-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/******************* OS Memory Descriptor APIS *****************************\
+* *
+* This contains routines to create and destroy OS memory descriptor *
+* *
+****************************************************************************/
+
+#include <nv.h> // NV device driver interface
+#include <os/os.h>
+#include <osapi.h>
+#include <nvos.h>
+#include <rmapi/client.h>
+#include <rmapi/rs_utils.h>
+#include <gpu_mgr/gpu_mgr.h>
+
+#include "gpu/bif/kernel_bif.h"
+
+static NV_STATUS osCreateOsDescriptorFromPageArray(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void **);
+static void osDestroyOsDescriptorPageArray(PMEMORY_DESCRIPTOR);
+
+static NV_STATUS osCreateOsDescriptorFromIoMemory(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static void osDestroyOsDescriptorFromIoMemory(PMEMORY_DESCRIPTOR);
+
+static NV_STATUS osCreateOsDescriptorFromPhysAddr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static void osDestroyOsDescriptorFromPhysAddr(PMEMORY_DESCRIPTOR);
+
+static NV_STATUS osCreateOsDescriptorFromFileHandle(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static NV_STATUS osCreateOsDescriptorFromDmaBufPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static void osDestroyOsDescriptorFromDmaBuf(PMEMORY_DESCRIPTOR);
+static NV_STATUS osCreateOsDescriptorFromSgtPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static void osDestroyOsDescriptorFromSgt(PMEMORY_DESCRIPTOR);
+
+static NV_STATUS osCheckGpuBarsOverlapAddrRange(NvRangeU64 addrRange);
+
+NV_STATUS
+osCreateMemFromOsDescriptor
+(
+ OBJGPU *pGpu,
+ NvP64 pDescriptor,
+ NvHandle hClient,
+ NvU32 flags,
+ NvU64 *pLimit,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ NvU32 descriptorType,
+ RS_PRIV_LEVEL privilegeLevel
+)
+{
+ RmClient* pClient;
+ NV_STATUS rmStatus;
+ void *pPrivate;
+
+ if ((pDescriptor == NvP64_NULL) ||
+ (*pLimit == 0) ||
+ (serverutilGetClientUnderLock(hClient, &pClient) != NV_OK))
+ {
+ return NV_ERR_INVALID_PARAM_STRUCT;
+ }
+
+ //
+ // For the sake of simplicity, unmatched RM and OS page
+ // sizes are not currently supported in this path, except for
+ // PPC64LE and aarch64.
+ //
+ // Also, the nvmap handle is sent which can be any random number so
+ // the virtual address alignment sanity check can't be done here.
+ //
+ if (!NVCPU_IS_PPC64LE &&
+ !NVCPU_IS_AARCH64 &&
+ (NV_RM_PAGE_SIZE != os_page_size))
+ {
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ //
+ // The two checks below use cached privilege because they
+ // concern the privilege level of the client, and not the
+ // privilege level of the calling context which may be
+ // overridden to KERNEL at some internal callsites.
+ //
+
+ //
+ // The RM cannot obtain a table of physical addresses
+ // for a kernel virtual address range on all of
+ // the supported UNIX platforms. Since this path is
+ // not really compelling for kernel allocations on any
+ // of those platforms, it is not supported.
+ // For UVM, they could have pre-allocated sysmem to register
+ // with RM so we put in an exception for that case.
+ //
+ if ((rmclientGetCachedPrivilege(pClient) >= RS_PRIV_LEVEL_KERNEL) &&
+ (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR) &&
+ (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE) &&
+ (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) &&
+ (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR))
+ {
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ //
+ // NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR and
+ // NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR can only be utilized by kernel space
+ // rm-clients.
+ //
+ if ((rmclientGetCachedPrivilege(pClient) < RS_PRIV_LEVEL_KERNEL) &&
+ ((descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) ||
+ (descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR)))
+ {
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ switch (descriptorType)
+ {
+ case NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS:
+ rmStatus = NV_ERR_NOT_SUPPORTED;
+ break;
+ case NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR:
+ if (privilegeLevel < RS_PRIV_LEVEL_KERNEL)
+ {
+ rmStatus = NV_ERR_NOT_SUPPORTED;
+ break;
+ }
+ rmStatus = osCreateOsDescriptorFromPhysAddr(pGpu, pDescriptor,
+ hClient, flags, pLimit, ppMemDesc, &pPrivate);
+ break;
+ case NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY:
+ rmStatus = osCreateOsDescriptorFromIoMemory(pGpu, pDescriptor,
+ hClient, flags, pLimit, ppMemDesc, &pPrivate);
+ break;
+ case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY:
+ rmStatus = osCreateOsDescriptorFromPageArray(pGpu, pDescriptor,
+ hClient, flags, pLimit, ppMemDesc, &pPrivate);
+ break;
+ case NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE:
+ rmStatus = osCreateOsDescriptorFromFileHandle(pGpu, pDescriptor,
+ hClient, flags, pLimit, ppMemDesc, &pPrivate);
+ break;
+ case NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR:
+ rmStatus = osCreateOsDescriptorFromDmaBufPtr(pGpu, pDescriptor,
+ hClient, flags, pLimit, ppMemDesc, &pPrivate);
+ break;
+ case NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR:
+ rmStatus = osCreateOsDescriptorFromSgtPtr(pGpu, pDescriptor,
+ hClient, flags, pLimit, ppMemDesc, &pPrivate);
+ break;
+ default:
+ rmStatus = NV_ERR_INVALID_ARGUMENT;
+ break;
+ }
+
+ return rmStatus;
+}
+
+static NV_STATUS
+osCreateMemdescFromPages
+(
+ OBJGPU *pGpu,
+ NvU64 size,
+ NvU32 flags,
+ NvU32 cacheType,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void *pImportPriv,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus;
+ MEMORY_DESCRIPTOR *pMemDesc;
+ NvU64 memdescFlags = MEMDESC_FLAGS_NONE;
+ NvU32 gpuCachedFlags;
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
+ {
+ memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
+ }
+
+ rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0,
+ NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM,
+ cacheType, memdescFlags);
+ if (rmStatus != NV_OK)
+ {
+ return rmStatus;
+ }
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
+ gpuCachedFlags = NV_MEMORY_CACHED;
+ else
+ gpuCachedFlags = NV_MEMORY_UNCACHED;
+
+ pMemDesc = *ppMemDesc;
+ rmStatus = nv_register_user_pages(NV_GET_NV_STATE(pGpu),
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ memdescGetPteArray(pMemDesc, AT_CPU), pImportPriv,
+ ppPrivate);
+ if (rmStatus != NV_OK)
+ {
+ memdescDestroy(pMemDesc);
+ return rmStatus;
+ }
+
+ memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
+ memdescSetAddress(pMemDesc, NvP64_NULL);
+ memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
+ memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
+
+ //
+ // If the OS layer doesn't think in RM page size, we need to inflate the
+ // PTE array into RM pages.
+ //
+ if ((NV_RM_PAGE_SIZE < os_page_size) &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ //
+ // memdescMapIommu() requires the OS-private data to be set on the memory
+ // descriptor, but we don't want to wire up the teardown callback just yet:
+ // that callback needs to unpin the pages, but that will already be done
+ // as part of failure handling further up the stack if memdescMapIommu()
+ // fails. So we only set up the priv-data cleanup callback once we're sure
+ // this call will succeed.
+ //
+ memdescSetMemData(pMemDesc, *ppPrivate, NULL);
+
+ rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+ if (rmStatus != NV_OK)
+ {
+ if ((NV_RM_PAGE_SIZE < os_page_size) &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ nv_unregister_user_pages(NV_GET_NV_STATE(pGpu),
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ NULL /* import_priv */, ppPrivate);
+ memdescDestroy(pMemDesc);
+ return rmStatus;
+ }
+
+ return NV_OK;
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromPageArray
+(
+ OBJGPU *pGpu,
+ NvP64 pDescriptor,
+ NvHandle hClient,
+ NvU32 flags,
+ NvU64 *pLimit,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus;
+
+ *ppPrivate = NvP64_VALUE(pDescriptor);
+
+ //
+ // Since the only type of memory permitted in this path
+ // is anonymous user memory, certain restrictions
+ // apply for the allocation flags:
+ //
+ // 1) anonymous memory is write-back cacheable, hence
+ // the _COHERENCY flag must match.
+ //
+ // 2) the RM has no control over the location of the
+ // associated pages in memory and thus cannot
+ // honor requests for contiguous memory.
+ //
+ // These restrictions are enforced here to avoid subtle
+ // bugs later on.
+ //
+ if ((!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, flags) &&
+ !FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, flags)) ||
+ FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
+ {
+ return NV_ERR_INVALID_FLAGS;
+ }
+
+ rmStatus = osCreateMemdescFromPages(pGpu, (*pLimit + 1), flags,
+ NV_MEMORY_CACHED, ppMemDesc,
+ NULL /* pImportPriv */, ppPrivate);
+ if (rmStatus != NV_OK)
+ {
+ return rmStatus;
+ }
+
+ // All is well - wire up the cleanup callback now
+ memdescSetMemData(*ppMemDesc, memdescGetMemData(*ppMemDesc),
+ osDestroyOsDescriptorPageArray);
+
+ return NV_OK;
+}
+
+/*!
+ * @brief Checks if the given address range overlaps with the BARs for any of
+ * the GPUs.
+ */
+static NV_STATUS
+osCheckGpuBarsOverlapAddrRange
+(
+ NvRangeU64 addrRange
+)
+{
+ NvRangeU64 gpuPhysAddrRange;
+ NvRangeU64 gpuPhysFbAddrRange;
+ NvRangeU64 gpuPhysInstAddrRange;
+ NvU32 gpuInstance;
+ OBJGPU *pGpu;
+ NvU32 gpuMask;
+ NV_STATUS rmStatus;
+
+ rmStatus = gpumgrGetGpuAttachInfo(NULL, &gpuMask);
+ NV_ASSERT_OR_RETURN(rmStatus == NV_OK, rmStatus);
+
+ gpuInstance = 0;
+ while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+ {
+ NV_INIT_RANGE(gpuPhysFbAddrRange, gpumgrGetGpuPhysFbAddr(pGpu),
+ gpumgrGetGpuPhysFbAddr(pGpu) + pGpu->fbLength -1);
+
+ NV_INIT_RANGE(gpuPhysAddrRange, pGpu->busInfo.gpuPhysAddr,
+ pGpu->busInfo.gpuPhysAddr + pGpu->deviceMappings[0].gpuNvLength -1);
+
+ NV_INIT_RANGE(gpuPhysInstAddrRange, pGpu->busInfo.gpuPhysInstAddr,
+ pGpu->busInfo.gpuPhysInstAddr + pGpu->instLength -1);
+
+ if (NV_IS_OVERLAPPING_RANGE(gpuPhysFbAddrRange, addrRange) ||
+ NV_IS_OVERLAPPING_RANGE(gpuPhysAddrRange, addrRange) ||
+ NV_IS_OVERLAPPING_RANGE(gpuPhysInstAddrRange, addrRange))
+ {
+ return NV_ERR_INVALID_ADDRESS;
+ }
+ }
+
+ return NV_OK;
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromIoMemory
+(
+ OBJGPU *pGpu,
+ NvP64 pDescriptor,
+ NvHandle hClient,
+ NvU32 flags,
+ NvU64 *pLimit,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus;
+ NvU32 gpuCachedFlags;
+ MEMORY_DESCRIPTOR *pMemDesc;
+ NvU64 *pPteArray;
+ NvRangeU64 physAddrRange;
+ NvU64 *base = 0;
+ NvBool bAllowMmap;
+
+ //
+ // Unlike the page array path, this one deals exclusively
+ // with I/O memory, which is expected to be contiguous
+ // physically, and which may only be accessed with uncached
+ // transactions.
+ //
+ if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags) ||
+ !FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
+ {
+ return NV_ERR_INVALID_FLAGS;
+ }
+
+ //
+ // _PEER_MAP_OVERRIDE flag is controlled by the RM and not the client.
+ //
+ // RM will set the _PEER_MAP_OVERRIDE_REQUIRED flag itself for IO memory
+ // memory imported with RmVidHeapControl.
+ //
+ if (FLD_TEST_DRF(OS02, _FLAGS, _PEER_MAP_OVERRIDE, _REQUIRED, flags))
+ {
+ //
+ // Don't allow MMIO mappings for unprivileged users
+ // This is a temporary WAR for bug 1630288 "[PeerSync] threat related
+ // to GPU"
+ //
+ KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+ if (!pKernelBif->peerMappingOverride && !osIsAdministrator())
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s(): permission denied, allowPeermapping=%d\n",
+ __FUNCTION__, pKernelBif->peerMappingOverride);
+ return NV_ERR_INSUFFICIENT_PERMISSIONS;
+ }
+ }
+
+ bAllowMmap = !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NEVER_MAP, flags);
+
+ base = (void *)(NvUPtr)pDescriptor;
+
+ //
+ // There is an architectural deadlock scenario involved when full-duplex P2P
+ // enabled over BAR1. See #3 in the description of bug 1571948 which explains
+ // the classic deadlock. So, make sure to error out usermode's memory
+ // registration if a memory range falls within any of the available GPU's
+ // BAR window.
+ //
+ physAddrRange.min = *base;
+ physAddrRange.max = *base + *pLimit;
+
+ rmStatus = osCheckGpuBarsOverlapAddrRange(physAddrRange);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s(): phys range 0x%016llx-0x%016llx overlaps with GPU BARs",
+ __FUNCTION__, physAddrRange.min, physAddrRange.max);
+ return rmStatus;
+ }
+
+ rmStatus = memdescCreate(ppMemDesc, pGpu, (*pLimit + 1), 0,
+ NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM,
+ NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s(): error %d while attempting to create the MMIO mapping\n",
+ __FUNCTION__, rmStatus);
+ return rmStatus;
+ }
+
+ pMemDesc = *ppMemDesc;
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
+ gpuCachedFlags = NV_MEMORY_CACHED;
+ else
+ gpuCachedFlags = NV_MEMORY_UNCACHED;
+
+ memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
+ memdescSetAddress(pMemDesc, NvP64_NULL);
+ memdescSetMemData(pMemDesc, NULL, NULL);
+ memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
+ memdescSetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM, NV_TRUE);
+
+ pPteArray = memdescGetPteArray(pMemDesc, AT_CPU);
+ pPteArray[0] = *base;
+
+ *ppPrivate = NULL;
+
+ if (bAllowMmap)
+ {
+ rmStatus = nv_register_peer_io_mem(NV_GET_NV_STATE(pGpu), pPteArray,
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ ppPrivate);
+ if (rmStatus != NV_OK)
+ {
+ memdescDestroy(pMemDesc);
+ return rmStatus;
+ }
+ }
+
+ memdescSetMemData(pMemDesc, *ppPrivate, NULL);
+
+ //
+ // memdescMapIommu() requires the OS-private data to be set on the memory
+ // descriptor, but we don't want to wire up the teardown callback just yet:
+ // that callback needs to unpin the pages, but that will already be done
+ // as part of failure handling further up the stack if memdescMapIommu()
+ // fails. So we only set up the priv-data cleanup callback once we're sure
+ // this call will succeed.
+ //
+ rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+ if (rmStatus != NV_OK)
+ {
+ if (*ppPrivate != NULL)
+ {
+ nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), *ppPrivate);
+ }
+ memdescDestroy(pMemDesc);
+ return rmStatus;
+ }
+
+ // All is well - wire up the cleanup callback now
+ memdescSetMemData(pMemDesc, memdescGetMemData(pMemDesc),
+ osDestroyOsDescriptorFromIoMemory);
+
+ return NV_OK;
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromPhysAddr
+(
+ OBJGPU *pGpu,
+ NvP64 pDescriptor,
+ NvHandle hClient,
+ NvU32 flags,
+ NvU64 *pLimit,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus;
+ MEMORY_DESCRIPTOR *pMemDesc;
+ NvU64 *pPteArray;
+ NvU64 base = 0;
+ NvU32 cache_type = NV_MEMORY_CACHED;
+ NvU64 memdescFlags = MEMDESC_FLAGS_NONE;
+ NvU64 *pPhys_addrs;
+ NvU64 num_os_pages;
+ NvU32 idx;
+ // Currently only work with contiguous sysmem allocations
+ if (!FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
+ {
+ return NV_ERR_INVALID_FLAGS;
+ }
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_TYPE_SYNCPOINT, _APERTURE, flags))
+ {
+ // Syncpoint memory is uncached, DMA mapping needs to skip CPU sync.
+ cache_type = NV_MEMORY_UNCACHED;
+ }
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
+ {
+ memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
+ }
+
+ base = (NvU64)pDescriptor;
+ rmStatus = memdescCreate(ppMemDesc, pGpu, (*pLimit + 1), 0,
+ NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM,
+ cache_type, memdescFlags);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s(): error %d while creating memdesc for kernel memory\n",
+ __FUNCTION__, rmStatus);
+ return rmStatus;
+ }
+
+ pMemDesc = *ppMemDesc;
+
+ memdescSetAddress(pMemDesc, NvP64_NULL);
+ memdescSetMemData(pMemDesc, NULL, NULL);
+ memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
+
+ pPteArray = memdescGetPteArray(pMemDesc, AT_CPU);
+ pPteArray[0] = base;
+
+ num_os_pages = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);
+ pPhys_addrs = portMemAllocNonPaged(sizeof(NvU64) * num_os_pages);
+ if (pPhys_addrs == NULL)
+ return NV_ERR_NO_MEMORY;
+
+ for (idx = 0; idx < num_os_pages; idx++)
+ {
+ pPhys_addrs[idx] = base + (idx * os_page_size);
+ }
+
+ *ppPrivate = NULL;
+ rmStatus = nv_register_phys_pages(NV_GET_NV_STATE(pGpu), pPhys_addrs,
+ num_os_pages,
+ memdescGetCpuCacheAttrib(pMemDesc),
+ ppPrivate);
+ if (rmStatus != NV_OK)
+ {
+ memdescDestroy(pMemDesc);
+ goto cleanup;
+ }
+
+ memdescSetMemData(pMemDesc, *ppPrivate,
+ osDestroyOsDescriptorFromPhysAddr);
+
+cleanup:
+ portMemFree(pPhys_addrs);
+
+ return rmStatus;
+}
+
+static NV_STATUS
+_createMemdescFromDmaBufSgtHelper
+(
+ OBJGPU *pGpu,
+ NvU32 flags,
+ void *pImportPriv,
+ struct sg_table *pImportSgt,
+ NvU32 size,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate,
+ MEM_DATA_RELEASE_CALL_BACK *pMemDataReleaseCallback
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+ NvU32 cacheType = NV_MEMORY_UNCACHED;
+ MEMORY_DESCRIPTOR *pMemDesc;
+ NvU64 memdescFlags = MEMDESC_FLAGS_NONE;
+ NvU32 gpuCachedFlags;
+
+ NV_ASSERT((pMemDataReleaseCallback == osDestroyOsDescriptorFromDmaBuf) ||
+ (pMemDataReleaseCallback == osDestroyOsDescriptorFromSgt));
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_COMBINE, flags))
+ {
+ cacheType = NV_MEMORY_WRITECOMBINED;
+ }
+ else if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags))
+ {
+ cacheType = NV_MEMORY_CACHED;
+ }
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
+ {
+ memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
+ }
+
+ rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0,
+ NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM,
+ cacheType, memdescFlags);
+ if (rmStatus != NV_OK)
+ {
+ return rmStatus;
+ }
+
+ if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
+ {
+ gpuCachedFlags = NV_MEMORY_CACHED;
+ }
+ else
+ {
+ gpuCachedFlags = NV_MEMORY_UNCACHED;
+ }
+
+ pMemDesc = *ppMemDesc;
+
+ memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
+ memdescSetAddress(pMemDesc, NvP64_NULL);
+ memdescSetMemData(pMemDesc, NULL, NULL);
+ memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
+ memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
+
+ *ppPrivate = NULL;
+ rmStatus = nv_register_sgt(NV_GET_NV_STATE(pGpu),
+ memdescGetPteArray(pMemDesc, AT_CPU),
+ NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
+ memdescGetCpuCacheAttrib(pMemDesc),
+ ppPrivate,
+ pImportSgt,
+ pImportPriv);
+ if (rmStatus != NV_OK)
+ {
+ memdescDestroy(pMemDesc);
+ return rmStatus;
+ }
+
+ //
+ // If the OS layer doesn't think in RM page size, we need to inflate the
+ // PTE array into RM pages.
+ //
+ if ((NV_RM_PAGE_SIZE < os_page_size) &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ memdescSetMemData(*ppMemDesc, *ppPrivate, NULL);
+
+ //
+ // memdescMapIommu() requires the OS-private data to be set on the memory
+ // descriptor, but we don't want to wire up the teardown callback just yet:
+ // that callback does teardown that will already be done as part of failure
+ // handling further up the stack if memdescMapIommu() fails. So we only
+ // setup the priv-data cleanup callback once we're sure this call will
+ // succeed.
+ //
+ rmStatus = memdescMapIommu(*ppMemDesc, pGpu->busInfo.iovaspaceId);
+ if (rmStatus != NV_OK)
+ {
+ if ((NV_RM_PAGE_SIZE < os_page_size) &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+ if (*ppPrivate != NULL)
+ {
+ nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
+ (void **) &pImportPriv, *ppPrivate);
+ }
+ memdescDestroy(pMemDesc);
+ return rmStatus;
+ }
+
+ // All is well - wire up the cleanup callback now
+ memdescSetMemData(*ppMemDesc, *ppPrivate, pMemDataReleaseCallback);
+
+ return rmStatus;
+}
+
+static NV_STATUS
+_createMemdescFromDmaBuf
+(
+ OBJGPU *pGpu,
+ NvU32 flags,
+ nv_dma_buf_t *pImportPriv,
+ void *pUserPages,
+ struct sg_table *pImportSgt,
+ NvU32 size,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus =
+ _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPriv, pImportSgt,
+ size, ppMemDesc, ppPrivate,
+ osDestroyOsDescriptorFromDmaBuf);
+
+ NV_ASSERT(pUserPages == NULL);
+
+ if (rmStatus != NV_OK)
+ {
+ nv_dma_release_dma_buf(NULL, pImportPriv);
+ }
+
+ return rmStatus;
+}
+
+static NV_STATUS
+_createMemdescFromSgt
+(
+ OBJGPU *pGpu,
+ NvU32 flags,
+ struct drm_gem_object *pImportPrivGem,
+ struct sg_table *pImportSgt,
+ NvU32 size,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus =
+ _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPrivGem,
+ pImportSgt, size, ppMemDesc,
+ ppPrivate,
+ osDestroyOsDescriptorFromSgt);
+ if (rmStatus != NV_OK)
+ {
+ nv_dma_release_sgt(pImportSgt, pImportPrivGem);
+ }
+
+ return rmStatus;
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromFileHandle
+(
+ OBJGPU *pGpu,
+ NvP64 pDescriptor,
+ NvHandle hClient,
+ NvU32 flags,
+ NvU64 *pLimit,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ NvU32 size = 0;
+ void *pUserPages = NULL;
+ nv_dma_buf_t *pImportPriv = NULL;
+ struct sg_table *pImportSgt = NULL;
+ NvS32 fd;
+
+ fd = (NvS32)((NvU64)pDescriptor);
+ if ((NvU64)fd != (NvU64)pDescriptor)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s(): fd must fit within a signed 32-bit integer!\n",
+ __FUNCTION__);
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ rmStatus = nv_dma_import_from_fd(nv->dma_dev, fd, &size,
+ &pUserPages, &pImportSgt, &pImportPriv);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s(): Error (%d) while trying to import fd!\n",
+ __FUNCTION__, rmStatus);
+ return rmStatus;
+ }
+
+ return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv,
+ pUserPages, pImportSgt,
+ size, ppMemDesc, ppPrivate);
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromSgtPtr
+(
+ OBJGPU *pGpu,
+ NvP64 pDescriptor,
+ NvHandle hClient,
+ NvU32 flags,
+ NvU64 *pLimit,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS *params =
+ (NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS*)((NvUPtr) pDescriptor);
+
+ struct sg_table *sgt = params->sgt;
+ struct drm_gem_object *gem = params->gem;
+
+ rmStatus = nv_dma_import_sgt(nv->dma_dev, sgt, gem);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s(): Error (%d) while trying to import sgt!\n",
+ __FUNCTION__, rmStatus);
+ return rmStatus;
+ }
+
+ return _createMemdescFromSgt(pGpu, flags, gem, sgt,
+ (*pLimit + 1), ppMemDesc, ppPrivate);
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromDmaBufPtr
+(
+ OBJGPU *pGpu,
+ NvP64 pDescriptor,
+ NvHandle hClient,
+ NvU32 flags,
+ NvU64 *pLimit,
+ MEMORY_DESCRIPTOR **ppMemDesc,
+ void **ppPrivate
+)
+{
+ NV_STATUS rmStatus = NV_OK;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ NvU32 size = 0;
+ void *pUserPages = NULL;
+ nv_dma_buf_t *pImportPriv = NULL;
+ struct sg_table *pImportSgt = NULL;
+ void *dmaBuf = (void*)((NvUPtr)pDescriptor);
+
+ rmStatus = nv_dma_import_dma_buf(nv->dma_dev, dmaBuf, &size,
+ &pUserPages, &pImportSgt, &pImportPriv);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "%s(): Error (%d) while trying to import dma_buf!\n",
+ __FUNCTION__, rmStatus);
+ return rmStatus;
+ }
+
+ return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv,
+ pUserPages, pImportSgt,
+ size, ppMemDesc, ppPrivate);
+}
+
+static void
+osDestroyOsDescriptorFromPhysAddr
+(
+ PMEMORY_DESCRIPTOR pMemDesc
+)
+{
+ OBJGPU *pGpu = pMemDesc->pGpu;
+ void *pPrivate;
+
+ pPrivate = memdescGetMemData(pMemDesc);
+ NV_ASSERT(pPrivate != NULL);
+
+ nv_unregister_phys_pages(NV_GET_NV_STATE(pGpu), pPrivate);
+}
+
+static void
+osDestroyOsDescriptorFromIoMemory
+(
+ PMEMORY_DESCRIPTOR pMemDesc
+)
+{
+ OBJGPU *pGpu = pMemDesc->pGpu;
+ void *pPrivate = memdescGetMemData(pMemDesc);
+
+ if (pPrivate == NULL)
+ {
+ return;
+ }
+
+ nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), pPrivate);
+}
+
+static void
+osDestroyOsDescriptorPageArray
+(
+ MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+ OBJGPU *pGpu = pMemDesc->pGpu;
+ NvU64 osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);
+ NV_STATUS status;
+ void *pPrivate;
+
+ pPrivate = memdescGetMemData(pMemDesc);
+
+ NV_ASSERT(pPrivate != NULL);
+
+ //
+ // TODO: Bug 1811006: Notably skip any IOMMU mapping management as the
+ // pMemDesc->pGpu might have been torn down already and the pGpu passed in
+ // doesn't necessarily have IOMMU mappings. For now just allow memdescDestroy()
+ // to clean up whatever is there (this may not work correctly either if any
+ // of the IOMMU mappings have outlasted their VASPACEs). This should
+ // be cleaned up once the fix for bug 1811006 is known.
+ //
+
+ if ((NV_RM_PAGE_SIZE < os_page_size) &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ nv_unregister_user_pages(NV_GET_NV_STATE(pGpu), osPageCount,
+ NULL /* import_priv */, &pPrivate);
+
+ if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_FOREIGN_PAGE) == NV_FALSE)
+ {
+ status = os_unlock_user_pages(osPageCount, pPrivate);
+ NV_ASSERT(status == NV_OK);
+ }
+ else
+ {
+ os_free_mem(pPrivate);
+ }
+}
+
+static void
+osDestroyOsDescriptorFromDmaBuf
+(
+ PMEMORY_DESCRIPTOR pMemDesc
+)
+{
+ OBJGPU *pGpu = pMemDesc->pGpu;
+ void *pPrivate = memdescGetMemData(pMemDesc);
+
+ struct sg_table *pImportSgt;
+ void *pImportPriv;
+
+ /*
+ * Unmap IOMMU now or we will get a kernel crash when it is unmapped after
+ * pImportSgt is freed.
+ */
+ memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+
+ if ((NV_RM_PAGE_SIZE < os_page_size) &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
+ &pImportPriv, pPrivate);
+
+ /*
+ * pImportSgt doesn't need to be passed to nv_dma_release_dma_buf() because
+ * the DMA-BUF associated with pImportPriv already has a reference to the
+ * SGT.
+ */
+
+ nv_dma_release_dma_buf(NULL, pImportPriv);
+}
+
+static void
+osDestroyOsDescriptorFromSgt
+(
+ PMEMORY_DESCRIPTOR pMemDesc
+)
+{
+ OBJGPU *pGpu = pMemDesc->pGpu;
+ void *pPrivate = memdescGetMemData(pMemDesc);
+
+ struct sg_table *pImportSgt;
+ struct drm_gem_object *pImportPrivGem;
+
+ NV_ASSERT(pPrivate != NULL);
+
+ /*
+ * Unmap IOMMU now or we will get a kernel crash when it is unmapped after
+ * pImportSgt is freed.
+ */
+ memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+
+ if ((NV_RM_PAGE_SIZE < os_page_size) &&
+ !memdescGetContiguity(pMemDesc, AT_CPU))
+ {
+ RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+ pMemDesc->PageCount);
+ }
+
+ nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
+ (void **) &pImportPrivGem, pPrivate);
+
+ nv_dma_release_sgt(pImportSgt, pImportPrivGem);
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osnvlink.c b/userspace/sdks/nv525/nvalloc/unix/src/osnvlink.c
new file mode 100644
index 0000000..2d597a1
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/osnvlink.c
@@ -0,0 +1,698 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include "kernel/gpu/nvlink/kernel_nvlink.h"
+#include "osapi.h"
+
+#if defined(INCLUDE_NVLINK_LIB)
+#include "nvlink.h"
+//
+// The functions in this file are a workaround for a significant design flaw
+// where RM callbacks for the nvlink library are built with the altstack
+// enabled, but the nvlink library is not built with altstack support. Whenever
+// the library calls a callback, the stack switching needs to be accounted for
+// or else we will observe corruption of data structures in the nvlink library
+// as data is pushed onto what the callback thinks is the stack. See bug
+// 1710300.
+//
+// This bug has also exposed other problems, such as the complete lack of
+// locking awareness by these callbacks (e.g., assumption that the RMAPI and
+// GPU locks are always held on entry, which is not a legitimate assumption).
+// For now, we ignore that just to unblock testing.
+//
+extern NvlStatus knvlinkCoreAddLinkCallback(struct nvlink_link *);
+extern NvlStatus knvlinkCoreRemoveLinkCallback(struct nvlink_link *);
+extern NvlStatus knvlinkCoreLockLinkCallback(struct nvlink_link *);
+extern void knvlinkCoreUnlockLinkCallback(struct nvlink_link *);
+extern NvlStatus knvlinkCoreQueueLinkChangeCallback(struct nvlink_link_change *);
+extern NvlStatus knvlinkCoreSetDlLinkModeCallback(struct nvlink_link *, NvU64, NvU32);
+extern NvlStatus knvlinkCoreGetDlLinkModeCallback(struct nvlink_link *, NvU64 *);
+extern NvlStatus knvlinkCoreSetTlLinkModeCallback(struct nvlink_link *, NvU64, NvU32);
+extern NvlStatus knvlinkCoreGetTlLinkModeCallback(struct nvlink_link *, NvU64 *);
+extern NvlStatus knvlinkCoreGetTxSublinkModeCallback(struct nvlink_link *, NvU64 *, NvU32 *);
+extern NvlStatus knvlinkCoreSetTxSublinkModeCallback(struct nvlink_link *, NvU64, NvU32);
+extern NvlStatus knvlinkCoreGetRxSublinkModeCallback(struct nvlink_link *, NvU64 *, NvU32 *);
+extern NvlStatus knvlinkCoreSetRxSublinkModeCallback(struct nvlink_link *, NvU64, NvU32);
+extern NvlStatus knvlinkCoreReadDiscoveryTokenCallback(struct nvlink_link *, NvU64 *);
+extern NvlStatus knvlinkCoreWriteDiscoveryTokenCallback(struct nvlink_link *, NvU64);
+extern void knvlinkCoreTrainingCompleteCallback(struct nvlink_link *);
+extern void knvlinkCoreGetUphyLoadCallback(struct nvlink_link *, NvBool*);
+
+/*!
+ * @brief Helper to allocate an alternate stack from within core RM.
+ *
+ * This needs to be an NV_API_CALL (built to use the original stack instead
+ * of the altstack) since it is called before we switch to using the altstack.
+ */
+static NV_STATUS NV_API_CALL osNvlinkAllocAltStack(nvidia_stack_t **pSp)
+{
+ NV_STATUS status = NV_OK;
+ nvidia_stack_t *sp = NULL;
+#if defined(NVCPU_X86_64) && defined(__use_altstack__)
+ status = os_alloc_mem((void **)&sp, sizeof(nvidia_stack_t));
+ if (status == NV_OK)
+ {
+ sp->size = sizeof(sp->stack);
+ sp->top = sp->stack + sp->size;
+ }
+#endif
+ *pSp = sp;
+ return status;
+}
+
+/*!
+ * @brief Helper to free an alternate stack from within core RM.
+ *
+ * This needs to be an NV_API_CALL (built to use the original stack instead
+ * of the altstack) since it is called after we've switched back to using the
+ * original stack.
+ */
+static void NV_API_CALL osNvlinkFreeAltStack(nvidia_stack_t *sp)
+{
+#if defined(NVCPU_X86_64) && defined(__use_altstack__)
+ os_free_mem(sp);
+#endif
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_add_link
+(
+ struct nvlink_link *link
+)
+{
+ void *fp;
+ NvlStatus status;
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp;
+
+ if (NV_OK != osNvlinkAllocAltStack(&sp))
+ {
+ return NVL_ERR_GENERIC;
+ }
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ status = knvlinkCoreAddLinkCallback(link);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ if (status == NVL_SUCCESS)
+ {
+ pLink->pOsInfo = sp;
+ }
+ else
+ {
+ osNvlinkFreeAltStack(sp);
+ }
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_remove_link
+(
+ struct nvlink_link *link
+)
+{
+ void *fp;
+ NvlStatus status;
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ pLink->pOsInfo = NULL;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ status = knvlinkCoreRemoveLinkCallback(link);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ osNvlinkFreeAltStack(sp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_lock_link
+(
+ struct nvlink_link *link
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreLockLinkCallback(link);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static void NV_API_CALL rm_nvlink_ops_unlock_link
+(
+ struct nvlink_link *link
+)
+{
+ void *fp;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ knvlinkCoreUnlockLinkCallback(link);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_queue_link_change
+(
+ struct nvlink_link_change *link_change
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link_change->master->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreQueueLinkChangeCallback(link_change);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_dl_link_mode
+(
+ struct nvlink_link *link,
+ NvU64 mode,
+ NvU32 flags
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreSetDlLinkModeCallback(link, mode, flags);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_dl_link_mode
+(
+ struct nvlink_link *link,
+ NvU64 *mode
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreGetDlLinkModeCallback(link, mode);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_tl_link_mode
+(
+ struct nvlink_link *link,
+ NvU64 mode,
+ NvU32 flags
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreSetTlLinkModeCallback(link, mode, flags);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_tl_link_mode
+(
+ struct nvlink_link *link,
+ NvU64 *mode
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreGetTlLinkModeCallback(link, mode);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_tx_mode
+(
+ struct nvlink_link *link,
+ NvU64 mode,
+ NvU32 flags
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreSetTxSublinkModeCallback(link, mode, flags);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_tx_mode
+(
+ struct nvlink_link *link,
+ NvU64 *mode,
+ NvU32 *subMode
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreGetTxSublinkModeCallback(link, mode, subMode);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_rx_mode
+(
+ struct nvlink_link *link,
+ NvU64 mode,
+ NvU32 flags
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreSetRxSublinkModeCallback(link, mode, flags);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_rx_mode
+(
+ struct nvlink_link *link,
+ NvU64 *mode,
+ NvU32 *subMode
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreGetRxSublinkModeCallback(link, mode, subMode);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_rx_detect
+(
+ struct nvlink_link *link,
+ NvU32 flags
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreSetRxSublinkDetectCallback(link, flags);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_rx_detect
+(
+ struct nvlink_link *link
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreGetRxSublinkDetectCallback(link);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static void NV_API_CALL rm_nvlink_get_uphy_load
+(
+ struct nvlink_link *link,
+ NvBool *bUnlocked
+)
+{
+ void *fp;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ knvlinkCoreGetUphyLoadCallback(link, bUnlocked);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_read_link_discovery_token
+(
+ struct nvlink_link *link,
+ NvU64 *token
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreReadDiscoveryTokenCallback(link, token);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_write_link_discovery_token
+(
+ struct nvlink_link *link,
+ NvU64 token
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreWriteDiscoveryTokenCallback(link, token);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+
+ return status;
+}
+
+static void NV_API_CALL rm_nvlink_ops_training_complete
+(
+ struct nvlink_link *link
+)
+{
+ void *fp;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK *pLink = link->link_info;
+ nvidia_stack_t *sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ knvlinkCoreTrainingCompleteCallback(link);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_ali_training
+(
+ struct nvlink_link *link
+)
+{
+ void *fp;
+ NvlStatus status;
+ THREAD_STATE_NODE threadState = {0};
+ KNVLINK_RM_LINK * pLink = link->link_info;
+ nvidia_stack_t * sp = (nvidia_stack_t *)pLink->pOsInfo;
+
+ NV_ENTER_RM_RUNTIME(sp, fp);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+ status = knvlinkCoreAliTrainingCallback(link);
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ NV_EXIT_RM_RUNTIME(sp, fp);
+ return status;
+}
+
+#endif /* defined(INCLUDE_NVLINK_LIB) */
+
+const struct nvlink_link_handlers* osGetNvlinkLinkCallbacks(void)
+{
+#if defined(INCLUDE_NVLINK_LIB)
+ static const struct nvlink_link_handlers rm_nvlink_link_ops =
+ {
+ .add = rm_nvlink_ops_add_link,
+ .remove = rm_nvlink_ops_remove_link,
+ .lock = rm_nvlink_ops_lock_link,
+ .unlock = rm_nvlink_ops_unlock_link,
+ .queue_link_change = rm_nvlink_ops_queue_link_change,
+ .set_dl_link_mode = rm_nvlink_ops_set_dl_link_mode,
+ .get_dl_link_mode = rm_nvlink_ops_get_dl_link_mode,
+ .set_tl_link_mode = rm_nvlink_ops_set_tl_link_mode,
+ .get_tl_link_mode = rm_nvlink_ops_get_tl_link_mode,
+ .set_tx_mode = rm_nvlink_ops_set_link_tx_mode,
+ .get_tx_mode = rm_nvlink_ops_get_link_tx_mode,
+ .set_rx_mode = rm_nvlink_ops_set_link_rx_mode,
+ .get_rx_mode = rm_nvlink_ops_get_link_rx_mode,
+ .set_rx_detect = rm_nvlink_ops_set_link_rx_detect,
+ .get_rx_detect = rm_nvlink_ops_get_link_rx_detect,
+ .write_discovery_token = rm_nvlink_ops_write_link_discovery_token,
+ .read_discovery_token = rm_nvlink_ops_read_link_discovery_token,
+ .training_complete = rm_nvlink_ops_training_complete,
+ .get_uphy_load = rm_nvlink_get_uphy_load,
+ .ali_training = rm_nvlink_ops_ali_training,
+ };
+
+ return &rm_nvlink_link_ops;
+#else
+ return NULL;
+#endif
+}
+
+/*
+ * @brief Verif only function to get the chiplib overrides for link connection
+ * state for all NVLINKs.
+ *
+ * If chiplib overrides exist, each link can either be enabled (1) or disabled (0)
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[in] maxLinks Size of pLinkConnection array
+ * @param[out] pLinkConnection array of pLinkConnection values to be populated by MODS
+ *
+ * @return NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available)
+ */
+NV_STATUS
+osGetForcedNVLinkConnection
+(
+ OBJGPU *pGpu,
+ NvU32 maxLinks,
+ NvU32 *pLinkConnection
+)
+{
+ int i, ret;
+ NV_STATUS status;
+ char path[64];
+ OBJSYS *pSys;
+ OBJOS *pOS;
+
+ NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER);
+ NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED);
+ NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT);
+
+ pSys = SYS_GET_INSTANCE();
+ pOS = SYS_GET_OS(pSys);
+ if (pOS == NULL || pOS->osSimEscapeRead == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "%s: escape reads not supported on platform\n",
+ __FUNCTION__);
+ return NV_ERR_NOT_SUPPORTED;
+ }
+
+ for (i = 0; i < maxLinks; i++)
+ {
+ ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|NVLink%u", i);
+ NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1)));
+
+ status = pOS->osSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]);
+ if (status == NV_OK)
+ {
+ NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__,
+ path, pLinkConnection[i]);
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_INFO, "%s: osSimEscapeRead for '%s' failed (%u)\n",
+ __FUNCTION__, path, status);
+ return NV_ERR_NOT_SUPPORTED;
+ }
+ }
+ return NV_OK;
+}
+
+/*
+ * @brief Get Platform suggested NVLink linerate
+ *
+ * NVLink will use this function to get the platform suggested linerate
+ * if available in FRU or device tree.
+ *
+ * @param[in] pGpu GPU object pointer
+ * @param[out] NvU32 * Suggested datarate
+ *
+ * @return NV_OK or NV_ERR_NOT_SUPPORTED (platform linerate data not available)
+ */
+NV_STATUS
+osGetPlatformNvlinkLinerate
+(
+ OBJGPU *pGpu,
+ NvU32 *lineRate
+)
+{
+#if defined(NVCPU_PPC64LE)
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
+
+ if (!pKernelNvlink)
+ return NV_ERR_INVALID_ARGUMENT;
+
+ return nv_get_nvlink_line_rate(nv, lineRate);
+#else
+ //TODO : FRU based method to be filled out by Bug 200285656
+ //*lineRate = 0;
+ //return NV_OK;
+ return NV_ERR_NOT_SUPPORTED;
+#endif
+}
+
+void
+osSetNVLinkSysmemLinkState
+(
+ OBJGPU *pGpu,
+ NvBool enabled
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ NV_ASSERT(enabled);
+ if (enabled)
+ nv_dma_enable_nvlink(nv->dma_dev);
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/osunix.c b/userspace/sdks/nv525/nvalloc/unix/src/osunix.c
new file mode 100644
index 0000000..7d70602
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/osunix.c
@@ -0,0 +1,88 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+/***************************** HW State Routines ***************************\
+* *
+* Fills in os specific function pointers for the Unix OS object. *
+* *
+\***************************************************************************/
+
+#include <osfuncs.h>
+#include <os/os.h>
+
+static void initOSSpecificFunctionPointers(OBJOS *);
+static void initMiscOSFunctionPointers(OBJOS *);
+static void initUnixOSFunctionPointers(OBJOS *);
+static void initOSSpecificProperties(OBJOS *);
+
+void
+osInitObjOS(OBJOS *pOS)
+{
+ initOSSpecificFunctionPointers(pOS);
+ initOSSpecificProperties(pOS);
+}
+
+static void
+initOSSpecificFunctionPointers(OBJOS *pOS)
+{
+ initMiscOSFunctionPointers(pOS);
+ initUnixOSFunctionPointers(pOS);
+}
+
+static void
+initMiscOSFunctionPointers(OBJOS *pOS)
+{
+ pOS->osQueueWorkItem = osQueueWorkItem;
+ pOS->osQueueWorkItemWithFlags = osQueueWorkItemWithFlags;
+ pOS->osQueueSystemWorkItem = osQueueSystemWorkItem;
+}
+
+static void
+initUnixOSFunctionPointers(OBJOS *pOS)
+{
+#if defined(NVCPU_X86_64)
+ pOS->osNv_rdcr4 = nv_rdcr4;
+ pOS->osNv_cpuid = nv_cpuid;
+#endif
+
+ pOS->osCallACPI_DSM = osCallACPI_DSM;
+ pOS->osCallACPI_DDC = osCallACPI_DDC;
+ pOS->osCallACPI_NVHG_ROM = osCallACPI_NVHG_ROM;
+ pOS->osCallACPI_DOD = osCallACPI_DOD;
+ pOS->osCallACPI_MXDM = osCallACPI_MXDM;
+ pOS->osCallACPI_MXDS = osCallACPI_MXDS;
+
+ pOS->osDbgBreakpointEnabled = osDbgBreakpointEnabled;
+}
+
+static void
+initOSSpecificProperties
+(
+ OBJOS *pOS
+)
+{
+ pOS->setProperty(pOS, PDB_PROP_OS_ONDEMAND_VBLANK_CONTROL_ENABLE_DEFAULT, NV_TRUE);
+ pOS->setProperty(pOS, PDB_PROP_OS_CACHED_MEMORY_MAPPINGS_FOR_ACPI_TABLE, NV_TRUE);
+ pOS->setProperty(pOS, PDB_PROP_OS_LIMIT_GPU_RESET, NV_TRUE);
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/registry.c b/userspace/sdks/nv525/nvalloc/unix/src/registry.c
new file mode 100644
index 0000000..522b937
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/registry.c
@@ -0,0 +1,705 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include <nv-priv.h>
+#include <nvos.h>
+
+#if defined(DEBUG_REGISTRY)
+#define DBG_REG_PRINTF(a, ...) \
+ NV_PRINTF(LEVEL_INFO, a, ##__VA_ARGS__)
+#else
+#define DBG_REG_PRINTF(a, ...)
+#endif
+
+static NvS32 stringCaseCompare(
+ const char *string1,
+ const char *string2
+)
+{
+ NvU8 c1, c2;
+
+ do
+ {
+ c1 = *string1, c2 = *string2;
+ if (c1 >= 'A' && c1 <= 'Z')
+ c1 += ('a' - 'A');
+ if (c2 >= 'A' && c2 <= 'Z')
+ c2 += ('a' - 'A');
+ string1++, string2++;
+ }
+ while ((c1 == c2) && (c1 != '\0'));
+
+ return (c1 - c2);
+}
+
+static nv_reg_entry_t *the_registry = NULL;
+
+static nv_reg_entry_t* regCreateNewRegistryKey(
+ nv_state_t *nv,
+ const char *regParmStr
+)
+{
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ nv_reg_entry_t *new_reg = NULL;
+ char *new_ParmStr = NULL;
+ NvU32 parm_size;
+
+ if (regParmStr == NULL)
+ {
+ DBG_BREAKPOINT();
+ return NULL;
+ }
+
+ new_reg = portMemAllocNonPaged(sizeof(nv_reg_entry_t));
+ if (NULL == new_reg)
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to grow registry\n");
+ return NULL;
+ }
+
+ portMemSet(new_reg, 0, sizeof(nv_reg_entry_t));
+
+ if (regParmStr != NULL)
+ {
+ parm_size = (portStringLength(regParmStr) + 1);
+ new_ParmStr = portMemAllocNonPaged(parm_size);
+ if (NULL == new_ParmStr)
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to allocate registry param string\n");
+ portMemFree(new_reg);
+ return NULL;
+ }
+
+ NV_ASSERT(parm_size <= NVOS38_MAX_REGISTRY_STRING_LENGTH);
+
+ if (portMemCopy(new_ParmStr, parm_size, regParmStr, parm_size) == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to copy registry param string\n");
+ portMemFree(new_ParmStr);
+ portMemFree(new_reg);
+ return NULL;
+ }
+ }
+
+ new_reg->regParmStr = new_ParmStr;
+ new_reg->type = NV_REGISTRY_ENTRY_TYPE_UNKNOWN;
+
+ if (nvp != NULL)
+ {
+ new_reg->next = nvp->pRegistry;
+ nvp->pRegistry = new_reg;
+ DBG_REG_PRINTF("local registry now at 0x%p\n", nvp->pRegistry);
+ }
+ else
+ {
+ new_reg->next = the_registry;
+ the_registry = new_reg;
+ DBG_REG_PRINTF("global registry now at 0x%p\n", the_registry);
+ }
+
+ return new_reg;
+}
+
+static NV_STATUS regFreeEntry(nv_reg_entry_t *tmp)
+{
+ portMemFree(tmp->regParmStr);
+ tmp->regParmStr = NULL;
+ {
+ portMemFree(tmp->pdata);
+ tmp->pdata = NULL;
+ tmp->len = 0;
+ }
+ portMemFree(tmp);
+
+ return NV_OK;
+}
+
+static nv_reg_entry_t* regFindRegistryEntry(
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU32 type,
+ NvBool *bGlobalEntry
+)
+{
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ nv_reg_entry_t *tmp;
+
+ DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+ if (nvp != NULL)
+ {
+ tmp = nvp->pRegistry;
+ DBG_REG_PRINTF(" local registry at 0x%p\n", tmp);
+
+ while ((tmp != NULL) && (tmp->regParmStr != NULL))
+ {
+ DBG_REG_PRINTF(" Testing against %s\n",
+ tmp->regParmStr);
+ if ((stringCaseCompare(tmp->regParmStr, regParmStr) == 0) &&
+ (type == tmp->type))
+ {
+ DBG_REG_PRINTF(" found a match!\n");
+ if (bGlobalEntry)
+ *bGlobalEntry = NV_FALSE;
+ return tmp;
+ }
+ tmp = tmp->next;
+ }
+ }
+
+ tmp = the_registry;
+ DBG_REG_PRINTF(" global registry at 0x%p\n", tmp);
+
+ while ((tmp != NULL) && (tmp->regParmStr != NULL))
+ {
+ DBG_REG_PRINTF(" Testing against %s\n",
+ tmp->regParmStr);
+ if ((stringCaseCompare(tmp->regParmStr, regParmStr) == 0) &&
+ (type == tmp->type))
+ {
+ DBG_REG_PRINTF(" found a match!\n");
+ if (bGlobalEntry)
+ *bGlobalEntry = NV_TRUE;
+ return tmp;
+ }
+ tmp = tmp->next;
+ }
+
+ DBG_REG_PRINTF(" no match\n");
+ return NULL;
+}
+
+NV_STATUS RmWriteRegistryDword(
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU32 Data
+)
+{
+ nv_reg_entry_t *tmp;
+ NvBool bGlobalEntry;
+
+ if (regParmStr == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ DBG_REG_PRINTF("%s: %s -> 0x%x\n", __FUNCTION__, regParmStr, Data);
+
+ tmp = regFindRegistryEntry(nv, regParmStr,
+ NV_REGISTRY_ENTRY_TYPE_DWORD, &bGlobalEntry);
+
+ // If we found an entry and we were looking for a global entry and
+ // found a global, or we were looking for a per-GPU entry and found a
+ // per-GPU entry
+ if (tmp != NULL &&
+ ((nv == NULL && bGlobalEntry) ||
+ (nv != NULL && !bGlobalEntry)))
+ {
+ tmp->data = Data;
+
+ if (stringCaseCompare(regParmStr, "ResmanDebugLevel") == 0)
+ {
+ os_dbg_set_level(Data);
+ }
+
+ return NV_OK;
+ }
+
+ tmp = regCreateNewRegistryKey(nv, regParmStr);
+ if (tmp == NULL)
+ return NV_ERR_GENERIC;
+
+ tmp->type = NV_REGISTRY_ENTRY_TYPE_DWORD;
+ tmp->data = Data;
+
+ return NV_OK;
+}
+
+NV_STATUS RmReadRegistryDword(
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU32 *Data
+)
+{
+ nv_reg_entry_t *tmp;
+
+ if ((regParmStr == NULL) || (Data == NULL))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+ tmp = regFindRegistryEntry(nv, regParmStr,
+ NV_REGISTRY_ENTRY_TYPE_DWORD, NULL);
+ if (tmp == NULL)
+ {
+ tmp = regFindRegistryEntry(nv, regParmStr,
+ NV_REGISTRY_ENTRY_TYPE_BINARY, NULL);
+ if ((tmp != NULL) && (tmp->len >= sizeof(NvU32)))
+ {
+ *Data = *(NvU32 *)tmp->pdata;
+ }
+ else
+ {
+ DBG_REG_PRINTF(" not found\n");
+ return NV_ERR_GENERIC;
+ }
+ }
+ else
+ {
+ *Data = tmp->data;
+ }
+
+ DBG_REG_PRINTF(" found in the_registry: 0x%x\n", *Data);
+
+ return NV_OK;
+}
+
+NV_STATUS RmReadRegistryBinary(
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU8 *Data,
+ NvU32 *cbLen
+)
+{
+ nv_reg_entry_t *tmp;
+ NV_STATUS status;
+
+ if ((regParmStr == NULL) || (Data == NULL) || (cbLen == NULL))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+ tmp = regFindRegistryEntry(nv, regParmStr,
+ NV_REGISTRY_ENTRY_TYPE_BINARY, NULL);
+ if (tmp == NULL)
+ {
+ DBG_REG_PRINTF(" not found\n");
+ return NV_ERR_GENERIC;
+ }
+
+ DBG_REG_PRINTF(" found\n");
+
+ if (*cbLen >= tmp->len)
+ {
+ portMemCopy((NvU8 *)Data, *cbLen, (NvU8 *)tmp->pdata, tmp->len);
+ *cbLen = tmp->len;
+ status = NV_OK;
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "buffer (length: %u) is too small (data length: %u)\n",
+ *cbLen, tmp->len);
+ status = NV_ERR_GENERIC;
+ }
+
+ return status;
+}
+
+NV_STATUS RmWriteRegistryBinary(
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU8 *Data,
+ NvU32 cbLen
+)
+{
+ nv_reg_entry_t *tmp;
+ NvBool bGlobalEntry;
+
+ if ((regParmStr == NULL) || (Data == NULL))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+ tmp = regFindRegistryEntry(nv, regParmStr,
+ NV_REGISTRY_ENTRY_TYPE_BINARY, &bGlobalEntry);
+
+ // If we found an entry and we were looking for a global entry and
+ // found a global, or we were looking for a per-GPU entry and found a
+ // per-GPU entry
+ if (tmp != NULL &&
+ ((nv == NULL && bGlobalEntry) ||
+ (nv != NULL && !bGlobalEntry)))
+ {
+ if (tmp->pdata != NULL)
+ {
+ portMemFree(tmp->pdata);
+ tmp->pdata = NULL;
+ tmp->len = 0;
+ }
+ }
+ else
+ {
+ tmp = regCreateNewRegistryKey(nv, regParmStr);
+ if (tmp == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to create binary registry entry\n");
+ return NV_ERR_GENERIC;
+ }
+ }
+
+ tmp->pdata = portMemAllocNonPaged(cbLen);
+ if (NULL == tmp->pdata)
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to write binary registry entry\n");
+ return NV_ERR_GENERIC;
+ }
+
+ tmp->type = NV_REGISTRY_ENTRY_TYPE_BINARY;
+ tmp->len = cbLen;
+ portMemCopy((NvU8 *)tmp->pdata, tmp->len, (NvU8 *)Data, cbLen);
+
+ return NV_OK;
+}
+
+NV_STATUS RmWriteRegistryString(
+ nv_state_t *nv,
+ const char *regParmStr,
+ const char *buffer,
+ NvU32 bufferLength
+)
+{
+ nv_reg_entry_t *tmp;
+ NvBool bGlobalEntry;
+
+ if ((regParmStr == NULL) || (buffer == NULL))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+ tmp = regFindRegistryEntry(nv, regParmStr,
+ NV_REGISTRY_ENTRY_TYPE_STRING, &bGlobalEntry);
+
+ // If we found an entry and we were looking for a global entry and
+ // found a global, or we were looking for a per-GPU entry and found a
+ // per-GPU entry
+ if (tmp != NULL &&
+ ((nv == NULL && bGlobalEntry) ||
+ (nv != NULL && !bGlobalEntry)))
+ {
+ if (tmp->pdata != NULL)
+ {
+ portMemFree(tmp->pdata);
+ tmp->len = 0;
+ tmp->pdata = NULL;
+ }
+ }
+ else
+ {
+ tmp = regCreateNewRegistryKey(nv, regParmStr);
+ if (tmp == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "failed to allocate a string registry entry!\n");
+ return NV_ERR_INSUFFICIENT_RESOURCES;
+ }
+ }
+
+ tmp->pdata = portMemAllocNonPaged(bufferLength);
+ if (tmp->pdata == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to write a string registry entry!\n");
+ return NV_ERR_NO_MEMORY;
+ }
+
+ tmp->type = NV_REGISTRY_ENTRY_TYPE_STRING;
+ tmp->len = bufferLength;
+ portMemCopy((void *)tmp->pdata, tmp->len, buffer, (bufferLength - 1));
+ tmp->pdata[bufferLength-1] = '\0';
+
+ return NV_OK;
+}
+
+NV_STATUS RmReadRegistryString(
+ nv_state_t *nv,
+ const char *regParmStr,
+ NvU8 *buffer,
+ NvU32 *pBufferLength
+)
+{
+ NvU32 bufferLength;
+ nv_reg_entry_t *tmp;
+
+ if ((regParmStr == NULL) || (buffer == NULL) || (pBufferLength == NULL))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+ bufferLength = *pBufferLength;
+ *pBufferLength = 0;
+ *buffer = '\0';
+
+ tmp = regFindRegistryEntry(nv, regParmStr,
+ NV_REGISTRY_ENTRY_TYPE_STRING, NULL);
+ if (tmp == NULL)
+ {
+ return NV_ERR_GENERIC;
+ }
+
+ if (bufferLength >= tmp->len)
+ {
+ portMemCopy((void *)buffer, bufferLength, (void *)tmp->pdata, tmp->len);
+ *pBufferLength = tmp->len;
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "buffer (length: %u) is too small (data length: %u)\n",
+ bufferLength, tmp->len);
+ return NV_ERR_BUFFER_TOO_SMALL;
+ }
+
+ return NV_OK;
+}
+
+NV_STATUS RmInitRegistry(void)
+{
+ NV_STATUS rmStatus;
+
+ rmStatus = os_registry_init();
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "failed to initialize the OS registry!\n");
+ }
+
+ return rmStatus;
+}
+
+NV_STATUS RmDestroyRegistry(nv_state_t *nv)
+{
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ nv_reg_entry_t *tmp;
+
+ if (nvp != NULL)
+ {
+ tmp = nvp->pRegistry;
+ nvp->pRegistry = NULL;
+ }
+ else
+ {
+ tmp = the_registry;
+ the_registry = NULL;
+ }
+
+ while (tmp != NULL)
+ {
+ nv_reg_entry_t *entry = tmp;
+ tmp = tmp->next;
+ regFreeEntry(entry);
+ }
+
+ return NV_OK;
+}
+
+static void regCountEntriesAndSize(
+ NvU32 *pNumEntries, // Pointer to number of entries
+ NvU32 *pSize, // Pointer to total size
+ nv_reg_entry_t *pRegEntry // Pointer local or global registry
+)
+{
+ //
+ // Note that *pNumEntries and *pSize are not initialized here. This is so
+ // we can accumulate totals of both global and local registries.
+ //
+ NvU32 numEntries = *pNumEntries;
+ NvU32 size = *pSize;
+
+ while ((pRegEntry != NULL) && (pRegEntry->regParmStr != NULL))
+ {
+ size += portStringLength(pRegEntry->regParmStr) + 1 + pRegEntry->len;
+ numEntries++;
+ pRegEntry = pRegEntry->next;
+ }
+
+ *pNumEntries = numEntries;
+ *pSize = size;
+}
+
+static NV_STATUS regCopyEntriesToPackedBuffer(
+ PACKED_REGISTRY_TABLE *pRegTable, // Pointer to packed record
+ nv_reg_entry_t *pRegEntry, // Pointer local or global registry
+ NvU32 *pEntryIndex, // Pointer to next index
+ NvU32 *pDataOffset // Pointer to offset of next data byte.
+)
+{
+ NvU8 *pByte = (NvU8 *)pRegTable; // Byte version of record pointer.
+ NV_STATUS nvStatus = NV_OK;
+ NvU32 entryIndex = *pEntryIndex;
+ NvU32 dataOffset = *pDataOffset;
+
+ // Walk the records and copy the data.
+ while ((pRegEntry != NULL) && (pRegEntry->regParmStr != NULL))
+ {
+ PACKED_REGISTRY_ENTRY *pEntry = &pRegTable->entries[entryIndex];
+ NvU32 slen = portStringLength(pRegEntry->regParmStr) + 1;
+
+ // Sanity check the data offset and index against counted totals.
+ if ((dataOffset + slen + pRegEntry->len > pRegTable->size) ||
+ (entryIndex >= pRegTable->numEntries))
+ {
+ // Something has changed since we counted them?
+ NV_PRINTF(LEVEL_ERROR, "Registry entry record is full\n");
+ nvStatus = NV_ERR_INVALID_STATE;
+ break;
+ }
+
+ // Copy registry entry name to data blob.
+ pEntry->nameOffset = dataOffset;
+ portMemCopy(&pByte[dataOffset], slen, pRegEntry->regParmStr, slen);
+ dataOffset += slen;
+
+ switch (pRegEntry->type)
+ {
+ case NV_REGISTRY_ENTRY_TYPE_DWORD:
+ pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_DWORD;
+ pEntry->length = sizeof(NvU32);
+ pEntry->data = pRegEntry->data;
+ break;
+
+ case NV_REGISTRY_ENTRY_TYPE_BINARY:
+ case NV_REGISTRY_ENTRY_TYPE_STRING:
+ pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_STRING;
+ if (pRegEntry->type == NV_REGISTRY_ENTRY_TYPE_BINARY)
+ pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_BINARY;
+
+ pEntry->length = pRegEntry->len;
+ pEntry->data = dataOffset;
+ portMemCopy(&pByte[dataOffset], pEntry->length,
+ pRegEntry->pdata, pRegEntry->len);
+ dataOffset += pRegEntry->len;
+ break;
+
+ default:
+ // We should never get here.
+ pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_UNKNOWN;
+ pEntry->length = 0;
+ pEntry->data = 0;
+ DBG_BREAKPOINT();
+ break;
+ }
+
+ pRegEntry = pRegEntry->next;
+ entryIndex++;
+ }
+
+ *pEntryIndex = entryIndex;
+ *pDataOffset = dataOffset;
+ return nvStatus;
+}
+
+// Package registry entries
+NV_STATUS RmPackageRegistry(
+ nv_state_t *nv,
+ PACKED_REGISTRY_TABLE *pRegTable,
+ NvU32 *pSize
+)
+{
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ nv_reg_entry_t *pLocalRegistry = NULL;
+ NV_STATUS nvStatus = NV_OK;
+ NvU32 totalSize;
+ NvU32 numEntries;
+
+ if (pSize == NULL)
+ return NV_ERR_INVALID_ARGUMENT;
+
+ // Use the local (per-device) registry if we have one.
+ if (nvp != NULL)
+ pLocalRegistry = nvp->pRegistry;
+
+ numEntries = 0;
+ totalSize = NV_OFFSETOF(PACKED_REGISTRY_TABLE, entries);
+
+ // Count the number of global entries and total size.
+ regCountEntriesAndSize(&numEntries, &totalSize, the_registry);
+
+ // Count the number of local entries and total size.
+ regCountEntriesAndSize(&numEntries, &totalSize, pLocalRegistry);
+
+ // Add table record size into total size.
+ totalSize += sizeof(PACKED_REGISTRY_ENTRY) * numEntries;
+
+ //
+ // If this function is called to only compute total size of registry table,
+ // then we are done here.
+ //
+ if (pRegTable == NULL)
+ {
+ *pSize = totalSize;
+ return NV_OK;
+ }
+
+ // Return warning if there are no registry entries.
+ if (numEntries == 0)
+ return NV_WARN_NOTHING_TO_DO;
+
+ if (totalSize > *pSize)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Registry entries overflow RPC record\n");
+ return NV_ERR_BUFFER_TOO_SMALL;
+ }
+
+ // Fill in our new structure with the first pass (counting) values.
+ pRegTable->size = totalSize;
+ *pSize = totalSize;
+ pRegTable->numEntries = numEntries;
+
+ // Offset of first byte after the registry entry table.
+ totalSize = NV_OFFSETOF(PACKED_REGISTRY_TABLE, entries) +
+ (sizeof(PACKED_REGISTRY_ENTRY) * numEntries);
+
+ // Starting index in the registry entry table.
+ numEntries = 0;
+
+ // Walk the global registry and copy the data.
+ nvStatus = regCopyEntriesToPackedBuffer(pRegTable,
+ the_registry, &numEntries, &totalSize);
+
+ // Walk the local registry and copy the data.
+ if (nvStatus == NV_OK)
+ {
+ nvStatus = regCopyEntriesToPackedBuffer(pRegTable,
+ pLocalRegistry, &numEntries, &totalSize);
+ }
+
+ // Sanity check second pass against first pass.
+ if ((numEntries != pRegTable->numEntries) || (totalSize != pRegTable->size))
+ {
+ NV_PRINTF(LEVEL_ERROR, "First/second pass mismatch\n");
+ nvStatus = NV_ERR_INVALID_STATE;
+ }
+
+ return nvStatus;
+}
+
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/rm-gpu-ops.c b/userspace/sdks/nv525/nvalloc/unix/src/rm-gpu-ops.c
new file mode 100644
index 0000000..d542393
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/rm-gpu-ops.c
@@ -0,0 +1,813 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include <os/os.h>
+#include <osapi.h>
+#include <core/thread_state.h>
+#include "rmapi/nv_gpu_ops.h"
+#include "gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h"
+
+NV_STATUS NV_API_CALL rm_gpu_ops_create_session(
+ nvidia_stack_t *sp,
+ struct gpuSession **session)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsCreateSession(session);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_destroy_session (
+ nvidia_stack_t *sp, gpuSessionHandle session)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDestroySession(session);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_device_create (
+ nvidia_stack_t *sp,
+ nvgpuSessionHandle_t session,
+ const gpuInfo *pGpuInfo,
+ const NvProcessorUuid *gpuUuid,
+ nvgpuDeviceHandle_t *device,
+ NvBool bCreateSmcPartition)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDeviceCreate(session, pGpuInfo, gpuUuid, device, bCreateSmcPartition);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_device_destroy (
+ nvidia_stack_t *sp,
+ gpuDeviceHandle device)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDeviceDestroy(device);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_address_space_create (
+ nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ NvU64 vaBase,
+ NvU64 vaSize,
+ gpuAddressSpaceHandle *vaSpace,
+ gpuAddressSpaceInfo *vaSpaceInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsAddressSpaceCreate(device, vaBase, vaSize, vaSpace,
+ vaSpaceInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_dup_address_space(
+ nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ NvHandle hUserClient,
+ NvHandle hUserVASpace,
+ gpuAddressSpaceHandle *dupedVaspace,
+ gpuAddressSpaceInfo *vaSpaceInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDupAddressSpace(device, hUserClient, hUserVASpace,
+ dupedVaspace, vaSpaceInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_address_space_destroy(nvidia_stack_t *sp,
+ gpuAddressSpaceHandle vaspace)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ nvGpuOpsAddressSpaceDestroy(vaspace);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_memory_alloc_fb(
+ nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace,
+ NvLength size, NvU64 *gpuOffset, gpuAllocInfo *allocInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsMemoryAllocFb(vaspace, size, gpuOffset, allocInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_get_p2p_caps(nvidia_stack_t *sp,
+ gpuDeviceHandle device1,
+ gpuDeviceHandle device2,
+ getP2PCapsParams *pP2pCapsParams)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsGetP2PCaps(device1, device2, pP2pCapsParams);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_memory_alloc_sys(
+ nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace,
+ NvLength size, NvU64 *gpuOffset, gpuAllocInfo *allocInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsMemoryAllocSys(vaspace, size, gpuOffset, allocInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_pma_register_callbacks(
+ nvidia_stack_t *sp,
+ void *pPma,
+ pmaEvictPagesCb_t evictPages,
+ pmaEvictRangeCb_t evictRange,
+ void *callbackData)
+{
+ THREAD_STATE_NODE threadState;
+ NV_STATUS rmStatus;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ rmStatus = pmaRegisterEvictionCb(pPma, evictPages, evictRange, callbackData);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+void NV_API_CALL rm_gpu_ops_pma_unregister_callbacks(
+ nvidia_stack_t *sp,
+ void *pPma)
+{
+ THREAD_STATE_NODE threadState;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ pmaUnregisterEvictionCb(pPma);
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_get_pma_object(
+ nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ void **pPma,
+ const nvgpuPmaStatistics_t *pPmaPubStats)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsGetPmaObject(device, pPma,
+ (const UvmPmaStatistics **)pPmaPubStats);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_pma_alloc_pages(
+ nvidia_stack_t *sp, void *pPma,
+ NvLength pageCount, NvU32 pageSize,
+ nvgpuPmaAllocationOptions_t pPmaAllocOptions,
+ NvU64 *pPages)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsPmaAllocPages(pPma, pageCount, pageSize,
+ pPmaAllocOptions, pPages);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_pma_pin_pages(
+ nvidia_stack_t *sp, void *pPma,
+ NvU64 *pPages, NvLength pageCount, NvU32 pageSize, NvU32 flags)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsPmaPinPages(pPma, pPages, pageCount, pageSize, flags);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_pma_unpin_pages(
+ nvidia_stack_t *sp, void *pPma,
+ NvU64 *pPages, NvLength pageCount, NvU32 pageSize)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsPmaUnpinPages(pPma, pPages, pageCount, pageSize);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_map(
+ nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace,
+ NvU64 gpuOffset, NvLength length, void **cpuPtr, NvU32 pageSize)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsMemoryCpuMap(vaspace, gpuOffset, length, cpuPtr,
+ pageSize);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_ummap(
+ nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, void* cpuPtr)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ nvGpuOpsMemoryCpuUnMap(vaspace, cpuPtr);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_channel_allocate(nvidia_stack_t *sp,
+ gpuAddressSpaceHandle vaspace,
+ const gpuChannelAllocParams *allocParams,
+ gpuChannelHandle *channel,
+ gpuChannelInfo *channelInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsChannelAllocate(vaspace, allocParams, channel,
+ channelInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_channel_destroy(nvidia_stack_t * sp,
+ nvgpuChannelHandle_t channel)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ nvGpuOpsChannelDestroy(channel);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_pma_free_pages(nvidia_stack_t *sp,
+ void *pPma, NvU64 *pPages, NvLength pageCount, NvU32 pageSize, NvU32 flags)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ nvGpuOpsPmaFreePages(pPma, pPages, pageCount, pageSize, flags);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_memory_free(
+ nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, NvU64 gpuOffset)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ nvGpuOpsMemoryFree(vaspace, gpuOffset);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_query_caps(nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuCaps * caps)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsQueryCaps(device, caps);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_query_ces_caps(nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuCesCaps *caps)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsQueryCesCaps(device, caps);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_get_gpu_info(nvidia_stack_t *sp,
+ const NvProcessorUuid *pUuid,
+ const gpuClientInfo *pGpuClientInfo,
+ gpuInfo *pGpuInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsGetGpuInfo(pUuid, pGpuClientInfo, pGpuInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_service_device_interrupts_rm(nvidia_stack_t *sp,
+ gpuDeviceHandle device)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsServiceDeviceInterruptsRM(device);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_set_page_directory (nvidia_stack_t *sp,
+ gpuAddressSpaceHandle vaSpace,
+ NvU64 physAddress, unsigned numEntries,
+ NvBool bVidMemAperture, NvU32 pasid)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsSetPageDirectory(vaSpace, physAddress, numEntries,
+ bVidMemAperture, pasid);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_unset_page_directory (nvidia_stack_t *sp,
+ gpuAddressSpaceHandle vaSpace)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsUnsetPageDirectory(vaSpace);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_dup_allocation(nvidia_stack_t *sp,
+ gpuAddressSpaceHandle srcVaSpace,
+ NvU64 srcAddress,
+ gpuAddressSpaceHandle dstVaSpace,
+ NvU64 dstVaAlignment,
+ NvU64 *dstAddress)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDupAllocation(srcVaSpace, srcAddress, dstVaSpace, dstVaAlignment, dstAddress);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_dup_memory (nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ NvHandle hClient,
+ NvHandle hPhysMemory,
+ NvHandle *hDupMemory,
+ nvgpuMemoryInfo_t gpuMemoryInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDupMemory(device, hClient, hPhysMemory, hDupMemory, gpuMemoryInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_free_duped_handle (nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ NvHandle hPhysHandle)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsFreeDupedHandle(device, hPhysHandle);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_get_fb_info (nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuFbInfo * fbInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsGetFbInfo(device, fbInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_get_ecc_info (nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuEccInfo * eccInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsGetEccInfo(device, eccInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+//
+// Please see the comments for nvUvmInterfaceOwnPageFaultIntr(), in
+// nv_uvm_interface.h, for the recommended way to use this routine.
+//
+// How it works:
+//
+// The rmGpuLocksAcquire call generally saves the current GPU interrupt
+// state, then disables interrupt generation for one (or all) GPUs.
+// Likewise, the rmGpuLocksRelease call restores (re-enables) those
+// interrupts to their previous state. However, the rmGpuLocksRelease
+// call does NOT restore interrupts that RM does not own.
+//
+// This is rather hard to find in the code, so: very approximately, the
+// following sequence happens: rmGpuLocksRelease, osEnableInterrupts,
+// intrRestoreNonStall_HAL, intrEncodeIntrEn_HAL, and that last one skips
+// over any interrupts that RM does not own.
+//
+// This means that things are a bit asymmetric, because this routine
+// actually changes that ownership in between the rmGpuLocksAcquire and
+// rmGpuLocksRelease calls. So:
+//
+// -- If you call this routine with bOwnInterrupts == NV_TRUE (UVM is
+// taking ownership from the RM), then rmGpuLocksAcquire disables all
+// GPU interrupts. Then the ownership is taken away from RM, so the
+// rmGpuLocksRelease call leaves the replayable page fault interrupts
+// disabled. It is then up to UVM (the caller) to enable replayable
+// page fault interrupts when it is ready.
+//
+// -- If you call this routine with bOwnInterrupts == NV_FALSE (UVM is
+// returning ownership to the RM), then rmGpuLocksAcquire disables
+// all GPU interrupts that RM owns. Then the ownership is returned to
+// RM, so the rmGpuLocksRelease call re-enables replayable page fault
+// interrupts. So, that implies that you need to disable replayable page
+// fault interrupts before calling this routine, in order to hand
+// over a GPU to RM that is not generating interrupts, until RM is
+// ready to handle the interrupts.
+//
+NV_STATUS NV_API_CALL rm_gpu_ops_own_page_fault_intr(nvidia_stack_t *sp,
+ struct gpuDevice *device,
+ NvBool bOwnInterrupts)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsOwnPageFaultIntr(device, bOwnInterrupts);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_init_fault_info (nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuFaultInfo *pFaultInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsInitFaultInfo(device, pFaultInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_destroy_fault_info (nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuFaultInfo *pFaultInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDestroyFaultInfo(device, pFaultInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+// Functions
+//
+// - rm_gpu_ops_has_pending_non_replayable_faults
+// - rm_gpu_ops_get_non_replayable_faults
+//
+// Cannot take the GPU/RM lock because it is called during fault servicing.
+// This could produce deadlocks if the UVM bottom half gets stuck behind a
+// stalling interrupt that cannot be serviced if UVM is holding the lock.
+//
+// However, these functions can be safely called with no locks because it is
+// just accessing the given client shadow fault buffer, which is implemented
+// using a lock-free queue. There is a different client shadow fault buffer
+// per GPU: RM top-half producer, UVM top/bottom-half consumer.
+
+NV_STATUS NV_API_CALL rm_gpu_ops_has_pending_non_replayable_faults(nvidia_stack_t *sp,
+ gpuFaultInfo *pFaultInfo,
+ NvBool *hasPendingFaults)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsHasPendingNonReplayableFaults(pFaultInfo, hasPendingFaults);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_get_non_replayable_faults(nvidia_stack_t *sp,
+ gpuFaultInfo *pFaultInfo,
+ void *faultBuffer,
+ NvU32 *numFaults)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsGetNonReplayableFaults(pFaultInfo, faultBuffer, numFaults);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_init_access_cntr_info(nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuAccessCntrInfo *accessCntrInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsInitAccessCntrInfo(device, accessCntrInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_destroy_access_cntr_info(nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuAccessCntrInfo *accessCntrInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDestroyAccessCntrInfo(device, accessCntrInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_enable_access_cntr(nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuAccessCntrInfo *accessCntrInfo,
+ gpuAccessCntrConfig *accessCntrConfig)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsEnableAccessCntr(device, accessCntrInfo, accessCntrConfig);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_disable_access_cntr(nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ gpuAccessCntrInfo *accessCntrInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsDisableAccessCntr(device, accessCntrInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_p2p_object_create(nvidia_stack_t *sp,
+ gpuDeviceHandle device1,
+ gpuDeviceHandle device2,
+ NvHandle *hP2pObject)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp, fp);
+ rmStatus = nvGpuOpsP2pObjectCreate(device1, device2, hP2pObject);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+ return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_p2p_object_destroy(nvidia_stack_t *sp,
+ nvgpuSessionHandle_t session,
+ NvHandle hP2pObject)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp, fp);
+ nvGpuOpsP2pObjectDestroy(session, hP2pObject);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_get_external_alloc_ptes(nvidia_stack_t* sp,
+ nvgpuAddressSpaceHandle_t vaSpace,
+ NvHandle hDupedMemory,
+ NvU64 offset,
+ NvU64 size,
+ nvgpuExternalMappingInfo_t gpuExternalMappingInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp, fp);
+ rmStatus = nvGpuOpsGetExternalAllocPtes(vaSpace, hDupedMemory, offset, size,
+ gpuExternalMappingInfo);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_retain_channel(nvidia_stack_t* sp,
+ nvgpuAddressSpaceHandle_t vaSpace,
+ NvHandle hClient,
+ NvHandle hChannel,
+ void **retainedChannel,
+ nvgpuChannelInstanceInfo_t channelInstanceInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp, fp);
+ rmStatus = nvGpuOpsRetainChannel(vaSpace, hClient, hChannel,
+ (gpuRetainedChannel **)retainedChannel,
+ channelInstanceInfo);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_bind_channel_resources(nvidia_stack_t* sp,
+ void *retainedChannel,
+ nvgpuChannelResourceBindParams_t channelResourceBindParams)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp, fp);
+ rmStatus = nvGpuOpsBindChannelResources(retainedChannel,
+ channelResourceBindParams);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+ return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_release_channel(nvidia_stack_t *sp, void *retainedChannel)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp, fp);
+ nvGpuOpsReleaseChannel(retainedChannel);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+void NV_API_CALL
+rm_gpu_ops_stop_channel(nvidia_stack_t * sp,
+ void *retainedChannel,
+ NvBool bImmediate)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ nvGpuOpsStopChannel(retainedChannel, bImmediate);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_get_channel_resource_ptes(nvidia_stack_t* sp,
+ nvgpuAddressSpaceHandle_t vaSpace,
+ NvP64 resourceDescriptor,
+ NvU64 offset,
+ NvU64 size,
+ nvgpuExternalMappingInfo_t gpuExternalMappingInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp, fp);
+ rmStatus = nvGpuOpsGetChannelResourcePtes(vaSpace, resourceDescriptor,
+ offset, size,
+ gpuExternalMappingInfo);
+ NV_EXIT_RM_RUNTIME(sp, fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_report_non_replayable_fault(nvidia_stack_t *sp,
+ nvgpuDeviceHandle_t device,
+ const void *pFaultPacket)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsReportNonReplayableFault(device, pFaultPacket);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_paging_channel_allocate(nvidia_stack_t *sp,
+ gpuDeviceHandle device,
+ const gpuPagingChannelAllocParams *allocParams,
+ gpuPagingChannelHandle *channel,
+ gpuPagingChannelInfo *channelInfo)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsPagingChannelAllocate(device, allocParams, channel,
+ channelInfo);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_paging_channel_destroy(nvidia_stack_t *sp,
+ gpuPagingChannelHandle channel)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ nvGpuOpsPagingChannelDestroy(channel);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_paging_channels_map(nvidia_stack_t *sp,
+ gpuAddressSpaceHandle srcVaSpace,
+ NvU64 srcAddress,
+ gpuDeviceHandle device,
+ NvU64 *dstAddress)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsPagingChannelsMap(srcVaSpace, srcAddress, device, dstAddress);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_paging_channels_unmap(nvidia_stack_t *sp,
+ gpuAddressSpaceHandle srcVaSpace,
+ NvU64 srcAddress,
+ gpuDeviceHandle device)
+{
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ nvGpuOpsPagingChannelsUnmap(srcVaSpace, srcAddress, device);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_paging_channel_push_stream(nvidia_stack_t *sp,
+ gpuPagingChannelHandle channel,
+ char *methodStream,
+ NvU32 methodStreamSize)
+{
+ NV_STATUS rmStatus;
+ void *fp;
+ NV_ENTER_RM_RUNTIME(sp,fp);
+ rmStatus = nvGpuOpsPagingChannelPushStream(channel, methodStream, methodStreamSize);
+ NV_EXIT_RM_RUNTIME(sp,fp);
+ return rmStatus;
+}
+
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/rmobjexportimport.c b/userspace/sdks/nv525/nvalloc/unix/src/rmobjexportimport.c
new file mode 100644
index 0000000..dcc8d94
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/rmobjexportimport.c
@@ -0,0 +1,634 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2016-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ * @file
+ *
+ * @brief Provides RmExportObject, RmImportObject, RmFreeObjExportHandle and
+ * RmGetExportObjectInfo interfaces :
+ *
+ * These interfaces allow rm clients to export their objects into
+ * a unique RmObjExportHandle which another rm client could
+ * import, even if the source rm client gets destroyed.
+ *
+ * RM's device instance may get destroyed asynchronously, in which
+ * case exported objects residing on that device instance also get
+ * destroyed. This means it is not possible to import it back, but the
+ * RmObjExportHandle into which the object had been exported still
+ * remains valid but no other object could get it.
+ *
+ * There are not init/fini routines, it is the responsibility of the
+ * rest of RM's eco-system to make sure that all RmObjExportHandles get
+ * freed during driver unload.
+ *
+ * The api lock is expected to be held before calling into
+ * rmobjexportimport.c; do not hold gpu or any other lock.
+ */
+
+#include "rmobjexportimport.h"
+#include "nvlimits.h"
+#include "gpu/device/device.h"
+
+#include "containers/map.h"
+#include "rmapi/rmapi.h"
+#include "rmapi/rs_utils.h"
+
+#include "class/cl0080.h"
+#include "class/cl2080.h"
+#include <ctrl/ctrl0000/ctrl0000unix.h>
+#include <ctrl/ctrl0000/ctrl0000client.h>
+
+//
+// A reference to an RmObjExportHandle
+// generated by function RmGenerateObjExportHandle().
+//
+typedef struct
+{
+ NvU32 deviceInstance;
+} RmObjExportHandleRef;
+MAKE_MAP(RmObjExportHandleMap, RmObjExportHandleRef);
+
+//
+// Memory allocator
+//
+PORT_MEM_ALLOCATOR *pMemAllocator;
+
+//
+// Map RmObjExportHandle -> RmObjExportHandleRef
+//
+RmObjExportHandleMap objExportHandleMap;
+
+//
+// Rm client to use to dup an object exported to RmObjExportHandle. The minimal
+// requirement for duping is to have a device object allocated. This rm client
+// is simply like any other external rm client and has no any special handling.
+//
+// We keep this rm client just like any other external rm client: if
+// gpu(s)/device gets powered-down/uninitialized, rm objects allocated by
+// external rm clients and located on that gpu(s)/device gets freed (the
+// os-layer does that). In that way, code in this file doesn't need to worry
+// about freeing exported objects located on that gpu(s)/device.
+//
+NvHandle hObjExportRmClient;
+
+//
+// Tracker for device and subdevice handles. For now only one subdevice
+// (instance 0) is supported per device.
+//
+typedef struct
+{
+ NvHandle hRmDevice;
+ NvHandle hRmSubDevice;
+} RmObjExportDevice;
+
+RmObjExportDevice objExportDevice[NV_MAX_DEVICES];
+
+//
+// Usage reference counter for static object in this file like rm client used to
+// dup an exported object, memory allocator, map etc.
+//
+NvU64 objExportImportRefCount;
+
+//
+// Static functions for internal use to code in this file.
+//
+static NV_STATUS RmRefObjExportImport (void);
+static void RmUnrefObjExportImport (void);
+
+static RmObjExportHandle RmGenerateObjExportHandle (NvU32 deviceInstance);
+static NV_STATUS RmUnrefObjExportHandle (RmObjExportHandle hObject);
+
+//
+// Free the RmObjExportHandle.
+//
+static NV_STATUS RmUnrefObjExportHandle(RmObjExportHandle hObject)
+{
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+ RmObjExportHandleRef *pHandleRef =
+ mapFind(&objExportHandleMap, hObject);
+
+ if (pHandleRef == NULL)
+ {
+ return NV_ERR_OBJECT_NOT_FOUND;
+ }
+
+ if (pRmApi->Free(pRmApi,
+ hObjExportRmClient,
+ (NvHandle)mapKey(&objExportHandleMap, pHandleRef)) != NV_OK)
+ {
+ NV_PRINTF(LEVEL_WARNING,
+ "Exported object trying to free was zombie in %s\n",
+ __FUNCTION__);
+ }
+
+ mapRemove(&objExportHandleMap, pHandleRef);
+
+ return NV_OK;
+}
+
+//
+// Generate unique RmObjExportHandle.
+//
+static RmObjExportHandle RmGenerateObjExportHandle(NvU32 deviceInstance)
+{
+ //
+ // The object export handle belongs to range of 0 to
+ // (MAX_OBJ_EXPORT_HANDLES - 1).
+ //
+ // Handle 0 is considered as invalid object handle, this function generates
+ // handle from range of 1 to (MAX_OBJ_EXPORT_HANDLES - 1).
+ //
+ #define MAX_OBJ_EXPORT_HANDLES 0x80000
+
+ static NvHandle hObjExportHandleNext = 1;
+
+ RmObjExportHandle hStartHandle = hObjExportHandleNext;
+ RmObjExportHandle hObject = 0;
+
+ do
+ {
+ RmObjExportHandleRef *pHandleRef;
+
+ hObject = hObjExportHandleNext++;
+ /* Reset hObjExportHandleNext to next valid handle */
+ if (hObjExportHandleNext == MAX_OBJ_EXPORT_HANDLES) {
+ hObjExportHandleNext = 1;
+ }
+
+ pHandleRef = mapFind(&objExportHandleMap, hObject);
+
+ if (hObject != hObjExportRmClient && pHandleRef == NULL)
+ {
+ break;
+ }
+ else
+ {
+ hObject = 0;
+ }
+
+ } while(hObjExportHandleNext != hStartHandle);
+
+ if (hObject != 0)
+ {
+ RmObjExportHandleRef *pHandleRef =
+ mapInsertNew(&objExportHandleMap, hObject);
+
+ if (pHandleRef != NULL)
+ {
+ pHandleRef->deviceInstance = deviceInstance;
+ }
+ else
+ {
+ hObject = 0;
+ }
+ }
+
+ return hObject;
+}
+
+//
+// Validate that the given hObject is not one of our internally used handles.
+//
+// Note that mapFind(&objExportHandleMap, hObject) could still fail; that is the
+// caller's responsibility.
+//
+static NvBool RmValidateHandleAgainstInternalHandles(RmObjExportHandle hObject)
+{
+ NvU32 i;
+
+ //
+ // No external RmObjExportHandle could be valid if hObjExportRmClient has
+ // not been allocated yet, or if it is equal to any of the handles used
+ // internally by code in this file.
+ //
+ if (objExportImportRefCount == 0 || hObjExportRmClient == 0 ||
+ hObject == hObjExportRmClient)
+ {
+ return NV_FALSE;
+ }
+
+ for (i = 0; i < NV_ARRAY_ELEMENTS(objExportDevice); i++)
+ {
+ if (objExportDevice[i].hRmDevice != 0 &&
+ (hObject == objExportDevice[i].hRmDevice ||
+ hObject == objExportDevice[i].hRmSubDevice))
+ {
+ return NV_FALSE;
+ }
+ }
+
+ return NV_TRUE;
+}
+
+//
+// Increment reference count of static objects internally
+// used by code in this file.
+//
+static NV_STATUS RmRefObjExportImport(void)
+{
+ NV_STATUS rmStatus = NV_OK;
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+ if ((objExportImportRefCount++) != 0)
+ {
+ NV_ASSERT(hObjExportRmClient != 0);
+ NV_ASSERT(pMemAllocator != NULL);
+ return NV_OK;
+ }
+
+ rmStatus = pRmApi->AllocWithHandle(pRmApi,
+ NV01_NULL_OBJECT,
+ NV01_NULL_OBJECT,
+ NV01_NULL_OBJECT,
+ NV01_ROOT,
+ &hObjExportRmClient);
+ if (rmStatus != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Unable to alloc root in %s\n", __FUNCTION__);
+ goto failed;
+ }
+
+ pMemAllocator = portMemAllocatorCreateNonPaged();
+
+ if (pMemAllocator == NULL)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to alloc memory allocator in %s\n",
+ __FUNCTION__);
+ goto failed;
+ }
+
+ mapInit(&objExportHandleMap, pMemAllocator);
+
+ return NV_OK;
+
+failed:
+
+ RmUnrefObjExportImport();
+
+ return rmStatus;
+}
+
+//
+// Decrement reference count of static objects internally used by code in this
+// file, and free them if reference count reaches to zero.
+//
+static void RmUnrefObjExportImport(void)
+{
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+
+ if ((--objExportImportRefCount) != 0)
+ {
+ return;
+ }
+
+ if (pMemAllocator != NULL)
+ {
+ NvU32 i;
+
+ for (i = 0; i < NV_ARRAY_ELEMENTS(objExportDevice); i++)
+ {
+ if (objExportDevice[i].hRmDevice != 0)
+ {
+ RmUnrefObjExportHandle(objExportDevice[i].hRmSubDevice);
+ objExportDevice[i].hRmSubDevice = 0;
+ RmUnrefObjExportHandle(objExportDevice[i].hRmDevice);
+ objExportDevice[i].hRmDevice = 0;
+ }
+ }
+
+ mapDestroy(&objExportHandleMap);
+
+ portMemAllocatorRelease(pMemAllocator);
+ pMemAllocator = NULL;
+ }
+
+ if (hObjExportRmClient != 0)
+ {
+ NV_STATUS rmStatus = pRmApi->Free(pRmApi,
+ hObjExportRmClient,
+ hObjExportRmClient);
+
+ NV_ASSERT(rmStatus == NV_OK);
+ hObjExportRmClient = 0;
+ }
+}
+
+NV_STATUS RmExportObject(NvHandle hSrcClient, NvHandle hSrcObject,
+ RmObjExportHandle *pDstObject, NvU32 *pDeviceInstance)
+{
+ RmObjExportHandle hDstObject;
+ NvU32 deviceInstance = NV_MAX_DEVICES;
+ NvHandle hTmpObject;
+ NvBool bClientAsDstParent = NV_FALSE;
+ NV_STATUS status;
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+ if (pDstObject == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ //
+ // Find the device instance on which the rm object exists.
+ //
+ hTmpObject = hSrcObject;
+ do
+ {
+ RsResourceRef *pResourceRef;
+
+ status = serverutilGetResourceRef(hSrcClient, hTmpObject, &pResourceRef);
+ if (status != NV_OK)
+ return status;
+
+ Device *pDevice = dynamicCast(pResourceRef->pResource, Device);
+ if (pDevice != NULL)
+ {
+ deviceInstance = pDevice->deviceInst;
+ break;
+ }
+
+ hTmpObject = pResourceRef->pParentRef ? pResourceRef->pParentRef->hResource : 0;
+ } while (hTmpObject != 0);
+
+ // If a memory object is not parented by a device, use client as a parent.
+ if ((hTmpObject == 0) || (deviceInstance >= NV_MAX_DEVICES))
+ {
+ bClientAsDstParent = NV_TRUE;
+ }
+
+ status = RmRefObjExportImport();
+
+ if (status != NV_OK)
+ {
+ return status;
+ }
+
+ if (!bClientAsDstParent &&
+ ((objExportDevice[deviceInstance].hRmDevice == 0) ||
+ serverutilValidateNewResourceHandle(hObjExportRmClient,
+ objExportDevice[deviceInstance].hRmDevice)))
+ {
+ //
+ // Device object has not been created or it got destroyed in the
+ // teardown path of device instance destruction; allocate a fresh device
+ // object.
+ //
+ NV0080_ALLOC_PARAMETERS params;
+ NV2080_ALLOC_PARAMETERS subdevParams;
+
+ if (objExportDevice[deviceInstance].hRmDevice == 0)
+ {
+ NV_ASSERT(objExportDevice[deviceInstance].hRmSubDevice == 0);
+
+ objExportDevice[deviceInstance].hRmDevice =
+ RmGenerateObjExportHandle(deviceInstance);
+ objExportDevice[deviceInstance].hRmSubDevice =
+ RmGenerateObjExportHandle(deviceInstance);
+
+ if (objExportDevice[deviceInstance].hRmDevice == 0 ||
+ objExportDevice[deviceInstance].hRmSubDevice == 0)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to allocate object handles in %s\n",
+ __FUNCTION__);
+
+ status = NV_ERR_NO_MEMORY;
+ goto done;
+ }
+ }
+
+ portMemSet(&params, 0, sizeof(NV0080_ALLOC_PARAMETERS));
+
+ params.deviceId = deviceInstance;
+
+ status = pRmApi->AllocWithHandle(pRmApi,
+ hObjExportRmClient,
+ hObjExportRmClient,
+ objExportDevice[deviceInstance].hRmDevice,
+ NV01_DEVICE_0,
+ &params);
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Unable to alloc device in %s\n",
+ __FUNCTION__);
+ goto done;
+ }
+
+ portMemSet(&subdevParams, 0, sizeof(NV2080_ALLOC_PARAMETERS));
+
+ subdevParams.subDeviceId = 0;
+
+ status = pRmApi->AllocWithHandle(pRmApi,
+ hObjExportRmClient,
+ objExportDevice[deviceInstance].hRmDevice,
+ objExportDevice[deviceInstance].hRmSubDevice,
+ NV20_SUBDEVICE_0,
+ &subdevParams);
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Unable to alloc subdevice in %s\n",
+ __FUNCTION__);
+
+ (void) pRmApi->Free(pRmApi, hObjExportRmClient,
+ objExportDevice[deviceInstance].hRmDevice);
+ goto done;
+ }
+ }
+
+ hDstObject = RmGenerateObjExportHandle(deviceInstance);
+
+ if (hDstObject == 0)
+ {
+ NV_PRINTF(LEVEL_ERROR, "Failed to allocate object handle in %s\n",
+ __FUNCTION__);
+ status = NV_ERR_NO_MEMORY;
+ goto done;
+ }
+
+ // If duping under device handle fails, try subdevice handle.
+ status = pRmApi->DupObject(pRmApi,
+ hObjExportRmClient,
+ bClientAsDstParent ? hObjExportRmClient :
+ objExportDevice[deviceInstance].hRmDevice,
+ &hDstObject,
+ hSrcClient,
+ hSrcObject,
+ 0 /* flags */);
+ if (status != NV_OK)
+ {
+ if (!bClientAsDstParent && (status == NV_ERR_INVALID_OBJECT_PARENT))
+ {
+ NV_PRINTF(LEVEL_INFO,
+ "pRmApi->DupObject(Dev, failed due to invalid parent in %s."
+ " Now attempting DupObject with Subdev handle.\n",
+ __FUNCTION__);
+
+ status = pRmApi->DupObject(pRmApi,
+ hObjExportRmClient,
+ objExportDevice[deviceInstance].hRmSubDevice,
+ &hDstObject,
+ hSrcClient,
+ hSrcObject,
+ 0 /* flags */);
+ if (status != NV_OK)
+ {
+ RmUnrefObjExportHandle(hDstObject);
+
+ NV_PRINTF(LEVEL_ERROR,
+ "pRmApi->DupObject(Subdev, failed with error code 0x%x in %s\n",
+ status, __FUNCTION__);
+ goto done;
+ }
+ }
+ else
+ {
+ RmUnrefObjExportHandle(hDstObject);
+
+ NV_PRINTF(LEVEL_ERROR,
+ "pRmApi->DupObject(Dev, failed with error code 0x%x in %s\n",
+ status, __FUNCTION__);
+ goto done;
+ }
+ }
+
+ if (pDeviceInstance != NULL)
+ {
+ *pDeviceInstance = deviceInstance;
+ }
+
+ *pDstObject = hDstObject;
+
+done:
+ if (status != NV_OK)
+ {
+ RmUnrefObjExportImport();
+ }
+
+ return status;
+}
+
+void RmFreeObjExportHandle(RmObjExportHandle hObject)
+{
+ if (!RmValidateHandleAgainstInternalHandles(hObject))
+ {
+ NV_PRINTF(LEVEL_ERROR, "Invalid handle to exported object in %s\n",
+ __FUNCTION__);
+ return;
+ }
+
+ RmUnrefObjExportHandle(hObject);
+
+ RmUnrefObjExportImport();
+}
+
+NV_STATUS RmImportObject(NvHandle hDstClient, NvHandle hDstParent,
+ NvHandle *phDstObject, RmObjExportHandle hSrcObject,
+ NvU8 *pObjectType)
+{
+ NV_STATUS status;
+ NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS params;
+ RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+ if (!RmValidateHandleAgainstInternalHandles(hSrcObject))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ if (mapFind(&objExportHandleMap, hSrcObject) == NULL)
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ if (pObjectType != NULL)
+ {
+ params.hObject = hSrcObject;
+ params.mapFlags = 0;
+ params.addrSpaceType = \
+ NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_INVALID;
+
+ status = pRmApi->Control(pRmApi, hObjExportRmClient, hObjExportRmClient,
+ NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE,
+ &params, sizeof(params));
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "GET_ADDR_SPACE_TYPE failed with error code 0x%x in %s\n",
+ status, __FUNCTION__);
+ return status;
+ }
+
+ switch (params.addrSpaceType)
+ {
+ case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_SYSMEM:
+ *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_SYSMEM;
+ break;
+ case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_VIDMEM:
+ *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_VIDMEM;
+ break;
+ case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC:
+ *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC;
+ break;
+#if defined(NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC) && \
+ defined(NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC)
+ case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC:
+ *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC;
+ break;
+#endif
+ default:
+ NV_ASSERT_OK_OR_RETURN(NV_ERR_INVALID_ARGUMENT);
+ }
+ }
+
+ status = pRmApi->DupObject(pRmApi, hDstClient, hDstParent, phDstObject,
+ hObjExportRmClient, hSrcObject,
+ 0 /* flags */);
+
+ if (status != NV_OK)
+ {
+ NV_PRINTF(LEVEL_ERROR,
+ "pRmApi->DupObject(pRmApi, failed with error code 0x%x in %s\n",
+ status, __FUNCTION__);
+ return status;
+ }
+
+ return NV_OK;
+}
+
+NV_STATUS RmGetExportObjectInfo(RmObjExportHandle hSrcObject, NvU32 *deviceInstance)
+{
+ RmObjExportHandleRef *pHandleRef = NULL;
+
+ if (!RmValidateHandleAgainstInternalHandles(hSrcObject))
+ {
+ return NV_ERR_INVALID_ARGUMENT;
+ }
+
+ pHandleRef = mapFind(&objExportHandleMap, hSrcObject);
+ if (pHandleRef == NULL)
+ {
+ return NV_ERR_OBJECT_NOT_FOUND;
+ }
+
+ *deviceInstance = pHandleRef->deviceInstance;
+ return NV_OK;
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/subdevice_ctrl_os_unix.c b/userspace/sdks/nv525/nvalloc/unix/src/subdevice_ctrl_os_unix.c
new file mode 100644
index 0000000..6989c7c
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/subdevice_ctrl_os_unix.c
@@ -0,0 +1,100 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h> // NV device driver interface
+#include <nv-priv.h>
+#include <os/os.h>
+#include <nvos.h>
+#include <osapi.h>
+#include "gpu/gpu.h"
+#include "gpu/gpu_resource.h"
+#include "gpu/subdevice/subdevice.h"
+#include <osfuncs.h>
+#include <diagnostics/journal.h>
+#include "gpu/mem_mgr/mem_desc.h"
+#include "mem_mgr/mem.h"
+#include <nvpcie.h>
+#include <core/locks.h>
+#include "rmapi/rs_utils.h"
+#include "rmapi/client_resource.h"
+#include <class/cl0000.h>
+#include <class/cl90cd.h>
+#include <class/cl0005.h> // NV01_EVENT
+#include <class/cl003e.h> // NV01_MEMORY_SYSTEM
+#include <class/cl844c.h> // G84_PERFBUFFER
+#include <ctrl/ctrl0000/ctrl0000gpu.h>
+#include <ctrl/ctrl0000/ctrl0000unix.h>
+#include <ctrl/ctrl2080/ctrl2080gpu.h>
+#include <ctrl/ctrl2080/ctrl2080unix.h>
+
+/*!
+ * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS
+ * RmControl request. It will check if the GPU video memory will be
+ * persistent during system suspend/resume cycle.
+ *
+ * @param[in] pSubdevice
+ * @param[in,out] pParams
+ *
+ * @return
+ * NV_OK Success
+ */
+NV_STATUS
+subdeviceCtrlCmdOsUnixVidmemPersistenceStatus_IMPL
+(
+ Subdevice *pSubdevice,
+ NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS *pParams
+)
+{
+ OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+ pParams->bVidmemPersistent = !gpuIsVidmemPreservationBrokenBug3172217(pGpu) &&
+ (nv->preserve_vidmem_allocations ||
+ nvp->s0ix_pm_enabled);
+
+ return NV_OK;
+}
+
+/*!
+ * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS
+ * RmControl request. It sets restore TGP flag which is used
+ * to restore TGP limits when client is killed.
+ *
+ * @param[in] pSubdevice
+ * @param[in] pParams
+ *
+ * @return
+ * NV_OK Success
+ */
+NV_STATUS
+subdeviceCtrlCmdOsUnixUpdateTgpStatus_IMPL
+(
+ Subdevice *pSubdevice,
+ NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS *pParams
+)
+{
+ pSubdevice->bUpdateTGP = pParams->bUpdateTGP;
+
+ return NV_OK;
+}
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/unix_console.c b/userspace/sdks/nv525/nvalloc/unix/src/unix_console.c
new file mode 100644
index 0000000..8b1c055
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/unix_console.c
@@ -0,0 +1,118 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <core/locks.h>
+#include <ctrl/ctrl0080/ctrl0080unix.h>
+#include <gpu/device/device.h>
+#include <gpu/gpu.h>
+#include <nv-priv.h>
+#include <nv.h>
+#include <osapi.h>
+#include <gpu/mem_mgr/mem_mgr.h>
+
+NV_STATUS deviceCtrlCmdOsUnixVTGetFBInfo_IMPL
+(
+ Device *pDevice,
+ NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS *pParams
+)
+{
+ OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice);
+ NvBool bContinue = NV_TRUE;
+
+ if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FB) == NV_OK)
+ {
+ // See if the console is on one of the subdevices of this device.
+ portMemSet(pParams, 0, sizeof(*pParams));
+
+ SLI_LOOP_START(SLI_LOOP_FLAGS_NONE)
+
+ MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+ if ((memmgrGetReservedConsoleMemDesc(pGpu, pMemoryManager) != NULL) && bContinue)
+ {
+ NvU64 baseAddr;
+
+ // There should only be one.
+ NV_ASSERT(pParams->width == 0);
+
+ pParams->subDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
+
+ // Console is either mapped to BAR1 or BAR2 + 16 MB
+ os_get_screen_info(&baseAddr, &pParams->width,
+ &pParams->height, &pParams->depth,
+ &pParams->pitch,
+ nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address,
+ nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000);
+
+ if (baseAddr != 0)
+ {
+ bContinue = NV_FALSE;
+ }
+ }
+
+ SLI_LOOP_END
+
+ rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+ else
+ {
+ NV_PRINTF(LEVEL_INFO,"%s: Failed to acquire GPU lock", __FUNCTION__);
+ }
+
+ return NV_OK;
+}
+
+
+void
+dispdeviceFillVgaSavedDisplayState
+(
+ OBJGPU *pGpu,
+ NvU64 vgaAddr,
+ NvU8 vgaMemType,
+ NvBool vgaValid,
+ NvU64 workspaceAddr,
+ NvU8 workspaceMemType,
+ NvBool workspaceValid,
+ NvBool baseValid,
+ NvBool workspaceBaseValid
+)
+{
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ NvBool use_vbios = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv);
+
+ if (use_vbios)
+ {
+ nvp->vga.base.addr = vgaAddr;
+ nvp->vga.base.memTarget = vgaMemType;
+ nvp->vga.base.valid = vgaValid;
+ nvp->vga.baseValid = baseValid;
+
+ nvp->vga.workspaceBase.addr = workspaceAddr;
+ nvp->vga.workspaceBase.memTarget = workspaceMemType;
+ nvp->vga.workspaceBase.valid = workspaceValid;
+ nvp->vga.workspaceBaseValid = workspaceBaseValid;
+ }
+}
+
diff --git a/userspace/sdks/nv525/nvalloc/unix/src/unix_intr.c b/userspace/sdks/nv525/nvalloc/unix/src/unix_intr.c
new file mode 100644
index 0000000..d60a30d
--- /dev/null
+++ b/userspace/sdks/nv525/nvalloc/unix/src/unix_intr.c
@@ -0,0 +1,700 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include <nv-priv.h>
+#include <osapi.h>
+#include <core/thread_state.h>
+#include <core/locks.h>
+#include <gpu/gpu.h>
+#include "kernel/gpu/intr/intr.h"
+#include "gpu/bif/kernel_bif.h"
+#include "gpu/mmu/kern_gmmu.h"
+#include "gpu/disp/kern_disp.h"
+#include <nv_sriov_defines.h>
+#include "objtmr.h"
+
+
+static NvBool osInterruptPending(
+ OBJGPU *pGpu,
+ NvBool *serviced,
+ THREAD_STATE_NODE *pThreadState
+)
+{
+ POBJDISP pDisp;
+ KernelDisplay *pKernelDisplay;
+ NvBool pending, sema_release;
+ THREAD_STATE_NODE threadState;
+ NvU32 gpuMask, gpuInstance;
+ Intr *pIntr = NULL;
+ MC_ENGINE_BITVECTOR intr0Pending;
+ MC_ENGINE_BITVECTOR intr1Pending;
+
+ *serviced = NV_FALSE;
+ pending = NV_FALSE;
+ sema_release = NV_TRUE;
+ OBJGPU *pDeviceLockGpu = pGpu;
+ NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer
+ PORT_MEM_ALLOCATOR *pIsrAllocator;
+
+ //
+ // GPU interrupt servicing ("top half")
+ //
+ // Top-level processing of GPU interrupts is performed using the
+ // steps below; although the code is straight forward, there
+ // are a few points to be aware of:
+ //
+ // 1) The GPUs lock is acquired for two reasons: to allow
+ // looping over GPUs atomically in SLI and to sanity
+ // check the PCI configuration space of any initialized
+ // GPUs. If the acquisition fails, the early return
+ // is acceptable since GPU interrupts are disabled while
+ // the lock is held; note that returning success
+ // in this case could interfere with the processing
+ // of third-party device interrupts if the IRQ is shared.
+ // Due to the above, some interrupts may be reported as
+ // unhandled if invocations of the ISR registered with
+ // the kernel are not serialized. This is bad, but
+ // ignored by currently supported kernels, provided most
+ // interrupts are handled.
+ //
+ // 2) Since acquisition of the lock disables interrupts
+ // on all initialized GPUs, NV_PMC_INTR_EN_0 can not be
+ // relied up on to determine whether interrupts are
+ // expected from a given GPU. The code below is therefore
+ // forced to rely on software state. NV_PMC_INTR_EN_0
+ // is read only as a sanity check to guard against
+ // invalid GPU state (lack of PCI memory access, etc.).
+ //
+ // 3) High priority interrupts (VBLANK, etc.), are serviced in
+ // this function, service of all other interrupts is
+ // deferred until a bottom half. If a bottom half needs
+ // to be scheduled, release of the GPUs lock is
+ // likewise deferred until completion of the bottom half.
+ //
+ // 4) To reduce the risk of starvation, an effort is made to
+ // consolidate processing of interrupts pending on
+ // all GPUs sharing a given IRQ.
+ //
+ // 5) Care is taken to ensure that the consolidated interrupt
+ // processing is performed in the context of a GPU
+ // that has interrupts pending. Else if additional ISR
+ // processing via a bottom-half is required, this
+ // bottom-half ISR might race against the GPU's shut-down
+ // path.
+ //
+
+ pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
+ tlsIsrInit(pIsrAllocator);
+
+ // For SWRL granular locking process the countdown timer interrupt.
+ if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_SWRL_GRANULAR_LOCKING))
+ {
+ threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+
+ gpuMask = gpumgrGetGpuMask(pDeviceLockGpu);
+
+ gpuInstance = 0;
+ while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+ {
+ pIntr = GPU_GET_INTR(pGpu);
+
+ if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))
+ {
+ // If interrupt enable is garbage the GPU is probably in a bad state
+ if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX)
+ {
+ continue;
+ }
+
+ intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
+ POBJTMR pTmr = GPU_GET_TIMER(pGpu);
+ *serviced = tmrServiceSwrlWrapper(pGpu, pTmr, &intr0Pending, &threadState);
+ }
+ }
+
+ threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+ }
+
+ // LOCK: try to acquire GPUs lock
+ if (rmDeviceGpuLocksAcquire(pDeviceLockGpu, GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_ISR) == NV_OK)
+ {
+ threadStateInitISRAndDeferredIntHandler(&threadState,
+ pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR);
+
+ gpuMask = gpumgrGetGpuMask(pDeviceLockGpu);
+
+ gpuInstance = 0;
+ while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+ {
+ pIntr = GPU_GET_INTR(pGpu);
+ pDisp = GPU_GET_DISP(pGpu);
+ pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+
+ if ((pDisp != NULL) && pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+ {
+ }
+ else if ((pIntr != NULL) && INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))
+ {
+ // If interrupt enable is garbage the GPU is probably in a bad state
+ if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX)
+ continue;
+
+ intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
+ if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))
+ {
+ if (pKernelDisplay != NULL)
+ {
+ kdispServiceVblank_HAL(pGpu, pKernelDisplay, 0,
+ (VBLANK_STATE_PROCESS_LOW_LATENCY |
+ VBLANK_STATE_PROCESS_CALLED_FROM_ISR),
+ &threadState);
+ *serviced = NV_TRUE;
+ intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
+ }
+ }
+
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) &&
+ !pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
+ {
+ pIntr = GPU_GET_INTR(pGpu);
+ if (pIntr != NULL)
+ {
+ NvBool bCtxswLog = NV_FALSE;
+ intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
+ intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog);
+ }
+ }
+
+ if (!bitVectorTestAllCleared(&intr0Pending) ||
+ !bitVectorTestAllCleared(&intr1Pending))
+ {
+ pending = NV_TRUE;
+ sema_release = NV_FALSE;
+ }
+ }
+ }
+ threadStateFreeISRAndDeferredIntHandler(&threadState,
+ pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR);
+
+ if (sema_release)
+ {
+ NV_ASSERT(!pending);
+
+ // UNLOCK: release GPUs lock
+ rmDeviceGpuLocksRelease(pDeviceLockGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+ else
+ {
+ rmDeviceGpuLockSetOwner(pDeviceLockGpu, GPUS_LOCK_OWNER_PENDING_DPC_REFRESH);
+ }
+ }
+
+ if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) &&
+ pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
+ {
+ threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+
+ gpuMask = gpumgrGetGpuMask(pDeviceLockGpu);
+ gpuInstance = 0;
+ while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+ {
+ pIntr = GPU_GET_INTR(pGpu);
+ if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)))
+ {
+ NvBool bCtxswLog = NV_FALSE;
+ intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
+ intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog);
+ if (!bitVectorTestAllCleared(&intr1Pending))
+ {
+ intrServiceNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
+ *serviced = NV_TRUE;
+ }
+ }
+ }
+ threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+ }
+
+ tlsIsrDestroy(pIsrAllocator);
+ portMemAllocatorRelease(pIsrAllocator);
+
+ return pending;
+}
+
+NV_STATUS osIsr(
+ OBJGPU *pGpu
+)
+{
+ NV_STATUS status = NV_OK;
+ nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ NvBool pending = NV_FALSE;
+ NvBool serviced = NV_FALSE;
+ Intr *pIntr;
+
+ if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
+ {
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+ {
+ pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */);
+ }
+ else
+ {
+ pIntr = GPU_GET_INTR(pGpu);
+ if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))
+ {
+ KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+ pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */);
+ kbifCheckAndRearmMSI(pGpu, pKernelBif);
+ }
+ }
+ }
+
+ if (!pending && (IS_VIRTUAL(pGpu) || !serviced))
+ status = NV_ERR_NO_INTR_PENDING;
+ else if (pending)
+ status = NV_WARN_MORE_PROCESSING_REQUIRED;
+
+ return status;
+}
+
+/*
+ * Helper function to determine when the RM SEMA/GPUS LOCK should toggle
+ * interrupts. Based on the state of the GPU - we must add cases here as we
+ * discover them.
+ *
+ * Noteworthy special cases:
+ *
+ * - Suspend/resume: the GPU could still be suspended and not accessible
+ * on the bus, while passive-level threads need to grab the GPUs
+ * lock, or other GPUs are being resumed and triggering interrupts.
+ *
+ * - SLI state transitions: interrupts are disabled manually prior to
+ * removing GPUs from the lock mask leading up to SLI link/unlink
+ * operations on UNIX, but since the GPUs lock is not held by design in
+ * these paths, it needs to be ensured that GPUs lock acquisitions
+ * occurring aynchronously do not re-enable interrupts on any of the
+ * GPUs undergoing the SLI state transition.
+ *
+ * @param[in] pGpu OBJGPU pointer
+ *
+ * @return NV_TRUE if the RM SEMA/GPUS LOCK should toggle interrupts, NV_FALSE
+ * otherwise.
+ */
+NvBool osLockShouldToggleInterrupts(OBJGPU *pGpu)
+{
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+ return NV_TRUE;
+
+ return (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH) &&
+ gpuIsStateLoaded(pGpu) &&
+ !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SLI_LINK_CODEPATH));
+}
+
+void osEnableInterrupts(OBJGPU *pGpu)
+{
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+ {
+ // enable irq through os call
+ nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_TRUE);
+ return;
+ }
+ else
+ {
+ Intr *pIntr = GPU_GET_INTR(pGpu);
+ NvU32 intrEn;
+
+ if (!pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING))
+ NV_ASSERT(intrGetIntrEnFromHw_HAL(pGpu, pIntr, NULL) == INTERRUPT_TYPE_DISABLED);
+
+ intrEn = intrGetIntrEn(pIntr);
+ intrSetIntrEnInHw_HAL(pGpu, pIntr, intrEn, NULL);
+
+ if (pIntr != NULL)
+ {
+ intrSetStall_HAL(pGpu, pIntr, intrEn, NULL);
+ }
+
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED))
+ {
+ if (pIntr != NULL)
+ {
+ intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL);
+ }
+ }
+
+ }
+}
+
+void osDisableInterrupts(
+ OBJGPU *pGpu,
+ NvBool bIsr
+)
+{
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+ {
+ // disable irq through os call
+ nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_FALSE);
+ return;
+ }
+ else
+ {
+ Intr *pIntr = GPU_GET_INTR(pGpu);
+ NvU32 new_intr_en_0 = INTERRUPT_TYPE_DISABLED;
+
+ intrSetIntrEnInHw_HAL(pGpu, pIntr, new_intr_en_0, NULL);
+
+ if (pIntr != NULL)
+ {
+ intrSetStall_HAL(pGpu, pIntr, new_intr_en_0, NULL);
+ }
+
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED))
+ {
+ if (pIntr != NULL)
+ {
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
+ {
+ intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL);
+ }
+ else
+ {
+ intrRestoreNonStall_HAL(pGpu, pIntr, new_intr_en_0, NULL);
+ }
+ }
+ }
+ }
+}
+
+static void RmIsrBottomHalf(
+ nv_state_t *pNv
+)
+{
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+ THREAD_STATE_NODE threadState;
+ OS_THREAD_HANDLE threadId;
+ NvU32 gpuMask, gpuInstance;
+ OBJGPU *pDeviceLockGpu = pGpu;
+ Intr *pIntr = NULL;
+ POBJDISP pDisp = NULL;
+ NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer
+ PORT_MEM_ALLOCATOR *pIsrAllocator;
+
+ pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
+ tlsIsrInit(pIsrAllocator);
+
+ //
+ // The owning thread changes as the ISR acquires the GPUs lock,
+ // but the bottom half releases it. Refresh the ThreadId owner to be
+ // correct here for the bottom half context.
+ //
+ osGetCurrentThread(&threadId);
+ rmDeviceGpuLockSetOwner(pDeviceLockGpu, threadId);
+
+ gpuMask = gpumgrGetGpuMask(pGpu);
+
+ gpuInstance = 0;
+ while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+ {
+
+ threadStateInitISRAndDeferredIntHandler(&threadState,
+ pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER);
+
+ pIntr = GPU_GET_INTR(pGpu);
+ pDisp = GPU_GET_DISP(pGpu);
+
+ //
+ // Call disp service incase of SOC Display,
+ // TODO : with multi interrupt handling based on irq aux interrupts are serviced by dpAuxService
+ // See JIRA task TDS-4253.
+ //
+ if ((pDisp != NULL) && pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+ {
+ }
+ else if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)))
+ {
+ intrServiceStall_HAL(pGpu, pIntr);
+
+ if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) &&
+ !pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
+ {
+ MC_ENGINE_BITVECTOR intrPending;
+ intrServiceNonStall_HAL(pGpu, pIntr, &intrPending, &threadState);
+ }
+ }
+
+ threadStateFreeISRAndDeferredIntHandler(&threadState,
+ pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER);
+ }
+
+ // UNLOCK: release GPUs lock
+ rmDeviceGpuLocksRelease(pDeviceLockGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+
+ tlsIsrDestroy(pIsrAllocator);
+ portMemAllocatorRelease(pIsrAllocator);
+}
+
+static void RmIsrBottomHalfUnlocked(
+ nv_state_t *pNv
+)
+{
+ OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+ Intr *pIntr;
+ THREAD_STATE_NODE threadState;
+
+ // In the GSP client scenario, the fatal fault interrupt is not shared
+ // by UVM and CPU-RM. Instead, it is handled entirely by GSP-RM. We
+ // therefore do not expect this function to be called. But if it is, bail
+ // without attempting to service interrupts.
+ if (IS_GSP_CLIENT(pGpu))
+ {
+ return;
+ }
+
+ // Grab GPU lock here as this kthread-item was enqueued without grabbing GPU lock
+ if (rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DPC) == NV_OK)
+ {
+ if (FULL_GPU_SANITY_CHECK(pGpu))
+ {
+ pIntr = GPU_GET_INTR(pGpu);
+
+ threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+ if (intrGetIntrEn(pIntr) != INTERRUPT_TYPE_DISABLED)
+ {
+ MC_ENGINE_BITVECTOR intrPending;
+ intrGetPendingStall_HAL(pGpu, pIntr, &intrPending, &threadState);
+ intrServiceNonStallBottomHalf(pGpu, pIntr, &intrPending, &threadState);
+ }
+
+ threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+ }
+
+ rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+ }
+}
+
+NvBool NV_API_CALL rm_isr(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvU32 *NeedBottomHalf
+)
+{
+ NV_STATUS status;
+ nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+ OBJGPU *pGpu;
+ NvBool retval;
+ void *fp;
+
+ if ((nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) == 0)
+ {
+ return NV_FALSE;
+ }
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ if (pGpu == NULL)
+ {
+ return NV_FALSE;
+ }
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ // call actual isr function here
+ status = isrWrapper(pGpu->testIntr, pGpu);
+
+ switch (status)
+ {
+ case NV_OK:
+ *NeedBottomHalf = NV_FALSE;
+ retval = NV_TRUE;
+ break;
+ case NV_WARN_MORE_PROCESSING_REQUIRED:
+ *NeedBottomHalf = NV_TRUE;
+ retval = NV_TRUE;
+ break;
+ case NV_ERR_NO_INTR_PENDING:
+ default:
+ *NeedBottomHalf = NV_FALSE;
+ retval = NV_FALSE;
+ break;
+ }
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return retval;
+}
+
+void NV_API_CALL rm_isr_bh(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ RmIsrBottomHalf(pNv);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_isr_bh_unlocked(
+ nvidia_stack_t *sp,
+ nv_state_t *pNv
+)
+{
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ RmIsrBottomHalfUnlocked(pNv);
+
+ NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvU32 *faultsCopied
+)
+{
+ NV_STATUS status = NV_OK;
+ OBJGPU *pGpu;
+ void *fp;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ if (pGpu == NULL || faultsCopied == NULL)
+ {
+ status = NV_ERR_OBJECT_NOT_FOUND;
+ goto done;
+ }
+
+ // Non-replayable faults are copied to the client shadow buffer by GSP-RM.
+ if (IS_GSP_CLIENT(pGpu))
+ {
+ status = NV_ERR_NOT_SUPPORTED;
+ goto done;
+ }
+
+done:
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return status;
+}
+
+//
+// Use this call when MMU faults needs to be copied
+// outisde of RM lock.
+//
+NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults_unlocked(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvU32 *faultsCopied
+)
+{
+ OBJGPU *pGpu;
+ void *fp;
+ NV_STATUS status = NV_OK;
+
+ NV_ENTER_RM_RUNTIME(sp,fp);
+
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+ if (pGpu == NULL || faultsCopied == NULL)
+ {
+ status = NV_ERR_OBJECT_NOT_FOUND;
+ goto done;
+ }
+
+ // Non-replayable faults are copied to the client shadow buffer by GSP-RM.
+ if (IS_GSP_CLIENT(pGpu))
+ {
+ status = NV_ERR_NOT_SUPPORTED;
+ goto done;
+ }
+
+done:
+ NV_EXIT_RM_RUNTIME(sp,fp);
+
+ return status;
+}
+
+//
+// Wrapper to handle calls to copy mmu faults
+//
+NV_STATUS rm_gpu_handle_mmu_faults(
+ nvidia_stack_t *sp,
+ nv_state_t *nv,
+ NvU32 *faultsCopied
+)
+{
+ NvU32 status = NV_OK;
+
+ *faultsCopied = 0;
+
+ OBJGPU *pGpu;
+ pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+ if (pGpu == NULL)
+ {
+ return NV_ERR_OBJECT_NOT_FOUND;
+ }
+
+ if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && !IS_VIRTUAL(pGpu))
+ {
+ THREAD_STATE_NODE threadState;
+
+ KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
+ Intr *pIntr = GPU_GET_INTR(pGpu);
+
+ NvU32 hw_put = 0;
+ NvU32 hw_get = 0;
+
+ threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+
+ kgmmuReadFaultBufferPutPtr_HAL(pGpu, pKernelGmmu, NON_REPLAYABLE_FAULT_BUFFER,
+ &hw_put, &threadState);
+
+ kgmmuReadFaultBufferGetPtr_HAL(pGpu, pKernelGmmu, NON_REPLAYABLE_FAULT_BUFFER,
+ &hw_get, &threadState);
+
+ if(hw_get != hw_put)
+ {
+ // We have to clear the top level interrupt bit here since otherwise
+ // the bottom half will attempt to service the interrupt on the CPU
+ // side before GSP recieves the notification and services it
+ kgmmuClearNonReplayableFaultIntr(pGpu, pKernelGmmu, &threadState);
+ status = intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_NON_REPLAYABLE_FAULT_HANDLE);
+
+ }
+
+ threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+ }
+ else
+ {
+ status = rm_gpu_copy_mmu_faults_unlocked(sp, nv, faultsCopied);
+ }
+ return status;
+}
+
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-31 04:41:30 +0000