Initial commit: Open-IOV vGPU-IO complete stack

- Firmware decompilation: GSP 570.144W (28.5MB ELF, RV64IMAC), 132 decompiled C files
- Kernel modules: nvidia.ko (34MB, OpenRM), open-iov-vfio.ko (2.8MB, VFIO mdev), gsp-vgpu.ko (1.8MB, guest driver)
- Userspace: libiov.so/libvgpu.so (76KB), example_vgpu (28KB), rpc_emulator.c
- PCI config space emulation (256B with PM, MSI, MSI-X, PCIe caps)
- 12 RPC types fully mapped from firmware decompilation
- BAR trap, WPR, scheduling, notification, migration emulation
- Helper scripts: load_vgpu_stack, create_mdev, launch_qemu_vgpu, test_vgpu_stack, setup_guest_driver, unload_vgpu_stack
- Architecture documentation
- Wiki: 10 technical documents
- Git LFS for large binaries (.ko, .bin, .iso, .img, .o, ghidra repos)
- nvidia-open-gpu-kernel-modules as submodule

21 files changed, 27467 insertions, 0 deletions

diff --git a/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_cpuid.c b/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_cpuid.c
new file mode 100644
index 0000000..fe6dfa9
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_cpuid.c
@@ -0,0 +1,51 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <os/os.h>
+
+/* inline assembler routines for UNIX platforms */
+
+#if defined(NVCPU_X86_64)
+
+int osNv_cpuid(
+    int    op,
+    int    subop,
+    NvU32  *eax,
+    NvU32  *ebx,
+    NvU32  *ecx,
+    NvU32  *edx
+)
+{
+    asm volatile ("   cpuid                     \n"
+                  : "=a" (*eax),
+                    "=b" (*ebx),
+                    "=c" (*ecx),
+                    "=d" (*edx)
+                  : "a" (op),
+                    "c" (subop)
+                  : "cc");
+
+    return 1;
+}
+
+#endif
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_rdcr.c b/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_rdcr.c
new file mode 100644
index 0000000..6115aa4
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/asm/x86/nv_rdcr.c
@@ -0,0 +1,37 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <os/os.h>
+
+/* inline assembler routines for UNIX platforms */
+
+#if defined(NVCPU_X86_64)
+
+NvU32 osNv_rdcr4(void)
+{
+    NvU64 val;
+    asm volatile ("movq %%cr4,%0" : "=r" (val));
+    return (NvU32)val;
+}
+
+#endif
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/dynamic-power.c b/userspace/sdks/nv570/nvalloc/unix/src/dynamic-power.c
new file mode 100644
index 0000000..fa61d94
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/dynamic-power.c
@@ -0,0 +1,3185 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <rmconfig.h>
+
+/*
+ * NOTE:
+ * The driver disables dynamic power management by effectively setting
+ * nv_dynamic_power_t::mode to NV_DYNAMIC_PM_NEVER at two places:
+ * 1) During module probe, when the _PRx methods are not supported
+ * by the SBIOS or when the kernel does not support runtime PM.
+ * 2) During RM Init, when the GPU is not Turing+ or when the system is not
+ * a laptop or when the VBIOS does not support RTD3/GC6.
+ *
+ * Thus, after RMInit, if the nv_dynamic_power_t::mode is still
+ * NV_DYNAMIC_PM_FINE, then we know for sure that the fine grained power
+ * control is active and running. In this case, the runtime D3 status is
+ * shown as "Enabled (fine-grained)".
+ *
+ * If the nv_dynamic_power_t::mode is NV_DYNAMIC_PM_NEVER, we have to
+ * distinguish between four cases:
+ * 1) The driver disabled dynamic power management due to lack of kernel/SBIOS
+ * support.
+ * 2) The driver disabled dynamic power management because the GPU does not
+ * support it or the system is not a laptop.
+ * 3) The user has explicitly disabled the feature.
+ * 4) Configuration is not supported by default.
+ *
+ * In order to differentiate between these four cases,
+ * we use the value of regkey NVreg_DynamicPowerManagement. If this value is
+ * set to 0, then we know that the user has explicitly disabled the
+ * feature. In this case we show the runtime D3 status as "Disabled". This
+ * handles case (3) above.
+ *
+ * For case (4), we look at the combination of nv_dynamic_power_t::mode and
+ * NVreg_DynamicPowerManagement value. If mode is never and regkey value is
+ * default, then status is shown as "Disabled by default".
+ *
+ * For remaining cases, (case (1) and (2)) we show status as "Not Supported".
+ *
+ * When the regkey is set to 0 and the driver disables the feature due to lack
+ * of some support, then precedence will be given to regkey value. Thus, when
+ * both "Not supported" and "Disabled" are possible values, precedence will be
+ * given to "Disabled" over "Not Supported".
+ *
+ * For coarse-grain power control, we show runtime D3 status as
+ * "Enabled (coarse-grained)".
+ *
+ * The runtime D3 status is shown as "?" when RM has not been initialized.
+ * This is because we do not have full knowledge regarding driver triggered
+ * disablement until RM is initialized.
+ */
+
+#include <nvlog_inc.h>
+#include <nv.h>
+#include <nv-priv.h>
+#include <nv-reg.h>
+#include <nv_ref.h>
+
+#include <osapi.h>
+
+#include <gpu/mem_mgr/mem_mgr.h>
+#include <gpu/kern_gpu_power.h>
+#include <gpu_mgr/gpu_mgr.h>
+#include <core/locks.h>
+#include <pex.h>
+#include <acpidsmguids.h>
+#include "kernel/gpu/intr/intr.h"
+
+#include <gpu/mem_sys/kern_mem_sys.h>
+#include <gpu/subdevice/subdevice.h>
+#include <ctrl/ctrl0080/ctrl0080gpu.h>
+#include <ctrl/ctrl2080/ctrl2080unix.h>
+#include <gpu/timer/objtmr.h>
+
+
+// mask holding dynamic power supported GPUs
+static NvU32 dynamicPowerSupportGpuMask = 0;
+
+//
+// Schedule timer based callback, to check for the complete GPU Idleness.
+// Windows has idle time from 70msec to 10sec, we opted for present duration
+// considering windows limit. Duration is not much aggressive or slow, hence
+// less thrashing.
+//
+#define GC6_PRECONDITION_CHECK_TIME    ((NvU64)5 * 1000 * 1000 * 1000)
+
+//
+// iGPU needs a more aggressive timeout value of precondition
+// check to quickly engage the Rail-Gating state on an embedded
+// system.
+// Considering some basic scenarios like GPU monitoring which could
+// poll the GPU every 1 second, Rail-Gating cycle should not be triggered
+// in between the pollings. So the timeout value is set to 500 ms.
+// and the shortest duration to trigger Deferred-Idle would be
+// IGPU_RG_PRECONDITION_CHECK_TIME + IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME
+//
+#define IGPU_RG_PRECONDITION_CHECK_TIME    ((NvU64)500 * 1000 * 1000)
+
+//
+// Timeout needed for back to back GC6 cycles.
+// Timeout is kept same as the timeout selected for GC6 precondition check.
+// There are cases where GPU is in GC6 and then kernel wakes GPU out of GC6
+// as part of say accessing pci tree through lspci and then again ask driver
+// to put GPU in GC6 state after access to device info is done.
+// Below are the reasons for having timeout limit for back to back GC6 cycle.
+// 1> Less chip life cycle.
+// 2> P-state is not reached to GC6 supported P-state, in some cycles.
+// P-state case returns error to kernel, resulting in corrupted sysfs entry
+// and then kernel never calls driver to put device in low power state.
+//
+#define GC6_CYCLE_IDLE_HOLDOFF_CHECK_TIME    GC6_PRECONDITION_CHECK_TIME
+
+//
+// Once GPU is found to be idle, driver will schedule another callback of
+// smaller duration. Driver needs to be sure that methods that are present
+// in host pipeline are flushed to the respective engines and engines become
+// idle upon consumption.
+//
+#define GC6_BAR1_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME (200 * 1000 * 1000)
+
+//
+// The purpose of this timeout value is similar to
+// GC6_BAR1_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME.
+// The difference between iGPU Rail-Gating and GC6 is that iGPU RG has a more
+// aggressive timeout value of precondition check to quickly engage the
+// Rail-Gating state on an embedded system. Meanwhile, the overall duration
+// of Deferred-Idle should be larger than 1 second considering some polling
+// scenarios which happens every 1 second.
+// The value here is extended to 700 ms so the overall Deferred-Idle duration.
+// could be keep within within the range:
+// - Lower Bound: IGPU_RG_PRECONDITION_CHECK_TIME + IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME
+// - Upper Bound: IGPU_RG_PRECONDITION_CHECK_TIME * 2 + IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME
+//
+#define IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME (700 * 1000 * 1000)
+
+//
+// Cap Maximum FB allocation size for GCOFF. If regkey value is greater
+// than this value then it will be capped to this value.
+//
+#define GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB      1024
+
+//
+// Safe/Fallback value of PEX reset delay for GCOFF/GPUOFF.
+// Used when programming upstream port as per sku specific PEX reset delay from vbios fails
+//
+#define DEFAULT_GCOFF_PEXRST_DELAY 0
+
+static void RmScheduleCallbackForIdlePreConditions(OBJGPU *);
+static void RmScheduleCallbackForIdlePreConditionsUnderGpuLock(OBJGPU *);
+static void RmScheduleCallbackToIndicateIdle(OBJGPU *);
+static NvBool RmCheckForGcxSupportOnCurrentState(OBJGPU *);
+static void RmScheduleCallbackToRemoveIdleHoldoff(OBJGPU *);
+static void RmQueueIdleSustainedWorkitem(OBJGPU *);
+
+/*!
+ * @brief Wrapper that checks lock order for the dynamic power mutex.  Locking
+ * order dictates that the GPU lock must not be taken before taking the dynamic
+ * power mutex.
+ *
+ * The GPU lock must not be held when calling this function.
+ *
+ * @param[in]   nvp     nv_priv_t pointer.
+ */
+static void acquireDynamicPowerMutex(nv_priv_t *nvp)
+{
+    /*
+     * Note that this may be called before nvp->pGpu has been initialized, for
+     * example at the very beginning of nv_start_device.
+     */
+    NV_ASSERT_CHECKED(!nvp->pGpu || !rmDeviceGpuLockIsOwner(nvp->pGpu->gpuInstance));
+
+    portSyncMutexAcquire(nvp->dynamic_power.mutex);
+}
+
+/*!
+ * @brief Wrapper to release the mutex, just for consistency with
+ * acquireDynamicPowerMutex() above.
+ *
+ * @param[in]   nvp     nv_priv_t pointer.
+ */
+static void releaseDynamicPowerMutex(nv_priv_t *nvp)
+{
+    portSyncMutexRelease(nvp->dynamic_power.mutex);
+}
+
+/*!
+ * @brief: Helper function to get a string for the given state.
+ *
+ * @param[in]   state   Dynamic power state.
+ *
+ * @return      String for the given state.
+ */
+const char *nv_dynamic_power_state_string(
+    nv_dynamic_power_state_t state
+)
+{
+    switch (state)
+    {
+#define DPCASE(_case) \
+    case NV_DYNAMIC_POWER_STATE_ ## _case: return # _case;
+
+    DPCASE(IN_USE);
+    DPCASE(IDLE_INSTANT);
+    DPCASE(IDLE_SUSTAINED);
+    DPCASE(IDLE_INDICATED);
+    DPCASE(UNKNOWN);
+#undef DPCASE
+    }
+    NV_ASSERT(0);
+    return "UNEXPECTED";
+}
+
+/*!
+ * @brief: Helper function to transition the GPU represented by the 'nv'
+ * pointer from the power state 'old_state' to the power state 'new_state'.
+ *
+ * An atomic compare-and-swap is used; the function returns true if the
+ * operation was successful.  If the function returns false, the state is
+ * unchanged.
+ *
+ * @param[in]   nv          nv_state_t pointer.
+ * @param[in]   old_state   Previous (current) state.
+ * @param[in]   new_state   New (requested) state.
+ *
+ * @return      TRUE if the operation was successful; FALSE otherwise.
+ */
+NvBool nv_dynamic_power_state_transition(
+    nv_state_t *nv,
+    nv_dynamic_power_state_t old_state,
+    nv_dynamic_power_state_t new_state
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+    NV_ASSERT(old_state != new_state);
+
+    ct_assert(sizeof(nv_dynamic_power_state_t) == sizeof(NvS32));
+    NvBool ret = portAtomicCompareAndSwapS32((PORT_ATOMIC NvS32*)&nvp->dynamic_power.state,
+                                             (NvS32)new_state, (NvS32)old_state);
+
+    if (ret)
+    {
+        NV_PRINTF(LEVEL_INFO, "%s: state transition %s -> %s\n",
+                  __FUNCTION__,
+                  nv_dynamic_power_state_string(old_state),
+                  nv_dynamic_power_state_string(new_state));
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_INFO, "%s: FAILED state transition %s -> %s\n",
+                  __FUNCTION__,
+                  nv_dynamic_power_state_string(old_state),
+                  nv_dynamic_power_state_string(new_state));
+    }
+
+    return ret;
+
+}
+
+/*!
+ * @brief: Check if the GPU hardware appears to be idle.
+ *
+ * Assumes the GPU lock is held.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ *
+ * @return      TRUE if the GPU appears to be currently idle; FALSE otherwise.
+ */
+static NvBool RmCanEnterGcxUnderGpuLock(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t    *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t     *nvp = NV_GET_NV_PRIV(nv);
+
+    NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance));
+
+    /*
+     * If GC6 cannot be achieved (either GC6 is unsupported or the upstream port is not configured),
+     * Check for GCOFF prerequisites. Exclude the check for IGPU which has RG support.
+     */
+    if ((!pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED) ||
+         !nvp->gc6_upstream_port_configured) &&
+        (!pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED)))
+    {
+        NvU64          usedFbSize     = 0;
+        MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+
+        /*
+         * In order to enter GCOFF, the FB memory threshold should be less than gcoff_max_fb_size
+         * and clients of RM should not hold the refcount to prevent GCOFF.
+         */
+        if (!((memmgrGetUsedRamSize(pGpu, pMemoryManager, &usedFbSize) == NV_OK) &&
+              (usedFbSize <= nvp->dynamic_power.gcoff_max_fb_size) &&
+              (nvp->dynamic_power.clients_gcoff_disallow_refcount == 0)))
+        {
+            return NV_FALSE;
+        }
+    }
+
+    // Check the instantaneous engine level idleness.
+    return RmCheckForGcxSupportOnCurrentState(pGpu);
+}
+
+/*!
+ * @brief: Check the feasibility of GPU engaging in a GCx cycle.
+ *
+ * Takes the GPU lock.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ *
+ * @return      TRUE if the GPU appears to be currently idle; FALSE otherwise.
+ */
+static NvBool RmCanEnterGcx(
+    OBJGPU *pGpu
+)
+{
+    NvBool      result = NV_FALSE;
+    NV_STATUS   status;
+    GPU_MASK    gpuMask;
+
+    // LOCK: acquire per device lock
+    status = rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
+                                   GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER,
+                                   &gpuMask);
+    if (status == NV_OK)
+    {
+        result = RmCanEnterGcxUnderGpuLock(pGpu);
+        // UNLOCK: release per device lock
+        rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+    }
+
+    return result;
+}
+
+/*!
+ * @brief: Force the GPU to be "not idle", if it had previously been indicated
+ * idle.  This is triggered by GPU usage (i.e., writing through a revoked
+ * mapping), not tracked through the refcount maintained through
+ * os_{ref,unref}_dynamic_power.  So, if we'd previously indicated the GPU was
+ * idle, we transition to the IDLE_INSTANT state.
+ *
+ * @param[in]   gpuInstance     GPU instance ID.
+ * @param[in]   pArgs           Unused callback closure.
+ */
+static void RmForceGpuNotIdle(
+    NvU32 gpuInstance,
+    void *pArgs
+)
+{
+    OBJGPU *pGpu = gpumgrGetGpu(gpuInstance);
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    nv_dynamic_power_state_t old_state;
+    NvBool ret;
+
+    acquireDynamicPowerMutex(nvp);
+
+    old_state = nvp->dynamic_power.state;
+
+    switch (old_state)
+    {
+    case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED:
+        nv_indicate_not_idle(nv);
+        NV_ASSERT(nvp->dynamic_power.deferred_idle_enabled);
+        RmScheduleCallbackForIdlePreConditions(pGpu);
+        /* fallthrough */
+    case NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED:
+        ret = nv_dynamic_power_state_transition(nv, old_state, NV_DYNAMIC_POWER_STATE_IDLE_INSTANT);
+        NV_ASSERT(ret);
+        break;
+    case NV_DYNAMIC_POWER_STATE_IDLE_INSTANT:
+    case NV_DYNAMIC_POWER_STATE_IN_USE:
+        break;
+    case NV_DYNAMIC_POWER_STATE_UNKNOWN:
+        NV_ASSERT(0);
+        break;
+    }
+
+    /*
+     * Now that the GPU is guaranteed to not be powered off, we can resume
+     * servicing mmap requests as usual.
+     */
+    nv_acquire_mmap_lock(nv);
+    nv_set_safe_to_mmap_locked(nv, NV_TRUE);
+    nv_release_mmap_lock(nv);
+
+    releaseDynamicPowerMutex(nvp);
+}
+
+/*!
+ * @brief: Work item to actually indicate the GPU is idle.  This rechecks the
+ * preconditions one last time and verifies nothing has attempted to write
+ * through any mappings which were revoked.
+ *
+ * @param[in]   gpuInstance     GPU instance ID.
+ * @param[in]   pArgs           Unused callback closure.
+ */
+static void RmIndicateIdle(
+    NvU32 gpuInstance,
+    void *pArgs
+)
+{
+    OBJGPU *pGpu = gpumgrGetGpu(gpuInstance);
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+    acquireDynamicPowerMutex(nvp);
+
+    nv_acquire_mmap_lock(nv);
+
+    if (nv_get_all_mappings_revoked_locked(nv) &&
+        nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED &&
+        RmCanEnterGcx(pGpu))
+    {
+        nv_set_safe_to_mmap_locked(nv, NV_FALSE);
+        nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED,
+                                              NV_DYNAMIC_POWER_STATE_IDLE_INDICATED);
+        nv_indicate_idle(nv);
+    }
+    else
+    {
+        RmScheduleCallbackForIdlePreConditions(pGpu);
+    }
+
+    nv_release_mmap_lock(nv);
+
+    releaseDynamicPowerMutex(nvp);
+}
+
+/*!
+ * @brief: Helper function to trigger the GPU to be forced indicated
+ * "non-idle", which powers it on if it had been previously powered off, and
+ * allows mmap processing to continue.
+ *
+ * Triggered when a revoked mapping is accessed.
+ *
+ * @param[in]   sp  nvidia_stack_t pointer.
+ * @param[in]   nv  nv_state_t pointer.
+ */
+NV_STATUS NV_API_CALL rm_schedule_gpu_wakeup(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    void *fp;
+    NV_STATUS ret;
+    OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    ret = osQueueWorkItem(pGpu,
+                          RmForceGpuNotIdle,
+                          NULL,
+                          (OsQueueWorkItemFlags){0});
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    return ret;
+}
+
+/*!
+ * @brief Function to increment/decrement refcount for GPU driving console.
+ *
+ * @params[in]  nv         nv_state_t pointer.
+ * @params[in]  bIncrease  If true, increase refcount else decrease.
+ */
+void
+RmUpdateGc6ConsoleRefCount
+(
+    nv_state_t *nv,
+    NvBool      bIncrease
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+    if (nv->console_device == NV_FALSE)
+    {
+        return;
+    }
+
+    acquireDynamicPowerMutex(nvp);
+
+    if (bIncrease)
+    {
+        /*
+         * Here refcount value should atleast be 2 due to below:
+         * 1> Refcount increased by NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT.
+         * 2> Refcount increased by the Rm entry points for current thread.
+         */
+        if (nvp->dynamic_power.refcount <= 1)
+        {
+            NV_PRINTF(LEVEL_INFO,
+                  "%s: Unexpected dynamic power refcount value\n", __FUNCTION__);
+        }
+        else
+        {
+            nvp->dynamic_power.refcount++;
+        }
+    }
+    else
+    {
+        /*
+         * Here refcount value should atleast be 3 due to below:
+         * 1> Refcount set by RM at the probe time for console device.
+         * 2> Refcount increased by NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT.
+         * 3> Refcount increased by the Rm entry points for current thread.
+         */
+        if (nvp->dynamic_power.refcount <= 2)
+        {
+            NV_PRINTF(LEVEL_INFO,
+                  "%s: Unexpected dynamic power refcount value\n", __FUNCTION__);
+        }
+        else
+        {
+            nvp->dynamic_power.refcount--;
+        }
+    }
+
+    releaseDynamicPowerMutex(nvp);
+}
+
+/*!
+ * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT RmControl
+ * request.
+ *
+ * @param[in]     pSubdevice
+ * @param[in,out] pParams
+ *
+ * @return
+ *      NV_OK                       Success
+ *      NV_ERR_INVALID_ARGUMENT     Invalid pParams->action
+ */
+NV_STATUS
+subdeviceCtrlCmdOsUnixGc6BlockerRefCnt_IMPL
+(
+    Subdevice *pSubdevice,
+    NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS *pParams
+)
+{
+    OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
+
+    /* TODO: keep track of inc/dec on a per-client basis, clean up after
+     * clients which are unbalanced. */
+    if (pParams->action == NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC)
+    {
+        return osRefGpuAccessNeeded(pGpu->pOsGpuInfo);
+    }
+    else if (pParams->action == NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC)
+    {
+        osUnrefGpuAccessNeeded(pGpu->pOsGpuInfo);
+    }
+    else
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    return NV_OK;
+}
+
+/*!
+ * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER RmControl
+ *        request. It will perform dynamic power management for GPU
+ *        HDA contoller.
+ *
+ * @param[in]     pSubdevice
+ * @param[in,out] pParams
+ *
+ * @return
+ *      NV_OK   Success
+ */
+NV_STATUS
+subdeviceCtrlCmdOsUnixAudioDynamicPower_IMPL
+(
+    Subdevice *pSubdevice,
+    NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS *pParams
+)
+{
+    OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
+
+    /*
+     * Ideally, we need to honor pParams->bEnter. If it is true, then send the
+     * request for suspending the HDA controller and if is false, then send the
+     * request for waking up the HDA controller. Currently, for unix based OS,
+     * we don't have any proper way to do this and it is implemented with
+     * workaround. This workaround will wake up the HDA controller for few
+     * seconds and then bring it back to suspended state. We are using this
+     * workaround for both the cases (bEnter is true or false). It will help
+     * in keeping HDA controller active for the duration, when GPU is
+     * processing HDA controller commands (like writing ELD data).
+     */
+    nv_audio_dynamic_power(NV_GET_NV_STATE(pGpu));
+
+    return NV_OK;
+}
+
+/*!
+ * @brief: Function to indicate if GC6/GC-OFF is supported
+ * or not on the SKU.
+ *
+ * @param[in]   pGpu           OBJGPU pointer.
+ * @param[in]   bGcxTypeGc6    If true, returns string indicating GC6 support
+ *                             otherwise returns GC-OFF support.
+ *
+ * @return      String indicating GC6/GC-OFF support status.
+ */
+static const char* RmGetGpuGcxSupport(
+    OBJGPU *pGpu,
+    NvBool  bGcxTypeGC6
+)
+{
+    if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_UNIX_DYNAMIC_POWER_SUPPORTED))
+    {
+        return "Not Supported";
+    }
+
+    if (bGcxTypeGC6)
+    {
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED))
+        {
+            return "Supported";
+        }
+    }
+    else
+    {
+        if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GCOFF_SUPPORTED)) ||
+            (pGpu->getProperty(pGpu, PDB_PROP_GPU_LEGACY_GCOFF_SUPPORTED)))
+        {
+            return "Supported";
+        }
+    }
+
+    return "Not Supported";
+}
+
+/*!
+ * @brief Function to increment/decrement global Gcoff disallow refcount.
+ *
+ * @params[in]  pOsGpuInfo        OS specific GPU information pointer.
+ * @params[in]  bIsGcoffDisallow  If true, increase refcount else decrease.
+ */
+void osClientGcoffDisallowRefcount(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvBool       bIsGcoffDisallow
+)
+{
+    nv_priv_t    *nvp = NV_GET_NV_PRIV(pOsGpuInfo);
+
+    if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_NEVER)
+    {
+        return;
+    }
+
+    acquireDynamicPowerMutex(nvp);
+
+    if (bIsGcoffDisallow)
+    {
+        nvp->dynamic_power.clients_gcoff_disallow_refcount++;
+    }
+    else
+    {
+        nvp->dynamic_power.clients_gcoff_disallow_refcount--;
+    }
+
+    releaseDynamicPowerMutex(nvp);
+}
+
+/*!
+ * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF RmControl
+ * request.
+ *
+ * @param[in]     pSubdevice
+ * @param[in,out] pParams
+ *
+ * @return
+ *      NV_OK                           Success
+ *      NV_ERR_INVALID_ARGUMENT         Invalid pParams->action
+ *      NV_ERR_INVALID_OBJECT_HANDLE    Invalid Object handle
+ *      NV_ERR_OBJECT_NOT_FOUND         Object not found
+ */
+NV_STATUS
+subdeviceCtrlCmdOsUnixAllowDisallowGcoff_IMPL
+(
+    Subdevice *pSubdevice,
+    NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS *pParams
+)
+{
+    NV_STATUS status;
+    OBJGPU   *pGpu = GPU_RES_GET_GPU(pSubdevice);
+
+    if (pParams->action == NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW)
+    {
+        if (pSubdevice->bGcoffDisallowed)
+        {
+            pSubdevice->bGcoffDisallowed = NV_FALSE;
+            osClientGcoffDisallowRefcount(pGpu->pOsGpuInfo, NV_FALSE);
+            status = NV_OK;
+       }
+       else
+       {
+            NV_ASSERT(0);
+            status = NV_ERR_INVALID_ARGUMENT;
+       }
+    }
+    else if (pParams->action == NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW)
+    {
+        if (!pSubdevice->bGcoffDisallowed)
+        {
+            pSubdevice->bGcoffDisallowed = NV_TRUE;
+            osClientGcoffDisallowRefcount(pGpu->pOsGpuInfo, NV_TRUE);
+            status = NV_OK;
+        }
+        else
+        {
+            NV_ASSERT(0);
+            status = NV_ERR_INVALID_ARGUMENT;
+        }
+    }
+    else
+    {
+        status = NV_ERR_INVALID_ARGUMENT;
+    }
+
+    return status;
+}
+
+/*!
+ * @brief Read and Parse the Dynamic Power Management regkey.
+ * This function modifies the default value of regkey to either enable
+ * or disable RTD3 as per System form factor and GPU architecture.
+ *
+ * @param[in]    pNv             nv_state_t pointer.
+ * @param[out]   pRegkeyValue    Original value of regkey.
+ * @param[out]   pOption         Modified value of regkey.
+ *
+ * @return       NV_STATUS code.
+ */
+
+#define NV_PMC_BOOT_42_CHIP_ID_GA102                     0x00000172
+
+static NV_STATUS
+rmReadAndParseDynamicPowerRegkey
+(
+    nv_state_t *pNv,
+    NvU32      *pRegkeyValue,
+    NvU32      *pOption
+)
+{
+    NV_STATUS  status;
+    NvU32      chipId;
+    nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+
+    status = osReadRegistryDword(NULL, NV_REG_DYNAMIC_POWER_MANAGEMENT, pRegkeyValue);
+
+    if (status != NV_OK)
+        return status;
+
+    // If User has set some value, honor that value
+    if (*pRegkeyValue != NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT)
+    {
+        *pOption = *pRegkeyValue;
+        return NV_OK;
+    }
+
+    chipId = decodePmcBoot42ChipId(pNvp->pmc_boot_42);
+
+    // From GA102+, we enable RTD3 only if system is found to be Notebook
+    if ((chipId >= NV_PMC_BOOT_42_CHIP_ID_GA102) &&
+        (rm_is_system_notebook()))
+    {
+        *pOption = NV_REG_DYNAMIC_POWER_MANAGEMENT_FINE;
+        return NV_OK;
+    }
+
+    // For Tegra PCI iGPU, support GPU Rail-Gating feature
+    if (pNv->supports_tegra_igpu_rg)
+    {
+        *pOption = NV_REG_DYNAMIC_POWER_MANAGEMENT_FINE;
+        return NV_OK;
+    }
+
+    *pOption = NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER;
+    return NV_OK;
+}
+#undef NV_PMC_BOOT_42_CHIP_ID_GA102
+
+/*!
+ * @brief Initialize state related to Tegra iGPU dynamic power management.
+ * Called once per GPU during driver initialization.
+ *
+ * @param[in]   sp  nvidia_stack_t pointer.
+ * @param[in]   nv  nv_state_t pointer.
+ */
+void NV_API_CALL rm_init_tegra_dynamic_power_management(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    void *fp;
+
+    if (!nv->supports_tegra_igpu_rg)
+        return;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    portMemSet(&nvp->dynamic_power, 0, sizeof(nvp->dynamic_power));
+
+    nv->is_tegra_pci_igpu_rg_enabled = NV_FALSE;
+
+    NV_PRINTF(LEVEL_INFO,
+                "NVRM: Tegra PCI iGPU detected, Rail-Gating is supported.\n");
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void rm_check_s0ix_regkey_and_platform_support(void)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    NvU32 data;
+    NvBool status = ((nv_platform_supports_s0ix()) &&
+                     ((osReadRegistryDword(NULL, NV_REG_ENABLE_S0IX_POWER_MANAGEMENT, &data) == NV_OK) && (data == 1)));
+
+    pSys->setProperty(pSys, PDB_PROP_SYS_SUPPORTS_S0IX, status);
+}
+
+/*!
+ * @brief Initialize state related to dynamic power management.
+ * Called once per GPU during driver initialization.
+ *
+ * @param[in]   sp  nvidia_stack_t pointer.
+ * @param[in]   nv  nv_state_t pointer.
+ */
+void NV_API_CALL rm_init_dynamic_power_management(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    NvBool bPr3AcpiMethodPresent
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    NvU32 option   = NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER;
+    NvU32 gcOffMaxFbSizeMb;
+    void *fp;
+    NvBool bUefiConsole;
+    NvU32 status;
+    NvU32 regkeyValue;
+    NvBool bIsTegraPciIgpuRgSupported = nv->supports_tegra_igpu_rg;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    /*
+     * Program an impossible value so that we show correct status
+     * during procfs read of runtime D3 status.
+     */
+    nvp->dynamic_power.dynamic_power_regkey = 0xFFFFFFFFu;
+
+    /*
+     * Dynamic power management will be disabled in either of
+     * following 3 cases:
+     * 1. The driver does not support dynamic PM for this
+     *    distro/kernel version.
+     * 2. _PR3 ACPI method is not supported by upstream port of GPU.
+     * 3. This feature is not enabled by regkey.
+     * Also see NOTE
+     */
+    status = rmReadAndParseDynamicPowerRegkey(nv, &regkeyValue, &option);
+
+    if (status == NV_OK)
+    {
+        nvp->dynamic_power.dynamic_power_regkey = regkeyValue;
+    }
+
+    nvp->pr3_acpi_method_present = bPr3AcpiMethodPresent;
+    if (!nv_dynamic_power_available(nv) || (status != NV_OK) ||
+        (!bPr3AcpiMethodPresent && !bIsTegraPciIgpuRgSupported))
+    {
+        NV_PRINTF(LEVEL_NOTICE,
+                  "%s: Disabling dynamic power management either due to lack"
+                  " of system support or due to error (%d) in reading regkey."
+                  "\n", __FUNCTION__, status);
+        option = NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER;
+    }
+
+    /*
+     * Read the OS registry for Maximum FB size during GCOFF based dynamic PM.
+     * This will be capped to GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB.
+     */
+    if (osReadRegistryDword(NULL,
+                            NV_REG_DYNAMIC_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD,
+                            &gcOffMaxFbSizeMb) == NV_OK)
+    {
+        if (gcOffMaxFbSizeMb > GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB)
+            gcOffMaxFbSizeMb = GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB;
+    }
+    else
+    {
+        gcOffMaxFbSizeMb = GCOFF_DYNAMIC_PM_MAX_FB_SIZE_MB;
+    }
+
+    nvp->dynamic_power.gcoff_max_fb_size =
+            (NvU64)gcOffMaxFbSizeMb * 1024 * 1024;
+
+    nvp->dynamic_power.mutex = portSyncMutexCreate(portMemAllocatorGetGlobalNonPaged());
+    if (nvp->dynamic_power.mutex == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s: failed to create power mutex\n", __FUNCTION__);
+        nvp->dynamic_power.mode = NV_DYNAMIC_PM_NEVER;
+        goto done;
+    }
+
+    switch (option)
+    {
+    case NV_REG_DYNAMIC_POWER_MANAGEMENT_FINE:
+        nvp->dynamic_power.mode = NV_DYNAMIC_PM_FINE;
+        break;
+    case NV_REG_DYNAMIC_POWER_MANAGEMENT_COARSE:
+        nvp->dynamic_power.mode = NV_DYNAMIC_PM_COARSE;
+        break;
+    default:
+        nv_printf(LEVEL_ERROR,
+                  "NVRM: Unknown DynamicPowerManagement value '%u' specified; "
+                  "disabling dynamic power management.\n", option);
+        // fallthrough
+    case NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER:
+        nvp->dynamic_power.mode = NV_DYNAMIC_PM_NEVER;
+        bIsTegraPciIgpuRgSupported = NV_FALSE;
+        break;
+    }
+
+    // Enable RTD3 infrastructure from OS side.
+    if ((nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE) &&
+        (nvp->dynamic_power.dynamic_power_regkey == NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT))
+    {
+        // Enable Dynamic Rail-Gating for Tegra PCI iGPU
+        if (bIsTegraPciIgpuRgSupported)
+        {
+            if (nv_pci_tegra_pm_init(nv))
+            {
+                nv_printf(NV_DBG_INFO, "NVRM: Tegra PCI iGPU railgating is enabled\n");
+                nv->is_tegra_pci_igpu_rg_enabled = NV_TRUE;
+
+                // Allow runtime suspend for Tegra PCI iGPU device with RG successfully enabled.
+                nv_allow_runtime_suspend(nv);
+            }
+        }
+        else
+        {
+            // Allow runtime suspend for the GPU which is not Tegra PCI iGPU device.
+            nv_allow_runtime_suspend(nv);
+        }
+    }
+
+    // UEFI case: where console is driven by GOP driver.
+    bUefiConsole = rm_get_uefi_console_status(nv);
+
+    nv->console_device = bUefiConsole || nv->primary_vga;
+
+    //
+    // Initialize refcount as 1 and state as IN_USE.
+    // rm_enable_dynamic_power_management() will decrease the refcount
+    // and change state to IDLE_INDICATED, if all prerequisites are met.
+    //
+    nvp->dynamic_power.refcount = 1;
+    nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_UNKNOWN,
+                                      NV_DYNAMIC_POWER_STATE_IN_USE);
+done:
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+/*!
+ * @brief Clean up state related to dynamic power management.
+ * Called during driver removal.
+ *
+ * @param[in]   sp  nvidia_stack_t pointer.
+ * @param[in]   nv  nv_state_t pointer.
+ */
+void NV_API_CALL rm_cleanup_dynamic_power_management(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    void *fp;
+    NvBool ret;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    // Disable RTD3 infrastructure from OS side.
+    if (((nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE) &&
+        (nvp->dynamic_power.dynamic_power_regkey == NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT)) ||
+        nv->is_tegra_pci_igpu_rg_enabled)
+    {
+        nv_disallow_runtime_suspend(nv);
+
+        // Disable Dynamic Rail-Gating for Tegra PCI iGPU
+        if (nv->is_tegra_pci_igpu_rg_enabled)
+        {
+            nv_pci_tegra_pm_deinit(nv);
+            nv->is_tegra_pci_igpu_rg_enabled = NV_FALSE;
+            nv_printf(NV_DBG_INFO, "NVRM: Tegra PCI iGPU railgating is disabled\n");
+        }
+    }
+
+    nv_dynamic_power_state_t old_state = nvp->dynamic_power.state;
+
+    switch (old_state)
+    {
+    case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED:
+        nv_indicate_not_idle(nv);
+        /* fallthrough */
+    case NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED:
+    case NV_DYNAMIC_POWER_STATE_IDLE_INSTANT:
+        ret = nv_dynamic_power_state_transition(nv, old_state,
+                                                NV_DYNAMIC_POWER_STATE_IN_USE);
+        /*
+         * Nothing else should be running asynchronous to teardown that could
+         * change the state so this should always succeed.
+         */
+        NV_ASSERT(ret);
+        break;
+    case NV_DYNAMIC_POWER_STATE_IN_USE:
+        break;
+    case NV_DYNAMIC_POWER_STATE_UNKNOWN:
+        NV_ASSERT(0);
+        break;
+    }
+
+    if (nvp->dynamic_power.mutex)
+    {
+        portSyncMutexDestroy(nvp->dynamic_power.mutex);
+        nvp->dynamic_power.mutex = NULL;
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+/*!
+ * @brief Initialize the dynamic power management refcount and enable GPUOFF,
+ * if all prerequisites are met.
+ * Called once per GPU during driver initialization.
+ *
+ * @param[in]   sp  nvidia_stack_t pointer.
+ * @param[in]   nv  nv_state_t pointer.
+ */
+void NV_API_CALL rm_enable_dynamic_power_management(
+    nvidia_stack_t *sp,
+    nv_state_t     *nv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    /*
+     * If dynamic power management is enabled and device is not driving console,
+     * then dynamic power management can be engaged (since we're not
+     * using the GPU yet, so it's okay to power it off).
+     */
+    if ((nvp->dynamic_power.mode != NV_DYNAMIC_PM_NEVER) && !nv->console_device)
+    {
+        os_unref_dynamic_power(nv, NV_DYNAMIC_PM_COARSE);
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+/*
+ * @brief Increment the dynamic power refcount to prevent the GPU from being
+ * powered off until a corresponding os_unref_dynamic_power() call is made.
+ * These two calls must always be balanced in pairs.
+ *
+ * This call may block until the GPU is powered on.
+ *
+ * The caller must not be holding the GPU lock, but may be holding the API
+ * lock.
+ *
+ * @param[in]   sp      nvidia_stack_t pointer.
+ * @param[in]   mode    Dynamic power mode this refcount bump is used for.
+ */
+NV_STATUS
+os_ref_dynamic_power(
+    nv_state_t *nv,
+    nv_dynamic_power_mode_t mode
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    NV_STATUS status = NV_OK;
+    NvS32 ref;
+
+    if (nvp == NULL)
+    {
+        NV_ASSERT(nv->removed);
+        return NV_OK;
+    }
+
+    if (mode > nvp->dynamic_power.mode)
+    {
+        return NV_OK;
+    }
+
+    acquireDynamicPowerMutex(nvp);
+
+    ref = nvp->dynamic_power.refcount++;
+
+    NV_ASSERT(ref >= 0);
+
+    if (ref > 0)
+    {
+        NV_ASSERT(nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IN_USE);
+    }
+    else
+    {
+        /*
+         * Just transitioned from 0 -> 1.
+         * The current dynamic power state could be any of:
+         *
+         * - IDLE_INDICATED: call nv_indicate_not_idle() to ensure
+         *   the GPU is at full power and ready to be used.
+         *
+         * - IDLE_INSTANT/IDLE_SUSTAINED: transition to IN_USE.
+         *   Note that in these states, callbacks may be operating
+         *   asynchronously, so care has to be taken to retry the write if it
+         *   fails.
+         */
+
+        nv_dynamic_power_state_t old_state = nvp->dynamic_power.state;
+        NvBool ret;
+
+        switch (old_state)
+        {
+        default:
+        case NV_DYNAMIC_POWER_STATE_IN_USE:
+            NV_PRINTF(LEVEL_ERROR, "Unexpected dynamic power state 0x%x\n",
+                      old_state);
+            /* fallthrough */
+        case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED:
+            status = nv_indicate_not_idle(nv);
+            if (status != NV_OK)
+            {
+                nvp->dynamic_power.refcount--;
+                break;
+            }
+            if (nvp->dynamic_power.deferred_idle_enabled)
+            {
+                RmScheduleCallbackForIdlePreConditions(NV_GET_NV_PRIV_PGPU(nv));
+            }
+            /* fallthrough */
+        case NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED:
+        case NV_DYNAMIC_POWER_STATE_IDLE_INSTANT:
+            do {
+                ret = nv_dynamic_power_state_transition(nv, old_state, NV_DYNAMIC_POWER_STATE_IN_USE);
+                if (!ret)
+                {
+                    old_state = nvp->dynamic_power.state;
+                    NV_ASSERT(old_state == NV_DYNAMIC_POWER_STATE_IDLE_INSTANT ||
+                              old_state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED);
+                }
+            } while (!ret);
+            break;
+        }
+    }
+
+    releaseDynamicPowerMutex(nvp);
+
+    return status;
+}
+
+/*
+ * @brief This function queries the dynamic boost support.
+ * This function returns -1 in case it failed to query the
+ * dynamic boost support. It returns 1 if dynamic boost is
+ * supported and returns 0 if dynamic boost is not supported.
+ *
+ * The caller must hold the API lock before calling this
+ * function.
+ *
+ * @param[in]   nv      nv_state_t pointer.
+ */
+static int os_get_dynamic_boost_support(
+    nv_state_t *nv    
+)
+{
+    NV_STATUS status = NV_OK;
+    RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS *pNvpcfParams;
+    OBJGPU *pGpu;
+    int ret;
+
+    status = os_alloc_mem((void**)&pNvpcfParams, sizeof(*pNvpcfParams));
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to allocate memory\n");
+        ret = -1;
+        goto end;
+    }
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    pNvpcfParams->gpuId = pGpu->gpuId;
+    pNvpcfParams->subFunc = 
+        NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_SUPPORTED_CASE;
+    status = pRmApi->Control(pRmApi,
+                           nv->rmapi.hClient,
+                           nv->rmapi.hClient,
+                           NV0000_CTRL_CMD_SYSTEM_NVPCF_GET_POWER_MODE_INFO,
+                           (void*)pNvpcfParams,
+                           sizeof(*pNvpcfParams));
+    if (status == NV_OK)
+    {
+        ret = 1;
+    }
+    else if (status == NV_ERR_NOT_SUPPORTED)
+    {
+        ret = 0;
+    }
+    else
+    {
+        ret = -1;
+    }
+
+    os_free_mem(pNvpcfParams);
+end:
+    return ret;
+}
+
+/*!
+ * @brief Wrapper around os_ref_dynamic_power() suitable for use from the
+ * per-OS layers.
+ *
+ * @param[in]   sp      nvidia_stack_t pointer.
+ * @param[in]   nv      nv_state_t pointer.
+ * @param[in]   mode    Dynamic power mode this refcount bump is used for.
+ */
+NV_STATUS NV_API_CALL rm_ref_dynamic_power(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    nv_dynamic_power_mode_t mode
+)
+{
+    NV_STATUS status;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    status = os_ref_dynamic_power(nv, mode);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return status;
+}
+
+/*!
+ * @brief Decrement the dynamic power refcount to release an earlier
+ * requirement that the GPU be powered on made by calling
+ * os_ref_dynamic_power().
+ * These two calls must always be balanced in pairs.
+ *
+ * This call may block until the GPU is powered on.
+ *
+ * The caller must not be holding the GPU lock, but may be holding the API
+ * lock.
+ *
+ * @param[in]   sp      nvidia_stack_t pointer.
+ * @param[in]   mode    Dynamic power mode this refcount bump is used for.
+ */
+void
+os_unref_dynamic_power(
+    nv_state_t *nv,
+    nv_dynamic_power_mode_t mode
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    NvS32 ref;
+
+    if (nvp == NULL)
+    {
+        NV_ASSERT(nv->removed);
+        return;
+    }
+
+    if (mode > nvp->dynamic_power.mode)
+    {
+        return;
+    }
+
+    acquireDynamicPowerMutex(nvp);
+
+    ref = --nvp->dynamic_power.refcount;
+
+    NV_ASSERT(ref >= 0);
+
+    if (ref == 0) {
+        NvBool ret;
+
+        /*
+         * Just transitioned from 1 -> 0.  We should have been in the IN_USE
+         * state previously; now we transition to an idle state: "instant" idle
+         * if deferred idle is enabled; otherwise indicate idle immediately.
+         *
+         * Nothing should be asynchronously changing the state from IN_USE,
+         * so the state transitions are not expected to fail.
+         */
+
+        NV_ASSERT(nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IN_USE);
+
+        if (nvp->dynamic_power.deferred_idle_enabled)
+        {
+            ret = nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IN_USE,
+                                                        NV_DYNAMIC_POWER_STATE_IDLE_INSTANT);
+            NV_ASSERT(ret);
+        }
+        else
+        {
+            nv_indicate_idle(nv);
+            ret = nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IN_USE,
+                                                        NV_DYNAMIC_POWER_STATE_IDLE_INDICATED);
+            NV_ASSERT(ret);
+        }
+    }
+
+    releaseDynamicPowerMutex(nvp);
+}
+
+/*!
+ * @brief Wrapper around os_unref_dynamic_power() suitable for use from the
+ * per-OS layers.
+ *
+ * @param[in]   sp      nvidia_stack_t pointer.
+ * @param[in]   nv      nv_state_t pointer.
+ * @param[in]   mode    Dynamic power mode this refcount bump is used for.
+ */
+void NV_API_CALL rm_unref_dynamic_power(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    nv_dynamic_power_mode_t mode
+)
+{
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    os_unref_dynamic_power(nv, mode);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+/*!
+ * @brief Wrapper around os_ref_dynamic_power() suitable for use from core RM.
+ *
+ * @params[in]  pOsGpuInfo        OS specific GPU information pointer
+ */
+NV_STATUS osRefGpuAccessNeeded(
+    OS_GPU_INFO *pOsGpuInfo
+)
+{
+    return os_ref_dynamic_power(pOsGpuInfo, NV_DYNAMIC_PM_FINE);
+}
+
+/*!
+ * @brief Wrapper around os_unref_dynamic_power() suitable for use from core RM.
+ *
+ * @params[in]  pOsGpuInfo        OS specific GPU information pointer
+ */
+void osUnrefGpuAccessNeeded(
+    OS_GPU_INFO *pOsGpuInfo
+)
+{
+    os_unref_dynamic_power(pOsGpuInfo, NV_DYNAMIC_PM_FINE);
+}
+
+/*!
+ * @brief Check if GCx is supported on current pstate and if engines are idle.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static NvBool RmCheckForGcxSupportOnCurrentState(
+    OBJGPU *pGpu
+)
+{
+    NV_STATUS   status = NV_OK;
+    nv_state_t *nv     = NV_GET_NV_STATE(pGpu);
+    RM_API     *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS entryPrerequisiteParams;
+
+    portMemSet(&entryPrerequisiteParams, 0, sizeof(entryPrerequisiteParams));
+
+    status = pRmApi->Control(pRmApi,
+                             nv->rmapi.hClient,
+                             nv->rmapi.hSubDevice,
+                             NV2080_CTRL_CMD_INTERNAL_GCX_ENTRY_PREREQUISITE,
+                             (void*)&entryPrerequisiteParams,
+                             sizeof(entryPrerequisiteParams));
+
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "NVRM, Failed to get GCx pre-requisite, status=0x%x\n",
+                  status);
+        return NV_FALSE;
+    }
+
+    return entryPrerequisiteParams.bIsGC6Satisfied || entryPrerequisiteParams.bIsGCOFFSatisfied;
+}
+
+/*!
+ * @brief: Work item to actually remove the idle holdoff which was applied
+ * on GC6 exit.
+ * This workitem is scheduled after the timeout limit for idle holdoff is
+ * finished.
+ *
+ * Queue with lock flags:
+ *     bLockGpuGroupSubdevice
+ *
+ * @param[in]   gpuInstance     GPU instance ID.
+ * @param[in]   pArgs           Unused callback closure.
+ */
+static void RmRemoveIdleHoldoff(
+   NvU32 gpuInstance,
+    void *pArgs
+)
+{
+    OBJGPU     *pGpu = gpumgrGetGpu(gpuInstance);
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+
+    if (nvp->dynamic_power.b_idle_holdoff == NV_TRUE)
+    {
+        if ((RmCheckForGcxSupportOnCurrentState(pGpu) == NV_TRUE) ||
+            (nvp->dynamic_power.idle_precondition_check_callback_scheduled))
+        {
+            nv_indicate_idle(nv);
+            nvp->dynamic_power.b_idle_holdoff = NV_FALSE;
+        }
+        else
+        {
+            RmScheduleCallbackToRemoveIdleHoldoff(pGpu);
+        }
+    }
+}
+
+/*!
+ * @brief Timer callback to schedule a work item to remove idle hold off
+ * that was applied after GC6 exit.
+ *
+ * @param[in]   pCallbackData   OBJGPU pointer (with void pointer type).
+ */
+static void timerCallbackToRemoveIdleHoldoff(
+    void *pCallbackData
+)
+{
+    NV_STATUS  status = NV_OK;
+    OBJGPU    *pGpu   = reinterpretCast(pCallbackData, OBJGPU *);
+
+    status = osQueueWorkItem(pGpu,
+                             RmRemoveIdleHoldoff,
+                             NULL,
+                             (OsQueueWorkItemFlags){.bLockGpuGroupSubdevice = NV_TRUE});
+
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "Queuing of remove idle holdoff work item failed with status : 0x%x\n",
+                  status);
+    }
+}
+
+/*!
+ * @brief Timer callback to check if all idle conditions remain met and
+ * if so schedule a work item to indicate the GPU is idle.
+ *
+ * @param[in]   pCallbackData   OBJGPU pointer (with void pointer type).
+ */
+static void timerCallbackToIndicateIdle(
+    void *pCallbackData
+)
+{
+    OBJGPU     *pGpu    = reinterpretCast(pCallbackData, OBJGPU *);
+    nv_state_t *nv      = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp     = NV_GET_NV_PRIV(nv);
+
+    nv_acquire_mmap_lock(nv);
+
+    if (nv_get_all_mappings_revoked_locked(nv) &&
+        nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED &&
+        RmCanEnterGcxUnderGpuLock(pGpu))
+    {
+        osQueueWorkItem(pGpu,
+                        RmIndicateIdle,
+                        NULL,
+                        (OsQueueWorkItemFlags){0});
+    }
+    else
+    {
+        if (nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED)
+        {
+            /*
+             * Something used the GPU between when we last sampled and now.
+             * Go back to the "instant" state to start waiting from scratch.
+             */
+            nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED,
+                                                  NV_DYNAMIC_POWER_STATE_IDLE_INSTANT);
+        }
+
+        RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu);
+    }
+
+    nv_release_mmap_lock(nv);
+}
+
+/*!
+ * @brief Timer callback to periodically check if all idle preconditions are
+ * met.  If so, the dynamic power state is advanced from IDLE_INSTANT to
+ * IDLE_SUSTAINED, or (if already in the IDLE_SUSTAINED state) mappings are
+ * revoked and timerCallbackToIndicateIdle() is scheduled.
+ *
+ * @param[in]   pCallbackData   OBJGPU pointer (with void pointer type).
+ */
+static void timerCallbackForIdlePreConditions(
+    void *pCallbackData
+)
+{
+    OBJGPU     *pGpu    = reinterpretCast(pCallbackData, OBJGPU *);
+    nv_state_t *nv      = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp     = NV_GET_NV_PRIV(nv);
+
+    nvp->dynamic_power.idle_precondition_check_callback_scheduled = NV_FALSE;
+
+    /*
+     * Note: It is not safe to take the dynamic power mutex here.
+     * Lock ordering dictates that the dynamic power mutex must be taken before
+     * the GPU lock, and this function is called with the GPU lock held.
+     *
+     * Therefore, this function is careful about *only* triggering specific
+     * transitions:
+     * NV_DYNAMIC_POWER_STATE_IDLE_INSTANT -> NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED
+     * NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED -> NV_DYNAMIC_POWER_STATE_IDLE_INSTANT
+     *
+     * Other codepaths are resilient to these state transitions occurring
+     * asynchronously.  The state transition itself is done with an atomic
+     * compare and swap, so there should be no danger of inadvertently
+     * triggering a transition from a different state due to a
+     * read-modify-write or "time of check to time of use" (TOCTTOU) race
+     * condition.
+     */
+
+    if (nvp->dynamic_power.state != NV_DYNAMIC_POWER_STATE_IN_USE)
+    {
+        if (RmCanEnterGcxUnderGpuLock(pGpu))
+        {
+            nv_dynamic_power_state_t state = nvp->dynamic_power.state;
+            switch (state)
+            {
+            case NV_DYNAMIC_POWER_STATE_UNKNOWN:
+                NV_ASSERT(0);
+                /* fallthrough */
+            case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED:
+                NV_PRINTF(LEVEL_ERROR, "unexpected dynamic power state 0x%x\n",
+                          nvp->dynamic_power.state);
+                /* fallthrough */
+            case NV_DYNAMIC_POWER_STATE_IN_USE:
+                break;
+            case NV_DYNAMIC_POWER_STATE_IDLE_INSTANT:
+                nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IDLE_INSTANT,
+                                                      NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED);
+                break;
+            case NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED:
+                RmQueueIdleSustainedWorkitem(pGpu);
+                return;
+            }
+
+        }
+        else
+        {
+            if (nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED)
+            {
+                /*
+                 * Something used the GPU between when we last sampled and now.
+                 * Go back to the "instant" state to start waiting from scratch.
+                 */
+                nv_dynamic_power_state_transition(nv, NV_DYNAMIC_POWER_STATE_IDLE_SUSTAINED,
+                                                      NV_DYNAMIC_POWER_STATE_IDLE_INSTANT);
+            }
+        }
+    }
+    RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu);
+}
+
+/*!
+ * @brief Cancel any in-flight callback to remove idle holdoff.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static void RmCancelCallbackToRemoveIdleHoldoff(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    TMR_EVENT_GENERAL_PARAMS cancelParams;
+
+    if (NvP64_VALUE(nvp->dynamic_power.remove_idle_holdoff) != NULL)
+    {
+        portMemSet(&cancelParams, 0, sizeof(cancelParams));
+
+        cancelParams.pEvent = nvp->dynamic_power.remove_idle_holdoff;
+
+        tmrCtrlCmdEventCancel(pGpu, &cancelParams);
+    }
+}
+
+/*!
+ * @brief Cancel any in-flight timer callbacks.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static void RmCancelDynamicPowerCallbacks(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    TMR_EVENT_GENERAL_PARAMS cancelParams;
+
+    if (NvP64_VALUE(nvp->dynamic_power.idle_precondition_check_event) != NULL)
+    {
+        // Cancel precondition check callback.
+        portMemSet(&cancelParams, 0, sizeof(cancelParams));
+
+        cancelParams.pEvent = nvp->dynamic_power.idle_precondition_check_event;
+
+        tmrCtrlCmdEventCancel(pGpu, &cancelParams);
+    }
+
+    if (NvP64_VALUE(nvp->dynamic_power.indicate_idle_event) != NULL)
+    {
+        // Cancel callback to indicate idle
+        portMemSet(&cancelParams, 0, sizeof(cancelParams));
+
+        cancelParams.pEvent = nvp->dynamic_power.indicate_idle_event;
+
+        tmrCtrlCmdEventCancel(pGpu, &cancelParams);
+    }
+
+    RmCancelCallbackToRemoveIdleHoldoff(pGpu);
+
+    nvp->dynamic_power.idle_precondition_check_callback_scheduled = NV_FALSE;
+}
+
+/*!
+ * @brief Free timer events allocated by CreateDynamicPowerCallbacks()
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static void RmDestroyDynamicPowerCallbacks(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    TMR_EVENT_GENERAL_PARAMS destroyParams;
+
+    if (NvP64_VALUE(nvp->dynamic_power.idle_precondition_check_event) != NULL)
+    {
+        // Destroy precondition check callback.
+        portMemSet(&destroyParams, 0, sizeof(destroyParams));
+
+        destroyParams.pEvent = nvp->dynamic_power.idle_precondition_check_event;
+
+        tmrCtrlCmdEventDestroy(pGpu, &destroyParams);
+        nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL;
+    }
+
+    if (NvP64_VALUE(nvp->dynamic_power.indicate_idle_event) != NULL)
+    {
+        // Destroy callback to indicate idle
+        portMemSet(&destroyParams, 0, sizeof(destroyParams));
+
+        destroyParams.pEvent = nvp->dynamic_power.indicate_idle_event;
+
+        tmrCtrlCmdEventDestroy(pGpu, &destroyParams);
+        nvp->dynamic_power.indicate_idle_event = NvP64_NULL;
+    }
+
+    if (NvP64_VALUE(nvp->dynamic_power.remove_idle_holdoff) != NULL)
+    {
+        // Destroy callback to decrease kernel refcount
+        portMemSet(&destroyParams, 0, sizeof(destroyParams));
+
+        destroyParams.pEvent = nvp->dynamic_power.remove_idle_holdoff;
+
+        tmrCtrlCmdEventDestroy(pGpu, &destroyParams);
+        nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL;
+    }
+
+    nvp->dynamic_power.deferred_idle_enabled = NV_FALSE;
+}
+
+/*
+ * @brief Adds a GPU to dynamicPowerSupportGpuMask
+ *
+ * @param[in]                  instance
+ */
+static void AddGpuDynamicPowerSupported
+(
+    NvU32 instance
+)
+{
+    dynamicPowerSupportGpuMask |= NVBIT(instance);
+}
+
+/*
+ * @brief Removes a GPU from dynamicPowerSupportGpuMask
+ *
+ * @param[in]                  instance
+ */
+static void RemoveGpuDynamicPowerSupported
+(
+    NvU32 instance
+)
+{
+    dynamicPowerSupportGpuMask &= ~NVBIT(instance);
+}
+
+/*
+ * @brief queries  dynamicPowerSupportGpuMask
+ *
+ * @param[in]                  void
+ */
+NvU32 osGetDynamicPowerSupportMask
+(
+    void
+)
+{
+    return dynamicPowerSupportGpuMask;
+}
+
+/*!
+ * @brief Cancel any outstanding callbacks, and free RM resources allocated by
+ * RmInitDeferredDynamicPowerManagement().
+ *
+ * @param[in]   nv      nv_state_t pointer.
+ */
+void RmDestroyDeferredDynamicPowerManagement(
+    nv_state_t *nv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    OBJGPU    *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    //
+    // Re-instate the dynamic power mode to fine if it was
+    // disabled due to lack of platform support
+    //
+    if (nvp->dynamic_power.b_fine_not_supported == NV_TRUE)
+    {
+        nvp->dynamic_power.mode = NV_DYNAMIC_PM_FINE;
+        nvp->dynamic_power.b_fine_not_supported = NV_FALSE;
+    }
+
+    //
+    // Before cancelling and destroying the callbacks, make sure to
+    // remove the idle holfoff that was applied during gc6 exit.
+    //
+    if (nvp->dynamic_power.b_idle_holdoff == NV_TRUE)
+    {
+        nv_indicate_idle(nv);
+        nvp->dynamic_power.b_idle_holdoff = NV_FALSE;
+    }
+
+    RmCancelDynamicPowerCallbacks(pGpu);
+    RmDestroyDynamicPowerCallbacks(pGpu);
+    RemoveGpuDynamicPowerSupported(gpuGetInstance(pGpu));
+}
+
+/*!
+ * @brief Allocate timer events to call the functions:
+ * timerCallbackForIdlePreConditions()
+ * timerCallbackToIndicateIdle()
+ *
+ * The callbacks must be scheduled separately.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static NV_STATUS CreateDynamicPowerCallbacks(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    TMR_EVENT_SET_PARAMS createEventParams;
+    NV_STATUS status;
+
+    // Creating PreCondition check callback.
+    portMemSet(&createEventParams, 0, sizeof(createEventParams));
+
+    createEventParams.ppEvent = &(nvp->dynamic_power.idle_precondition_check_event);
+    createEventParams.pTimeProc = NV_PTR_TO_NvP64(timerCallbackForIdlePreConditions);
+    createEventParams.pCallbackData = NV_PTR_TO_NvP64(pGpu);
+    createEventParams.flags = TMR_FLAG_USE_OS_TIMER;
+
+    status = tmrCtrlCmdEventCreate(pGpu, &createEventParams);
+
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "Error creating dynamic power precondition check callback\n");
+        nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL;
+        nvp->dynamic_power.indicate_idle_event = NvP64_NULL;
+        nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL;
+        return status;
+    }
+
+    // Create callback to indicate idle..
+    portMemSet(&createEventParams, 0, sizeof(createEventParams));
+
+    createEventParams.ppEvent = &(nvp->dynamic_power.indicate_idle_event);
+    createEventParams.pTimeProc = NV_PTR_TO_NvP64(timerCallbackToIndicateIdle);
+    createEventParams.pCallbackData = NV_PTR_TO_NvP64(pGpu);
+    createEventParams.flags = TMR_FLAG_USE_OS_TIMER;
+
+    status = tmrCtrlCmdEventCreate(pGpu, &createEventParams);
+
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "Error creating callback to indicate GPU idle\n");
+        nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL;
+        nvp->dynamic_power.indicate_idle_event = NvP64_NULL;
+        nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL;
+        return status;
+    }
+
+    // Create callback to remove idle holdoff
+    portMemSet(&createEventParams, 0, sizeof(createEventParams));
+
+    createEventParams.ppEvent = &(nvp->dynamic_power.remove_idle_holdoff);
+    createEventParams.pTimeProc = NV_PTR_TO_NvP64(timerCallbackToRemoveIdleHoldoff);
+    createEventParams.pCallbackData = NV_PTR_TO_NvP64(pGpu);
+    createEventParams.flags = TMR_FLAG_USE_OS_TIMER;
+
+    status = tmrCtrlCmdEventCreate(pGpu, &createEventParams);
+
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "Error creating callback to decrease kernel refcount\n");
+        nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL;
+        nvp->dynamic_power.indicate_idle_event = NvP64_NULL;
+        nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL;
+        return status;
+    }
+
+    return NV_OK;
+}
+
+/*!
+ * @brief Schedule the timerCallbackForIdlePreConditions() function to be called.
+ *
+ * If it is already scheduled, the function is a noop.
+ *
+ * The GPU lock must be held when calling this function.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static void RmScheduleCallbackForIdlePreConditionsUnderGpuLock(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    TMR_EVENT_SCHEDULE_PARAMS scheduleEventParams;
+    NV_STATUS status;
+
+    NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance));
+
+    if (nvp->dynamic_power.idle_precondition_check_callback_scheduled)
+    {
+        return;
+    }
+
+    if (NvP64_VALUE(nvp->dynamic_power.idle_precondition_check_event) != NULL)
+    {
+        portMemSet(&scheduleEventParams, 0, sizeof(scheduleEventParams));
+
+        scheduleEventParams.pEvent = nvp->dynamic_power.idle_precondition_check_event;
+        scheduleEventParams.bUseTimeAbs = NV_FALSE;
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED))
+        {
+            /*
+             * iGPU which supports Rail-Gating may want shorter idleness
+             * threshold to engage the deferred-idle transition.
+             * Set the threshold for precondition check to 500 ms.
+             */
+            scheduleEventParams.timeNs = IGPU_RG_PRECONDITION_CHECK_TIME;
+        }
+        else
+        {
+            scheduleEventParams.timeNs = GC6_PRECONDITION_CHECK_TIME;
+        }
+
+        status = tmrCtrlCmdEventSchedule(pGpu, &scheduleEventParams);
+
+        if (status == NV_OK)
+        {
+            nvp->dynamic_power.idle_precondition_check_callback_scheduled = NV_TRUE;
+        }
+        else
+        {
+            NV_PRINTF(LEVEL_ERROR, "Error scheduling precondition callback\n");
+        }
+    }
+}
+
+/*!
+ * @brief Schedule the timerCallbackForIdlePreConditions() function to be called.
+ *
+ * If it is already scheduled, the function is a noop.
+ *
+ * This function takes the GPU lock.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static void RmScheduleCallbackForIdlePreConditions(
+    OBJGPU *pGpu
+)
+{
+    NV_STATUS   status;
+    GPU_MASK    gpuMask;
+
+    // LOCK: acquire per device lock
+    status = rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
+                                   GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER,
+                                   &gpuMask);
+    if (status == NV_OK)
+    {
+        RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu);
+        // UNLOCK: release per device lock
+        rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+    }
+}
+
+/*!
+ * @brief Schedule the timerCallbackToIndicateIdle() function to be called.
+ *
+ * The GPU lock must be held when calling this function.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static void RmScheduleCallbackToIndicateIdle(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    TMR_EVENT_SCHEDULE_PARAMS scheduleEventParams;
+    NV_STATUS status;
+
+    NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance));
+
+    if (NvP64_VALUE(nvp->dynamic_power.indicate_idle_event) != NULL)
+    {
+        portMemSet(&scheduleEventParams, 0, sizeof(scheduleEventParams));
+
+        scheduleEventParams.pEvent = nvp->dynamic_power.indicate_idle_event;
+        scheduleEventParams.bUseTimeAbs = NV_FALSE;
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED))
+        {
+            /*
+             * iGPU which supports Rail-Gating may want shorter idleness
+             * threshold to engage the deferred-idle transition.
+             * Set the threshold for idle indicate to 700 ms.
+             */
+            scheduleEventParams.timeNs = IGPU_RG_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME;
+        }
+        else
+        {
+            scheduleEventParams.timeNs = GC6_BAR1_BLOCKER_CHECK_AND_METHOD_FLUSH_TIME;
+        }
+
+        status = tmrCtrlCmdEventSchedule(pGpu, &scheduleEventParams);
+
+        if (status != NV_OK)
+            NV_PRINTF(LEVEL_ERROR, "Error scheduling indicate idle callback\n");
+    }
+}
+
+/*!
+ * @brief Schedule the timerCallbackToRemoveIdleHoldoff() function.
+ *
+ * The GPU lock must be held when calling this function.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static void RmScheduleCallbackToRemoveIdleHoldoff(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    TMR_EVENT_SCHEDULE_PARAMS scheduleEventParams;
+    NV_STATUS status;
+
+    NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance));
+
+    if (NvP64_VALUE(nvp->dynamic_power.remove_idle_holdoff) != NULL)
+    {
+        portMemSet(&scheduleEventParams, 0, sizeof(scheduleEventParams));
+
+        scheduleEventParams.pEvent = nvp->dynamic_power.remove_idle_holdoff;
+        scheduleEventParams.timeNs = GC6_CYCLE_IDLE_HOLDOFF_CHECK_TIME;
+        scheduleEventParams.bUseTimeAbs = NV_FALSE;
+
+        status = tmrCtrlCmdEventSchedule(pGpu, &scheduleEventParams);
+
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "Error scheduling kernel refcount decrement callback\n");
+        }
+        else
+        {
+            nvp->dynamic_power.b_idle_holdoff = NV_TRUE;
+        }
+    }
+}
+
+/*!
+ * @brief Check if the system supports RTD3-GC6.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ */
+static NvBool RmCheckRtd3GcxSupport(
+    nv_state_t *pNv,
+    NvBool     *bGC6Support,
+    NvBool     *bGCOFFSupport
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(pNv);
+    OBJGPU    *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+    RM_API    *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    NV_STATUS  status;
+    NV0080_CTRL_GPU_GET_VIRTUALIZATION_MODE_PARAMS virtModeParams = { 0 };
+    if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_UNIX_DYNAMIC_POWER_SUPPORTED))
+    {
+        NV_PRINTF(LEVEL_NOTICE, "RTD3 is not supported for this arch\n");
+        return NV_FALSE;
+    }
+
+    *bGC6Support = pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED);
+    *bGCOFFSupport = pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GCOFF_SUPPORTED) ||
+                     pGpu->getProperty(pGpu, PDB_PROP_GPU_LEGACY_GCOFF_SUPPORTED);
+
+    if (!(*bGC6Support) &&
+        !(*bGCOFFSupport) &&
+        !(pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED)))
+    {
+        NV_PRINTF(LEVEL_NOTICE,
+                  "Disabling RTD3. [GC6 support=%d GCOFF support=%d RG support=%d]\n",
+                  *bGC6Support,
+                  *bGCOFFSupport,
+                  pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED));
+        return NV_FALSE;
+    }
+
+    status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDevice,
+                             NV0080_CTRL_CMD_GPU_GET_VIRTUALIZATION_MODE,
+                             &virtModeParams, sizeof(virtModeParams));
+
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "Failed to get Virtualization mode, status=0x%x\n",
+                  status);
+        return NV_FALSE;
+    }
+
+    if ((virtModeParams.virtualizationMode != NV0080_CTRL_GPU_VIRTUALIZATION_MODE_NONE) &&
+        (virtModeParams.virtualizationMode != NV0080_CTRL_GPU_VIRTUALIZATION_MODE_NMOS))
+    {
+        NV_PRINTF(LEVEL_NOTICE, "RTD3 is not supported on VM\n");
+        return NV_FALSE;
+    }
+
+    if (!nvp->pr3_acpi_method_present &&
+        !(pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED)))
+    {
+        NV_PRINTF(LEVEL_NOTICE, "RTD3 ACPI support is not available.\n");
+        return NV_FALSE;
+    }
+
+    return NV_TRUE;
+}
+
+/*!
+ * @brief Allocate resources needed to track deferred power management, and
+ * schedule the initial periodic callbacks.
+ *
+ * @param[in]   nv      nv_state_t pointer.
+ */
+static void RmInitDeferredDynamicPowerManagement(
+    nv_state_t *nv,
+    NvBool      bRtd3Support
+)
+{
+    NV_STATUS  status;
+    nv_priv_t *nvp  = NV_GET_NV_PRIV(nv);
+
+    if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE)
+    {
+        OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+        if (!bRtd3Support)
+        {
+             nvp->dynamic_power.mode = NV_DYNAMIC_PM_NEVER;
+             nvp->dynamic_power.b_fine_not_supported = NV_TRUE;
+             NV_PRINTF(LEVEL_NOTICE, "RTD3 is not supported.\n");
+             return;
+        }
+        AddGpuDynamicPowerSupported(gpuGetInstance(pGpu));
+        nvp->dynamic_power.b_fine_not_supported = NV_FALSE;
+        status = CreateDynamicPowerCallbacks(pGpu);
+
+        if (status == NV_OK)
+        {
+            RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu);
+
+            nvp->dynamic_power.deferred_idle_enabled = NV_TRUE;
+            // RM's default is GCOFF allow
+            nvp->dynamic_power.clients_gcoff_disallow_refcount = 0;
+        }
+        else
+        {
+            NV_PRINTF(LEVEL_ERROR, "Failed to register for dynamic power callbacks\n");
+        }
+    }
+}
+
+/*!
+ * @brief Check if GCOFF state can be used for Power Management.
+ *
+ * @param[in]   pGpu            OBJGPU pointer.
+ * @param[in]   usedFbSize      Used FB Allocation size.
+ * @param[in]   bIsDynamicPM    If true, check for dynamic PM; otherwise, check
+ *                              for system PM (suspend/resume).
+ *
+ * returns NV_TRUE if GCOFF state can be used, otherwise NV_FALSE.
+ *
+ */
+static NvBool RmCheckForGcOffPM(
+    OBJGPU *pGpu,
+    NvU64   usedFbSize,
+    NvBool  bIsDynamicPM
+)
+{
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    NvU64       gcoff_max_fb_size;
+
+    if (bIsDynamicPM)
+    {
+        /*
+         * GCOFF won't be enaged for Dynamic PM if X server does not allow.
+         * The dynamic_power.clients_gcoff_disallow_refcount is being accessed
+         * without acquiring dynamic power mutex since this value can be
+         * modified only when GPU is in active state. The os_ref_dynamic_power()
+         * function will be called to hold a reference to the GPU whenever
+         * RM is processing osClientGcoffDisallowRefcount() calls. This either
+         * prevents the GPU from going into a low power state or will wait for
+         * it to be resumed before proceeding.
+         */
+        if (nvp->dynamic_power.clients_gcoff_disallow_refcount != 0)
+            return NV_FALSE;
+
+        gcoff_max_fb_size = nvp->dynamic_power.gcoff_max_fb_size;
+    }
+    else
+    {
+        gcoff_max_fb_size = nvp->s0ix_gcoff_max_fb_size;
+    }
+
+    /*
+     * GCOFF will be engaged whenever the following necessary preconditions
+     * are met:
+     *
+     * 1. The GCOFF has not been disabled with regkey by setting it to zero.
+     * 2. Used FB allocation size are within limits.
+     */
+    return (gcoff_max_fb_size > 0) &&
+           (usedFbSize <= gcoff_max_fb_size);
+}
+
+//
+// Function to update fixed fbsr modes to support multiple vairants such as
+// GCOFF and cuda S3/resume.
+//
+static void
+RmUpdateFixedFbsrModes(OBJGPU *pGpu)
+{
+    MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+    nv_state_t    *nv             = NV_GET_NV_STATE(pGpu);
+
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERING))
+    {
+        pMemoryManager->fixedFbsrModesMask = NVBIT(FBSR_TYPE_DMA);
+    }
+    else if (nv->preserve_vidmem_allocations)
+    {
+        pMemoryManager->fixedFbsrModesMask = NVBIT(FBSR_TYPE_FILE);
+    }
+}
+
+static NV_STATUS
+RmPowerManagement(
+    OBJGPU *pGpu,
+    nv_pm_action_t pmAction
+)
+{
+    // default to NV_OK. there may cases where resman is loaded, but
+    // no devices are allocated (we're still at the console). in these
+    // cases, it's fine to let the system do whatever it wants.
+    NV_STATUS rmStatus = NV_OK;
+
+    if (pGpu)
+    {
+        nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+        nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+        NvBool bcState = gpumgrGetBcEnabledStatus(pGpu);
+        Intr *pIntr = GPU_GET_INTR(pGpu);
+        MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+
+        if ((pmAction == NV_PM_ACTION_HIBERNATE) || (pmAction == NV_PM_ACTION_STANDBY))
+        {
+            //
+            // pFb object store the FBSR mode through which FB state unload has happened,
+            // so os layer doesn't need to set FBSR mode on resume.
+            //
+            RmUpdateFixedFbsrModes(pGpu);
+        }
+
+        switch (pmAction)
+        {
+            case NV_PM_ACTION_HIBERNATE:
+                nvp->pm_state.InHibernate = NV_TRUE;
+                nvp->pm_state.IntrEn = intrGetIntrEn(pIntr);
+                intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED);
+                gpumgrSetBcEnabledStatus(pGpu, NV_FALSE);
+
+                rmStatus = gpuEnterHibernate(pGpu);
+                gpumgrSetBcEnabledStatus(pGpu, bcState);
+
+                break;
+
+            case NV_PM_ACTION_STANDBY:
+                nvp->pm_state.InHibernate = NV_FALSE;
+                nvp->pm_state.IntrEn = intrGetIntrEn(pIntr);
+                intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED);
+                gpumgrSetBcEnabledStatus(pGpu, NV_FALSE);
+
+                rmStatus = gpuEnterStandby(pGpu);
+                gpumgrSetBcEnabledStatus(pGpu, bcState);
+
+                break;
+
+            case NV_PM_ACTION_RESUME:
+                gpumgrSetBcEnabledStatus(pGpu, NV_FALSE);
+
+                if (nvp->pm_state.InHibernate)
+                {
+                    gpuResumeFromHibernate(pGpu);
+                }
+                else
+                {
+                    gpuResumeFromStandby(pGpu);
+                }
+                intrSetIntrEn(pIntr, nvp->pm_state.IntrEn);
+                gpumgrSetBcEnabledStatus(pGpu, bcState);
+                NvU32 ac_plugged = 0;
+
+                if (IsMobile(pGpu))
+                {
+                    if (nv_acpi_get_powersource(&ac_plugged) == NV_OK)
+                    {
+                        //
+                        // As we have already acquired API and GPU lock here, we are
+                        // directly calling RmPowerSourceChangeEvent.
+                        //
+                        RmPowerSourceChangeEvent(nv, !ac_plugged);
+                    }
+                    RmRequestDNotifierState(nv);
+                }
+                break;
+
+            default:
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                break;
+        }
+
+        pMemoryManager->fixedFbsrModesMask = 0;
+    }
+
+    return rmStatus;
+}
+
+/*!
+ * @brief Call core RM to perform the GCx (GC6/GCOF) entry sequence (before
+ * powering off the GPU) or exit sequence (after powering the GPU back on).
+ *
+ * The GPU lock should be held when calling this function.
+ *
+ * @param[in]   pGpu            OBJGPU pointer.
+ * @param[in]   bEnter          If true, perform entry sequence; exit sequence
+ *                              otherwise.
+ * @param[in]   bIsDynamicPM    If true, it is for dynamic PM; otherwise, it is
+ *                              for system PM (suspend/resume).
+ *
+ * @return      NV_STATUS code indicating if the operation was successful.
+ */
+NV_STATUS RmGcxPowerManagement(
+    OBJGPU *pGpu,
+    NvBool  bEnter,
+    NvBool  bIsDynamicPM,
+    NvBool *bTryAgain
+)
+{
+    MemoryManager *pMemoryManager           = GPU_GET_MEMORY_MANAGER(pGpu);
+    KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
+    nv_state_t         *nv                  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t          *nvp                 = NV_GET_NV_PRIV(nv);
+    NV_STATUS           status              = NV_OK;
+
+    if (pGpu->acpiMethodData.jtMethodData.bSBIOSCaps &&
+        !pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_MOBILE))
+    {
+        //
+        // AML override is expected only on NB platforms to support GCOFF-1.0(
+        // which is legacy GCOFF) and GC6-3.0
+        // GC6 3.0 is possible without AML override but due to changes required
+        // for GCOFF-1.0 in SBIOS & HW, AML override is needed for GC6-3.0 also.
+        //
+        NV_PRINTF(LEVEL_INFO,"AML overrides present in Desktop");
+    }
+
+    nv->d0_state_in_suspend = NV_FALSE;
+
+    if (bEnter)
+    {
+        NvU64          usedFbSize = 0;
+        NvBool         bCanUseGc6 = NV_FALSE;
+        NV_STATUS      fbsrStatus = NV_ERR_GENERIC;
+
+        //
+        // If the GPU supports GC6, then check if GC6 can be used for
+        // the current power management request.
+        // 1. For dynamic PM, GC6 can be used if it is supported by the GPU.
+        // 2. For system PM with s2idle, GC6 can be used if it is
+        //    supported by the GPU.
+        //
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED) &&
+            nvp->gc6_upstream_port_configured)
+        {
+            bCanUseGc6 = bIsDynamicPM ? NV_TRUE : nv_s2idle_pm_configured();
+        }
+
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED))
+        {
+            pGpu->setProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERING, NV_TRUE);
+            status = RmPowerManagement(pGpu, NV_PM_ACTION_STANDBY);
+            pGpu->setProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERING, NV_FALSE);
+            if (status == NV_OK)
+            {
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERED, NV_TRUE);
+            }
+        }
+        //
+        // If GC6 cannot be used, then no need to compare the used FB size with
+        // threshold value and select GCOFF irrespective of FB size.
+        //
+        else if ((memmgrGetUsedRamSize(pGpu, pMemoryManager, &usedFbSize) == NV_OK) &&
+            (!bCanUseGc6 || RmCheckForGcOffPM(pGpu, usedFbSize, bIsDynamicPM)) &&
+            ((fbsrStatus = fbsrReserveSysMemoryForPowerMgmt(pGpu, pMemoryManager->pFbsr[FBSR_TYPE_DMA],
+                                             usedFbSize)) == NV_OK))
+        {
+            pGpu->setProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERING, NV_TRUE);
+
+            //
+            // Set 'bPreserveComptagBackingStoreOnSuspend' so that comptag
+            // related handling can be done during state unload/load.
+            //
+            if (!pMemoryManager->bUseVirtualCopyOnSuspend)
+            {
+                pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = NV_TRUE;
+            }
+
+            status = RmPowerManagement(pGpu, NV_PM_ACTION_STANDBY);
+            pGpu->setProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERING, NV_FALSE);
+
+            if (status == NV_OK)
+            {
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED, NV_TRUE);
+            }
+            else
+            {
+                /*
+                 * Normally the NV_PM_ACTION_RESUME resume should free the
+                 * the reserved memory while doing FB state load. But if the
+                 * failure has happened at very early stage in FB state unload
+                 * while doing NV_PM_ACTION_STANDBY, then reserved system memory
+                 * might not be freed.
+                 */
+                fbsrFreeReservedSysMemoryForPowerMgmt(pMemoryManager->pFbsr[FBSR_TYPE_DMA]);
+
+                if (!pMemoryManager->bUseVirtualCopyOnSuspend)
+                {
+                    pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = nv->preserve_vidmem_allocations;
+                }
+            }
+        }
+        else if (bCanUseGc6)
+        {
+            NV2080_CTRL_GC6_ENTRY_PARAMS entryParams;
+            portMemSet(&entryParams, 0, sizeof(entryParams));
+
+            entryParams.flavorId = NV2080_CTRL_GC6_FLAVOR_ID_MSHYBRID;
+            entryParams.stepMask = NVBIT(NV2080_CTRL_GC6_STEP_ID_GPU_OFF);
+            entryParams.params.bIsRTD3Transition = NV_TRUE;
+
+            //
+            // Currently if the GPU is not in the Lowest P-State, then the
+            // GC6 entry will fail in its internal sanity check. The system
+            // suspend can be triggered in any P-State, so sanity check
+            // for the lowest P-State needs to be skipped. In this case, during
+            // Perf state unload time, the forceful P-State switch will
+            // happen which will bring the GPU into lowest P-State.
+            //
+            entryParams.params.bSkipPstateSanity = !bIsDynamicPM;
+
+            status = gpuGc6Entry(pGpu, &entryParams);
+
+            //
+            // The GC6 state requires SW involvement before each transition
+            // from D0 to D3cold. In case of system suspend with pass-through
+            // mode, the VM suspend should happen before host suspend. During
+            // VM suspend, the NVIDIA driver running in the VM side will be
+            // involved in D3cold transition, but in the host side, it will
+            // go through D3cold->D0->D3cold transition and will make the
+            // second D3cold transition without NVIDIA driver. To handle this
+            // use-case, during VM suspend, use the D0 state and the host linux
+            // kernel will put the device into D3cold state.
+            //
+            if ((status == NV_OK) && ((nv->flags & NV_FLAG_PASSTHRU) != 0))
+            {
+                nv->d0_state_in_suspend = NV_TRUE;
+            }
+        }
+        //
+        // The else condition below will hit in the following cases: 
+        // Case 1. During system suspend transition: For systems that support s2idle but are configured
+        // for deep sleep, this "else" condition will be hit when the system memory 
+        // is not sufficient. In this case, we should unset bTryAgain to abort the current suspend entry. 
+        // Case 2. During runtime suspend transition: For systems that do not support GC6 but support 
+        // GCOFF, this "else" condition will be hit when system memory is not sufficent, In this case, we 
+        // should set bTryagain so that the kernel can reschedule the callback later. 
+        //
+        else
+        {
+            if (bIsDynamicPM)
+            {
+                if (fbsrStatus == NV_ERR_NO_MEMORY)
+                {
+                    *bTryAgain = NV_TRUE;
+                }
+                status = fbsrStatus;
+            }
+            else
+            {
+                status = NV_ERR_NOT_SUPPORTED;
+                NV_PRINTF(LEVEL_ERROR,
+                          "System suspend failed with current system suspend configuration. "
+                          "Please change the system suspend configuration to s2idle in /sys/power/mem_sleep.\n");
+            }
+        }
+    }
+    else
+    {
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED))
+        {
+            status = RmPowerManagement(pGpu, NV_PM_ACTION_RESUME);
+            pGpu->setProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED, NV_FALSE);
+            if (!pMemoryManager->bUseVirtualCopyOnSuspend)
+            {
+                pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = nv->preserve_vidmem_allocations;
+            }
+        }
+        else if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERED))
+        {
+            status = RmPowerManagement(pGpu, NV_PM_ACTION_RESUME);
+            if (status == NV_OK)
+            {
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_RG_STATE_ENTERED, NV_FALSE);
+            }
+        }
+        else
+        {
+            NV2080_CTRL_GC6_EXIT_PARAMS exitParams;
+            portMemSet(&exitParams, 0, sizeof(exitParams));
+
+            exitParams.flavorId = NV2080_CTRL_GC6_FLAVOR_ID_MSHYBRID;
+            exitParams.params.bIsRTD3Transition = NV_TRUE;
+
+            status = gpuGc6Exit(pGpu, &exitParams);
+        }
+    }
+
+    return status;
+}
+
+NV_STATUS NV_API_CALL rm_power_management(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    nv_pm_action_t pmAction
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus = NV_OK;
+    void *fp;
+    NvBool bTryAgain = NV_FALSE;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_DEVICE_INIT);
+
+    NV_ASSERT_OK(os_flush_work_queue(pNv->queue, pmAction != NV_PM_ACTION_RESUME));
+    //
+    // For GPU driving console, disable console access here, to ensure no console
+    // writes through BAR1 can interfere with physical RM's setup of BAR1
+    //
+    if (pNv->client_managed_console && !pNv->sysmem_mapped_console)
+    {
+        os_disable_console_access();
+    }
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK)
+    {
+        OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+        if (pGpu != NULL)
+        {
+            if ((rmStatus = os_ref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE)) == NV_OK)
+            {
+                // LOCK: acquire GPUs lock
+                if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK)
+                {
+                    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+                    {
+                        rmStatus = RmPowerManagementTegra(pGpu, pmAction);
+
+                        //
+                        // RmPowerManagementTegra() is most likely to fail due to
+                        // gpuStateUnload() failures deep in the RM's GPU power
+                        // management paths.  However, those paths make no
+                        // attempt to unwind in case of errors.  Rather, they
+                        // soldier on and simply report an error at the very end.
+                        // GPU software state meanwhile will indicate the GPU
+                        // has been suspended.
+                        //
+                        // Sadly, in case of an error during suspend/hibernate,
+                        // the only path forward here is to attempt to resume the
+                        // GPU, accepting that the odds of success will vary.
+                        //
+                        if (rmStatus != NV_OK && pmAction != NV_PM_ACTION_RESUME)
+                        {
+                            RmPowerManagementTegra(pGpu, NV_PM_ACTION_RESUME);
+                        }
+                    }
+                    else
+                    {
+                        nv_priv_t *nvp = NV_GET_NV_PRIV(pNv);
+
+                        //
+                        // Before going to S3 or S4, remove idle holdoff which was
+                        // applied during gc6 exit.
+                        //
+                        if ((pmAction != NV_PM_ACTION_RESUME) &&
+                            (nvp->dynamic_power.b_idle_holdoff == NV_TRUE))
+                        {
+                            nv_indicate_idle(pNv);
+                            RmCancelCallbackToRemoveIdleHoldoff(pGpu);
+                            nvp->dynamic_power.b_idle_holdoff = NV_FALSE;
+                        }
+
+                        //
+                        // Use GCx (GCOFF/GC6) power management if S0ix-based PM is
+                        // enabled and the request is for system suspend/resume.
+                        // Otherwise, use the existing mechanism.
+                        //
+                        if (nvp->s0ix_pm_enabled &&
+                            (pmAction == NV_PM_ACTION_STANDBY ||
+                            (pmAction == NV_PM_ACTION_RESUME &&
+                             !nvp->pm_state.InHibernate)))
+                        {
+                            rmStatus = RmGcxPowerManagement(pGpu,
+                                            pmAction == NV_PM_ACTION_STANDBY,
+                                            NV_FALSE, &bTryAgain);
+
+                        }
+                        else
+                        {
+                            rmStatus = RmPowerManagement(pGpu, pmAction);
+                        }
+                    }
+
+                    // UNLOCK: release GPUs lock
+                    rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+                }
+                os_unref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE);
+            }
+        }
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    if (pNv->client_managed_console && !pNv->sysmem_mapped_console)
+    {
+        os_enable_console_access();
+    }
+
+    NV_ASSERT_OK(os_flush_work_queue(pNv->queue, pmAction != NV_PM_ACTION_RESUME));
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+/*!
+ * @brief Wrapper around RmGcxPowerManagement() and removes the Idle holdoff
+ * during exit sequence.
+ *
+ * The GPU lock should be held when calling this function.
+ *
+ * @param[in]   pGpu    OBJGPU pointer.
+ * @param[in]   bEnter  If true, perform entry sequence; exit sequence
+ *                      otherwise.
+ *
+ * @return      NV_STATUS code indicating if the operation was successful.
+ */
+static NV_STATUS RmTransitionDynamicPower(
+    OBJGPU *pGpu,
+    NvBool  bEnter,
+    NvBool *bTryAgain
+)
+{
+    nv_state_t *nv   = NV_GET_NV_STATE(pGpu);
+    NV_STATUS   status;
+
+    status = RmGcxPowerManagement(pGpu, bEnter, NV_TRUE, bTryAgain);
+
+    if (!bEnter && status == NV_OK)
+    {
+        nv_idle_holdoff(nv);
+        RmScheduleCallbackToRemoveIdleHoldoff(pGpu);
+    }
+
+    return status;
+}
+
+/*!
+ * @brief Wrapper around RmTransitionDynamicPower() which sets up the RM
+ * runtime (thread state and altstack), and takes the GPU lock for the duration
+ * of the operation.
+ *
+ * @param[in]   sp      nvidia_stack_t pointer.
+ * @param[in]   nv      nv_state_t pointer.
+ * @param[in]   bEnter  If true, perform entry sequence; exit sequence
+ *                      otherwise.
+ *
+ * @return      NV_STATUS code indicating if the operation was successful.
+ */
+NV_STATUS NV_API_CALL rm_transition_dynamic_power(
+    nvidia_stack_t *sp,
+    nv_state_t     *nv,
+    NvBool          bEnter,
+    NvBool         *bTryAgain
+)
+{
+    OBJGPU             *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    NV_STATUS           status = NV_OK;
+    THREAD_STATE_NODE   threadState;
+    void               *fp;
+
+    if (NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        return NV_OK;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if (!bEnter &&
+        !threadState.bValid &&
+        (threadState.threadId == 0) &&
+        pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_RG_SUPPORTED))
+    {
+        /*
+         * If the thread node is already present in the database,
+         * reset the thread timeout before the resume operation starts
+         * for IGPU.
+         */
+        threadStateResetTimeout(pGpu);
+    }
+
+    if (nv->client_managed_console && !nv->sysmem_mapped_console)
+    {
+        os_disable_console_access();
+    }
+
+    // LOCK: acquire GPUs lock
+    status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER);
+    if (status == NV_OK)
+    {
+        rmapiEnterRtd3PmPath();
+
+        status = RmTransitionDynamicPower(pGpu, bEnter, bTryAgain);
+
+        rmapiLeaveRtd3PmPath();
+
+        // UNLOCK: release GPUs lock
+        rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+    }
+
+    if (nv->client_managed_console && !nv->sysmem_mapped_console)
+    {
+        os_enable_console_access();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return status;
+}
+
+/*!
+ * @brief: Notify client about hot-plugged/unplugged displays.
+ * Called after GPU exits GC6 because of display hot-plug/unplug.
+ *
+ * @param[in]   nv    nv_state_t pointer.
+ */
+static void RmNotifyClientAboutHotplug(
+    nv_state_t *nv
+)
+{
+    NV_STATUS status;
+    GPU_MASK gpuMask;
+    OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    Nv2080HotplugNotification hotplugNotificationParams;
+
+    portMemSet(&hotplugNotificationParams, 0, sizeof(hotplugNotificationParams));
+
+    // LOCK: acquire per device lock
+    status = rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
+                                   GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HOTPLUG,
+                                   &gpuMask);
+    if (status == NV_OK)
+    {
+        gpuNotifySubDeviceEvent(pGpu, NV2080_NOTIFIERS_HOTPLUG,
+                                &hotplugNotificationParams,
+                                sizeof(hotplugNotificationParams), 0, 0);
+        // UNLOCK: release per device lock
+        rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+    }
+}
+
+/*!
+ * @brief If GPU is in GC6, exit GC6 state and
+ * notify client about display hot-plug/unplug.
+ *
+ * @param[in]   sp    nvidia_stack_t pointer.
+ * @param[in]   nv    nv_state_t pointer.
+ */
+void RmHandleDisplayChange(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    if (pGpu &&
+        (IS_GPU_GC6_STATE_ENTERED(pGpu) ||
+         pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED)))
+    {
+        if (rm_ref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE) == NV_OK)
+        {
+            RmNotifyClientAboutHotplug(nv);
+            rm_unref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE);
+        }
+        else
+            NV_PRINTF(LEVEL_ERROR,
+                      "Failed to increment dynamic power refcount\n");
+    }
+}
+
+/*!
+ * @brief RmGetDynamicBoostSupport() gets the notebook DB
+ * support. DB support is static information for vast majority
+ * of notebooks and hence we fetch it only once (during the first 
+ * call to this function). This way, the GPU remains in RTD3 state
+ * when procfs read is done.
+ *
+ *  @param[in]   nv    nv_state_t pointer.
+ */
+static const char* RmGetDynamicBoostSupport(
+    nv_state_t *pNv
+)
+{
+    nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+    const char* DbStatus = "?";
+
+    if (pNvp->db_supported == -1)
+    {
+        pNvp->db_supported = os_get_dynamic_boost_support(pNv);
+    }
+    
+    if (pNvp->db_supported == 0)
+    {
+        DbStatus = "Not Supported";
+    }
+    else if (pNvp->db_supported == 1)
+    { 
+        DbStatus = "Supported";
+    }
+    return DbStatus;
+}
+
+/*!
+ * @brief: Function to query Dynamic Power Management
+ *
+ * @param[in]   nvp     nv_priv_t pointer.
+ *
+ * @return      String indicating Dynamic Power Management status.
+ */
+static const char* RmGetDynamicPowerManagementStatus(
+        nv_priv_t *nvp
+)
+{
+   /*
+    * If the platform/driver does not support Dynamic Power Management,
+    * we set mode as NV_DYNAMIC_PM_NEVER. Hence, after RmInit if the
+    * mode is still NV_DYNAMIC_PM_FINE, we are sure that it is
+    * supported and enabled. Also see NOTE.
+    */
+    if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE)
+    {
+        return "Enabled (fine-grained)";
+    }
+    else if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_COARSE)
+    {
+        return "Enabled (coarse-grained)";
+    }
+    else if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_NEVER)
+    {
+
+        if (nvp->dynamic_power.dynamic_power_regkey ==
+            NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER)
+            return "Disabled";
+        else if (nvp->dynamic_power.dynamic_power_regkey ==
+                 NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT)
+            return "Disabled by default";
+    }
+
+    return "Not supported";
+}
+
+static void RmHandleIdleSustained(
+    NvU32 gpuInstance,
+    void *pArgs
+)
+{
+    OBJGPU     *pGpu = gpumgrGetGpu(gpuInstance);
+    nv_state_t *nv   = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp  = NV_GET_NV_PRIV(nv);
+
+    nv_revoke_gpu_mappings(NV_GET_NV_STATE(pGpu));
+    RmScheduleCallbackToIndicateIdle(pGpu);
+    nvp->dynamic_power.b_idle_sustained_workitem_queued = NV_FALSE;
+}
+
+/*
+ * Queue a workitem for revoking GPU mappings, and scheduling a callback to indicate idle.
+ */
+static void RmQueueIdleSustainedWorkitem(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv   = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp  = NV_GET_NV_PRIV(nv);
+    NV_STATUS status = NV_OK;
+
+    if (!nvp->dynamic_power.b_idle_sustained_workitem_queued)
+    {
+        status = osQueueWorkItem(pGpu,
+            RmHandleIdleSustained,
+            NULL,
+            (OsQueueWorkItemFlags){.bLockGpuGroupSubdevice = NV_TRUE});
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_WARNING,
+                      "Failed to queue RmHandleIdleSustained() workitem.\n");
+            RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu);
+            return;
+        }
+        nvp->dynamic_power.b_idle_sustained_workitem_queued = NV_TRUE;
+    }
+}
+
+/*
+ * Allocate resources needed for S0ix-based system power management.
+ */
+static void
+RmInitS0ixPowerManagement(
+    nv_state_t *nv,
+    NvBool bRtd3Support,
+    NvBool bGC6Support
+)
+{
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    NvU32       data;
+
+    // S0ix-based S2Idle, on desktops, is not supported yet. Return early for desktop SKUs
+    if (!nvp->b_mobile_config_enabled)
+    {
+        return;
+    }
+
+    /*
+     * The GPU S0ix-based system power management will be enabled
+     * only if all the following necessary requirements are met:
+     *
+     * 1. The GPU has RTD3 support.
+     * 2. The platform has support for s0ix.
+     * 3. Feature regkey EnableS0ixPowerManagement is enabled.
+     */
+    if (bRtd3Support && bGC6Support &&
+        nv_platform_supports_s0ix() &&
+        (osReadRegistryDword(NULL, NV_REG_ENABLE_S0IX_POWER_MANAGEMENT,
+                             &data) == NV_OK) && (data == 1))
+    {
+        nvp->s0ix_pm_enabled = NV_TRUE;
+
+        /*
+         * Read the OS registry for Maximum FB size for S0ix-based PM
+         * which will be expressed in Megabytes (1048576 bytes) and
+         * convert it into bytes.
+         */
+        if ((osReadRegistryDword(NULL,
+                                 NV_REG_S0IX_POWER_MANAGEMENT_VIDEO_MEMORY_THRESHOLD,
+                                 &data) == NV_OK))
+        {
+            nvp->s0ix_gcoff_max_fb_size = (NvU64)data * 1024 * 1024;
+        }
+    }
+}
+
+/*!
+ * @brief: Helper function to abstract the execution and return value verification
+ *         of PEX DSM methods, necessary for GC6 to work in Desktops.
+ *
+ * This function is capable to invoke three _DSM methods, as listed below:
+ *   0x0: _DSM for "Query supported functions"
+ *        (for more details, refer to section 9.1.1 of the ACPI 6.3 Specification).
+ *   0xA: _DSM for "Requesting D3cold Aux Power Limit"
+ *        (for more details, refer to section 4.6.10 of PCI Firmware Specification v3.3)
+ *   0xB: _DSM for "Adding PERST# Assertion Delay"
+ *        (for more details, refer to section 4.6.11 of PCI Firmware Specification v3.3)
+ *
+ * @param[in]   nv                 nv_state_t pointer.
+ * @param[in]   dsmSubFunc         subfunction to invoke.
+ * @param[in]   data               input data for dsmSubFunc.
+ *
+ * @return      TRUE if dsmSubFunc executed successfully and return value is as expected;
+ *              FALSE otherwise.
+ */
+static NvBool RmAcpiD3ColdDsm
+(
+    nv_state_t *nv,
+    NvU32       dsmSubFunc,
+    NvU32       data
+)
+{
+    NvU32 inData = data;
+
+    /*
+     * Third argument (i.e. acpiDsmRev) is sent as 4 because PEX_FUNC_AUXPOWERLIMIT and
+     * PEX_FUNC_PEXRST_DELAY sub-functions were introduced in PCI Firmware Specification 3.2
+     * and they are supported only for _DSM revision 4 onwards. But, RM still use _DSM
+     * revision 2 (see drivers/common/inc/pex.h). Hence, the need to hardcode 4 here.
+     */
+    if (nv_acpi_d3cold_dsm_for_upstream_port(nv, (NvU8 *)&PEX_DSM_GUID, 4, dsmSubFunc, &data) != NV_OK)
+    {
+        NV_PRINTF(LEVEL_NOTICE, "%s: PEX _DSM subfunction: 0x%X failed.\n", __FUNCTION__, dsmSubFunc);
+        return NV_FALSE;
+    }
+
+    switch (dsmSubFunc)
+    {
+        case NV_ACPI_ALL_FUNC_SUPPORT:
+            return ((data & NVBIT(PEX_FUNC_PEXRST_DELAY)) &&
+                    (data & NVBIT(PEX_FUNC_AUXPOWERLIMIT)));
+
+        case PEX_FUNC_PEXRST_DELAY:
+            return (inData == data);
+
+        case PEX_FUNC_AUXPOWERLIMIT:
+            return ((data == NV_AUX_POWER_REQUEST_STATUS_GRANTED_WITH_12V_POWER) ||
+                    (data == NV_AUX_POWER_REQUEST_STATUS_GRANTED_WITHOUT_12V_POWER));
+
+        default:
+            return NV_FALSE;
+    }
+
+    return NV_FALSE;
+}
+
+static NvBool RmConfigureUpstreamPortForRTD3(
+    nv_state_t *nv,
+    NvBool      bConfigureForGC6
+)
+{
+    nv_priv_t *nvp  = NV_GET_NV_PRIV(nv);
+    OBJGPU    *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    if (nvp->b_mobile_config_enabled)
+        return NV_TRUE;
+
+    if (!RmAcpiD3ColdDsm(nv, NV_ACPI_ALL_FUNC_SUPPORT, 0))
+    {
+        return NV_FALSE;
+    }
+
+    if (!RmAcpiD3ColdDsm(nv, PEX_FUNC_PEXRST_DELAY,
+                         bConfigureForGC6 ? pGpu->gc6State.GC6PerstDelay : DEFAULT_GCOFF_PEXRST_DELAY))
+    {
+        return NV_FALSE;
+    }
+
+    if (!RmAcpiD3ColdDsm(nv, PEX_FUNC_AUXPOWERLIMIT,
+                         bConfigureForGC6 ? pGpu->gc6State.GC6TotalBoardPower : 0))
+    {
+        if (bConfigureForGC6)
+            RmAcpiD3ColdDsm(nv, PEX_FUNC_PEXRST_DELAY, DEFAULT_GCOFF_PEXRST_DELAY);
+
+        return NV_FALSE;
+    }
+
+    NV_PRINTF(LEVEL_NOTICE,
+              "Aux Power and Pex delay settings %s successfully.\n",
+              bConfigureForGC6? "applied": "cleared");
+
+    return NV_TRUE;
+}
+
+void RmInitPowerManagement(
+    nv_state_t *nv
+)
+{
+    // LOCK: acquire GPUs lock
+    if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK)
+    {
+        NvBool bGC6Support   = NV_FALSE;
+        NvBool bGCOFFSupport = NV_FALSE;
+        NvBool bRtd3Support  = RmCheckRtd3GcxSupport(nv, &bGC6Support, &bGCOFFSupport);
+        nv_priv_t *nvp       = NV_GET_NV_PRIV(nv);
+        OBJGPU *pGpu         = NV_GET_NV_PRIV_PGPU(nv);
+
+        RmInitDeferredDynamicPowerManagement(nv, bRtd3Support);
+        RmInitS0ixPowerManagement(nv, bRtd3Support, bGC6Support);
+
+        if (bRtd3Support && bGC6Support &&
+            (nvp->s0ix_pm_enabled || (nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE)))
+        {
+            nvp->gc6_upstream_port_configured = RmConfigureUpstreamPortForRTD3(nv, NV_TRUE);
+        }
+
+        nv->is_pm_unsupported = pGpu->getProperty(pGpu, PDB_PROP_GPU_POWER_MANAGEMENT_UNSUPPORTED);
+
+        // UNLOCK: release GPUs lock
+        rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+    }
+}
+
+void RmDestroyPowerManagement(
+    nv_state_t *nv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+    if (nvp->gc6_upstream_port_configured)
+    {
+        RmConfigureUpstreamPortForRTD3(nv, NV_FALSE);
+    }
+
+    RmDestroyDeferredDynamicPowerManagement(nv);
+}
+
+void NV_API_CALL rm_get_power_info(
+    nvidia_stack_t  *sp,
+    nv_state_t      *pNv,
+    nv_power_info_t *powerInfo
+)
+{
+    THREAD_STATE_NODE threadState;
+    void              *fp;
+    GPU_MASK          gpuMask;
+    const char       *pVidmemPowerStatus = "?";
+    const char       *pGc6Supported = "?";
+    const char       *pGcoffSupported = "?";
+    const char       *pDynamicPowerStatus = "?";
+    const char       *pS0ixStatus = "?";
+    const char       *pDbStatus = "?";
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK)
+    {
+        OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+        // LOCK: acquire per device lock
+        if ((pGpu != NULL) &&
+           ((rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
+                                   GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER,
+                                   &gpuMask)) == NV_OK))
+        {
+            nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+
+            pVidmemPowerStatus = pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED) ?
+                                    "Off" : "Active";
+            pDynamicPowerStatus = RmGetDynamicPowerManagementStatus(pNvp);
+            pGc6Supported = RmGetGpuGcxSupport(pGpu, NV_TRUE);
+            pGcoffSupported = RmGetGpuGcxSupport(pGpu, NV_FALSE);
+            pS0ixStatus = pNvp->s0ix_pm_enabled ? "Enabled" : "Disabled";
+            pDbStatus = RmGetDynamicBoostSupport(pNv);
+            // UNLOCK: release per device lock
+            rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+        }
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    powerInfo->vidmem_power_status = pVidmemPowerStatus;
+    powerInfo->dynamic_power_status = pDynamicPowerStatus;
+    powerInfo->gc6_support = pGc6Supported;
+    powerInfo->gcoff_support = pGcoffSupported;
+    powerInfo->s0ix_status = pS0ixStatus;
+    powerInfo->db_support = pDbStatus;
+}
+
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/escape.c b/userspace/sdks/nv570/nvalloc/unix/src/escape.c
new file mode 100644
index 0000000..fae7b8d
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/escape.c
@@ -0,0 +1,1017 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+
+//***************************** Module Header **********************************
+//
+// This code is linked into the resource manager proper.  It receives the
+//    ioctl from the resource manager's customer, unbundles the args and
+//    calls the correct resman routines.
+//
+//******************************************************************************
+
+#include <core/prelude.h>
+#include <core/locks.h>
+#include <nv.h>
+#include <nv_escape.h>
+#include <osapi.h>
+#include <rmapi/exports.h>
+#include <nv-unix-nvos-params-wrappers.h>
+
+#include <nvos.h>
+#include <class/cl0000.h> // NV01_ROOT
+#include <class/cl0001.h> // NV01_ROOT_NON_PRIV
+#include <class/cl0005.h> // NV01_EVENT
+#include <class/cl003e.h> // NV01_MEMORY_SYSTEM
+#include <class/cl0071.h> // NV01_MEMORY_SYSTEM_OS_DESCRIPTOR
+
+#include "rmapi/client_resource.h"
+#include "nvlog/nvlog.h"
+#include <nv-ioctl-lockless-diag.h>
+
+#include <ctrl/ctrl00fd.h>
+
+#include <ctrl/ctrl00e0.h>
+
+#define NV_CTL_DEVICE_ONLY(nv)                 \
+{                                              \
+    if (((nv)->flags & NV_FLAG_CONTROL) == 0)  \
+    {                                          \
+        rmStatus = NV_ERR_INVALID_ARGUMENT;    \
+        goto done;                             \
+    }                                          \
+}
+
+#define NV_ACTUAL_DEVICE_ONLY(nv)              \
+{                                              \
+    if (((nv)->flags & NV_FLAG_CONTROL) != 0)  \
+    {                                          \
+        rmStatus = NV_ERR_INVALID_ARGUMENT;    \
+        goto done;                             \
+    }                                          \
+}
+
+static NV_STATUS RmGetDeviceFd(NVOS54_PARAMETERS *pApi, NvS32 *pFd,
+                               NvBool *pSkipDeviceRef)
+{
+    RMAPI_PARAM_COPY paramCopy;
+    void *pKernelParams;
+    NvU32 paramSize;
+    NV_STATUS status;
+
+    *pFd = -1;
+    *pSkipDeviceRef = NV_TRUE;
+
+    switch(pApi->cmd)
+    {
+        case NV00FD_CTRL_CMD_ATTACH_GPU:
+            paramSize = sizeof(NV00FD_CTRL_ATTACH_GPU_PARAMS);
+            break;
+        case NV00E0_CTRL_CMD_EXPORT_MEM:
+            paramSize = sizeof(NV00E0_CTRL_EXPORT_MEM_PARAMS);
+            break;
+        default:
+            return NV_OK;
+    }
+
+    RMAPI_PARAM_COPY_INIT(paramCopy, pKernelParams, pApi->params, paramSize, 1);
+
+    status = rmapiParamsAcquire(&paramCopy, NV_TRUE);
+    if (status != NV_OK)
+        return status;
+
+    switch(pApi->cmd)
+    {
+        case NV00FD_CTRL_CMD_ATTACH_GPU:
+            {
+                NV00FD_CTRL_ATTACH_GPU_PARAMS *pAttachGpuParams = pKernelParams;
+
+                *pSkipDeviceRef = NV_FALSE;
+                 *pFd = (NvS32)pAttachGpuParams->devDescriptor;
+            }
+            break;
+        case NV00E0_CTRL_CMD_EXPORT_MEM:
+            {
+                NV00E0_CTRL_EXPORT_MEM_PARAMS *pExportMemParams = pKernelParams;
+
+                // If hParent is client, no need to reference device.
+                *pSkipDeviceRef = (pExportMemParams->hParent == pApi->hClient);
+                *pFd = (NvS32)pExportMemParams->devDescriptor;
+            }
+            break;
+        default:
+            NV_ASSERT(0);
+            break;
+    }
+
+    NV_ASSERT(rmapiParamsRelease(&paramCopy) == NV_OK);
+
+    return status;
+}
+
+// Only return errors through pApi->status
+static void RmCreateOsDescriptor(NVOS32_PARAMETERS *pApi, API_SECURITY_INFO secInfo)
+{
+    NV_STATUS rmStatus;
+    NvBool writable;
+    NvU32 flags = 0;
+    NvU64 allocSize, pageCount, *pPteArray = NULL;
+    void *pDescriptor, *pPageArray = NULL;
+    NVOS32_PARAMETERS allocApi = *pApi;
+    NVOS32_PARAMETERS *pAllocApi = &allocApi;
+
+    pDescriptor = NvP64_VALUE(pApi->data.AllocOsDesc.descriptor);
+    if (((NvUPtr)pDescriptor & ~os_page_mask) != 0)
+    {
+        rmStatus = NV_ERR_NOT_SUPPORTED;
+        goto done;
+    }
+
+    // Check to prevent an NvU64 overflow
+    if ((pApi->data.AllocOsDesc.limit + 1) == 0)
+    {
+        rmStatus = NV_ERR_INVALID_LIMIT;
+        goto done;
+    }
+
+    allocSize = (pApi->data.AllocOsDesc.limit + 1);
+    pageCount = (1 + ((allocSize - 1) / os_page_size));
+
+    writable = FLD_TEST_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_WRITE, pApi->data.AllocOsDesc.attr2);
+
+    flags = FLD_SET_DRF_NUM(_LOCK_USER_PAGES, _FLAGS, _WRITE, writable, flags);
+    rmStatus = os_lock_user_pages(pDescriptor, pageCount, &pPageArray, flags);
+    if (rmStatus == NV_OK)
+    {
+        pAllocApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPageArray;
+        pAllocApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY;
+    }
+    else if (rmStatus == NV_ERR_INVALID_ADDRESS)
+    {
+        rmStatus = os_lookup_user_io_memory(pDescriptor, pageCount, &pPteArray);
+        if (rmStatus == NV_OK)
+        {
+            pAllocApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPteArray;
+            pAllocApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY;
+        }
+    }
+    if (rmStatus != NV_OK)
+        goto done;
+
+    Nv04VidHeapControlWithSecInfo(pAllocApi, secInfo);
+
+    pApi->status = pAllocApi->status;
+    pApi->data.AllocOsDesc.hMemory = pAllocApi->data.AllocOsDesc.hMemory;
+
+    if ((pAllocApi->status != NV_OK) &&
+        (pAllocApi->data.AllocOsDesc.descriptor != NvP64_NULL))
+    {
+        switch (pAllocApi->data.AllocOsDesc.descriptorType)
+        {
+            default:
+                break;
+            case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY:
+                os_unlock_user_pages(pageCount, pPageArray, flags);
+                break;
+        }
+    }
+
+done:
+    if (rmStatus != NV_OK)
+        pApi->status = rmStatus;
+}
+
+// Only return errors through pApi->status
+static void RmAllocOsDescriptor(NVOS02_PARAMETERS *pApi, API_SECURITY_INFO secInfo)
+{
+    NV_STATUS rmStatus = NV_OK;
+    NvU32 flags, attr, attr2;
+    NVOS32_PARAMETERS *pVidHeapParams;
+
+    if (!FLD_TEST_DRF(OS02, _FLAGS, _LOCATION, _PCI, pApi->flags) ||
+        !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NO_MAP, pApi->flags))
+    {
+        rmStatus = NV_ERR_INVALID_FLAGS;
+        goto done;
+    }
+
+    attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI);
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, pApi->flags) ||
+        FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, pApi->flags))
+    {
+        attr = FLD_SET_DRF(OS32, _ATTR, _COHERENCY, _WRITE_BACK, attr);
+    }
+    else if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, pApi->flags))
+        attr = FLD_SET_DRF(OS32, _ATTR, _COHERENCY, _UNCACHED, attr);
+    else {
+        rmStatus = NV_ERR_INVALID_FLAGS;
+        goto done;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, pApi->flags))
+        attr = FLD_SET_DRF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS, attr);
+    else
+        attr = FLD_SET_DRF(OS32, _ATTR, _PHYSICALITY, _NONCONTIGUOUS, attr);
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, pApi->flags))
+        attr2 = DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _YES);
+    else
+        attr2 = DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _NO);
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _MEMORY_PROTECTION, _UNPROTECTED, pApi->flags))
+        attr2 = FLD_SET_DRF(OS32, _ATTR2, _MEMORY_PROTECTION, _UNPROTECTED, attr2);
+
+    pVidHeapParams = portMemAllocNonPaged(sizeof(NVOS32_PARAMETERS));
+    if (pVidHeapParams == NULL)
+    {
+        rmStatus = NV_ERR_NO_MEMORY;
+        goto done;
+    }
+    portMemSet(pVidHeapParams, 0, sizeof(NVOS32_PARAMETERS));
+
+    pVidHeapParams->hRoot = pApi->hRoot;
+    pVidHeapParams->hObjectParent = pApi->hObjectParent;
+    pVidHeapParams->function = NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR;
+
+    flags = (NVOS32_ALLOC_FLAGS_MEMORY_HANDLE_PROVIDED |
+             NVOS32_ALLOC_FLAGS_MAP_NOT_REQUIRED);
+
+    if (DRF_VAL(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, pApi->flags))
+        attr2 = FLD_SET_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_ONLY, attr2);
+
+    // Currently CPU-RO memory implies GPU-RO as well
+    if (DRF_VAL(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, pApi->flags) ||
+        DRF_VAL(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, pApi->flags))
+        attr2 = FLD_SET_DRF(OS32, _ATTR2, _PROTECTION_DEVICE, _READ_ONLY, attr2);
+
+    pVidHeapParams->data.AllocOsDesc.hMemory = pApi->hObjectNew;
+    pVidHeapParams->data.AllocOsDesc.flags = flags;
+    pVidHeapParams->data.AllocOsDesc.attr = attr;
+    pVidHeapParams->data.AllocOsDesc.attr2 = attr2;
+    pVidHeapParams->data.AllocOsDesc.descriptor = pApi->pMemory;
+    pVidHeapParams->data.AllocOsDesc.limit = pApi->limit;
+    pVidHeapParams->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS;
+
+    RmCreateOsDescriptor(pVidHeapParams, secInfo);
+
+    pApi->status = pVidHeapParams->status;
+
+    portMemFree(pVidHeapParams);
+
+done:
+    if (rmStatus != NV_OK)
+        pApi->status = rmStatus;
+}
+
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hRoot) == NV_OFFSETOF(NVOS64_PARAMETERS, hRoot));
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hObjectParent) == NV_OFFSETOF(NVOS64_PARAMETERS, hObjectParent));
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hObjectNew) == NV_OFFSETOF(NVOS64_PARAMETERS, hObjectNew));
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, hClass) == NV_OFFSETOF(NVOS64_PARAMETERS, hClass));
+ct_assert(NV_OFFSETOF(NVOS21_PARAMETERS, pAllocParms) == NV_OFFSETOF(NVOS64_PARAMETERS, pAllocParms));
+
+
+NV_STATUS RmValidateIoctl(NvU32 cmd, NvU32 size)
+{
+    int i = 0;
+    const unsigned int arg_cmd = (cmd & 0xFF);
+    static const struct {
+        unsigned int cmdKey;
+        unsigned int paramSize;
+    } RmIoctlsTable[] = {
+    
+    #define _RM_ESC_IOCTL_ENTRY(_cmd, _type) \
+        { .cmdKey = ((_cmd) & 0xFF), .paramSize = sizeof(_type) }
+
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_STATUS_CODE, nv_ioctl_status_code_t),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_ALLOC_MEMORY, nv_ioctl_nvos02_parameters_with_fd),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_ALLOC_OBJECT, NVOS05_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_FREE, NVOS00_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_CONTROL, NVOS54_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_DUP_OBJECT, NVOS55_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_SHARE, NVOS57_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_I2C_ACCESS, NVOS_I2C_ACCESS_PARAMS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_IDLE_CHANNELS, NVOS30_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_VID_HEAP_CONTROL, NVOS32_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_ACCESS_REGISTRY, NVOS38_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_MAP_MEMORY, nv_ioctl_nvos33_parameters_with_fd),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_UNMAP_MEMORY, NVOS34_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_ALLOC_CONTEXT_DMA2, NVOS39_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_MAP_MEMORY_DMA, NVOS46_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_UNMAP_MEMORY_DMA, NVOS47_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_BIND_CONTEXT_DMA, NVOS49_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO, NVOS56_PARAMETERS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_RM_LOCKLESS_DIAGNOSTIC, NV_LOCKLESS_DIAGNOSTIC_PARAMS),
+        _RM_ESC_IOCTL_ENTRY(NV_ESC_REGISTER_FD, nv_ioctl_register_fd_t),
+    };
+
+    if (arg_cmd == NV_ESC_RM_ALLOC)
+    {
+        if (size == sizeof(NVOS64_PARAMETERS) || size == sizeof(NVOS21_PARAMETERS))
+        {
+            return NV_OK;
+        }
+        else
+        {
+            nv_printf(NV_DBG_ERRORS, "NVRM: invalid %d structure size!\n", arg_cmd);
+
+            return NV_ERR_INVALID_ARGUMENT;
+        }
+    }
+
+    for (i = 0; i < NV_ARRAY_ELEMENTS(RmIoctlsTable); ++i)
+    {
+        if (arg_cmd == RmIoctlsTable[i].cmdKey)
+        {
+            if (size != RmIoctlsTable[i].paramSize)
+            {
+                nv_printf(NV_DBG_ERRORS, "NVRM: invalid %d structure size, expected %d, got %d!\n",
+                          RmIoctlsTable[i].cmdKey, RmIoctlsTable[i].paramSize, size);
+                // If IOCTL is found and size is mismatched, return NV_ERR_INVALID_ARGUMENT
+                return NV_ERR_INVALID_ARGUMENT;
+            }
+            // If IOCTL is found and size is valid, return NV_OK
+            return NV_OK;
+        }
+    }
+    // If IOCTL is not found, return NV_ERR_INVALID_COMMAND
+    return NV_ERR_INVALID_COMMAND;
+}
+
+NV_STATUS RmIoctl(
+    nv_state_t  *nv,
+    nv_file_private_t *nvfp,
+    NvU32        cmd,
+    void        *data,
+    NvU32        dataSize
+)
+{
+    // nvfp->ctl_nvfp
+    ct_assert(sizeof(PORT_ATOMIC nv_file_private_t *) == sizeof(nv_file_private_t *));
+    ct_assert(_Alignof(PORT_ATOMIC nv_file_private_t *) == _Alignof(nv_file_private_t *));
+
+    // nvfp->register_or_refcount
+    ct_assert(sizeof(PORT_ATOMIC NvU32) == sizeof(NvU32));
+    ct_assert(_Alignof(PORT_ATOMIC NvU32) == _Alignof(NvU32));
+
+    NV_STATUS            rmStatus = NV_ERR_GENERIC;
+    API_SECURITY_INFO    secInfo = { };
+
+    secInfo.privLevel = osIsAdministrator() ? RS_PRIV_LEVEL_USER_ROOT : RS_PRIV_LEVEL_USER;
+    secInfo.paramLocation = PARAM_LOCATION_USER;
+    secInfo.pProcessToken = NULL;
+    secInfo.gpuOsInfo = NULL;
+    secInfo.clientOSInfo = nvfp->ctl_nvfp;
+    if (secInfo.clientOSInfo == NULL)
+        secInfo.clientOSInfo = nvfp;
+
+    switch (cmd)
+    {
+        case NV_ESC_RM_ALLOC_MEMORY:
+        {
+            nv_ioctl_nvos02_parameters_with_fd *pApi;
+            NVOS02_PARAMETERS *pParms;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            pApi = data;
+            pParms = &pApi->params;
+
+            NV_ACTUAL_DEVICE_ONLY(nv);
+
+            if (pParms->hClass == NV01_MEMORY_SYSTEM_OS_DESCRIPTOR)
+                RmAllocOsDescriptor(pParms, secInfo);
+            else
+            {
+                NvU32 flags = pParms->flags;
+
+                Nv01AllocMemoryWithSecInfo(pParms, secInfo);
+
+                //
+                // If the system memory is going to be mapped immediately,
+                // create the mmap context for it now.
+                //
+                if ((pParms->hClass == NV01_MEMORY_SYSTEM) &&
+                    (!FLD_TEST_DRF(OS02, _FLAGS, _ALLOC, _NONE, flags)) &&
+                    (!FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NO_MAP, flags)) &&
+                    (pParms->status == NV_OK))
+                {
+                    if (rm_create_mmap_context(pParms->hRoot,
+                            pParms->hObjectParent, pParms->hObjectNew,
+                            pParms->pMemory, pParms->limit + 1, 0,
+                            NV_MEMORY_DEFAULT,
+                            pApi->fd) != NV_OK)
+                    {
+                        NV_PRINTF(LEVEL_WARNING,
+                                  "could not create mmap context for %p\n",
+                                  NvP64_VALUE(pParms->pMemory));
+                        rmStatus = NV_ERR_INVALID_ARGUMENT;
+                        goto done;
+                    }
+                }
+            }
+
+            break;
+        }
+
+        case NV_ESC_RM_ALLOC_OBJECT:
+        {
+            NVOS05_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv01AllocObjectWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_ALLOC:
+        {
+            NVOS21_PARAMETERS *pApi = data;
+            NVOS64_PARAMETERS *pApiAccess = data;
+            NvBool bAccessApi = (dataSize == sizeof(NVOS64_PARAMETERS));
+
+            if ((dataSize != sizeof(*pApi)) &&
+                (dataSize != sizeof(*pApiAccess)))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            switch (pApi->hClass)
+            {
+                case NV01_ROOT:
+                case NV01_ROOT_CLIENT:
+                case NV01_ROOT_NON_PRIV:
+                {
+                    NV_CTL_DEVICE_ONLY(nv);
+
+                    // Force userspace client allocations to be the _CLIENT class.
+                    pApi->hClass = NV01_ROOT_CLIENT;
+                    break;
+                }
+                case NV01_EVENT:
+                case NV01_EVENT_OS_EVENT:
+                case NV01_EVENT_KERNEL_CALLBACK:
+                case NV01_EVENT_KERNEL_CALLBACK_EX:
+                {
+                    break;
+                }
+                default:
+                {
+                    NV_CTL_DEVICE_ONLY(nv);
+                    break;
+                }
+            }
+
+            if (!bAccessApi)
+            {
+                Nv04AllocWithSecInfo(pApi, secInfo);
+            }
+            else
+            {
+                Nv04AllocWithAccessSecInfo(pApiAccess, secInfo);
+            }
+
+            break;
+        }
+
+        case NV_ESC_RM_FREE:
+        {
+            NVOS00_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv01FreeWithSecInfo(pApi, secInfo);
+
+            if (pApi->status == NV_OK &&
+                pApi->hObjectOld == pApi->hRoot)
+            {
+                rm_client_free_os_events(pApi->hRoot);
+            }
+
+            break;
+        }
+
+        case NV_ESC_RM_VID_HEAP_CONTROL:
+        {
+            NVOS32_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            if (pApi->function == NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR)
+                RmCreateOsDescriptor(pApi, secInfo);
+            else
+                Nv04VidHeapControlWithSecInfo(pApi, secInfo);
+
+            break;
+        }
+
+        case NV_ESC_RM_I2C_ACCESS:
+        {
+            NVOS_I2C_ACCESS_PARAMS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_ACTUAL_DEVICE_ONLY(nv);
+
+            Nv04I2CAccessWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_IDLE_CHANNELS:
+        {
+            NVOS30_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv04IdleChannelsWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_MAP_MEMORY:
+        {
+            nv_ioctl_nvos33_parameters_with_fd *pApi;
+            NVOS33_PARAMETERS *pParms;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            pApi = data;
+            pParms = &pApi->params;
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            // Don't allow userspace to override the caching type
+            pParms->flags = FLD_SET_DRF(OS33, _FLAGS, _CACHING_TYPE, _DEFAULT, pParms->flags);
+            Nv04MapMemoryWithSecInfo(pParms, secInfo);
+
+            if (pParms->status == NV_OK)
+            {
+                pParms->status = rm_create_mmap_context(pParms->hClient,
+                                 pParms->hDevice, pParms->hMemory,
+                                 pParms->pLinearAddress, pParms->length,
+                                 pParms->offset,
+                                 DRF_VAL(OS33, _FLAGS, _CACHING_TYPE, pParms->flags),
+                                 pApi->fd);
+                if (pParms->status != NV_OK)
+                {
+                    NVOS34_PARAMETERS params;
+                    portMemSet(&params, 0, sizeof(params));
+                    params.hClient        = pParms->hClient;
+                    params.hDevice        = pParms->hDevice;
+                    params.hMemory        = pParms->hMemory;
+                    params.pLinearAddress = pParms->pLinearAddress;
+                    params.flags          = pParms->flags;
+                    Nv04UnmapMemoryWithSecInfo(&params, secInfo);
+                }
+            }
+            break;
+        }
+
+        case NV_ESC_RM_UNMAP_MEMORY:
+        {
+            NVOS34_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv04UnmapMemoryWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_ACCESS_REGISTRY:
+        {
+            NVOS38_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            pApi->status = rm_access_registry(pApi->hClient,
+                                              pApi->hObject,
+                                              pApi->AccessType,
+                                              pApi->pDevNode,
+                                              pApi->DevNodeLength,
+                                              pApi->pParmStr,
+                                              pApi->ParmStrLength,
+                                              pApi->pBinaryData,
+                                              &pApi->BinaryDataLength,
+                                              &pApi->Data,
+                                              &pApi->Entry);
+            break;
+        }
+
+        case NV_ESC_RM_ALLOC_CONTEXT_DMA2:
+        {
+            NVOS39_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv04AllocContextDmaWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_BIND_CONTEXT_DMA:
+        {
+            NVOS49_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv04BindContextDmaWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_MAP_MEMORY_DMA:
+        {
+            NVOS46_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv04MapMemoryDmaWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_UNMAP_MEMORY_DMA:
+        {
+            NVOS47_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv04UnmapMemoryDmaWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_DUP_OBJECT:
+        {
+            NVOS55_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv04DupObjectWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_RM_SHARE:
+        {
+            NVOS57_PARAMETERS *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            Nv04ShareWithSecInfo(pApi, secInfo);
+            break;
+        }
+
+        case NV_ESC_STATUS_CODE:
+        {
+            nv_state_t *pNv;
+            nv_ioctl_status_code_t *pApi = data;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            pNv = nv_get_adapter_state(pApi->domain, pApi->bus, pApi->slot);
+            if (pNv == NULL)
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            rmStatus = rm_get_adapter_status(pNv, &pApi->status);
+
+            if (rmStatus != NV_OK)
+                goto done;
+
+            break;
+        }
+
+        case NV_ESC_RM_CONTROL:
+        {
+            NVOS54_PARAMETERS *pApi = data;
+            void *priv = NULL;
+            nv_file_private_t *dev_nvfp = NULL;
+            NvS32 fd;
+            NvBool bSkipDeviceRef;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            rmStatus = RmGetDeviceFd(pApi, &fd, &bSkipDeviceRef);
+            if (rmStatus != NV_OK)
+            {
+                goto done;
+            }
+
+            if (!bSkipDeviceRef)
+            {
+                dev_nvfp = nv_get_file_private(fd, NV_FALSE, &priv);
+                if (dev_nvfp == NULL)
+                {
+                    rmStatus = NV_ERR_INVALID_DEVICE;
+                    goto done;
+                }
+
+                // Check to avoid cyclic dependency with NV_ESC_REGISTER_FD
+                if (!portAtomicCompareAndSwapU32(&dev_nvfp->register_or_refcount,
+                                                 NVFP_TYPE_REFCOUNTED,
+                                                 NVFP_TYPE_NONE))
+                {
+                    // Is this already refcounted...
+                    if (dev_nvfp->register_or_refcount != NVFP_TYPE_REFCOUNTED)
+                    {
+                        nv_put_file_private(priv);
+                        rmStatus = NV_ERR_IN_USE;
+                        goto done;
+                    }
+                }
+
+                secInfo.gpuOsInfo = priv;
+            }
+
+            Nv04ControlWithSecInfo(pApi, secInfo);
+
+            if ((pApi->status != NV_OK) && (priv != NULL))
+            {
+                //
+                // No need to reset `register_or_refcount` as it might be set
+                // for previous successful calls. We let it clear with FD close.
+                //
+                nv_put_file_private(priv);
+
+                secInfo.gpuOsInfo = NULL;
+            }
+
+            break;
+        }
+
+        case NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO:
+        {
+            NVOS56_PARAMETERS *pApi = data;
+            void *pOldCpuAddress;
+            void *pNewCpuAddress;
+
+            if (dataSize != sizeof(*pApi))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            pOldCpuAddress = NvP64_VALUE(pApi->pOldCpuAddress);
+            pNewCpuAddress = NvP64_VALUE(pApi->pNewCpuAddress);
+
+            pApi->status = rm_update_device_mapping_info(pApi->hClient,
+                                                         pApi->hDevice,
+                                                         pApi->hMemory,
+                                                         pOldCpuAddress,
+                                                         pNewCpuAddress);
+            break;
+        }
+
+        case NV_ESC_RM_LOCKLESS_DIAGNOSTIC:
+        {
+            NV_LOCKLESS_DIAGNOSTIC_PARAMS *pParams = data;
+
+            if (dataSize != sizeof(*pParams))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            NV_CTL_DEVICE_ONLY(nv);
+
+            if (!osIsAdministrator())
+            {
+                rmStatus = NV_ERR_INSUFFICIENT_PERMISSIONS;
+                goto done;
+            }
+
+            switch (pParams->cmd)
+            {
+                // Do not use NVOC _DISPATCH here as it dereferences NULL RmClientResource*
+                case NV0000_CTRL_CMD_NVD_GET_NVLOG_INFO:
+                    rmStatus = cliresCtrlCmdNvdGetNvlogInfo_IMPL(NULL, &pParams->params.getNvlogInfo);
+                    break;
+                case NV0000_CTRL_CMD_NVD_GET_NVLOG_BUFFER_INFO:
+                    rmStatus = cliresCtrlCmdNvdGetNvlogBufferInfo_IMPL(NULL, &pParams->params.getNvlogBufferInfo);
+                    break;
+                case NV0000_CTRL_CMD_NVD_GET_NVLOG:
+                    rmStatus = cliresCtrlCmdNvdGetNvlog_IMPL(NULL, &pParams->params.getNvlog);
+                    break;
+                default:
+                    rmStatus = NV_ERR_NOT_SUPPORTED;
+                    break;
+            }
+
+            pParams->status = rmStatus;
+            goto done;
+        }
+
+        case NV_ESC_REGISTER_FD:
+        {
+            nv_ioctl_register_fd_t *params = data;
+            void *priv = NULL;
+            nv_file_private_t *ctl_nvfp;
+
+            if (dataSize != sizeof(*params))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            // LOCK: acquire API lock
+            rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+            if (rmStatus != NV_OK)
+                goto done;
+
+            // If there is already a ctl fd registered on this nvfp, fail.
+            if (nvfp->ctl_nvfp != NULL)
+            {
+                // UNLOCK: release API lock
+                rmapiLockRelease();
+                rmStatus = NV_ERR_INVALID_STATE;
+                goto done;
+            }
+
+            //
+            // Note that this call is valid for both "actual" devices and ctrl
+            // devices.  In particular, NV_ESC_ALLOC_OS_EVENT can be used with
+            // both types of devices.
+            // But, the ctl_fd passed in should always correspond to a control FD.
+            //
+            ctl_nvfp = nv_get_file_private(params->ctl_fd,
+                                           NV_TRUE, /* require ctl fd */
+                                           &priv);
+            if (ctl_nvfp == NULL)
+            {
+                // UNLOCK: release API lock
+                rmapiLockRelease();
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            // Disallow self-referential links, and disallow links to FDs that
+            // themselves have a link.
+            if ((ctl_nvfp == nvfp) || (ctl_nvfp->ctl_nvfp != NULL))
+            {
+                nv_put_file_private(priv);
+                // UNLOCK: release API lock
+                rmapiLockRelease();
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                goto done;
+            }
+
+            // Check to avoid cyclic dependency with device refcounting
+            if (!portAtomicCompareAndSwapU32(&nvfp->register_or_refcount,
+                                             NVFP_TYPE_REGISTERED,
+                                             NVFP_TYPE_NONE))
+            {
+                nv_put_file_private(priv);
+                // UNLOCK: release API lock
+                rmapiLockRelease();
+                rmStatus = NV_ERR_IN_USE;
+                goto done;
+            }
+
+            //
+            // nvfp->ctl_nvfp is read outside the lock, so set it atomically.
+            // Note that once set, this can never be removed until the fd
+            // associated with nvfp is closed.  We hold on to 'priv' until the
+            // fd is closed, too, to ensure that the fd associated with
+            // ctl_nvfp remains valid.
+            //
+            portAtomicSetSize(&nvfp->ctl_nvfp, ctl_nvfp);
+            nvfp->ctl_nvfp_priv = priv;
+
+            // UNLOCK: release API lock
+            rmapiLockRelease();
+
+            // NOTE: nv_put_file_private(priv) is not called here.  It MUST be
+            // called during cleanup of this nvfp.
+            rmStatus = NV_OK;
+            break;
+        }
+
+        default:
+        {
+            NV_PRINTF(LEVEL_ERROR, "unknown NVRM ioctl command: 0x%x\n", cmd);
+            goto done;
+        }
+    }
+
+    rmStatus = NV_OK;
+done:
+
+    return rmStatus;
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/exports-stubs.c b/userspace/sdks/nv570/nvalloc/unix/src/exports-stubs.c
new file mode 100644
index 0000000..ff40300
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/exports-stubs.c
@@ -0,0 +1,30 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include "os/os.h"
+#include <rmconfig.h>
+#include <gpu/subdevice/subdevice.h>
+#include <ctrl/ctrl0080/ctrl0080unix.h>
+#include <ctrl/ctrl2080/ctrl2080unix.h>
+
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/gcc_helper.c b/userspace/sdks/nv570/nvalloc/unix/src/gcc_helper.c
new file mode 100644
index 0000000..3c1037c
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/gcc_helper.c
@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvtypes.h>
+#include <os-interface.h>
+
+void* memset(void* s, int c, NvUPtr n)
+{
+    return os_mem_set(s, (NvU8)c, (NvU32)n);
+}
+
+void* memcpy(void* dest, const void* src, NvUPtr n)
+{
+    return os_mem_copy(dest, src, (NvU32)n);
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/os-hypervisor.c b/userspace/sdks/nv570/nvalloc/unix/src/os-hypervisor.c
new file mode 100644
index 0000000..26c468f
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/os-hypervisor.c
@@ -0,0 +1,994 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2014-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ *
+ * @file os-hypervisor.c
+ * @brief OS specific Hypervisor interfaces for RM
+ *
+ */
+
+#include "os/os.h"
+#include "nv.h"
+#include "nv-priv.h"
+#include <nvrm_registry.h>
+#include <virtualization/hypervisor/hypervisor.h>
+#include "core/thread_state.h"
+#include "core/locks.h"
+#include "virtualization/kernel_vgpu_mgr.h"
+#include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
+#include "kernel/gpu/fifo/kernel_fifo.h"
+#include "osapi.h"
+#include "virtualization/kernel_hostvgpudeviceapi.h"
+#include "gpu/timer/objtmr.h"
+#include "gpu/bif/kernel_bif.h"
+#include "gpu/bus/kern_bus.h"
+#include <nv_ref.h>               // NV_PMC_BOOT_1_VGPU
+#include "nvdevid.h"
+#include "ctrl/ctrl0000/ctrl0000vgpu.h"
+
+#include "g_vgpu_chip_flags.h"    // vGPU device names
+
+#define NV_VFIO_PCI_BAR0_REGION_INDEX 0
+#define NV_VFIO_PCI_BAR1_REGION_INDEX 1
+#define NV_VFIO_PCI_BAR2_REGION_INDEX 2
+#define NV_VFIO_PCI_BAR3_REGION_INDEX 3
+
+static NV_STATUS nv_parse_config_params(const char *, const char *, const char, NvU32 *);
+
+
+void hypervisorSetHypervVgpuSupported_IMPL(OBJHYPERVISOR *pHypervisor)
+{
+    pHypervisor->bIsHypervVgpuSupported = NV_TRUE;
+}
+
+NvBool hypervisorIsVgxHyper_IMPL(void)
+{
+    return os_is_vgx_hyper();
+}
+
+NV_STATUS hypervisorInjectInterrupt_IMPL
+(
+    OBJHYPERVISOR *pHypervisor,
+    VGPU_NS_INTR   *pVgpuNsIntr
+)
+{
+    NV_STATUS status = NV_ERR_NOT_SUPPORTED;
+
+    if (osIsVgpuVfioPresent() == NV_OK)
+        return NV_ERR_NOT_SUPPORTED;
+    else
+    {
+        if (pVgpuNsIntr->guestMSIAddr && pVgpuNsIntr->guestMSIData)
+        {
+            status = os_inject_vgx_msi((NvU16)pVgpuNsIntr->guestDomainId,
+                                       pVgpuNsIntr->guestMSIAddr,
+                                       pVgpuNsIntr->guestMSIData);
+        }
+    }
+
+    return status;
+}
+
+HYPERVISOR_TYPE NV_API_CALL nv_get_hypervisor_type(void)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
+    return hypervisorGetHypervisorType(pHypervisor);
+}
+
+#define MAX_STR_LEN 256
+NV_STATUS  NV_API_CALL nv_vgpu_get_type_info(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    NvU32 vgpuTypeId,
+    char *buffer,
+    int type_info,
+    NvU8 devfn
+)
+{
+    THREAD_STATE_NODE threadState;
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+    NV_STATUS rmStatus = NV_OK;
+    VGPU_TYPE *vgpuTypeInfo;
+    NvU32 pgpuIndex, i, avail_instances = 0;
+    OBJGPU *pGpu = NULL;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+    {
+        if ((rmStatus = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex)) ==
+            NV_OK)
+        {
+            for (i = 0; i < MAX_VGPU_TYPES_PER_PGPU; i++)
+            {
+                vgpuTypeInfo = pKernelVgpuMgr->pgpuInfo[pgpuIndex].vgpuTypes[i];
+                if (vgpuTypeInfo == NULL)
+                    break;
+
+                if (vgpuTypeInfo->vgpuTypeId != vgpuTypeId)
+                    continue;
+
+                switch (type_info)
+                {
+                    case VGPU_TYPE_INSTANCES:
+                        pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+                        if (pGpu == NULL)
+                        {
+                            NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n",
+                                      __FUNCTION__);
+                            rmStatus = NV_ERR_INVALID_STATE;
+                            goto exit;
+                        }
+
+                        rmStatus = kvgpumgrGetAvailableInstances(&avail_instances, pGpu,
+                                                                 vgpuTypeInfo,
+                                                                 pgpuIndex, devfn);
+                        if (rmStatus != NV_OK)
+                            goto exit;
+
+                        os_snprintf(buffer, MAX_STR_LEN, "%d\n", avail_instances);
+                        break;
+                    default:
+                        rmStatus = NV_ERR_INVALID_ARGUMENT;
+                }
+                break;
+            }
+        }
+
+exit:
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_get_type_ids(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    NvU32 *numVgpuTypes,
+    NvU32 *vgpuTypeIds,
+    NvBool isVirtfn,
+    NvU8 devfn,
+    NvBool getCreatableTypes
+)
+{
+    THREAD_STATE_NODE threadState;
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJGPU *pGpu = NULL;
+    KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+    NV_STATUS rmStatus = NV_OK;
+    NvU32 pgpuIndex, i, avail_instances = 0;
+    NvU32 numSupportedVgpuTypes = 0;
+    VGPU_TYPE *vgpuTypeInfo;
+    void *fp;
+
+    if (!vgpuTypeIds || !numVgpuTypes)
+        return NV_ERR_INVALID_ARGUMENT;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+    {
+        if ((rmStatus = kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pNv->gpu_id, &pgpuIndex)) ==
+            NV_OK)
+        {
+            if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].sriovEnabled && !isVirtfn)
+            {
+                *numVgpuTypes = 0;
+            }
+            else
+            {
+                numSupportedVgpuTypes = pKernelVgpuMgr->pgpuInfo[pgpuIndex].numVgpuTypes;
+                *numVgpuTypes = 0;
+
+                for (i = 0; i < numSupportedVgpuTypes; i++)
+                {
+                    vgpuTypeInfo = pKernelVgpuMgr->pgpuInfo[pgpuIndex].vgpuTypes[i];
+
+                    if (!getCreatableTypes)
+                    {
+                        // Return all available types
+                        vgpuTypeIds[*numVgpuTypes] = vgpuTypeInfo->vgpuTypeId;
+                        (*numVgpuTypes)++;
+                        continue;
+                    }
+
+                    pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+                    if (pGpu == NULL)
+                    {
+                        NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n",
+                                  __FUNCTION__);
+                        goto exit;
+                    }
+
+                    rmStatus = kvgpumgrGetAvailableInstances(&avail_instances, pGpu,
+                                                             vgpuTypeInfo, pgpuIndex,
+                                                             devfn);
+                    if (rmStatus != NV_OK)
+                    {
+                        NV_PRINTF(LEVEL_ERROR, "Failed to get available instances for vGPU ID: %d, status: 0x%x\n",
+                                  vgpuTypeInfo->vgpuTypeId, rmStatus);
+                        continue;
+                    }
+
+                    if (avail_instances == 0)
+                        continue;
+
+                    vgpuTypeIds[*numVgpuTypes] = vgpuTypeInfo->vgpuTypeId;
+                    (*numVgpuTypes)++;
+                }
+            }
+        }
+
+exit:
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_delete(
+    nvidia_stack_t *sp,
+    const NvU8 *pVgpuDevName,
+    NvU16 vgpuId
+)
+{
+    THREAD_STATE_NODE threadState;
+    void         *fp = NULL;
+    NvU32        rmStatus = NV_OK;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+    {
+        rmStatus = kvgpumgrDeleteRequestVgpu(pVgpuDevName, vgpuId);
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL nv_vgpu_process_vf_info(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    NvU8 cmd,
+    NvU32 domain,
+    NvU8 bus,
+    NvU8 slot,
+    NvU8 function,
+    NvBool isMdevAttached,
+    void *vf_pci_info
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS     rmStatus = NV_OK;
+    void         *fp = NULL;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+    {
+        rmStatus = kvgpumgrProcessVfInfo(pNv->gpu_id, cmd, domain, bus, slot, function, isMdevAttached, (vgpu_vf_pci_info *) vf_pci_info);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL nv_vgpu_create_request(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    const NvU8 *pVgpuDevName,
+    NvU32 vgpuTypeId,
+    NvU16 *vgpuId,
+    NvU32 *gpu_instance_id,
+    NvU32 *placement_id,
+    NvU32 gpuPciBdf
+)
+{
+    THREAD_STATE_NODE threadState;
+    void          *fp          = NULL;
+    NV_STATUS     rmStatus     = NV_OK;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+    {
+        rmStatus = kvgpumgrCreateRequestVgpu(pNv->gpu_id, pVgpuDevName,
+                                             vgpuTypeId, vgpuId, gpu_instance_id, placement_id, gpuPciBdf);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+static NV_STATUS is_bar_64bit(
+    OBJGPU *pGpu,
+    NvU32 regionIndex,
+    NvBool *isBar64bit
+)
+{
+    NV_STATUS   rmStatus = NV_OK;
+
+    if (pGpu == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__);
+        rmStatus = NV_ERR_INVALID_STATE;
+        goto exit;
+    }
+
+    switch (regionIndex)
+    {
+        case NV_VFIO_PCI_BAR0_REGION_INDEX:
+            *isBar64bit = pGpu->sriovState.b64bitVFBar0;
+            break;
+
+        case NV_VFIO_PCI_BAR1_REGION_INDEX:
+            *isBar64bit = pGpu->sriovState.b64bitVFBar1;
+            break;
+
+        case NV_VFIO_PCI_BAR2_REGION_INDEX:
+            *isBar64bit = pGpu->sriovState.b64bitVFBar2;
+            break;
+
+        default:
+            NV_PRINTF(LEVEL_ERROR, "BAR%d region doesn't exist!\n", regionIndex);
+            rmStatus = NV_ERR_INVALID_ARGUMENT;
+            goto exit;
+            break;
+    }
+
+    NV_PRINTF(LEVEL_INFO, "BAR%d region is_64bit: %d\n", regionIndex, *isBar64bit);;
+
+exit:
+    return rmStatus;
+}
+
+static NV_STATUS
+_nv_vgpu_get_bar_size(OBJGPU *pGpu, KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice,
+                      NvU32 regionIndex, NvU64 *size, NvU8 *configParams)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
+    NV_STATUS status;
+    KernelBus *pKernelBus;
+    NvU32 value = 0;
+
+    pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+    *size = kbusGetPciBarSize(pKernelBus, regionIndex);
+
+    if (regionIndex == NV_VFIO_PCI_BAR1_REGION_INDEX)
+    {
+        VGPU_TYPE *vgpuTypeInfo;
+        NvU32     pgpuIndex = 0;
+        NvBool    bOverrideBar1Size = NV_FALSE;
+
+        // Read BAR1 length from vgpuTypeInfo
+        NV_ASSERT_OK_OR_RETURN(kvgpumgrGetVgpuTypeInfo(pKernelHostVgpuDevice->vgpuType,
+                                                       &vgpuTypeInfo));
+
+        *size = vgpuTypeInfo->bar1Length << 20;
+        NV_ASSERT_OK_OR_RETURN(kvgpumgrGetPgpuIndex(pKernelVgpuMgr, pGpu->gpuId, &pgpuIndex));
+
+        /*
+         * check for 'override_bar1_size' param in vgpuExtraParams list first,
+         * if param is missing there then check it in vgpu_params list
+         */
+        status = nv_parse_config_params((const char*)vgpuTypeInfo->vgpuExtraParams,
+                                        "override_bar1_size", ';', &value);
+        if (status == NV_OK && value)
+        {
+            bOverrideBar1Size = NV_TRUE;
+        }
+        else if (status == NV_ERR_OBJECT_NOT_FOUND)
+        {
+            status = nv_parse_config_params((const char *)configParams,
+                                            "override_bar1_size", ',', &value);
+            if (status == NV_OK && value)
+                bOverrideBar1Size = NV_TRUE;
+        }
+
+        if (gpuIsVfResizableBAR1Supported(pGpu))
+        {
+            if ((*size > pGpu->sriovState.vfBarSize[1]) ||
+                (!portStringCompare("Compute", (const char *)vgpuTypeInfo->vgpuClass, 7)))
+            {
+                *size = pGpu->sriovState.vfBarSize[1];
+            }
+        }
+
+        if (bOverrideBar1Size)
+        {
+            NvU64 bar1SizeInBytes, guestBar1;
+            NvU64 gpuBar1LowerLimit = 256 * 1024 * 1024; // bar1 lower limit for override_bar1_length parameter
+            bar1SizeInBytes = kbusGetPciBarSize(pKernelBus, NV_VFIO_PCI_BAR1_REGION_INDEX);
+
+            if (pKernelVgpuMgr->pgpuInfo[pgpuIndex].sriovEnabled)
+            {
+                *size = pGpu->sriovState.vfBarSize[1];
+            }
+            else if (bar1SizeInBytes > gpuBar1LowerLimit)
+            {
+                guestBar1 = bar1SizeInBytes / vgpuTypeInfo->maxInstance;
+                *size = nvPrevPow2_U64(guestBar1);
+            }
+        }
+    }
+    else if (regionIndex == NV_VFIO_PCI_BAR2_REGION_INDEX ||
+             regionIndex == NV_VFIO_PCI_BAR3_REGION_INDEX)
+    {
+        status = nv_parse_config_params((const char *)configParams,
+                                        "address64", ',', &value);
+
+        if ((status != NV_OK) || ((status == NV_OK) && (value != 0)))
+        {
+            if (regionIndex == NV_VFIO_PCI_BAR2_REGION_INDEX)
+                *size = 0;
+            else if (regionIndex == NV_VFIO_PCI_BAR3_REGION_INDEX)
+                *size = kbusGetPciBarSize(pKernelBus, NV_VFIO_PCI_BAR2_REGION_INDEX);
+        }
+    }
+
+    return NV_OK;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_get_bar_info
+(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    const NvU8 *pVgpuDevName,
+    NvU64 *barSizes,
+    NvU64 *sparseOffsets,
+    NvU64 *sparseSizes,
+    NvU32 *sparseCount,
+    NvBool *isBar064bit,
+    NvU8 *configParams
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus = NV_OK;
+    OBJGPU *pGpu = NULL;
+    void *fp = NULL;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+                        rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit);
+
+    pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+    if (pGpu == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__);
+        rmStatus = NV_ERR_INVALID_STATE;
+        goto release_lock;
+    }
+
+    NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+                        nv_vgpu_rm_get_bar_info(pGpu, pVgpuDevName, barSizes,
+                                                sparseOffsets, sparseSizes,
+                                                sparseCount, isBar064bit,
+                                                configParams),
+                        release_lock);
+release_lock:
+    // UNLOCK: release API lock
+    rmapiLockRelease();
+
+exit:
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_update_sysfs_info
+(
+    nvidia_stack_t      *sp,
+    nv_state_t          *pNv,
+    const NvU8          *pVgpuDevName,
+    NvU32                mode,
+    NvU32                *sysfs_val
+)
+{
+    THREAD_STATE_NODE       threadState;
+    void                   *fp              = NULL;
+    OBJSYS                 *pSys            = SYS_GET_INSTANCE();
+    KernelVgpuMgr          *pKernelVgpuMgr  = SYS_GET_KERNEL_VGPUMGR(pSys);
+    REQUEST_VGPU_INFO_NODE *pRequestVgpu    = NULL;
+    NV_STATUS               rmStatus        = NV_OK;
+    VGPU_SYSFS_OP           updateMode      = (VGPU_SYSFS_OP)mode;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+                        rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit);
+
+    // Check the corresponding REQUEST_VGPU_INFO_NODE
+    for (pRequestVgpu = listHead(&pKernelVgpuMgr->listRequestVgpuHead);
+         pRequestVgpu != NULL;
+         pRequestVgpu = listNext(&pKernelVgpuMgr->listRequestVgpuHead, pRequestVgpu))
+    {
+        if (portMemCmp(pRequestVgpu->vgpuDevName, pVgpuDevName, VM_UUID_SIZE) == 0)
+            break;
+    }
+
+    if (pRequestVgpu == NULL)
+    {
+        rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+        goto release_lock;
+    }
+
+    if (updateMode == SET_GPU_INSTANCE_ID)
+        rmStatus = kvgpumgrSetGpuInstanceId(pRequestVgpu, *sysfs_val);
+    else if (updateMode == SET_PLACEMENT_ID)
+        rmStatus = kvgpumgrSetPlacementId(pRequestVgpu, *sysfs_val);
+    else if (updateMode == GET_PLACEMENT_ID)
+       *sysfs_val = pRequestVgpu->placementId;
+
+release_lock:
+    // UNLOCK: release API lock
+    rmapiLockRelease();
+
+exit:
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_vgpu_get_hbm_info(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    const NvU8 *pVgpuDevName,
+    NvU64 *hbmAddr,
+    NvU64 *size
+)
+{
+    NV_STATUS    rmStatus = NV_OK;
+    THREAD_STATE_NODE threadState;
+    OBJGPU      *pGpu = NULL;
+    KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice;
+    void         *fp = NULL;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if ((size == NULL) || (hbmAddr == NULL))
+    {
+        rmStatus = NV_ERR_INVALID_ARGUMENT;
+        goto exit;
+    }
+
+    // LOCK: acquire API lock
+    NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT, rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit);
+
+    pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+    if (pGpu == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "%s GPU handle is not valid \n", __FUNCTION__);
+        rmStatus = NV_ERR_INVALID_STATE;
+        goto release_lock;
+    }
+
+    NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+                        kvgpumgrGetHostVgpuDeviceFromVgpuDevName(pNv->gpu_id,
+                                                                 pVgpuDevName,
+                                                                 &pKernelHostVgpuDevice), release_lock);
+    if (pKernelHostVgpuDevice->numValidHbmRegions > 1)
+    {
+        NV_PRINTF(LEVEL_NOTICE, "non contiguous HBM region is not supported\n");
+        NV_ASSERT(0);
+    }
+
+    if (pKernelHostVgpuDevice->hbmRegionList != NULL)
+    {
+        *hbmAddr = pKernelHostVgpuDevice->hbmRegionList[0].hbmBaseAddr;
+        *size = pKernelHostVgpuDevice->hbmRegionList[0].size;
+    }
+    else
+    {
+        *hbmAddr = 0;
+        *size = 0;
+    }
+
+release_lock:
+    // UNLOCK: release API lock
+    rmapiLockRelease();
+exit:
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+static NV_STATUS nv_parse_config_params(
+    const char *config_params,
+    const char *key,
+    const char delim,
+    NvU32 *config_value
+)
+{
+    char *ptr, *configParams = rm_remove_spaces(config_params);
+    char *token, *value, *name;
+    NvU32 data;
+    NV_STATUS rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+
+    ptr = configParams;
+    while ((token = rm_string_token(&ptr, delim)) != NULL)
+    {
+        if (!(name = rm_string_token(&token, '=')) || !os_string_length(name))
+            continue;
+
+        if (!(value = rm_string_token(&token, '=')) || !os_string_length(value))
+            continue;
+
+        data = os_strtoul(value, NULL, 0);
+
+        if (os_string_compare(name, key) == 0)
+        {
+            rmStatus = NV_OK;
+            *config_value = data;
+        }
+    }
+
+   // Free the memory allocated by rm_remove_spaces()
+   os_free_mem(configParams);
+
+   return rmStatus;
+}
+
+static NV_STATUS _nv_vgpu_get_sparse_mmap(
+    OBJGPU *pGpu,
+    KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice,
+    NvU64 *offsets,
+    NvU64 *sizes,
+    NvU32 *numAreas,
+    NvU8 *configParams
+)
+{
+    NV_STATUS rmStatus = NV_OK, status;
+    OBJTMR *pTmr = GPU_GET_TIMER(pGpu);;
+    KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);;
+    KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+    NvU32 value = 0;
+
+    *numAreas = 0;
+    if (pKernelHostVgpuDevice->gfid != 0)
+    {
+        rmStatus = kbifGetVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice,
+                                                  os_page_size, numAreas, NULL, NULL);
+        if (rmStatus == NV_OK)
+        {
+            if (*numAreas > NVA081_MAX_SPARSE_REGION_COUNT)
+            {
+                NV_PRINTF(LEVEL_ERROR, "Not enough space for sparse mmap region info\n");
+                return NV_ERR_INSUFFICIENT_RESOURCES;
+            }
+
+
+            rmStatus = kbifGetVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice, os_page_size,
+                                                      numAreas, offsets, sizes);
+            if (rmStatus != NV_OK)
+                return rmStatus;
+        }
+    }
+    else
+    {
+        status = nv_parse_config_params((const char *)configParams,
+                                        "direct_gpu_timer_access", ',', &value);
+        if ((status == NV_OK) && (value != 0))
+        {
+            NvU64 offset = 0;
+            NvU32 size = 0;
+
+            rmStatus = tmrGetTimerBar0MapInfo_HAL(pGpu, pTmr, &offset, &size);
+
+            if (rmStatus == NV_OK)
+            {
+                offsets[*numAreas] = offset;
+                sizes[*numAreas] = size;
+                (*numAreas)++;
+            }
+        }
+
+        value = 0;
+        {
+            NvU64 offset = 0;
+            NvU32 size = 0;
+
+            status = kfifoGetUsermodeMapInfo_HAL(pGpu, pKernelFifo, &offset, &size);
+
+            if (status == NV_OK)
+            {
+                offsets[*numAreas] = offset;
+                sizes[*numAreas] = size;
+                (*numAreas)++;
+            }
+        }
+    }
+
+    return rmStatus;
+}
+
+NV_STATUS nv_vgpu_rm_get_bar_info
+(
+    OBJGPU     *pGpu,
+    const NvU8 *pVgpuDevName,
+    NvU64      *barSizes,
+    NvU64      *sparseOffsets,
+    NvU64      *sparseSizes,
+    NvU32      *sparseCount,
+    NvBool     *isBar064bit,
+    NvU8       *configParams
+)
+{
+    KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice;
+    NV_STATUS rmStatus;
+    NvU32 i = 0;
+
+    NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+                        is_bar_64bit(pGpu, NV_VFIO_PCI_BAR0_REGION_INDEX, isBar064bit),
+                        exit);
+
+    NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+                        kvgpumgrGetHostVgpuDeviceFromVgpuDevName(pGpu->gpuId,
+                                                                 pVgpuDevName,
+                                                                 &pKernelHostVgpuDevice),
+                        exit);
+
+    for (i = 0; i < NVA081_MAX_BAR_REGION_COUNT; i++)
+    {
+        /*
+         * For SRIOV, only VF BAR1 is queried via RM, others BARs are directly
+         * queried via VF config space in vgpu-vfio
+         */
+        if (gpuIsSriovEnabled(pGpu) && (i != NV_VFIO_PCI_BAR1_REGION_INDEX))
+        {
+            barSizes[i] = 0;
+            continue;
+        }
+
+        rmStatus = _nv_vgpu_get_bar_size(pGpu, pKernelHostVgpuDevice, i,
+                                         &barSizes[i], configParams);
+        if (rmStatus != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR, "Failed to query BAR size for index %u 0x%x\n",
+                      i, rmStatus);
+            goto exit;
+        }
+    }
+
+    NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
+                        _nv_vgpu_get_sparse_mmap(pGpu, pKernelHostVgpuDevice,
+                                                 sparseOffsets, sparseSizes,
+                                                 sparseCount, configParams),
+                        exit);
+
+exit:
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_gpu_unbind_event
+(
+    nvidia_stack_t *sp,
+    NvU32 gpuId,
+    NvBool *isEventNotified
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus = NV_OK;
+    void *fp = NULL;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+    {
+        NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_UNBIND eventData = { 0 };
+
+        // Send gpu_id so that nvidia-vgpu-mgr daemon knows which GPU is being unbound
+        eventData.gpuId = gpuId;
+        CliAddSystemEvent(NV0000_NOTIFIERS_VGPU_UNBIND_EVENT, &eventData, isEventNotified);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL nv_gpu_bind_event(
+    nvidia_stack_t *sp,
+    NvU32 gpuId,
+    NvBool *isEventNotified
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus = NV_OK;
+    void *fp = NULL;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
+    {
+        NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_BIND eventData = { 0 };
+
+        eventData.gpuId = gpuId;
+        CliAddSystemEvent(NV0000_NOTIFIERS_VGPU_BIND_EVENT, &eventData, isEventNotified);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS osIsVgpuVfioPresent(void)
+{
+    vgpu_vfio_info vgpu_info;
+
+    return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VGPU_VFIO_PRESENT);
+}
+
+NV_STATUS osIsVfioPciCorePresent(void)
+{
+    vgpu_vfio_info vgpu_info;
+
+    return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VFIO_PCI_CORE_PRESENT);
+}
+
+void osWakeRemoveVgpu(NvU32 gpuId, NvU32 returnStatus)
+{
+    vgpu_vfio_info vgpu_info;
+
+    vgpu_info.return_status = returnStatus;
+    vgpu_info.domain = GPU_32_BIT_ID_DECODE_DOMAIN(gpuId);
+    vgpu_info.bus = GPU_32_BIT_ID_DECODE_BUS(gpuId);
+    vgpu_info.device = GPU_32_BIT_ID_DECODE_DEVICE(gpuId);
+
+    os_call_vgpu_vfio((void *)&vgpu_info, CMD_VFIO_WAKE_REMOVE_GPU);
+}
+
+NvBool osIsGridSupported(OBJGPU *pGpu)
+{
+    return (os_is_grid_supported() || os_get_grid_csp_support() || hypervisorIsVgxHyper());
+}
+
+NvU32 osGetGridCspSupport(void)
+{
+    return os_get_grid_csp_support();
+}
+
+void initVGXSpecificRegistry(OBJGPU *pGpu)
+{
+    NvU32 data32;
+    osWriteRegistryDword(pGpu, NV_REG_STR_RM_POWER_FEATURES, 0x55455555);
+    osWriteRegistryDword(pGpu, NV_REG_STR_RM_INFOROM_DISABLE_BBX,
+                               NV_REG_STR_RM_INFOROM_DISABLE_BBX_YES);
+    if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_DUMP_NVLOG, &data32) != NV_OK))
+    {
+        osWriteRegistryDword(pGpu, NV_REG_STR_RM_DUMP_NVLOG,
+                             NV_REG_STR_RM_DUMP_NVLOG_ENABLE);
+    }
+    osWriteRegistryDword(pGpu, NV_REG_STR_RM_RC_WATCHDOG,
+                               NV_REG_STR_RM_RC_WATCHDOG_DISABLE);
+    osWriteRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P,
+                         DRF_DEF(_REG_STR, _CL_FORCE_P2P, _READ, _DISABLE) |
+                         DRF_DEF(_REG_STR, _CL_FORCE_P2P, _WRITE, _DISABLE));
+}
+
+
+NV_STATUS rm_is_vgpu_supported_device(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32       pmc_boot_1,
+    NvU32       pmc_boot_42
+)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
+    NvBool is_sriov_enabled = FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1);
+    NvU32     i;
+
+    // if not running in vGPU mode (guest VM) return NV_OK
+    if (!(pHypervisor && pHypervisor->bIsHVMGuest &&
+          (FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _PV, pmc_boot_1) ||
+           is_sriov_enabled)))
+    {
+        return NV_OK;
+    }
+
+    if (!is_sriov_enabled)
+    {
+        return NV_OK;
+    }
+
+    if (os_is_grid_supported() && !gpumgrIsVgxRmFirmwareCapableChip(pmc_boot_42))
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    if (os_is_grid_supported())
+    {
+        for (i = 0; i < NV_ARRAY_ELEMENTS(sVgpuUsmTypes); i++)
+        {
+            if (pOsGpuInfo->pci_info.device_id == sVgpuUsmTypes[i].ulDevID &&
+                    pOsGpuInfo->subsystem_id == sVgpuUsmTypes[i].ulSubID)
+            {
+                return NV_OK;
+            }
+        }
+    }
+
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osIsVgpuDeviceVmPresent(void)
+{
+    return os_device_vm_present();
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/os.c b/userspace/sdks/nv570/nvalloc/unix/src/os.c
new file mode 100644
index 0000000..b441a66
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/os.c
@@ -0,0 +1,6359 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>                     // NV device driver interface
+#include <nv-priv.h>
+#include <nv-caps.h>
+#include <os/os.h>
+#include <nvos.h>
+#include <osapi.h>
+#include <ctrl/ctrl0000/ctrl0000gpu.h>
+#include <ctrl/ctrl0000/ctrl0000unix.h>
+#include <class/cl90cd.h> // NV_EVENT_BUFFER_BIND
+#include <nvdevid.h>
+
+#include <nverror.h>
+#include <gpu/device/device.h>
+
+#include "gpu/gpu.h"
+#include <gpu_mgr/gpu_mgr.h>
+#include <platform/chipset/chipset.h>
+
+#include "nverror.h"
+#include "kernel/gpu/bif/kernel_bif.h"
+
+#include "gpu/mig_mgr/kernel_mig_manager.h"
+
+#include "gpu/mem_sys/kern_mem_sys.h"
+
+#include "mem_mgr/io_vaspace.h"
+#include <diagnostics/journal.h>
+#include "kernel/diagnostics/xid_context.h"
+#include "gpu/mem_mgr/mem_desc.h"
+#include "gpu/mem_mgr/mem_mgr.h"
+#include "core/thread_state.h"
+#include <nvacpitypes.h>
+#include <platform/acpi_common.h>
+#include <core/locks.h>
+#include <ctrl/ctrl2080/ctrl2080gpu.h>
+#include "virtualization/hypervisor/hypervisor.h"
+#include "rmobjexportimport.h"
+#include <nvi2c.h>
+#include "rmapi/rs_utils.h"
+#include "rmapi/client_resource.h"
+#include "os/dce_rm_client_ipc.h"
+#include "mem_mgr/mem.h"
+#include "gpu/mem_mgr/virt_mem_allocator_common.h"
+#include "nvrm_registry.h"
+
+#include "vgpu/vgpu_util.h"
+
+#include "platform/chipset/chipset.h"
+
+#include <acpidsmguids.h>
+#include <pex.h>
+#include "gps.h"
+#include "jt.h"
+
+
+
+
+extern const char *ppOsBugCheckBugcodeStr[];
+
+
+ct_assert(NV_RM_PAGE_SIZE == RM_PAGE_SIZE);
+ct_assert(NV_RM_PAGE_MASK == RM_PAGE_MASK);
+ct_assert(NV_RM_PAGE_SHIFT == RM_PAGE_SHIFT);
+
+typedef struct
+{
+    NvU32 euid;
+    NvU32 pid;
+} TOKEN_USER, *PTOKEN_USER;
+
+struct OS_RM_CAPS
+{
+    NvU32 count;
+
+    // This should be the last element
+    nv_cap_t **caps;
+};
+
+NvBool osIsRaisedIRQL(void)
+{
+    return (!os_semaphore_may_sleep());
+}
+
+NvBool osIsISR(void)
+{
+    return os_is_isr();
+}
+
+NV_STATUS osGetDriverBlock
+(
+    OS_GPU_INFO     *pOsGpuInfo,
+    OS_DRIVER_BLOCK *pBlock
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NvU64 osGetMonotonicTimeNs(void)
+{
+    return os_get_monotonic_time_ns();
+}
+
+NV_STATUS osGetPerformanceCounter(NvU64 *pTimeInNs)
+{
+    *pTimeInNs = os_get_monotonic_time_ns_hr();
+    return NV_OK;
+}
+
+NV_STATUS osGetSystemTime(
+    NvU32 *pSeconds,
+    NvU32 *pMicroSeconds
+)
+{
+    return os_get_system_time(pSeconds, pMicroSeconds);
+}
+
+/*!
+ * @brief Get timestamp for logging.
+ *
+ * Everything that logs a time stamp should use this routine for consistency.
+ *
+ * The returned value is OS dependent.  We want the time stamp to use
+ * KeQueryPerformanceCounter on Windows so it matches the DirectX timestamps.
+ * Linux uses microseconds since 1970 (osGetSystemTime), since matching DirectX
+ * is not a priority.
+ *
+ * osGetTimestampFreq returns the frequency required to decode the time stamps.
+ *
+ * @returns   system dependent timestamp.
+ */
+NvU64 osGetTimestamp(void)
+{
+    NvU32 sec  = 0;
+    NvU32 usec = 0;
+    osGetSystemTime(&sec, &usec);
+    return (NvU64)sec * 1000000 + usec;
+}
+
+/*!
+ * @brief Get timestamp frequency.
+ *
+ * Timestamps are OS dependent.  This call returns the frequency
+ * required to decode them.
+ *
+ * @returns   Timestamp frequency.  For example, 1000000 for MHz.
+ */
+NvU64 osGetTimestampFreq(void)
+{
+    return 1000000;
+}
+
+NV_STATUS osDelay(NvU32 milliseconds)
+{
+    return os_delay(milliseconds);
+}
+
+NV_STATUS osDelayUs(NvU32 microseconds)
+{
+    return os_delay_us(microseconds);
+}
+
+NV_STATUS osDelayNs(NvU32 nanoseconds)
+{
+    NvU32 microseconds = NV_MAX(1, (nanoseconds / 1000));
+    return os_delay_us(microseconds);
+}
+
+NvU32 osGetCpuFrequency(void)
+{
+    /* convert os_get_cpu_frequency()'s return value from Hz to MHz */
+    return ((NvU32)(os_get_cpu_frequency() / 1000000ULL));
+}
+
+void* osPciInitHandle(
+    NvU32  Domain,
+    NvU8   Bus,
+    NvU8   Slot,
+    NvU8   Function,
+    NvU16 *pVendor,
+    NvU16 *pDevice
+)
+{
+    //
+    // Check if the BDF is for a GPU that's already been attached, for which
+    // we should already have a handle cached. This won't catch devices that
+    // have been probed but not yet attached, but that shouldn't be a common
+    // occurrence.
+    //
+    // More importantly, having this check here means we don't need to check
+    // a global list of devices in the kernel interface layer, which could
+    // have the implication of taking another lock, causing hairy lock
+    // ordering issues.
+    //
+    if (Function == 0)
+    {
+        OBJGPU *pGpu = gpumgrGetGpuFromBusInfo(Domain, Bus, Slot);
+        if (pGpu != NULL)
+        {
+            nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+            if (pVendor) *pVendor = nv->pci_info.vendor_id;
+            if (pDevice) *pDevice = nv->pci_info.device_id;
+            return nv->handle;
+        }
+    }
+
+    return os_pci_init_handle(Domain, Bus, Slot, Function, pVendor, pDevice);
+}
+
+NvU8 osPciReadByte(
+    void *pHandle,
+    NvU32 Offset
+)
+{
+    NvU8 val;
+    os_pci_read_byte(pHandle, Offset, &val);
+    return val;
+}
+
+NvU16 osPciReadWord(
+    void *pHandle,
+    NvU32 Offset
+)
+{
+    NvU16 val;
+    os_pci_read_word(pHandle, Offset, &val);
+    return val;
+}
+
+NvU32 osPciReadDword(
+    void *pHandle,
+    NvU32 Offset
+)
+{
+    NvU32 val;
+    os_pci_read_dword(pHandle, Offset, &val);
+    return val;
+}
+
+void osPciWriteByte(
+    void *pHandle,
+    NvU32 Offset,
+    NvU8  Value
+)
+{
+    os_pci_write_byte(pHandle, Offset, Value);
+}
+
+void osPciWriteWord(
+    void  *pHandle,
+    NvU32 Offset,
+    NvU16 Value
+)
+{
+    os_pci_write_word(pHandle, Offset, Value);
+}
+
+void osPciWriteDword(
+    void  *pHandle,
+    NvU32 Offset,
+    NvU32 Value
+)
+{
+    os_pci_write_dword(pHandle, Offset, Value);
+}
+
+void* osMapKernelSpace(
+    RmPhysAddr Start,
+    NvU64      Size,
+    NvU32      Mode,
+    NvU32      Protect
+)
+{
+    NvU64 offset;
+    NvU8 *ptr;
+
+    if (0 == Size)
+    {
+        NV_ASSERT(Size != 0);
+        return NULL;
+    }
+
+    offset = (Start & ~os_page_mask);
+    Start &= os_page_mask;
+
+    if (!portSafeAddU64(Size, offset, &Size) ||
+        !portSafeAddU64(Size, ~os_page_mask, &Size))
+    {
+        return NULL;
+    }
+    Size &= os_page_mask;
+
+    ptr = os_map_kernel_space(Start, Size, Mode);
+    if (ptr != NULL)
+        return (ptr + offset);
+
+    return NULL;
+}
+
+void osUnmapKernelSpace(
+    void *pAddress,
+    NvU64 Size
+)
+{
+    NvU64 offset;
+    NvUPtr ptr = (NvUPtr)pAddress;
+
+    if (0 == Size)
+    {
+        NV_ASSERT(Size != 0);
+        return;
+    }
+
+    offset = (ptr & ~os_page_mask);
+    ptr &= os_page_mask;
+    Size = ((Size + offset + ~os_page_mask) & os_page_mask);
+    os_unmap_kernel_space((void *)ptr, Size);
+}
+
+static NV_STATUS setNumaPrivData
+(
+    KernelMemorySystem      *pKernelMemorySystem,
+    nv_state_t              *nv,
+    MEMORY_DESCRIPTOR       *pMemDesc
+)
+{
+    NV_STATUS   rmStatus      = NV_OK;
+    void       *pAllocPrivate = NULL;
+    NvU64      *addrArray     = NULL;
+    NvU64       numPages      = pMemDesc->PageCount;
+    NvU64       numOsPages    = numPages;
+    RmPhysAddr *pteArray      = memdescGetPteArray(pMemDesc, AT_CPU);
+    NvU64       i;
+
+    //
+    // Since this function will also deal with the memory allocated from PMA,
+    // we have to perform page size inflate/deflate here.
+    //
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        NvU64 size = pMemDesc->ActualSize;
+        numOsPages = size / NV_MIN(os_page_size, pMemDesc->pageArrayGranularity);
+    }
+
+    addrArray = portMemAllocNonPaged(numOsPages * sizeof(NvU64));
+    if (addrArray == NULL)
+    {
+        return NV_ERR_NO_MEMORY;
+    }
+
+    if (!IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc) && NV_RM_PAGE_SIZE < os_page_size)
+    {
+        numOsPages = NV_RM_PAGES_TO_OS_PAGES(numPages);
+    }
+
+    if (!memdescGetContiguity(pMemDesc, AT_CPU))
+    {
+        NvU64 copyByteSize = 0;
+        if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+        {
+            // pMemDesc->pageArraySize is calculated by 4KB, it should NOT smaller than `numOSPages`.
+            NV_ASSERT_OR_ELSE(pMemDesc->pageArraySize >= numOsPages, rmStatus = NV_ERR_INVALID_STATE; goto errors);
+            copyByteSize = numOsPages * sizeof(NvU64);
+        }
+        else
+        {
+            copyByteSize = numPages * sizeof(NvU64);
+        }
+
+        portMemCopy((void*)addrArray,
+            copyByteSize,
+            (void*)pteArray,
+            copyByteSize);
+
+        if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+        {
+            NvU64 size = pMemDesc->ActualSize;
+            //
+            // Because of PMA case, we need to convert page array 
+            // from RM native granularity to OS granularity.
+            //
+            convertPageArrayBetweenOsAndRm(
+                addrArray, 
+                size / pMemDesc->pageArrayGranularity, 
+                pMemDesc->pageArrayGranularity,
+                size / os_page_size, 
+                os_page_size,
+                &numOsPages
+            );
+        }
+        else if (NV_RM_PAGE_SIZE < os_page_size)
+        {
+            RmDeflateRmToOsPageArray(addrArray, numPages);
+        }
+
+        for (i = 0; i < numOsPages; i++)
+        {
+            // Update GPA to system physical address
+            addrArray[i] += pKernelMemorySystem->coherentCpuFbBase;
+        }
+    }
+    else
+    {
+        //
+        // Original PTE array in contiguous memdesc only holds start address.
+        // We need to fill the OS page array with adjacent page addresses to
+        // map contiguously.
+        //
+        NvU64 offset = pteArray[0] + pKernelMemorySystem->coherentCpuFbBase;
+
+        for (i = 0; i < numOsPages; i++, offset += os_page_size)
+        {
+            addrArray[i] = offset;
+        }
+    }
+
+    rmStatus = nv_register_phys_pages(nv, addrArray, numOsPages, NV_MEMORY_CACHED, &pAllocPrivate);
+    if (rmStatus != NV_OK)
+    {
+        goto errors;
+    }
+
+    memdescSetMemData(pMemDesc, pAllocPrivate, NULL);
+
+errors:
+    portMemFree(addrArray);
+
+    return rmStatus;
+}
+
+NV_STATUS osGetNumMemoryPages
+(
+    MEMORY_DESCRIPTOR *pMemDesc,
+    NvU32 *pNumPages
+)
+{
+    void *pAllocPrivate = NULL;
+
+    pAllocPrivate = memdescGetMemData(pMemDesc);
+    if (pAllocPrivate == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
+        return NV_ERR_INVALID_STATE;
+    }
+
+    return nv_get_num_phys_pages(pAllocPrivate, pNumPages);
+}
+
+NV_STATUS osGetMemoryPages
+(
+    MEMORY_DESCRIPTOR *pMemDesc,
+    void *pPages,
+    NvU32 *pNumPages
+)
+{
+    void *pAllocPrivate = NULL;
+
+    pAllocPrivate = memdescGetMemData(pMemDesc);
+    if (pAllocPrivate == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
+        return NV_ERR_INVALID_STATE;
+    }
+
+    return nv_get_phys_pages(pAllocPrivate, pPages, pNumPages);
+}
+
+NV_STATUS osMapSystemMemory
+(
+    MEMORY_DESCRIPTOR *pMemDesc,
+    NvU64 Offset,
+    NvU64 Length,
+    NvBool Kernel,
+    NvU32 Protect,
+    NvP64 *ppAddress,
+    NvP64 *ppPrivate
+)
+{
+    OBJGPU *pGpu = pMemDesc->pGpu;
+
+    NvU64     rootOffset = 0;
+    pMemDesc = memdescGetRootMemDesc(pMemDesc, &rootOffset);
+    Offset  += rootOffset;
+
+    RmPhysAddr userAddress;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    NV_STATUS rmStatus = NV_OK;
+    void *pAllocPrivate = NULL;
+    void *pAddress;
+    void *pPrivate = NULL;
+    NvU64 pageIndex;
+    NvU32 pageOffset = (Offset & ~os_page_mask);
+
+    if (pGpu != NULL &&
+        pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) &&
+        memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)
+    {
+        KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
+
+        rmStatus = setNumaPrivData(pKernelMemorySystem, nv, pMemDesc);
+        if (rmStatus != NV_OK)
+            return rmStatus;
+
+        pageOffset = memdescGetPhysAddr(pMemDesc, FORCE_VMMU_TRANSLATION(pMemDesc, AT_GPU), Offset) & ~os_page_mask;
+    }
+
+    *ppAddress = NvP64_NULL;
+    *ppPrivate = NvP64_NULL;
+
+    if ((Offset + Length) < Length)
+        return NV_ERR_INVALID_ARGUMENT;
+    if ((Offset + Length) > pMemDesc->Size)
+        return NV_ERR_INVALID_ARGUMENT;
+
+    pageIndex = (Offset >> os_page_shift);
+
+    pAllocPrivate = memdescGetMemData(pMemDesc);
+    if (!pAllocPrivate)
+    {
+        NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
+        return NV_ERR_INVALID_STATE;
+    }
+
+    if (Kernel)
+    {
+        pAddress = nv_alloc_kernel_mapping(nv, pAllocPrivate,
+                pageIndex, pageOffset, Length, &pPrivate);
+        if (pAddress == NULL)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "failed to create system memory kernel mapping!\n");
+            rmStatus = NV_ERR_GENERIC;
+        }
+        else
+        {
+            *ppAddress = NV_PTR_TO_NvP64(pAddress);
+            *ppPrivate = NV_PTR_TO_NvP64(pPrivate);
+        }
+    }
+    else
+    {
+        rmStatus = nv_alloc_user_mapping(nv, pAllocPrivate,
+                pageIndex, pageOffset, Length, Protect, &userAddress,
+                &pPrivate);
+        if (rmStatus != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "failed to create system memory user mapping!\n");
+        }
+        else
+        {
+            *ppAddress = (NvP64)(userAddress);
+            *ppPrivate = NV_PTR_TO_NvP64(pPrivate);
+        }
+    }
+
+    return rmStatus;
+}
+
+void osUnmapSystemMemory
+(
+    MEMORY_DESCRIPTOR *pMemDesc,
+    NvBool Kernel,
+    NvP64  pAddress,
+    NvP64  pPrivate
+)
+{
+    void *pAllocPrivate;
+    OBJGPU *pGpu = pMemDesc->pGpu;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    pMemDesc = memdescGetRootMemDesc(pMemDesc, NULL);
+    pAllocPrivate = memdescGetMemData(pMemDesc);
+
+    if (Kernel)
+    {
+        nv_free_kernel_mapping(nv, pAllocPrivate, NvP64_VALUE(pAddress),
+                NvP64_VALUE(pPrivate));
+    }
+    else
+    {
+        nv_free_user_mapping(nv, pAllocPrivate, (NvU64)pAddress,
+                NvP64_VALUE(pPrivate));
+    }
+
+    if (pGpu != NULL &&
+        pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) &&
+        memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM &&
+        pAllocPrivate != NULL)
+    {
+        nv_unregister_phys_pages(nv, pAllocPrivate);
+        memdescSetMemData(pMemDesc, NULL, NULL);
+    }
+}
+
+void osIoWriteByte(
+    NvU32   Address,
+    NvU8    Value
+)
+{
+    os_io_write_byte(Address, Value);
+}
+
+NvU16 osIoReadWord(
+    NvU32   Address
+)
+{
+    return os_io_read_word(Address);
+}
+
+void osIoWriteWord(
+    NvU32 Address,
+    NvU16 Value
+)
+{
+    os_io_write_word(Address, Value);
+}
+
+NvU8 osIoReadByte(
+    NvU32   Address
+)
+{
+    return os_io_read_byte(Address);
+}
+
+NvBool osIsAdministrator(void)
+{
+    return os_is_administrator();
+}
+
+NvBool osCheckAccess(RsAccessRight accessRight)
+{
+    return os_check_access(accessRight);
+}
+
+NvU32 osGetCurrentProcess(void)
+{
+    return os_get_current_process();
+}
+
+void osGetCurrentProcessName(char *ProcName, NvU32 Length)
+{
+    return os_get_current_process_name(ProcName, Length);
+}
+
+NV_STATUS osGetCurrentThread(OS_THREAD_HANDLE *pThreadId)
+{
+    NV_STATUS rmStatus;
+    NvU64 threadId = 0;
+
+    if (pThreadId == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    rmStatus = os_get_current_thread(&threadId);
+    if (rmStatus == NV_OK)
+    {
+        *pThreadId = threadId;
+    }
+    else
+    {
+        *pThreadId = 0;
+    }
+
+    return rmStatus;
+}
+
+void* osGetPidInfo(void)
+{
+    return os_get_pid_info();
+}
+
+void osPutPidInfo(void *pOsPidInfo)
+{
+    os_put_pid_info(pOsPidInfo);
+}
+
+NV_STATUS osFindNsPid(void *pOsPidInfo, NvU32 *pNsPid)
+{
+    return os_find_ns_pid(pOsPidInfo, pNsPid);
+}
+
+NvBool osIsInitNs(void)
+{
+    return os_is_init_ns();
+}
+
+NV_STATUS osAttachToProcess(void** ppProcessInfo, NvU32 ProcessId)
+{
+    //
+    // This function is used by RmUnmapMemory() to attach to the
+    // process for which a given device memory mapping was
+    // created, in order to be able to unmap it. On Linux/UNIX
+    // platforms, we can't "attach" to a random process, but
+    // since we don't create/destroy user mappings in the RM, we
+    // don't need to, either.
+    //
+    // Report success to the caller to keep RmUnmapMemory() from
+    // failing, and memory from being leaked as a result.
+    //
+    *ppProcessInfo = NULL;
+    return NV_OK;
+}
+
+void osDetachFromProcess(void* pProcessInfo)
+{
+    // stub
+    return;
+}
+
+NV_STATUS osAcquireRmSema(void *pSema)
+{
+    return NV_OK;
+}
+
+NV_STATUS osCondAcquireRmSema(void *pSema)
+{
+    return NV_OK;
+}
+
+NvU32 osReleaseRmSema(void *pSema, OBJGPU *pDpcGpu)
+{
+    return NV_SEMA_RELEASE_SUCCEED;
+}
+
+void osSpinLoop(void)
+{
+    
+}
+
+NvU64 osGetMaxUserVa(void)
+{
+    return os_get_max_user_va();
+}
+
+NV_STATUS osSchedule(void)
+{
+    return os_schedule();
+}
+
+NV_STATUS osQueueWorkItem(
+    OBJGPU *pGpu,
+    OSWorkItemFunction pFunction,
+    void *pParams,
+    OsQueueWorkItemFlags flags
+)
+{
+    nv_work_item_t *pWi;
+    nv_state_t *nv;
+    NV_STATUS status;
+
+    pWi = portMemAllocNonPaged(sizeof(nv_work_item_t));
+
+    if (NULL == pWi)
+    {
+        return NV_ERR_NO_MEMORY;
+    }
+
+    pWi->flags = (OsQueueWorkItemFlags){0};
+    pWi->flags.bDontFreeParams         = flags.bDontFreeParams;
+    pWi->flags.bLockSema               = flags.bLockSema;
+    pWi->flags.apiLock                 = flags.apiLock;
+    pWi->flags.bLockGpus               = flags.bLockGpus;
+    pWi->flags.bLockGpuGroupDevice     = flags.bLockGpuGroupDevice;
+    pWi->flags.bLockGpuGroupSubdevice  = flags.bLockGpuGroupSubdevice;
+    pWi->flags.bFullGpuSanity          = flags.bFullGpuSanity;
+    pWi->flags.bDropOnUnloadQueueFlush = flags.bDropOnUnloadQueueFlush;
+    pWi->flags.bSetExpandedGpuVisibility = flags.bSetExpandedGpuVisibility;
+
+    pWi->flags.bRequiresGpu = NV_TRUE;
+    pWi->gpuInstance = gpuGetInstance(pGpu);
+
+    pWi->func.pGpuFunction = pFunction;
+    pWi->pData = pParams;
+    nv = NV_GET_NV_STATE(pGpu);
+
+    status = os_queue_work_item(nv ? nv->queue : NULL, pWi);
+
+    if (NV_OK != status)
+    {
+        portMemFree((void *)pWi);
+    }
+
+    return status;
+}
+
+NV_STATUS osQueueSystemWorkItem(
+    OSSystemWorkItemFunction pFunction,
+    void *pParams
+)
+{
+    nv_work_item_t *pWi;
+    NV_STATUS status;
+
+    pWi = portMemAllocNonPaged(sizeof(nv_work_item_t));
+
+    if (NULL == pWi)
+    {
+        return NV_ERR_NO_MEMORY;
+    }
+
+    pWi->flags = (OsQueueWorkItemFlags){0};
+    pWi->func.pSystemFunction = pFunction;
+    pWi->pData = pParams;
+
+    status = os_queue_work_item(NULL, pWi);
+
+    if (NV_OK != status)
+    {
+        portMemFree((void *)pWi);
+    }
+
+    return status;
+}
+
+NV_STATUS osQueueSystemWorkItemWithFlags(
+    OSSystemWorkItemFunction pFunction,
+    void *pParams,
+    OsQueueWorkItemFlags flags
+)
+{
+    nv_work_item_t *pWi;
+    NV_STATUS status;
+
+    pWi = portMemAllocNonPaged(sizeof(nv_work_item_t));
+
+    if (NULL == pWi)
+    {
+        return NV_ERR_NO_MEMORY;
+    }
+
+    pWi->flags = (OsQueueWorkItemFlags){0};
+    pWi->flags.bDontFreeParams         = flags.bDontFreeParams;
+    pWi->flags.bLockSema               = flags.bLockSema;
+    pWi->flags.apiLock                 = flags.apiLock;
+    pWi->flags.bLockGpus               = flags.bLockGpus;
+    pWi->flags.bLockGpuGroupDevice     = flags.bLockGpuGroupDevice;
+    pWi->flags.bLockGpuGroupSubdevice  = flags.bLockGpuGroupSubdevice;
+    pWi->flags.bFullGpuSanity          = flags.bFullGpuSanity;
+    pWi->flags.bDropOnUnloadQueueFlush = flags.bDropOnUnloadQueueFlush;
+    pWi->func.pSystemFunction = pFunction;
+    pWi->pData = pParams;
+
+    status = os_queue_work_item(NULL, pWi);
+
+    if (NV_OK != status)
+    {
+        portMemFree((void *)pWi);
+    }
+
+    return status;
+}
+
+void osQueueMMUFaultHandler(OBJGPU *pGpu)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    nv_schedule_uvm_isr(nv);
+}
+
+NvBool osGpuSupportsAts(OBJGPU *pGpu)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    // Checks ATS support from both OS side and RM side.
+    return nv->ats_support && 
+        pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED);
+}
+
+NV_STATUS osQueueDrainP2PHandler(NvU8 *pUuid)
+{
+    return nv_schedule_uvm_drain_p2p(pUuid);
+}
+
+void osQueueResumeP2PHandler(NvU8 *pUuid)
+{
+    nv_schedule_uvm_resume_p2p(pUuid);
+}
+
+static inline nv_dma_device_t* osGetDmaDeviceForMemDesc(
+    OS_GPU_INFO *pOsGpuInfo,
+    MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+    return (pOsGpuInfo->niso_dma_dev != NULL) &&
+           memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) ?
+           pOsGpuInfo->niso_dma_dev : pOsGpuInfo->dma_dev;
+}
+
+//
+// Set the DMA address size for the given GPU
+//
+// This is a global device setting and care would need to be taken if it was to
+// be modified outside of GPU initialization. At least on Linux other drivers,
+// like UVM, might be requesting its own DMA mappings for the same GPU after
+// the GPU has been initialized.
+//
+void osDmaSetAddressSize(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32        bits
+)
+{
+    nv_set_dma_address_size(pOsGpuInfo, bits);
+}
+
+static NV_STATUS osGetPagesInfo(
+    MEMORY_DESCRIPTOR *pMemDesc,
+    NvU64 *pageSize,
+    NvU64 *osPageCount,
+    NvU64 *rmPageCount
+)
+{
+    NvU64 osPageSize  = osGetPageSize();
+    NvU64 maxPageSize = NV_MAX(osPageSize, RM_PAGE_SIZE);
+    NvU64 alignedSize = NV_ALIGN_UP(pMemDesc->Size, maxPageSize);
+
+    *osPageCount = alignedSize >> BIT_IDX_32(osPageSize);
+    *rmPageCount = alignedSize >> RM_PAGE_SHIFT;
+    *pageSize = memdescGetAdjustedPageSize(pMemDesc);
+
+    // In the non-contig case need to protect against page array overflows.
+    if (!memdescGetContiguity(pMemDesc, AT_CPU))
+        NV_ASSERT_OR_RETURN(*rmPageCount <= pMemDesc->pageArraySize, NV_ERR_INVALID_ARGUMENT);
+
+    if (*osPageCount > NV_U32_MAX || *rmPageCount > NV_U32_MAX)
+        return NV_ERR_INVALID_LIMIT;
+
+    return NV_OK;
+}
+
+NV_STATUS osAllocPagesInternal(
+    MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+    OBJSYS           *pSys      = SYS_GET_INSTANCE();
+    OBJGPU           *pGpu      = pMemDesc->pGpu;
+    nv_state_t       *nv        = NV_GET_NV_STATE(pGpu);
+    void             *pMemData  = NULL;
+    NV_STATUS         status;
+    NvS32             nodeId    = memdescGetNumaNode(pMemDesc);
+    NV_ADDRESS_SPACE  addrSpace = memdescGetAddressSpace(pMemDesc);
+    NvU64             pageSize;
+    NvU64             osPageCount;
+    NvU64             rmPageCount;
+    NvU32             cpuCacheAttrib = memdescGetCpuCacheAttrib(pMemDesc);
+
+    memdescSetMemData(pMemDesc, NULL, NULL);
+
+#if (defined(NVCPU_AARCH64) && RMCFG_MODULE_CL)
+    {
+        OBJCL   *pCl       = SYS_GET_CL(pSys);
+
+        if ((pCl != NULL) &&
+             pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT) &&
+             !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_NON_IO_COHERENT))
+        {
+            cpuCacheAttrib = NV_MEMORY_CACHED;
+        }
+    }
+#endif
+
+    //
+    // For carveout, the memory is already reserved so we don't have
+    // to allocate memory.
+    //
+    if (memdescIsCarveoutMemory(pMemDesc) ||
+        memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
+    {
+        // We only support carveout with contiguous memory.
+        if (memdescIsCarveoutMemory(pMemDesc) &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+        {
+            status = NV_ERR_NOT_SUPPORTED;
+            goto done;
+        }
+
+        status = osGetPagesInfo(pMemDesc, &pageSize, &osPageCount, &rmPageCount);
+        if (status != NV_OK)
+            goto done;
+
+        if (!IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc) && 
+            NV_RM_PAGE_SIZE < os_page_size &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+        {
+            RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+                                    rmPageCount);
+        }
+
+        status = nv_alias_pages(
+            NV_GET_NV_STATE(pGpu),
+            osPageCount,
+            pageSize,
+            memdescGetContiguity(pMemDesc, AT_CPU),
+            cpuCacheAttrib,
+            memdescGetGuestId(pMemDesc),
+            memdescGetPteArray(pMemDesc, AT_CPU),
+            memdescIsCarveoutMemory(pMemDesc),
+            &pMemData);
+    }
+    else
+    {
+        NvBool unencrypted = 0;
+
+        if (nv && (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_32BIT_ADDRESSABLE)))
+            nv->force_dma32_alloc = NV_TRUE;
+
+        //
+        // If AMD SEV is enabled but APM(Ampere protected) or CC(Confidential
+        // Compute) mode is not enabled on the GPU, all RM and client
+        // allocations must be an unprotected sysmem.  If APM is enabled and RM
+        // is allocating sysmem for its internal use, then it has to be
+        // unprotected as protected sysmem is not accessible to GPU.
+        //
+        unencrypted = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY);
+
+        //
+        // OBJGPU/nv may be NULL if constructing NV01_MEMORY_DEVICELESS. Assume that
+        // NV01_MEMORY_DEVICELESS won't be used for display.
+        //
+        if (pGpu != NULL &&
+            pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY) &&
+            addrSpace == ADDR_SYSMEM)
+        {
+            if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) &&
+                !NV_SOC_IS_ISO_IOMMU_PRESENT(nv))
+            {
+                NV_PRINTF(LEVEL_INFO, "Forcing physically contiguous flags for ISO\n");
+                memdescSetContiguity(pMemDesc, AT_CPU, NV_TRUE);
+            }
+
+            if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) &&
+                !NV_SOC_IS_NISO_IOMMU_PRESENT(nv))
+            {
+                NV_PRINTF(LEVEL_INFO, "Forcing physically contiguous flags for NISO\n");
+                memdescSetContiguity(pMemDesc, AT_CPU, NV_TRUE);
+            }
+        }
+
+        status = osGetPagesInfo(pMemDesc, &pageSize, &osPageCount, &rmPageCount);
+        if (status != NV_OK)
+            goto done;
+
+        status = nv_alloc_pages(
+            NV_GET_NV_STATE(pGpu),
+            osPageCount,  // TODO: This call needs to receive the page count param at the requested page size.
+            pageSize,
+            memdescGetContiguity(pMemDesc, AT_CPU),
+            cpuCacheAttrib,
+            pSys->getProperty(pSys,
+                PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS),
+            unencrypted,
+            nodeId,
+            memdescGetPteArray(pMemDesc, AT_CPU),
+            &pMemData);
+
+        if (nv && nv->force_dma32_alloc)
+            nv->force_dma32_alloc = NV_FALSE;
+    }
+
+    if (status != NV_OK)
+    {
+        goto done;
+    }
+
+    // Guest allocated memory is already initialized
+    if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
+    {
+        if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+        {
+            NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, osPageCount * os_page_size, os_page_size));
+        }
+        else
+        {
+            NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, rmPageCount * RM_PAGE_SIZE, RM_PAGE_SIZE));
+        }
+    }
+
+    //
+    // If the OS layer doesn't think in RM page size, we need to inflate the
+    // PTE array into RM pages.
+    //
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        NV_ASSERT_OR_RETURN(pMemDesc->pageArrayGranularity == os_page_size, NV_ERR_INVALID_ARGUMENT);
+    }
+    else if (NV_RM_PAGE_SIZE < os_page_size &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+    {
+        RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+                                pMemDesc->PageCount);
+    }
+
+    memdescSetMemData(pMemDesc, pMemData, NULL);
+
+    if ((pGpu != NULL) && IS_VIRTUAL(pGpu))
+        NV_ASSERT_OK_OR_RETURN(vgpuUpdateGuestSysmemPfnBitMap(pGpu, pMemDesc, NV_TRUE));
+done:
+    return status;
+}
+
+void osFreePagesInternal(
+    MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+    OBJGPU *pGpu = pMemDesc->pGpu;
+    NV_STATUS rmStatus;
+    NvU64 osPageCount;
+
+    if ((pGpu != NULL) && IS_VIRTUAL(pGpu))
+        NV_ASSERT_OR_RETURN_VOID(vgpuUpdateGuestSysmemPfnBitMap(pGpu, pMemDesc, NV_FALSE) == NV_OK);
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        NV_ASSERT(pMemDesc->pageArrayGranularity == os_page_size);
+        osPageCount = pMemDesc->PageCount;
+    }
+    else if (NV_RM_PAGE_SIZE < os_page_size &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+    {
+        osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);
+        RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+                                pMemDesc->PageCount);
+    }
+    else
+    {
+        osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);
+    }
+
+    rmStatus = nv_free_pages(NV_GET_NV_STATE(pGpu),
+        osPageCount,
+        memdescGetContiguity(pMemDesc, AT_CPU),
+        memdescGetCpuCacheAttrib(pMemDesc),
+        memdescGetMemData(pMemDesc));
+    NV_ASSERT(rmStatus == NV_OK);
+}
+
+NV_STATUS osLockMem(
+    MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+    // Not supported on this OS.
+    DBG_BREAKPOINT();
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osUnlockMem(
+    MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+    // Not supported on this OS.
+    DBG_BREAKPOINT();
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osMapPciMemoryAreaUser
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    MemoryArea memArea,
+    NvU32 protect,
+    NvU32 mode,
+    NvP64 *pVirtualAddress,
+    NvP64 *pPriv
+)
+{
+    NV_STATUS status = NV_OK;
+    NvU64 origStart;
+    NvU64 origSize;
+    NvU64 diffStart;
+
+    if (memArea.numRanges == 0)
+    {
+        *pVirtualAddress = (NvP64) NULL;
+        *pPriv = NULL;
+        return NV_OK;
+    }
+
+    // Fix alignment insofar as we can (middle blocks can't be force-aligned this way)
+    origStart = memArea.pRanges[0].start;
+    memArea.pRanges[0].start = NV_ALIGN_DOWN64(origStart, os_page_size);
+    diffStart = origStart - memArea.pRanges[0].start;
+    memArea.pRanges[0].size += diffStart;
+    origSize = memArea.pRanges[memArea.numRanges - 1llu].size;
+    memArea.pRanges[memArea.numRanges - 1llu].size = NV_ALIGN_UP64(origSize, os_page_size);
+
+    {
+        nv_usermap_access_params_t **ppNvuap, tNvuap;
+        NvU64 totalRangeSize = sizeof(MemoryRange) * memArea.numRanges;
+
+        portMemSet(&tNvuap, 0, sizeof(nv_usermap_access_params_t));
+
+        tNvuap.memArea = memArea;
+        // access_size is only for caching, we can use os_page_size for now until linux has been properly plumbed
+        tNvuap.access_start = memArea.pRanges[0].start;
+        tNvuap.access_size = os_page_size;
+
+        tNvuap.caching = mode;
+        tNvuap.contig = NV_TRUE;
+
+        ppNvuap = (nv_usermap_access_params_t **) tlsEntryAcquire(TLS_ENTRY_ID_MAPPING_CONTEXT);
+        NV_ASSERT_OR_RETURN(ppNvuap != NULL, NV_ERR_INVALID_STATE);
+
+        NV_ASSERT_OK_OR_GOTO(status,
+            os_alloc_mem((void**) ppNvuap, sizeof(nv_alloc_mapping_context_t)),
+            free_tls);
+
+        portMemCopy(*ppNvuap, sizeof(nv_usermap_access_params_t),
+            &tNvuap, sizeof(nv_usermap_access_params_t));
+
+        NV_ASSERT_OK_OR_GOTO(status, 
+            os_alloc_mem((void**) &((*ppNvuap)->memArea.pRanges), totalRangeSize),
+            free_nvuap);
+
+        portMemCopy((*ppNvuap)->memArea.pRanges, totalRangeSize, memArea.pRanges, totalRangeSize);
+
+        *pVirtualAddress = (NvP64) (memArea.pRanges[0].start + diffStart);
+        goto unalign_and_return;
+
+free_nvuap:
+        os_free_mem(*ppNvuap);
+free_tls:
+        tlsEntryRelease(TLS_ENTRY_ID_MAPPING_CONTEXT);
+    }
+unalign_and_return:
+    memArea.pRanges[memArea.numRanges - 1llu].size =  origSize;
+    memArea.pRanges[0].size -= diffStart;
+    memArea.pRanges[0].start = origStart;
+    return status;
+}
+
+NV_STATUS osMapPciMemoryUser(
+    OS_GPU_INFO *pOsGpuInfo,
+    RmPhysAddr   busAddress,
+    NvU64        length,
+    NvU32        Protect,
+    NvP64       *pVirtualAddress,
+    NvP64       *pPriv,
+    NvU32        modeFlag
+)
+{
+    MemoryArea memArea;
+    MemoryRange memRange;
+
+    memArea.numRanges = 1;
+    memArea.pRanges = &memRange;
+
+    memRange.start = busAddress;
+    memRange.size = length;
+
+    return osMapPciMemoryAreaUser(pOsGpuInfo, memArea, Protect, modeFlag, pVirtualAddress, pPriv);
+}
+
+void osUnmapPciMemoryUser(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvP64        virtualAddress,
+    NvU64        length,
+    NvP64        pPriv
+)
+{
+}
+
+NV_STATUS osMapPciMemoryKernelOld
+(
+    OBJGPU    *pGpu,
+    RmPhysAddr busAddress,
+    NvU64      length,
+    NvU32      Protect,
+    void     **pVirtualAddress,
+    NvU32      modeFlag
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_kern_mapping_t *mapping;
+
+    if (pVirtualAddress == NULL)
+        return NV_ERR_GENERIC;
+
+    *pVirtualAddress = os_map_kernel_space(busAddress, length, modeFlag);
+    if (*pVirtualAddress == NULL)
+        return NV_ERR_GENERIC;
+
+    mapping = portMemAllocNonPaged(sizeof(nv_kern_mapping_t));
+    if (NULL == mapping)
+    {
+        os_unmap_kernel_space(*pVirtualAddress, length);
+        *pVirtualAddress = 0;
+        return NV_ERR_GENERIC;
+    }
+
+    mapping->addr = *pVirtualAddress;
+    mapping->size = length;
+    mapping->modeFlag = modeFlag;
+
+    mapping->next = nv->kern_mappings;
+    nv->kern_mappings = mapping;
+
+    return NV_OK;
+}
+
+NV_STATUS osMapPciMemoryKernel64
+(
+    OBJGPU    *pGpu,
+    RmPhysAddr busAddress,
+    NvU64      length,
+    NvU32      Protect,
+    NvP64     *pVirtualAddress,
+    NvU32      modeFlag
+)
+{
+    void *tmppVirtualAddress = NvP64_VALUE(pVirtualAddress);
+    NV_STATUS rc;
+
+    rc = osMapPciMemoryKernelOld(pGpu,
+                                 busAddress,
+                                 length,
+                                 Protect,
+                                 &tmppVirtualAddress,
+                                 modeFlag);
+
+    *pVirtualAddress = NV_PTR_TO_NvP64(tmppVirtualAddress);
+
+    return rc;
+}
+
+void osUnmapPciMemoryKernelOld
+(
+    OBJGPU *pGpu,
+    void*   virtualAddress
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_kern_mapping_t *mapping, *tmp;
+
+    // this can happen, for example, during a call to RmShutdownAdapter()
+    // from a failed RmInitAdapter()
+    if (virtualAddress == NULL)
+    {
+        return;
+    }
+
+    tmp = mapping = nv->kern_mappings;
+    while (mapping)
+    {
+        if (mapping->addr == virtualAddress)
+        {
+            if (mapping == nv->kern_mappings)
+            {
+                nv->kern_mappings = mapping->next;
+            }
+            else
+            {
+                tmp->next = mapping->next;
+            }
+
+            os_unmap_kernel_space(mapping->addr, mapping->size);
+
+            portMemFree(mapping);
+            return;
+        }
+        tmp = mapping;
+        mapping = mapping->next;
+    }
+
+    DBG_BREAKPOINT();
+}
+
+void osUnmapPciMemoryKernel64
+(
+    OBJGPU *pGpu,
+    NvP64   virtualAddress
+)
+{
+    osUnmapPciMemoryKernelOld(pGpu, NvP64_VALUE(virtualAddress));
+}
+
+NV_STATUS osMapGPU(
+    OBJGPU        *pGpu,
+    RS_PRIV_LEVEL  privLevel,
+    NvU64          offset,
+    NvU64          length,
+    NvU32          Protect,
+    NvP64         *pAddress,
+    NvP64         *pPriv
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+
+    if (privLevel >= RS_PRIV_LEVEL_KERNEL)
+    {
+        if (!portSafeAddU64((NvUPtr)pGpu->deviceMappings[0].gpuNvAddr, offset, (NvU64*)pAddress))
+        {
+            rmStatus = NV_ERR_INVALID_LIMIT;
+        }
+    }
+    else
+    {
+        RmPhysAddr busAddress;
+        if (!portSafeAddU64(pGpu->busInfo.gpuPhysAddr, offset, &busAddress))
+        {
+            rmStatus = NV_ERR_INVALID_LIMIT;
+        }
+        else
+        {
+            rmStatus = osMapPciMemoryUser(pGpu->pOsGpuInfo,
+                                          busAddress,
+                                          length,
+                                          Protect,
+                                          pAddress,
+                                          pPriv,
+                                          NV_MEMORY_UNCACHED);
+        }
+    }
+
+    return rmStatus;
+}
+
+void osUnmapGPU(
+    OS_GPU_INFO   *pOsGpuInfo,
+    RS_PRIV_LEVEL  privLevel,
+    NvP64          address,
+    NvU64          length,
+    NvP64          priv
+)
+{
+    if (privLevel < RS_PRIV_LEVEL_KERNEL)
+    {
+        osUnmapPciMemoryUser(pOsGpuInfo, address, length, priv);
+    }
+}
+
+static NV_STATUS referenceEventForPost(
+    nv_event_t *event,
+    NvBool bDoRefCount,
+    nv_event_t *eventSnapshot,
+    nv_file_private_t **ppNvfp
+)
+{
+    nv_state_t *nv = nv_get_ctl_state();
+
+    portSyncSpinlockAcquire(nv->event_spinlock);
+
+    if (event->active == 0)
+    {
+        portSyncSpinlockRelease(nv->event_spinlock);
+        return NV_ERR_INVALID_EVENT;
+    }
+
+    if (bDoRefCount)
+        ++event->refcount;
+
+    // Snapshot while event_spinlock still protects the event lifetime.
+    *eventSnapshot = *event;
+    *ppNvfp = eventSnapshot->nvfp;
+    portAtomicIncrementS32(&(*ppNvfp)->event_posting_refcount);
+
+    portSyncSpinlockRelease(nv->event_spinlock);
+    return NV_OK;
+}
+
+static void dereferenceEventForPost(
+    nv_event_t *event,
+    NvBool bDoRefCount,
+    nv_file_private_t *nvfp
+)
+{
+    if (bDoRefCount)
+    {
+        nv_state_t *nv = nv_get_ctl_state();
+
+        portSyncSpinlockAcquire(nv->event_spinlock);
+
+        NV_ASSERT(event->refcount > 0);
+        if (--event->refcount == 0 && !event->active)
+            portMemFree(event);
+
+        portSyncSpinlockRelease(nv->event_spinlock);
+    }
+
+    portAtomicDecrementS32(&nvfp->event_posting_refcount);
+}
+
+static void postEvent(
+    nv_event_t *event,
+    NvU32 hEvent,
+    NvU32 notifyIndex,
+    NvU32 info32,
+    NvU16 info16,
+    NvBool dataValid,
+    NvBool bDoRefCount
+)
+{
+    nv_event_t *eventToPost = event;
+
+    nv_file_private_t *nvfp = NULL;
+    nv_event_t eventSnapshot;
+
+    if (referenceEventForPost(event, bDoRefCount, &eventSnapshot, &nvfp) != NV_OK)
+        return;
+
+    eventToPost = &eventSnapshot;
+
+    nv_post_event(eventToPost, hEvent, notifyIndex,
+                  info32, info16, dataValid);
+
+    dereferenceEventForPost(bDoRefCount ? event : NULL, bDoRefCount, nvfp);
+}
+
+NvU32 osSetEvent
+(
+    OBJGPU   *pGpu,
+    NvP64     eventID
+)
+{
+    nv_event_t *event = NvP64_VALUE(eventID);
+    postEvent(event, 0, 0, 0, 0, NV_FALSE, NV_TRUE);
+    return 1;
+}
+
+NV_STATUS osNotifyEvent(
+    OBJGPU             *pGpu,
+    PEVENTNOTIFICATION  NotifyEvent,
+    NvU32               Method,
+    NvU32               Data,
+    NV_STATUS           Status,
+    NvBool              bDoRefCount
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+
+    // notify the event
+    switch (NotifyEvent->NotifyType)
+    {
+        case NV01_EVENT_OS_EVENT:
+        {
+            nv_event_t *event = NvP64_VALUE(NotifyEvent->Data);
+            postEvent(event,
+                      NotifyEvent->hEvent,
+                      NotifyEvent->NotifyIndex,
+                      0, 0,
+                      NotifyEvent->bEventDataRequired,
+                      bDoRefCount);
+            break;
+        }
+
+        // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX.
+        case NV01_EVENT_KERNEL_CALLBACK:
+        {
+            MINIPORT_CALLBACK callBackToMiniport =
+                (MINIPORT_CALLBACK)NvP64_VALUE(NotifyEvent->Data);
+
+            // perform a direct callback to the miniport
+            if (callBackToMiniport)
+                callBackToMiniport(NV_GET_NV_STATE(pGpu));
+            break;
+        }
+
+        case NV01_EVENT_KERNEL_CALLBACK_EX:
+        {
+            NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(NotifyEvent->Data);
+
+            // passes two arguments (arg, params) to the kernel callback instead of one (arg).
+            if (kc && kc->func)
+            {
+                kc->func(kc->arg, NULL, NotifyEvent->hEvent, Data, Status);
+            }
+            break;
+        }
+
+
+        default:
+        {
+            rmStatus = NV_ERR_GENERIC;
+            break;
+        }
+    }
+
+    return rmStatus;
+
+} // end of osNotifyEvent()
+
+// Allow CPL Events to be callback or events
+NV_STATUS osEventNotification
+(
+    OBJGPU   *pGpu,
+    PEVENTNOTIFICATION pNotifyEvent,
+    NvU32 notifyIndex,
+    void * pEventData,
+    NvU32 eventDataSize
+)
+{
+    return osEventNotificationWithInfo(pGpu, pNotifyEvent, notifyIndex, 0, 0,
+                                       pEventData, eventDataSize);
+}
+
+NV_STATUS osEventNotificationWithInfo
+(
+    OBJGPU   *pGpu,
+    PEVENTNOTIFICATION pNotifyEvent,
+    NvU32 notifyIndex,
+    NvU32 info32,
+    NvU16 info16,
+    void * pEventData,
+    NvU32 eventDataSize
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+    NvBool bMatchSubdev = NV_FALSE;
+
+    if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) &&
+        (OS_EVENT_NOTIFICATION_INDEX_MATCH_SUBDEV & notifyIndex))
+    {
+        bMatchSubdev = NV_TRUE;
+        notifyIndex &= ~OS_EVENT_NOTIFICATION_INDEX_MATCH_SUBDEV;
+    }
+
+    // walk this object's event list and find any matches for this specific notify
+    for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next)
+    {
+        // notifyIndex must match if request isn't for all
+        if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) &&
+            (pNotifyEvent->NotifyIndex != notifyIndex))
+        {
+            continue;
+        }
+
+        if (bMatchSubdev && (pGpu->subdeviceInstance != pNotifyEvent->subdeviceInst))
+            continue;
+
+        switch (pNotifyEvent->NotifyType)
+        {
+            case NV_EVENT_BUFFER_BIND:
+            case NV01_EVENT_WIN32_EVENT:
+            {
+                nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data);
+                postEvent(event,
+                          pNotifyEvent->hEvent,
+                          pNotifyEvent->NotifyIndex,
+                          info32, info16,
+                          pNotifyEvent->bEventDataRequired,
+                          NV_TRUE);
+                break;
+            }
+
+            // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX.
+            case NV01_EVENT_KERNEL_CALLBACK:
+            {
+                MINIPORT_CALLBACK callBackToMiniport =
+                    (MINIPORT_CALLBACK)NvP64_VALUE(pNotifyEvent->Data);
+
+                // perform a direct callback to the miniport
+                if (callBackToMiniport)
+                    callBackToMiniport(NV_GET_NV_STATE(pGpu));
+                break;
+            }
+
+            case NV01_EVENT_KERNEL_CALLBACK_EX:
+            {
+                NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data);
+
+                if (kc && kc->func)
+                {
+                    kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+    return rmStatus;
+}
+
+// Allow CPL Events to be callback or events
+NV_STATUS osObjectEventNotification
+(
+    NvHandle            hClient,
+    NvHandle            hObject,
+    NvU32               hClass,
+    PEVENTNOTIFICATION  pNotifyEvent,
+    NvU32               notifyIndex,
+    void                *pEventData,
+    NvU32               eventDataSize
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+
+    NV_PRINTF(LEVEL_INFO, "%s()\n", __FUNCTION__);
+    // walk this object's event list and find any matches for this specific notify
+    for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next)
+    {
+        // notifyIndex must match if request isn't for all
+        if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) &&
+            (pNotifyEvent->NotifyIndex != notifyIndex))
+        {
+            continue;
+        }
+
+        switch (pNotifyEvent->NotifyType)
+        {
+            case NV01_EVENT_OS_EVENT:
+            {
+                nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data);
+                postEvent(event,
+                          pNotifyEvent->hEvent,
+                          pNotifyEvent->NotifyIndex,
+                          0, 0,
+                          pNotifyEvent->bEventDataRequired,
+                          NV_TRUE);
+                break;
+            }
+
+            case NV01_EVENT_KERNEL_CALLBACK_EX:
+            {
+                NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data);
+
+                if (kc && kc->func)
+                {
+                    kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+    return rmStatus;
+}
+
+NV_STATUS osReferenceObjectCount(void *pEvent)
+{
+    nv_state_t *nv = nv_get_ctl_state();
+    nv_event_t *event = pEvent;
+
+    portSyncSpinlockAcquire(nv->event_spinlock);
+    // If event->active is false, don't allow any more reference
+    if (event->active == 0)
+    {
+        portSyncSpinlockRelease(nv->event_spinlock);
+        return NV_ERR_INVALID_EVENT;
+    }
+    ++event->refcount;
+    portSyncSpinlockRelease(nv->event_spinlock);
+    return NV_OK;
+}
+
+NV_STATUS osDereferenceObjectCount(void *pOSEvent)
+{
+    nv_state_t *nv = nv_get_ctl_state();
+    nv_event_t *event = pOSEvent;
+
+    portSyncSpinlockAcquire(nv->event_spinlock);
+    NV_ASSERT(event->refcount > 0);
+    // If event->refcount == 0 but event->active is true, the client
+    // has not yet freed the OS event.  free_os_event will free its
+    // memory when they do, or else when the client itself is freed.
+    if (--event->refcount == 0 && !event->active)
+        portMemFree(event);
+    portSyncSpinlockRelease(nv->event_spinlock);
+
+    return NV_OK;
+}
+
+NV_STATUS osUserHandleToKernelPtr(NvHandle hClient, NvP64 hEvent, NvP64 *pEvent)
+{
+    nv_state_t *nv = nv_get_ctl_state();
+    NvU32 fd = (NvU64)hEvent;
+    NV_STATUS result;
+
+    portSyncSpinlockAcquire(nv->event_spinlock);
+    nv_event_t *e = nv->event_list;
+    while (e != NULL)
+    {
+        if (e->fd == fd && e->hParent == hClient)
+            break;
+        e = e->next;
+    }
+
+    if (e != NULL)
+    {
+        ++e->refcount;
+        *pEvent = NV_PTR_TO_NvP64(e);
+        result = NV_OK;
+    }
+    else
+        result = NV_ERR_OBJECT_NOT_FOUND;
+    portSyncSpinlockRelease(nv->event_spinlock);
+
+    return result;
+}
+
+NV_STATUS osFlushCpuCache(void)
+{
+    return os_flush_cpu_cache_all();
+}
+
+void osFlushCpuWriteCombineBuffer(void)
+{
+    os_flush_cpu_write_combine_buffer();
+}
+
+
+//
+// Evict GPU memory range from the CPU caches.
+//
+// On some platforms (e.g. P9+V100), the CPU can coherently cache GPU memory
+// and RM takes advantage of that. Most everything is handled transparently,
+// but there are two exceptions that require explicitly flushing any CPU cache
+// lines of GPU memory. These are:
+//
+// 1) Flushing memory backing ACR regions before they get locked.
+//
+// Otherwise the cache could get flushed while the regions are locked causing a
+// region violation physical fault. See more details in
+// acrFlushRegionsFromGpuCoherentCpuCache_IMPL().
+//
+// 2) Flushing all of FB before GPU reset (NVLink going down)
+//
+// Leaving cache entries on the CPU causes fatal errors when the CPU tries
+// flushing them later while the link is down. See more details in
+// nvlinkStatePostUnload_IMPL().
+//
+void osFlushGpuCoherentCpuCacheRange
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU64        cpuVirtual,
+    NvU64        size
+)
+{
+    nv_flush_coherent_cpu_cache_range(pOsGpuInfo, cpuVirtual, size);
+}
+
+void osErrorLogV(OBJGPU *pGpu, XidContext context, const char * pFormat, va_list arglist)
+{
+    NV_STATUS        rmStatus;
+    nv_state_t      *nv             = NV_GET_NV_STATE(pGpu);
+
+    if ((pFormat == NULL) || (*pFormat == '\0'))
+    {
+        return;
+    }
+
+    rmStatus = nv_log_error(nv, context.xid, pFormat, arglist);
+    NV_ASSERT(rmStatus == NV_OK);
+}
+
+void osErrorLog(OBJGPU *pGpu, NvU32 num, const char* pFormat, ...)
+{
+    va_list arglist;
+    va_start(arglist, pFormat);
+    osErrorLogV(pGpu, (XidContext){.xid = num}, pFormat, arglist);
+    va_end(arglist);
+}
+
+NvU32
+osPollHotkeyState
+(
+    OBJGPU  *pGpu
+)
+{
+    return 0;
+}
+
+void osDevWriteReg008(
+     OBJGPU            *pGpu,
+     DEVICE_MAPPING    *pMapping,
+     NvU32              thisAddress,
+     NvV8               thisValue
+)
+{
+    if (thisAddress >= pMapping->gpuNvLength)
+    {
+        NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+        return;
+    }
+
+    NV_PRIV_REG_WR08(pMapping->gpuNvAddr, thisAddress, thisValue);
+}
+
+void osDevWriteReg016(
+     OBJGPU            *pGpu,
+     DEVICE_MAPPING    *pMapping,
+     NvU32              thisAddress,
+     NvV16              thisValue
+)
+{
+    if (thisAddress >= pMapping->gpuNvLength)
+    {
+        NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+        return;
+    }
+
+    NV_PRIV_REG_WR16(pMapping->gpuNvAddr, thisAddress, thisValue);
+}
+
+void osDevWriteReg032(
+     OBJGPU            *pGpu,
+     DEVICE_MAPPING    *pMapping,
+     NvU32              thisAddress,
+     NvV32              thisValue
+)
+{
+    NvBool vgpuHandled = NV_FALSE;
+
+    vgpuDevWriteReg032(pGpu, thisAddress, thisValue, &vgpuHandled);
+    if (vgpuHandled)
+    {
+        return;
+    }
+
+    if (thisAddress >= pMapping->gpuNvLength)
+    {
+        NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+        return;
+    }
+
+    NV_PRIV_REG_WR32(pMapping->gpuNvAddr, thisAddress, thisValue);
+}
+
+NvU8 osDevReadReg008(
+    OBJGPU             *pGpu,
+    DEVICE_MAPPING     *pMapping,
+    NvU32               thisAddress
+)
+{
+    NvU8 retval = 0;
+
+    if (thisAddress >= pMapping->gpuNvLength)
+    {
+        NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+    }
+    else
+        retval = NV_PRIV_REG_RD08(pMapping->gpuNvAddr, thisAddress);
+
+    return retval;
+}
+
+NvU16 osDevReadReg016(
+    OBJGPU             *pGpu,
+    DEVICE_MAPPING     *pMapping,
+    NvU32               thisAddress
+)
+{
+    NvU16 retval = 0;
+
+    if (thisAddress >= pMapping->gpuNvLength)
+    {
+        NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+    }
+    else
+        retval = NV_PRIV_REG_RD16(pMapping->gpuNvAddr, thisAddress);
+
+    return retval;
+}
+
+NvU32 osDevReadReg032(
+    OBJGPU             *pGpu,
+    DEVICE_MAPPING     *pMapping,
+    NvU32               thisAddress
+)
+{
+    NvU32 retval = 0;
+    NvBool vgpuHandled = NV_FALSE;
+
+    retval = vgpuDevReadReg032(pGpu, thisAddress, &vgpuHandled);
+    if (vgpuHandled)
+    {
+        return retval;
+    }
+
+    if (thisAddress >= pMapping->gpuNvLength)
+    {
+        NV_ASSERT(thisAddress < pMapping->gpuNvLength);
+    }
+    else
+        retval = NV_PRIV_REG_RD32(pMapping->gpuNvAddr, thisAddress);
+
+    return retval;
+}
+
+NV_STATUS osReadRegistryDwordBase(
+    OBJGPU     *pGpu,
+    const char *regParmStr,
+    NvU32      *Data
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    return RmReadRegistryDword(nv, regParmStr, Data);
+}
+
+NV_STATUS osWriteRegistryDword(
+    OBJGPU     *pGpu,
+    const char *regParmStr,
+    NvU32       Data
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    return RmWriteRegistryDword(nv, regParmStr, Data);
+}
+
+NV_STATUS osReadRegistryBinary(
+    OBJGPU     *pGpu,
+    const char *regParmStr,
+    NvU8       *Data,
+    NvU32      *cbLen
+)
+{
+#if defined(DEBUG) || defined(DEVELOP)
+    NV_ASSERT_OK_OR_RETURN(osValidateRegistryKeyPrefix(regParmStr));
+#endif
+
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    return RmReadRegistryBinary(nv, regParmStr, Data, cbLen);
+}
+
+NV_STATUS osWriteRegistryBinary(
+    OBJGPU     *pGpu,
+    const char *regParmStr,
+    NvU8       *Data,
+    NvU32       cbLen
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    return RmWriteRegistryBinary(nv, regParmStr, Data, cbLen);
+}
+
+NV_STATUS osWriteRegistryVolatile(
+    OBJGPU     *pGpu,
+    const char *regParmStr,
+    NvU8       *Data,
+    NvU32       cbLen
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osReadRegistryVolatile
+(
+    OBJGPU     *pGpu,
+    const char *regParmStr,
+    NvU8       *Data,
+    NvU32       cbLen
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osReadRegistryVolatileSize
+(
+    OBJGPU     *pGpu,
+    const char *regParmStr,
+    NvU32      *pSize
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS osReadRegistryStringBase(
+    OBJGPU     *pGpu,
+    const char *regParmStr,
+    NvU8       *buffer,
+    NvU32      *pBufferLength
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    return RmReadRegistryString(nv, regParmStr, buffer, pBufferLength);
+}
+
+NV_STATUS osPackageRegistry(
+    OBJGPU                 *pGpu,
+    PACKED_REGISTRY_TABLE  *pRegTable,
+    NvU32                  *pSize
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    return RmPackageRegistry(nv, pRegTable, pSize);
+}
+
+NvU32 osGetCpuCount(void)
+{
+    return os_get_cpu_count();   // Total number of logical CPUs.
+}
+
+NvU32 osGetCurrentProcessorNumber(void)
+{
+    return os_get_cpu_number();
+}
+
+ClientGroupID osClientGroupID(NvU64 process, void *pidInfo)
+{
+    return os_cgroup_for_pid(process, pidInfo);
+}
+
+NvCgroupImpl osCgroupImplementation(void)
+{
+    switch (os_cgroup_implementation())
+    {
+    case OS_CGROUP_IMPL_DMEM: return CGROUP_IMPL_OS;
+    case OS_CGROUP_IMPL_MISC: return CGROUP_IMPL_FALLBACK;
+    default:                  return CGROUP_IMPL_NONE;
+    }
+}
+
+void *osCgroupRegisterRegion(OBJGPU *pGpu, NvU64 size)
+{
+    char name[32];
+    nvDbgSnprintf(name, sizeof name, "nvidia/%04x:%02x:%02x/vidmem",
+        gpuGetDomain(pGpu),
+        gpuGetBus(pGpu),
+        gpuGetDevice(pGpu));
+    return os_dmem_cgroup_register_region(size, name);
+}
+
+void osCgroupUnregisterRegion(void *region)
+{
+    os_dmem_cgroup_unregister_region(region);
+}
+
+NV_STATUS osMemacctTryCharge(void *pRegion, NvLength size, void **ppPool)
+{
+    return os_dmem_cgroup_try_charge(pRegion, size, ppPool, NULL);
+}
+
+void osMemacctReleaseCharge(void *pool, NvLength size)
+{
+    os_dmem_cgroup_uncharge(pool, size);
+}
+
+void *osCgroupGetFromFd(NvS32 fd)
+{
+    return os_cgroup_get_from_fd(fd);
+}
+
+void osCgroupPut(void *cgroup)
+{
+    os_cgroup_put(cgroup);
+}
+
+void osGetTimeoutParams(OBJGPU *pGpu, NvU32 *pTimeoutUs, NvU32 *pScale, NvU32 *pFlags)
+{
+    NvU32 gpuMode   = gpuGetMode(pGpu);
+
+    NV_ASSERT((NV_GPU_MODE_GRAPHICS_MODE == gpuMode) ||
+              (NV_GPU_MODE_COMPUTE_MODE  == gpuMode));
+
+    if (hypervisorIsVgxHyper())
+    {
+        //
+        // 1.8 seconds is chosen because it is 90% of the overall hard limit of 2.0
+        // seconds, imposed by WDDM driver rules.
+        // Currently primary use case of VGX is Windows, so setting 1.8 as default
+        //
+        *pTimeoutUs = 1.8 * 1000000;
+    }
+    else
+    {
+        switch (gpuMode)
+        {
+        default:
+        case NV_GPU_MODE_GRAPHICS_MODE:
+            *pTimeoutUs = 4 * 1000000;
+            break;
+
+        case NV_GPU_MODE_COMPUTE_MODE:
+            *pTimeoutUs = 30 * 1000000;
+            break;
+        }
+    }
+
+    *pFlags = GPU_TIMEOUT_FLAGS_OSTIMER;
+
+    *pScale = 1;
+    if (IS_EMULATION(pGpu) || IS_SIMULATION(pGpu))
+    {
+        *pScale = 60;       // 1s -> 1m
+    }
+
+    return;
+}
+
+void osFlushLog(void)
+{
+    // Not implemented
+}
+
+static NvU32 _osGetTegraPlatform(void)
+{
+    NV_STATUS status;
+    NvU32 mode;
+
+    status = os_get_tegra_platform(&mode);
+    if (status != NV_ERR_NOT_SUPPORTED)
+    {
+        return mode;
+    }
+
+    return NV_OS_TEGRA_PLATFORM_SILICON;
+}
+
+NvU32 osGetSimulationMode(void)
+{
+    NvU32 mode;
+
+    switch (_osGetTegraPlatform())
+    {
+        case NV_OS_TEGRA_PLATFORM_SIM:
+            mode = NV_SIM_MODE_CMODEL;
+            break;
+        case NV_OS_TEGRA_PLATFORM_FPGA:
+            mode = NV_SIM_MODE_TEGRA_FPGA;
+            break;
+        case NV_OS_TEGRA_PLATFORM_SILICON:
+        default:
+            mode = NV_SIM_MODE_HARDWARE;
+            break;
+    }
+
+    return mode;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixFlushUserCache_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS *pAddressSpaceParams
+)
+{
+    Memory *pMemory;
+    MEMORY_DESCRIPTOR *pMemDesc;
+    NvU64 start, end;
+    NvBool bInvalidateOnly;
+
+    NV_CHECK_OK_OR_RETURN(LEVEL_SILENT,
+        memGetByHandle(RES_GET_CLIENT(pRmCliRes),
+                       pAddressSpaceParams->hObject,
+                       &pMemory));
+
+    pMemDesc = pMemory->pMemDesc;
+
+    if (memdescGetAddressSpace(pMemDesc) != ADDR_SYSMEM)
+    {
+        NV_PRINTF(LEVEL_ERROR, "%s: wrong address space %d\n",
+                  __FUNCTION__, memdescGetAddressSpace(pMemDesc));
+        return NV_ERR_INVALID_COMMAND;
+    }
+
+    if (memdescGetCpuCacheAttrib(pMemDesc) != NV_MEMORY_CACHED)
+    {
+        NV_PRINTF(LEVEL_ERROR, "%s: wrong caching type %d\n",
+                  __FUNCTION__, memdescGetCpuCacheAttrib(pMemDesc));
+        return NV_ERR_INVALID_COMMAND;
+    }
+
+    start = pAddressSpaceParams->offset;
+    end = start + pAddressSpaceParams->length;
+
+    switch(pAddressSpaceParams->cacheOps)
+    {
+        case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH_INVALIDATE:
+        case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH:
+            bInvalidateOnly = NV_FALSE;
+            break;
+
+        case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_INVALIDATE:
+            bInvalidateOnly = NV_TRUE;
+            break;
+
+        default:
+            NV_PRINTF(LEVEL_ERROR, "%s: cacheOps not specified\n", __FUNCTION__);
+            return NV_ERR_INVALID_COMMAND;
+    }
+
+    if ((end - start) > pMemory->Length)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s: end address 0x%llx exceeded buffer length: 0x%llx\n",
+                  __FUNCTION__, end, pMemory->Length);
+        return NV_ERR_INVALID_LIMIT;
+    }
+
+    if (bInvalidateOnly)
+    {
+        //
+        // XXX: this seems fishy - I'm not sure if invalidating by the kernel
+        // VA only as nv_dma_cache_invalidate() does here is sufficient for
+        // this control call.
+        // pAddressSpaceParams->internalOnly is expected to be the RM client
+        // VA for this control call; if we wanted to invalidate the user VA we
+        // could do so using that.
+        //
+        // For I/O coherent platforms this won't actually do anything.
+        // On non-I/O-coherent platforms, there's no need to do a second
+        // invalidation after the full flush.
+        //
+        nv_state_t *nv = NV_GET_NV_STATE(pMemDesc->pGpu);
+        if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
+        {
+            PIOVAMAPPING pIovaMapping = memdescGetIommuMap(pMemDesc, nv->iovaspace_id);
+            //
+            // This should only be called for devices that map memory descriptors
+            // through the nv-dma library, where the memory descriptor data
+            // contains all the kernel-specific context we need for the
+            // invalidation.
+            //
+            // (These checks match those in osIovaUnmap() leading up to
+            // nv_dma_unmap_alloc()).
+            //
+            if (pIovaMapping == NULL ||
+                pIovaMapping->pOsData == NULL ||
+                memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED) ||
+                memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
+            {
+                return NV_ERR_INVALID_ARGUMENT;
+            }
+
+            nv_dma_cache_invalidate(nv->dma_dev, pIovaMapping->pOsData);
+        }
+        else
+        {
+            return NV_ERR_INVALID_ARGUMENT;
+        }
+    }
+    else
+    {
+        return os_flush_user_cache();
+    }
+
+    return NV_OK;
+}
+
+static NV_STATUS
+_initializeExportObjectFd
+(
+    nv_file_private_t *nvfp,
+    NvHandle           hClient,
+    NvHandle           hDevice,
+    NvU16              maxObjects,
+    NvU8              *metadata
+)
+{
+    NV_STATUS      status;
+    RsResourceRef *pResourceRef;
+    Device        *pDevice;
+    NvU32          deviceInstance = NV_MAX_DEVICES;
+    NvU32          gpuInstanceId = NV_U32_MAX;
+
+    if (nvfp->handles != NULL)
+    {
+        return NV_ERR_STATE_IN_USE;
+    }
+
+    if (hDevice != 0)
+    {
+        OBJGPU *pGpu;
+        MIG_INSTANCE_REF ref;
+
+        status = serverutilGetResourceRef(hClient, hDevice, &pResourceRef);
+        if (status != NV_OK)
+        {
+            return status;
+        }
+
+        pDevice = dynamicCast(pResourceRef->pResource, Device);
+        if (pDevice == NULL)
+        {
+            return NV_ERR_INVALID_PARAMETER;
+        }
+
+        deviceInstance = pDevice->deviceInst;
+        pGpu = GPU_RES_GET_GPU(pDevice);
+
+        if (IS_MIG_IN_USE(pGpu))
+        {
+            KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
+            status = kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager,
+                                                     pDevice, &ref);
+            if (status == NV_OK)
+            {
+                gpuInstanceId = ref.pKernelMIGGpuInstance->swizzId;
+            }
+        }
+    }
+
+    NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void **)&nvfp->handles,
+                           sizeof(nvfp->handles[0]) * maxObjects));
+
+    os_mem_set(nvfp->handles, 0,
+               sizeof(nvfp->handles[0]) * maxObjects);
+
+    nvfp->maxHandles        = maxObjects;
+    nvfp->deviceInstance    = deviceInstance;
+    nvfp->gpuInstanceId     = gpuInstanceId;
+
+    if (metadata != NULL)
+    {
+        os_mem_copy(nvfp->metadata, metadata, sizeof(nvfp->metadata));
+    }
+
+    return NV_OK;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixExportObjectToFd_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS *pParams
+)
+{
+    NvHandle           hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+    RmObjExportHandle  hExportHandle = 0;
+    nv_file_private_t *nvfp = NULL;
+    void              *priv = NULL;
+    NV_STATUS          status = NV_OK;
+
+    /*
+     * This flag is intended to be implemented entirely in the rmapi library in
+     * userspace, we should never encounter it here.
+     */
+    if (FLD_TEST_DRF(0000_CTRL, _OS_UNIX_EXPORT_OBJECT_TO_FD_FLAGS,
+                     _EMPTY_FD, _TRUE, pParams->flags))
+    {
+        return NV_ERR_INVALID_PARAMETER;
+    }
+
+    if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM ||
+        pParams->fd == -1)
+    {
+        return NV_ERR_INVALID_PARAMETER;
+    }
+
+    status = RmExportObject(hClient,
+                            pParams->object.data.rmObject.hObject,
+                            &hExportHandle, NULL);
+    if (status != NV_OK)
+    {
+        goto done;
+    }
+    NV_ASSERT(hExportHandle != 0);
+
+    nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+    if (nvfp == NULL)
+    {
+        NV_ASSERT(priv == NULL);
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    status = _initializeExportObjectFd(nvfp, hClient,
+                                       pParams->object.data.rmObject.hDevice,
+                                       1, NULL);
+    if (status != NV_OK)
+    {
+        goto done;
+    }
+
+    nvfp->handles[0] = hExportHandle;
+
+done:
+
+    if (status != NV_OK && hExportHandle != 0)
+    {
+        RmFreeObjExportHandle(hExportHandle);
+    }
+
+    if (priv != NULL)
+    {
+        nv_put_file_private(priv);
+    }
+
+    return status;
+}
+
+// This control call has been deprecated. It will be deleted soon.
+NV_STATUS
+cliresCtrlCmdOsUnixCreateExportObjectFd_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS *pParams
+)
+{
+    NV_STATUS          status;
+    NvHandle           hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+    nv_file_private_t *nvfp = NULL;
+    void              *priv = NULL;
+
+    ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata));
+
+    if (pParams->maxObjects == 0)
+    {
+        return NV_ERR_INVALID_PARAMETER;
+    }
+
+    nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+    if (nvfp == NULL)
+    {
+        NV_ASSERT(priv == NULL);
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice,
+                                       pParams->maxObjects, pParams->metadata);
+
+done:
+    if (priv != NULL)
+    {
+        nv_put_file_private(priv);
+    }
+
+    return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixExportObjectsToFd_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS *pParams
+)
+{
+    NvHandle           hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+    RmObjExportHandle *pExportHandle;
+    nv_file_private_t *nvfp = NULL;
+    void              *priv = NULL;
+    NV_STATUS          status = NV_OK;
+    NvU32              i;
+    NvU32              deviceInstance;
+    NvU32              result;
+    NvHandle          *exportHandles = NULL;
+    NvBool             bFdSetup = NV_FALSE;
+
+    nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+    if (nvfp == NULL)
+    {
+        NV_ASSERT(priv == NULL);
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata));
+
+    /* Setup export FD if not done */
+    if (nvfp->handles == NULL)
+    {
+        if (pParams->maxObjects == 0)
+        {
+            status = NV_ERR_INVALID_PARAMETER;
+            goto done;
+        }
+
+        status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice,
+                                           pParams->maxObjects,
+                                           pParams->metadata);
+        if (status != NV_OK)
+        {
+            goto done;
+        }
+
+        bFdSetup = NV_TRUE;
+    }
+
+    if ((nvfp->handles == NULL) ||
+        (pParams->numObjects >
+            NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS))
+    {
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) ||
+        (result > nvfp->maxHandles))
+    {
+        status = NV_ERR_OUT_OF_RANGE;
+        goto done;
+    }
+
+    status = os_alloc_mem((void **)&exportHandles,
+                          sizeof(exportHandles[0]) *
+                            pParams->numObjects);
+    if (status != NV_OK)
+    {
+        goto done;
+    }
+
+    for (i = 0; i < pParams->numObjects; i++)
+    {
+        exportHandles[i] = 0;
+
+        if (pParams->objects[i] == 0)
+        {
+            continue;
+        }
+
+        status = RmExportObject(hClient,
+                                pParams->objects[i],
+                                &exportHandles[i],
+                                &deviceInstance);
+        if (status != NV_OK)
+        {
+            goto done;
+        }
+
+        NV_ASSERT(exportHandles[i] != 0);
+
+        if (deviceInstance != nvfp->deviceInstance)
+        {
+            status = NV_ERR_INVALID_PARAMETER;
+            goto done;
+        }
+    }
+
+    for (i = 0; i < pParams->numObjects; i++)
+    {
+        pExportHandle = &nvfp->handles[i + pParams->index];
+
+        // If the handle already exists in this position, free it
+        if (*pExportHandle != 0)
+        {
+            RmFreeObjExportHandle(*pExportHandle);
+            *pExportHandle = 0;
+        }
+
+        *pExportHandle = exportHandles[i];
+    }
+
+done:
+
+    if ((status != NV_OK) && (exportHandles != NULL))
+    {
+        for (i = 0; i < pParams->numObjects; i++)
+        {
+            if (exportHandles[i] != 0)
+            {
+                RmFreeObjExportHandle(exportHandles[i]);
+            }
+        }
+    }
+
+    if (exportHandles != NULL)
+    {
+        os_free_mem(exportHandles);
+    }
+
+    if ((status != NV_OK) && bFdSetup)
+    {
+        os_free_mem(nvfp->handles);
+        nvfp->handles = NULL;
+        nvfp->maxHandles = 0;
+    }
+
+    if (priv != NULL)
+    {
+        nv_put_file_private(priv);
+    }
+
+    return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixImportObjectFromFd_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS *pParams
+)
+{
+    NvHandle           hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+    nv_file_private_t *nvfp = NULL;
+    void              *priv = NULL;
+    NV_STATUS          status = NV_OK;
+
+    if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM ||
+        pParams->fd == -1)
+    {
+        return NV_ERR_INVALID_PARAMETER;
+    }
+
+    nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+    if (nvfp == NULL)
+    {
+        NV_ASSERT(priv == NULL);
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    if ((nvfp->handles == NULL) || (nvfp->handles[0] == 0) ||
+        (nvfp->maxHandles < 1))
+    {
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    status = RmImportObject(hClient,
+                            pParams->object.data.rmObject.hParent,
+                            &pParams->object.data.rmObject.hObject,
+                            nvfp->handles[0], NULL);
+
+done:
+    if (priv != NULL)
+    {
+        nv_put_file_private(priv);
+    }
+
+    return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixImportObjectsFromFd_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS *pParams
+)
+{
+    NvHandle           hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
+    nv_file_private_t *nvfp = NULL;
+    void              *priv = NULL;
+    NV_STATUS          status = NV_OK;
+    NvU32              i = 0;
+    RmObjExportHandle  hImportHandle;
+    NvU32              result;
+    RM_API            *pRmApi;
+
+    nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+    if (nvfp == NULL)
+    {
+        NV_ASSERT(priv == NULL);
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    if ((nvfp->handles == NULL) ||
+        (pParams->numObjects >
+           NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS))
+    {
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) ||
+        (result > nvfp->maxHandles))
+    {
+        status = NV_ERR_OUT_OF_RANGE;
+        goto done;
+    }
+
+    for (i = 0; i < pParams->numObjects; i++)
+    {
+        hImportHandle = nvfp->handles[i + pParams->index];
+
+        /* Nothing to import, just continue */
+        if (hImportHandle == 0)
+        {
+            pParams->objectTypes[i] = \
+                NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_NONE;
+            continue;
+        }
+
+        status = RmImportObject(hClient,
+                                pParams->hParent,
+                                &pParams->objects[i],
+                                hImportHandle,
+                                &pParams->objectTypes[i]);
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR, "%s: Unable to import handle (%x, %x, %x)\n",
+                __FUNCTION__, pParams->hParent, pParams->objects[i], hImportHandle);
+            goto done;
+        }
+    }
+
+done:
+
+    if (status != NV_OK)
+    {
+        pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+        while (i > 0)
+        {
+            i--;
+
+            if (pParams->objects[i] != 0)
+            {
+                pRmApi->Free(pRmApi, hClient, pParams->objects[i]);
+            }
+        }
+    }
+
+    if (priv != NULL)
+    {
+        nv_put_file_private(priv);
+    }
+
+    return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixGetExportObjectInfo_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS *pParams
+)
+{
+    nv_file_private_t *nvfp = NULL;
+    void              *priv = NULL;
+    NV_STATUS         status = NV_OK;
+
+    if (pParams->fd < 0)
+    {
+        return NV_ERR_INVALID_PARAMETER;
+    }
+
+    nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
+    if (nvfp == NULL)
+    {
+        NV_ASSERT(priv == NULL);
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    if (nvfp->handles == NULL)
+    {
+        status = NV_ERR_INVALID_PARAMETER;
+        goto done;
+    }
+
+    pParams->maxObjects = nvfp->maxHandles;
+    pParams->deviceInstance = nvfp->deviceInstance;
+    pParams->gpuInstanceId = nvfp->gpuInstanceId;
+
+    os_mem_copy(pParams->metadata, nvfp->metadata, sizeof(nvfp->metadata));
+
+done:
+    if (priv != NULL)
+    {
+        nv_put_file_private(priv);
+    }
+
+    return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixMemacctSetLimits_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_MEMACCT_SET_LIMITS_PARAMS *pParams
+)
+{
+    void *cgroup = osCgroupGetFromFd(pParams->cgroupFd);
+    if (cgroup == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    NV_STATUS status = memacctSetLimits((ClientGroupID)cgroup, pParams->gpuId, pParams->softLimit, pParams->hardLimit);
+    osCgroupPut(cgroup);
+    return status;
+}
+
+NV_STATUS
+cliresCtrlCmdOsUnixMemacctGetLimits_IMPL
+(
+    RmClientResource *pRmCliRes,
+    NV0000_CTRL_OS_UNIX_MEMACCT_GET_LIMITS_PARAMS *pParams
+)
+{
+    NvU32     gpuMask        = 0;
+    NvU32     gpuIdx         = 0;
+    OBJGPU   *pGpu           = NULL;
+    NV_STATUS status         = NV_OK;
+
+    pParams->count = 0;
+    NV_ASSERT_OK_OR_RETURN(gpumgrGetGpuAttachInfo(NULL, &gpuMask));
+
+    void *cgroup = osCgroupGetFromFd(pParams->cgroupFd);
+    if (cgroup == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    NvU32 i = 0;
+    while ((i < NV0000_CTRL_GPU_MAX_ATTACHED_GPUS) &&
+        ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuIdx)) != NULL))
+    {
+        NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
+            memacctGetLimits((ClientGroupID)cgroup, pGpu->gpuId,
+                &pParams->limits[i].soft,
+                &pParams->limits[i].hard,
+                &pParams->limits[i].current),
+            out);
+        pParams->limits[i].gpuId = pGpu->gpuId;
+        i++;
+    }
+    pParams->count = i;
+out:
+    osCgroupPut(cgroup);
+    return status;
+}
+
+/*!
+ * osAcpiDsm
+ *
+ * @brief Handles os specific _DSM method function calls.
+ *
+ * Input parameters:
+ * @param[in]     pGpu   : OBJGPU pointer
+ * @param[in]     acpiDsmFunction    : ACPI DSM function
+ * @param[in]     acpiDsmSubFunction : ACPI DSM subfunction
+ * @param[in/out] pInOut : in/out buffer, caller should make sure the buffer is large enough.
+ * @param[in]     pSize  : when input, size of data that the caller wants to read, in bytes.
+ *                         when output, size of valid data in pInOuta in bytes.
+ */
+NV_STATUS osCallACPI_DSM
+(
+    OBJGPU            *pGpu,
+    ACPI_DSM_FUNCTION  acpiDsmFunction,
+    NvU32              acpiDsmSubFunction,
+    NvU32             *pInOut,
+    NvU16             *pSize
+)
+{
+    NV_STATUS   status;
+    NvU8       *pAcpiDsmGuid = NULL;
+    NvU32       acpiDsmRev;
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+    NvU16       acpiDsmInArgSize = 4;
+    NvBool      acpiNvpcfDsmFunction = NV_FALSE;
+
+    // do any handling/remapping of guid needed.
+    status = checkDsmCall(pGpu,
+                          (ACPI_DSM_FUNCTION *) &acpiDsmFunction,
+                          &acpiDsmSubFunction,
+                          pInOut,
+                          pSize);
+
+    // return if subfunction is not supported or we're returning cache data
+    if (status != NV_WARN_MORE_PROCESSING_REQUIRED)
+    {
+        return status;
+    }
+
+    switch ((NvU32) acpiDsmFunction)
+    {
+        case ACPI_DSM_FUNCTION_NBSI:
+            pAcpiDsmGuid = (NvU8 *) &NBSI_DSM_GUID;
+            acpiDsmRev  = NBSI_REVISION_ID;
+            break;
+        case ACPI_DSM_FUNCTION_NVHG:
+            pAcpiDsmGuid = (NvU8 *) &NVHG_DSM_GUID;
+            acpiDsmRev  = NVHG_REVISION_ID;
+            break;
+        case ACPI_DSM_FUNCTION_MXM:
+            pAcpiDsmGuid = (NvU8 *) &DSM_MXM_GUID;
+            acpiDsmRev  = ACPI_MXM_REVISION_ID;
+            break;
+        case ACPI_DSM_FUNCTION_NBCI:
+            pAcpiDsmGuid = (NvU8 *) &NBCI_DSM_GUID;
+            acpiDsmRev  = NBCI_REVISION_ID;
+            break;
+        case ACPI_DSM_FUNCTION_NVOP:
+            pAcpiDsmGuid = (NvU8 *) &NVOP_DSM_GUID;
+            acpiDsmRev  = NVOP_REVISION_ID;
+            break;
+        case ACPI_DSM_FUNCTION_PCFG:
+            pAcpiDsmGuid = (NvU8 *) &PCFG_DSM_GUID;
+            acpiDsmRev  = PCFG_REVISION_ID;
+            break;
+        case ACPI_DSM_FUNCTION_GPS_2X:
+            pAcpiDsmGuid = (NvU8 *) &GPS_DSM_GUID;
+            acpiDsmRev = GPS_2X_REVISION_ID;
+            acpiDsmInArgSize = (*pSize);
+            break;
+        case ACPI_DSM_FUNCTION_GPS:
+            if (IsTU10X(pGpu))
+            {
+                pAcpiDsmGuid = (NvU8 *) &GPS_DSM_GUID;
+                acpiDsmRev = GPS_REVISION_ID;
+                acpiDsmInArgSize = (*pSize);
+            }
+            else
+            {
+                return NV_ERR_NOT_SUPPORTED;
+            }
+            break;
+        case ACPI_DSM_FUNCTION_PEX:
+            pAcpiDsmGuid = (NvU8 *) &PEX_DSM_GUID;
+            acpiDsmRev   = PEX_REVISION_ID;
+            if (acpiDsmSubFunction == PEX_FUNC_SETLTRLATENCY)
+            {
+                acpiDsmInArgSize = (3 + *pSize);
+            }
+            break;
+        case (ACPI_DSM_FUNCTION_JT):
+            pAcpiDsmGuid = (NvU8 *) &JT_DSM_GUID;
+            acpiDsmRev = JT_REVISION_ID;
+            break;
+        case ACPI_DSM_FUNCTION_NVPCF:
+            {
+                pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID;
+                acpiDsmRev = NVPCF_ACPI_DSM_REVISION_ID;
+                acpiDsmInArgSize = (*pSize);
+                acpiNvpcfDsmFunction = NV_TRUE;
+                break;
+            }
+        case ACPI_DSM_FUNCTION_NVPCF_2X:
+            pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID;
+            acpiDsmRev = NVPCF_2X_ACPI_DSM_REVISION_ID;
+            acpiDsmInArgSize = (*pSize);
+            acpiNvpcfDsmFunction = NV_TRUE;
+            break;
+
+        default:
+            return NV_ERR_NOT_SUPPORTED;
+            break;
+    }
+
+    status = nv_acpi_dsm_method(nv,
+                                pAcpiDsmGuid,
+                                acpiDsmRev,
+                                acpiNvpcfDsmFunction,
+                                acpiDsmSubFunction,
+                                pInOut,
+                                acpiDsmInArgSize,
+                                NULL,
+                                pInOut,
+                                pSize);
+
+    if (status == NV_OK)
+    {
+        if (acpiDsmSubFunction == NV_ACPI_ALL_FUNC_SUPPORT)
+        {
+            // if handling get supported functions list... cache it for later calls
+            cacheDsmSupportedFunction(pGpu, acpiDsmFunction, acpiDsmSubFunction, pInOut, *pSize);
+        }
+    }
+    else if (nvp->b_mobile_config_enabled)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "osCallACPI_DSM: Error during 0x%x DSM subfunction 0x%x! status=0x%x\n",
+                  acpiDsmFunction, acpiDsmSubFunction, status);
+    }
+
+    return status;
+}
+
+NV_STATUS osCallACPI_DOD
+(
+    OBJGPU  *pGpu,
+    NvU32   *pOut,
+    NvU32   *pSize
+)
+{
+    NV_STATUS rmStatus;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    if ((pOut == NULL) || (pSize == NULL))
+    {
+        return NV_ERR_INVALID_POINTER;
+    }
+
+    rmStatus = nv_acpi_dod_method(nv, pOut, pSize);
+
+    return rmStatus;
+}
+
+//
+// osAcpiDdc
+//
+// Handles os specific _DDC method function calls. _DDC is to get EDID from SBIOS.
+//
+NV_STATUS osCallACPI_DDC
+(
+    OBJGPU *pGpu,
+    NvU32   ulAcpiId,
+    NvU8   *pOutData,
+    NvU32  *pOutSize,
+    NvBool  bReadMultiBlock
+)
+{
+    NV_STATUS rmStatus;
+
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    if ((pOutData == NULL) || (pOutSize == NULL))
+    {
+        return NV_ERR_INVALID_POINTER;
+    }
+
+    portMemSet(pOutData, 0, *pOutSize);
+
+    rmStatus = nv_acpi_ddc_method(nv, pOutData, pOutSize, bReadMultiBlock);
+
+    return rmStatus;
+}
+
+// osCallACPI_NVHG_ROM
+// Making ACPI Call into SBIOS with ROM method to get display device's ROM data.
+//
+NV_STATUS  osCallACPI_NVHG_ROM
+(
+    OBJGPU *pGpu,
+    NvU32 *pInData,
+    NvU32 *pOutData
+)
+{
+    NV_STATUS rmStatus;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    if ((pOutData == NULL) || (pInData == NULL))
+    {
+        return NV_ERR_INVALID_POINTER;
+    }
+
+    if (pInData[1] > ROM_METHOD_MAX_RETURN_BUFFER_SIZE)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    rmStatus = nv_acpi_rom_method(nv, pInData, pOutData);
+
+    return rmStatus;
+}
+
+void osInitSystemStaticConfig(SYS_STATIC_CONFIG *pConfig)
+{
+    pConfig->bIsNotebook = rm_is_system_notebook();
+    pConfig->bOsCCEnabled = os_cc_enabled;
+    pConfig->bOsCCSevSnpEnabled = os_cc_sev_snp_enabled;
+    pConfig->bOsCCSmeEnabled = os_cc_sme_enabled;
+    pConfig->bOsCCSnpVtomEnabled = os_cc_snp_vtom_enabled;
+    pConfig->bOsCCTdxEnabled = os_cc_tdx_enabled;
+}
+
+NvU32 osApiLockAcquireConfigureFlags(NvU32 flags)
+{
+    return flags;
+}
+
+NV_STATUS osGpuLocksQueueRelease(OBJGPU *pGpu, NvU32 dpcGpuLocksRelease)
+{
+    return NV_SEMA_RELEASE_FAILED;
+}
+
+void osSyncWithRmDestroy(void)
+{
+}
+
+void osSyncWithGpuDestroy(NvBool bEntry)
+{
+}
+
+void osModifyGpuSwStatePersistence
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvBool       bEnable
+)
+{
+    if (!osIsLegacyPersistenceModeSupported(pOsGpuInfo))
+        return;
+
+    if (bEnable)
+    {
+        pOsGpuInfo->flags |= NV_FLAG_PERSISTENT_SW_STATE;
+    }
+    else
+    {
+        pOsGpuInfo->flags &= ~NV_FLAG_PERSISTENT_SW_STATE;
+    }
+}
+
+//
+//osCallACPI_MXDS
+//
+//Handles OS specific MXDS function call.
+//
+NV_STATUS osCallACPI_MXDS
+(
+    OBJGPU *pGpu,
+    NvU32 acpiId,
+    NvU32 *pInOut
+)
+{
+    NV_STATUS rmStatus;
+
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    if (pInOut == NULL)
+    {
+        return NV_ERR_INVALID_POINTER;
+    }
+
+    rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDS");
+
+    return rmStatus;
+}
+
+//
+//osCallACPI_MXDM
+//
+//Handles OS specific MXDM function call.
+//
+NV_STATUS osCallACPI_MXDM
+(
+    OBJGPU *pGpu,
+    NvU32 acpiId,
+    NvU32 *pInOut
+)
+{
+    NV_STATUS rmStatus;
+
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    if (pInOut == NULL)
+    {
+        return NV_ERR_INVALID_POINTER;
+    }
+
+    rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDM");
+
+    return rmStatus;
+}
+
+#include "lib/protobuf/prb.h"
+#include "lib/protobuf/prb_util.h"
+#include "g_nvdebug_pb.h"
+
+NV_STATUS osGetVersionDump(void * pVoid)
+{
+    PRB_ENCODER * pPrbEnc = (PRB_ENCODER *)pVoid;
+    NV_STATUS rmStatus;
+    os_version_info * pOsVersionInfo = NULL;
+    const char NV_UNKNOWN_BUILD_VERSION[] = "Unknown build version";
+    const char NV_UNKNOWN_BUILD_DATE[]    = "Unknown build date";
+
+    NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void**)&pOsVersionInfo,
+                                        sizeof(os_version_info)));
+    portMemSet(pOsVersionInfo, 0, sizeof(os_version_info));
+
+    prbEncAddUInt32(pPrbEnc,
+                    NVDEBUG_SYSTEMINFO_OSINFO_FAMILY,
+                    NVDEBUG_OS_UNIX);
+
+    rmStatus = os_get_version_info(pOsVersionInfo);
+    if (rmStatus != NV_OK)
+    {
+        goto cleanup;
+    }
+
+    prbEncAddUInt32(pPrbEnc,
+                    NVDEBUG_SYSTEMINFO_OSINFO_OSMAJORVERSION,
+                    pOsVersionInfo->os_major_version);
+
+    prbEncAddUInt32(pPrbEnc,
+                    NVDEBUG_SYSTEMINFO_OSINFO_OSMINORVERSION,
+                    pOsVersionInfo->os_minor_version);
+
+    prbEncAddUInt32(pPrbEnc,
+                    NVDEBUG_SYSTEMINFO_OSINFO_OSBLDNUM,
+                    pOsVersionInfo->os_build_number);
+
+    if (NULL == pOsVersionInfo->os_build_version_str)
+    {
+        pOsVersionInfo->os_build_version_str = NV_UNKNOWN_BUILD_VERSION;
+    }
+
+    prbEncAddString(pPrbEnc,
+                    NVDEBUG_SYSTEMINFO_OSINFO_BUILDVERSION,
+                    pOsVersionInfo->os_build_version_str);
+
+    if (NULL == pOsVersionInfo->os_build_date_plus_str)
+    {
+        pOsVersionInfo->os_build_date_plus_str = NV_UNKNOWN_BUILD_DATE;
+    }
+
+    prbEncAddString(pPrbEnc,
+                    NVDEBUG_SYSTEMINFO_OSINFO_BUILDDATEPLUS,
+                    pOsVersionInfo->os_build_date_plus_str);
+
+cleanup:
+    os_free_mem(pOsVersionInfo);
+    return rmStatus;
+}
+
+NV_STATUS osTegraiGpuPerfBoost(OBJGPU *pGpu, NvBool enable, NvU32 duration)
+{
+    return os_tegra_igpu_perf_boost(pGpu->pOsGpuInfo, enable, duration);
+}
+
+NV_STATUS osGetVersion(NvU32 *majorVer, NvU32 *minorVer, NvU32 *buildNum, NvU16 *unusedPatchVersion, NvU16 *unusedProductType)
+{
+    os_version_info osVersionInfo;
+    NV_STATUS rmStatus;
+
+    portMemSet(&osVersionInfo, 0, sizeof(osVersionInfo));
+
+    rmStatus = os_get_version_info(&osVersionInfo);
+    if (rmStatus == NV_OK)
+    {
+        if (majorVer)
+            *majorVer = osVersionInfo.os_major_version;
+        if (minorVer)
+            *minorVer = osVersionInfo.os_minor_version;
+        if (buildNum)
+            *buildNum = osVersionInfo.os_build_number;
+    }
+
+    return rmStatus;
+}
+
+NV_STATUS osGetIsOpenRM(NvBool *bOpenRm)
+{
+    return os_get_is_openrm(bOpenRm);
+}
+
+NvBool osIsBifResetSupported(OS_GPU_INFO *pOsGpuInfo)
+{
+    return os_is_bif_reset_supported((void *)pOsGpuInfo);
+}
+
+NV_STATUS
+osGetCarveoutInfo
+(
+    NvU64 *pAddr,
+    NvU64 *pSize
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osGetVPRInfo
+(
+    NvU64 *pAddr,
+    NvU64 *pSize
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osAllocInVPR
+(
+    MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osGetGenCarveout
+(
+    NvU64  *pAddr,
+    NvU64  *pSize,
+    NvU32   id,
+    NvU64   align
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osI2CClosePorts
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32        numPorts
+)
+{
+    nv_i2c_unregister_clients(pOsGpuInfo);
+    return NV_OK;
+}
+
+static NvU32
+i2cSwPortMapping(
+    nv_state_t *nv,
+    NvU32 physicalI2CPort
+)
+{
+    NvU32 linuxI2CSwPort = NV_U32_MAX;
+
+    /**
+     *  For T23X, Linux Tegra I2C controller driver uses logical port(controller) number
+     *  where logical port number of I2C1(Gen1) controller is 0, logical port number for
+     *  I2C2(Gen2) controller is 1 and so on. But RM passes I2C physical port(controller)
+     *  number i.e RM passes "1" for I2C1(Gen1), 2 for I2C2(Gen2), etc. So convert
+     *  physical port number to logical port number(linuxI2CSwPort).
+     *
+     *  For other chips, the above mentioned logic does not apply and we do not need
+     *  conversion as the physical controller number was updated to 0 and RM passes the
+     *  same.
+     */
+    if ((nv->disp_sw_soc_chip_id == NV_CHIP_ID_T234)
+        || (nv->disp_sw_soc_chip_id == NV_CHIP_ID_T239)
+        )
+    {
+        linuxI2CSwPort = physicalI2CPort - 1;
+    }
+    else
+    {
+        linuxI2CSwPort = physicalI2CPort;
+    }
+
+    return linuxI2CSwPort;
+}
+
+NV_STATUS
+osI2CTransfer
+(
+    OBJGPU *pGpu,
+    NvU32 Port,
+    NvU8 Address,
+    nv_i2c_msg_t *nv_i2c_msgs,
+    NvU32 count
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    if (NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        Port = i2cSwPortMapping(nv, Port);
+        return nv_i2c_transfer(nv, Port, Address,
+                               nv_i2c_msgs, count);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+NV_STATUS
+osTegraI2CGetBusState
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32 port,
+    NvS32 *scl,
+    NvS32 *sda
+)
+{
+    if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo))
+    {
+        port = i2cSwPortMapping(pOsGpuInfo, port);
+        return nv_i2c_bus_status(pOsGpuInfo, port, scl, sda);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+NV_STATUS
+osReadI2CBufferDirect
+(
+    OBJGPU *pGpu,
+    NvU32   Port,
+    NvU8    Address,
+    void   *pOutputBuffer,
+    NvU32   OutputSize,
+    void   *pInputBuffer,
+    NvU32   InputSize
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osWriteI2CBufferDirect
+(
+    OBJGPU  *pGpu,
+    NvU32    Port,
+    NvU8     Address,
+    void    *pOutputBuffer0,
+    NvU32    OutputSize0,
+    void    *pOutputBuffer1,
+    NvU32    OutputSize1
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+NV_STATUS
+osGC6PowerControl
+(
+    OBJGPU *pGpu,
+    NvU32   cmd,
+    NvU32  *pOut
+)
+{
+    NV_STATUS status;
+    NvU32 inOut   = cmd;
+    NvU16 rtnSize = sizeof(inOut);
+
+    if (FLD_TEST_DRF(_JT_FUNC, _POWERCONTROL, _GPU_POWER_CONTROL, _GSS, inOut))
+    {
+        if (!pOut)
+        {
+            return NV_ERR_INVALID_ARGUMENT;
+        }
+    }
+
+    status = osCallACPI_DSM(pGpu,
+                            ACPI_DSM_FUNCTION_JT,
+                            JT_FUNC_POWERCONTROL,
+                            &inOut,
+                            &rtnSize);
+
+    if ((status != NV_OK) || !pOut)
+    {
+        return status;
+    }
+
+    *pOut = inOut;
+
+    return NV_OK;
+}
+
+NvBool osTestPcieExtendedConfigAccess(void *handle, NvU32 offset)
+{
+    OBJGPU    *pGpu;
+    KernelBif *pKernelBif;
+    NvU32      nvXveId       = 0;
+    NvU32      nvXveVccapHdr = 0;
+    NvU32      pciStart      = 0;
+    NvU32      pcieStart     = 0;
+
+    static NvBool retryAllowed = NV_TRUE;
+    static NvBool configAccess = NV_FALSE;
+
+    //
+    // Return early for offset within PCI space
+    // and does not requires extended config space access
+    //
+    if (offset < 0x100)
+    {
+        return NV_TRUE;
+    }
+
+    if (!retryAllowed)
+    {
+        return configAccess;
+    }
+
+    pGpu = gpumgrGetSomeGpu();
+    if (pGpu == NULL)
+    {
+        return configAccess;
+    }
+
+    retryAllowed = NV_FALSE;
+
+    pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+    if (pKernelBif == NULL || gpuGetBusIntfType_HAL(pGpu) !=
+                                  NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS)
+    {
+        return configAccess;
+    }
+
+    // Now verify PCI and PCIe config start registers.
+    kbifGetPcieConfigAccessTestRegisters_HAL(pGpu, pKernelBif, &pciStart, &pcieStart);
+    os_pci_read_dword(handle, pciStart,  &nvXveId);
+    os_pci_read_dword(handle, pcieStart, &nvXveVccapHdr);
+
+    if (NV_OK == kbifVerifyPcieConfigAccessTestRegisters_HAL(pGpu,
+                                                             pKernelBif,
+                                                             nvXveId,
+                                                             nvXveVccapHdr))
+    {
+        configAccess = NV_TRUE;
+    }
+
+    return configAccess;
+}
+
+static NvBool skipIovaMappingForTegra
+(
+    PIOVAMAPPING pIovaMapping,
+    nv_state_t *nv
+)
+{
+    //
+    // TODO: When ISO SMMU is not present, dma mapping of imported ISO memory
+    //       causes crash during __clean_dcache_area_poc. dma mapping of ISO
+    //       memory allocated by RM (via __get_free_pages) still works.
+    //       Skip dma mapping of imported ISO memory to unblock Tegra Display in
+    //       AV+L. Bug 200765629
+    //
+
+    NV_ASSERT(nv != NULL);
+
+    if (NV_IS_SOC_DISPLAY_DEVICE(nv) &&
+        !NV_SOC_IS_ISO_IOMMU_PRESENT(nv) &&
+        !memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) &&
+        memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM))
+    {
+        NV_PRINTF(LEVEL_INFO, "%s: Skip memdescMapIommu mapping\n", __FUNCTION__);
+        return NV_TRUE;
+    }
+
+    return NV_FALSE;
+}
+
+static NvBool osMemDescRequiresReadOnlyDeviceDmaMap
+(
+    MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+    //
+    // MEMDESC_FLAGS_USER_READ_ONLY is also used by RM-owned buffers that the
+    // kernel writes and user/GSP reads. Restrict RO DMA direction to external
+    // OS descriptor mappings.
+    //
+    return memdescGetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM) &&
+           !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE) &&
+           (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY) ||
+            memdescGetFlag(pMemDesc, MEMDESC_FLAGS_DEVICE_READ_ONLY));
+}
+
+/*!
+ * @brief Map memory into an IOVA space according to the given mapping info.
+ *
+ * @param[in]   pIovaMapping    IOVA mapping info
+ *
+ * @return      NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osIovaMap
+(
+    PIOVAMAPPING pIovaMapping
+)
+{
+    OBJGPU *pGpu;
+    nv_state_t *nv, *peer;
+    NV_STATUS status;
+    RmPhysAddr base;
+    NvBool bIsBar0;
+    PMEMORY_DESCRIPTOR pRootMemDesc;
+    NvBool bIsFbOffset = NV_FALSE;
+    NvBool bIsIndirectPeerMapping = NV_FALSE;
+    NvBool bIsContig;
+    NV_ADDRESS_SPACE addressSpace;
+    NvU32 osPageCount;
+
+    if (pIovaMapping == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId);
+    if (pGpu == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    pRootMemDesc = memdescGetRootMemDesc(pIovaMapping->pPhysMemDesc, NULL);
+    addressSpace = memdescGetAddressSpace(pIovaMapping->pPhysMemDesc);
+    if (gpumgrCheckIndirectPeer(pGpu, pRootMemDesc->pGpu) &&
+        (addressSpace == ADDR_FBMEM))
+    {
+        bIsIndirectPeerMapping = NV_TRUE;
+    }
+
+    if ((addressSpace != ADDR_SYSMEM) && !bIsIndirectPeerMapping)
+    {
+        NV_PRINTF(LEVEL_INFO,
+                  "%s passed memory descriptor in an unsupported address space (%s)\n",
+                  __FUNCTION__,
+                  memdescGetApertureString(memdescGetAddressSpace(pIovaMapping->pPhysMemDesc)));
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    //
+    // For guest-allocated memory, we don't actually want to do any remapping,
+    // since the physical address is already the DMA address to be used by the
+    // GPU.
+    //
+    // For carveout memory, we setup identity mapping, so physical
+    // address is same as the DMA address.
+    //
+    //
+    if (memdescIsCarveoutMemory(pIovaMapping->pPhysMemDesc) ||
+        memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
+    {
+        return NV_OK;
+    }
+
+    nv = NV_GET_NV_STATE(pGpu);
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc))
+    {
+        osPageCount = pIovaMapping->pPhysMemDesc->PageCount;
+        NV_ASSERT_OR_RETURN(os_page_size == pIovaMapping->pPhysMemDesc->pageArrayGranularity, NV_ERR_INVALID_PARAMETER);
+    }
+    else
+    {
+        osPageCount = NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount);
+    }
+
+    //
+    // Intercept peer IO type memory. These are contiguous allocations, so no
+    // need to adjust pages.
+    //
+    if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
+    {
+        NV_ASSERT(memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU));
+
+        status = nv_dma_map_mmio(nv->dma_dev,
+            osPageCount,
+            &pIovaMapping->iovaArray[0]);
+
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s: failed to map peer IO mem (status = 0x%x)\n",
+                      __FUNCTION__, status);
+        }
+
+        return status;
+    }
+
+    //
+    // We need to check against the "root" GPU, e.g., the GPU that owns this
+    // allocation. If we're trying to map one of its BARs for a peer, we need
+    // to handle it differently because it wouldn't have gone through our system
+    // memory page allocation paths, obviously, and wouldn't have alloc private
+    // data associated with it.
+    //
+    peer = NV_GET_NV_STATE(pRootMemDesc->pGpu);
+    bIsContig = memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU);
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc))
+    {
+        NV_ASSERT_OR_RETURN(pIovaMapping->pPhysMemDesc->pageArrayGranularity == os_page_size, NV_ERR_INVALID_ARGUMENT);
+    }
+    else if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig)
+    {
+        RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0],
+                                pIovaMapping->pPhysMemDesc->PageCount);
+    }
+
+    base = memdescGetPhysAddr(pIovaMapping->pPhysMemDesc, AT_CPU, 0);
+    bIsBar0 = IS_REG_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size);
+
+    bIsFbOffset = IS_FB_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size);
+
+    void *pPriv = memdescGetMemData(pIovaMapping->pPhysMemDesc);
+
+    if (!bIsBar0 && !bIsFbOffset)
+    {
+        if (pPriv == NULL)
+        {
+            return NV_ERR_INVALID_STATE;
+        }
+    }
+    else if(bIsIndirectPeerMapping)
+    {
+        NV_ASSERT(!bIsBar0 && bIsFbOffset);
+        //
+        // TODO: Align onlined GPU memory allocation paths with system memory allocation
+        //       That way pMemDesc->pMemData is setup correctly when we try to create mapping
+        //       to onlined memory of indirect peer. After that we can also get rid of some
+        //       extra code in nv_dma_map_alloc. See bug 190324 for details
+        //
+
+        status = memdescGetNvLinkGpa(pRootMemDesc->pGpu, (bIsContig ? 1 : osPageCount),
+                                    &pIovaMapping->iovaArray[0]);
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR, "%s Failed to get SPA\n", __FUNCTION__);
+            return status;
+        }
+    }
+
+    if (skipIovaMappingForTegra(pIovaMapping, nv))
+    {
+        return NV_OK;
+    }
+
+    if (!bIsBar0 && (!bIsFbOffset || bIsIndirectPeerMapping))
+    {
+        NvBool bReadOnlyDeviceMap =
+            osMemDescRequiresReadOnlyDeviceDmaMap(pIovaMapping->pPhysMemDesc);
+
+        status = nv_dma_map_alloc(
+                    osGetDmaDeviceForMemDesc(nv, pIovaMapping->pPhysMemDesc),
+                    osPageCount,
+                    &pIovaMapping->iovaArray[0],
+                    bIsContig, bReadOnlyDeviceMap, &pPriv);
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s: failed to map allocation (status = 0x%x)\n",
+                      __FUNCTION__, status);
+            return status;
+        }
+
+        pIovaMapping->pOsData = pPriv;
+    }
+    else if (peer != nv)
+    {
+        status = nv_dma_map_peer(nv->dma_dev, peer->dma_dev, bIsBar0 ? 0 : 1,
+                                 osPageCount, &pIovaMapping->iovaArray[0]);
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_INFO,
+                      "%s: failed to map peer (base = 0x%llx, status = 0x%x)\n",
+                      __FUNCTION__, base, status);
+            return status;
+        }
+
+        //
+        // pOsData must be NULL to distinguish a peer DMA mapping from a
+        // system memory mapping in osIovaUnmap(), so make sure to set it
+        // accordingly here.
+        //
+        pIovaMapping->pOsData = NULL;
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_INFO, "cannot map a GPU's BAR to itself\n");
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    //
+    // If the OS layer doesn't think in RM page size, we need to inflate the
+    // PTE array into RM pages.
+    //
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc))
+    {
+        NV_ASSERT_OR_RETURN(pIovaMapping->pPhysMemDesc->pageArrayGranularity == os_page_size, NV_ERR_INVALID_ARGUMENT);
+    }
+    else if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig)
+    {
+        RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0],
+                                pIovaMapping->pPhysMemDesc->PageCount);
+    }
+
+    return NV_OK;
+}
+
+/*!
+ * @brief Unmap memory from an IOVA space according to the given mapping info.
+ *
+ * This mapping info must have been previously mapped by osIovaMap().
+ *
+ * @param[in]   pIovaMapping    IOVA mapping info
+ *
+ */
+void
+osIovaUnmap
+(
+    PIOVAMAPPING pIovaMapping
+)
+{
+    OBJGPU *pGpu;
+    nv_state_t *nv;
+    void *pPriv;
+    NV_STATUS status;
+    NvU64 osPageCount;
+
+    if (pIovaMapping == NULL)
+    {
+        return;
+    }
+
+    pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId);
+    if (pGpu == NULL)
+    {
+        return;
+    }
+
+    //
+    // For guest-allocated or carveout memory, we never actually remapped the
+    // memory, so we shouldn't try to unmap it here.
+    //
+    if (memdescIsCarveoutMemory(pIovaMapping->pPhysMemDesc) ||
+        memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
+    {
+        return;
+    }
+
+    nv = NV_GET_NV_STATE(pGpu);
+
+    if (skipIovaMappingForTegra(pIovaMapping, nv))
+    {
+        return;
+    }
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc))
+    {
+        osPageCount = pIovaMapping->pPhysMemDesc->PageCount;
+        NV_ASSERT(os_page_size == pIovaMapping->pPhysMemDesc->pageArrayGranularity);
+    }
+    else
+    {
+        osPageCount = NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount);
+    }
+
+    if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
+    {
+        nv_dma_unmap_mmio(nv->dma_dev,
+            osPageCount,
+            pIovaMapping->iovaArray[0]);
+
+        return;
+    }
+
+    //
+    // TODO: Formalize the interface with the OS layers so we can use a common
+    // definition of OS_IOVA_MAPPING_DATA.
+    //
+    pPriv = (void *)pIovaMapping->pOsData;
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc))
+    {
+        NV_ASSERT(pIovaMapping->pPhysMemDesc->pageArrayGranularity == os_page_size);
+    }
+    else if (NV_RM_PAGE_SIZE < os_page_size &&
+            !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU))
+    {
+        RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0],
+                                pIovaMapping->pPhysMemDesc->PageCount);
+    }
+
+    if (pPriv != NULL)
+    {
+        status = nv_dma_unmap_alloc(nv->dma_dev,
+            osPageCount,
+            &pIovaMapping->iovaArray[0], &pPriv);
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s: failed to unmap allocation (status = 0x%x)\n",
+                      __FUNCTION__, status);
+        }
+    }
+    else
+    {
+        nv_dma_unmap_peer(nv->dma_dev,
+            osPageCount,
+            pIovaMapping->iovaArray[0]);
+    }
+
+    //
+    // If the OS layer doesn't think in RM page size, we need to fluff out the
+    // PTE array into RM pages.
+    //
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pIovaMapping->pPhysMemDesc))
+    {
+        NV_ASSERT(pIovaMapping->pPhysMemDesc->pageArrayGranularity == os_page_size);
+    }
+    else if (NV_RM_PAGE_SIZE < os_page_size &&
+            !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU))
+    {
+        RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0],
+                                pIovaMapping->pPhysMemDesc->PageCount);
+    }
+
+    pIovaMapping->pOsData = NULL;
+}
+
+/*!
+ * @brief Set the GPU Rail Voltage in Tegra SoC. Currently not supported
+ *
+ * @param[in]  pGpu            GPU object pointer
+ * @param[in]  reqVoltageuV    Rail Voltage requested in uV
+ * @param[out] pSetVoltageuV   Rail Voltage set in uV
+ *
+ * @return    NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osSetGpuRailVoltage
+(
+   OBJGPU   *pGpu,
+   NvU32     reqVoltageuV,
+   NvU32    *pSetVoltageuV
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Get the GPU Rail Voltage in Tegra SoC. Currently not supported
+ *
+ * @param[in]  pGpu        GPU object pointer
+ * @param[out] voltageuV   Rail Voltage in uV
+ *
+ * @return    NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osGetGpuRailVoltage
+(
+    OBJGPU   *pGpu,
+    NvU32    *pVoltageuV
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Bring down system in a controlled manner on known error conditions.
+ *
+ * @bugCode[in] Error code / reason.
+ */
+void osBugCheck(NvU32 bugCode)
+{
+    if (bugCode > OS_BUG_CHECK_BUGCODE_LAST)
+    {
+        bugCode = OS_BUG_CHECK_BUGCODE_UNKNOWN;
+    }
+
+    os_bug_check(bugCode, ppOsBugCheckBugcodeStr[bugCode]);
+}
+
+/*!
+ * @brief Perform an action at assertion failure.
+ */
+void osAssertFailed(void)
+{
+    os_dump_stack();
+}
+
+/*!
+ * @brief Get the GPU Chip Info - Speedo and IDDQ values
+ *
+ *
+ * @param[in]   pGpu           GPU object pointer
+ * @param[out]  pGpuSpeedoHv   Pointer to GPU Speedo value at high voltage corner.
+ * @param[out]  pGpuSpeedoLv   Pointer to GPU Speedo value at low voltage corner.
+ * @param[out]  pGpuIddq       Pointer to GPU Iddq Value
+ * @param[out]  pChipSkuId     SKU ID for the chip
+ *
+ * @return     NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osGetChipInfo
+(
+    OBJGPU   *pGpu,
+    NvU32    *pGpuSpeedoHv,
+    NvU32    *pGpuSpeedoLv,
+    NvU32    *pGpuIddq,
+    NvU32    *pChipSkuId
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+/*
+ * @brief Get the GPU Rail Voltage Info (i.e. Min, Max and StepSize) in Tegra SoC.
+ *
+ * @param[in]  pGpu            GPU object pointer
+ * @param[out] pMinVoltageuV   Minimum Voltage supported on the Rail in Micro Volts
+ * @param[out] pMaxVoltageuV   Maximum Voltage supported on the Rail in Micro Volts
+ * @param[out] pStepVoltageuV  Voltage Step-size supported on the Rail in Micro Volts
+ *
+ * @return    NV_ERR_NOT_SUPPORTED
+ */
+NV_STATUS
+osGetGpuRailVoltageInfo
+(
+    OBJGPU    *pGpu,
+    NvU32     *pMinVoltageuV,
+    NvU32     *pMaxVoltageuV,
+    NvU32     *pStepVoltageuV
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Get the current opaque security token.
+ *
+ * For Linux the security token is the effective UID of a process and process ID
+ *
+ * Note: This function allocates memory for the token. The onus is on the calling
+ * function to free the memory associated with the token once its done with it.
+ *
+ * @return pointer to the security token.
+ */
+PSECURITY_TOKEN
+osGetSecurityToken(void)
+{
+    NV_STATUS rmStatus;
+    TOKEN_USER *pTokenUser;
+
+    pTokenUser = portMemAllocNonPaged(sizeof(TOKEN_USER));
+    if (pTokenUser == NULL)
+    {
+        return NULL;
+    }
+    rmStatus = os_get_euid(&pTokenUser->euid);
+    if (rmStatus != NV_OK)
+    {
+        portMemFree(pTokenUser);
+        return NULL;
+    }
+
+    pTokenUser->pid = os_get_current_process();
+
+    return (PSECURITY_TOKEN)pTokenUser;
+}
+
+PUID_TOKEN
+osGetCurrentUidToken(void)
+{
+    NV_STATUS rmStatus;
+    NvU32 *pUidToken;
+
+    pUidToken = portMemAllocNonPaged(sizeof(NvU32));
+    if (pUidToken == NULL)
+    {
+        return NULL;
+    }
+
+    rmStatus = os_get_euid(pUidToken);
+    if (rmStatus != NV_OK)
+    {
+        portMemFree(pUidToken);
+        return NULL;
+    }
+
+    return (PUID_TOKEN)pUidToken;
+}
+
+/*!
+ * @brief Interface function to validate the token for the current client
+ *
+ * This function takes two tokens as parameters, validates them and checks
+ * if either the PID or EUID from client database matches the current PID or EUID.
+ *
+ * @param[in] pClientSecurityToken   security token cached in the client db
+ * @param[in] pCurrentSecurityToken  security token of the current client
+ * @return             NV_OK    if the validation is successful
+ *                     NV_ERR_INVALID_CLIENT if the tokens do not match
+ *                     NV_ERR_INVALID_POINTER if the tokens are invalid
+ */
+NV_STATUS
+osValidateClientTokens
+(
+   PSECURITY_TOKEN pClientSecurityToken,
+   PSECURITY_TOKEN pCurrentSecurityToken
+)
+{
+    PTOKEN_USER  pClientTokenUser  = (PTOKEN_USER)pClientSecurityToken;
+    PTOKEN_USER  pCurrentTokenUser = (PTOKEN_USER)pCurrentSecurityToken;
+
+    if (pClientTokenUser == NULL || pCurrentTokenUser == NULL)
+        return NV_ERR_INVALID_POINTER;
+
+    if ((pClientTokenUser->euid != pCurrentTokenUser->euid) &&
+        (pClientTokenUser->pid != pCurrentTokenUser->pid))
+    {
+        NV_PRINTF(LEVEL_INFO,
+                  "NVRM: %s: Current security token doesn't match the one in the client database. "
+                  "Current EUID: %d, PID: %d; Client DB EUID: %d, PID: %d\n",
+                  __FUNCTION__, pCurrentTokenUser->euid, pCurrentTokenUser->pid,
+                  pClientTokenUser->euid, pClientTokenUser->pid);
+        return NV_ERR_INVALID_CLIENT;
+    }
+
+    return NV_OK;
+}
+
+/*!
+ * @brief Interface function to compare the tokens for two client
+ *
+ * This function takes two tokens as parameters, validates them and checks
+ * if the EUIDs of each token match.
+ *
+ * @param[in] pToken1  Token to compare
+ * @param[in] pToken2  Token to compare
+ * @return             NV_TRUE if the tokens match
+ *                     NV_FALSE if the tokens do not match
+ */
+NvBool
+osUidTokensEqual
+(
+   PUID_TOKEN pUidToken1,
+   PUID_TOKEN pUidToken2
+)
+{
+    NvU32 * pTokenUser1 = (NvU32*)pUidToken1;
+    NvU32 * pTokenUser2 = (NvU32*)pUidToken2;
+
+    NV_ASSERT_OR_RETURN((pTokenUser1 != NULL), NV_FALSE);
+    NV_ASSERT_OR_RETURN((pTokenUser2 != NULL), NV_FALSE);
+
+    if (*pTokenUser1 != *pTokenUser2)
+    {
+        return NV_FALSE;
+    }
+
+    return NV_TRUE;
+}
+
+NvBool
+osRemoveGpuSupported
+(
+    void
+)
+{
+    return os_pci_remove_supported();
+}
+
+/*
+ * @brief Get the address ranges assigned to local or peer GPUs on a system that
+ * supports hardware address translation services (ATS) over NVLink/C2C.
+ *
+ * @note
+ * - All address values are in the System Physical Address (SPA) space
+ * - Targets can either be "Local" (bIsPeer=False) or for a specified "Peer"
+ *   (bIsPeer=True, peerIndex=#) GPU
+ * - Target address and mask values have a specified bit width, and represent
+ *   the higher order bits above the target address granularity
+ *
+ * @param[in]   pGpu                GPU object pointer
+ * @param[out]  pAddrSysPhys        Pointer to hold SPA
+ * @param[out]  pAddrWidth          Address range width value pointer
+ * @param[out]  pMask               Mask value pointer
+ * @param[out]  pMaskWidth          Mask width value pointer
+ * @param[in]   bIsPeer             NV_TRUE if this is a peer, local GPU otherwise
+ * @param[in]   peerIndex           Peer index
+ *
+ * @return      NV_OK or NV_ERR_NOT_SUPPORTED
+ *
+ *              A return value of NV_ERR_NOT_SUPPORTED for the local GPU would
+ *              indicate that the system does not support ATS over NVLink/C2C
+ */
+NV_STATUS
+osGetAtsTargetAddressRange
+(
+    OBJGPU *pGpu,
+    NvU64   *pAddrSysPhys,
+    NvU32   *pAddrWidth,
+    NvU32   *pMask,
+    NvU32   *pMaskWidth,
+    NvBool  bIsPeer,
+    NvU32   peerIndex
+)
+{
+#if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && defined(NVCPU_AARCH64)
+    KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
+    KernelBif    *pKernelBif    = GPU_GET_KERNEL_BIF(pGpu);
+    nv_state_t   *nv;
+    const int addrMaskWidth = 0x10;
+
+    if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP))
+        return NV_ERR_INVALID_ARGUMENT;
+
+    nv = NV_GET_NV_STATE(pGpu);
+
+    //
+    // TODO : Bug 1848958 restricts peer device tree parsing. Return early if
+    // peer values are requested. This should be fixed by passing correct pGpu
+    // pointer of the peer GPU retrieved using peerIds.
+    //
+    if (bIsPeer)
+    {
+        const int addrWidth = 0x10;
+
+        *pAddrSysPhys = 0;
+        *pAddrWidth = addrWidth;
+        *pMask = 0;
+        *pMaskWidth = addrMaskWidth;
+        return NV_OK;
+    }
+    else
+    {
+        NV_STATUS status = nv_get_device_memory_config(nv, pAddrSysPhys, NULL, NULL, NULL,
+                                                       pAddrWidth, NULL);
+        if (status == NV_OK)
+        {
+            *pMask = NVBIT(*pAddrWidth) - 1U;
+            *pMaskWidth = addrMaskWidth;
+        }
+        return status;
+    }
+
+    return NV_OK;
+#endif
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+/*
+ * @brief Get the physical address in CPU address map and NUMA node id
+ * of the GPU memory.
+ *
+ * @note
+ * - The physical address is System Physical Address (SPA) in baremetal/host
+ *   and Intermediate Physical Address(IPA) or Guest Physical Address(GPA)
+ *   inside a VM.
+ *
+ * @param[in]   pGpu             GPU object pointer
+ * @param[out]  pAddrPhys        Pointer to hold the physical address of FB in
+ *                               CPU address map
+ * @param[out]  pSizePhys        Pointer to hold the size of FB mapped to CPU
+ * @param[out]  pNodeId          NUMA nodeID of respective GPU memory
+ *
+ * @return      NV_OK or NV_ERR_NOT_SUPPORTED
+ *
+ */
+NV_STATUS
+osGetFbNumaInfo
+(
+    OBJGPU *pGpu,
+    NvU64  *pAddrPhys,
+    NvU64  *pSizePhys,
+    NvU64  *pAddrRsvdPhys,
+    NvS32  *pNodeId
+)
+{
+#if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && defined(NVCPU_AARCH64)
+    KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
+    KernelBif    *pKernelBif    = GPU_GET_KERNEL_BIF(pGpu);
+    nv_state_t   *nv;
+
+    *pNodeId = NV0000_CTRL_NO_NUMA_NODE;
+
+    if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP))
+        return NV_ERR_INVALID_ARGUMENT;
+
+    nv = NV_GET_NV_STATE(pGpu);
+
+    NV_STATUS status = nv_get_device_memory_config(nv, NULL, pAddrPhys, pSizePhys,
+                                                   pAddrRsvdPhys, NULL, pNodeId);
+
+    return status;
+#endif
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+
+/*
+ * @brief Verif only function to get the chiplib overrides for link connection
+ *        state for all C2C links.
+ *
+ * If chiplib overrides exist, each link can either be enabled (1) or disabled (0)
+ *
+ * @param[in]   pGpu                GPU object pointer
+ * @param[in]   maxLinks            Size of pLinkConnection array
+ * @param[out]  pLinkConnection     array of pLinkConnection values to be populated by MODS
+ *
+ * @return      NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available)
+ */
+NV_STATUS
+osGetForcedC2CConnection
+(
+    OBJGPU *pGpu,
+    NvU32   maxLinks,
+    NvU32   *pLinkConnection
+)
+{
+    int i, ret;
+    NV_STATUS status;
+    char path[64];
+
+    NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER);
+    NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED);
+    NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT);
+
+    for (i = 0; i < maxLinks; i++)
+    {
+        ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|C2C%u", i);
+        NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1)));
+
+        status = gpuSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]);
+        if (status == NV_OK)
+        {
+            NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__,
+                      path, pLinkConnection[i]);
+        }
+        else
+        {
+            NV_PRINTF(LEVEL_INFO, "%s: gpuSimEscapeRead for '%s' failed (%u)\n",
+                      __FUNCTION__, path, status);
+            return NV_ERR_NOT_SUPPORTED;
+        }
+    }
+    return NV_OK;
+}
+
+static NV_STATUS
+osGetSmbiosTableInfo
+(
+    const NvU8 *pMappedAddr,
+    NvU64 *pBaseAddr,
+    NvU64 *pLength,
+    NvU64 *pNumSubTypes,
+    NvU32 *pVersion
+)
+{
+    *pBaseAddr = 0;
+    *pLength = 0;
+    *pNumSubTypes = 0;
+    *pVersion = 0;
+
+    if (portMemCmp(pMappedAddr, "_SM3_", 5) == 0)
+    {
+        *pVersion = (pMappedAddr[7] << 8) | pMappedAddr[8];
+        portMemCopy(pLength, 4, pMappedAddr + 12, 4);
+        portMemCopy(pBaseAddr, 8, pMappedAddr + 16, 8);
+
+        *pNumSubTypes = *pLength / 4;
+
+        return NV_OK;
+    }
+
+    if (portMemCmp(pMappedAddr, "_SM_", 4) == 0)
+    {
+        *pVersion = (pMappedAddr[6] << 8) | pMappedAddr[7];
+
+        pMappedAddr += 16;
+
+        if (portMemCmp(pMappedAddr, "_DMI_", 5) == 0)
+        {
+            portMemCopy(pLength, 2, pMappedAddr + 6, 2);
+            portMemCopy(pBaseAddr, 4, pMappedAddr + 8, 4);
+            portMemCopy(pNumSubTypes, 2, pMappedAddr + 12, 2);
+
+            if (!*pVersion)
+                *pVersion = (pMappedAddr[14] & 0xF0) << 4 |
+                            (pMappedAddr[14] & 0x0F);
+
+            return NV_OK;
+        }
+    }
+
+    return NV_ERR_INVALID_ADDRESS;
+}
+
+
+/*
+ * @brief Function to export SMBIOS table. Also, maps table in kernel-space.
+ *
+ * @param[out]   ppBaseVAddr         Base virtual address of SMBIOS table.
+ * @param[out]   pLength             Size of SMBIOS table.
+ * @param[out]   pNumSubTypes        Count of structures (types) embedded in
+ *                                   the SMBIOS table.
+ * @param[out]   pVersion            SMBIOS version
+ *
+ * @return NV_OK, NV_ERR_INSUFFICIENT_RESOURCES or NV_ERR_INVALID_ADDRESS
+ *          or errors from OS layer
+ */
+NV_STATUS
+osGetSmbiosTable
+(
+    void **ppBaseVAddr,
+    NvU64 *pLength,
+    NvU64 *pNumSubTypes,
+    NvU32 *pVersion
+)
+{
+    NV_STATUS status = NV_OK;
+    NvU64 physSmbiosAddr = ~0ull;
+    void *pMappedAddr = NULL;
+    NvU64 basePAddr = 0;
+
+    if (!NVCPU_IS_X86_64 && !NVCPU_IS_AARCH64)
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    status = os_get_smbios_header(&physSmbiosAddr);
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_INFO, "%s: Failed query SMBIOS table with error: %x \n",
+                  __FUNCTION__, status);
+        return status;
+    }
+
+    NV_ASSERT(physSmbiosAddr != ~0ull);
+
+    pMappedAddr = osMapKernelSpace(physSmbiosAddr,
+                                   os_page_size,
+                                   NV_MEMORY_CACHED,
+                                   NV_PROTECT_READ_WRITE);
+    if (!pMappedAddr)
+    {
+        return NV_ERR_INSUFFICIENT_RESOURCES;
+    }
+
+    status = osGetSmbiosTableInfo(pMappedAddr,
+                                  &basePAddr,
+                                  pLength,
+                                  pNumSubTypes,
+                                  pVersion);
+
+    osUnmapKernelSpace(pMappedAddr, os_page_size);
+
+    if (status != NV_OK)
+    {
+        return status;
+    }
+
+    *ppBaseVAddr = osMapKernelSpace(basePAddr,
+                                    *pLength,
+                                    NV_MEMORY_CACHED,
+                                    NV_PROTECT_READ_WRITE);
+    if (!*ppBaseVAddr)
+    {
+        return NV_ERR_INSUFFICIENT_RESOURCES;
+    }
+
+    return NV_OK;
+}
+
+/*
+ * @brief Function to free SMBIOS table mappings
+ *
+ * @param[in]   pBaseVAddr        Base virtual address of SMBIOS table.
+ * @param[in]   length            Size of SMBIOS table.
+ *
+ */
+void
+osPutSmbiosTable
+(
+    void *pBaseVAddr,
+    NvU64 length
+)
+{
+    osUnmapKernelSpace(pBaseVAddr, length);
+}
+
+NV_STATUS
+osGetAcpiRsdpFromUefi
+(
+    NvU32  *pRsdpAddr
+)
+{
+    return os_get_acpi_rsdp_from_uefi(pRsdpAddr);
+}
+
+/*
+ *  @brief Returns NV_TRUE if NvSwitch device is present in the system.
+ */
+NvBool
+osIsNvswitchPresent
+(
+    void
+)
+{
+    return os_is_nvswitch_present();
+}
+
+/*
+ *  @brief Function to add crashlog buffer entry.
+ *
+ *  @param[in] pBuffer              virt_addr of nvlog buffer
+ *  @param[in] length               size of nvlog buffer
+ */
+void
+osAddRecordForCrashLog
+(
+    void *pBuffer,
+    NvU32 length
+)
+{
+    os_add_record_for_crashLog(pBuffer, length);
+}
+
+/*
+ *  @brief Function to delete crashlog buffer entry.
+ *
+ *  @param[in] pBuffer              virt_addr of nvlog buffer
+ */
+void
+osDeleteRecordForCrashLog
+(
+    void *pBuffer
+)
+{
+    os_delete_record_for_crashLog(pBuffer);
+}
+
+/*
+ *  @brief Queries the sysfs interface to get memblock size
+ *  @param[out] memblock_size Pointer to the memblock_size
+ */
+NV_STATUS
+osNumaMemblockSize
+(
+    NvU64 *memblock_size
+)
+{
+    return os_numa_memblock_size(memblock_size);
+}
+
+NvBool
+osNumaOnliningEnabled
+(
+    OS_GPU_INFO *pOsGpuInfo
+)
+{
+    NvS32 numaNodeId = NV0000_CTRL_NO_NUMA_NODE;
+
+    //
+    // Note that this numaNodeId value fetched from Linux layer might not be
+    // accurate since it is possible to overwrite it with regkey on some configs
+    //
+    if (nv_get_device_memory_config(pOsGpuInfo, NULL, NULL, NULL, NULL, NULL,
+                                    &numaNodeId) != NV_OK)
+    {
+        return NV_FALSE;
+    }
+
+    return (numaNodeId != NV0000_CTRL_NO_NUMA_NODE);
+}
+
+/*
+ *  @brief Function to call NUMA allocation entry.
+ *
+ *  @param[in]  nid       NUMA node id
+ *  @param[in]  size      Allocation size
+ *  @param[in]  flag      Allocation flags
+ *  @param[out] pAddress  Ptr to the allocated physical address
+ */
+NV_STATUS
+osAllocPagesNode
+(
+    NvS32       nid,
+    NvLength    size,
+    NvU32       flag,
+    NvU64      *pAddress
+)
+{
+    NV_STATUS  status    = NV_OK;
+    NvU32      localFlag = NV_ALLOC_PAGES_NODE_NONE;
+
+    if (pAddress == NULL || nid < 0 || size > NV_U32_MAX)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    // Translate the flags
+    if (flag & OS_ALLOC_PAGES_NODE_SKIP_RECLAIM)
+    {
+        localFlag |= NV_ALLOC_PAGES_NODE_SKIP_RECLAIM;
+    }
+
+    status = os_alloc_pages_node(nid, (NvU32)size, localFlag, pAddress);
+    return status;
+}
+
+void
+osAllocAcquirePage
+(
+    NvU64 pAddress,
+    NvU32 pageCount
+)
+{
+    NvU32 i;
+
+    for (i = 0; i < pageCount; i++)
+    {
+        os_get_page(pAddress + (i << os_page_shift));
+    }
+}
+
+void
+osAllocReleasePage
+(
+    NvU64 pAddress,
+    NvU32 pageCount
+)
+{
+    NvU32 i;
+
+    for (i = 0; i < pageCount; i++)
+    {
+        os_put_page(pAddress + (i << os_page_shift));
+    }
+}
+
+/*
+ *  @brief Function to return refcount on a page
+ *  @param[in] address  The physical address of the page
+ */
+NvU32
+osGetPageRefcount
+(
+    NvU64       pAddress
+)
+{
+    return os_get_page_refcount(pAddress);
+}
+
+/*
+ *  @brief Function to return the number of tail pages if the address is
+ *  referring to a compound page; For non-compound pages, 1 is returned.
+ *  @param[in] address  The physical address of the page
+ */
+NvU32
+osCountTailPages
+(
+    NvU64       pAddress
+)
+{
+    return os_count_tail_pages(pAddress);
+}
+
+NvU64
+osGetPageSize(void)
+{
+    return os_page_size;
+}
+
+NvU64
+osGetSupportedSysmemPageSizeMask(void)
+{
+    //
+    // We assume that the kernel can support all power-of-two pagesizes 
+    // between os_page_size and os_max_page_size (inclusive).  Return a 
+    // bitmask containing all of those.
+    //
+    return (((os_max_page_size << 1) - 1) & (~(os_page_size - 1)));
+}
+
+NvU8
+osGetPageShift(void)
+{
+    return os_page_shift;
+}
+
+
+
+/*
+ * @brief Opens a new temporary file for reading and writing
+ *
+ * @param[in]  ppFile      void double pointer
+ *
+ * @returns NV_STATUS, NV_OK if success,
+                       NV_ERR_GENERIC, if error
+                       NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osOpenTemporaryFile
+(
+    void **ppFile
+)
+{
+    return os_open_temporary_file(ppFile);
+}
+
+/*
+ * @brief Closes the specified temporary file
+ *
+ * @param[in]  pFile      Pointer to file
+ *
+ * @returns void
+ */
+void
+osCloseFile
+(
+    void *pFile
+)
+{
+    os_close_file(pFile);
+}
+
+/*
+ * @brief Writes the buffer to the specified file at the given offset
+ *
+ * @param[in]  pFile            Pointer to file (void)
+ * @param[in]  pBuffer          Pointer to buffer from which to copy
+ * @param[in]  size             Size of the copy
+ * @parma[in]  offset           offset in the file
+ *
+ * @returns NV_STATUS, NV_OK if success,
+                       NV_ERR_GENERIC, if error
+                       NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osWriteToFile
+(
+    void *pFile,
+    NvU8 *pBuffer,
+    NvU64 size,
+    NvU64 offset
+)
+{
+    return os_write_file(pFile, pBuffer, size, offset);
+}
+
+/*
+ * @brief Reads from the specified file at the given offset
+ *
+ * @param[in]  pFile            Pointer to file (void *)
+ * @param[in]  pBuffer          Pointer to buffer to which the data is copied
+ * @param[in]  size             Size of the copy
+ * @parma[in]  offset           offset in the file
+ *
+ * @returns NV_STATUS, NV_OK if success,
+                       NV_ERR_GENERIC, if error
+                       NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osReadFromFile
+(
+    void *pFile,
+    NvU8 *pBuffer,
+    NvU64 size,
+    NvU64 offset
+)
+{
+    return os_read_file(pFile, pBuffer, size, offset);
+}
+
+/*
+ * @brief Unregisters caps from the capability framework.
+ *        The function assumes that the caps are allocated and stored in the
+ *        hierarchical order. If they aren't, OS (Linux kernel) would warn and
+ *        leak the caps.
+ *
+ * @param[in]  pOsRmCaps           caps of interest
+ */
+void
+osRmCapUnregister
+(
+    OS_RM_CAPS **ppOsRmCaps
+)
+{
+    OS_RM_CAPS *pOsRmCaps = *ppOsRmCaps;
+    NvS32 i;
+
+    if (pOsRmCaps == NULL)
+    {
+        return;
+    }
+
+    for (i = pOsRmCaps->count - 1; i >= 0; i--)
+    {
+        if (pOsRmCaps->caps[i] != NULL)
+        {
+            os_nv_cap_destroy_entry(pOsRmCaps->caps[i]);
+        }
+    }
+
+    os_free_mem(pOsRmCaps->caps);
+    os_free_mem(pOsRmCaps);
+
+    *ppOsRmCaps = NULL;
+}
+
+static NV_STATUS
+_allocOsRmCaps
+(
+    OS_RM_CAPS **ppOsRmCaps,
+    NvU32        count
+)
+{
+    NV_STATUS   status;
+    OS_RM_CAPS *pOsRmCaps;
+
+    *ppOsRmCaps = NULL;
+
+    status = os_alloc_mem((void**)&pOsRmCaps, sizeof(OS_RM_CAPS));
+    if (status != NV_OK)
+        return status;
+
+    pOsRmCaps->count = count;
+
+    status = os_alloc_mem((void**)&pOsRmCaps->caps, sizeof(pOsRmCaps->caps[0]) * count);
+    if (status != NV_OK)
+    {
+        os_free_mem(pOsRmCaps);
+        return status;
+    }
+
+    os_mem_set(pOsRmCaps->caps, 0, sizeof(pOsRmCaps->caps[0]) * count);
+
+    *ppOsRmCaps = pOsRmCaps;
+    return NV_OK;
+}
+
+#define OS_RM_CAP_GPU_DIR 0
+#define OS_RM_CAP_GPU_MIG_DIR 1
+#define OS_RM_CAP_GPU_COUNT 2
+
+/*
+ * @brief Registers OBJGPU with the capability framework.
+ *
+ * @param[in]  pOsGpuInfo  OS specific GPU information pointer
+ * @param[out] ppOsRmCaps  GPU OS specific capabilities pointer
+ */
+NV_STATUS
+osRmCapRegisterGpu
+(
+    OS_GPU_INFO   *pOsGpuInfo,
+    OS_RM_CAPS   **ppOsRmCaps
+)
+{
+    NvU32 minor = nv_get_dev_minor(pOsGpuInfo);
+    char name[16];
+    NV_STATUS   status;
+    OS_RM_CAPS *pOsRmCaps;
+    nv_cap_t   *parent;
+    nv_cap_t   *cap;
+
+    // Return success on the unsupported platforms.
+    if (nvidia_caps_root == NULL)
+    {
+        return NV_OK;
+    }
+
+    if (*ppOsRmCaps != NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_GPU_COUNT);
+    if (status != NV_OK)
+        return status;
+
+    *ppOsRmCaps = pOsRmCaps;
+
+    os_snprintf(name, sizeof(name), "gpu%u", minor);
+    name[sizeof(name) - 1] = '\0';
+    parent = nvidia_caps_root;
+
+    cap = os_nv_cap_create_dir_entry(parent, name, (OS_RUGO | OS_XUGO));
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to setup gpu%u directory\n", minor);
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+
+    pOsRmCaps->caps[OS_RM_CAP_GPU_DIR] = cap;
+    parent = cap;
+
+    // TODO: Bug 2679591: Add MIG directory only if SMC is enabled.
+    // For now, always add "mig" directory.
+    cap = os_nv_cap_create_dir_entry(parent, "mig", (OS_RUGO | OS_XUGO));
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to setup mig directory\n");
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+
+    pOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR] = cap;
+
+    return NV_OK;
+
+failed:
+    osRmCapUnregister(ppOsRmCaps);
+
+    return status;
+}
+
+#define OS_RM_CAP_SMC_PART_DIR 0
+#define OS_RM_CAP_SMC_PART_ACCESS_FILE 1
+#define OS_RM_CAP_SMC_PART_COUNT 2
+
+/*
+ * @brief Registers SMC partition (a.k.a. GPU instance) with the capability
+ *        framework
+ *
+ * @param[in]  pGpuOsRmCaps         GPU OS specific capabilities pointer
+ * @param[out] ppPartitionOsRmCaps  OS specific capabilities pointer for SMC partition
+ * @param[in]  swizzId              SMC partition swizz ID
+ */
+NV_STATUS
+osRmCapRegisterSmcPartition
+(
+    OS_RM_CAPS  *pGpuOsRmCaps,
+    OS_RM_CAPS **ppPartitionOsRmCaps,
+    NvU32        swizzId
+)
+{
+    char        name[16];
+    NV_STATUS   status;
+    nv_cap_t   *parent;
+    nv_cap_t   *cap;
+    OS_RM_CAPS *pOsRmCaps;
+
+    // Return success as there is nothing to do.
+    if (pGpuOsRmCaps == NULL)
+    {
+        return NV_OK;
+    }
+
+    if (*ppPartitionOsRmCaps != NULL || swizzId >= NV_U32_MAX)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    parent = pGpuOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR];
+    if (parent == NULL)
+    {
+        return NV_ERR_INVALID_STATE;
+    }
+
+    status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_PART_COUNT);
+    if (status != NV_OK)
+        return status;
+
+    *ppPartitionOsRmCaps = pOsRmCaps;
+
+    os_snprintf(name, sizeof(name), "gi%u", swizzId);
+    name[sizeof(name) - 1] = '\0';
+
+    cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to setup gi%u directory\n",
+                  swizzId);
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+
+    pOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR] = cap;
+    parent = cap;
+
+    cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n",
+                  swizzId);
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+
+    pOsRmCaps->caps[OS_RM_CAP_SMC_PART_ACCESS_FILE] = cap;
+
+    return NV_OK;
+
+failed:
+    osRmCapUnregister(ppPartitionOsRmCaps);
+
+    return status;
+}
+
+#define OS_RM_CAP_SMC_EXEC_PART_DIR 0
+#define OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE 1
+#define OS_RM_CAP_SMC_EXEC_PART_COUNT 2
+
+/*
+ * @brief Registers SMC execution partition (a.k.a. compute instance) with the
+ *        capability framework
+ *
+ * @param[in]  pPartitionOsRmCaps       OS specific capabilities pointer for SMC partition
+ * @param[out] ppExecPartitionOsRmCaps  OS specific capabilities pointer for SMC execution partition
+ * @param[in]  execPartitionId          SMC execution partition ID
+ */
+NV_STATUS
+osRmCapRegisterSmcExecutionPartition
+(
+    OS_RM_CAPS  *pPartitionOsRmCaps,
+    OS_RM_CAPS **ppExecPartitionOsRmCaps,
+    NvU32        execPartitionId
+)
+{
+    char        name[16];
+    NV_STATUS   status;
+    nv_cap_t   *parent;
+    nv_cap_t   *cap;
+    OS_RM_CAPS *pOsRmCaps;
+
+    // Return success as there is nothing to do.
+    if (pPartitionOsRmCaps == NULL)
+    {
+        return NV_OK;
+    }
+
+    if ((*ppExecPartitionOsRmCaps != NULL) || (execPartitionId >= NV_U32_MAX))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    parent = pPartitionOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR];
+    if (parent == NULL)
+    {
+        return NV_ERR_INVALID_STATE;
+    }
+
+    status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_EXEC_PART_COUNT);
+    if (status != NV_OK)
+    {
+        return status;
+    }
+
+    *ppExecPartitionOsRmCaps = pOsRmCaps;
+
+    os_snprintf(name, sizeof(name), "ci%u", execPartitionId);
+    name[sizeof(name) - 1] = '\0';
+
+    cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to setup ci%u directory\n",
+                  execPartitionId);
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+
+    pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_DIR] = cap;
+    parent = cap;
+
+    cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n",
+                  execPartitionId);
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+
+    pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE] = cap;
+
+    return NV_OK;
+
+failed:
+    osRmCapUnregister(ppExecPartitionOsRmCaps);
+
+    return status;
+}
+
+/*
+ * @brief Release the acquired capability
+ *
+ * @param[in]  dupedCapDescriptor  descriptor to be released
+ */
+void
+osRmCapRelease
+(
+    NvU64 dupedCapDescriptor
+)
+{
+    if (dupedCapDescriptor == NV_U64_MAX)
+    {
+        return;
+    }
+
+    os_nv_cap_close_fd((int)dupedCapDescriptor);
+}
+
+#define OS_RM_CAP_SYS_MIG_DIR                   0
+#define OS_RM_CAP_SYS_SMC_CONFIG_FILE           1
+#define OS_RM_CAP_SYS_SMC_MONITOR_FILE          2
+#define OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE     3
+#define OS_RM_CAP_SYS_PROFILER_DEVICE_FILE      4
+#define OS_RM_CAP_SYS_PROFILER_CONTEXT_FILE     5
+#define OS_RM_CAP_SYS_TRACE_DEVICE_FILE         6
+#define OS_RM_CAP_SYS_COUNT                     7
+
+NV_STATUS
+osRmCapRegisterSys
+(
+    OS_RM_CAPS **ppOsRmCaps
+)
+{
+    nv_cap_t  **ppCaps;
+    nv_cap_t   *parent;
+    nv_cap_t   *cap;
+    NV_STATUS   status;
+    OS_RM_CAPS *pOsRmCaps;
+
+    if (nvidia_caps_root == NULL)
+        return NV_ERR_NOT_SUPPORTED;
+
+    status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SYS_COUNT);
+    if (status != NV_OK)
+        return status;
+
+    *ppOsRmCaps = pOsRmCaps;
+
+    ppCaps = pOsRmCaps->caps;
+
+    parent = os_nv_cap_create_dir_entry(nvidia_caps_root, "mig", OS_RUGO | OS_XUGO);
+    if (parent == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to create mig directory\n");
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+    ppCaps[OS_RM_CAP_SYS_MIG_DIR] = parent;
+
+    cap = os_nv_cap_create_file_entry(parent, "config", OS_RUSR);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to create mig config file\n");
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+    ppCaps[OS_RM_CAP_SYS_SMC_CONFIG_FILE] = cap;
+
+    cap = os_nv_cap_create_file_entry(parent, "monitor", OS_RUGO);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to create mig monitor file\n");
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+    ppCaps[OS_RM_CAP_SYS_SMC_MONITOR_FILE] = cap;
+
+    cap = os_nv_cap_create_file_entry(nvidia_caps_root, "fabric-imex-mgmt", OS_RUSR);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to create imex file\n");
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+    ppCaps[OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE] = cap;
+
+    cap = os_nv_cap_create_file_entry(nvidia_caps_root, "profiler-device", OS_RUSR);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to create profiler-device file\n");
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+    ppCaps[OS_RM_CAP_SYS_PROFILER_DEVICE_FILE] = cap;
+
+    cap = os_nv_cap_create_file_entry(nvidia_caps_root, "profiler-context", OS_RUSR);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to create profiler-context file\n");
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+    ppCaps[OS_RM_CAP_SYS_PROFILER_CONTEXT_FILE] = cap;
+
+    cap = os_nv_cap_create_file_entry(nvidia_caps_root, "trace-device", OS_RUSR);
+    if (cap == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to create trace-device file\n");
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto failed;
+    }
+    ppCaps[OS_RM_CAP_SYS_TRACE_DEVICE_FILE] = cap;
+
+    return NV_OK;
+
+failed:
+    osRmCapUnregister(ppOsRmCaps);
+    return status;
+}
+
+/*
+ * @brief Acquire the requested capability
+ *
+ * @param[in]  pOsRmCaps           opaque pointer to the caps.
+ * @param[in]  rmCap               the capability to be acquired.
+ * @param[in]  capDescriptor       descriptor to be used for validation
+ * @param[out] dupedCapDescriptor  returns duplicated descriptor if validation
+ *                                 is successful
+ *
+ * Note: On Linux, duplicating fd is helpful to let administrators know about
+ * the capability users. See https://linux.die.net/man/8/lsof usage.
+ */
+NV_STATUS
+osRmCapAcquire
+(
+    OS_RM_CAPS *pOsRmCaps,
+    NvU32       rmCap,
+    NvU64       capDescriptor,
+    NvU64      *dupedCapDescriptor
+)
+{
+    nv_cap_t *cap;
+    int fd = (int)capDescriptor;
+    int duped_fd;
+    NvU32 index;
+    NV_STATUS status;
+
+    *dupedCapDescriptor = NV_U64_MAX;
+
+    switch (rmCap)
+    {
+        case NV_RM_CAP_SMC_PARTITION_ACCESS:
+        {
+            index = OS_RM_CAP_SMC_PART_ACCESS_FILE;
+            break;
+        }
+        case NV_RM_CAP_EXT_FABRIC_MGMT:
+        {
+            status = nv_acquire_fabric_mgmt_cap(fd, &duped_fd);
+            if (status != NV_OK)
+            {
+                return status;
+            }
+
+            goto done;
+        }
+        case NV_RM_CAP_SMC_EXEC_PARTITION_ACCESS:
+        {
+            index = OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE;
+            break;
+        }
+        case NV_RM_CAP_SYS_SMC_CONFIG:
+        {
+            index = OS_RM_CAP_SYS_SMC_CONFIG_FILE;
+            break;
+        }
+        case NV_RM_CAP_SYS_SMC_MONITOR:
+        {
+            index = OS_RM_CAP_SYS_SMC_MONITOR_FILE;
+            break;
+        }
+        case NV_RM_CAP_SYS_FABRIC_IMEX_MGMT:
+        {
+            index = OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE;
+            break;
+        }
+        case NV_RM_CAP_SYS_PROFILER_DEVICE:
+        {
+            index = OS_RM_CAP_SYS_PROFILER_DEVICE_FILE;
+            break;
+        }
+        case NV_RM_CAP_SYS_PROFILER_CONTEXT:
+        {
+            index = OS_RM_CAP_SYS_PROFILER_CONTEXT_FILE;
+            break;
+        }
+        case NV_RM_CAP_SYS_TRACE_DEVICE:
+        {
+            index = OS_RM_CAP_SYS_TRACE_DEVICE_FILE;
+            break;
+        }
+        default:
+        {
+            return NV_ERR_INVALID_ARGUMENT;
+        }
+    }
+
+    if (pOsRmCaps == NULL)
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    if (index >= pOsRmCaps->count)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    cap = pOsRmCaps->caps[index];
+
+    duped_fd = os_nv_cap_validate_and_dup_fd(cap, fd);
+    if (duped_fd < 0)
+    {
+        return NV_ERR_INSUFFICIENT_PERMISSIONS;
+    }
+
+done:
+    *dupedCapDescriptor = duped_fd;
+
+    return NV_OK;
+}
+
+/*
+ * @brief Initializes capability descriptor
+ *
+ * @param[out] pCapDescriptor        descriptor to be used
+ *
+ */
+void
+osRmCapInitDescriptor
+(
+    NvU64 *pCapDescriptor
+)
+{
+    *pCapDescriptor = NV_U64_MAX;
+}
+
+/*
+ * @brief Checks if IMEX channel support is present.
+ */
+NvBool
+osImexChannelIsSupported(void)
+{
+    return os_imex_channel_is_supported;
+}
+
+/*
+ * @brief Returns IMEX channel count.
+ */
+NvS32
+osImexChannelCount
+(
+    void
+)
+{
+    return os_imex_channel_count();
+}
+
+/*
+ * @brief Returns IMEX channel number.
+ *
+ * @param[in] descriptor   OS specific descriptor to query channel number.
+ *
+ */
+NvS32
+osImexChannelGet(NvU64 descriptor)
+{
+    return os_imex_channel_get(descriptor);
+}
+
+/*
+ * @brief Generates random bytes which can be used as a universally unique
+ *        identifier.
+ *
+ * This function may sleep (interruptible).
+ *
+ * @param[out] pBytes        Array of random bytes
+ * @param[in]  numBytes      Size of the array
+ */
+NV_STATUS
+osGetRandomBytes
+(
+    NvU8 *pBytes,
+    NvU16 numBytes
+)
+{
+    return os_get_random_bytes(pBytes, numBytes);
+}
+
+/*
+ * @brief Get current process flags..
+ */
+NvU32
+osGetCurrentProcessFlags
+(
+    void
+)
+{
+    return os_get_current_process_flags();
+}
+
+/*
+ * @brief Allocate wait queue
+ *
+ * @param[out] ppWq        Wait queue
+ */
+NV_STATUS
+osAllocWaitQueue
+(
+    OS_WAIT_QUEUE **ppWq
+)
+{
+    return os_alloc_wait_queue(ppWq);
+}
+
+/*
+ * @brief Free wait queue
+ *
+ * @param[in] pWq        Wait queue
+ */
+void
+osFreeWaitQueue
+(
+    OS_WAIT_QUEUE *pWq
+)
+{
+    os_free_wait_queue(pWq);
+}
+
+/*
+ * @brief Put thread to uninterruptible sleep
+ *
+ * @param[in] pWq        Wait queue
+ */
+void
+osWaitUninterruptible
+(
+    OS_WAIT_QUEUE *pWq
+)
+{
+    os_wait_uninterruptible(pWq);
+}
+
+/*
+ * @brief Put thread to interruptible sleep
+ *
+ * @param[in] pWq        Wait queue
+ */
+void
+osWaitInterruptible
+(
+    OS_WAIT_QUEUE *pWq
+)
+{
+    os_wait_interruptible(pWq);
+}
+
+/*
+ * @brief Wake up thread from uninterruptible sleep
+ *
+ * @param[in] pWq        Wait queue
+ */
+void
+osWakeUp
+(
+    OS_WAIT_QUEUE *pWq
+)
+{
+    os_wake_up(pWq);
+}
+
+NV_STATUS
+osReadPFPciConfigInVF
+(
+    NvU32 addr,
+    NvU32 *data
+)
+{
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Callback function to notify RM when unix layer receives an event
+ *
+ * This function is basically a wrapper to call the Core RM layer and is
+ * being called from DCE KMD when an event is received from DCE RM.
+ *
+ * @param[in]  handle         handle allocated for corresponding IPC type with DCE
+ * @param[in]  interfaceType  RM IPC interface type
+ * @param[in]  length         length of the message passed from DCE
+ * @param[in]  data           any specific data if present
+ * @param[in]  usrCtx         any specific user context if present
+ *
+ * @returns void
+ */
+static void
+osTegraDceClientIpcCallback
+(
+    NvU32 handle,
+    NvU32 interfaceType,
+    NvU32 length,
+    void *data,
+    void *usrCtx
+)
+{
+    THREAD_STATE_NODE threadState;
+    NvU32 rmInterfaceType = nv_tegra_get_rm_interface_type(interfaceType);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if (rmLocksAcquireAll(RM_LOCK_MODULES_KERNEL_RM_EVENTS) == NV_OK)
+    {
+        dceclientHandleAsyncRpcCallback(handle, rmInterfaceType, length, data, usrCtx);
+
+        rmLocksReleaseAll();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+}
+
+/*!
+ * @brief Performs IPC Client registration with DCE
+ *
+ * This function is basically a wrapper to call the unix/linux layer.
+ *
+ * @param[in]  interfaceType  RM IPC interface type
+ * @param[in]  usrCtx         any specific user context if present
+ * @param[out] clientId       unique ID registered with DCE for IPC
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ *          other errors as may be returned by subfunctions.
+ */
+NV_STATUS
+osTegraDceRegisterIpcClient
+(
+    NvU32 interfaceType,
+    void *usrCtx,
+    NvU32 *clientId
+)
+{
+    if (interfaceType == DCE_CLIENT_RM_IPC_TYPE_SYNC)
+        return nv_tegra_dce_register_ipc_client(interfaceType, usrCtx, NULL, clientId);
+    else if (interfaceType == DCE_CLIENT_RM_IPC_TYPE_EVENT)
+        return nv_tegra_dce_register_ipc_client(interfaceType, usrCtx, osTegraDceClientIpcCallback, clientId);
+    else
+        return NV_ERR_INVALID_ARGUMENT;
+}
+
+/*!
+ * @brief Performs IPC Client destroy with DCE
+ *
+ * This function is basically a wrapper to call the unix/linux layer.
+ *
+ * @param[in] clientId  unique ID registered with DCE for IPC
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ *          other errors as may be returned by subfunctions.
+ */
+NV_STATUS
+osTegraDceUnregisterIpcClient
+(
+    NvU32 clientId
+)
+{
+    return nv_tegra_dce_unregister_ipc_client(clientId);
+}
+
+/*!
+ * @brief Performs IPC Send/Receive to/from DCE
+ *
+ * This function is basically a wrapper to call the unix/linux layer.
+ *
+ * @param[in]  clientId   unique ID registered with DCE KMD for corresponding IPC type
+ * @param[in]  msg        structure to hold dce ipc message info
+ * @param[in]  msgLength  length of the message
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ *          other errors as may be returned by subfunctions.
+ */
+NV_STATUS
+osTegraDceClientIpcSendRecv
+(
+    NvU32 clientId,
+    void *msg,
+    NvU32 msgLength
+)
+{
+    return nv_tegra_dce_client_ipc_send_recv(clientId, msg, msgLength);
+}
+
+/*!
+ * @brief Sends an MRQ (message-request) to BPMP
+ *
+ * The request, response, and ret parameters of this function correspond to the
+ * components of the tegra_bpmp_message struct, which BPMP uses to receive
+ * MRQs.
+ *
+ * @param[in]  pGpu               OBJGPU pointer
+ * @param[in]  mrq                MRQ_xxx ID specifying what is requested
+ * @param[in]  pRequestData       Pointer to request input data
+ * @param[in]  requestDataSize    Size of structure pointed to by pRequestData
+ * @param[out] pResponseData      Pointer to response output data
+ * @param[in]  responseDataSize   Size of structure pointed to by pResponseData
+ * @param[out] ret                MRQ return code (from "ret" element of
+ *                                  tegra_bpmp_message struct)
+ * @param[out] apiRet             Return code from tegra_bpmp_transfer call
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available,
+ *          NV_ERR_INVALID_POINTER if the tegra_bpmp struct pointer could not
+ *            be obtained from nv, or
+ *          NV_ERR_GENERIC if the tegra_bpmp_transfer call failed (see apiRet
+ *            for Linux error code).
+ */
+NV_STATUS
+osTegraSocBpmpSendMrq
+(
+    OBJGPU      *pGpu,
+    NvU32        mrq,
+    const void  *pRequestData,
+    NvU32        requestDataSize,
+    void        *pResponseData,
+    NvU32        responseDataSize,
+    NvS32       *pRet,
+    NvS32       *pApiRet
+)
+{
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_SOC_SDM))
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+    else if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+    {
+        return nv_bpmp_send_mrq(pGpu->pOsGpuInfo,
+                                mrq,
+                                pRequestData,
+                                requestDataSize,
+                                pResponseData,
+                                responseDataSize,
+                                pRet,
+                                pApiRet);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/*!
+ * @brief Returns IMP-relevant data collected from other modules
+ *
+ * This function is basically a wrapper to call the unix/linux layer.
+ *
+ * @param[out]  pTegraImpImportData  Structure to receive the data
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_BUFFER_TOO_SMALL if the array in TEGRA_IMP_IMPORT_DATA is
+ *            too small,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ *          other errors as may be returned by subfunctions.
+ */
+NV_STATUS
+osTegraSocGetImpImportData
+(
+    OBJGPU *pGpu,
+    TEGRA_IMP_IMPORT_DATA *pTegraImpImportData
+)
+{
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+    {
+        return nv_imp_get_import_data(pTegraImpImportData);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/*!
+ * @brief Returns IMP-relevant data collected from UEFI
+ *
+ * This function is basically a wrapper to call the unix/linux layer.
+ *
+ * @param[in]   pOsGpuInfo    Per GPU Linux state
+ * @param[out]  pIsoBwKbps    ISO BW set by UEFI
+ * @param[out]  pFloorBwKbps  DRAM Floor BW set by UEFI
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ *          other errors as may be returned by subfunctions.
+ */
+NV_STATUS
+osTegraSocGetImpUefiData
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32 *pIsoBwKbps,
+    NvU32 *pFloorBwKbps
+)
+{
+    return nv_imp_get_uefi_data(pOsGpuInfo, pIsoBwKbps, pFloorBwKbps);
+}
+
+/*!
+ * @brief Tells BPMP whether or not RFL is valid
+ *
+ * Display HW generates an ok_to_switch signal which asserts when mempool
+ * occupancy is high enough to be able to turn off memory long enough to
+ * execute a dramclk frequency switch without underflowing display output.
+ * ok_to_switch drives the RFL ("request for latency") signal in the memory
+ * unit, and the switch sequencer waits for this signal to go active before
+ * starting a dramclk switch.  However, if the signal is not valid (e.g., if
+ * display HW or SW has not been initialized yet), the switch sequencer ignores
+ * the signal.  This API tells BPMP whether or not the signal is valid.
+ *
+ * @param[in] pOsGpuInfo    Per GPU Linux state
+ * @param[in] bEnable       True if RFL will be valid; false if invalid
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ *          NV_ERR_GENERIC if some other kind of error occurred.
+ */
+NV_STATUS
+osTegraSocEnableDisableRfl
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvBool       bEnable
+)
+{
+    if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo))
+    {
+        return nv_imp_enable_disable_rfl(pOsGpuInfo, bEnable);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/*!
+ * @brief Returns max rates for display clocks passed by UEFI
+ *
+ * @param[in]  pOsGpuInfo                   Per GPU Linux state
+ * @param[out] pMaxDispClkRateDisppll       disp clock maxrate with disppll as parent
+ * @param[out] pMaxDispClkRateSppllClkouta  disp clock maxrate with sppllclkouta as parent
+ * @param[out] pMaxHubClkRateSppllClkoutb   hub clock maxrate with sppllclkoutb as parent
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available
+ */
+NV_STATUS
+osTegraSocGetDispClockRates
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32       *pMaxDispClkRateDisppll,
+    NvU32       *pMaxDispClkRateSppllClkouta,
+    NvU32       *pMaxHubClkRateSppllClkoutb
+)
+{
+    nv_state_t *nv = pOsGpuInfo;
+
+    if (NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        *pMaxDispClkRateDisppll = nv->clocks.max_dispclk_rate_using_disppllkhz;
+        *pMaxDispClkRateSppllClkouta = nv->clocks.max_dispclk_rate_using_sppll0clkoutakhz;
+        *pMaxHubClkRateSppllClkoutb = nv->clocks.max_hubclk_rate_using_sppll0clkoutbkhz;
+
+        return NV_OK;
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/*!
+ * @brief Allocates a specified amount of ISO memory bandwidth for display
+ *
+ * floorBandwidthKBPS is the minimum required (i.e., floor) dramclk frequency
+ * multiplied by the width of the pipe over which the display data will travel.
+ * (It is understood that the bandwidth calculated by multiplying the clock
+ * frequency by the pipe width will not be realistically achievable, due to
+ * overhead in the memory subsystem.  The infrastructure will not actually use
+ * the bandwidth value, except to reverse the calculation to get the required
+ * dramclk frequency.)
+ *
+ * This function is basically a wrapper to call the unix/linux layer.
+ *
+ * @param[in]   pOsGpuInfo            OS specific GPU information pointer
+ * @param[in]   averageBandwidthKBPS  Amount of ISO memory bandwidth requested
+ * @param[in]   floorBandwidhtKBPS    Min required dramclk freq * pipe width
+ *
+ * @returns NV_OK if successful,
+ *          NV_ERR_INSUFFICIENT_RESOURCES if one of the bandwidth values is too
+ *            high, and bandwidth cannot be allocated,
+ *          NV_ERR_NOT_SUPPORTED if the functionality is not available, or
+ *          NV_ERR_GENERIC if some other kind of error occurred.
+ */
+NV_STATUS
+osTegraAllocateDisplayBandwidth
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32        averageBandwidthKBPS,
+    NvU32        floorBandwidthKBPS
+)
+{
+    if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo))
+    {
+        return nv_imp_icc_set_bw(pOsGpuInfo,
+                                 averageBandwidthKBPS,
+                                 floorBandwidthKBPS);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/*!
+ * @brief Creates or sets up platform specific nano second resolution timer
+ *
+ * @param[in]     pOsGpuInfo     OS specific GPU information pointer
+ * @param[in]     pTmrEvent      Pointer to timer event information
+ * @param[in/out] pTimer         pointer to hold high resolution timer object
+ */
+NV_STATUS
+osCreateNanoTimer
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    void *pTmrEvent,
+    void **pTimer
+)
+{
+    nv_create_nano_timer(pOsGpuInfo, pTmrEvent, (nv_nano_timer_t **)pTimer);
+    return NV_OK;
+}
+
+/*!
+ * @brief Starts platform specific nano second resolution timer
+ *
+ * @param[in] pOsGpuInfo     OS specific GPU information pointer
+ * @param[in] pTimer         pointer to high resolution timer object
+ * @param[in] timeNs         Relative time in nano seconds
+ */
+NV_STATUS
+osStartNanoTimer
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    void *pTimer,
+    NvU64 timeNs
+)
+{
+    nv_start_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer, timeNs);
+    return NV_OK;
+}
+
+/*!
+ * @brief Cancels platform specific nano second resolution timer
+ *
+ * @param[in] pOsGpuInfo     OS specific GPU information pointer
+ * @param[in] pTimer       pointer to timer object
+ */
+NV_STATUS
+osCancelNanoTimer
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    void *pTimer
+)
+{
+    nv_cancel_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer);
+    return NV_OK;
+}
+
+/*!
+ * @brief Destroys & cancels platform specific nano second resolution timer
+ *
+ *
+ * @param[in] pGpu         Device of interest
+ * @param[in] pTimer       pointer to timer object
+ */
+NV_STATUS
+osDestroyNanoTimer
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    void *pTimer
+)
+{
+    nv_destroy_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer);
+    return NV_OK;
+}
+
+
+NvBool osIsNonPreemptableDebuggerSessionActive(OBJGPU *pGpu)
+{
+    NvU32 data = 0;
+    if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_NON_PREEMTABLE_DEBUGGER_SESSION_ACTIVE, &data) == NV_OK) && (data == NV_REG_STR_RM_NON_PREEMTABLE_DEBUGGER_SESSION_ACTIVE_ENABLE)) 
+    {
+        return NV_TRUE;
+    }
+
+    return NV_FALSE;
+}
+
+/*!
+ * @brief Get number of dpAux instances.
+ * It is wrapper function to call unix/linux layer.
+ *
+ * @param[in]  pGpu           GPU object pointer
+ * @param[out]  pNumIntances  Number of dpAux instances.
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ *                     NV_ERR_GENERIC, if error
+ *                     NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+
+NV_STATUS
+osGetTegraNumDpAuxInstances
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32  *pNumIntances
+)
+{
+    if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo))
+    {
+        return nv_get_num_dpaux_instances(pOsGpuInfo, pNumIntances);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/*
+ * @brief Return the priv Data of current IRQ.
+ * It is wrapper function to call unix/linux layer.
+ *
+ * @param[in]  pGpu           Device of interest
+ * @param[out] pPrivData      privData of current IRQ
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ *                     NV_ERR_GENERIC, if error
+ *                     NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osGetCurrentIrqPrivData
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32  *pPrivData
+)
+{
+    if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo))
+    {
+        return nv_get_current_irq_priv_data(pOsGpuInfo, pPrivData);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/*!
+ * @brief Get the brightness level
+ * It is wrapper function to call unix/linux layer.
+ *
+ * @param[in]  pGpu           GPU object pointer
+ * @param[out] brightness     Pointer to brightness level
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ *                     NV_ERR_GENERIC, if error
+ *                     NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osGetTegraBrightnessLevel
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32 *brightness
+)
+{
+    if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo))
+    {
+        return nv_get_tegra_brightness_level(pOsGpuInfo, brightness);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/*!
+ * @brief Set the brightness level
+ * It is wrapper function to call unix/linux layer.
+ *
+ * @param[in]  pGpu           GPU object pointer
+ * @param[out] brightness     brightness level
+ *
+ * @returns NV_STATUS, NV_OK if success,
+ *                     NV_ERR_GENERIC, if error
+ *                     NV_ERR_NOT_SUPPORTED, for unsupported platforms
+ */
+NV_STATUS
+osSetTegraBrightnessLevel
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32 brightness
+)
+{
+    if (NV_IS_SOC_DISPLAY_DEVICE(pOsGpuInfo))
+    {
+        return nv_set_tegra_brightness_level(pOsGpuInfo, brightness);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+/* @brief Gets syncpoint aperture information
+ *
+ * @param[in] OS_GPU_INFO OS specific GPU information pointer
+ * @param[in]  syncpointId
+ * @param[out] *physAddr
+ * @param[out] *limit
+ * @param[out] *offset
+ */
+NV_STATUS
+osGetSyncpointAperture
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32 syncpointId,
+    NvU64 *physAddr,
+    NvU64 *limit,
+    NvU32 *offset
+)
+{
+    return nv_get_syncpoint_aperture(syncpointId, physAddr, limit, offset);
+}
+
+/*!
+ * @brief Enable PCIe AtomicOp Requester Enable and return
+ * the completer side capabilities that the requester can send.
+ *
+ * @param[in]    pOsGpuInfo   OS_GPU_INFO OS specific GPU information pointer
+ * @param[out]   pMask        mask of supported atomic size, including one or more of:
+ *                            OS_PCIE_CAP_MASK_REQ_ATOMICS_32
+ *                            OS_PCIE_CAP_MASK_REQ_ATOMICS_64
+ *                            OS_PCIE_CAP_MASK_REQ_ATOMICS_128
+ *
+ * @returns NV_STATUS, NV_OK if success
+ *                     NV_ERR_NOT_SUPPORTED if platform doesn't support this
+ *                     feature.
+ *                     NV_ERR_GENERIC for any other error
+ */
+
+NV_STATUS
+osConfigurePcieReqAtomics
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU32       *pMask
+)
+{
+    if (pMask)
+    {
+        *pMask = 0U;
+        if (pOsGpuInfo)
+        {
+            if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
+                                          OS_INTF_PCIE_REQ_ATOMICS_32BIT) == NV_OK)
+                *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_32;
+            if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
+                                          OS_INTF_PCIE_REQ_ATOMICS_64BIT) == NV_OK)
+                *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_64;
+            if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
+                                          OS_INTF_PCIE_REQ_ATOMICS_128BIT) == NV_OK)
+                *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_128;
+
+            if (*pMask != 0)
+                return NV_OK;
+        }
+    }
+    return NV_ERR_NOT_SUPPORTED;
+}
+
+/*!
+ * @brief Check GPU is accessible or not
+ *
+ * @param[in]  pGpu           GPU object pointer
+ *
+ * @returns NVBool, Returns TRUE if the GPU is accessible,
+ *                  FALSE, if error
+ */
+NvBool
+osIsGpuAccessible
+(
+    OBJGPU *pGpu
+)
+{
+    return nv_is_gpu_accessible(NV_GET_NV_STATE(pGpu));
+}
+
+/*!
+ * @brief Check whether GPU has received a shutdown notification from the OS
+ */
+NvBool
+osIsGpuShutdown
+(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    return nv ? nv->is_shutdown : NV_TRUE;
+}
+
+/*!
+ * @brief Check GPU OS info matches
+ *
+ * @param[in]  pGpu           GPU object pointer
+ *
+ * @returns NVBool, Returns TRUE if matched.
+ */
+NvBool
+osMatchGpuOsInfo
+(
+    OBJGPU *pGpu,
+    void   *pOsInfo
+)
+{
+    return nv_match_gpu_os_info(NV_GET_NV_STATE(pGpu), pOsInfo);
+}
+
+/*!
+ * @brief Release GPU OS info.
+ *
+ * @param[in]  pOsInfo        GPU OS info pointer
+ *
+ * @returns void
+ */
+void
+osReleaseGpuOsInfo
+(
+    void   *pOsInfo
+)
+{
+    nv_put_file_private(pOsInfo);
+}
+
+/*!
+ * @brief Get free, total memory of a NUMA node by NUMA node ID from kernel.
+ *
+ * @param[in]      numaId              NUMA node ID.
+ * @param[out]     free_memory_bytes   free memory in bytes.
+ * @param[out]     total_memory_bytes  total memory in bytes.
+ *
+ */
+void
+osGetNumaMemoryUsage
+(
+    NvS32 numaId,
+    NvU64 *free_memory_bytes,
+    NvU64 *total_memory_bytes
+)
+{
+    NV_STATUS status = os_get_numa_node_memory_usage(numaId,
+                                                     free_memory_bytes,
+                                                     total_memory_bytes);
+    NV_ASSERT(status == NV_OK);
+}
+
+/*!
+ * @brief Add GPU memory as a NUMA node.
+ *
+ * @param[in/out]  pOsGpuInfo   OS specific GPU information pointer
+ * @param[in]      offset       start offset of the partition within FB
+ * @param[in]      size         size of the partition
+ * @param[out]     pNumaNodeId  OS NUMA node id for the added memory.
+ *
+ * @returns NV_OK if all is okay.  Otherwise an error-specific value.
+ *
+ */
+NV_STATUS
+osNumaAddGpuMemory
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU64       offset,
+    NvU64       size,
+    NvU32       *pNumaNodeId
+)
+{
+    nv_state_t *nv = pOsGpuInfo;
+
+    return os_numa_add_gpu_memory(nv->handle, offset, size, pNumaNodeId);
+}
+
+/*!
+ * @brief Remove a particular SMC partition's GPU memory from OS kernel.
+ *
+ * Remove GPU memory from the OS kernel that is earlier added as a NUMA node
+ * to the kernel in platforms where GPU is coherently connected to the CPU.
+ *
+ * @param[in/out]  pOsGpuInfo   OS_GPU_INFO OS specific GPU information pointer
+ * @param[in]      offset       start offset of the partition within FB
+ * @param[in]      size         size of the partition
+ * @param[in]      numaNodeId   OS NUMA node id of the memory to be removed.
+ *
+ */
+void
+osNumaRemoveGpuMemory
+(
+    OS_GPU_INFO *pOsGpuInfo,
+    NvU64       offset,
+    NvU64       size,
+    NvU32       numaNodeId
+)
+{
+    nv_state_t *nv = pOsGpuInfo;
+
+    NV_STATUS status = os_numa_remove_gpu_memory(nv->handle, offset, size, numaNodeId);
+    NV_ASSERT(status == NV_OK);
+
+    return;
+}
+
+NvBool
+osDmabufIsSupported(void)
+{
+    return os_dma_buf_enabled;
+}
+
+NV_STATUS
+osGetEgmInfo
+(
+    OBJGPU *pGpu,
+    NvU64  *pPhysAddr,
+    NvU64  *pSize,
+    NvS32  *pNodeId
+)
+{
+    return nv_get_egm_info(NV_GET_NV_STATE(pGpu), pPhysAddr, pSize, pNodeId);
+}
+
+/*!
+ * @brief Offline (i.e., blacklist) the page containing a given address from OS kernel.
+ *
+ * @param[in]      address   Address (SPA) of the page to be offlined
+ *
+ */
+NV_STATUS
+osOfflinePageAtAddress
+(
+    NvU64       address
+)
+{
+    return os_offline_page_at_address(address);
+}
+
+void osAllocatedRmClient(void *pOsInfo)
+{
+    nv_file_private_t* nvfp = (nv_file_private_t*)pOsInfo;
+
+    if (nvfp != NULL)
+        nvfp->bCleanupRmapi = NV_TRUE;
+}
+
+/*!
+ * @brief Update variable to indicate console managed by drm driver.
+ *
+ * @param[in]  OBJGPU  GPU object pointer
+ *
+ * @returns void
+ */
+void
+osDisableConsoleManagement
+(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv  = NV_GET_NV_STATE(pGpu);
+
+    nv->client_managed_console = NV_TRUE;
+}
+
+/*
+ * @brief Return NV_TRUE if GPU is configured as CXL device.
+ *
+ * @param[in]   pGpu             GPU object pointer
+ */
+NvBool
+osGpuIsCxlDevice
+(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    return nv->is_cxl_dev;
+}
+
+/*
+ * @brief Disabel CXL cache and perform writeback and invalidate for CXL device.
+ *
+ * @param[in]   pGpu             GPU object pointer
+ * @param[in]   bEnableCache     Enable/Disable flag
+ */
+void
+osCxlSetCaching
+(
+    OBJGPU *pGpu,
+    NvBool  bEnableCache
+)
+{
+
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    nv_pci_cxl_set_caching(nv, bEnableCache);
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osapi.c b/userspace/sdks/nv570/nvalloc/unix/src/osapi.c
new file mode 100644
index 0000000..a3d614d
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/osapi.c
@@ -0,0 +1,6284 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+
+
+#include <nv_ref.h>
+#include <nv.h>
+#include <nv_escape.h>
+#include <nv-priv.h>
+#include <os/os.h>
+#include <osapi.h>
+#include <class/cl0000.h>
+#include <rmosxfac.h> // Declares RmInitRm().
+#include "gpu/gpu.h"
+#include "gps.h"
+#include <platform/chipset/chipset.h>
+
+#include <gpu/timer/objtmr.h>
+#include <gpu/subdevice/subdevice.h>
+#include <mem_mgr/mem.h>
+#include "kernel/gpu/mem_mgr/mem_mgr.h"
+
+#include <gpu/mem_sys/kern_mem_sys.h>
+
+#include <diagnostics/journal.h>
+#include <nvrm_registry.h>
+
+#include <nvUnixVersion.h>
+#include <gpu_mgr/gpu_mgr.h>
+#include <core/thread_state.h>
+#include <platform/acpi_common.h>
+#include <core/locks.h>
+#include "platform/sli/sli.h"
+
+#include <mem_mgr/p2p.h>
+
+#include "rmapi/exports.h"
+#include "rmapi/rmapi_utils.h"
+#include "rmapi/rs_utils.h"
+#include "rmapi/resource_fwd_decls.h"
+#include <nv-kernel-rmapi-ops.h>
+#include <rmobjexportimport.h>
+#include "nv-reg.h"
+#include "nv-firmware-registry.h"
+#include "core/hal_mgr.h"
+#include "gpu/device/device.h"
+
+#include "resserv/rs_server.h"
+#include "resserv/rs_client.h"
+#include "resserv/rs_resource.h"
+#include "gpu/gpu_uuid.h"
+
+#include "platform/chipset/pci_pbi.h"
+
+#include "ctrl/ctrl0000/ctrl0000system.h"
+#include "ctrl/ctrl0073/ctrl0073dp.h"
+#include "ctrl/ctrl0073/ctrl0073system.h"
+#include "ctrl/ctrl0073/ctrl0073specific.h"
+#include "ctrl/ctrl2080/ctrl2080bios.h"
+#include "ctrl/ctrl2080/ctrl2080fb.h"
+#include "ctrl/ctrl2080/ctrl2080perf.h"
+#include "ctrl/ctrl2080/ctrl2080gpu.h"
+#include "ctrl/ctrl402c.h"
+
+#include "g_nv_name_released.h"   // released chip entries from nvChipAutoFlags.h
+
+#include <virtualization/hypervisor/hypervisor.h>
+
+#include "gpu/bus/kern_bus.h"
+
+//
+// If timer callback comes when PM resume is in progress, then it can't be
+// serviced. The timer needs to be rescheduled in this case. This time controls
+// the duration of rescheduling.
+//
+#define TIMER_RESCHED_TIME_DURING_PM_RESUME_NS      (100 * 1000 * 1000)
+
+static void RmInflatePageArray(
+    RmPhysAddr *pteArray, 
+    NvU64 pageCountFrom,
+    NvU64 pageSizeFrom,
+    NvU64 pageCountTo,
+    NvU64 pageSizeTo);
+static void RmDeflatePageArray(
+    RmPhysAddr *pteArray, 
+    NvU64 pageCountFrom,
+    NvU64 pageSizeFrom,
+    NvU64 pageCountTo,
+    NvU64 pageSizeTo);
+
+static void RmInflatePageArray(
+    RmPhysAddr *pteArray, 
+    NvU64 pageCountFrom,
+    NvU64 pageSizeFrom,
+    NvU64 pageCountTo,
+    NvU64 pageSizeTo)
+{
+    NV_ASSERT(pageCountFrom <= pageCountTo);
+    NV_ASSERT(pageSizeFrom >= pageSizeTo);
+    NvUPtr dstPageIdx, dstPageOffset;
+    NvU64 i;
+    // convert the idx from pageCount_to to pageCount_from.
+    NvU64 dstPageShift = NV_SRC_TO_DST_PAGE_SHIFT(pageSizeTo, pageSizeFrom);
+
+    //
+    // We can do the translation in place by moving backwards, since there
+    // will always be more RM pages than OS pages
+    //
+    for (i = pageCountTo - 1; i != NV_U64_MAX; i--)
+    {
+        dstPageIdx = i >> dstPageShift;
+        dstPageOffset = (i & ((1 << dstPageShift) - 1)) *
+                pageSizeTo;
+        pteArray[i] = pteArray[dstPageIdx] + dstPageOffset;
+    }
+}
+
+static void RmDeflatePageArray(
+    RmPhysAddr *pteArray, 
+    NvU64 pageCountFrom,
+    NvU64 pageSizeFrom,
+    NvU64 pageCountTo,
+    NvU64 pageSizeTo)
+{
+    NV_ASSERT(pageCountFrom >= pageCountTo);
+    NV_ASSERT(pageSizeFrom <= pageSizeTo);
+    NvU64 i;
+
+    for (i = 0; i < pageCountTo; i++)
+    {
+        pteArray[i] = pteArray[(i << NV_SRC_TO_DST_PAGE_SHIFT(pageSizeFrom, pageSizeTo))];
+    }
+
+    // Zero out the rest of the addresses, which are now invalid
+    portMemSet(pteArray + i, 0, sizeof(*pteArray) * (pageCountFrom - i));
+}
+
+void convertPageArrayBetweenOsAndRm(
+    RmPhysAddr *pteArray, 
+    NvU64 pageCountFrom,
+    NvU64 pageSizeFrom,
+    NvU64 pageCountTo,
+    NvU64 pageSizeTo,
+    NvU64 *newPageCountTo)
+{
+    if (pageSizeFrom < pageSizeTo)
+    {
+        RmDeflatePageArray(pteArray, 
+            pageCountFrom,
+            pageSizeFrom, 
+            pageCountTo, 
+            pageSizeTo
+        );
+        if (newPageCountTo)
+        {
+            *newPageCountTo = pageCountTo;
+        }
+    }
+    else if (pageSizeFrom > pageSizeTo)
+    {
+        RmInflatePageArray(pteArray, 
+            pageCountFrom, 
+            pageSizeFrom,
+            pageCountTo,
+            pageSizeTo
+        );
+    }
+}
+
+//
+// Helper function which can be called before doing any RM control
+// This function:
+//
+// a. Performs threadStateInit().
+// b. Acquires API lock.
+// c. Checks if RMAPI client handle is valid (i.e. RM is initialized) and
+//    returns early if RMAPI client handle is invalid.
+// d. Increments the dynamic power refcount. If GPU is in RTD3 suspended
+//    state, then it will wake-up the GPU.
+// e. Returns the RMAPI interface handle.
+//
+// This function should be called only when caller doesn't have acquired API
+// lock. Caller needs to use RmUnixRmApiEpilogue() after RM control, if
+// RmUnixRmApiPrologue() is successful.
+//
+RM_API *RmUnixRmApiPrologue(nv_state_t *pNv, THREAD_STATE_NODE *pThreadNode, NvU32 module)
+{
+    threadStateInit(pThreadNode, THREAD_STATE_FLAGS_NONE);
+
+    if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, module)) == NV_OK)
+    {
+        if ((pNv->rmapi.hClient != 0) &&
+            (os_ref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE) == NV_OK))
+        {
+            return rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+        }
+
+        rmapiLockRelease();
+    }
+
+    threadStateFree(pThreadNode, THREAD_STATE_FLAGS_NONE);
+
+    return NULL;
+}
+
+//
+// Helper function which can be called after doing RM control, if
+// caller has used RmUnixRmApiPrologue() helper function. This function:
+//
+// a. Decrements the dynamic power refcount.
+// b. Release API lock.
+// c. Performs threadStateFree().
+//
+void RmUnixRmApiEpilogue(nv_state_t *pNv, THREAD_STATE_NODE *pThreadNode)
+{
+    os_unref_dynamic_power(pNv, NV_DYNAMIC_PM_FINE);
+    rmapiLockRelease();
+    threadStateFree(pThreadNode, THREAD_STATE_FLAGS_NONE);
+}
+
+NvBool RmGpuHasIOSpaceEnabled(nv_state_t * nv)
+{
+    NvU16 val;
+    NvBool has_io;
+    os_pci_read_word(nv->handle, NV_CONFIG_PCI_NV_1, &val);
+    has_io = FLD_TEST_DRF(_CONFIG, _PCI_NV_1, _IO_SPACE, _ENABLED, val);
+    return has_io;
+}
+
+// This is a stub function for unix
+void osHandleDeferredRecovery(
+    OBJGPU *pGpu
+)
+{
+
+}
+
+// This is a stub function for unix
+NvBool osIsSwPreInitOnly
+(
+    OS_GPU_INFO *pOsGpuInfo
+)
+{
+    return NV_FALSE;
+}
+
+const NvU8 * RmGetGpuUuidRaw(
+    nv_state_t *pNv
+)
+{
+    NV_STATUS rmStatus;
+    OBJGPU *pGpu = NULL;
+    NvU32 gidFlags;
+    NvBool isApiLockTaken = NV_FALSE;
+
+    if (pNv->nv_uuid_cache.valid)
+        return pNv->nv_uuid_cache.uuid;
+
+    if (!rmapiLockIsOwner())
+    {
+        rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU);
+        if (rmStatus != NV_OK)
+        {
+            return NULL;
+        }
+
+        isApiLockTaken = NV_TRUE;
+    }
+
+    pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+    //
+    // PBI is not present in simulation and the loop inside
+    // pciPbiReadUuid takes up considerable amount of time in
+    // simulation environment during RM load.
+    //
+    if (pGpu && IS_SIMULATION(pGpu))
+    {
+        rmStatus = NV_ERR_NOT_SUPPORTED;
+    }
+    else
+    {
+        if (!pNv->nv_uuid_cache.pci_uuid_read_attempted)
+        {
+            rmStatus = pciPbiReadUuid(pNv->handle, pNv->nv_uuid_cache.uuid);
+            pNv->nv_uuid_cache.pci_uuid_read_attempted = NV_TRUE;
+            pNv->nv_uuid_cache.pci_uuid_status = rmStatus;
+        }
+        else
+        {
+            rmStatus = pNv->nv_uuid_cache.pci_uuid_status;
+        }
+    }
+
+    if (rmStatus == NV_OK)
+    {
+        rmStatus = gpumgrSetUuid(pNv->gpu_id, pNv->nv_uuid_cache.uuid);
+        if (rmStatus != NV_OK)
+        {
+            goto err;
+        }
+
+        pNv->nv_uuid_cache.valid = NV_TRUE;
+        goto done;
+    }
+    else if (rmStatus == NV_ERR_NOT_SUPPORTED)
+    {
+        nv_printf(NV_DBG_INFO,
+                  "NVRM: PBI is not supported for GPU " NV_PCI_DEV_FMT "\n",
+                  NV_PCI_DEV_FMT_ARGS(pNv));
+    }
+
+    gidFlags = DRF_DEF(2080_GPU_CMD,_GPU_GET_GID_FLAGS,_TYPE,_SHA1)
+             | DRF_DEF(2080_GPU_CMD,_GPU_GET_GID_FLAGS,_FORMAT,_BINARY);
+
+    if (!pGpu)
+        goto err;
+
+    rmStatus = gpuGetGidInfo(pGpu, NULL, NULL, gidFlags);
+    if (rmStatus != NV_OK)
+        goto err;
+
+    if (!pGpu->gpuUuid.isInitialized)
+        goto err;
+
+    // copy the uuid from the OBJGPU uuid cache
+    os_mem_copy(pNv->nv_uuid_cache.uuid, pGpu->gpuUuid.uuid, GPU_UUID_LEN);
+    pNv->nv_uuid_cache.valid = NV_TRUE;
+
+done:
+    if (isApiLockTaken)
+    {
+        rmapiLockRelease();
+    }
+
+    return pNv->nv_uuid_cache.uuid;
+
+err:
+    if (isApiLockTaken)
+    {
+        rmapiLockRelease();
+    }
+
+    return NULL;
+}
+
+static NV_STATUS RmGpuUuidRawToString(
+    const NvU8     *pGidData,
+    char           *pBuf,
+    NvU32           bufLen
+)
+{
+    NvU8     *pGidString;
+    NvU32     GidStrlen;
+    NV_STATUS rmStatus;
+    NvU32     gidFlags;
+
+    gidFlags = DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _ASCII) |
+               DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1);
+
+    rmStatus = transformGidToUserFriendlyString(pGidData, RM_SHA1_GID_SIZE,
+                                                &pGidString, &GidStrlen,
+                                                gidFlags,
+                                                RM_UUID_PREFIX_GPU);
+    if (rmStatus != NV_OK)
+        return rmStatus;
+
+    if (bufLen >= GidStrlen)
+        portMemCopy(pBuf, bufLen, pGidString, GidStrlen);
+    else
+        rmStatus = NV_ERR_BUFFER_TOO_SMALL;
+
+    portMemFree((void *)pGidString);
+
+    return rmStatus;
+}
+
+// This function should be called with the API and GPU locks already acquired.
+void
+RmLogGpuCrash(OBJGPU *pGpu)
+{
+    NvBool bGpuIsLost, bGpuIsConnected;
+
+    //
+    // Re-evaluate whether or not the GPU is accessible. This could be called
+    // from a recovery context where the OS has re-enabled MMIO for the device.
+    // This happens during EEH processing on IBM Power + Linux, and marking
+    // the device as connected again will allow rcdbAddRmGpuDump() to collect
+    // more GPU state.
+    //
+    bGpuIsLost = pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_LOST);
+    bGpuIsConnected = pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED);
+    if (!bGpuIsConnected || bGpuIsLost)
+    {
+        nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+        nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+        NvU32 pmcBoot0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0);
+        if (pmcBoot0 == nvp->pmc_boot_0)
+        {
+            pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED, NV_TRUE);
+            pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_LOST, NV_FALSE);
+        }
+    }
+
+    {
+        //
+        // Log the engine data to the Journal object, to be pulled out later. This
+        // will return NV_WARN_MORE_PROCESSING_REQUIRED if the dump needed to be
+        // deferred to a passive IRQL. We still log the crash dump as being created
+        // in that case since it (should) be created shortly thereafter, and
+        // there's currently not a good way to print the below notification
+        // publicly from the core RM when the DPC completes.
+        //
+        NV_STATUS status = rcdbAddRmGpuDump(pGpu);
+        if (status != NV_OK && status != NV_WARN_MORE_PROCESSING_REQUIRED)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s: failed to save GPU crash data\n", __FUNCTION__);
+        }
+        else
+        {
+            nv_printf(NV_DBG_ERRORS,
+                "NVRM: A GPU crash dump has been created. If possible, please run\n"
+                "NVRM: nvidia-bug-report.sh as root to collect this data before\n"
+                "NVRM: the NVIDIA kernel module is unloaded.\n");
+            if (!IS_GSP_CLIENT(pGpu) && hypervisorIsVgxHyper())
+            {
+                nv_printf(NV_DBG_ERRORS, "NVRM: Dumping nvlogs buffers\n");
+                nvlogDumpToKernelLog(NV_FALSE);
+            }
+        }
+    }
+
+    // Restore the disconnected properties, if they were reset
+    pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED, bGpuIsConnected);
+    pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_LOST, bGpuIsLost);
+
+    // Restore persistence mode to the way it was prior to the crash
+    osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo,
+        pGpu->getProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE));
+}
+
+static void free_os_event_under_lock(nv_event_t *event)
+{
+    portAtomicSetS32(&event->active, 0);
+
+    // If refcount > 0, event will be freed by osDereferenceObjectCount
+    // when the last associated RM event is freed.
+    if (event->refcount == 0)
+        portMemFree(event);
+}
+
+static void free_os_events(
+    nv_file_private_t *nvfp,
+    NvHandle client
+)
+{
+    nv_state_t *nv = nv_get_ctl_state();
+    nv_event_t **pprev;
+
+    portSyncSpinlockAcquire(nv->event_spinlock);
+
+    pprev = &nv->event_list;
+    while (*pprev != NULL)
+    {
+        nv_event_t *cur = *pprev;
+        //
+        // XXX We must be called from either rm_client_free_os_events() or
+        // RmFreeUnusedClients() for this to work.
+        //
+        if ((cur->hParent == client) || (cur->nvfp == nvfp))
+        {
+            *pprev = cur->next;
+            free_os_event_under_lock(cur);
+        }
+        else
+        {
+            pprev = &cur->next;
+        }
+    }
+
+    portSyncSpinlockRelease(nv->event_spinlock);
+}
+
+static NV_STATUS get_os_event_data(
+    nv_file_private_t  *nvfp,
+    NvP64               pEvent,
+    NvU32              *MoreEvents
+)
+{
+    nv_event_t        nv_event;
+    NvUnixEvent      *nv_unix_event;
+    NV_STATUS         status;
+
+    status = os_alloc_mem((void**)&nv_unix_event, sizeof(NvUnixEvent));
+    if (status != NV_OK)
+        return status;
+
+    status = nv_get_event(nvfp, &nv_event, MoreEvents);
+    if (status != NV_OK)
+    {
+        status = NV_ERR_OPERATING_SYSTEM;
+        goto done;
+    }
+
+    os_mem_set(nv_unix_event, 0, sizeof(NvUnixEvent));
+    nv_unix_event->hObject     = nv_event.hObject;
+    nv_unix_event->NotifyIndex = nv_event.index;
+    nv_unix_event->info32      = nv_event.info32;
+    nv_unix_event->info16      = nv_event.info16;
+
+    status = os_memcpy_to_user(NvP64_VALUE(pEvent), nv_unix_event, sizeof(NvUnixEvent));
+done:
+    os_free_mem(nv_unix_event);
+    return status;
+}
+
+void rm_client_free_os_events(
+    NvHandle client
+)
+{
+    free_os_events(NULL, client);
+}
+
+void RmFreeUnusedClients(
+    nv_state_t         *nv,
+    nv_file_private_t  *nvfp
+)
+{
+    NvU32 *pClientList;
+    NvU32 numClients, i;
+    NV_STATUS status;
+    RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+    //
+    // The 'nvfp' pointer uniquely identifies an open instance in kernel space
+    // and the kernel interface layer guarantees that we are not called before
+    // the associated nvfp descriptor is closed. We can thus safely free
+    // abandoned clients with matching 'nvfp' pointers.
+    //
+    status = rmapiGetClientHandlesFromOSInfo(nvfp, &pClientList, &numClients);
+    if (status != NV_OK)
+    {
+        numClients = 0;
+    }
+
+    for (i = 0; i < numClients; ++i)
+    {
+        NV_PRINTF(LEVEL_INFO, "freeing abandoned client 0x%x\n",
+                  pClientList[i]);
+
+    }
+
+    if (numClients != 0)
+    {
+        pRmApi->DisableClients(pRmApi, pClientList, numClients);
+
+        portMemFree(pClientList);
+    }
+
+    // Clean up any remaining events using this nvfp.
+    free_os_events(nvfp, 0);
+}
+
+static void RmUnbindLock(
+    nv_state_t *nv
+)
+{
+    OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    if ((pGpu == NULL) || (gpuGetUserClientCount(pGpu) == 0))
+    {
+        nv->flags |= NV_FLAG_UNBIND_LOCK;
+    }
+}
+
+static NV_STATUS allocate_os_event(
+    NvHandle            hParent,
+    nv_file_private_t  *nvfp,
+    NvU32               fd
+)
+{
+    nv_state_t *nv = nv_get_ctl_state();
+    NvU32 status = NV_OK;
+    nv_event_t *event;
+
+    nv_event_t *new_event = portMemAllocNonPaged(sizeof(nv_event_t));
+    if (new_event == NULL)
+    {
+        status = NV_ERR_NO_MEMORY;
+        goto done;
+    }
+
+    new_event->hParent  = hParent;
+    new_event->nvfp     = nvfp;
+    new_event->fd       = fd;
+    portAtomicSetS32(&new_event->active, 1);
+    new_event->refcount = 0;
+
+    portSyncSpinlockAcquire(nv->event_spinlock);
+    for (event = nv->event_list; event; event = event->next)
+    {
+        // Only one event may be associated with a given fd.
+        if (event->hParent == hParent && event->fd == fd)
+        {
+            status = NV_ERR_INVALID_ARGUMENT;
+            portSyncSpinlockRelease(nv->event_spinlock);
+            goto done;
+        }
+    }
+
+    new_event->next = nv->event_list;
+    nv->event_list = new_event;
+    nvfp->bCleanupRmapi = NV_TRUE;
+    portSyncSpinlockRelease(nv->event_spinlock);
+
+done:
+    if (status == NV_OK)
+    {
+        NV_PRINTF(LEVEL_INFO, "allocated OS event:\n");
+        NV_PRINTF(LEVEL_INFO, "   hParent: 0x%x\n", hParent);
+        NV_PRINTF(LEVEL_INFO, "   fd: %d\n", fd);
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_INFO, "failed to allocate OS event: 0x%08x\n", status);
+        status = NV_ERR_INSUFFICIENT_RESOURCES;
+        portMemFree(new_event);
+    }
+
+    return status;
+}
+
+static NV_STATUS free_os_event(
+    NvHandle    hParent,
+    NvU32       fd
+)
+{
+    nv_state_t *nv = nv_get_ctl_state();
+    nv_event_t *event, *tmp;
+    NV_STATUS result;
+
+    portSyncSpinlockAcquire(nv->event_spinlock);
+    tmp = event = nv->event_list;
+    while (event)
+    {
+        if ((event->fd == fd) && (event->hParent == hParent))
+        {
+            if (event == nv->event_list)
+                nv->event_list = event->next;
+            else
+                tmp->next = event->next;
+            break;
+        }
+        tmp = event;
+        event = event->next;
+    }
+
+    if (event != NULL)
+    {
+        free_os_event_under_lock(event);
+        result = NV_OK;
+    }
+    else
+        result = NV_ERR_INVALID_EVENT;
+    portSyncSpinlockRelease(nv->event_spinlock);
+
+    if (result == NV_OK)
+    {
+        NV_PRINTF(LEVEL_INFO, "freed OS event:\n");
+        NV_PRINTF(LEVEL_INFO, "   hParent: 0x%x\n", hParent);
+        NV_PRINTF(LEVEL_INFO, "   fd: %d\n", fd);
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_ERROR, "failed to find OS event:\n");
+        NV_PRINTF(LEVEL_ERROR, "   hParent: 0x%x\n", hParent);
+        NV_PRINTF(LEVEL_ERROR, "   fd: %d\n", fd);
+    }
+
+    return result;
+}
+
+static void RmExecuteWorkItem(
+    void *pWorkItem
+)
+{
+    nv_work_item_t *pWi = (nv_work_item_t *)pWorkItem;
+    NvU32 gpuMask = 0;
+    OsQueueWorkItemFlags releaseLocks = {0};
+
+    if (!(pWi->flags.bRequiresGpu) &&
+        (pWi->flags.bLockGpus ||
+         pWi->flags.bLockGpuGroupDevice ||
+         pWi->flags.bLockGpuGroupSubdevice ||
+         pWi->flags.bFullGpuSanity))
+    {
+        // Requesting one of the GPU locks without providing a GPU instance
+        NV_ASSERT(0);
+        goto done;
+    }
+
+    // Get locks requested by workitem
+    if (NV_OK != workItemLocksAcquire(pWi->gpuInstance, pWi->flags,
+                                      &releaseLocks, &gpuMask))
+    {
+        goto done;
+    }
+
+    // Some work items may not require a valid GPU instance
+    if (pWi->flags.bRequiresGpu)
+    {
+        // Make sure that pGpu is present
+        OBJGPU *pGpu = gpumgrGetGpu(pWi->gpuInstance);
+        if (pGpu != NULL)
+        {
+            nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+            NV_ASSERT_OR_GOTO(nv != NULL, done);
+
+            NvU32 prevGpuInst = gpumgrSetCurrentGpuInstance(pGpu->gpuInstance);
+
+            if (!(pWi->flags.bDropOnUnloadQueueFlush && os_is_queue_flush_ongoing(nv->queue)))
+            {
+                pWi->func.pGpuFunction(pWi->gpuInstance, pWi->pData);
+            }
+
+            gpumgrSetCurrentGpuInstance(prevGpuInst);
+        }
+        else
+        {
+            NV_PRINTF(LEVEL_ERROR, "Invalid GPU instance for workitem\n");
+        }
+    }
+    else
+    {
+        if (!(pWi->flags.bDropOnUnloadQueueFlush && os_is_queue_flush_ongoing(NULL)))
+        {
+            pWi->func.pSystemFunction(pWi->pData);
+        }
+    }
+
+done:
+    // Release any locks taken
+    workItemLocksRelease(releaseLocks, gpuMask);
+
+    if ((pWi->pData != NULL) && !(pWi->flags.bDontFreeParams))
+    {
+        portMemFree(pWi->pData);
+    }
+
+    portMemFree((void *)pWi);
+}
+
+static NV_STATUS RmAccessRegistry(
+    NvHandle   hClient,
+    NvHandle   hObject,
+    NvU32      AccessType,
+    NvP64      clientDevNodeAddress,
+    NvU32      DevNodeLength,
+    NvP64      clientParmStrAddress,
+    NvU32      ParmStrLength,
+    NvP64      clientBinaryDataAddress,
+    NvU32      *pBinaryDataLength,
+    NvU32      *Data,
+    NvU32      *Entry
+)
+{
+    NvU32         gpuMask = 0, gpuInstance = 0;
+    OBJGPU       *pGpu;
+    NvBool        isDevice = NV_FALSE;
+    NV_STATUS     RmStatus = NV_ERR_OPERATING_SYSTEM;
+    CLIENT_ENTRY *pClientEntry;
+    Device       *pDevice;
+    Subdevice    *pSubdevice;
+
+    RMAPI_PARAM_COPY devNodeParamCopy;
+    NvU8      *tmpDevNode = NULL;
+    NvU32      copyOutDevNodeLength = 0;
+
+    RMAPI_PARAM_COPY parmStrParamCopy;
+    char      *tmpParmStr = NULL;
+    NvU32      copyOutParmStrLength = 0;
+
+    RMAPI_PARAM_COPY binaryDataParamCopy;
+    NvU8      *tmpBinaryData = NULL;
+    NvU32      BinaryDataLength = 0;
+    NvU32      copyOutBinaryDataLength = 0;
+
+    if (NV_OK != serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE,
+            &pClientEntry))
+    {
+        return NV_ERR_INVALID_CLIENT;
+    }
+
+    if (hClient == hObject)
+    {
+        pGpu = NULL;
+    }
+    else
+    {
+        RsClient *pClient = pClientEntry->pClient;
+
+        RmStatus = deviceGetByHandle(pClient, hObject, &pDevice);
+        if (RmStatus != NV_OK)
+        {
+            RmStatus = subdeviceGetByHandle(pClient, hObject, &pSubdevice);
+            if (RmStatus != NV_OK)
+                goto done;
+
+            RmStatus = rmGpuGroupLockAcquire(pSubdevice->subDeviceInst,
+                    GPU_LOCK_GRP_SUBDEVICE,
+                    GPUS_LOCK_FLAGS_NONE,
+                    RM_LOCK_MODULES_GPU,
+                    &gpuMask);
+            if (RmStatus != NV_OK)
+            {
+                gpuMask = 0;
+                goto done;
+            }
+
+            GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
+            pGpu = GPU_RES_GET_GPU(pSubdevice);
+        }
+        else
+        {
+            RmStatus = rmGpuGroupLockAcquire(pDevice->deviceInst,
+                    GPU_LOCK_GRP_DEVICE,
+                    GPUS_LOCK_FLAGS_NONE,
+                    RM_LOCK_MODULES_GPU,
+                    &gpuMask);
+            if (RmStatus != NV_OK)
+            {
+                gpuMask = 0;
+                goto done;
+            }
+
+            GPU_RES_SET_THREAD_BC_STATE(pDevice);
+            pGpu = GPU_RES_GET_GPU(pDevice);
+            isDevice = NV_TRUE;
+        }
+    }
+
+    if (pBinaryDataLength)
+    {
+        BinaryDataLength = *pBinaryDataLength;
+    }
+
+    // a passed-in devNode
+    if (DevNodeLength)
+    {
+        // the passed-in DevNodeLength does not account for '\0'
+        DevNodeLength++;
+
+        if (DevNodeLength > NVOS38_MAX_REGISTRY_STRING_LENGTH)
+        {
+            RmStatus = NV_ERR_INVALID_STRING_LENGTH;
+            goto done;
+        }
+
+        // get access to client's DevNode
+        RMAPI_PARAM_COPY_INIT(devNodeParamCopy, tmpDevNode, clientDevNodeAddress, DevNodeLength, 1);
+        devNodeParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_ZERO_BUFFER;
+        RmStatus = rmapiParamsAcquire(&devNodeParamCopy, NV_TRUE);
+        if (RmStatus != NV_OK)
+        {
+            RmStatus = NV_ERR_OPERATING_SYSTEM;
+            goto done;
+        }
+    }
+
+    // a passed-in parmStr
+    if (ParmStrLength)
+    {
+        // the passed-in ParmStrLength does not account for '\0'
+        ParmStrLength++;
+
+        if ((ParmStrLength == 0) || (ParmStrLength > NVOS38_MAX_REGISTRY_STRING_LENGTH))
+        {
+            RmStatus = NV_ERR_INVALID_STRING_LENGTH;
+            goto done;
+        }
+        // get access to client's parmStr
+        RMAPI_PARAM_COPY_INIT(parmStrParamCopy, tmpParmStr, clientParmStrAddress, ParmStrLength, 1);
+        RmStatus = rmapiParamsAcquire(&parmStrParamCopy, NV_TRUE);
+        if (RmStatus != NV_OK)
+        {
+            RmStatus = NV_ERR_OPERATING_SYSTEM;
+            goto done;
+        }
+        if (tmpParmStr[ParmStrLength - 1] != '\0')
+        {
+            RmStatus = NV_ERR_INVALID_ARGUMENT;
+            goto done;
+        }
+    }
+
+    if ((AccessType == NVOS38_ACCESS_TYPE_READ_BINARY) ||
+        (AccessType == NVOS38_ACCESS_TYPE_WRITE_BINARY))
+    {
+        if ((BinaryDataLength > NVOS38_MAX_REGISTRY_BINARY_LENGTH) ||
+            (BinaryDataLength == 0))
+        {
+            RmStatus = NV_ERR_INVALID_STRING_LENGTH;
+            goto done;
+        }
+
+        // get access to client's binaryData
+        RMAPI_PARAM_COPY_INIT(binaryDataParamCopy, tmpBinaryData, clientBinaryDataAddress, BinaryDataLength, 1);
+        if (AccessType == NVOS38_ACCESS_TYPE_READ_BINARY)
+            binaryDataParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYIN;
+        RmStatus = rmapiParamsAcquire(&binaryDataParamCopy, NV_TRUE);
+        if (RmStatus != NV_OK)
+        {
+            RmStatus = NV_ERR_OPERATING_SYSTEM;
+            goto done;
+        }
+    }
+
+    switch (AccessType)
+    {
+        case NVOS38_ACCESS_TYPE_READ_DWORD:
+            RmStatus = osReadRegistryDword(pGpu,
+                                tmpParmStr, Data);
+            break;
+
+        case NVOS38_ACCESS_TYPE_WRITE_DWORD:
+            if (!osIsAdministrator())
+            {
+                RmStatus = NV_ERR_INSUFFICIENT_PERMISSIONS;
+                break;
+            }
+            if (isDevice)
+            {
+                while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+                {
+                    RmStatus = osWriteRegistryDword(pGpu,
+                                    tmpParmStr, *Data);
+
+                    if (RmStatus != NV_OK)
+                        goto done;
+                }
+                break;
+            }
+
+            RmStatus = osWriteRegistryDword(pGpu,
+                                    tmpParmStr, *Data);
+            break;
+
+        case NVOS38_ACCESS_TYPE_READ_BINARY:
+            RmStatus = osReadRegistryBinary(pGpu,
+                                tmpParmStr, tmpBinaryData, &BinaryDataLength);
+
+            if (RmStatus != NV_OK)
+            {
+                goto done;
+            }
+
+            if (BinaryDataLength)
+                copyOutBinaryDataLength = BinaryDataLength;
+
+            break;
+
+        case NVOS38_ACCESS_TYPE_WRITE_BINARY:
+            if (!osIsAdministrator())
+            {
+                RmStatus = NV_ERR_INSUFFICIENT_PERMISSIONS;
+                break;
+            }
+            if (isDevice)
+            {
+                while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+                {
+                    RmStatus = osWriteRegistryBinary(pGpu,
+                                tmpParmStr, tmpBinaryData,
+                                BinaryDataLength);
+
+                    if (RmStatus != NV_OK)
+                        goto done;
+                }
+                break;
+            }
+
+            RmStatus = osWriteRegistryBinary(pGpu,
+                                tmpParmStr, tmpBinaryData,
+                                BinaryDataLength);
+            break;
+
+        default:
+            RmStatus = NV_ERR_INVALID_ACCESS_TYPE;
+    }
+
+ done:
+    if (gpuMask != 0)
+        rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+
+    if (tmpDevNode != NULL)
+    {
+        // skip copyout on error
+        if ((RmStatus != NV_OK) || (copyOutDevNodeLength == 0))
+            devNodeParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
+        devNodeParamCopy.paramsSize = copyOutDevNodeLength;
+        if (NV_OK != rmapiParamsRelease(&devNodeParamCopy))
+            if (RmStatus == NV_OK)
+                RmStatus = NV_ERR_OPERATING_SYSTEM;
+    }
+    if (tmpParmStr != NULL)
+    {
+        // skip copyout on error
+        if ((RmStatus != NV_OK) || (copyOutParmStrLength == 0))
+            parmStrParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
+        parmStrParamCopy.paramsSize = copyOutParmStrLength;
+        if (NV_OK != rmapiParamsRelease(&parmStrParamCopy))
+            if (RmStatus == NV_OK)
+                RmStatus = NV_ERR_OPERATING_SYSTEM;
+    }
+    if (tmpBinaryData != NULL)
+    {
+        // skip copyout on error
+        if ((RmStatus != NV_OK) || (copyOutBinaryDataLength == 0))
+            binaryDataParamCopy.flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
+        binaryDataParamCopy.paramsSize = copyOutBinaryDataLength;
+        if (NV_OK != rmapiParamsRelease(&binaryDataParamCopy))
+            if (RmStatus == NV_OK)
+                RmStatus = NV_ERR_OPERATING_SYSTEM;
+        *pBinaryDataLength = copyOutBinaryDataLength;
+    }
+
+    serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClientEntry);
+    return RmStatus;
+}
+
+static NV_STATUS RmUpdateDeviceMappingInfo(
+    NvHandle    hClient,
+    NvHandle    hDevice,
+    NvHandle    hMappable,
+    void       *pOldCpuAddress,
+    void       *pNewCpuAddress
+)
+{
+    NV_STATUS status;
+    CLIENT_ENTRY *pClientEntry;
+    RsClient *pClient;
+    RsResourceRef *pMappableRef;
+    RsCpuMapping *pCpuMapping;
+    Device *pDevice;
+    Subdevice *pSubdevice;
+    NvU32 gpuMask = 0;
+
+    status = serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClientEntry);
+    if (status != NV_OK)
+        return status;
+
+    pClient = pClientEntry->pClient;
+
+    status = deviceGetByHandle(pClient, hDevice, &pDevice);
+    if (status != NV_OK)
+    {
+        status = subdeviceGetByHandle(pClient, hDevice, &pSubdevice);
+        if (status != NV_OK)
+            goto done;
+
+        status = rmGpuGroupLockAcquire(pSubdevice->subDeviceInst,
+                                       GPU_LOCK_GRP_SUBDEVICE,
+                                       GPUS_LOCK_FLAGS_NONE,
+                                       RM_LOCK_MODULES_GPU,
+                                       &gpuMask);
+        if (status != NV_OK)
+            goto done;
+
+        GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
+    }
+    else
+    {
+        status = rmGpuGroupLockAcquire(pDevice->deviceInst,
+                                       GPU_LOCK_GRP_DEVICE,
+                                       GPUS_LOCK_FLAGS_NONE,
+                                       RM_LOCK_MODULES_GPU,
+                                       &gpuMask);
+        if (status != NV_OK)
+            goto done;
+
+        GPU_RES_SET_THREAD_BC_STATE(pDevice);
+    }
+
+    status = clientGetResourceRef(pClient, hMappable, &pMappableRef);
+    if (status != NV_OK)
+        goto done;
+
+    if ((objDynamicCastById(pMappableRef->pResource, classId(Memory)) == NULL) &&
+        (objDynamicCastById(pMappableRef->pResource, classId(KernelCcuApi)) == NULL) &&
+        (objDynamicCastById(pMappableRef->pResource, classId(KernelChannel)) == NULL))
+    {
+        status = NV_ERR_INVALID_OBJECT_HANDLE;
+        goto done;
+    }
+
+    status = refFindCpuMappingWithFilter(pMappableRef,
+                                         NV_PTR_TO_NvP64(pOldCpuAddress),
+                                         serverutilMappingFilterCurrentUserProc,
+                                         &pCpuMapping);
+    if (status != NV_OK)
+        goto done;
+
+    pCpuMapping->pLinearAddress = NV_PTR_TO_NvP64(pNewCpuAddress);
+
+done:
+
+    if (gpuMask != 0)
+        rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+
+    serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClientEntry);
+    return status;
+}
+
+static void RmSanitizeLogString(
+    char *pString,
+    NvU32 stringLength
+)
+{
+    NvU32 i;
+
+    if (pString == NULL)
+        return;
+
+    for (i = 0; i < stringLength; i++)
+    {
+        NvU8 ch = (NvU8)pString[i];
+
+        if (ch == '\0')
+            return;
+
+        if ((ch < 0x20) || (ch == 0x7f))
+        {
+            pString[i] = '?';
+        }
+    }
+}
+
+static NV_STATUS RmPerformVersionCheck(
+    void *pData,
+    NvU32 dataSize
+)
+{
+    nv_ioctl_rm_api_version_t *pParams;
+    char clientCh, rmCh;
+    const char *rmStr = NV_VERSION_STRING;
+    NvBool relaxed = NV_FALSE;
+    NvU32 i;
+    NvU32 procId;
+    char procName[32];
+
+    //
+    // rmStr (i.e., NV_VERSION_STRING) must be null-terminated and fit within
+    // NV_RM_API_VERSION_STRING_LENGTH, so that:
+    //
+    // (1) If the versions don't match, we can return rmStr in
+    //     pParams->versionString.
+    // (2) The below loop is guaranteed to not overrun rmStr.
+    //
+    ct_assert(sizeof(NV_VERSION_STRING) <= NV_RM_API_VERSION_STRING_LENGTH);
+
+    if (dataSize != sizeof(nv_ioctl_rm_api_version_t))
+        return NV_ERR_INVALID_ARGUMENT;
+
+    pParams = pData;
+
+    //
+    // write the reply value, so that the client knows we recognized
+    // the request
+    //
+    pParams->reply = NV_RM_API_VERSION_REPLY_RECOGNIZED;
+
+    //
+    // the client is just querying the version, not verifying against expected.
+    //
+    if (pParams->cmd == NV_RM_API_VERSION_CMD_QUERY)
+    {
+        os_string_copy(pParams->versionString, rmStr);
+        return NV_OK;
+    }
+
+    //
+    // the client requested relaxed version checking; we will only
+    // compare the strings until the first decimal point.
+    //
+    if (pParams->cmd == NV_RM_API_VERSION_CMD_RELAXED)
+    {
+        relaxed = NV_TRUE;
+    }
+
+    for (i = 0; i < NV_RM_API_VERSION_STRING_LENGTH; i++)
+    {
+        clientCh = pParams->versionString[i];
+        rmCh = rmStr[i];
+
+        //
+        // fail if the current character is not the same
+        //
+        if (clientCh != rmCh)
+        {
+            break;
+        }
+
+        //
+        // if relaxed matching was requested, succeed when we find the
+        // first decimal point
+        //
+        if ((relaxed) && (clientCh == '.'))
+        {
+            return NV_OK;
+        }
+
+        //
+        // we found the end of the strings: succeed
+        //
+        if (clientCh == '\0')
+        {
+            return NV_OK;
+        }
+    }
+
+    //
+    // the version strings did not match: print an error message and
+    // copy the RM's version string into pParams->versionString, so
+    // that the client can report the mismatch; explicitly NULL
+    // terminate the client's string, since we cannot trust it
+    //
+    pParams->versionString[NV_RM_API_VERSION_STRING_LENGTH - 1] = '\0';
+
+    procId = os_get_current_process();
+    os_get_current_process_name(procName, sizeof(procName));
+
+    RmSanitizeLogString(pParams->versionString, NV_RM_API_VERSION_STRING_LENGTH);
+    RmSanitizeLogString(procName, sizeof(procName));
+
+    nv_printf(NV_DBG_ERRORS,
+              "NVRM: API mismatch: the client '%s' (pid %u)\n"
+              "NVRM: has the version %s, but this kernel module has\n"
+              "NVRM: the version %s.  Please make sure that this\n"
+              "NVRM: kernel module and all NVIDIA driver components\n"
+              "NVRM: have the same version.\n",
+              procName, procId,
+              pParams->versionString, NV_VERSION_STRING);
+
+    os_string_copy(pParams->versionString, rmStr);
+
+    return NV_ERR_GENERIC;
+}
+
+NV_STATUS RmPowerSourceChangeEvent(
+    nv_state_t *pNv,
+    NvU32       event_val
+)
+{
+    NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS params = {0};
+    RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+
+    params.powerStateInfo.powerState = event_val ? NV2080_CTRL_PERF_POWER_SOURCE_BATTERY :
+                                                   NV2080_CTRL_PERF_POWER_SOURCE_AC;
+
+    return pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+                           pNv->rmapi.hSubDevice,
+                           NV2080_CTRL_CMD_PERF_SET_POWERSTATE,
+                           &params, sizeof(params));
+}
+
+/*!
+ * @brief Function to request latest D-Notifier status from SBIOS.
+ *
+ * Handle certain scenarios (like a fresh boot or suspend/resume
+ * of the system) when RM is not available to receive the Dx notifiers.
+ * This function gets the latest D-Notifier status from SBIOS
+ * when RM is ready to receive and handle those events.
+ * Use GPS_FUNC_REQUESTDXSTATE subfunction to invoke current Dx state.
+ *
+ * @param[in]   pNv   nv_state_t pointer.
+ */
+void RmRequestDNotifierState(
+    nv_state_t *pNv
+)
+{
+    OBJGPU *pGpu         = NV_GET_NV_PRIV_PGPU(pNv);
+    NvU32 supportedFuncs = 0;
+    NvU16 dsmDataSize    = sizeof(supportedFuncs);
+    NV_STATUS status     = NV_OK;
+
+    status = osCallACPI_DSM(pGpu, ACPI_DSM_FUNCTION_GPS_2X,
+                            GPS_FUNC_REQUESTDXSTATE, &supportedFuncs,
+                            &dsmDataSize);
+    if (status != NV_OK)
+    {
+        //
+        // Call for 'GPS_FUNC_REQUESTDXSTATE' subfunction may fail if the
+        // SBIOS/EC does not have the corresponding implementation.
+        //
+        NV_PRINTF(LEVEL_INFO,
+                  "%s: Failed to request Dx event update, status 0x%x\n",
+                  __FUNCTION__, status);
+    }
+}
+
+/*!
+ * @brief Deal with D-notifier events to apply a performance
+ * level based on the requested auxiliary power-state.
+ * Read confluence page "D-Notifiers on Linux" for more details.
+ *
+ * @param[in]   pNv          nv_state_t pointer.
+ * @param[in]   event_type   NvU32 Event type.
+ */
+static void RmHandleDNotifierEvent(
+    nv_state_t *pNv,
+    NvU32       event_type
+)
+{
+    NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS params = { 0 };
+    RM_API            *pRmApi;
+    THREAD_STATE_NODE  threadState;
+    NV_STATUS          rmStatus = NV_OK;
+
+    switch (event_type)
+    {
+        case ACPI_NOTIFY_POWER_LEVEL_D1:
+            params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P0;
+            break;
+        case ACPI_NOTIFY_POWER_LEVEL_D2:
+            params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P1;
+            break;
+        case ACPI_NOTIFY_POWER_LEVEL_D3:
+            params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P2;
+            break;
+        case ACPI_NOTIFY_POWER_LEVEL_D4:
+            params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P3;
+            break;
+        case ACPI_NOTIFY_POWER_LEVEL_D5:
+            params.powerState = NV2080_CTRL_PERF_AUX_POWER_STATE_P4;
+            break;
+        default:
+            return;
+    }
+
+    pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_ACPI);
+    if (pRmApi == NULL)
+    {
+        return;
+    }
+
+    rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+                               pNv->rmapi.hSubDevice,
+                               NV2080_CTRL_CMD_PERF_SET_AUX_POWER_STATE,
+                               &params, sizeof(params));
+
+    RmUnixRmApiEpilogue(pNv, &threadState);
+
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s: Failed to handle ACPI D-Notifier event, status=0x%x\n",
+                  __FUNCTION__, rmStatus);
+    }
+}
+
+static NvU32 RmDmabufMmapGetCpuCacheType(
+    OBJGPU            *pGpu,
+    MEMORY_DESCRIPTOR *pMemDesc,
+    NvU8               mappingType
+)
+{
+    if (pGpu->pGpuArch->bGpuArchIsZeroFb)
+    {
+        return memdescGetCpuCacheAttrib(pMemDesc);
+    }
+
+#if defined(NVCPU_X86_64)
+    return NV_MEMORY_WRITECOMBINED;
+#elif defined(NVCPU_AARCH64)
+    if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) &&
+        (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT))
+    {
+        return NV_MEMORY_CACHED;
+    }
+    else
+    {
+        return NV_MEMORY_UNCACHED;
+    }
+#else
+    // Default to uncached for other platforms for now
+    return NV_MEMORY_UNCACHED;
+#endif
+}
+
+static NV_STATUS
+RmDmabufVerifyMemHandle(
+    OBJGPU             *pGpu,
+    NvHandle            hSrcClient,
+    NvHandle            hMemory,
+    NvU64               offset,
+    NvU64               size,
+    void               *pGpuInstanceInfo,
+    MEMORY_DESCRIPTOR **ppMemDesc
+)
+{
+    NV_STATUS status;
+    RsClient *pClient = NULL;
+    RsResourceRef *pSrcMemoryRef = NULL;
+    Memory *pSrcMemory = NULL;
+    MEMORY_DESCRIPTOR *pMemDesc = NULL;
+
+    NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hSrcClient, &pClient));
+
+    status = clientGetResourceRef(pClient, hMemory, &pSrcMemoryRef);
+    if (status != NV_OK)
+    {
+        return status;
+    }
+
+    pSrcMemory = dynamicCast(pSrcMemoryRef->pResource, Memory);
+    if (pSrcMemory == NULL)
+    {
+        return NV_ERR_INVALID_OBJECT;
+    }
+
+    pMemDesc = pSrcMemory->pMemDesc;
+    if (pMemDesc == NULL)
+    {
+        return NV_ERR_INVALID_STATE;
+    }
+
+    //
+    // We skip the partitionable heap check when the source memory is in
+    // sysmem as there is no valid heap (pHeap will be NULL).
+    //
+    if (pGpuInstanceInfo != NULL && memdescGetAddressSpace(pMemDesc) != ADDR_SYSMEM)
+    {
+        KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance;
+        pKernelMIGGpuInstance = (KERNEL_MIG_GPU_INSTANCE *) pGpuInstanceInfo;
+
+        if ((pKernelMIGGpuInstance->pMemoryPartitionHeap != pSrcMemory->pHeap))
+        {
+            return NV_ERR_INVALID_OBJECT_PARENT;
+        }
+    }
+
+    // Check if hMemory belongs to the same pGpu
+    if ((pMemDesc->pGpu != pGpu) ||
+        (pSrcMemory->pGpu != pGpu))
+    {
+        return NV_ERR_INVALID_OBJECT_PARENT;
+    }
+
+    // Offset and size must be aligned to OS page-size
+    if (!NV_IS_ALIGNED64(offset, os_page_size) ||
+        !NV_IS_ALIGNED64(size, os_page_size))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    // Only supported for vidmem and sysmem(only for 0FB) handles
+    if ((memdescGetAddressSpace(pMemDesc) != ADDR_FBMEM) &&
+        (!pGpu->pGpuArch->bGpuArchIsZeroFb))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    if ((size == 0) ||
+        (size > memdescGetSize(pMemDesc)) ||
+        (offset > (memdescGetSize(pMemDesc) - size)))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    *ppMemDesc = pMemDesc;
+
+    return NV_OK;
+}
+
+static NV_STATUS
+RmDmabufGetClientAndDevice(
+    OBJGPU   *pGpu,
+    NvHandle  hClient,
+    NvHandle  hMemory,
+    NvU8      mappingType,
+    NvHandle *phClient,
+    NvHandle *phDevice,
+    NvHandle *phSubdevice,
+    void    **ppGpuInstanceInfo
+)
+{
+    MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+
+    // No dma-buf support for SLI-enabled GPU
+    if (IsSLIEnabled(pGpu))
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    //
+    // MAPPING_TYPE_FORCE_PCIE is to be used only on coherent systems with a
+    // direct PCIe connection between the exporter and importer.
+    // MIG is not a supported use-case for dma-buf on these systems.
+    //
+    if (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE)
+    {
+        KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+
+        if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) ||
+            pKernelBus->bBar1Disabled ||
+            IS_MIG_ENABLED(pGpu))
+        {
+            return NV_ERR_NOT_SUPPORTED;
+        }
+    }
+
+    if (IS_MIG_ENABLED(pGpu))
+    {
+        MIG_INSTANCE_REF ref;
+        RsClient *pClient;
+        Memory *pMemory;
+        KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
+
+        NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
+                              serverGetClientUnderLock(&g_resServ, hClient, &pClient));
+
+        NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
+                              memGetByHandle(pClient, hMemory, &pMemory));
+
+        NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
+                              kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager,
+                                                              pMemory->pDevice, &ref));
+
+        NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
+                              kmigmgrIncRefCount(ref.pKernelMIGGpuInstance->pShare));
+
+        *phClient    = ref.pKernelMIGGpuInstance->instanceHandles.hClient;
+        *phDevice    = ref.pKernelMIGGpuInstance->instanceHandles.hDevice;
+        *phSubdevice = ref.pKernelMIGGpuInstance->instanceHandles.hSubdevice;
+        *ppGpuInstanceInfo = (void *) ref.pKernelMIGGpuInstance;
+
+        return NV_OK;
+    }
+
+    *phClient    = pMemoryManager->hClient;
+    *phDevice    = pMemoryManager->hDevice;
+    *phSubdevice = pMemoryManager->hSubdevice;
+    *ppGpuInstanceInfo = NULL;
+
+    return NV_OK;
+}
+
+static void
+RmDmabufPutClientAndDevice(
+    OBJGPU   *pGpu,
+    NvHandle  hClient,
+    NvHandle  hDevice,
+    NvHandle  hSubdevice,
+    void     *pGpuInstanceInfo
+)
+{
+    KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance;
+
+    if (pGpuInstanceInfo == NULL)
+    {
+        return;
+    }
+
+    pKernelMIGGpuInstance = (KERNEL_MIG_GPU_INSTANCE *) pGpuInstanceInfo;
+
+    NV_ASSERT_OK(kmigmgrDecRefCount(pKernelMIGGpuInstance->pShare));
+}
+
+static NV_STATUS RmP2PDmaMapPages(
+    nv_state_t      *pNv,
+    nv_dma_device_t *peer,
+    NvU64            pageSize,
+    NvU32            pageCount,
+    NvU64           *pDmaAddresses,
+    void           **ppPriv
+)
+{
+    NV_STATUS status;
+    NvU64 *pOsDmaAddresses = NULL;
+    NvU32 osPagesPerP2PPage, osPageCount, count;
+    NvBool bDmaMapped = NV_FALSE;
+    NvU32 i, j, index;
+    void *pgmap = NULL;
+    NvBool bPageMapRef = NV_FALSE;
+
+    NV_ASSERT_OR_RETURN((pageSize >= os_page_size), NV_ERR_INVALID_ARGUMENT);
+
+    if (pNv->coherent_gpu_mem_mode == NV_COHERENT_GPU_MEM_MODE_DRIVER)
+    {
+        // Take a refcount on the dev_pagemap created for GPU memory by UVM
+        pgmap = nv_dma_get_dev_pagemap(pDmaAddresses[0]);
+        if (pgmap == NULL)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "Failed to get a reference on dev_pagemap in CDMM(driver) mode\n");
+            return NV_ERR_NOT_SUPPORTED;
+        }
+
+        bPageMapRef = NV_TRUE;
+    }
+
+    if (pageSize == os_page_size)
+    {
+        status = nv_dma_map_alloc(peer, pageCount, pDmaAddresses, NV_FALSE, NV_FALSE, ppPriv);
+
+        goto put_pgmap;
+    }
+
+    //
+    // If P2P page_size returned from nvidia_p2p_get_pages is bigger than
+    // OS page size, the page array need to be inflated to correctly do the DMA mapping.
+    // For example, if page_size = 64K and OS page size = 4K,
+    // there are 16 OS pages to this 64K P2P page.
+    //
+
+    osPagesPerP2PPage = pageSize / os_page_size;
+    osPageCount = osPagesPerP2PPage * pageCount;
+
+    status = os_alloc_mem((void **)&pOsDmaAddresses,
+                          (osPageCount * sizeof(NvU64)));
+    if (status != NV_OK)
+    {
+        goto failed;
+    }
+
+    index = 0;
+    for (i = 0; i < pageCount; i++)
+    {
+        pOsDmaAddresses[index] = pDmaAddresses[i];
+        index++;
+
+        for (j = 1; j < osPagesPerP2PPage; j++)
+        {
+            pOsDmaAddresses[index] = pOsDmaAddresses[index - 1] + os_page_size;
+            index++;
+        }
+    }
+
+    status = nv_dma_map_alloc(peer, osPageCount, pOsDmaAddresses, NV_FALSE, NV_FALSE, ppPriv);
+    if (status != NV_OK)
+    {
+        goto failed;
+    }
+    bDmaMapped = NV_TRUE;
+
+    //
+    // The driver breaks down the size into submaps and dma-maps those individually.
+    // This may result in non-contiguous IOVA segments allocated to each submap.
+    // We check for OS page contiguity within the bigger pageSize since we need to
+    // return IOVA chunks at pageSize granularity.
+    //
+    count = 0;
+    for (i = 1; i < osPageCount; i++)
+    {
+        count++;
+        if (count == osPagesPerP2PPage)
+        {
+            count = 0;
+            continue;
+        }
+
+        if (pOsDmaAddresses[i] != (pOsDmaAddresses[i - 1] + os_page_size))
+        {
+            status = NV_ERR_INVALID_STATE;
+            goto failed;
+        }
+    }
+
+    //
+    // Fill pDmaAddresses at pageSize granularity
+    // from expanded pOsDmaAddresses.
+    //
+    index = 0;
+    for (i = 0; i < pageCount; i++)
+    {
+        pDmaAddresses[i] = pOsDmaAddresses[index];
+        index += osPagesPerP2PPage;
+    }
+
+    os_free_mem(pOsDmaAddresses);
+
+    if (bPageMapRef)
+    {
+        nv_dma_put_dev_pagemap(pgmap);
+    }
+
+    return NV_OK;
+
+failed:
+    if (bDmaMapped)
+    {
+        NV_ASSERT_OK(nv_dma_unmap_alloc(peer, osPageCount,
+                                        pOsDmaAddresses, ppPriv));
+    }
+
+    if (pOsDmaAddresses != NULL)
+    {
+        os_free_mem(pOsDmaAddresses);
+    }
+
+put_pgmap:
+    if (bPageMapRef)
+    {
+        nv_dma_put_dev_pagemap(pgmap);
+    }
+
+    return status;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ *
+ * The routines below are part of the interface between the kernel interface
+ * layer and the kernel-agnostic portions of the resource manager.
+ *
+ * ---------------------------------------------------------------------------
+ */
+
+NvBool NV_API_CALL rm_init_private_state(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    NvBool retval;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    retval = RmInitPrivateState(pNv);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return retval;
+}
+
+void NV_API_CALL rm_free_private_state(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    RmFreePrivateState(pNv);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NvBool NV_API_CALL rm_init_adapter(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    THREAD_STATE_NODE threadState;
+    NvBool     retval = NV_FALSE;
+    void      *fp;
+    NvBool     bEnabled;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_DEVICE_INIT);
+
+    //
+    // If GPU is driving any frame buffer console(vesafb, efifb etc)
+    // mark the console as client driven
+    //
+    pNv->client_managed_console = rm_get_uefi_console_status(pNv);
+
+    //
+    // For GPU driving console, disable console access here, to ensure no console
+    // writes through BAR1 can interfere with physical RM's setup of BAR1
+    //
+    if (pNv->client_managed_console && !pNv->sysmem_mapped_console)
+    {
+        os_disable_console_access();
+    }
+
+    // LOCK: acquire API lock
+    if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK)
+    {
+        if (!((gpumgrQueryGpuDrainState(pNv->gpu_id, &bEnabled, NULL) == NV_OK)
+              && bEnabled))
+        {
+            if (pNv->flags & NV_FLAG_PERSISTENT_SW_STATE)
+            {
+                retval = RmPartiallyInitAdapter(pNv);
+            }
+            else
+            {
+                retval = RmInitAdapter(pNv);
+            }
+        }
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    if (pNv->client_managed_console && !pNv->sysmem_mapped_console)
+    {
+        os_enable_console_access();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return retval;
+}
+
+void NV_API_CALL rm_disable_adapter(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    THREAD_STATE_NODE threadState;
+    void      *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_ASSERT_OK(os_flush_work_queue(pNv->queue, NV_TRUE));
+
+    if (pNv->client_managed_console && !pNv->sysmem_mapped_console)
+    {
+        os_disable_console_access();
+    }
+
+    // LOCK: acquire API lock
+    if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+    {
+        if (pNv->flags & NV_FLAG_PERSISTENT_SW_STATE)
+        {
+            RmPartiallyDisableAdapter(pNv);
+        }
+        else
+        {
+            RmDisableAdapter(pNv);
+        }
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    if (pNv->client_managed_console && !pNv->sysmem_mapped_console)
+    {
+        os_enable_console_access();
+    }
+
+    NV_ASSERT_OK(os_flush_work_queue(pNv->queue, NV_TRUE));
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_shutdown_adapter(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    THREAD_STATE_NODE threadState;
+    void      *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if (pNv->client_managed_console && !pNv->sysmem_mapped_console)
+    {
+        os_disable_console_access();
+    }
+
+    // LOCK: acquire API lock
+    if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+    {
+        RmShutdownAdapter(pNv);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    if (pNv->client_managed_console && !pNv->sysmem_mapped_console)
+    {
+        os_enable_console_access();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_exclude_adapter(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    NV_STATUS rmStatus;
+    THREAD_STATE_NODE threadState;
+    void      *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    rmStatus = RmExcludeAdapter(pNv);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_acquire_api_lock(
+    nvidia_stack_t *sp
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_release_api_lock(
+    nvidia_stack_t *sp
+)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // UNLOCK: release API lock
+    rmapiLockRelease();
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_acquire_gpu_lock(
+    nvidia_stack_t *sp,
+    nv_state_t     *nv
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire GPU lock
+    rmStatus = rmDeviceGpuLocksAcquire(NV_GET_NV_PRIV_PGPU(nv),
+                                       GPUS_LOCK_FLAGS_NONE,
+                                       RM_LOCK_MODULES_OSAPI);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_release_gpu_lock(
+    nvidia_stack_t *sp,
+    nv_state_t     *nv
+)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // UNLOCK: release GPU lock
+    rmDeviceGpuLocksRelease(NV_GET_NV_PRIV_PGPU(nv), GPUS_LOCK_FLAGS_NONE, NULL);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_acquire_all_gpus_lock(
+    nvidia_stack_t *sp
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire all GPUs lock
+    rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_release_all_gpus_lock(
+    nvidia_stack_t *sp
+)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // UNLOCK: release all GPUs lock
+    rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return NV_OK;
+}
+
+/*!
+ * @brief Handle ACPI_NOTIFY_GPS_STATUS_CHANGE event.
+ *
+ * This function is called for GPS when SBIOS trigger
+ * gps STATUS_CHANGE event, which calls rm control call
+ * NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL to init the GPS
+ * data from SBIOS.
+ */
+static void RmHandleGPSStatusChange
+(
+    nv_state_t *pNv
+)
+{
+    NV_STATUS          rmStatus;
+    RM_API            *pRmApi;
+    THREAD_STATE_NODE  threadState;
+    NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS gpsControl = { 0 };
+
+    pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_ACPI);
+    if (pRmApi == NULL)
+    {
+        return;
+    }
+
+    gpsControl.command = NV0000_CTRL_CMD_SYSTEM_GPS_DATA_INIT_USING_SBIOS_AND_ACK;
+    gpsControl.locale  = NV0000_CTRL_CMD_SYSTEM_GPS_LOCALE_SYSTEM;
+
+    //
+    // NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL is not a subdevice command.
+    // But, Inside GPS module, the first GPU, which has GPS
+    // enabled will be grabbed with GPU manager. The RMAPI_API_LOCK_INTERNAL
+    // will internally grab the GPU locks and wake-up all the GPUs. Ideally
+    // this control call should be GPU specific command, if internally
+    // it uses the GPU.
+    //
+    rmStatus = pRmApi->Control(pRmApi,
+                               pNv->rmapi.hClient,
+                               pNv->rmapi.hClient,
+                               NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL,
+                               (void *)&gpsControl,
+                               sizeof(NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS));
+
+    RmUnixRmApiEpilogue(pNv, &threadState);
+
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s: Failed to handle ACPI GPS status change event, status=0x%x\n",
+                  __FUNCTION__, rmStatus);
+    }
+}
+
+/*!
+ * @brief Function to handle device specific ACPI events.
+ *
+ * @param[in]   sp           nvidia_stack_t pointer.
+ * @param[in]   nv           nv_state_t pointer.
+ * @param[in]   event_type   NvU32 Event type.
+ */
+void NV_API_CALL rm_acpi_notify(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    NvU32 event_type
+)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    switch (event_type)
+    {
+        case ACPI_VIDEO_NOTIFY_PROBE:
+        {
+            THREAD_STATE_NODE threadState;
+
+            threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+            RmHandleDisplayChange(sp, nv);
+            threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+            break;
+        }
+
+        case ACPI_NOTIFY_GPS_STATUS_CHANGE:
+            RmHandleGPSStatusChange(nv);
+            break;
+
+        case ACPI_NOTIFY_POWER_LEVEL_D1:    /* fallthrough */
+        case ACPI_NOTIFY_POWER_LEVEL_D2:    /* fallthrough */
+        case ACPI_NOTIFY_POWER_LEVEL_D3:    /* fallthrough */
+        case ACPI_NOTIFY_POWER_LEVEL_D4:    /* fallthrough */
+        case ACPI_NOTIFY_POWER_LEVEL_D5:
+            RmHandleDNotifierEvent(nv, event_type);
+            break;
+
+        default:
+            NV_PRINTF(LEVEL_INFO, "No support for 0x%x event\n", event_type);
+            NV_ASSERT(0);
+            break;
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+static void nv_align_mmap_offset_length(
+    nv_usermap_access_params_t *nvuap)
+{
+    NvU64 page_size = os_page_size;
+    NvU64 end = nvuap->size + (nvuap->addr & (page_size - 1));
+
+    nvuap->memArea.numRanges = 1;
+    nvuap->memArea.pRanges[0].start = NV_ALIGN_DOWN(nvuap->addr, page_size);
+    nvuap->memArea.pRanges[0].size = NV_ALIGN_UP(end, page_size);
+    nvuap->offset = NV_ALIGN_DOWN(nvuap->offset, page_size);
+}
+
+static inline NV_STATUS RmGetArrayMinMax(
+    NvU64 *array,
+    NvU64 count,
+    NvU64 *min,
+    NvU64 *max
+)
+{
+    NvU64 i;
+
+    if (array == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    *min = array[0];
+    *max = array[0];
+
+    if (count == 1)
+        return NV_OK;
+
+    for (i = 1; i < count; i++)
+    {
+        if (array[i] > *max)
+            *max = array[i];
+
+        if (array[i] < *min)
+            *min = array[i];
+    }
+
+    return NV_OK;
+}
+
+static NV_STATUS RmGetAllocPrivate(RmClient *, NvU32, NvU64, NvU64, NvU32 *, void **,
+                                   NvU64 *);
+
+static NV_STATUS RmValidateMmapRequest(
+    nv_state_t *pNv,
+    NvU64       offset,
+    NvU64       length,
+    NvU32      *pProtection
+)
+{
+    NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS params = { 0 };
+    RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    NV_STATUS status;
+
+    if (osIsAdministrator())
+    {
+        *pProtection = NV_PROTECT_READ_WRITE;
+        return NV_OK;
+    }
+
+    params.addressStart = offset;
+    params.addressLength = length;
+
+    status = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+                             pNv->rmapi.hSubDevice,
+                             NV2080_CTRL_CMD_GPU_VALIDATE_MEM_MAP_REQUEST,
+                             &params, sizeof(params));
+
+    if (status == NV_OK)
+    {
+        *pProtection = params.protection;
+    }
+
+    return status;
+}
+
+static NV_STATUS RmSetUserMapAccessRange(
+    nv_usermap_access_params_t *nvuap
+)
+{
+    NV_STATUS status = NV_OK;
+
+    if (nvuap->contig)
+    {
+         nvuap->access_start = nvuap->memArea.pRanges[0].start;
+         nvuap->access_size = nvuap->memArea.pRanges[0].size;
+    }
+    else
+    {
+        NvU64 highest_address_mapped;
+        NvU64 lowest_address_mapped;
+
+        status = RmGetArrayMinMax(nvuap->page_array, nvuap->num_pages,
+                                  &lowest_address_mapped,
+                                  &highest_address_mapped);
+        if (status != NV_OK)
+        {
+            return status;
+        }
+
+        nvuap->access_start = lowest_address_mapped;
+        nvuap->access_size = (highest_address_mapped + os_page_size) - lowest_address_mapped;
+    }
+
+    return status;
+}
+
+static NV_STATUS RmGetMmapPteArray(
+    KernelMemorySystem         *pKernelMemorySystem,
+    RsClient                   *pClient,
+    NvHandle                    hMemory,
+    nv_usermap_access_params_t *nvuap
+)
+{
+    NV_STATUS status = NV_OK;
+    NvU64 pages = 0;
+    Memory *pMemory = NULL;
+    PMEMORY_DESCRIPTOR pMemDesc  = NULL;
+    RsResourceRef *pResourceRef;
+    NvU64 i;
+    NvU64 *pteArray;
+    NvU64 index;
+
+    if (nvuap == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    //
+    // If we're mapping a memory handle, we can get the data from the
+    // descriptor (required if the allocation being mapped is discontiguous).
+    //
+    if (clientGetResourceRef(pClient, hMemory, &pResourceRef) == NV_OK)
+    {
+        pMemory = dynamicCast(pResourceRef->pResource, Memory);
+        if (pMemory != NULL)
+        {
+            pMemDesc = pMemory->pMemDesc;
+            nvuap->contig = memdescGetContiguity(pMemDesc, AT_CPU);
+        }
+    }
+
+    //
+    // In the discontiguous case, the page array needs to be large enough to hold
+    // the 4K-page-sized addresses that we will then deflate to OS page addresses.
+    // For the contiguous case, we can create the OS-page-sized addresses
+    // directly in the array.
+    //
+    if (nvuap->contig)
+    {
+        pages = nvuap->memArea.pRanges[0].size / os_page_size;
+    }
+    else
+    {
+        if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+        {
+            pages = nvuap->memArea.pRanges[0].size / NV_MIN(os_page_size, pMemDesc->pageArrayGranularity);
+        }
+        else
+        {
+            pages = nvuap->memArea.pRanges[0].size / NV_RM_PAGE_SIZE;
+        }
+    }
+
+    NV_ASSERT_OR_RETURN(pages != 0, NV_ERR_INVALID_ARGUMENT);
+
+    status = os_alloc_mem((void **)&nvuap->page_array, (pages * sizeof(NvU64)));
+    if (status != NV_OK)
+    {
+        return status;
+    }
+
+    if (!nvuap->contig)
+    {
+        pteArray = memdescGetPteArray(pMemDesc, AT_CPU);
+        if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+        {
+            index = nvuap->offset / NV_MIN(os_page_size, pMemDesc->pageArrayGranularity);
+        }
+        else
+        {
+            index = nvuap->offset / NV_RM_PAGE_SIZE;
+        }
+
+        //
+        // We're guaranteed to have a MEMORY_DESCRIPTOR in the discontiguous
+        // case. Copy over the addresses now.
+        //
+        NvU64 copyByteSize = NV_MIN(pages, pMemDesc->pageArraySize) * sizeof(NvU64);
+        portMemCopy((void *)nvuap->page_array,
+                    copyByteSize, (void *)&pteArray[index],
+                    copyByteSize);
+
+        if (!IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+        {
+            if (NV_RM_PAGE_SIZE < os_page_size)
+            {
+                RmDeflateRmToOsPageArray(nvuap->page_array, pages);
+                pages = NV_RM_PAGES_TO_OS_PAGES(pages);
+            }
+        }
+        else
+        {
+            //
+            // Because of PMA case, we need to convert page array 
+            // from RM native granularity to OS granularity.
+            //
+            convertPageArrayBetweenOsAndRm(
+                nvuap->page_array,
+                nvuap->memArea.pRanges[0].size / pMemDesc->pageArrayGranularity,
+                pMemDesc->pageArrayGranularity, 
+                nvuap->memArea.pRanges[0].size / os_page_size,
+                os_page_size,
+                &pages
+            );
+        }
+
+        //
+        // Convert the GPU physical addresses to system physical addresses,
+        // if applicable.
+        //
+        for (i = 0; i < pages; i++)
+        {
+            nvuap->page_array[i] += pKernelMemorySystem->coherentCpuFbBase;
+        }
+    }
+    else
+    {
+        // Offset is accounted in nvuap->memArea.pRanges[0].start.
+        for (nvuap->page_array[0] = nvuap->memArea.pRanges[0].start, i = 1;
+             i < pages; i++)
+        {
+            nvuap->page_array[i] = nvuap->page_array[i-1] + os_page_size;
+        }
+    }
+
+    nvuap->num_pages = pages;
+
+    return status;
+}
+
+/* Must be called with the API lock and the GPU locks */
+static NV_STATUS RmCreateMmapContextLocked(
+    RmClient   *pRmClient,
+    NvHandle    hDevice,
+    NvHandle    hMemory,
+    NvP64       address,
+    NvU64       size,
+    NvU64       offset,
+    NvU32       cachingType,
+    NvU32       fd
+)
+{
+    NV_STATUS status = NV_OK;
+    void *pAllocPriv = NULL;
+    OBJGPU *pGpu = NULL;
+    RsClient *pClient = staticCast(pRmClient, RsClient);
+    KernelMemorySystem *pKernelMemorySystem = NULL;
+    NvBool bCoherentAtsCpuOffset = NV_FALSE;
+    NvBool bHostCoherentFbOffset = NV_FALSE;
+    nv_state_t *pNv = NULL;
+    NvU64 addr = (NvU64)address;
+    NvU32 prot = 0;
+    NvU64 pageIndex = 0;
+    nv_usermap_access_params_t *nvuap = NULL;
+    NvBool bClientMap = (pClient->hClient == hDevice);
+
+    if (!bClientMap)
+    {
+        if (CliSetGpuContext(pClient->hClient, hDevice, &pGpu, NULL) != NV_OK)
+        {
+            Subdevice *pSubdevice;
+
+            status = subdeviceGetByHandle(pClient, hDevice, &pSubdevice);
+            if (status != NV_OK)
+                return status;
+
+            pGpu = GPU_RES_GET_GPU(pSubdevice);
+
+            GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
+        }
+    }
+    nvuap = tlsEntryGet(TLS_ENTRY_ID_MAPPING_CONTEXT);
+    if (nvuap != NULL)
+    {
+        NV_ASSERT(tlsEntryRelease(TLS_ENTRY_ID_MAPPING_CONTEXT) == 0);
+        prot = NV_PROTECT_READ_WRITE;
+        //
+        // TODO: Add this to the mapping context so we don't need to do gymnastics
+        // to figure out what the correct pNv is
+        //
+        pNv = NV_GET_NV_STATE(pGpu);
+        goto add_ctx_to_file;
+    }
+
+    NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void**)&nvuap, sizeof(nv_usermap_access_params_t)));
+    portMemSet(nvuap, 0, sizeof(nv_usermap_access_params_t));
+
+    NV_ASSERT_OK_OR_GOTO(status, os_alloc_mem((void**)&(nvuap->memArea.pRanges), sizeof(MemoryRange)), free_nvuap);
+
+    nvuap->addr = addr;
+    nvuap->size = size;
+    nvuap->offset = offset;
+    nvuap->caching = cachingType;
+
+    //
+    // Assume the allocation is contiguous until RmGetMmapPteArray
+    // determines otherwise.
+    //
+    nvuap->contig = NV_TRUE;
+    nv_align_mmap_offset_length(nvuap);
+
+    if (pGpu != NULL)
+    {
+        pNv = NV_GET_NV_STATE(pGpu);
+        pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
+        bCoherentAtsCpuOffset = IS_COHERENT_CPU_ATS_OFFSET(pGpu, pKernelMemorySystem, addr, size);
+        bHostCoherentFbOffset = !bCoherentAtsCpuOffset &&
+            IS_COHERENT_FB_OFFSET(pGpu, pKernelMemorySystem, addr, size);
+    }
+    //
+    // If no device is given, or the address isn't in the given device's BARs,
+    // validate this as a system memory mapping and associate it with the
+    // control device.
+    //
+    if ((pNv == NULL) ||
+        (!IS_REG_OFFSET(pNv, addr, size) &&
+         !IS_FB_OFFSET(pNv, addr, size) &&
+         !(bCoherentAtsCpuOffset || bHostCoherentFbOffset) &&
+         !IS_IMEM_OFFSET(pNv, addr, size)))
+    {
+        pNv = nv_get_ctl_state();
+
+        //
+        // Validate the mapping request by looking up the underlying sysmem
+        // allocation.
+        //
+        status = RmGetAllocPrivate(pRmClient, hMemory, addr, size, &prot, &pAllocPriv,
+                                   &pageIndex);
+
+        if (status != NV_OK)
+        {
+            goto done;
+        }
+    }
+    else
+    {
+        //
+        // Validate the mapping request for ATS and get pteArray
+        //
+        if (bCoherentAtsCpuOffset)
+        {
+            status = RmGetMmapPteArray(pKernelMemorySystem, pClient, hMemory, nvuap);
+            if (status != NV_OK)
+            {
+                goto done;
+            }
+        }
+        else if (bHostCoherentFbOffset)
+        {
+            status = RmGetMmapPteArray(pKernelMemorySystem, pClient, hMemory, nvuap);
+            if (status != NV_OK)
+            {
+                goto done;
+            }
+
+            //
+            // nvuap->page_array(allocated in RmGetMmapPteArray) is not assigned
+            // to nvamc->page_array if onlining status is false(which is the case with
+            // bHostCoherentFbOffset) and so doesn't get freed if not done here.
+            // The call to RmGetMmapPteArray is for getting the contig and num
+            // pages of the allocation.
+            //
+            os_free_mem(nvuap->page_array);
+            nvuap->page_array = NULL;
+        }
+
+        NV_ASSERT_OK_OR_GOTO(status, RmSetUserMapAccessRange(nvuap), done);
+
+        // Validate the mapping request for BAR's.
+        status = RmValidateMmapRequest(pNv, nvuap->access_start,
+                                       nvuap->access_size, &prot);
+        if (status != NV_OK)
+        {
+            goto done;
+        }
+    }
+
+add_ctx_to_file:
+    status = nv_add_mapping_context_to_file(pNv, nvuap, prot, pAllocPriv,
+                                            pageIndex, fd);
+
+done:
+    os_free_mem(nvuap->memArea.pRanges);
+free_nvuap:
+    os_free_mem(nvuap);
+    return status;
+}
+
+// TODO: Bug 1802250: [uvm8] Use an alt stack in all functions in unix/src/osapi.c
+NV_STATUS rm_create_mmap_context(
+    NvHandle    hClient,
+    NvHandle    hDevice,
+    NvHandle    hMemory,
+    NvP64       address,
+    NvU64       size,
+    NvU64       offset,
+    NvU32       cachingType,
+    NvU32       fd
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) == NV_OK)
+    {
+        CLIENT_ENTRY *pClientEntry;
+        RmClient *pRmClient;
+
+        if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClientEntry,
+                &pRmClient))
+        {
+            // UNLOCK: release API lock
+            rmapiLockRelease();
+            return NV_ERR_INVALID_CLIENT;
+        }
+
+        if (pRmClient->ProcID != osGetCurrentProcess())
+        {
+            rmStatus = NV_ERR_INVALID_CLIENT;
+        }
+        // LOCK: acquire GPUs lock
+        else if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) == NV_OK)
+        {
+            rmStatus = RmCreateMmapContextLocked(pRmClient, hDevice,
+                                                 hMemory, address, size, offset,
+                                                 cachingType, fd);
+            // UNLOCK: release GPUs lock
+            rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+        }
+
+        serverutilReleaseClient(LOCK_ACCESS_READ, pClientEntry);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    return rmStatus;
+}
+
+static NV_STATUS RmGetAllocPrivate(
+    RmClient   *pRmClient,
+    NvU32       hMemory,
+    NvU64       offset,
+    NvU64       length,
+    NvU32      *pProtection,
+    void      **ppPrivate,
+    NvU64      *pPageIndex
+)
+{
+    NV_STATUS rmStatus;
+    PMEMORY_DESCRIPTOR pMemDesc;
+    NvU32 pageOffset;
+    NvU64 pageCount;
+    NvU64 endingOffset;
+    RsResourceRef *pResourceRef;
+    RmResource *pRmResource;
+    void *pMemData;
+    NvBool bPeerIoMem;
+    NvBool bReadOnlyMem;
+    *pProtection = NV_PROTECT_READ_WRITE;
+    *ppPrivate = NULL;
+
+    pageOffset = (offset & ~os_page_mask);
+    offset &= os_page_mask;
+
+    NV_ASSERT_OR_RETURN(rmapiLockIsOwner(), NV_ERR_INVALID_LOCK_STATE);
+
+    rmStatus = clientGetResourceRef(staticCast(pRmClient, RsClient),
+        hMemory, &pResourceRef);
+    if (rmStatus != NV_OK)
+        goto done;
+
+    pRmResource = dynamicCast(pResourceRef->pResource, RmResource);
+    if (!pRmResource)
+    {
+        rmStatus = NV_ERR_INVALID_OBJECT;
+        goto done;
+    }
+
+    rmStatus = rmresGetMemoryMappingDescriptor(pRmResource, &pMemDesc);
+    if (rmStatus != NV_OK)
+        goto done;
+
+    bReadOnlyMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY);
+    bPeerIoMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM);
+
+    if (!(pMemDesc->Allocated || bPeerIoMem ||
+          memdescIsCarveoutMemory(pMemDesc) ||
+          (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOW_EXT_SYSMEM_USER_CPU_MAPPING))))
+    {
+        NV_PRINTF(LEVEL_ERROR, "Mmap is not allowed\n");
+        rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+        goto done;
+    }
+
+    switch (memdescGetAddressSpace(pMemDesc))
+    {
+        case ADDR_SYSMEM:
+            break;
+        default:
+            rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+            goto done;
+    }
+
+    pMemData = memdescGetMemData(pMemDesc);
+    if (pMemData == NULL)
+    {
+        rmStatus = NV_ERR_OBJECT_NOT_FOUND;
+        goto done;
+    }
+
+    rmStatus = os_match_mmap_offset(pMemData, offset, pPageIndex);
+    if (rmStatus != NV_OK)
+        goto done;
+
+    if (!portSafeAddU64(pageOffset, length, &endingOffset))
+    {
+        rmStatus = NV_ERR_INVALID_ARGUMENT;
+        goto done;
+    }
+
+    pageCount = (endingOffset / os_page_size);
+
+    if (!portSafeAddU64(*pPageIndex + ((endingOffset % os_page_size) ? 1 : 0),
+                        pageCount, &pageCount))
+    {
+        rmStatus = NV_ERR_INVALID_ARGUMENT;
+        goto done;
+    }
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        NV_ASSERT_OR_ELSE(pageCount <= pMemDesc->PageCount, rmStatus = NV_ERR_INVALID_ARGUMENT; goto done);
+    }
+    else if (pageCount > NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount))
+    {
+        rmStatus = NV_ERR_INVALID_ARGUMENT;
+        goto done;
+    }
+
+    if (bReadOnlyMem)
+       *pProtection = NV_PROTECT_READABLE;
+
+    *ppPrivate = pMemData;
+
+done:
+    return rmStatus;
+}
+
+NV_STATUS rm_get_adapter_status(
+    nv_state_t *pNv,
+    NvU32      *pStatus
+)
+{
+    NV_STATUS rmStatus = NV_ERR_OPERATING_SYSTEM;
+
+    // LOCK: acquire API lock
+    if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI) == NV_OK)
+    {
+        rmStatus = RmGetAdapterStatus(pNv, pStatus);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_get_adapter_status_external(
+    nvidia_stack_t *sp,
+    nv_state_t     *pNv
+)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    if (rm_get_adapter_status(pNv, &rmStatus) != NV_OK)
+    {
+        rmStatus = NV_ERR_OPERATING_SYSTEM;
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NvBool NV_API_CALL rm_init_rm(
+    nvidia_stack_t *sp
+)
+{
+    NvBool retval;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    retval = RmInitRm();
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return retval;
+}
+
+void NV_API_CALL rm_shutdown_rm(
+    nvidia_stack_t *sp
+)
+{
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    RmShutdownRm();
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NvBool NV_API_CALL rm_init_event_locks(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    void *fp;
+    NvBool ret;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pNv->event_spinlock = portSyncSpinlockCreate(portMemAllocatorGetGlobalNonPaged());
+    ret = (pNv->event_spinlock != NULL);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return ret;
+}
+
+void NV_API_CALL rm_destroy_event_locks(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    if (pNv && pNv->event_spinlock)
+        portSyncSpinlockDestroy(pNv->event_spinlock);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_stop_user_channels(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS params = { 0 };
+    RM_API            *pRmApi;
+    THREAD_STATE_NODE  threadState;
+    NV_STATUS          rmStatus = NV_ERR_INVALID_STATE;
+    void              *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_FIFO);
+    if (pRmApi != NULL)
+    {
+        params.bDisable = NV_TRUE;
+        rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+                                   pNv->rmapi.hSubDevice,
+                                   NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS,
+                                   &params, sizeof(params));
+
+        RmUnixRmApiEpilogue(pNv, &threadState);
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_restart_user_channels(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS params = { 0 };
+    RM_API            *pRmApi;
+    THREAD_STATE_NODE  threadState;
+    NV_STATUS          rmStatus = NV_ERR_INVALID_STATE;
+    void              *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pRmApi = RmUnixRmApiPrologue(pNv, &threadState, RM_LOCK_MODULES_FIFO);
+    if (pRmApi != NULL)
+    {
+        params.bDisable = NV_FALSE;
+        rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hSubDevice,
+                                   NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS,
+                                   &params, sizeof(params));
+
+        RmUnixRmApiEpilogue(pNv, &threadState);
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+//
+// Use this call to check if the chipset is io coherent
+//
+NvBool NV_API_CALL rm_is_chipset_io_coherent(
+    nvidia_stack_t  *sp)
+{
+    void *fp;
+    OBJSYS *pSys;
+    OBJCL *pCl;
+    NvBool bIoCoherent = NV_FALSE;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pSys = SYS_GET_INSTANCE();
+    pCl = SYS_GET_CL(pSys);
+    if (pCl == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s: no CL object found, setting io coherent by default\n",
+                  __FUNCTION__);
+        goto done;
+    }
+
+    bIoCoherent = pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT);
+
+done:
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return bIoCoherent;
+}
+
+NV_STATUS NV_API_CALL rm_validate_ioctls(
+    NvU32               command,
+    NvU32               dataSize
+)
+{
+    size_t arg_size = dataSize;
+    int arg_cmd = (command & 0xFF);
+    int i;
+    NV_STATUS status;
+    static const struct {
+        unsigned int cmdKey;
+        size_t paramSize;
+    } rm_ioctls_table[] = {
+    #define _RM_IOCTL_ENTRY(_cmd, _type) \
+        { .cmdKey = ((_cmd) & 0xFF), .paramSize = sizeof(_type),}
+        /*rm_ioctls */
+        _RM_IOCTL_ENTRY(NV_ESC_ALLOC_OS_EVENT, nv_ioctl_alloc_os_event_t),
+        _RM_IOCTL_ENTRY(NV_ESC_FREE_OS_EVENT, nv_ioctl_free_os_event_t),
+        _RM_IOCTL_ENTRY(NV_ESC_RM_GET_EVENT_DATA, NVOS41_PARAMETERS),
+    };
+
+    status = RmValidateIoctl(command, dataSize);
+    if (status != NV_ERR_INVALID_COMMAND)
+    {
+        return status; 
+    }
+
+    for (i = 0; i < NV_ARRAY_ELEMENTS(rm_ioctls_table); ++i)
+    {
+        if (arg_cmd == rm_ioctls_table[i].cmdKey)
+        {
+            if (arg_size == rm_ioctls_table[i].paramSize)
+            {
+                return NV_OK;
+            }
+            nv_printf(NV_DBG_ERRORS, "NVRM: invalid %d structure size, expected %d, got %d!\n",
+                      rm_ioctls_table[i].cmdKey, rm_ioctls_table[i].paramSize, arg_size);
+
+            return NV_ERR_INVALID_ARGUMENT;
+        }
+    }
+    // Unknown IOCTL case. Not found in any rm and RM Tables.
+    return NV_ERR_INVALID_COMMAND;
+}
+
+NV_STATUS NV_API_CALL rm_ioctl(
+    nvidia_stack_t     *sp,
+    nv_state_t         *pNv,
+    nv_file_private_t  *nvfp,
+    NvU32               Command,
+    void               *pData,
+    NvU32               dataSize
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+    THREAD_STATE_NODE threadState;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    //
+    // Some ioctls are handled entirely inside the OS layer and don't need to
+    // suffer the overhead of calling into RM core.
+    //
+    switch (Command)
+    {
+        case NV_ESC_ALLOC_OS_EVENT:
+        {
+            nv_ioctl_alloc_os_event_t *pApi = pData;
+
+            if (dataSize != sizeof(nv_ioctl_alloc_os_event_t))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                break;
+            }
+
+            pApi->Status = allocate_os_event(pApi->hClient, nvfp, pApi->fd);
+            break;
+        }
+        case NV_ESC_FREE_OS_EVENT:
+        {
+            nv_ioctl_free_os_event_t *pApi = pData;
+
+            if (dataSize != sizeof(nv_ioctl_free_os_event_t))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                break;
+            }
+
+            pApi->Status = free_os_event(pApi->hClient, pApi->fd);
+            break;
+        }
+        case NV_ESC_RM_GET_EVENT_DATA:
+        {
+            NVOS41_PARAMETERS *pApi = pData;
+
+            if (dataSize != sizeof(NVOS41_PARAMETERS))
+            {
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                break;
+            }
+
+            pApi->status = get_os_event_data(nvfp,
+                                             pApi->pEvent,
+                                             &pApi->MoreEvents);
+            break;
+        }
+        default:
+        {
+            threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+            rmStatus = RmIoctl(pNv, nvfp, Command, pData, dataSize);
+            threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+            break;
+        }
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+static void _deferredClientListFreeCallback(void *unused)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    NV_STATUS status = serverFreeDisabledClients(&g_resServ, 0, pSys->clientListDeferredFreeLimit);
+    //
+    // Possible return values:
+    //   NV_WARN_MORE_PROCESSING_REQUIRED - Iteration limit reached, need to call again
+    //   NV_ERR_IN_USE - Already running on another thread, try again later
+    // In both cases, schedule a worker to clean up anything that remains
+    //
+    if (status != NV_OK)
+    {
+        status = osQueueSystemWorkItem(_deferredClientListFreeCallback, unused);
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_NOTICE, "Failed to schedule deferred free callback. Freeing immediately.\n");
+            serverFreeDisabledClients(&g_resServ, 0, 0);
+        }
+    }
+}
+
+void NV_API_CALL rm_cleanup_file_private(
+    nvidia_stack_t     *sp,
+    nv_state_t         *pNv,
+    nv_file_private_t  *nvfp
+)
+{
+    THREAD_STATE_NODE threadState;
+    void      *fp;
+    RM_API *pRmApi;
+    RM_API_CONTEXT rmApiContext = {0};
+    NvU32 i;
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    //
+    // Skip cleaning up this fd if:
+    // - no RMAPI clients and events were ever allocated on this fd
+    // - no RMAPI object handles were exported on this fd
+    // Access nvfp->handles without locking as fd cleanup is synchronised by the kernel
+    //
+    if (!nvfp->bCleanupRmapi && nvfp->handles == NULL)
+        goto done;
+
+    pRmApi = rmapiGetInterface(RMAPI_EXTERNAL);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    threadStateSetTimeoutOverride(&threadState, 10 * 1000);
+
+    if (rmapiPrologue(pRmApi, &rmApiContext) != NV_OK) {
+        NV_EXIT_RM_RUNTIME(sp,fp);
+        return;
+    }
+
+    // LOCK: acquire API lock. Low priority so cleanup doesn't block active threads
+    if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_LOW_PRIORITY, RM_LOCK_MODULES_OSAPI) == NV_OK)
+    {
+        // Unref any object which was exported on this file.
+        if (nvfp->handles != NULL)
+        {
+            for (i = 0; i < nvfp->maxHandles; i++)
+            {
+                if (nvfp->handles[i] == 0)
+                {
+                    continue;
+                }
+
+                RmFreeObjExportHandle(nvfp->handles[i]);
+                nvfp->handles[i] = 0;
+            }
+
+            os_free_mem(nvfp->handles);
+            nvfp->handles = NULL;
+            nvfp->maxHandles = 0;
+        }
+
+        // Disable any RM clients associated with this file.
+        RmFreeUnusedClients(pNv, nvfp);
+
+        // Unless configured otherwise, immediately free all disabled clients
+        if (!pSys->bUseDeferredClientListFree)
+            serverFreeDisabledClients(&g_resServ, RM_LOCK_STATES_API_LOCK_ACQUIRED, 0);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    // Start the deferred free callback if necessary
+    if (pSys->bUseDeferredClientListFree)
+        _deferredClientListFreeCallback(NULL);
+
+    rmapiEpilogue(pRmApi, &rmApiContext);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+done:
+    if (nvfp->ctl_nvfp != NULL)
+    {
+        nv_put_file_private(nvfp->ctl_nvfp_priv);
+        nvfp->ctl_nvfp = NULL;
+        nvfp->ctl_nvfp_priv = NULL;
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_unbind_lock(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    THREAD_STATE_NODE threadState;
+    void      *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK)
+    {
+        RmUnbindLock(pNv);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_read_registry_dword(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU32      *Data
+)
+{
+    OBJGPU    *pGpu = NULL;
+    NV_STATUS  RmStatus;
+    void      *fp;
+    NvBool     isApiLockTaken = NV_FALSE;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    //
+    // We can be called from different contexts:
+    //
+    // 1) early initialization without device state.
+    // 2) from outside the RM API (without the lock held)
+    //
+    // In context 1)the API lock is not needed and
+    // in context 2), it needs to be acquired.
+    //
+    if (nv != NULL)
+    {
+        // LOCK: acquire API lock
+        if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) != NV_OK)
+        {
+            NV_EXIT_RM_RUNTIME(sp,fp);
+            return RmStatus;
+        }
+
+        isApiLockTaken = NV_TRUE;
+    }
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    // Skipping the NULL check as osReadRegistryDword takes care of it.
+    RmStatus = osReadRegistryDword(pGpu, regParmStr, Data);
+
+    if (isApiLockTaken == NV_TRUE)
+    {
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return RmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_write_registry_dword(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU32       Data
+)
+{
+    NV_STATUS  RmStatus;
+    void      *fp;
+    NvBool     isApiLockTaken = NV_FALSE;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    if (nv != NULL)
+    {
+        // LOCK: acquire API lock
+        if ((RmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK)
+        {
+            NV_EXIT_RM_RUNTIME(sp,fp);
+            return RmStatus;
+        }
+
+        isApiLockTaken = NV_TRUE;
+    }
+
+    RmStatus = RmWriteRegistryDword(nv, regParmStr, Data);
+
+    if (isApiLockTaken == NV_TRUE)
+    {
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return RmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_write_registry_binary(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU8       *Data,
+    NvU32       cbLen
+)
+{
+    NV_STATUS  RmStatus;
+    void      *fp;
+    NvBool     isApiLockTaken = NV_FALSE;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    if (nv != NULL)
+    {
+        // LOCK: acquire API lock
+        if ((RmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK)
+        {
+            NV_EXIT_RM_RUNTIME(sp,fp);
+            return RmStatus;
+        }
+
+        isApiLockTaken = NV_TRUE;
+    }
+
+    RmStatus = RmWriteRegistryBinary(nv, regParmStr, Data, cbLen);
+
+    if (isApiLockTaken == NV_TRUE)
+    {
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return RmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_write_registry_string(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    const char *regParmStr,
+    const char *string,
+    NvU32       stringLength
+)
+{
+    NV_STATUS  rmStatus;
+    void      *fp;
+    NvBool     isApiLockTaken = NV_FALSE;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    if (nv != NULL)
+    {
+        // LOCK: acquire API lock
+        if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) != NV_OK)
+        {
+            NV_EXIT_RM_RUNTIME(sp,fp);
+            return rmStatus;
+        }
+
+        isApiLockTaken = NV_TRUE;
+    }
+
+    rmStatus = RmWriteRegistryString(nv, regParmStr, string, (stringLength + 1));
+
+    if (isApiLockTaken == NV_TRUE)
+    {
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+static NvBool NV_API_CALL rm_is_space(const char ch)
+{
+    //
+    // return true if it is a:
+    // ' '  : (space - decimal 32.)
+    // '\t' : (TAB - decimal 9)
+    // 'LF' : (Line feed, new line - decimal 10)
+    // 'VT' : (Vertical TAB - decimal 11)
+    // 'FF' : (Form feed, new page - decimal 12)
+    // '\r' : (carriage return - decimal 13)
+    //
+    return ((ch == ' ') || ((ch >= '\t') && (ch <= '\r')));
+}
+
+char* NV_API_CALL rm_remove_spaces(const char *in)
+{
+    unsigned int len = os_string_length(in) + 1;
+    const char *in_ptr;
+    char *out, *out_ptr;
+
+    if (os_alloc_mem((void **)&out, len) != NV_OK)
+        return NULL;
+
+    in_ptr = in;
+    out_ptr = out;
+
+    while (*in_ptr != '\0')
+    {
+        if (!rm_is_space(*in_ptr))
+            *out_ptr++ = *in_ptr;
+        in_ptr++;
+    }
+    *out_ptr = '\0';
+
+    return out;
+}
+
+char* NV_API_CALL rm_string_token(char **strp, const char delim)
+{
+    char *s, *token;
+
+    if ((strp == NULL) || (*strp == NULL))
+        return NULL;
+
+    s = token = *strp;
+    *strp = NULL;
+
+    for (; *s != '\0'; s++) {
+        if (*s == delim) {
+            *s = '\0';
+            *strp = ++s;
+            break;
+        }
+    }
+
+    return token;
+}
+
+// Parse string passed in NVRM as module parameter.
+void NV_API_CALL rm_parse_option_string(nvidia_stack_t *sp, const char *nvRegistryDwords)
+{
+    unsigned int i;
+    nv_parm_t *entry;
+    char *option_string = NULL;
+    char *ptr, *token;
+    char *name, *value;
+    NvU32 data;
+
+    if (nvRegistryDwords != NULL)
+    {
+        if ((option_string = rm_remove_spaces(nvRegistryDwords)) == NULL)
+        {
+            return;
+        }
+
+        ptr = option_string;
+
+        while ((token = rm_string_token(&ptr, ';')) != NULL)
+        {
+            if (!(name = rm_string_token(&token, '=')) || !os_string_length(name))
+            {
+                continue;
+            }
+
+            if (!(value = rm_string_token(&token, '=')) || !os_string_length(value))
+            {
+                continue;
+            }
+
+            if (rm_string_token(&token, '=') != NULL)
+            {
+                continue;
+            }
+
+            data = os_strtoul(value, NULL, 0);
+
+            for (i = 0; (entry = &nv_parms[i])->name != NULL; i++)
+            {
+                if (os_string_compare(entry->name, name) == 0)
+                    break;
+            }
+
+            if (!entry->name)
+                rm_write_registry_dword(sp, NULL, name, data);
+            else
+                *entry->data = data;
+        }
+
+        // Free the memory allocated by rm_remove_spaces()
+        os_free_mem(option_string);
+    }
+}
+
+NV_STATUS NV_API_CALL rm_run_rc_callback(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    OBJGPU *pGpu;
+    void   *fp;
+
+    /* make sure our timer's not still running when it shouldn't be */
+    if (nv == NULL)
+        return NV_ERR_GENERIC;
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    if (pGpu == NULL)
+        return NV_ERR_GENERIC;
+
+    if (nv->rc_timer_enabled == 0)
+        return NV_ERR_GENERIC;
+
+    if (!FULL_GPU_SANITY_CHECK(pGpu))
+    {
+        return NV_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    osRun1HzCallbacksNow(pGpu);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return NV_OK;
+}
+
+static void _tmrEventServiceTimerWorkItem
+(
+    NvU32 gpuInstance,
+    void *pArgs
+)
+{
+    OBJGPU *pGpu = gpumgrGetGpu(gpuInstance);
+    OBJTMR *pTmr = GPU_GET_TIMER(pGpu);
+    NV_STATUS status = NV_OK;
+
+    status = tmrEventServiceTimer(pGpu, pTmr, (TMR_EVENT *)pArgs);
+
+    if (status != NV_OK)
+        NV_PRINTF(LEVEL_ERROR, "Timer event failed from OS timer callback workitem with status :0x%x\n", status);
+}
+
+static NV_STATUS RmRunNanoTimerCallback(
+    OBJGPU *pGpu,
+    void *pTmrEvent
+)
+{
+    THREAD_STATE_NODE   threadState;
+    NV_STATUS         status = NV_OK;
+
+    threadStateInitISRAndDeferredIntHandler(&threadState, pGpu,
+        THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER);
+
+    //
+    // OS timers come in ISR context, hence schedule workitem for timer event service.
+    // GPU timer events are also handled in same mammer allowing us to have same functionality
+    // for callback functions.
+    //
+    status = osQueueWorkItem(pGpu,
+                             _tmrEventServiceTimerWorkItem,
+                             pTmrEvent,
+                             (OsQueueWorkItemFlags){
+                               .bLockGpuGroupDevice = NV_TRUE,
+                               .bDontFreeParams = NV_TRUE});
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,"Queuing workitem for timer event failed with status :0x%x\n", status);
+    }
+
+    // Out of conflicting thread
+    threadStateFreeISRAndDeferredIntHandler(&threadState,
+        pGpu, THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER);
+
+    return status;
+}
+
+NV_STATUS NV_API_CALL rm_run_nano_timer_callback
+(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    void *pTmrEvent
+)
+{
+    NV_STATUS status;
+    OBJGPU *pGpu = NULL;
+    void   *fp;
+
+    if (nv == NULL)
+        return NV_ERR_GENERIC;
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    if (pGpu == NULL)
+        return NV_ERR_GENERIC;
+
+    if (!FULL_GPU_SANITY_FOR_PM_RESUME(pGpu))
+    {
+        return NV_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    status = RmRunNanoTimerCallback(pGpu, pTmrEvent);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return status;
+}
+
+void NV_API_CALL rm_execute_work_item(
+    nvidia_stack_t *sp,
+    void *pNvWorkItem
+)
+{
+    void *fp;
+    THREAD_STATE_NODE threadState;
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    RmExecuteWorkItem(pNvWorkItem);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+const char* NV_API_CALL rm_get_device_name(
+     NvU16       device,
+     NvU16       subsystem_vendor,
+     NvU16       subsystem_device
+)
+{
+    unsigned int i;
+    const char *tmpName = NULL;
+
+    for (i = 0; i < NV_ARRAY_ELEMENTS(sChipsReleased); i++)
+    {
+        // if the device ID doesn't match, go to the next entry
+        if (device != sChipsReleased[i].devID)
+        {
+            continue;
+        }
+
+        // if the entry has 0 for the subsystem IDs, then the device
+        // ID match is sufficient, but continue scanning through
+        // sChipsReleased[] in case there is a subsystem ID match later
+        // in the table
+        if (sChipsReleased[i].subSystemVendorID == 0 &&
+            sChipsReleased[i].subSystemID == 0)
+        {
+            tmpName = sChipsReleased[i].name;
+            continue;
+        }
+
+        if (subsystem_vendor == sChipsReleased[i].subSystemVendorID &&
+            subsystem_device == sChipsReleased[i].subSystemID)
+        {
+            tmpName = sChipsReleased[i].name;
+            break;
+        }
+    }
+
+    return (tmpName != NULL) ? tmpName : "Unknown";
+}
+
+NV_STATUS rm_access_registry(
+    NvHandle   hClient,
+    NvHandle   hObject,
+    NvU32      AccessType,
+    NvP64      clientDevNodeAddress,
+    NvU32      DevNodeLength,
+    NvP64      clientParmStrAddress,
+    NvU32      ParmStrLength,
+    NvP64      clientBinaryDataAddress,
+    NvU32      *pBinaryDataLength,
+    NvU32      *Data,
+    NvU32      *Entry
+)
+{
+    NV_STATUS RmStatus;
+    NvBool bReadOnly = (AccessType == NVOS38_ACCESS_TYPE_READ_DWORD) ||
+                       (AccessType == NVOS38_ACCESS_TYPE_READ_BINARY);
+
+    // LOCK: acquire API lock
+    if ((RmStatus = rmapiLockAcquire(bReadOnly ? RMAPI_LOCK_FLAGS_READ : RMAPI_LOCK_FLAGS_NONE,
+                                     RM_LOCK_MODULES_OSAPI)) == NV_OK)
+    {
+        RmStatus = RmAccessRegistry(hClient,
+                                    hObject,
+                                    AccessType,
+                                    clientDevNodeAddress,
+                                    DevNodeLength,
+                                    clientParmStrAddress,
+                                    ParmStrLength,
+                                    clientBinaryDataAddress,
+                                    pBinaryDataLength,
+                                    Data,
+                                    Entry);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    return RmStatus;
+}
+
+NV_STATUS rm_update_device_mapping_info(
+    NvHandle    hClient,
+    NvHandle    hDevice,
+    NvHandle    hMemory,
+    void       *pOldCpuAddress,
+    void       *pNewCpuAddress
+)
+{
+    NV_STATUS RmStatus;
+
+    // LOCK: acquire API lock
+    if ((RmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU)) == NV_OK)
+    {
+        RmStatus = RmUpdateDeviceMappingInfo(hClient,
+                                             hDevice,
+                                             hMemory,
+                                             pOldCpuAddress,
+                                             pNewCpuAddress);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    return RmStatus;
+}
+
+static NvBool NV_API_CALL rm_is_legacy_device(
+    NvU16       device_id,
+    NvU16       subsystem_vendor,
+    NvU16       subsystem_device,
+    NvBool      print_warning
+)
+{
+    return NV_FALSE;
+}
+
+static NvBool NV_API_CALL rm_is_legacy_arch(
+    NvU32 pmc_boot_0,
+    NvU32 pmc_boot_42
+)
+{
+    NvBool      legacy = NV_FALSE;
+
+    return legacy;
+}
+
+NV_STATUS NV_API_CALL rm_is_supported_device(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS   rmStatus;
+    OBJSYS     *pSys;
+    OBJHALMGR  *pHalMgr;
+    GPUHWREG   *reg_mapping;
+    NvU32       myHalPublicID;
+    void       *fp;
+    NvU32       pmc_boot_0;
+    NvU32       pmc_boot_42;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    pSys = SYS_GET_INSTANCE();
+    pHalMgr = SYS_GET_HALMGR(pSys);
+
+    reg_mapping = osMapKernelSpace(pNv->regs->cpu_address,
+                                   os_page_size,
+                                   NV_MEMORY_UNCACHED,
+                                   NV_PROTECT_READABLE);
+
+    if (reg_mapping == NULL)
+    {
+        nv_printf(NV_DBG_ERRORS, "NVRM: failed to map registers!\n");
+        rmStatus = NV_ERR_OPERATING_SYSTEM;
+        goto threadfree;
+    }
+    NvU32 pmc_boot_1 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_1);
+    pmc_boot_0 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_0);
+    pmc_boot_42 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_42);
+
+    osUnmapKernelSpace(reg_mapping, os_page_size);
+
+    if ((pmc_boot_0 == 0xFFFFFFFF) && (pmc_boot_42 == 0xFFFFFFFF))
+    {
+        nv_printf(NV_DBG_ERRORS,
+            "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x\n"
+            "NVRM: (PCI ID: %04x:%04x) installed in this system has\n"
+            "NVRM: fallen off the bus and is not responding to commands.\n",
+            pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
+            pNv->pci_info.function, pNv->pci_info.vendor_id,
+            pNv->pci_info.device_id);
+        rmStatus = NV_ERR_GPU_IS_LOST;
+        goto threadfree;
+    }
+
+    /*
+     * For legacy architectures, rm_is_legacy_arch() prints "legacy" message.
+     * We do not want to print "unsupported" message for legacy architectures
+     * to avoid confusion. Also, the probe should not continue for legacy
+     * architectures. Hence, we set rmStatus to NV_ERR_NOT_SUPPORTED and
+     * goto threadfree.
+     */
+    if (rm_is_legacy_arch(pmc_boot_0, pmc_boot_42))
+    {
+        rmStatus = NV_ERR_NOT_SUPPORTED;
+        goto threadfree;
+    }
+
+    rmStatus = halmgrGetHalForGpu(pHalMgr, pmc_boot_0, pmc_boot_42, &myHalPublicID);
+
+    if (rmStatus != NV_OK)
+    {
+        NvBool bIsFirmwareCapable;
+
+        bIsFirmwareCapable = gpumgrIsDeviceRmFirmwareCapable(pmc_boot_42,
+                                                             FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1),
+                                                             NV_IS_SOC_DISPLAY_DEVICE(pNv),
+                                                             NULL,
+                                                             NV_FALSE /* bIsTccOrMcdm */);
+        if (!bIsFirmwareCapable)
+        {
+            if (hypervisorIsVgxHyper())
+            {
+                nv_printf(NV_DBG_ERRORS,
+                        "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
+                        "NVRM: installed in this vGPU host system is not supported by\n"
+                        "NVRM: open nvidia.ko.\n"
+                        "NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n"
+                        "NVRM: Firmware' sections in the NVIDIA Virtual GPU (vGPU)\n"
+                        "NVRM: Software documentation, available at docs.nvidia.com.\n",
+                        pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
+                        pNv->pci_info.function, pNv->pci_info.vendor_id,
+                        pNv->pci_info.device_id);
+            }
+            else
+            {
+                nv_printf(NV_DBG_ERRORS,
+                        "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
+                        "NVRM: installed in this system is not supported by open\n"
+                        "NVRM: nvidia.ko because it does not include the required GPU\n"
+                        "NVRM: System Processor (GSP).\n"
+                        "NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n"
+                        "NVRM: Firmware' sections in the driver README, available on\n"
+                        "NVRM: the Linux graphics driver download page at\n"
+                        "NVRM: www.nvidia.com.\n",
+                        pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
+                        pNv->pci_info.function, pNv->pci_info.vendor_id,
+                        pNv->pci_info.device_id);
+            }
+            goto threadfree;
+        }
+        goto print_unsupported;
+    }
+
+    rmStatus = rm_is_vgpu_supported_device(pNv, pmc_boot_1, pmc_boot_42);
+
+    if (rmStatus != NV_OK)
+    {
+        nv_printf(NV_DBG_ERRORS,
+                "NVRM: The NVIDIA vGPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
+                "NVRM: installed in this system is not supported by open\n"
+                "NVRM: nvidia.ko.\n"
+                "NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n"
+                "NVRM: Firmware' sections in the NVIDIA Virtual GPU (vGPU)\n"
+                "NVRM: Software documentation, available at docs.nvidia.com.\n",
+                pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
+                pNv->pci_info.function, pNv->pci_info.vendor_id,
+                pNv->pci_info.device_id);
+        goto threadfree;
+    }
+    goto threadfree;
+
+print_unsupported:
+    nv_printf(NV_DBG_ERRORS,
+       "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
+       "NVRM: installed in this system is not supported by the\n"
+       "NVRM: NVIDIA %s driver release.\n"
+       "NVRM: Please see 'Appendix A - Supported NVIDIA GPU Products'\n"
+       "NVRM: in this release's README, available on the operating system\n"
+       "NVRM: specific graphics driver download page at www.nvidia.com.\n",
+       pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
+       pNv->pci_info.function, pNv->pci_info.vendor_id,
+       pNv->pci_info.device_id, NV_VERSION_STRING);
+
+threadfree:
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NvBool NV_API_CALL rm_is_supported_pci_device(
+    NvU8   pci_class,
+    NvU8   pci_subclass,
+    NvU16  vendor,
+    NvU16  device,
+    NvU16  subsystem_vendor,
+    NvU16  subsystem_device,
+    NvBool print_legacy_warning
+)
+{
+    const NvU16 nv_pci_vendor_id            = 0x10DE;
+    const NvU16 nv_pci_id_riva_tnt          = 0x0020;
+    const NvU8  nv_pci_class_display        = 0x03;
+    const NvU8  nv_pci_subclass_display_vga = 0x00;
+    const NvU8  nv_pci_subclass_display_3d  = 0x02;
+
+    if (pci_class != nv_pci_class_display)
+    {
+        return NV_FALSE;
+    }
+
+    if ((pci_subclass != nv_pci_subclass_display_vga) &&
+        (pci_subclass != nv_pci_subclass_display_3d))
+    {
+        return NV_FALSE;
+    }
+
+    if (vendor != nv_pci_vendor_id)
+    {
+        return NV_FALSE;
+    }
+
+    if (device < nv_pci_id_riva_tnt)
+    {
+        return NV_FALSE;
+    }
+
+    if (rm_is_legacy_device(
+            device,
+            subsystem_vendor,
+            subsystem_device,
+            print_legacy_warning))
+    {
+        return NV_FALSE;
+    }
+
+    return NV_TRUE;
+}
+
+/*
+ * Performs the I2C transfers which are related with DP AUX channel
+ */
+static NV_STATUS RmDpAuxI2CTransfer
+(
+    nv_state_t  *pNv,
+    NvU32       displayId,
+    NvU8        addr,
+    NvU32       len,
+    NvU8       *pData,
+    NvBool      bWrite
+)
+{
+    NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_CTRL_PARAMS *pParams;
+    RM_API    *pRmApi   = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    NV_STATUS  status;
+
+    if (len > NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s: requested I2C transfer length %u is greater than maximum supported length %u\n",
+                  __FUNCTION__, len, NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE);
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    pParams = portMemAllocNonPaged(sizeof(*pParams));
+    if (pParams == NULL)
+    {
+        return NV_ERR_NO_MEMORY;
+    }
+
+    portMemSet(pParams, 0, sizeof(*pParams));
+
+    pParams->subDeviceInstance = 0;
+    pParams->displayId         = displayId;
+    pParams->addr              = addr;
+    pParams->size              = len;
+    pParams->bWrite            = bWrite;
+
+    if (bWrite)
+    {
+        portMemCopy(pParams->data, NV0073_CTRL_DP_AUXCH_I2C_TRANSFER_MAX_DATA_SIZE,
+                    pData, len);
+    }
+
+    status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp,
+                             NV0073_CTRL_CMD_DP_AUXCH_I2C_TRANSFER_CTRL,
+                             pParams, sizeof(*pParams));
+
+    if ((status == NV_OK) && !bWrite)
+    {
+        portMemCopy(pData, len, pParams->data, pParams->size);
+    }
+
+    portMemFree(pParams);
+
+    return status;
+}
+
+/*
+ * Performs the I2C transfers which are not related with DP AUX channel
+ */
+static NV_STATUS RmNonDPAuxI2CTransfer
+(
+    nv_state_t *pNv,
+    NvU8        portId,
+    nv_i2c_cmd_t type,
+    NvU8        addr,
+    NvU8        command,
+    NvU32       len,
+    NvU8       *pData
+)
+{
+    NV402C_CTRL_I2C_TRANSACTION_PARAMS *params;
+    RM_API    *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    NV_STATUS  rmStatus = NV_OK;
+
+    params = portMemAllocNonPaged(sizeof(*params));
+    if (params == NULL)
+    {
+        return NV_ERR_NO_MEMORY;
+    }
+
+    portMemSet(params, 0, sizeof(*params));
+
+    params->portId = portId;
+    // precondition our address (our stack requires this)
+    params->deviceAddress = addr << 1;
+
+    switch (type)
+    {
+        case NV_I2C_CMD_WRITE:
+            params->transData.i2cBlockData.bWrite = NV_TRUE;
+            /* fall through*/
+
+        case NV_I2C_CMD_READ:
+            params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_I2C_BLOCK_RW;
+            params->transData.i2cBlockData.messageLength = len;
+            params->transData.i2cBlockData.pMessage = pData;
+            break;
+
+        case NV_I2C_CMD_SMBUS_WRITE:
+            if (len == 2)
+            {
+                params->transData.smbusWordData.bWrite = NV_TRUE;
+            }
+            else
+            {
+                params->transData.smbusByteData.bWrite = NV_TRUE;
+            }
+            /* fall through*/
+
+        case NV_I2C_CMD_SMBUS_READ:
+            if (len == 2)
+            {
+                params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_WORD_RW;
+                params->transData.smbusWordData.message = pData[0] | ((NvU16)pData[1] << 8);
+                params->transData.smbusWordData.registerAddress = command;
+            }
+            else
+            {
+                params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BYTE_RW;
+                params->transData.smbusByteData.message = pData[0];
+                params->transData.smbusByteData.registerAddress = command;
+            }
+            break;
+
+        case NV_I2C_CMD_SMBUS_BLOCK_WRITE:
+            if (pData[0] >= len) {
+                portMemFree(params);
+                return NV_ERR_INVALID_ARGUMENT;
+            }
+            params->transData.smbusBlockData.bWrite = NV_TRUE;
+            params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BLOCK_RW;
+            params->transData.smbusBlockData.registerAddress = command;
+            params->transData.smbusBlockData.messageLength = pData[0];
+            params->transData.smbusBlockData.pMessage = pData + 1;
+            break;
+
+        case NV_I2C_CMD_SMBUS_BLOCK_READ:
+            params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_BLOCK_RW;
+            params->transData.smbusBlockData.registerAddress = command;
+            params->transData.smbusBlockData.messageLength = len;
+            params->transData.smbusBlockData.pMessage = pData;
+            break;
+
+        case NV_I2C_CMD_SMBUS_QUICK_WRITE:
+            params->transData.smbusQuickData.bWrite = NV_TRUE;
+            /* fall through*/
+
+        case NV_I2C_CMD_SMBUS_QUICK_READ:
+            params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_SMBUS_QUICK_RW;
+            break;
+
+        case NV_I2C_CMD_BLOCK_WRITE:
+            params->transData.i2cBufferData.bWrite = NV_TRUE;
+            /* fall through */
+
+        case NV_I2C_CMD_BLOCK_READ:
+            params->transType = NV402C_CTRL_I2C_TRANSACTION_TYPE_I2C_BUFFER_RW;
+            params->transData.i2cBufferData.registerAddress = command;
+            params->transData.i2cBufferData.messageLength = len;
+            params->transData.i2cBufferData.pMessage = pData;
+            break;
+
+        default:
+            portMemFree(params);
+            return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+                               pNv->rmapi.hI2C,
+                               NV402C_CTRL_CMD_I2C_TRANSACTION,
+                               params, sizeof(*params));
+
+    //
+    // For NV_I2C_CMD_SMBUS_READ, copy the read data to original
+    // data buffer.
+    //
+    if (rmStatus == NV_OK && type == NV_I2C_CMD_SMBUS_READ)
+    {
+        if (len == 2)
+        {
+            pData[0] = (params->transData.smbusWordData.message & 0xff);
+            pData[1] = params->transData.smbusWordData.message >> 8;
+        }
+        else
+        {
+            pData[0] = params->transData.smbusByteData.message;
+        }
+    }
+
+    portMemFree(params);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_i2c_transfer(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    void       *pI2cAdapter,
+    nv_i2c_cmd_t type,
+    NvU8        addr,
+    NvU8        command,
+    NvU32       len,
+    NvU8       *pData
+)
+{
+    THREAD_STATE_NODE threadState;
+    nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+    NV_STATUS  rmStatus = NV_OK;
+    OBJGPU    *pGpu  = NULL;
+    NvBool     unlockApi = NV_FALSE;
+    NvBool     unlockGpu = NV_FALSE;
+    NvU32      x;
+    void      *fp;
+    NvU32      numDispId = 0;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if (pNvp->flags & NV_INIT_FLAG_PUBLIC_I2C)
+    {
+        // LOCK: acquire API lock
+        if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK)
+            goto finish;
+
+        unlockApi = NV_TRUE;
+
+        // LOCK: acquire GPUs lock
+        if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK)
+            goto finish;
+
+        unlockGpu = NV_TRUE;
+    }
+
+    pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+    if (!pGpu)
+    {
+        rmStatus = NV_ERR_GENERIC;
+        goto finish;
+    }
+
+    for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+    {
+        if (pNvp->i2c_adapters[x].pOsAdapter == pI2cAdapter)
+        {
+            break;
+        }
+    }
+
+    if (x == MAX_I2C_ADAPTERS)
+    {
+        rmStatus = NV_ERR_GENERIC;
+        goto finish;
+    }
+
+    for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+    {
+        NvU32 displayId = pNvp->i2c_adapters[x].displayId[numDispId];
+
+        if (displayId == INVALID_DISP_ID)
+        {
+            continue;
+        }
+
+        // Handle i2c-over-DpAux adapters separately from regular i2c adapters
+        if (displayId == 0)
+        {
+            rmStatus = RmNonDPAuxI2CTransfer(pNv, pNvp->i2c_adapters[x].port,
+                                             type, addr, command, len, pData);
+        }
+        else
+        {
+            if ((type != NV_I2C_CMD_READ) && (type != NV_I2C_CMD_WRITE))
+            {
+                rmStatus = NV_ERR_NOT_SUPPORTED;
+                goto semafinish;
+            }
+
+            rmStatus = RmDpAuxI2CTransfer(pNv, displayId, addr, len, pData,
+                                          type == NV_I2C_CMD_WRITE);
+        }
+semafinish:
+        if (rmStatus == NV_OK)
+        {
+            break;
+        }
+    }
+
+finish:
+    if (unlockGpu)
+    {
+        // UNLOCK: release GPU lock
+        rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+    }
+
+    if (unlockApi)
+    {
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+static void rm_i2c_add_adapter(
+    nv_state_t *pNv,
+    NvU32       port,
+    NvU32       displayId
+)
+{
+    NvU32 y, free;
+    nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+    NvU32 numDispId = 0;
+
+    for (y = 0, free = MAX_I2C_ADAPTERS; y < MAX_I2C_ADAPTERS; y++)
+    {
+        if (pNvp->i2c_adapters[y].pOsAdapter == NULL)
+        {
+            // Only find the first free entry, and ignore the rest
+            if (free == MAX_I2C_ADAPTERS)
+            {
+                free = y;
+            }
+        }
+        else if (pNvp->i2c_adapters[y].port == port)
+        {
+            break;
+        }
+    }
+
+    if (y < MAX_I2C_ADAPTERS)
+    {
+        for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+        {
+            if (pNvp->i2c_adapters[y].displayId[numDispId] == INVALID_DISP_ID)
+            {
+                pNvp->i2c_adapters[y].displayId[numDispId] = displayId;
+                break;
+            }
+            else
+            {
+                NV_PRINTF(LEVEL_INFO,
+                          "%s: adapter  already exists (port=0x%x, displayId=0x%x)\n",
+                          __FUNCTION__, port,
+                          pNvp->i2c_adapters[y].displayId[numDispId]);
+            }
+        }
+
+        if (numDispId == MAX_DISP_ID_PER_ADAPTER)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s: no more free display Id entries in adapter\n",
+                      __FUNCTION__);
+        }
+
+        return;
+    }
+
+    if (free == MAX_I2C_ADAPTERS)
+    {
+        NV_PRINTF(LEVEL_ERROR, "%s: no more free adapter entries exist\n",
+                  __FUNCTION__);
+        return;
+    }
+
+    pNvp->i2c_adapters[free].pOsAdapter = nv_i2c_add_adapter(pNv, port);
+    pNvp->i2c_adapters[free].port       = port;
+    // When port is added, numDispId will be 0.
+    pNvp->i2c_adapters[free].displayId[numDispId] = displayId;
+}
+
+void RmI2cAddGpuPorts(nv_state_t * pNv)
+{
+    NvU32      x = 0;
+    nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
+    RM_API    *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+    NvU32      displayMask;
+    NV_STATUS  status;
+    OBJGPU    *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+    NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS systemGetSupportedParams = { 0 };
+
+    if (IS_VIRTUAL(pGpu) || NV_IS_SOC_DISPLAY_DEVICE(pNv))
+        return;
+
+    // Make displayId as Invalid.
+    for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+    {
+        NvU32 numDispId;
+
+        for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+        {
+            pNvp->i2c_adapters[x].displayId[numDispId] = INVALID_DISP_ID;
+        }
+    }
+
+    // First, set up the regular i2c adapters - one per i2c port
+    if (pNv->rmapi.hI2C != 0)
+    {
+        NV402C_CTRL_I2C_GET_PORT_INFO_PARAMS i2cPortInfoParams = { 0 };
+
+        status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hI2C,
+                                 NV402C_CTRL_CMD_I2C_GET_PORT_INFO,
+                                 &i2cPortInfoParams, sizeof(i2cPortInfoParams));
+
+        if (status == NV_OK)
+        {
+            for (x = 0; x < NV_ARRAY_ELEMENTS(i2cPortInfoParams.info); x++)
+            {
+                //
+                // Check if this port is implemented and RM I2C framework has
+                // validated this port. Only limited amount of ports can
+                // be added to the OS framework.
+                //
+                if (FLD_TEST_DRF(402C_CTRL, _I2C_GET_PORT_INFO, _IMPLEMENTED,
+                                 _YES, i2cPortInfoParams.info[x]) &&
+                    FLD_TEST_DRF(402C_CTRL, _I2C_GET_PORT_INFO, _VALID,
+                                 _YES, i2cPortInfoParams.info[x]))
+                {
+                    rm_i2c_add_adapter(pNv, x, 0);
+                }
+            }
+        }
+    }
+
+    //
+    // Now set up the i2c-over-DpAux adapters - one per DP OD
+    //
+    // 1. Perform NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS RM control which
+    //    will return the mask for all the display ID's.
+    // 2. Loop for all the display ID's and do
+    //    NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO RM control call. For each
+    //    output resource, check for the following requirements:
+    //    a. It must be DisplayPort.
+    //    b. It must be internal to the GPU (ie, not on the board)
+    //    c. It must be directly connected to the physical connector (ie, no DP
+    //       1.2 multistream ODs).
+    // 3. Perform NV0073_CTRL_CMD_SPECIFIC_GET_I2C_PORTID RM control for
+    //    getting the I2C port data.
+    //
+    // With these restrictions, we should only end up with at most one OD
+    // per DP connector.
+    //
+
+    if (pNv->rmapi.hDisp == 0)
+    {
+        return;
+    }
+
+    systemGetSupportedParams.subDeviceInstance = 0;
+    status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp,
+                             NV0073_CTRL_CMD_SYSTEM_GET_SUPPORTED,
+                             &systemGetSupportedParams, sizeof(systemGetSupportedParams));
+
+    if (status != NV_OK)
+    {
+        return;
+    }
+
+    for (displayMask = systemGetSupportedParams.displayMask;
+         displayMask != 0;
+         displayMask &= ~LOWESTBIT(displayMask))
+    {
+        NV0073_CTRL_SPECIFIC_OR_GET_INFO_PARAMS orInfoParams = { 0 };
+        NvU32 displayId = LOWESTBIT(displayMask);
+
+        orInfoParams.subDeviceInstance = 0;
+        orInfoParams.displayId         = displayId;
+
+        status = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hDisp,
+                                NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO,
+                                &orInfoParams, sizeof(orInfoParams));
+
+        if ((status == NV_OK) &&
+            (orInfoParams.type == NV0073_CTRL_SPECIFIC_OR_TYPE_SOR) &&
+            ((orInfoParams.protocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DP_A) ||
+             (orInfoParams.protocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DP_B)) &&
+            (orInfoParams.location == NV0073_CTRL_SPECIFIC_OR_LOCATION_CHIP) &&
+            (!orInfoParams.bIsDispDynamic))
+        {
+            NV0073_CTRL_SPECIFIC_GET_I2C_PORTID_PARAMS i2cPortIdParams = { 0 };
+
+            i2cPortIdParams.subDeviceInstance = 0;
+            i2cPortIdParams.displayId = displayId;
+
+            status = pRmApi->Control(pRmApi,
+                                     pNv->rmapi.hClient,
+                                     pNv->rmapi.hDisp,
+                                     NV0073_CTRL_CMD_SPECIFIC_GET_I2C_PORTID,
+                                     &i2cPortIdParams,
+                                     sizeof(i2cPortIdParams));
+
+            if ((status == NV_OK) &&
+                (i2cPortIdParams.ddcPortId != NV0073_CTRL_SPECIFIC_I2C_PORT_NONE))
+            {
+                rm_i2c_add_adapter(pNv, i2cPortIdParams.ddcPortId - 1, displayId);
+            }
+        }
+    }
+}
+
+void NV_API_CALL rm_i2c_remove_adapters(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    //
+    // Cycle through all adapter entries, and first remove the adapter
+    // from the list from the kernel, then remove the i2c adapter
+    // list once that is completed. This should only be used from exit
+    // module time. Otherwise it could fail to remove some of the
+    // kernel adapters and subsequent transfer requests would result
+    // in crashes.
+    //
+    NvU32 x = 0;
+    nv_priv_t  *pNvp = NV_GET_NV_PRIV(pNv);
+    NvU32 numDispId;
+
+    for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+    {
+        if (pNvp->i2c_adapters[x].pOsAdapter != NULL)
+        {
+            nv_i2c_del_adapter(pNv, pNvp->i2c_adapters[x].pOsAdapter);
+
+            pNvp->i2c_adapters[x].pOsAdapter = NULL;
+            pNvp->i2c_adapters[x].port       = 0;
+            for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+            {
+                pNvp->i2c_adapters[x].displayId[numDispId] = INVALID_DISP_ID;
+            }
+        }
+    }
+}
+
+NvBool NV_API_CALL rm_i2c_is_smbus_capable(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    void       *pI2cAdapter
+)
+{
+    THREAD_STATE_NODE threadState;
+    nv_priv_t  *pNvp = NV_GET_NV_PRIV(pNv);
+    NV_STATUS   rmStatus = NV_OK;
+    OBJGPU     *pGpu = NULL;
+    NvBool      unlock = NV_FALSE;
+    NvU32       x;
+    NvBool      ret = NV_FALSE;
+    void       *fp;
+    NvU32       numDispId = 0;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if (pNvp->flags & NV_INIT_FLAG_PUBLIC_I2C)
+    {
+        // LOCK: acquire API lock
+        if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_I2C)) != NV_OK)
+            goto semafinish;
+
+        unlock = NV_TRUE;
+    }
+
+    pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+    if (!pGpu)
+    {
+        goto semafinish;
+    }
+
+    for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+    {
+        if (pNvp->i2c_adapters[x].pOsAdapter == pI2cAdapter)
+        {
+            break;
+        }
+    }
+
+    if (x == MAX_I2C_ADAPTERS)
+    {
+        goto semafinish;
+    }
+
+    // we do not support smbus functions on i2c-over-DPAUX
+    for (numDispId = 0; numDispId < MAX_DISP_ID_PER_ADAPTER; numDispId++)
+    {
+        if (pNvp->i2c_adapters[x].displayId[numDispId] == 0x0)
+        {
+            ret = NV_TRUE;
+        }
+    }
+
+semafinish:
+    if (unlock)
+    {
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return ret;
+}
+
+NV_STATUS NV_API_CALL rm_perform_version_check(
+    nvidia_stack_t *sp,
+    void       *pData,
+    NvU32       dataSize
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    rmStatus = RmPerformVersionCheck(pData, dataSize);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+//
+// Handles the Power Source Change event(AC/DC) for Notebooks.
+// Notebooks from Maxwell have only one Gpu, so this functions grabs first Gpu
+// from GpuMgr and call subdevice RmControl.
+//
+void NV_API_CALL rm_power_source_change_event(
+    nvidia_stack_t *sp,
+    NvU32 event_val
+)
+{
+    THREAD_STATE_NODE threadState;
+    void           *fp;
+    nv_state_t     *nv;
+    static NvBool   first_event_seen = NV_FALSE;
+    static NvU32    last_event_val = 0;
+    NV_STATUS       rmStatus = NV_OK;
+
+    // Some systems generate spurious power source changed ACPI events.
+    // Before entering the PM runtime and waking the GPU, see if the power
+    // state has actually changed. If not, return without waking the GPU.
+    if ((first_event_seen != NV_FALSE) && (last_event_val == event_val))
+    {
+        return;
+    }
+    else
+    {
+        first_event_seen = NV_TRUE;
+        last_event_val = event_val;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_EVENT)) == NV_OK)
+    {
+        OBJGPU *pGpu = gpumgrGetGpu(0);
+        if (pGpu != NULL)
+        {
+            nv = NV_GET_NV_STATE(pGpu);
+            if ((rmStatus = os_ref_dynamic_power(nv, NV_DYNAMIC_PM_FINE)) ==
+                                                                         NV_OK)
+            {
+                // LOCK: acquire GPU lock
+                if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_EVENT)) ==
+                                                                         NV_OK)
+                {
+                    rmStatus = RmPowerSourceChangeEvent(nv, event_val);
+
+                    // UNLOCK: release GPU lock
+                    rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+                }
+                os_unref_dynamic_power(nv, NV_DYNAMIC_PM_FINE);
+            }
+            // UNLOCK: release API lock
+            rmapiLockRelease();
+        }
+    }
+
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s: Failed to handle Power Source change event, status=0x%x\n",
+                  __FUNCTION__, rmStatus);
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_request_dnotifier_state(
+    nv_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(pNv);
+
+    if (nvp->b_mobile_config_enabled)
+    {
+        THREAD_STATE_NODE threadState;
+        void              *fp;
+        GPU_MASK          gpuMask;
+
+        NV_ENTER_RM_RUNTIME(sp,fp);
+        threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+        // LOCK: acquire API lock
+        if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_ACPI)) == NV_OK)
+        {
+            OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+            // LOCK: acquire per device lock
+            if ((pGpu != NULL) &&
+                ((rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
+                                       GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_ACPI,
+                                       &gpuMask)) == NV_OK))
+            {
+                RmRequestDNotifierState(pNv);
+
+                // UNLOCK: release per device lock
+                rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
+            }
+
+            // UNLOCK: release API lock
+            rmapiLockRelease();
+        }
+
+        threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+        NV_EXIT_RM_RUNTIME(sp,fp);
+    }
+}
+
+NV_STATUS NV_API_CALL rm_p2p_dma_map_pages(
+    nvidia_stack_t  *sp,
+    nv_dma_device_t *peer,
+    NvU8            *pGpuUuid,
+    NvU64            pageSize,
+    NvU32            pageCount,
+    NvU64           *pDmaAddresses,
+    void           **ppPriv
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+    NvU32 i, j;
+    OBJGPU *pGpu;
+    nv_state_t *pNv;
+
+    if (ppPriv == NULL)
+        return NV_ERR_INVALID_ARGUMENT;
+
+    *ppPriv = NULL;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P);
+    if (rmStatus != NV_OK)
+        goto done;
+
+    pGpu = gpumgrGetGpuFromUuid(pGpuUuid,
+                DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1) |
+                DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _BINARY));
+    if (pGpu == NULL)
+    {
+        rmStatus = NV_ERR_INVALID_ARGUMENT;
+        goto unlock;
+    }
+
+    pNv = NV_GET_NV_STATE(pGpu);
+
+    if (pNv->coherent)
+    {
+        rmStatus = RmP2PDmaMapPages(pNv, peer, pageSize, pageCount,
+                                    pDmaAddresses, ppPriv);
+        goto unlock;
+    }
+
+    // Check if the topology is supported for third party p2p
+    if (!nv_grdma_pci_topology_supported(pNv, peer))
+    {
+        rmStatus = NV_ERR_NOT_SUPPORTED;
+        goto unlock;
+    }
+
+    for (i = 0; i < pageCount; i++)
+    {
+        // Map BAR1 PFNs for non-coherent platforms
+        rmStatus = nv_dma_map_peer(peer, pNv->dma_dev, 0x1,
+                                   pageSize / os_page_size,
+                                   &pDmaAddresses[i]);
+        if (rmStatus != NV_OK)
+        {
+            for (j = 0; j < i; j++)
+            {
+                nv_dma_unmap_peer(peer, pageSize / os_page_size,
+                                  pDmaAddresses[j]);
+            }
+            goto unlock;
+        }
+    }
+
+unlock:
+    rmapiLockRelease();
+
+done:
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_get_gpu_info(
+    nvidia_stack_t *sp,
+    NvU64           gpuVirtualAddress,
+    NvU64           length,
+    NvU8          **ppGpuUuid,
+    void          **ppGpuInfo
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P);
+    if (rmStatus == NV_OK)
+    {
+        OBJGPU *pGpu;
+        rmStatus = RmP2PGetGpuByAddress(gpuVirtualAddress,
+                                        length,
+                                        &pGpu);
+        if (rmStatus == NV_OK)
+        {
+            nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+            const NvU8 *pGid;
+
+            pGid = RmGetGpuUuidRaw(nv);
+            if (pGid == NULL)
+            {
+                rmStatus = NV_ERR_GPU_UUID_NOT_FOUND;
+            }
+            else
+            {
+                rmStatus = os_alloc_mem((void **)ppGpuUuid, GPU_UUID_LEN);
+                if (rmStatus == NV_OK)
+                {
+                    os_mem_copy(*ppGpuUuid, pGid, GPU_UUID_LEN);
+                }
+            }
+
+            *ppGpuInfo = (void *) pGpu;
+        }
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent(
+    nvidia_stack_t *sp,
+    NvU64           gpuVirtualAddress,
+    NvU64           length,
+    void          **p2pObject,
+    NvU64          *pPhysicalAddresses,
+    NvU32          *pEntries,
+    NvBool          bForcePcie,
+    void           *pPlatformData,
+    void           *pGpuInfo,
+    void          **ppMigInfo,
+    NvBool         *pMemCpuCacheable
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+    {
+        rmStatus = RmP2PGetPagesPersistent(gpuVirtualAddress,
+                                           length,
+                                           p2pObject,
+                                           pPhysicalAddresses,
+                                           pEntries,
+                                           pMemCpuCacheable,
+                                           bForcePcie,
+                                           pPlatformData,
+                                           pGpuInfo,
+                                           ppMigInfo);
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_get_pages(
+    nvidia_stack_t *sp,
+    NvU64       p2pToken,
+    NvU32       vaSpaceToken,
+    NvU64       gpuVirtualAddress,
+    NvU64       length,
+    NvU64      *pPhysicalAddresses,
+    NvU32      *pWreqMbH,
+    NvU32      *pRreqMbH,
+    NvU32      *pEntries,
+    NvU8      **ppGpuUuid,
+    void       *pPlatformData,
+    NvBool     *pMemCpuCacheable
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+    {
+        OBJGPU *pGpu;
+        rmStatus = RmP2PGetPagesWithoutCallbackRegistration(p2pToken,
+                                                            vaSpaceToken,
+                                                            gpuVirtualAddress,
+                                                            length,
+                                                            pPhysicalAddresses,
+                                                            pWreqMbH,
+                                                            pRreqMbH,
+                                                            pEntries,
+                                                            pMemCpuCacheable,
+                                                            &pGpu,
+                                                            pPlatformData);
+        if (rmStatus == NV_OK)
+        {
+            nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+            const NvU8 *pGid;
+
+            pGid = RmGetGpuUuidRaw(nv);
+            if (pGid == NULL)
+            {
+                NV_ASSERT_OK(RmP2PPutPages(p2pToken, vaSpaceToken,
+                                           gpuVirtualAddress,
+                                           pPlatformData));
+                rmStatus = NV_ERR_GENERIC;
+            }
+            else
+            {
+                rmStatus = os_alloc_mem((void **)ppGpuUuid, GPU_UUID_LEN);
+                if (rmStatus == NV_OK)
+                    os_mem_copy(*ppGpuUuid, pGid, GPU_UUID_LEN);
+            }
+
+            *pMemCpuCacheable = nv->coherent;
+        }
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_register_callback(
+    nvidia_stack_t *sp,
+    NvU64       p2pToken,
+    NvU64       gpuVirtualAddress,
+    NvU64       length,
+    void       *pPlatformData,
+    void      (*pFreeCallback)(void *pData),
+    void       *pData
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+    {
+        rmStatus = RmP2PRegisterCallback(p2pToken, gpuVirtualAddress, length,
+                                         pPlatformData, pFreeCallback, pData);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(
+    nvidia_stack_t *sp,
+    void           *p2pObject,
+    void           *pKey,
+    void           *pMigInfo
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+    {
+        rmStatus = RmP2PPutPagesPersistent(p2pObject, pKey, pMigInfo);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_p2p_put_pages(
+    nvidia_stack_t *sp,
+    NvU64       p2pToken,
+    NvU32       vaSpaceToken,
+    NvU64       gpuVirtualAddress,
+    void       *pKey
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
+    {
+        rmStatus = RmP2PPutPages(p2pToken,
+                                 vaSpaceToken,
+                                 gpuVirtualAddress,
+                                 pKey);
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+char* NV_API_CALL rm_get_gpu_uuid(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    NV_STATUS rmStatus;
+    const NvU8 *pGid;
+    char *pGidString;
+
+    THREAD_STATE_NODE threadState;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // Allocate space for the ASCII string
+    rmStatus = os_alloc_mem((void **)&pGidString, GPU_UUID_ASCII_LEN);
+    if (rmStatus != NV_OK)
+    {
+        pGidString = NULL;
+        goto done;
+    }
+
+    // Get the raw UUID; note the pGid is cached, so we do not need to free it
+    pGid = RmGetGpuUuidRaw(nv);
+
+    if (pGid != NULL)
+    {
+        // Convert the raw UUID to ASCII
+        rmStatus = RmGpuUuidRawToString(pGid, pGidString, GPU_UUID_ASCII_LEN);
+        if (rmStatus != NV_OK)
+        {
+            os_free_mem(pGidString);
+            pGidString = NULL;
+        }
+    }
+    else
+    {
+        const char *pTmpString = "GPU-???????\?-???\?-???\?-???\?-????????????";
+
+        portStringCopy(pGidString, GPU_UUID_ASCII_LEN, pTmpString,
+                       portStringLength(pTmpString) + 1);
+    }
+
+done:
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return pGidString;
+}
+
+//
+// This function will return the UUID in the binary format
+//
+const NvU8 * NV_API_CALL rm_get_gpu_uuid_raw(
+    nvidia_stack_t *sp,
+    nv_state_t *nv)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+    const NvU8 *pGid;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    pGid = RmGetGpuUuidRaw(nv);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return pGid;
+}
+
+static void rm_set_firmware_logs(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    void *fp;
+    NvU32 enableFirmwareLogsRegVal = NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    (void) RmReadRegistryDword(nv, NV_REG_ENABLE_GPU_FIRMWARE_LOGS,
+                               &enableFirmwareLogsRegVal);
+    nv->enable_firmware_logs = gpumgrGetRmFirmwareLogsEnabled(enableFirmwareLogsRegVal);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+static NvBool rm_get_is_gsp_capable_vgpu(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    void *fp;
+    THREAD_STATE_NODE threadState;
+    NvBool isVgpu = NV_FALSE;
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if ((pHypervisor != NULL) && pHypervisor->bIsHVMGuest)
+    {
+        GPUHWREG *reg_mapping;
+        reg_mapping = osMapKernelSpace(nv->regs->cpu_address,
+                                       os_page_size,
+                                       NV_MEMORY_UNCACHED,
+                                       NV_PROTECT_READABLE);
+
+        if (reg_mapping != NULL)
+        {
+            NvU32 pmc_boot_1 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_1);
+            NvU32 pmc_boot_42 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_42);
+
+            osUnmapKernelSpace(reg_mapping, os_page_size);
+
+            if (FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1) &&
+                gpumgrIsVgxRmFirmwareCapableChip(pmc_boot_42))
+            {
+                isVgpu = NV_TRUE;
+            }
+        }
+    }
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return isVgpu;
+}
+
+void NV_API_CALL rm_set_rm_firmware_requested(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    nv->request_firmware = NV_TRUE;
+    nv->allow_fallback_to_monolithic_rm = NV_FALSE;
+    if (rm_get_is_gsp_capable_vgpu(sp, nv))
+    {
+        nv->request_firmware = NV_FALSE;
+    }
+
+    // Check if we want firmware logs
+    if (nv->request_firmware)
+        rm_set_firmware_logs(sp, nv);
+}
+
+//
+// disable GPU SW state persistence
+//
+
+void NV_API_CALL rm_disable_gpu_state_persistence(nvidia_stack_t *sp, nv_state_t *nv)
+{
+    THREAD_STATE_NODE threadState;
+    OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    pGpu->setProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE, NV_FALSE);
+    osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, NV_FALSE);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void  NV_API_CALL  rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd)
+{
+    nvidia_kernel_rmapi_ops_t *ops = ops_cmd;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    switch (ops->op)
+    {
+        case NV01_FREE:
+            Nv01FreeKernel(&ops->params.free);
+            break;
+
+        case NV01_ALLOC_MEMORY:
+            Nv01AllocMemoryKernel(&ops->params.allocMemory64);
+            break;
+
+        case NV04_ALLOC:
+            Nv04AllocWithAccessKernel(&ops->params.alloc);
+            break;
+
+        case NV04_VID_HEAP_CONTROL:
+            Nv04VidHeapControlKernel(ops->params.pVidHeapControl);
+            break;
+
+        case NV04_MAP_MEMORY:
+        {
+            //
+            // We need to free NVUAP for MEM_SPACE_USER mappings,
+            // since we're not going through the regular path
+            //
+            nv_usermap_access_params_t *pNvuap = NULL;
+            Nv04MapMemoryKernel(&ops->params.mapMemory);
+            pNvuap = (nv_usermap_access_params_t *) tlsEntryGet(TLS_ENTRY_ID_MAPPING_CONTEXT);
+            if (pNvuap != NULL)
+            {
+                NV_ASSERT(tlsEntryRelease(TLS_ENTRY_ID_MAPPING_CONTEXT) == 0);
+                os_free_mem(pNvuap->memArea.pRanges);
+                os_free_mem(pNvuap);
+            }
+            break;
+        }
+
+        case NV04_UNMAP_MEMORY:
+            Nv04UnmapMemoryKernel(&ops->params.unmapMemory);
+            break;
+
+        case NV04_ALLOC_CONTEXT_DMA:
+            Nv04AllocContextDmaKernel(&ops->params.allocContextDma2);
+            break;
+
+        case NV04_MAP_MEMORY_DMA:
+            Nv04MapMemoryDmaKernel(&ops->params.mapMemoryDma);
+            break;
+
+        case NV04_UNMAP_MEMORY_DMA:
+            Nv04UnmapMemoryDmaKernel(&ops->params.unmapMemoryDma);
+            break;
+
+        case NV04_BIND_CONTEXT_DMA:
+            Nv04BindContextDmaKernel(&ops->params.bindContextDma);
+            break;
+
+        case NV04_CONTROL:
+            Nv04ControlKernel(&ops->params.control);
+            break;
+
+        case NV04_DUP_OBJECT:
+            Nv04DupObjectKernel(&ops->params.dupObject);
+            break;
+
+        case NV04_SHARE:
+            Nv04ShareKernel(&ops->params.share);
+            break;
+
+        case NV04_ADD_VBLANK_CALLBACK:
+            Nv04AddVblankCallbackKernel(&ops->params.addVblankCallback);
+            break;
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+//
+// ACPI method (NVIF/_DSM/WMMX/MXM*/etc.) initialization
+//
+void RmInitAcpiMethods(OBJOS *pOS, OBJSYS *pSys, OBJGPU *pGpu)
+{
+    NvU32 handlesPresent;
+
+    if (pSys->getProperty(pSys, PDB_PROP_SYS_NVIF_INIT_DONE))
+        return;
+
+    nv_acpi_methods_init(&handlesPresent);
+
+    acpiDsmInit(pGpu);
+}
+
+//
+// ACPI method (NVIF/_DSM/WMMX/MXM*/etc.) teardown
+//
+void RmUnInitAcpiMethods(OBJSYS *pSys)
+{
+    pSys->setProperty(pSys, PDB_PROP_SYS_NVIF_INIT_DONE, NV_FALSE);
+
+    nv_acpi_methods_uninit();
+}
+
+//
+// Converts an array of OS page address to an array of RM page addresses.This
+// assumes that:
+//   (1) The pteArray is at least pageCount entries large,
+//   (2) The pageCount is given in RM pages, and
+//   (3) The OS page entries start at index 0.
+//
+void RmInflateOsToRmPageArray(RmPhysAddr *pteArray, NvU64 pageCount)
+{
+    RmInflatePageArray(
+        pteArray, 
+        NV_RM_PAGES_TO_OS_PAGES(pageCount),
+        os_page_size, 
+        pageCount, 
+        NV_RM_PAGE_SIZE
+    );
+}
+
+void RmDeflateRmToOsPageArray(RmPhysAddr *pteArray, NvU64 pageCount)
+{
+    RmDeflatePageArray(
+        pteArray, 
+        pageCount, 
+        NV_RM_PAGE_SIZE,
+        NV_RM_PAGES_TO_OS_PAGES(pageCount),
+        os_page_size
+    );
+}
+
+NvBool NV_API_CALL
+rm_get_device_remove_flag
+(
+    nvidia_stack_t * sp,
+    NvU32 gpu_id
+)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+    NvBool bRemove;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if (gpumgrQueryGpuDrainState(gpu_id, NULL, &bRemove) != NV_OK)
+    {
+        bRemove = NV_FALSE;
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return bRemove;
+}
+
+//
+// This API updates only the following fields in nv_ioctl_numa_info_t:
+// - nid
+// - numa_mem_addr
+// - numa_mem_size
+// - offline_addresses
+//
+// Rest of the fields should be updated by caller.
+NV_STATUS NV_API_CALL rm_get_gpu_numa_info(
+    nvidia_stack_t          *sp,
+    nv_state_t              *nv,
+    nv_ioctl_numa_info_t    *numa_info
+)
+{
+    NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS *pParams;
+    RM_API             *pRmApi;
+    THREAD_STATE_NODE   threadState;
+    void               *fp;
+    NV_STATUS           status = NV_OK;
+
+    ct_assert(NV_ARRAY_ELEMENTS(numa_info->offline_addresses.addresses) >=
+          NV_ARRAY_ELEMENTS(pParams->numaOfflineAddresses));
+
+    if ((numa_info == NULL) ||
+        (numa_info->offline_addresses.numEntries >
+         NV_ARRAY_ELEMENTS(pParams->numaOfflineAddresses)))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    if (NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        numa_info->nid = NV0000_CTRL_NO_NUMA_NODE;
+        numa_info->numa_mem_addr = 0;
+        numa_info->numa_mem_size = 0;
+        numa_info->offline_addresses.numEntries = 0;
+        return NV_OK;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pParams = portMemAllocNonPaged(sizeof(*pParams));
+    if (pParams == NULL)
+    {
+        NV_EXIT_RM_RUNTIME(sp,fp);
+        return NV_ERR_NO_MEMORY;
+    }
+
+    portMemSet(pParams, 0, sizeof(*pParams));
+    pParams->numaOfflineAddressesCount =
+        numa_info->offline_addresses.numEntries;
+
+    pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM);
+    if (pRmApi == NULL)
+    {
+        status = NV_ERR_INVALID_STATE;
+        goto finish;
+    }
+
+    status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice,
+                             NV2080_CTRL_CMD_FB_GET_NUMA_INFO,
+                             pParams, sizeof(*pParams));
+
+    RmUnixRmApiEpilogue(nv, &threadState);
+
+    if (status == NV_OK)
+    {
+        NvU32 i;
+
+        numa_info->nid = pParams->numaNodeId;
+        numa_info->numa_mem_addr = pParams->numaMemAddr;
+        numa_info->numa_mem_size = pParams->numaMemSize;
+        numa_info->offline_addresses.numEntries =
+            pParams->numaOfflineAddressesCount;
+
+        for (i = 0; i < pParams->numaOfflineAddressesCount; i++)
+        {
+            numa_info->offline_addresses.addresses[i] =
+                pParams->numaOfflineAddresses[i];
+        }
+    }
+
+finish:
+    portMemFree(pParams);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return status;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_numa_online(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS params = { 0 };
+    RM_API            *pRmApi;
+    THREAD_STATE_NODE  threadState;
+    void              *fp;
+    NV_STATUS          status = NV_OK;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM);
+    if (pRmApi == NULL)
+    {
+        status = NV_ERR_INVALID_STATE;
+        goto finish;
+    }
+
+    params.bOnline = NV_TRUE;
+
+    status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice,
+                             NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS,
+                             &params, sizeof(params));
+
+    RmUnixRmApiEpilogue(nv, &threadState);
+
+finish:
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return status;
+}
+
+
+NV_STATUS NV_API_CALL rm_gpu_numa_offline(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS params = { 0 };
+    RM_API            *pRmApi;
+    THREAD_STATE_NODE  threadState;
+    void              *fp;
+    NV_STATUS          status = NV_OK;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pRmApi = RmUnixRmApiPrologue(nv, &threadState, RM_LOCK_MODULES_MEM);
+    if (pRmApi == NULL)
+    {
+        status = NV_ERR_INVALID_STATE;
+        goto finish;
+    }
+
+    params.bOnline = NV_FALSE;
+
+    status = pRmApi->Control(pRmApi, nv->rmapi.hClient,
+                             nv->rmapi.hSubDevice,
+                             NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS,
+                             &params, sizeof(params));
+
+    RmUnixRmApiEpilogue(nv, &threadState);
+
+finish:
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return status;
+}
+
+//
+// A device is considered "sequestered" if it has drain state enabled for it.
+// The kernel interface layer can use this to check the drain state of a device
+// in paths outside of initialization, e.g., when clients attempt to reference
+// count the device.
+//
+NvBool NV_API_CALL rm_is_device_sequestered(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+    NvBool bDrain = NV_FALSE;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    //
+    // If gpumgrQueryGpuDrainState succeeds, bDrain will be set as needed.
+    // If gpumgrQueryGpuDrainState fails, bDrain will stay false; we assume
+    // that if core RM can't tell us the drain state, it must not be
+    // attached and the "sequestered" question is not relevant.
+    //
+    (void) gpumgrQueryGpuDrainState(pNv->gpu_id, &bDrain, NULL);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return bDrain;
+}
+
+void NV_API_CALL rm_check_for_gpu_surprise_removal(
+    nvidia_stack_t *sp,
+    nv_state_t *nv
+)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+    NV_STATUS rmStatus;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock.
+    if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU)) == NV_OK)
+    {
+        OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+        if ((rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GPU)) == NV_OK)
+        {
+            osHandleGpuLost(pGpu, NV_TRUE);
+            rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+        }
+
+        // UNLOCK: release api lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_set_external_kernel_client_count(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv,
+    NvBool bIncr
+)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+    OBJGPU *pGpu;
+    NV_STATUS rmStatus = NV_OK;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+
+    if (pGpu != NULL)
+    {
+        rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE,
+                                           RM_LOCK_MODULES_GPU);
+        if (rmStatus == NV_OK)
+        {
+            rmStatus = gpuSetExternalKernelClientCount(pGpu, bIncr);
+            rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+        }
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+NvBool rm_get_uefi_console_status(
+    nv_state_t *nv
+)
+{
+    NvU32 fbWidth, fbHeight, fbDepth, fbPitch;
+    NvU64 fbSize;
+    NvU64 fbBaseAddress = 0;
+    NvBool bConsoleDevice = NV_FALSE;
+
+    //
+    // nv_get_screen_info() will return dimensions and an address for
+    // any fbdev driver (e.g., efifb, vesafb, etc).
+    //
+    nv_get_screen_info(nv, &fbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch, &fbSize);
+
+    bConsoleDevice = (fbSize != 0);
+
+    return bConsoleDevice;
+}
+
+NvU64 rm_get_uefi_console_size(
+    nv_state_t *nv,
+    NvU64      *pFbBaseAddress
+)
+{
+    NvU32 fbWidth, fbHeight, fbDepth, fbPitch;
+    NvU64 fbSize;
+
+    fbSize = fbWidth = fbHeight = fbDepth = fbPitch = 0;
+
+    //
+    // nv_get_screen_info() will return dimensions and an address for
+    // any fbdev driver (e.g., efifb, vesafb, etc).
+    //
+    nv_get_screen_info(nv, pFbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch, &fbSize);
+
+    return fbSize;
+}
+
+NvBool NV_API_CALL rm_disable_iomap_wc(void)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJCL *pCl = pSys->pCl;
+    return pCl->getProperty(pCl, PDB_PROP_CL_DISABLE_IOMAP_WC) == NV_TRUE;
+}
+
+//
+// Verifies the handle, offset and size and dups hMemory.
+// Must be called with API lock and GPU lock held.
+//
+NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle(
+    nvidia_stack_t   *sp,
+    nv_state_t       *nv,
+    NvHandle          hSrcClient,
+    NvHandle          hDstClient,
+    NvHandle          hDevice,
+    NvHandle          hSubdevice,
+    void             *pGpuInstanceInfo,
+    NvHandle          hMemory,
+    NvU64             offset,
+    NvU64             size,
+    NvU8              mappingType,
+    NvHandle         *phMemoryDuped,
+    void            **ppMemInfo,
+    NvBool           *pbCanMmap,
+    NvU32            *pCacheType,
+    NvBool           *pbReadOnlyMem,
+    nv_memory_type_t *pMemoryType
+)
+{
+    MEMORY_DESCRIPTOR *pMemDesc;
+    THREAD_STATE_NODE *pThreadState;
+    NV_STATUS rmStatus = NV_ERR_NO_MEMORY;
+    OBJGPU *pGpu;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    pThreadState = threadStateAlloc(THREAD_STATE_FLAGS_NONE);
+    NV_ASSERT_OR_GOTO(pThreadState != NULL, Done);
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    NV_ASSERT(rmapiLockIsOwner());
+
+    NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
+
+    rmStatus = RmDmabufVerifyMemHandle(pGpu, hSrcClient, hMemory,
+                                       offset, size, pGpuInstanceInfo,
+                                       &pMemDesc);
+    if (rmStatus == NV_OK)
+    {
+        RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+        NvHandle hMemoryDuped = 0;
+
+        rmStatus = pRmApi->DupObject(pRmApi,
+                                     hDstClient,
+                                     hDevice,
+                                     &hMemoryDuped,
+                                     hSrcClient,
+                                     hMemory,
+                                     0);
+        if (rmStatus == NV_OK)
+        {
+            *phMemoryDuped = hMemoryDuped;
+        }
+        else if (rmStatus == NV_ERR_INVALID_OBJECT_PARENT)
+        {
+            hMemoryDuped = 0;
+
+            // If duping under Device fails, try duping under Subdevice
+            rmStatus = pRmApi->DupObject(pRmApi,
+                                         hDstClient,
+                                         hSubdevice,
+                                         &hMemoryDuped,
+                                         hSrcClient,
+                                         hMemory,
+                                         0);
+            if (rmStatus == NV_OK)
+            {
+                *phMemoryDuped = hMemoryDuped;
+            }
+        }
+        *ppMemInfo = (void *) pMemDesc;
+
+        *pCacheType    = RmDmabufMmapGetCpuCacheType(pGpu, pMemDesc, mappingType);
+        *pbReadOnlyMem = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY);
+
+        if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
+        {
+            *pMemoryType = NV_MEMORY_TYPE_DEVICE_MMIO;
+        }
+        else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM)
+        {
+            *pMemoryType = NV_MEMORY_TYPE_SYSTEM;
+        }
+        else
+        {
+            // TODO: Assume NV_MEMORY_TYPE_FRAMEBUFFER is for now.
+            // Add Proper handling for NV_MEMORY_TYPE_REGISTERS.
+            *pMemoryType = NV_MEMORY_TYPE_FRAMEBUFFER;
+        }
+
+        // mmap is allowed on 0FB chips(iGPU) for sysmem and on dGPU for vidmem only
+        *pbCanMmap = (pGpu->pGpuArch->bGpuArchIsZeroFb ||
+                      (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM));
+    }
+
+Done:
+    if (pThreadState != NULL)
+        threadStateFree(pThreadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+//
+// Frees dup'd hMemory.
+// Must be called with API lock and GPU lock held.
+//
+void NV_API_CALL rm_dma_buf_undup_mem_handle(
+    nvidia_stack_t  *sp,
+    nv_state_t      *nv,
+    NvHandle         hClient,
+    NvHandle         hMemory
+)
+{
+    THREAD_STATE_NODE threadState;
+    RM_API *pRmApi;
+    OBJGPU *pGpu;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    NV_ASSERT(rmapiLockIsOwner());
+
+    NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
+
+    pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+
+    pRmApi->Free(pRmApi, hClient, hMemory);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+//
+// Maps a handle to system physical addresses:
+//   C2C for coherent platforms with DEFAULT mapping type
+//   BAR1(static & dynamic) for non-coherent platforms and for
+//     coherent platforms with mapping type FORCE_PCIE
+// Must be called with API lock and GPU lock held for dynamic BAR1.
+//
+NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle(
+    nvidia_stack_t        *sp,
+    nv_state_t            *nv,
+    NvHandle               hClient,
+    NvHandle               hMemory,
+    MemoryRange            memRange,
+    NvU8                   mappingType,
+    void                  *pMemInfo,
+    NvBool                 bStaticPhysAddrs,
+    MemoryArea            *pMemArea
+)
+{
+    THREAD_STATE_NODE *pThreadState;
+    NV_STATUS rmStatus = NV_ERR_INVALID_ARGUMENT;
+    OBJGPU *pGpu;
+    MEMORY_DESCRIPTOR *pMemDesc;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    pThreadState = threadStateAlloc(THREAD_STATE_FLAGS_NONE);
+    if (pThreadState == NULL)
+    {
+        NV_ASSERT(0);
+        rmStatus = NV_ERR_NO_MEMORY;
+        goto Done;
+    }
+
+    NV_ASSERT_OR_GOTO(((pMemArea != NULL) &&
+                       (pMemInfo != NULL)) &&
+                       (memRange.size != 0), Done);
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    pMemDesc = (MEMORY_DESCRIPTOR *) pMemInfo;
+
+    if (((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) ||
+        (pGpu->pGpuArch->bGpuArchIsZeroFb)) &&
+        (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT))
+    {
+        KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
+        NvBool contiguity = memdescCheckContiguity(pMemDesc, AT_CPU);
+
+        // On localized allocations over C2C/nvlink mappings, RDMA is not supported
+        if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_AS_LOCALIZED))
+        {
+            NV_PRINTF(LEVEL_ERROR, "RDMA is not supported for localized memory"
+                                  " over coherent mappings\n");
+            rmStatus = NV_ERR_NOT_SUPPORTED;
+            goto Done;
+        }
+
+        if (contiguity)
+        {
+            NvU64 physAddr = memdescGetPhysAddr(pMemDesc, AT_CPU, memRange.start);
+
+            NV_ASSERT_OK_OR_GOTO(rmStatus, os_alloc_mem((void **) &pMemArea->pRanges,
+                sizeof(MemoryRange)), Done);
+
+            pMemArea->pRanges[0].start = pKernelMemorySystem->coherentCpuFbBase + physAddr;
+            pMemArea->pRanges[0].size  = memRange.size;
+            pMemArea->numRanges = 1;
+        }
+        else
+        {
+            NvU64 idx = 0;
+            NvU64 memdescPageSize = memdescGetPageSize(pMemDesc, AT_CPU);
+            NvU64 origEnd = memRange.start + memRange.size;
+            NvU64 realStart = NV_ALIGN_DOWN64(memRange.start, memdescPageSize);
+            NvU64 realEnd = NV_ALIGN_UP64(memRange.start + memRange.size, memdescPageSize);
+            NvU64 realSize = realEnd - realStart;
+            NvU64 pageCount = realSize / memdescPageSize;
+
+            NV_ASSERT_OK_OR_GOTO(rmStatus, os_alloc_mem((void **) &pMemArea->pRanges,
+                pageCount * sizeof(MemoryRange)), Done);
+
+            for(idx = 0; idx < pageCount; idx++)
+            {
+                NvU64 physAddr = memdescGetPhysAddr(pMemDesc, AT_CPU,
+                    realStart + (idx * memdescPageSize));
+                pMemArea->pRanges[idx].start = pKernelMemorySystem->coherentCpuFbBase + physAddr;
+                pMemArea->pRanges[idx].size  = memdescPageSize;
+            }
+
+            pMemArea->pRanges[0].start += (memRange.start - realStart);
+            pMemArea->pRanges[0].size -= (memRange.start - realStart);
+            pMemArea->pRanges[pageCount - 1llu].size -= (realEnd - origEnd);
+
+            pMemArea->numRanges = pageCount;
+        }
+    }
+    else
+    {
+        Device *pDevice = NULL;
+        RsClient *pClient;
+        NvU64 idx;
+        NvU64 barOffset;
+        KernelBus *pKernelBus;
+        NvBool bForcePcie;
+
+        pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+
+        if (!bStaticPhysAddrs)
+        {
+            // Dynamic BAR1 requires RMAPI and GPU lock to be held
+            NV_ASSERT(rmapiLockIsOwner());
+            NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
+
+            NV_ASSERT_OK_OR_GOTO(rmStatus,
+                serverGetClientUnderLock(&g_resServ, hClient, &pClient),
+                Done);
+
+            NV_ASSERT_OK_OR_GOTO(rmStatus,
+                deviceGetByGpu(pClient, pGpu, NV_TRUE, &pDevice),
+                Done);
+        }
+
+        NV_CHECK_OR_GOTO(LEVEL_ERROR,
+                         memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM,
+                         Done);
+
+        //
+        // We don't take locks for static BAR1.
+        // Thus, we operate directly on the memdesc passed to
+        // bypass the client lookup which requires client locks.
+        //
+        NV_ASSERT_OK_OR_GOTO(rmStatus,
+            kbusMapFbAperture_HAL(pGpu, pKernelBus, pMemDesc, memRange, pMemArea,
+                    (BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_ALLOW_DISCONTIG),
+                    pDevice),
+            Done);
+
+        bForcePcie = (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE);
+
+        NV_ASSERT_OK_OR_GOTO(rmStatus,
+                             kbusGetGpuFbPhysAddressForRdma(pGpu, pKernelBus,
+                                                            bForcePcie, &barOffset),
+                             Done);
+
+        for (idx = 0; idx < pMemArea->numRanges; idx++)
+        {
+            pMemArea->pRanges[idx].start += barOffset;
+        }
+    }
+
+Done:
+    if (pThreadState != NULL)
+        threadStateFree(pThreadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+//
+// Unmaps a handle that was mapped to system physical addresses.
+// Must be called with API lock and GPU lock held for dynamic BAR1.
+//
+void NV_API_CALL rm_dma_buf_unmap_mem_handle(
+    nvidia_stack_t        *sp,
+    nv_state_t            *nv,
+    NvHandle               hClient,
+    NvHandle               hMemory,
+    NvU8                   mappingType,
+    void                  *pMemInfo,
+    NvBool                 bStaticPhysAddrs,
+    MemoryArea             memArea
+)
+{
+    THREAD_STATE_NODE threadState;
+    OBJGPU *pGpu;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+    if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) ||
+        pGpu->pGpuArch->bGpuArchIsZeroFb) &&
+        (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT))
+    {
+        os_free_mem(memArea.pRanges);
+    }
+    else
+    {
+        KernelBus *pKernelBus;
+        NvU64 idx;
+        NvU64 barOffset;
+        MEMORY_DESCRIPTOR *pMemDesc = (MEMORY_DESCRIPTOR *) pMemInfo;
+        NvBool bForcePcie = (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE);
+
+        pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+
+        NV_ASSERT_OK(kbusGetGpuFbPhysAddressForRdma(pGpu, pKernelBus,
+                                                    bForcePcie, &barOffset));
+
+        for (idx = 0; idx < memArea.numRanges; idx++)
+        {
+            memArea.pRanges[idx].start -= barOffset;
+        }
+
+        if (!bStaticPhysAddrs)
+        {
+            // Dynamic BAR1 requires RMAPI and GPU lock to be held
+            NV_ASSERT(rmapiLockIsOwner());
+            NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
+        }
+
+        //
+        // We don't take locks for static BAR1.
+        // Thus, we operate directly on the memdesc passed to
+        // bypass the client lookup which requires client locks.
+        //
+        NV_ASSERT_OK(kbusUnmapFbAperture_HAL(pGpu, pKernelBus, pMemDesc, memArea,
+                                             BUS_MAP_FB_FLAGS_MAP_UNICAST));
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(
+    nvidia_stack_t *sp,
+    nv_state_t     *nv,
+    NvHandle        hClient,
+    NvHandle        hMemory,
+    NvU8            mappingType,
+    NvHandle       *phClient,
+    NvHandle       *phDevice,
+    NvHandle       *phSubdevice,
+    void          **ppGpuInstanceInfo,
+    NvBool         *pbStaticPhysAddrs,
+    NvBool         *pbAcquireReleaseAllGpuLockOnDup
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+    if (rmStatus == NV_OK)
+    {
+        OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+        rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+        if (rmStatus == NV_OK)
+        {
+            rmStatus = RmDmabufGetClientAndDevice(pGpu, hClient, hMemory, mappingType,
+                                                  phClient, phDevice,
+                                                  phSubdevice, ppGpuInstanceInfo);
+            if (rmStatus == NV_OK)
+            {
+                *pbStaticPhysAddrs = ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) ||
+                                       pGpu->pGpuArch->bGpuArchIsZeroFb) &&
+                                      (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT)) ||
+                                      kbusIsStaticBar1Enabled(pGpu, GPU_GET_KERNEL_BUS(pGpu));
+            }
+
+            *pbAcquireReleaseAllGpuLockOnDup = pGpu->pGpuArch->bGpuArchIsZeroFb;
+
+            rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+        }
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return rmStatus;
+}
+
+void NV_API_CALL rm_dma_buf_put_client_and_device(
+    nvidia_stack_t *sp,
+    nv_state_t     *nv,
+    NvHandle        hClient,
+    NvHandle        hDevice,
+    NvHandle        hSubdevice,
+    void           *pGpuInstanceInfo
+)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+    if (rmStatus == NV_OK)
+    {
+        OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+
+        rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
+        if (rmStatus == NV_OK)
+        {
+            RmDmabufPutClientAndDevice(pGpu, hClient, hDevice, hSubdevice,
+                                       pGpuInstanceInfo);
+
+            rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+        }
+
+        // UNLOCK: release API lock
+        rmapiLockRelease();
+    }
+    NV_ASSERT_OK(rmStatus);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+//
+// Fetches GSP ucode data for usage during RM Init
+// NOTE: Used only on VMWware
+//
+
+void NV_API_CALL rm_vgpu_vfio_set_driver_vm(
+    nvidia_stack_t *sp,
+    NvBool is_driver_vm
+)
+{
+    OBJSYS *pSys;
+    OBJHYPERVISOR *pHypervisor;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pSys = SYS_GET_INSTANCE();
+    pHypervisor = SYS_GET_HYPERVISOR(pSys);
+
+    pHypervisor->setProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED, is_driver_vm);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NvBool NV_API_CALL rm_is_altstack_in_use(void)
+{
+#if defined(__use_altstack__)
+    return NV_TRUE;
+#else
+    return NV_FALSE;
+#endif
+}
+
+void NV_API_CALL rm_acpi_nvpcf_notify(
+    nvidia_stack_t *sp
+)
+{
+    void              *fp;
+    THREAD_STATE_NODE threadState;
+    NV_STATUS         rmStatus = NV_OK;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // LOCK: acquire API lock
+    if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE,
+                                     RM_LOCK_MODULES_EVENT)) == NV_OK)
+    {
+        OBJGPU *pGpu = gpumgrGetGpu(0);
+        if (pGpu != NULL)
+        {
+            nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+            if ((rmStatus = os_ref_dynamic_power(nv, NV_DYNAMIC_PM_FINE)) ==
+                                                                         NV_OK)
+            {
+               gpuNotifySubDeviceEvent(pGpu, NV2080_NOTIFIERS_NVPCF_EVENTS,
+                                       NULL, 0, 0, 0);
+            }
+            os_unref_dynamic_power(nv, NV_DYNAMIC_PM_FINE);
+        }
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+static void rm_notify_gpu_addition_removal_helper(
+    nv_state_t *nv,
+    NvBool bBind)
+{
+    THREAD_STATE_NODE threadState;
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK)
+    {
+        NV0000_CTRL_SYSTEM_EVENT_DATA_GPU_BIND_UNBIND eventData = { 0 };
+
+        eventData.gpuId = nv->gpu_id;
+        eventData.bBind = bBind;
+
+        CliAddSystemEvent(NV0000_NOTIFIERS_GPU_BIND_UNBIND_EVENT, &eventData, NULL);
+        rmapiLockRelease();
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_ERROR, "Fail to acquire rmApi lock. Skip notification.");
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+}
+
+void NV_API_CALL rm_notify_gpu_addition(
+    nvidia_stack_t *sp,
+    nv_state_t *nv)
+{
+    void *fp = NULL;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rm_notify_gpu_addition_removal_helper(nv, NV_TRUE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_notify_gpu_removal(
+    nvidia_stack_t *sp,
+    nv_state_t *nv)
+{
+    void *fp = NULL;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rm_notify_gpu_addition_removal_helper(nv, NV_FALSE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NvBool NV_API_CALL rm_wait_for_bar_firewall(
+    nvidia_stack_t * sp,
+    NvU32 domain,
+    NvU8  bus,
+    NvU8  device,
+    NvU8  function,
+    NvU16 devId,
+    NvU16 subsystemId
+)
+{
+    NvBool ret;
+    void *fp = NULL;
+
+    // no state set up yet for threadstate or RM locks
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    ret = gpumgrWaitForBarFirewall(domain, bus, device, function, devId, subsystemId);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return ret;
+}
+
+static NvU32 devfreq_clk_to_domain(
+    TEGRASOC_DEVFREQ_CLK devfreqClk
+)
+{
+    NvU32 clkDomain = NV2080_CTRL_CLK_DOMAIN_TEGRA_UNDEFINED;
+    switch (devfreqClk)
+    {
+        case TEGRASOC_DEVFREQ_CLK_GPC:
+            clkDomain = NV2080_CTRL_CLK_DOMAIN_TEGRA_GPCCLK;
+            break;
+
+        case TEGRASOC_DEVFREQ_CLK_NVD:
+            clkDomain = NV2080_CTRL_CLK_DOMAIN_TEGRA_NVDCLK;
+            break;
+
+        default:
+            break;
+    }
+
+    return clkDomain;
+}
+
+NV_STATUS NV_API_CALL rm_pmu_perfmon_get_load(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    NvU32 *load,
+    TEGRASOC_DEVFREQ_CLK devfreqClk
+)
+{
+    NV2080_CTRL_PERF_GET_TEGRA_PERFMON_SAMPLE_PARAMS params = { 0 };
+    NvU32 clkDomain = devfreq_clk_to_domain(devfreqClk);
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    RM_API *pRmApi;
+    NV_STATUS status;
+    void *fp;
+
+    if (clkDomain == NV2080_CTRL_CLK_DOMAIN_TEGRA_UNDEFINED)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    if (nvp->dynamic_power.state == NV_DYNAMIC_POWER_STATE_IDLE_INDICATED)
+    {
+        *load = 0;
+        return NV_OK;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI) == NV_OK)
+    {
+        pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+        if (pRmApi == NULL)
+        {
+            status = NV_ERR_INVALID_STATE;
+            rmapiLockRelease();
+            goto rm_pmu_perfmon_get_load_exit;
+        }
+
+        params.clkDomain = clkDomain;
+        status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice,
+                                 NV2080_CTRL_CMD_PERF_GET_TEGRA_PERFMON_SAMPLE,
+                                 &params, sizeof(params));
+        if (status == NV_OK)
+        {
+            *load = params.clkPercentBusy;
+        }
+
+        rmapiLockRelease();
+    }
+    else
+    {
+        status = NV_ERR_INVALID_STATE;
+    }
+
+rm_pmu_perfmon_get_load_exit:
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    return status;
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osinit.c b/userspace/sdks/nv570/nvalloc/unix/src/osinit.c
new file mode 100644
index 0000000..b3fd343
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/osinit.c
@@ -0,0 +1,2768 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**************************************************************************************************************
+*
+*   Description:
+*       UNIX-general, device-independent initialization code for
+*       the resource manager.
+*
+*
+**************************************************************************************************************/
+
+#include <nv_ref.h>
+#include <nv.h>                 // NV device driver interface
+#include <nv-reg.h>
+#include <nv-priv.h>
+#include <nvos.h>
+#include <nvrm_registry.h>
+#include <platform/sli/sli.h>
+#include <nverror.h>
+#include <osapi.h>
+#include <core/system.h>
+#include <os/os.h>
+#include "gpu/gpu.h"
+#include <gpu/timer/objtmr.h>
+#include "gpu/bus/kern_bus.h"
+#include "nverror.h"
+#include <nv-nb-regs.h>
+#include <gpu/bif/kernel_bif.h>
+#include <gpu/mem_mgr/mem_mgr.h>
+#include <gpu/mem_sys/kern_mem_sys.h>
+#include "kernel/gpu/intr/intr.h"
+#include <mem_mgr/io_vaspace.h>
+#include <gpu/disp/kern_disp.h>
+#include <platform/chipset/chipset.h>
+#include <kernel/gpu/rc/kernel_rc.h>
+#include <kernel/gpu/fifo/kernel_fifo.h>
+#include <nv-firmware-chip-family-select.h>
+#include <kernel/gpu/gsp/kernel_gsp.h>
+#include "liblogdecode.h"
+
+#include <mem_mgr/virt_mem_mgr.h>
+#include <virtualization/kernel_vgpu_mgr.h>
+
+#include <rmosxfac.h>
+#include <gpu_mgr/gpu_mgr.h>
+#include <core/thread_state.h>
+#include <core/locks.h>
+#include <rmapi/client.h>
+#include <ctrl/ctrl2080/ctrl2080gpu.h>
+#include <class/cl00f2.h>
+
+#include <platform/chipset/pci_pbi.h>
+
+#include "platform/nbsi/nbsi_read.h"
+#include "gpu_mgr/gpu_db.h"
+#include <class/cl0080.h>
+#include <class/cl0073.h>
+#include <class/cl2080.h>
+#include <class/cl402c.h>
+#include <ctrl/ctrl2080/ctrl2080gsp.h>
+
+#include <gpu/dce_client/dce_client.h>
+// RMCONFIG: need definition of REGISTER_ALL_HALS()
+#include "g_hal_register.h"
+
+typedef enum
+{
+   RM_INIT_OK,
+
+   /* general os errors */
+   RM_INIT_REG_SETUP_FAILED            =  0x10,
+   RM_INIT_SYS_ENVIRONMENT_FAILED,
+
+   /* gpu errors */
+   RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED =  0x20,
+   RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED,
+   RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED,
+   RM_INIT_GPU_PRE_INIT_FAILED,
+   RM_INIT_GPU_STATE_INIT_FAILED,
+   RM_INIT_GPU_LOAD_FAILED,
+   RM_INIT_GPU_DMA_CONFIGURATION_FAILED,
+   RM_INIT_GPU_GPUMGR_EXPANDED_VISIBILITY_FAILED,
+
+   /* vbios errors */
+   RM_INIT_VBIOS_FAILED                =  0x30,
+   RM_INIT_VBIOS_POST_FAILED,
+   RM_INIT_VBIOS_X86EMU_FAILED,
+
+   /* scalability errors */
+   RM_INIT_SCALABILITY_FAILED          =  0x40,
+
+   /* general core rm errors */
+   RM_INIT_WATCHDOG_FAILED,
+   RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED,
+   RM_GPUDB_REGISTER_FAILED,
+
+   RM_INIT_ALLOC_RMAPI_FAILED,
+   RM_INIT_GPUINFO_WITH_RMAPI_FAILED,
+
+   /* rm firmware errors */
+   RM_INIT_FIRMWARE_POLICY_FAILED      = 0x60,
+   RM_INIT_FIRMWARE_FETCH_FAILED,
+   RM_INIT_FIRMWARE_INIT_FAILED,
+
+   RM_INIT_MAX_FAILURES
+} rm_init_status;
+
+typedef rm_init_status RM_INIT_STATUS;
+
+typedef struct {
+    RM_INIT_STATUS initStatus;
+    NV_STATUS      rmStatus;
+    NvU32          line;
+} UNIX_STATUS;
+
+#define INIT_UNIX_STATUS   { RM_INIT_OK, NV_OK, 0 }
+#define RM_INIT_SUCCESS(init)  ((init) == RM_INIT_OK)
+
+#define RM_SET_ERROR(status, err)  { (status).initStatus = (err); \
+                                     (status).line = __LINE__; }
+
+
+static inline NvU64 nv_encode_pci_info(nv_pci_info_t *pci_info)
+{
+    return gpuEncodeDomainBusDevice(pci_info->domain, pci_info->bus, pci_info->slot);
+}
+
+static inline NvU32 nv_generate_id_from_pci_info(nv_pci_info_t *pci_info)
+{
+    return gpuGenerate32BitId(pci_info->domain, pci_info->bus, pci_info->slot);
+}
+
+static void nv_set_probed_gpu_flags(nv_state_t *nv)
+{
+    NvU32 flags = 0;
+
+    if (NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        flags |= DRF_DEF(0000, _CTRL_GPU_PROBED_ID_FLAGS, _SOC_DISPLAY, _TRUE);
+    }
+
+    gpumgrSetProbedFlags(nv->gpu_id, flags);
+}
+
+static inline void nv_os_map_kernel_space(nv_state_t *nv, nv_aperture_t *aperture)
+{
+    NV_ASSERT(aperture->map == NULL);
+
+    // let's start off assuming a standard device and map the registers
+    // normally. It is unfortunate to hard-code the register size here, but we don't
+    // want to fail trying to map all of a multi-devices' register space
+    aperture->map = osMapKernelSpace(aperture->cpu_address,
+                                     aperture->size,
+                                     NV_MEMORY_UNCACHED,
+                                     NV_PROTECT_READ_WRITE);
+    aperture->map_u = (nv_phwreg_t)aperture->map;
+}
+
+// local prototypes
+static NV_STATUS   initCoreLogic(OBJGPU *);
+static void        initVendorSpecificRegistry(OBJGPU *, NvU16);
+static NV_STATUS   teardownCoreLogic(OBJOS *, OBJGPU *);
+static void        initUnixSpecificRegistry(OBJGPU *);
+
+NV_STATUS osRmInitRm(void)
+{
+    OBJSYS    *pSys = SYS_GET_INSTANCE();
+    NV_STATUS  status;
+
+    if (os_is_efi_enabled())
+    {
+        pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_TRUE);
+    }
+
+    // have to init this before the debug subsystem, which will
+    // try to check the value of ResmanDebugLevel
+    RmInitRegistry();
+
+    NvU32 data = 0;
+    if ((osReadRegistryDword(NULL,
+                NV_REG_ENABLE_PCIE_GEN3, &data) == NV_OK) &&
+            (data != 0))
+    {
+        data = DRF_DEF(_REG_STR, _RM_PCIE, _LINK_SPEED_ALLOW_GEN3, _ENABLE);
+
+        osWriteRegistryDword(NULL,
+                NV_REG_STR_RM_PCIE_LINK_SPEED, data);
+    }
+
+    if (!(osReadRegistryDword(NULL,
+                NV_REG_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS,
+                &data) == NV_OK) || data)
+    {
+        pSys->setProperty(pSys,
+                PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS,
+                NV_TRUE);
+    }
+
+    if ((osReadRegistryDword(NULL,
+                NV_REG_ENABLE_STREAM_MEMOPS,
+                &data) == NV_OK) && (data != 0))
+    {
+        data = DRF_DEF(_REG_STR, _RM_STREAM_MEMOPS, _ENABLE, _YES);
+        osWriteRegistryDword(NULL, NV_REG_STR_RM_STREAM_MEMOPS, data);
+    }
+
+    if ((osReadRegistryDword(NULL,
+                NV_REG_STR_RM_FORCE_BAR_PATH,
+                &data) == NV_OK) && (data == 1))
+    {
+        //
+        // Exposing the GPU memory as NUMA node memory requires coherent CPU
+        // mappings over NVLink. If those mappings are disabled, also disable
+        // NUMA-onlining of the device memory.
+        //
+        osWriteRegistryDword(NULL,
+                NV_REG_ENABLE_USER_NUMA_MANAGEMENT, 0);
+    }
+
+    if ((osReadRegistryDword(NULL,
+                NV_REG_NVLINK_DISABLE, &data) == NV_OK) &&
+            (data == 1))
+    {
+        osWriteRegistryDword(NULL, NV_REG_STR_RM_NVLINK_CONTROL, 0x1);
+    }
+
+    if ((osReadRegistryDword(NULL,
+                NV_REG_ENABLE_PCIE_RELAXED_ORDERING_MODE, &data) == NV_OK) &&
+            (data == 1))
+    {
+        osWriteRegistryDword(NULL,
+                             NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING,
+                             NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING_ENABLE);
+    }
+
+    // init the debug subsystem if necessary
+    os_dbg_init();
+    nvDbgInitRmMsg(NULL);
+
+    // Force nvlog reinit since module params are now available
+    NVLOG_UPDATE();
+
+    // Register all supported hals
+    status = REGISTER_ALL_HALS();
+    if (status != NV_OK)
+    {
+        RmDestroyRegistry(NULL);
+        return status;
+    }
+
+    rm_check_s0ix_regkey_and_platform_support();
+
+    // Setup any ThreadState defaults
+    threadStateInitSetupFlags(THREAD_STATE_SETUP_FLAGS_ENABLED |
+                              THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED |
+                              THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED |
+                              THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED);
+
+    return NV_OK;
+}
+
+void RmShutdownRm(void)
+{
+    NV_PRINTF(LEVEL_INFO, "shutdown rm\n");
+
+    RmDestroyRegistry(NULL);
+
+    // Free objects created with RmInitRm, including the system object
+    RmDestroyRm();
+}
+
+//
+// osAttachGpu
+//
+// This routine is used as a callback by the gpumgrAttachGpu
+// interface to allow os-dependent code to set up any state
+// before engine construction begins.
+//
+NV_STATUS osAttachGpu(
+    OBJGPU    *pGpu,
+    void      *pOsGpuInfo
+)
+{
+    nv_state_t *nv = (nv_state_t *)pOsGpuInfo;
+    nv_priv_t  *nvp;
+
+    nvp = NV_GET_NV_PRIV(nv);
+
+    nvp->pGpu = pGpu;
+
+    NV_SET_NV_STATE(pGpu, (void *)nv);
+
+    initUnixSpecificRegistry(pGpu);
+
+    // Assign default values to Registry keys for VGX
+    if (os_is_vgx_hyper())
+    {
+        initVGXSpecificRegistry(pGpu);
+    }
+
+    return NV_OK;
+}
+
+NV_STATUS osDpcAttachGpu(
+    OBJGPU    *pGpu,
+    void      *pOsGpuInfo
+)
+{
+    return NV_OK; // Nothing to do for unix
+}
+
+void osDpcDetachGpu(
+    OBJGPU    *pGpu
+)
+{
+    return; // Nothing to do for unix
+}
+
+NV_STATUS
+osHandleGpuLost
+(
+    OBJGPU *pGpu,
+    NvBool  bEmitXid
+)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    NvU32 pmc_boot_0;
+
+    // Determine if we've already run the handler
+    if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED))
+    {
+        return NV_OK;
+    }
+
+    pmc_boot_0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0);
+    if (pmc_boot_0 != nvp->pmc_boot_0)
+    {
+        //
+        // This doesn't support PEX Reset and Recovery yet.
+        // This will help to prevent accessing registers of a GPU
+        // which has fallen off the bus.
+        //
+        if (bEmitXid)
+        {
+            nvErrorLog_va((void *)pGpu, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS,
+                          "GPU has fallen off the bus.");
+        }
+
+        gpuNotifySubDeviceEvent(pGpu, NV2080_NOTIFIERS_GPU_UNAVAILABLE, NULL,
+                                0, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, 0);
+
+        NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU has fallen off the bus.\n");
+
+        if (pGpu->boardInfo != NULL && pGpu->boardInfo->serialNumber[0] != '\0')
+        {
+            NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
+                          "GPU serial number is %s.\n",
+                          pGpu->boardInfo->serialNumber);
+        }
+
+        gpuSetDisconnectedProperties(pGpu);
+
+        if (IS_GSP_CLIENT(pGpu))
+        {
+            KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu);
+
+            // Notify all channels of the error so that UVM can fail gracefully
+            if (pKernelRc != NULL)
+            {
+                krcRcAndNotifyAllChannels(pGpu, pKernelRc, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, NV_FALSE);
+            }
+        }
+
+        // Initiate a crash dump immediately.
+        RmLogGpuCrash(pGpu);
+
+        // Set SURPRISE_REMOVAL flag for eGPU to help in device removal.
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_EXTERNAL_GPU))
+        {
+            nv->flags |= NV_FLAG_IN_SURPRISE_REMOVAL;
+        }
+
+        DBG_BREAKPOINT();
+    }
+
+    return NV_OK;
+}
+
+/*!
+ * @brief Traverse bus topology till Gpu's root port.
+ * If any of the intermediate bridge has TB3 supported vendorId and hotplug
+ * capability(not necessarily same bridge), mark the Gpu as External Gpu.
+ *
+ * @params[in]    pGpu    OBJGPU pointer
+ * @params[in]    pCl     OBJCL pointer
+ *
+* @return NV_OK
+*      Identified to be eGPU
+* @return others
+ *     Not an eGPU / error on identfying
+ *
+ */
+NvBool
+RmCheckForExternalGpu
+(
+    OBJGPU *pGpu,
+    OBJCL *pCl
+)
+{
+    NvU8 bus;
+    NvU32 domain;
+    void *handleUp;
+    NvU8 busUp, devUp, funcUp;
+    NvU16 vendorIdUp, deviceIdUp;
+    NvU32 portCaps, pciCaps, slotCaps;
+    NvU32 PCIECapPtr;
+    RM_API *pRmApi;
+    NV_STATUS status, rmStatus;
+    NvBool bTb3Bridge = NV_FALSE, bSlotHotPlugSupport = NV_FALSE;
+    NvBool iseGPUBridge = NV_FALSE;
+
+    pRmApi  = GPU_GET_PHYSICAL_RMAPI(pGpu);
+    domain  = gpuGetDomain(pGpu);
+    bus     = gpuGetBus(pGpu);
+    do
+    {
+        // Find the upstream bridge
+        handleUp = clFindP2PBrdg(pCl, domain, bus, &busUp, &devUp, &funcUp, &vendorIdUp, &deviceIdUp);
+        if (!handleUp)
+        {
+            return iseGPUBridge;
+        }
+
+        if (vendorIdUp == PCI_VENDOR_ID_INTEL)
+        {
+            // Check for the supported TB3(ThunderBolt 3) bridges.
+            NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS params = { 0 };
+
+            // LOCK: acquire GPUs lock
+            rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE,
+                                         RM_LOCK_MODULES_INIT);
+            if (rmStatus != NV_OK)
+            {
+                return iseGPUBridge;
+            }
+            params.pciDeviceId = deviceIdUp;
+            status = pRmApi->Control(pRmApi,
+                                     pGpu->hInternalClient,
+                                     pGpu->hInternalSubdevice,
+                                     NV2080_CTRL_CMD_INTERNAL_GET_EGPU_BRIDGE_INFO,
+                                     &params,
+                                     sizeof(params));
+            // UNLOCK: release GPUs lock
+            rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+            if (status != NV_OK)
+            {
+                NV_PRINTF(LEVEL_ERROR,
+                          "Error 0x%08x on eGPU Approval for Bridge ID: 0x%08x\n", status, deviceIdUp);
+                DBG_BREAKPOINT();
+                return iseGPUBridge;
+            }
+            else
+            {
+                // Check for the approved eGPU BUS TB3
+                if (params.iseGPUBridge &&
+                    params.approvedBusType == NV2080_CTRL_INTERNAL_EGPU_BUS_TYPE_TB3)
+                {
+                    bTb3Bridge =  NV_TRUE;
+                }
+            }
+        }
+
+        if (NV_OK != clSetPortPcieCapOffset(pCl, handleUp, &PCIECapPtr))
+        {
+            // PCIE bridge but no cap pointer.
+            break;
+        }
+
+        // Get the PCIE capabilities.
+        pciCaps = osPciReadDword(handleUp, CL_PCIE_CAP - CL_PCIE_BEGIN + PCIECapPtr);
+        if (CL_PCIE_CAP_SLOT & pciCaps)
+        {
+            // Get the slot capabilities.
+            slotCaps = osPciReadDword(handleUp, CL_PCIE_SLOT_CAP - CL_PCIE_BEGIN + PCIECapPtr);
+
+            if ((CL_PCIE_SLOT_CAP_HOTPLUG_CAPABLE & slotCaps) &&
+                (CL_PCIE_SLOT_CAP_HOTPLUG_SURPRISE & slotCaps))
+            {
+                bSlotHotPlugSupport = NV_TRUE;
+            }
+        }
+
+        if (bTb3Bridge && bSlotHotPlugSupport)
+        {
+            iseGPUBridge = NV_TRUE;
+            break;
+        }
+
+        bus = busUp;
+
+        // Get port caps to check if PCIE bridge is the root port
+        portCaps = osPciReadDword(handleUp, CL_PCIE_CAP - CL_PCIE_BEGIN + PCIECapPtr);
+
+    } while (!CL_IS_ROOT_PORT(portCaps));
+    return iseGPUBridge;
+}
+
+static NV_STATUS
+RmGetFirmwareVersion
+(
+    nv_state_t *pNv,
+    RM_API     *pRmApi
+)
+{
+    NV2080_CTRL_GSP_GET_FEATURES_PARAMS gsp_params;
+    NV_STATUS          rmStatus = NV_OK;
+    pNv->cached_gpu_info.firmware_version[0] = '\0';
+
+    rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hSubDevice,
+                               NV2080_CTRL_CMD_GSP_GET_FEATURES, &gsp_params,
+                               sizeof(gsp_params));
+
+    if (rmStatus != NV_OK)
+    {
+        if (RMCFG_FEATURE_GSP_CLIENT_RM && pNv->request_firmware)
+        {
+            const char *pTmpString = "N/A";
+            portStringCopy(pNv->cached_gpu_info.firmware_version,
+                           sizeof(pNv->cached_gpu_info.firmware_version),
+                           pTmpString, portStringLength(pTmpString) + 1);
+        }
+        NV_PRINTF(LEVEL_INFO,
+                  "%s: Failed to query gpu firmware version, status=0x%x\n",
+                  __FUNCTION__, rmStatus);
+    }
+    else
+    {
+        portMemCopy(pNv->cached_gpu_info.firmware_version,
+                    sizeof(pNv->cached_gpu_info.firmware_version),
+                    gsp_params.firmwareVersion, sizeof(gsp_params.firmwareVersion));
+    }
+
+    return NV_OK;
+}
+
+static NV_STATUS
+RmGetVbiosVersion
+(
+    nv_state_t *pNv,
+    RM_API     *pRmApi
+)
+{
+    NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS *vbios_params;
+    const size_t vbiosStringLen = 15; // "xx.xx.xx.xx.xx"
+    NV_STATUS rmStatus = NV_OK;
+
+    vbios_params = portMemAllocNonPaged(sizeof(*vbios_params));
+    if (vbios_params == NULL)
+    {
+        return NV_ERR_NO_MEMORY;;
+    }
+
+    portMemSet(vbios_params, 0, sizeof(*vbios_params));
+
+    vbios_params->biosInfoList[0].index = NV2080_CTRL_BIOS_INFO_INDEX_REVISION;
+    vbios_params->biosInfoList[1].index = NV2080_CTRL_BIOS_INFO_INDEX_OEM_REVISION;
+    vbios_params->biosInfoListSize      = 2;
+
+    rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient,
+                               pNv->rmapi.hSubDevice, NV2080_CTRL_CMD_BIOS_GET_INFO_V2,
+                               vbios_params, sizeof(*vbios_params));
+
+    if (rmStatus == NV_OK)
+    {
+        const NvU32 biosRevision = vbios_params->biosInfoList[0].data;
+        const NvU32 biosOEMRevision = vbios_params->biosInfoList[1].data;
+
+        os_snprintf(pNv->cached_gpu_info.vbios_version, vbiosStringLen,
+                    "%02x.%02x.%02x.%02x.%02x",
+                    (biosRevision & 0xff000000) >> 24,
+                    (biosRevision & 0x00ff0000) >> 16,
+                    (biosRevision & 0x0000ff00) >>  8,
+                    (biosRevision & 0x000000ff) >>  0,
+                    biosOEMRevision);
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_INFO,
+                  "%s: Failed to query vbios version, status=0x%x\n",
+                  __FUNCTION__, rmStatus);
+    }
+    portMemFree(vbios_params);
+
+    return NV_OK;
+}
+
+/*
+ * Initialize the required GPU information by doing RMAPI control calls
+ * and store the same in the UNIX specific data structures.
+ */
+static NV_STATUS
+RmInitGpuInfoWithRmApi
+(
+    OBJGPU *pGpu
+)
+{
+    RM_API     *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    nv_state_t *nv     = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp    = NV_GET_NV_PRIV(nv);
+    NV2080_CTRL_GPU_GET_INFO_V2_PARAMS *pGpuInfoParams = { 0 };
+    NV_STATUS   status;
+    const NvU8  *uuid;
+
+    // LOCK: acquire GPUs lock
+    status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT);
+    if (status != NV_OK)
+    {
+        return status;
+    }
+
+    pGpuInfoParams = portMemAllocNonPaged(sizeof(*pGpuInfoParams));
+    if (pGpuInfoParams == NULL)
+    {
+        // UNLOCK: release GPUs lock
+        rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+        return NV_ERR_NO_MEMORY;
+    }
+
+
+    portMemSet(pGpuInfoParams, 0, sizeof(*pGpuInfoParams));
+
+    pGpuInfoParams->gpuInfoListSize = 3;
+    pGpuInfoParams->gpuInfoList[0].index = NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED;
+    pGpuInfoParams->gpuInfoList[1].index = NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY;
+    pGpuInfoParams->gpuInfoList[2].index = NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE;
+
+    status = pRmApi->Control(pRmApi, nv->rmapi.hClient,
+                             nv->rmapi.hSubDevice,
+                             NV2080_CTRL_CMD_GPU_GET_INFO_V2,
+                             pGpuInfoParams, sizeof(*pGpuInfoParams));
+
+    if (status == NV_OK)
+    {
+        nvp->b_mobile_config_enabled =
+            (pGpuInfoParams->gpuInfoList[0].data ==
+             NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_YES);
+        nv->dma_buf_supported =
+            (pGpuInfoParams->gpuInfoList[1].data ==
+             NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES);
+    }
+
+    nv->coherent =
+        (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) ||
+         pGpu->pGpuArch->bGpuArchIsZeroFb);
+
+    //
+    // If coherent GPU memory mode is NONE, then GPU memory has struct page
+    // on coherent platforms and no struct page on non-coherent ones.
+    // If the mode is enabled and _NUMA, then GPU memory has struct page.
+    // If the mode is enabled and _DRIVER, then GPU memory doesn't have struct page.
+    // Tegra iGPU also falls under struct page category, although
+    // COHERENT_GPU_MEMORY_MODE doesn't apply there.
+    //
+    if (pGpuInfoParams->gpuInfoList[2].data ==
+        NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NONE)
+    {
+        nv->mem_has_struct_page = nv->coherent;
+    }
+    else
+    {
+        // If mode is not _NONE, we're already on a PDB_PROP_GPU_COHERENT_CPU_MAPPING platform.
+        nv->mem_has_struct_page = pGpu->pGpuArch->bGpuArchIsZeroFb ||
+                                  (pGpuInfoParams->gpuInfoList[2].data ==
+                                   NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NUMA);
+    }
+
+    switch (pGpuInfoParams->gpuInfoList[2].data)
+    {
+        case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NONE:
+            nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_NONE;
+            break;
+        case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NUMA:
+            nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_NUMA;
+            break;
+        case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_DRIVER:
+            nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_DRIVER;
+            break;
+    }
+
+    portMemFree(pGpuInfoParams);
+
+    if (status != NV_OK)
+    {
+        goto finish;
+    }
+
+    uuid = RmGetGpuUuidRaw(nv);
+    if (uuid == NULL)
+    {
+        status = NV_ERR_GPU_UUID_NOT_FOUND;
+        goto finish;
+    }
+
+    status = RmGetFirmwareVersion(nv, pRmApi);
+    if (status != NV_OK)
+    {
+        goto finish;
+    }
+
+    status = RmGetVbiosVersion(nv, pRmApi);
+
+finish:
+    // UNLOCK: release GPUs lock
+    rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+    return status;
+}
+
+static void RmSetSocDispDeviceMappings(
+    GPUATTACHARG *gpuAttachArg,
+    nv_state_t *nv
+)
+{
+    gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvAddr = (GPUHWREG*) nv->regs->map;
+    gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvPAddr = nv->regs->cpu_address;
+    gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvLength = (NvU32) nv->regs->size;
+}
+
+static void RmSetSocDpauxDeviceMappings(
+    GPUATTACHARG *gpuAttachArg,
+    nv_state_t *nv
+)
+{
+}
+
+static void RmSetSocHdacodecDeviceMappings(
+    GPUATTACHARG *gpuAttachArg,
+    nv_state_t *nv
+)
+{
+}
+
+static void RmSetSocMipiCalDeviceMappings(
+    GPUATTACHARG *gpuAttachArg,
+    nv_state_t *nv
+)
+{
+}
+
+static void
+osInitNvMapping(
+    nv_state_t *nv,
+    NvU32 *pDeviceReference,
+    UNIX_STATUS *status
+)
+{
+    OBJGPU *pGpu;
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    GPUATTACHARG *gpuAttachArg;
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    NvU32 deviceInstance;
+    NvU32 data = 0;
+    NvU32 dispIsoStreamId;
+    NvU32 dispNisoStreamId;
+
+    NV_PRINTF(LEVEL_INFO, "osInitNvMapping:\n");
+
+    // allocate the next available gpu device number
+    status->rmStatus = gpumgrAllocGpuInstance(pDeviceReference);
+    if (status->rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "*** Cannot get valid gpu instance\n");
+        RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED);
+        return;
+    }
+
+    // RM_BASIC_LOCK_MODEL: allocate GPU lock
+    status->rmStatus = rmGpuLockAlloc(*pDeviceReference);
+    if (status->rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "*** cannot allocate GPU lock\n");
+        RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED);
+        return;
+    }
+
+    // attach default single-entry broadcast device for this gpu
+    status->rmStatus = gpumgrCreateDevice(&deviceInstance, NVBIT(*pDeviceReference), NULL);
+    if (status->rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "*** Cannot attach bc gpu\n");
+        RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED);
+        // RM_BASIC_LOCK_MODEL: free GPU lock
+        rmGpuLockFree(*pDeviceReference);
+        return;
+    }
+
+    // init attach state
+    gpuAttachArg = portMemAllocNonPaged(sizeof(GPUATTACHARG));
+    if (gpuAttachArg == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "*** Cannot allocate gpuAttachArg\n");
+        RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED);
+        // RM_BASIC_LOCK_MODEL: free GPU lock
+        rmGpuLockFree(*pDeviceReference);
+        return;
+    }
+
+    portMemSet(gpuAttachArg, 0, sizeof(GPUATTACHARG));
+
+    if (NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        gpuAttachArg->socDeviceArgs.specified = NV_TRUE;
+
+        RmSetSocDispDeviceMappings(gpuAttachArg, nv);
+
+        RmSetSocDpauxDeviceMappings(gpuAttachArg, nv);
+
+        RmSetSocHdacodecDeviceMappings(gpuAttachArg, nv);
+
+        RmSetSocMipiCalDeviceMappings(gpuAttachArg, nv);
+
+        gpuAttachArg->socDeviceArgs.socChipId0 = nv->disp_sw_soc_chip_id;
+
+        gpuAttachArg->socDeviceArgs.iovaspaceId = nv->iovaspace_id;
+    }
+    else
+    {
+        gpuAttachArg->fbPhysAddr      = nv->fb->cpu_address;
+        gpuAttachArg->fbBaseAddr      = (GPUHWREG*) 0; // not mapped
+        gpuAttachArg->devPhysAddr     = nv->regs->cpu_address;
+        gpuAttachArg->regBaseAddr     = (GPUHWREG*) nv->regs->map;
+        gpuAttachArg->intLine         = 0;             // don't know yet
+        gpuAttachArg->instPhysAddr    = nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address;
+        gpuAttachArg->instBaseAddr    = (GPUHWREG*) 0; // not mapped
+
+        gpuAttachArg->regLength       = nv->regs->size;
+        gpuAttachArg->fbLength        = nv->fb->size;
+        gpuAttachArg->instLength      = nv->bars[NV_GPU_BAR_INDEX_IMEM].size;
+
+        gpuAttachArg->iovaspaceId     = nv->iovaspace_id;
+        gpuAttachArg->cpuNumaNodeId   = nv->cpu_numa_node_id;
+
+        if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
+        {
+            // Default - PCIe GPUs are connected via NISO IOMMU
+            nv->iommus.iso_iommu_present  = NV_FALSE;
+            nv->iommus.niso_iommu_present = NV_TRUE;
+        }
+    }
+
+    //
+    // we need this to check if we are running on virtual GPU
+    // in gpuBindHal function later.
+    //
+    if (!NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        gpuAttachArg->nvDomainBusDeviceFunc = nv_encode_pci_info(&nv->pci_info);
+    }
+
+    gpuAttachArg->bRequestFwClientRm = nv->request_fw_client_rm;
+
+    gpuAttachArg->pOsAttachArg    = (void *)nv;
+
+    // use gpu manager to attach gpu
+    status->rmStatus = gpumgrAttachGpu(*pDeviceReference, gpuAttachArg);
+    portMemFree(gpuAttachArg);
+    if (status->rmStatus != NV_OK)
+    {
+        gpumgrDestroyDevice(deviceInstance);
+        RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED);
+        NV_PRINTF(LEVEL_ERROR, "*** Cannot attach gpu\n");
+        // RM_BASIC_LOCK_MODEL: free GPU lock
+        rmGpuLockFree(*pDeviceReference);
+        return;
+    }
+    nvp->flags |= NV_INIT_FLAG_GPUMGR_ATTACH;
+
+    pGpu = gpumgrGetGpu(*pDeviceReference);
+
+    sysInitRegistryOverrides(pSys);
+
+    sysApplyLockingPolicy(pSys);
+
+    pGpu->busInfo.IntLine = nv->interrupt_line;
+
+    if (nv->fb != NULL)
+    {
+        pGpu->registerAccess.gpuFbAddr = (GPUHWREG*) nv->fb->map;
+        pGpu->busInfo.gpuPhysFbAddr = nv->fb->cpu_address;
+    }
+
+    // set default parent gpu
+    gpumgrSetParentGPU(pGpu, pGpu);
+
+    NV_PRINTF(LEVEL_INFO, "device instance          : %d\n", *pDeviceReference);
+    NV_PRINTF(LEVEL_INFO, "NV regs using linear address  : 0x%p\n",
+              pGpu->deviceMappings[SOC_DEV_MAPPING_DISP].gpuNvAddr);
+    NV_PRINTF(LEVEL_INFO,
+              "NV fb using linear address  : 0x%p\n", pGpu->registerAccess.gpuFbAddr);
+
+    if (!os_is_vgx_hyper())
+    {
+        pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE);
+    }
+    else
+    {
+        {
+            pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE);
+        }
+    }
+
+    if ((osReadRegistryDword(NULL,
+                             NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS,
+                             &data) == NV_OK) && data)
+    {
+        if (data == NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_ENABLED)
+        {
+            nv->preserve_vidmem_allocations = NV_TRUE;
+        }
+        else if (data == NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_AUTO)
+        {
+            /* If the kernel supports suspend notifiers, enable video memory allocations */
+            nv->preserve_vidmem_allocations = os_supports_kernel_suspend_notifiers();
+        }
+
+        if (nv->preserve_vidmem_allocations && nv_dev_needs_vidmem_preservation(nv))
+        {
+            KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
+            MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+
+            memmgrSetPmaForcePersistence(pMemoryManager, NV_TRUE);
+
+            // Monolithic uses Fifolite channel which doesn't support Virtual mode
+            if (!pMemoryManager->bUseVirtualCopyOnSuspend)
+            {
+                pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = NV_TRUE;
+            }
+        }
+    }
+
+    // Check if SMMU can be enabled on PushBuffer Aperture
+    nv_get_disp_smmu_stream_ids(nv, &dispIsoStreamId, &dispNisoStreamId);
+    if (dispNisoStreamId != NV_U32_MAX)
+    {
+        pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_FALSE);
+    }
+    else
+    {
+        pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_TRUE);
+    }
+
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TRIGGER_PCIE_FLR))
+    {
+        nv->flags |= NV_FLAG_TRIGGER_FLR;
+    }
+
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_SOC_SDM))
+    {
+        nv->flags |= NV_FLAG_HAS_CONSOLE_IN_SYSMEM_CARVEOUT;
+    }
+}
+
+static NV_STATUS
+osInitScalability(
+    OBJGPU *pGpu
+)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJCL *pCl = SYS_GET_CL(pSys);
+
+    // We need PCI-E chipset information very early.
+    if (pCl != NULL)
+    {
+        return clInitPcie(pGpu, pCl);
+    }
+    else
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+}
+
+static NV_STATUS
+osTeardownScalability(
+    OBJGPU *pGpu
+)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJCL *pCl = SYS_GET_CL(pSys);
+
+    return clTeardownPcie(pGpu, pCl);
+}
+
+static NV_STATUS
+RmInitNvHal(
+    nv_state_t *nv,
+    NvU32 deviceReference,
+    UNIX_STATUS *status
+)
+{
+    OBJGPU             *pGpu      = gpumgrGetGpu(deviceReference);
+    nv_priv_t          *nvp       = NV_GET_NV_PRIV(nv);
+
+    PORT_UNREFERENCED_VARIABLE(pGpu);
+
+    nvp->flags |= NV_INIT_FLAG_HAL;
+
+    nvp->flags |= NV_INIT_FLAG_HAL_COMPONENTS;
+
+    return NV_OK;
+}
+
+static void
+RmDeterminePrimaryDevice(OBJGPU *pGpu)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+    // Skip updating nv->primary_vga while RM is recovering after GPU reset
+    if (nv->flags & NV_FLAG_IN_RECOVERY)
+    {
+        return;
+    }
+
+    //
+    // In case of Passthru, GPU will always be secondary
+    //
+    if (IS_PASSTHRU(pGpu))
+    {
+        return;
+    }
+
+    //
+    // In case of VIRTUAL GPU, there is no display, hence it will be secondary
+    //
+    if (IS_VIRTUAL(pGpu))
+    {
+        return;
+    }
+
+    //
+    // If GPU is driving any frame buffer console(vesafb, efifb etc)
+    // mark the console as client driven and GPU as Primary.
+    //
+    nv->client_managed_console = rm_get_uefi_console_status(nv);
+
+    pGpu->setProperty(pGpu, PDB_PROP_GPU_PRIMARY_DEVICE,
+                      (nv->client_managed_console || !!nv->primary_vga));
+}
+
+static void
+RmSetConsolePreservationParams(OBJGPU *pGpu)
+{
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    NvU64 fbBaseAddress = 0;
+    NvU64 fbConsoleSize = 0;
+    MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+    KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
+    //
+    // PDB_PROP_GPU_PRIMARY_DEVICE should be NV_FALSE for vGPU configuration so
+    // return early
+    //
+    if (os_is_vgx_hyper() || IS_VIRTUAL(pGpu))
+        return;
+
+    if (!gpuFuseSupportsDisplay_HAL(pGpu))
+    {
+        return;
+    }
+
+    //
+    // If this is the zero FB SOC GPU with a console in system carveout, then
+    // the console memory is not mapped onto the BAR. The kernel can directly
+    // access the console memory, while the GPU accesses system carveout memory
+    // through the SMMU.
+    // The carveout is a reserved region of system memory that doesn't get used
+    // in standard memory allocation.
+    //
+    // In this scenario, you don't need to reserve console memory or BAR mapping.
+    //
+    if (NV_HAS_CONSOLE_IN_SYSMEM_CARVEOUT(nv))
+    {
+        return;
+    }
+
+    //
+    // Check the OS layer for any video memory used by a console
+    // driver that should be reserved.
+    //
+    // If the console driver is using some amount of memory, its
+    // mapping is presumably either pointing at the start of BAR1
+    // (this is the mapping the EFI GOP driver provides for
+    // efifb), or the upper 16MB of BAR2 (this is the mapping the
+    // VBIOS provides for, e.g., vesafb).
+    //
+    // In the BAR2 case, there is nothing more to do: the upper
+    // 16MB of BAR2 will remain in physical mode
+    // (see the documentation for NV_PBUS_BAR2_BLOCK in dev_bus.ref),
+    // and the console driver can continue to access it.
+    //
+    // In the BAR1 case, RM will put all of BAR1 into virtual
+    // mode, so we need to create a BAR1 mapping for the console
+    // driver to use.
+    //
+    fbConsoleSize = rm_get_uefi_console_size(nv, &fbBaseAddress);
+
+    if ((fbConsoleSize > 0) && (fbBaseAddress != 0) && (pKernelBus != NULL))
+    {
+        pKernelBus->bPreserveBar1ConsoleEnabled =
+                          (fbBaseAddress == nv->fb->cpu_address);
+    }
+
+    //
+    // Case for legacy console, where OS will not give console size
+    // and primary_vga will be set for the device.
+    // Hence, reserving default size for VGA text modes.
+    //
+    if ((fbConsoleSize == 0) && nv->primary_vga)
+    {
+        fbConsoleSize = 0x40000;
+    }
+
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+    {
+        pMemoryManager->Ram.ReservedConsoleDispMemSize = fbConsoleSize;
+        pMemoryManager->Ram.ReservedConsoleDispMemBase = fbBaseAddress;
+    }
+    else
+    {
+        pMemoryManager->Ram.ReservedConsoleDispMemSize = NV_ALIGN_UP(fbConsoleSize, 0x10000);
+    }
+}
+
+static NV_STATUS
+RmInitDeviceDma(
+    nv_state_t *nv
+)
+{
+    if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
+    {
+        OBJSYS *pSys = SYS_GET_INSTANCE();
+        OBJVMM *pVmm = SYS_GET_VMM(pSys);
+        OBJVASPACE *pIOVAS;
+        NV_STATUS status = vmmCreateVaspace(pVmm, IO_VASPACE_A,
+                                            nv->iovaspace_id, 0, 0ULL, ~0ULL,
+                                            0ULL, 0ULL,
+                                            NULL, VASPACE_FLAGS_ENABLE_VMM,
+                                            &pIOVAS);
+        if (status != NV_OK)
+        {
+            return status;
+        }
+    }
+
+    return NV_OK;
+}
+
+static void
+RmTeardownDeviceDma(
+    nv_state_t *nv
+)
+{
+    if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
+    {
+        OBJSYS *pSys = SYS_GET_INSTANCE();
+        OBJVMM *pVmm = SYS_GET_VMM(pSys);
+        OBJVASPACE *pIOVAS;
+
+        if (NV_OK == vmmGetVaspaceFromId(pVmm, nv->iovaspace_id, IO_VASPACE_A, &pIOVAS))
+        {
+            vmmDestroyVaspace(pVmm, pIOVAS);
+        }
+    }
+}
+
+static void
+RmEnableDeviceClks(nv_state_t *nv)
+{
+    NvU32 freqKHz = 0;
+    NvU32 i;
+    NV_STATUS ret;
+
+    for (i = TEGRASOC_WHICH_CLK_GPU_FIRST; i <= TEGRASOC_WHICH_CLK_GPU_LAST; i++)
+    {
+        ret = nv_get_max_freq(nv, i, &freqKHz);
+        if (ret != NV_OK)
+        {
+            NV_PRINTF(LEVEL_INFO, "NVRM: Max Freq fetch failed for Clk:%d\n", i);
+            continue;
+        }
+
+        ret = nv_enable_clk(nv, i);
+        if (ret != NV_OK)
+        {
+            NV_PRINTF(LEVEL_INFO, "NVRM: Clk prepare enable failed for Clk:%d\n", i);
+            continue;
+        }
+
+        ret = nv_set_freq(nv, i, freqKHz);
+        if (ret != NV_OK)
+        {
+            NV_PRINTF(LEVEL_INFO, "NVRM: Set Freq failed for Clk:%d\n", i);
+        }
+        else
+        {
+            NV_PRINTF(LEVEL_INFO, "NVRM: Set Freq:%d for Clk:%d\n", freqKHz, i);
+        }
+    }
+}
+
+static void
+RmDisableDeviceClks(nv_state_t *nv)
+{
+    NvU32 i;
+
+    for (i = TEGRASOC_WHICH_CLK_GPU_FIRST; i <= TEGRASOC_WHICH_CLK_GPU_LAST; i++)
+    {
+        nv_disable_clk(nv, i);
+    }
+}
+
+static void
+RmInitNvDevice(
+    NvU32 deviceReference,
+    UNIX_STATUS *status
+)
+{
+    // set the device context
+    OBJGPU *pGpu = gpumgrGetGpu(deviceReference);
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJCL  *pCl  = SYS_GET_CL(pSys);
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+
+    NV_PRINTF(LEVEL_INFO, "RmInitNvDevice:\n");
+
+    NV_PRINTF(LEVEL_INFO,
+              "device instance          : 0x%08x\n", deviceReference);
+
+    // initialize all engines -- calls back osInitMapping()
+    status->rmStatus = gpumgrStatePreInitGpu(pGpu);
+    if (status->rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "*** Cannot pre-initialize the device\n");
+        RM_SET_ERROR(*status, RM_INIT_GPU_PRE_INIT_FAILED);
+        return;
+    }
+
+    // Configure eGPU setting
+    if (RmCheckForExternalGpu(pGpu, pCl))
+    {
+        pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_EXTERNAL_GPU, NV_TRUE);
+        nv->is_external_gpu = NV_TRUE;
+    }
+    status->rmStatus = gpumgrStateInitGpu(pGpu);
+    if (status->rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize the device\n");
+        RM_SET_ERROR(*status, RM_INIT_GPU_STATE_INIT_FAILED);
+        return;
+    }
+    nvp->flags |= NV_INIT_FLAG_GPU_STATE;
+
+    KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+    //
+    // Make sure bifIsMSIEnabled() gets a chance to look up the OS PCI
+    // handle for this GPU. This must happen after GPU state
+    // initialization.
+    //
+    if (pKernelBif != NULL)
+    {
+        kbifCheckAndRearmMSI(pGpu, pKernelBif);
+    }
+
+    // Set RM's interrupt enable state to zero here so that interrupts
+    // won't be enabled during loading
+    Intr *pIntr = GPU_GET_INTR(pGpu);
+    if (pIntr != NULL)
+    {
+        intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED);
+    }
+
+    status->rmStatus = gpumgrStateLoadGpu(pGpu, GPU_STATE_DEFAULT);
+    if (status->rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "*** Cannot load state into the device\n");
+        RM_SET_ERROR(*status, RM_INIT_GPU_LOAD_FAILED);
+        return;
+    }
+    nvp->flags |= NV_INIT_FLAG_GPU_STATE_LOAD;
+
+    // Setup GPU scalability
+    (void) RmInitScalability(pGpu);
+
+    return;
+}
+
+NV_STATUS osInitMapping(
+    OBJGPU *pGpu
+)
+{
+    OBJSYS     *pSys = SYS_GET_INSTANCE();
+    OBJCL      *pCl = SYS_GET_CL(pSys);
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+
+    NV_PRINTF(LEVEL_INFO, "osInitMapping:\n");
+
+    // Some PCI BIOSs leave the ROM mapped.  This causes problems if it overlays system RAM.  Just disable it.
+    GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_12, 0);
+
+    // make sure our PCI latency timer is sufficient (max it out)
+    GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_3,
+        DRF_DEF(_CONFIG, _PCI_NV_3, _LATENCY_TIMER, _248_CLOCKS));
+
+    if (pCl != NULL)
+    {
+        initCoreLogic(pGpu);
+    }
+    nvp->flags |= NV_INIT_FLAG_CORE_LOGIC;
+
+    return NV_OK;
+
+} // end of osInitMapping()
+
+static NV_STATUS
+initCoreLogic(
+    OBJGPU *pGpu
+)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJCL *pCl = SYS_GET_CL(pSys);
+
+    return clInit(pGpu, pCl);
+}
+
+static NV_STATUS
+teardownCoreLogic(
+    OBJOS  *pOS,
+    OBJGPU *pGpu
+)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJCL *pCl = SYS_GET_CL(pSys);
+
+    return clTeardown(pGpu, pCl);
+}
+
+static void RmTeardownDpauxRegisters(
+    nv_state_t *nv
+)
+{
+}
+
+static void RmTeardownHdacodecRegisters(
+    nv_state_t *nv
+)
+{
+}
+
+static void RmTeardownMipiCalRegisters(
+    nv_state_t *nv
+)
+{
+}
+
+static NV_STATUS
+RmTeardownRegisters(
+    nv_state_t *nv
+)
+{
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Tearing down registers\n");
+
+    if (nv->regs && nv->regs->map)
+    {
+        osUnmapKernelSpace(nv->regs->map, nv->regs->size);
+        nv->regs->map = 0;
+        nv->regs->map_u = NULL;
+    }
+
+    RmTeardownDpauxRegisters(nv);
+
+    RmTeardownHdacodecRegisters(nv);
+
+    RmTeardownMipiCalRegisters(nv);
+
+    return NV_OK;
+}
+
+static NV_STATUS
+RmSetupDpauxRegisters(
+    nv_state_t *nv,
+    UNIX_STATUS *status
+)
+{
+
+    return NV_OK;
+}
+
+static NV_STATUS
+RmSetupHdacodecRegisters(
+    nv_state_t *nv,
+    UNIX_STATUS *status
+)
+{
+
+    return NV_OK;
+}
+
+static NV_STATUS
+RmSetupMipiCalRegisters(
+    nv_state_t *nv,
+    UNIX_STATUS *status
+)
+{
+
+    return NV_OK;
+}
+
+static void
+RmSetupRegisters(
+    nv_state_t *nv,
+    UNIX_STATUS *status
+)
+{
+    NV_STATUS ret;
+
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmSetupRegisters for 0x%x:0x%x\n",
+              nv->pci_info.vendor_id, nv->pci_info.device_id);
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "pci config info:\n");
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "   registers look  like: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx,
+              nv->regs->cpu_address, nv->regs->size);
+
+    if (nv->fb != NULL)
+    {
+        NV_DEV_PRINTF(NV_DBG_SETUP, nv, "   fb        looks like: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx "\n",
+                nv->fb->cpu_address, nv->fb->size);
+    }
+
+    nv_os_map_kernel_space(nv, nv->regs);
+
+    if (nv->regs->map == NULL)
+    {
+        NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "Failed to map regs registers!!\n");
+        RM_SET_ERROR(*status, RM_INIT_REG_SETUP_FAILED);
+        status->rmStatus   = NV_ERR_OPERATING_SYSTEM;
+        return;
+    }
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Successfully mapped framebuffer and registers\n");
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "final mappings:\n");
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "    regs: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx " 0x%p\n",
+              nv->regs->cpu_address, nv->regs->size, nv->regs->map);
+
+    ret = RmSetupDpauxRegisters(nv, status);
+    if (ret != NV_OK)
+        goto err_unmap_disp_regs;
+
+    ret = RmSetupHdacodecRegisters(nv, status);
+    if (ret != NV_OK)
+    {
+        RmTeardownDpauxRegisters(nv);
+        goto err_unmap_disp_regs;
+    }
+
+    ret = RmSetupMipiCalRegisters(nv, status);
+    if (ret != NV_OK)
+    {
+        RmTeardownHdacodecRegisters(nv);
+        RmTeardownDpauxRegisters(nv);
+        goto err_unmap_disp_regs;
+    }
+
+    return;
+
+err_unmap_disp_regs:
+    if (nv->regs && nv->regs->map)
+    {
+        osUnmapKernelSpace(nv->regs->map, nv->regs->size);
+        nv->regs->map = 0;
+    }
+
+    return;
+}
+
+NvBool RmInitPrivateState(
+    nv_state_t *pNv
+)
+{
+    nv_priv_t *nvp;
+    NvU32 gpuId;
+    NvU32 socChipId0 = 0;
+    NvU32 pmc_boot_0 = 0;
+    NvU32 pmc_boot_1 = 0;
+    NvU32 pmc_boot_42 = 0;
+    NvU32 dmaAddrWidth = 0;
+    NvU64 gpuDomainBusDevice = 0;
+
+    NV_SET_NV_PRIV(pNv, NULL);
+
+    if (NV_IS_SOC_DISPLAY_DEVICE(pNv))
+        socChipId0 = pNv->disp_sw_soc_chip_id;
+    else
+    {
+        pNv->regs->map_u = os_map_kernel_space(pNv->regs->cpu_address,
+                                               os_page_size,
+                                               NV_MEMORY_UNCACHED);
+        if (pNv->regs->map_u == NULL)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "failed to map GPU registers (DISABLE_INTERRUPTS).\n");
+            return NV_FALSE;
+        }
+
+        pmc_boot_0 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_0);
+        pmc_boot_1 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_1);
+        pmc_boot_42 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_42);
+
+        os_unmap_kernel_space(pNv->regs->map_u, os_page_size);
+        pNv->regs->map_u = NULL;
+    }
+
+    if (os_alloc_mem((void **)&nvp, sizeof(*nvp)) != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "failed to allocate private device state.\n");
+        return NV_FALSE;
+    }
+
+    if (!NV_IS_SOC_DISPLAY_DEVICE(pNv))
+    {
+        gpuId = nv_generate_id_from_pci_info(&pNv->pci_info);
+        gpuDomainBusDevice = nv_encode_pci_info(&pNv->pci_info);
+    }
+    else
+    {
+        gpuId = NV0000_CTRL_GPU_INVALID_ID - 1;
+        gpuDomainBusDevice = 0;
+    }
+
+    if (gpumgrRegisterGpuId(gpuId, gpuDomainBusDevice) != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "failed to register GPU with GPU manager.\n");
+        os_free_mem(nvp);
+        return NV_FALSE;
+    }
+
+    pNv->gpu_id = gpuId;
+
+    nv_set_probed_gpu_flags(pNv);
+
+    pNv->iovaspace_id = nv_requires_dma_remap(pNv) ? gpuId :
+                                                     NV_IOVA_DOMAIN_NONE;
+    pNv->cpu_numa_node_id = NV0000_CTRL_NO_NUMA_NODE;
+
+    // Get the GpuArch instance for this architecture to determine the DMA address width.
+    GpuArch *pGpuArch = gpumgrGetGpuArch(pmc_boot_42, socChipId0, TEGRA_CHIP_TYPE_DEFAULT);
+    if (pGpuArch == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "failed to get GpuArch for 0x%x/0x%x.\n",
+                  pmc_boot_42, socChipId0);
+        gpumgrUnregisterGpuId(gpuId);
+        os_free_mem(nvp);
+        return NV_FALSE;
+    }
+
+    dmaAddrWidth = gpuarchGetDmaAddrWidth(pGpuArch);
+    if (dmaAddrWidth == 0)
+    {
+        dmaAddrWidth = gpuarchGetSystemPhysAddrWidth(pGpuArch);
+    }
+    nv_set_dma_address_size(pNv, dmaAddrWidth);
+
+    pNv->is_tegra_pci_igpu = !NV_IS_SOC_DISPLAY_DEVICE(pNv) && pGpuArch->bGpuArchIsZeroFb;
+    //  Only certain Tegra PCI iGPUs support Rail-Gating
+    pNv->supports_tegra_igpu_rg = pNv->is_tegra_pci_igpu && pGpuArch->bGpuarchSupportsIgpuRg;
+
+    pNv->sysmem_mapped_console = pGpuArch->bGpuArchIsZeroFb;
+
+    kvgpumgrAttachGpu(pNv->gpu_id);
+
+    os_mem_set(nvp, 0, sizeof(*nvp));
+    nvp->status = NV_ERR_INVALID_STATE;
+    nvp->pmc_boot_0 = pmc_boot_0;
+    nvp->pmc_boot_1 = pmc_boot_1;
+    nvp->pmc_boot_42 = pmc_boot_42;
+    nvp->db_supported = -1;
+    NV_SET_NV_PRIV(pNv, nvp);
+
+    return NV_TRUE;
+}
+
+void RmClearPrivateState(
+    nv_state_t *pNv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(pNv);
+    NvU32 status;
+    void *pVbiosCopy = NULL;
+    void *pRegistryCopy = NULL;
+    NvU32 vbiosSize;
+    nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS];
+    nv_dynamic_power_t dynamicPowerCopy;
+    NvU32 x = 0;
+    NvU32 pmc_boot_0, pmc_boot_1, pmc_boot_42;
+    NvBool pr3_acpi_method_present = 0;
+    int db_supported;
+
+    //
+    // Do not clear private state after GPU resets, it is used while
+    // recovering the GPU. Only clear the pGpu pointer, which is
+    // restored during next initialization cycle.
+    //
+    if (pNv->flags & NV_FLAG_IN_RECOVERY)
+    {
+        nvp->pGpu = NULL;
+    }
+
+    status = nvp->status;
+    pVbiosCopy = nvp->pVbiosCopy;
+    vbiosSize = nvp->vbiosSize;
+    pRegistryCopy = nvp->pRegistry;
+    dynamicPowerCopy = nvp->dynamic_power;
+    pmc_boot_0 = nvp->pmc_boot_0;
+    pmc_boot_1 = nvp->pmc_boot_1;
+    pmc_boot_42 = nvp->pmc_boot_42;
+    pr3_acpi_method_present = nvp->pr3_acpi_method_present;
+    db_supported = nvp->db_supported;
+
+    for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+    {
+        i2c_adapters[x] = nvp->i2c_adapters[x];
+    }
+
+    portMemSet(nvp, 0, sizeof(nv_priv_t));
+
+    nvp->status = status;
+    nvp->pVbiosCopy = pVbiosCopy;
+    nvp->vbiosSize = vbiosSize;
+    nvp->pRegistry = pRegistryCopy;
+    nvp->dynamic_power = dynamicPowerCopy;
+    nvp->pmc_boot_0 = pmc_boot_0;
+    nvp->pmc_boot_1 = pmc_boot_1;
+    nvp->pmc_boot_42 = pmc_boot_42;
+    nvp->pr3_acpi_method_present = pr3_acpi_method_present;
+    nvp->db_supported = db_supported;
+
+    for (x = 0; x < MAX_I2C_ADAPTERS; x++)
+    {
+        nvp->i2c_adapters[x] = i2c_adapters[x];
+    }
+
+    nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C;
+}
+
+void RmFreePrivateState(
+    nv_state_t *pNv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(pNv);
+
+    gpumgrUnregisterGpuId(pNv->gpu_id);
+
+    kvgpumgrDetachGpu(pNv->gpu_id);
+
+    RmDestroyRegistry(pNv);
+
+    if (nvp != NULL)
+    {
+        portMemFree(nvp->pVbiosCopy);
+        os_free_mem(nvp);
+    }
+
+    NV_SET_NV_PRIV(pNv, NULL);
+}
+
+NvBool RmPartiallyInitAdapter(
+    nv_state_t *nv
+)
+{
+    NV_PRINTF(LEVEL_INFO, "%s: %04x:%02x:%02x.0\n", __FUNCTION__,
+              nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot);
+
+    nv_start_rc_timer(nv);
+
+    return NV_TRUE;
+}
+
+static NV_STATUS
+RmInitX86Emu(
+    OBJGPU *pGpu
+)
+{
+    NV_STATUS status = NV_OK;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    PORT_UNREFERENCED_VARIABLE(nv);
+
+#if NVCPU_IS_X86_64
+    status = RmInitX86EmuState(pGpu);
+#else
+    // We don't expect a "primary VGA" adapter on non-amd64 platforms
+    NV_ASSERT(!NV_PRIMARY_VGA(nv));
+#endif
+
+    return status;
+}
+
+static NV_STATUS RmRegisterGpudb(
+    OBJGPU       *pGpu
+)
+{
+    NV_STATUS     rmStatus;
+    const NvU8   *pGid;
+    nv_state_t   *pNv = NV_GET_NV_STATE(pGpu);
+
+    pGid = RmGetGpuUuidRaw(pNv);
+    if (pGid == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to get UUID\n");
+        return NV_ERR_OPERATING_SYSTEM;
+    }
+
+    rmStatus = gpudbRegisterGpu(pGid, &pGpu->gpuClData.upstreamPort.addr,
+                                pGpu->busInfo.nvDomainBusDeviceFunc);
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to register GPU with GPU data base\n");
+    }
+
+    return rmStatus;
+}
+
+static void RmUnixFreeRmApi(
+    nv_state_t *nv
+)
+{
+    if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+    {
+        RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+
+        if (nv->rmapi.hClient != 0)
+        {
+            pRmApi->Free(pRmApi, nv->rmapi.hClient, nv->rmapi.hClient);
+        }
+
+        portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi));
+
+        rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+    }
+}
+
+static NvBool RmUnixAllocRmApi(
+    nv_state_t *nv,
+    NvU32 deviceId
+)
+{
+    NV_STATUS status = NV_OK;
+    NV0080_ALLOC_PARAMETERS deviceParams = { 0 };
+    NV2080_ALLOC_PARAMETERS subDeviceParams = { 0 };
+    RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+
+    portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi));
+
+    // LOCK: acquire GPUs lock
+    status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT);
+    if (status != NV_OK)
+    {
+        return status;
+    }
+
+    if (pRmApi->AllocWithHandle(
+                    pRmApi,
+                    NV01_NULL_OBJECT,
+                    NV01_NULL_OBJECT,
+                    NV01_NULL_OBJECT,
+                    NV01_ROOT,
+                    &nv->rmapi.hClient,
+                    sizeof(nv->rmapi.hClient)) != NV_OK)
+    {
+        goto fail;
+    }
+
+    deviceParams.deviceId = deviceId;
+
+    if (pRmApi->Alloc(
+                    pRmApi,
+                    nv->rmapi.hClient,
+                    nv->rmapi.hClient,
+                    &nv->rmapi.hDevice,
+                    NV01_DEVICE_0,
+                    &deviceParams,
+                    sizeof(deviceParams)) != NV_OK)
+    {
+        goto fail;
+    }
+
+    subDeviceParams.subDeviceId = 0;
+
+    if (pRmApi->Alloc(
+                    pRmApi,
+                    nv->rmapi.hClient,
+                    nv->rmapi.hDevice,
+                    &nv->rmapi.hSubDevice,
+                    NV20_SUBDEVICE_0,
+                    &subDeviceParams,
+                    sizeof(subDeviceParams)) != NV_OK)
+    {
+        goto fail;
+    }
+
+    //
+    // The NV40_I2C allocation expected to fail, if it is disabled
+    // with RM config.
+    //
+    if (pRmApi->Alloc(
+                    pRmApi,
+                    nv->rmapi.hClient,
+                    nv->rmapi.hSubDevice,
+                    &nv->rmapi.hI2C,
+                    NV40_I2C,
+                    NULL,
+                    0) != NV_OK)
+    {
+        nv->rmapi.hI2C = 0;
+    }
+
+    //
+    // The NV04_DISPLAY_COMMON allocation expected to fail for displayless
+    // system. nv->rmapi.hDisp value needs to be checked before doing display
+    // related control calls.
+    //
+    if (pRmApi->Alloc(
+                    pRmApi,
+                    nv->rmapi.hClient,
+                    nv->rmapi.hDevice,
+                    &nv->rmapi.hDisp,
+                    NV04_DISPLAY_COMMON,
+                    NULL,
+                    0) != NV_OK)
+    {
+        nv->rmapi.hDisp = 0;
+    }
+
+    // UNLOCK: release GPUs lock
+    rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+    return NV_TRUE;
+
+fail:
+    // UNLOCK: release GPUs lock
+    rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+    RmUnixFreeRmApi(nv);
+    return NV_FALSE;
+}
+
+static NV_STATUS RmFetchGspRmImages
+(
+    nv_state_t    *nv,
+    GSP_FIRMWARE  *pGspFw,
+    const void   **gspFwHandle,
+    const void   **gspFwLogHandle,
+    const void   **gspUcodesHandle
+)
+{
+    nv_firmware_chip_family_t chipFamily;
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    NvU32 gpuArch = (decodePmcBoot42Architecture(nvp->pmc_boot_42) <<
+                     GPU_ARCH_SHIFT);
+    NvU32 gpuImpl = DRF_VAL(_PMC, _BOOT_42, _IMPLEMENTATION, nvp->pmc_boot_42);
+
+    chipFamily = nv_firmware_get_chip_family(gpuArch, gpuImpl);
+
+    portMemSet(pGspFw, 0, sizeof(*pGspFw));
+
+    *gspFwHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP,
+                                   chipFamily,
+                                   &pGspFw->pBuf,
+                                   &pGspFw->size);
+
+    NvBool bFirmwareLoadSuccessful = *gspFwHandle != NULL;
+
+    //
+    // GR-3428 development: Temporarily gated by regkey so this code can be
+    // decoupled from the build infra changes to package up ucodes.bin.
+    //
+    // Because loading the bindata ends up calling request_firmware() on Linux,
+    // and that function will always print an error to dmesg if the requested
+    // file is not found, we use this to skip trying to load it.
+    //
+    // Check will be removed once ucodes.bin becomes part of the driver package.
+    //
+    NvU32 data = 0;
+    if ((osReadRegistryDword(NULL, "GR3428ReadUcodesBin", &data) == NV_OK) && data)
+    {
+        *gspUcodesHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_UCODES,
+                                           chipFamily,
+                                           &pGspFw->pUcodesBin,
+                                           &pGspFw->ucodesBinSize);
+        bFirmwareLoadSuccessful &= *gspUcodesHandle != NULL;
+    }
+
+    if (!bFirmwareLoadSuccessful && !nv->allow_fallback_to_monolithic_rm)
+    {
+        NV_PRINTF(LEVEL_ERROR, "No firmware image found\n");
+        return NV_ERR_NOT_SUPPORTED;
+    }
+    else if (bFirmwareLoadSuccessful)
+    {
+#if LIBOS_LOG_DECODE_ENABLE
+        if (nv->enable_firmware_logs)
+        {
+            *gspFwLogHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP_LOG,
+                                              chipFamily,
+                                              &pGspFw->pLogElf,
+                                              &pGspFw->logElfSize);
+            if (*gspFwLogHandle == NULL)
+            {
+                NV_PRINTF(LEVEL_ERROR, "Failed to load gsp_log_*.bin, no GSP-RM logs will be printed (non-fatal)\n");
+            }
+        }
+#endif
+        nv->request_fw_client_rm = NV_TRUE;
+    }
+    else
+    {
+        nv->request_fw_client_rm = NV_FALSE;
+    }
+
+    return NV_OK;
+}
+
+static void _checkP2pChipsetSupport(
+    nv_state_t *nv
+)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+    OBJCL  *pCl  = SYS_GET_CL(pSys);
+
+    // Check for PCIe P2P chipset support
+    if (!pCl->bPciePeerWriteCapable ||
+        !pCl->bPciePeerReadCapable)
+    {
+        nv->flags |= NV_FLAG_PCI_P2P_UNSUPPORTED_CHIPSET;
+    }
+}
+
+NvBool RmInitAdapter(
+    nv_state_t *nv
+)
+{
+    NvU32           devicereference = 0;
+    UNIX_STATUS     status = INIT_UNIX_STATUS;
+    nv_priv_t      *nvp;
+    NvBool          retVal = NV_FALSE;
+    OBJSYS         *pSys;
+    OBJGPU         *pGpu = NULL;
+    OBJOS          *pOS;
+    KernelDisplay  *pKernelDisplay;
+    const void     *gspFwHandle = NULL;
+    const void     *gspFwLogHandle = NULL;
+    const void     *gspUcodesHandle = NULL;
+
+    GSP_FIRMWARE    gspFw = {0};
+    PORT_UNREFERENCED_VARIABLE(gspFw);
+
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter\n");
+
+    nv->flags &= ~NV_FLAG_PASSTHRU;
+    nv->flags &= ~NV_FLAG_PCI_P2P_UNSUPPORTED_CHIPSET;
+
+    RmSetupRegisters(nv, &status);
+    if (! RM_INIT_SUCCESS(status.initStatus) )
+        goto failed;
+
+    nvp = NV_GET_NV_PRIV(nv);
+    nvp->status = NV_ERR_OPERATING_SYSTEM;
+
+    status.rmStatus = RmInitDeviceDma(nv);
+    if (status.rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Cannot configure the device for DMA\n");
+        RM_SET_ERROR(status, RM_INIT_GPU_DMA_CONFIGURATION_FAILED);
+        goto shutdown;
+    }
+
+    nvp->flags |= NV_INIT_FLAG_DMA;
+
+    pSys = SYS_GET_INSTANCE();
+
+    //
+    // Get firmware from the OS, if requested, and decide if RM will run as a
+    // firmware client.
+    //
+    if (nv->request_firmware)
+    {
+        if (!NV_IS_SOC_DISPLAY_DEVICE(nv))
+        {
+            status.rmStatus = RmFetchGspRmImages(nv, &gspFw, &gspFwHandle, &gspFwLogHandle, &gspUcodesHandle);
+            if (status.rmStatus != NV_OK)
+            {
+                RM_SET_ERROR(status, RM_INIT_FIRMWARE_FETCH_FAILED);
+                goto shutdown;
+            }
+        }
+        else
+        {
+            nv->request_fw_client_rm = NV_TRUE;
+        }
+    }
+
+    //
+    // Initialization path requires expanded GPU visibility in GPUMGR
+    // in order to access the GPU undergoing initialization.
+    //
+    status.rmStatus = gpumgrThreadEnableExpandedGpuVisibility();
+    if (status.rmStatus != NV_OK)
+    {
+        RM_SET_ERROR(status, RM_INIT_GPU_GPUMGR_EXPANDED_VISIBILITY_FAILED);
+        goto shutdown;
+    }
+
+
+    // initialize the RM device register mapping
+    osInitNvMapping(nv, &devicereference, &status);
+    if (! RM_INIT_SUCCESS(status.initStatus) )
+    {
+        switch (status.rmStatus)
+        {
+            case NV_ERR_NOT_SUPPORTED:
+                nvp->status = NV_ERR_NOT_SUPPORTED;
+                break;
+        }
+        NV_PRINTF(LEVEL_ERROR,
+                  "osInitNvMapping failed, bailing out of RmInitAdapter\n");
+        goto shutdown;
+    }
+
+    //
+    // now we can have a pdev for the first time...
+    //
+    pGpu   = gpumgrGetGpu(devicereference);
+
+    gpumgrSetCurrentGpuInstance(pGpu->gpuInstance);
+
+    pOS    = SYS_GET_OS(pSys);
+
+    status.rmStatus = osInitScalability(pGpu);
+    if (status.rmStatus == NV_OK)
+    {
+        nvp->flags |= NV_INIT_FLAG_SCALABILITY;
+    }
+    else if (status.rmStatus != NV_ERR_NOT_SUPPORTED)
+    {
+        RM_SET_ERROR(status, RM_INIT_SCALABILITY_FAILED);
+        goto shutdown;
+    }
+
+    // Boot GSP-RM proxy through COT command either via FSP or SEC2
+    if (!IS_GSP_CLIENT(pGpu) && !IS_VIRTUAL(pGpu))
+    {
+        status.rmStatus = gpuBootGspRmProxy(pGpu);
+        if (status.rmStatus != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR, "GSP-RM proxy boot command failed.\n");
+            RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED);
+            goto shutdown;
+        }
+    }
+
+    RmDeterminePrimaryDevice(pGpu);
+
+    RmSetConsolePreservationParams(pGpu);
+
+    RmInitAcpiMethods(pOS, pSys, pGpu);
+
+    // This needs to run before GSP-RM is booted, or else it will timeout
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_CLKS_IN_TEGRA_SOC))
+    {
+        NV_PRINTF(LEVEL_INFO, "Enable Clocks to Max\n");
+        RmEnableDeviceClks(nv);
+    }
+
+    //
+    // If GSP fw RM support is enabled then start the GSP microcode
+    // (including the task running the full instance of the RM) and
+    // exchange the necessary initial RPC messages before continuing
+    // with GPU initialization here.
+    //
+    if (IS_GSP_CLIENT(pGpu))
+    {
+        status.rmStatus = kgspInitRm(pGpu, GPU_GET_KERNEL_GSP(pGpu), &gspFw);
+        if (status.rmStatus != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR, "Cannot initialize GSP firmware RM\n");
+            RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED);
+            goto shutdown;
+        }
+    }
+    else if (IS_DCE_CLIENT(pGpu))
+    {
+        status.rmStatus = dceclientDceRmInit(pGpu, GPU_GET_DCECLIENTRM(pGpu), NV_TRUE);
+        if (status.rmStatus != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR, "Cannot initialize DCE firmware RM\n");
+            RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED);
+            goto shutdown;
+        }
+    }
+    else if (nv->request_fw_client_rm)
+    {
+        // We were expecting to enable GSP-RM but something went wrong.
+        if (!nv->allow_fallback_to_monolithic_rm)
+        {
+            RM_SET_ERROR(status, RM_INIT_FIRMWARE_POLICY_FAILED);
+            goto shutdown;
+        }
+        else
+        {
+            NV_PRINTF(LEVEL_NOTICE, "Falling back to monolithic RM\n");
+        }
+    }
+
+    pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+    if (pKernelDisplay != NULL)
+    {
+        kdispSetWarPurgeSatellitesOnCoreFree(pKernelDisplay, NV_TRUE);
+    }
+
+    if (IS_PASSTHRU(pGpu))
+        nv->flags |= NV_FLAG_PASSTHRU;
+
+    RmInitNvHal(nv, devicereference, &status);
+    if (!RM_INIT_SUCCESS(status.initStatus))
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "RmInitNvHal() failed, bailing out of RmInitAdapter!\n");
+        goto shutdown;
+    }
+
+    status.rmStatus = RmInitX86Emu(pGpu);
+    if (status.rmStatus != NV_OK)
+    {
+        RM_SET_ERROR(status, RM_INIT_VBIOS_X86EMU_FAILED);
+        NV_PRINTF(LEVEL_ERROR,
+                  "RmInitX86Emu failed, bailing out of RmInitAdapter\n");
+        goto shutdown;
+    }
+
+
+    initVendorSpecificRegistry(pGpu, nv->pci_info.device_id);
+
+    if (!IS_VIRTUAL(pGpu) && !NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        initNbsiTable(pGpu);
+    }
+
+    // finally, initialize the device
+    RmInitNvDevice(devicereference, &status);
+    if (! RM_INIT_SUCCESS(status.initStatus) )
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "RmInitNvDevice failed, bailing out of RmInitAdapter\n");
+        switch (status.rmStatus)
+        {
+            case NV_ERR_INSUFFICIENT_POWER:
+                nvp->status = NV_ERR_INSUFFICIENT_POWER;
+                NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
+                    "GPU does not have the necessary power cables connected.\n");
+                break;
+        }
+        goto shutdown;
+    }
+
+    //
+    // Expanded GPU visibility in GPUMGR is no longer needed once the
+    // GPU is initialized.
+    //
+    gpumgrThreadDisableExpandedGpuVisibility();
+
+    // LOCK: acquire GPUs lock
+    status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE,
+                                        RM_LOCK_MODULES_INIT);
+    if (status.rmStatus != NV_OK)
+    {
+        goto shutdown;
+    }
+
+    status.rmStatus = osVerifySystemEnvironment(pGpu);
+
+    // UNLOCK: release GPUs lock
+    rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+    if (status.rmStatus != NV_OK)
+    {
+        RM_SET_ERROR(status, RM_INIT_SYS_ENVIRONMENT_FAILED);
+        switch (status.rmStatus)
+        {
+            case NV_ERR_IRQ_NOT_FIRING:
+                nvp->status = NV_ERR_IRQ_NOT_FIRING;
+                break;
+        }
+        NV_PRINTF(LEVEL_ERROR, "osVerifySystemEnvironment failed, bailing!\n");
+        goto shutdown;
+    }
+
+    Intr *pIntr = GPU_GET_INTR(pGpu);
+    if (pIntr != NULL)
+    {
+        intrSetIntrEn(pIntr, INTERRUPT_TYPE_HARDWARE);
+    }
+
+    // LOCK: acquire GPUs lock
+    status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE,
+                                        RM_LOCK_MODULES_INIT);
+    if (status.rmStatus != NV_OK)
+    {
+        goto shutdown;
+    }
+
+    KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu);
+    // initialize the watchdog (disabled by default)
+    status.rmStatus = pKernelRc != NULL ? krcWatchdogInit_HAL(pGpu, pKernelRc, NULL) :
+                                          NV_ERR_NOT_SUPPORTED;
+    if (status.rmStatus == NV_OK)
+    {
+        krcWatchdogDisable(pKernelRc, NULL);
+        nvp->flags |= NV_INIT_FLAG_FIFO_WATCHDOG;
+    }
+    else if (status.rmStatus == NV_ERR_NOT_SUPPORTED)
+    {
+        NV_PRINTF(LEVEL_INFO,
+            "krcWatchdogInit returned _NOT_SUPPORTED. For Kepler GPUs in PGOB mode, this is normal\n");
+    }
+    else
+    {
+        RM_SET_ERROR(status, RM_INIT_WATCHDOG_FAILED);
+        NV_PRINTF(LEVEL_ERROR,
+                  "krcWatchdogInit failed, bailing out of RmInitAdapter\n");
+        // UNLOCK: release GPUs lock
+        rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+        goto shutdown;
+    }
+
+    // UNLOCK: release GPUs lock
+    rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+    nv_start_rc_timer(nv);
+
+    nvp->status = NV_OK;
+
+    if (!RmUnixAllocRmApi(nv, devicereference)) {
+        RM_SET_ERROR(status, RM_INIT_ALLOC_RMAPI_FAILED);
+        status.rmStatus = NV_ERR_GENERIC;
+        goto shutdown;
+    }
+
+    if (!NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        status.rmStatus = RmInitGpuInfoWithRmApi(pGpu);
+        if (status.rmStatus != NV_OK)
+        {
+            RM_SET_ERROR(status, RM_INIT_GPUINFO_WITH_RMAPI_FAILED);
+            goto shutdown;
+        }
+    }
+
+    // i2c only on master device??
+    RmI2cAddGpuPorts(nv);
+    nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C;
+
+    // This fifo hal call will fail on pre-fermi gpus. In that case, userd
+    // info will remain 0 and skipped by fb mmap code
+    KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
+    NvU64       udaddr = 0;
+    NvU32       udsize = 0;
+    if (pKernelFifo != NULL)
+    {
+        status.rmStatus = kfifoGetUserdBar1MapInfo_HAL(pGpu, pKernelFifo, &udaddr, &udsize);
+        if (status.rmStatus == NV_OK)
+        {
+            nv->ud.cpu_address = (nv->fb->cpu_address + udaddr);
+            nv->ud.size = udsize;
+        }
+        else if (status.rmStatus != NV_ERR_NOT_SUPPORTED)
+        {
+            RM_SET_ERROR(status, RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED);
+            NV_PRINTF(LEVEL_ERROR,
+                    "kfifoGetUserdBar1MapInfo failed, bailing out of RmInitAdapter\n");
+            goto shutdown;
+        }
+    }
+
+    nv->flags &= ~NV_FLAG_IN_RECOVERY;
+
+    pSys->setProperty(pSys, PDB_PROP_SYS_SYSTEM_EVENTS_SUPPORTED, NV_TRUE);
+
+    RmInitPowerManagement(nv);
+
+    if (!NV_IS_SOC_DISPLAY_DEVICE(nv))
+    {
+        status.rmStatus = RmRegisterGpudb(pGpu);
+        if (status.rmStatus != NV_OK)
+        {
+            RM_SET_ERROR(status, RM_GPUDB_REGISTER_FAILED);
+            goto shutdown;
+        }
+    }
+
+    if (nvp->b_mobile_config_enabled)
+    {
+        NvU32 ac_plugged = 0;
+        if (nv_acpi_get_powersource(&ac_plugged) == NV_OK)
+        {
+            // LOCK: acquire GPU lock
+            if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_NONE) == NV_OK)
+            {
+                //
+                // As we have already acquired the API Lock here, we are calling
+                // RmPowerSourceChangeEvent directly instead of rm_power_source_change_event.
+                //
+                RmPowerSourceChangeEvent(nv, !ac_plugged);
+
+                // UNLOCK: release GPU lock
+                rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+            }
+        }
+    }
+
+    _checkP2pChipsetSupport(nv);
+
+    NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter succeeded!\n");
+
+    retVal = NV_TRUE;
+    goto done;
+
+ shutdown:
+    nv->flags &= ~NV_FLAG_IN_RECOVERY;
+
+    gpumgrThreadDisableExpandedGpuVisibility();
+
+    // call ShutdownAdapter to undo anything we've done above
+    RmShutdownAdapter(nv);
+
+ failed:
+    NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "RmInitAdapter failed! (0x%x:0x%x:%d)\n",
+        status.initStatus, status.rmStatus, status.line);
+
+done:
+    nv_put_firmware(gspFwHandle);
+    nv_put_firmware(gspFwLogHandle);
+    nv_put_firmware(gspUcodesHandle);
+
+    gpumgrSetCurrentGpuInstance(NV_U32_MAX);
+
+    return retVal;
+}
+
+void RmShutdownAdapter(
+    nv_state_t *nv
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    NV_STATUS rmStatus;
+
+    if ((pGpu != NULL) && (nvp->flags & NV_INIT_FLAG_GPUMGR_ATTACH))
+    {
+        NvU32 gpuInstance    = gpuGetInstance(pGpu);
+        NvU32 deviceInstance = gpuGetDeviceInstance(pGpu);
+        OBJSYS         *pSys = SYS_GET_INSTANCE();
+
+        RmUnixFreeRmApi(nv);
+
+        nv->ud.cpu_address = 0;
+        nv->ud.size = 0;
+
+        //
+        // LOCK: lock all clients in case of eGPU hot unplug, which
+        // will not wait for all existing RM clients to stop using the GPU.
+        //
+        if (!nv->is_external_gpu || serverLockAllClients(&g_resServ) == NV_OK)
+        {
+            // LOCK: acquire GPUs lock
+            if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+            {
+                //
+                // Shutdown path requires expanded GPU visibility in GPUMGR in order
+                // to access the GPU undergoing shutdown which may not be fully
+                // initialized, and to continue accessing the GPU undergoing shutdown
+                // after state destroy.
+                //
+                NV_ASSERT_OK(gpumgrThreadEnableExpandedGpuVisibility());
+
+                RmDestroyPowerManagement(nv);
+
+                freeNbsiTable(pGpu);
+
+                OBJCL          *pCl  = SYS_GET_CL(pSys);
+                OBJOS          *pOS  = GPU_GET_OS(pGpu);
+                if (pCl != NULL)
+                {
+                    if (nvp->flags & NV_INIT_FLAG_CORE_LOGIC)
+                    {
+                        teardownCoreLogic(pOS, pGpu);
+                    }
+                    else if (nvp->flags & NV_INIT_FLAG_SCALABILITY)
+                    {
+                        osTeardownScalability(pGpu);
+                    }
+                }
+
+            if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG)
+            {
+                krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu), NULL);
+                nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG;
+            }
+
+                rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance));
+                rmapiDelPendingDevices(NVBIT(gpuInstance));
+
+                if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
+                {
+                    rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT);
+                    NV_ASSERT(rmStatus == NV_OK);
+                }
+
+                if (nvp->flags & NV_INIT_FLAG_GPU_STATE)
+                {
+                    rmStatus = gpuStateDestroy(pGpu);
+                    NV_ASSERT(rmStatus == NV_OK);
+                }
+
+                if (IS_DCE_CLIENT(pGpu))
+                {
+                    rmStatus = dceclientDceRmInit(pGpu, GPU_GET_DCECLIENTRM(pGpu), NV_FALSE);
+                    if (rmStatus != NV_OK)
+                    {
+                        NV_PRINTF(LEVEL_ERROR, "DCE firmware RM Shutdown failure\n");
+                    }
+                }
+
+                if (pGpu->getProperty(pGpu, PDB_PROP_GPU_CLKS_IN_TEGRA_SOC))
+                {
+                    NV_PRINTF(LEVEL_INFO, "Disable Clocks\n");
+                    RmDisableDeviceClks(nv);
+                }
+
+                //if (nvp->flags & NV_INIT_FLAG_HAL)
+                //  destroyHal(pDev);
+
+#if NVCPU_IS_X86_64
+                RmFreeX86EmuState(pGpu);
+#endif
+
+                gpumgrDetachGpu(gpuInstance);
+                gpumgrDestroyDevice(deviceInstance);
+
+                //
+                // Expanded GPU visibility in GPUMGR is no longer needed once the
+                // GPU is removed from GPUMGR.
+                //
+                gpumgrThreadDisableExpandedGpuVisibility();
+
+                if (nvp->flags & NV_INIT_FLAG_DMA)
+                {
+                    RmTeardownDeviceDma(nv);
+                }
+
+                RmClearPrivateState(nv);
+
+                RmUnInitAcpiMethods(pSys);
+
+                // UNLOCK: release GPUs lock
+                rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+
+                // RM_BASIC_LOCK_MODEL: free GPU lock
+                rmGpuLockFree(deviceInstance);
+            }
+
+            // UNLOCK: unlock all clients for eGPU hot unplug path
+            if (nv->is_external_gpu)
+                serverUnlockAllClients(&g_resServ);
+        }
+    }
+    else
+    {
+        RmClearPrivateState(nv);
+    }
+
+    RmTeardownRegisters(nv);
+}
+
+void RmPartiallyDisableAdapter(
+    nv_state_t *nv
+)
+{
+    NV_PRINTF(LEVEL_INFO, "%s: RM is in SW Persistence mode\n", __FUNCTION__);
+
+    nv_stop_rc_timer(nv);
+}
+
+void RmDisableAdapter(
+    nv_state_t *nv
+)
+{
+    NV_STATUS  rmStatus;
+    OBJGPU    *pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    NvU32      gpuMask;
+    nv_priv_t *nvp  = NV_GET_NV_PRIV(nv);
+
+    if (pGpu == NULL)
+    {
+        return;
+    }
+
+    gpumgrSetCurrentGpuInstance(pGpu->gpuInstance);
+
+    //
+    // Normally, we re-enable interrupts when we release the lock
+    // after RmDisableAdapter(), before disabling the bottom-half ISR.
+    // If an interrupt came in in that time window, but the bottom half
+    // wasn't scheduled to run in time, the semaphore acquired by the
+    // top-half would never be released afterwards (since the bottom-half
+    // would never run). This call to intrSetIntrEn() ensures that this won't
+    // happen, by making sure interrupts remain disabled on the GPU in question after
+    // RmDisableAdapter().
+    //
+    Intr *pIntr = GPU_GET_INTR(pGpu);
+    if (pIntr != NULL)
+    {
+        intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED);
+    }
+
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SECONDARY_BUS_RESET) ||
+        pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_FULLCHIP_RESET))
+    {
+        pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_TIMEOUT_RECOVERY, NV_TRUE);
+        nv->flags |= NV_FLAG_IN_RECOVERY;
+    }
+
+    //
+    // LOCK: lock all clients in case of eGPU hot unplug, which
+    // will not wait for all existing RM clients to stop using the GPU.
+    //
+    if (!nv->is_external_gpu || serverLockAllClients(&g_resServ) == NV_OK)
+    {
+
+        // LOCK: acquire GPUs lock
+        if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
+        {
+            //
+            // Free the client allocated resources.
+            //
+            // Note this doesn't free RM internal resource allocations. Those are
+            // freed during gpuStateUnload.
+            //
+            gpuMask = gpumgrGetGpuMask(pGpu);
+
+            rmapiSetDelPendingClientResourcesFromGpuMask(gpuMask);
+            rmapiDelPendingDevices(gpuMask);
+
+            nv_stop_rc_timer(nv);
+
+            OBJSYS    *pSys = SYS_GET_INSTANCE();
+            OBJCL     *pCl  = SYS_GET_CL(pSys);
+            OBJOS     *pOS  = SYS_GET_OS(pSys);
+            if (pCl != NULL)
+            {
+                teardownCoreLogic(pOS, pGpu);
+                nvp->flags &= ~NV_INIT_FLAG_CORE_LOGIC;
+            }
+
+            if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG)
+            {
+                krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu), NULL);
+                nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG;
+            }
+
+            if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
+            {
+                rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT);
+                NV_ASSERT(rmStatus == NV_OK);
+                nvp->flags &= ~NV_INIT_FLAG_GPU_STATE_LOAD;
+            }
+
+            // UNLOCK: release GPUs lock
+            rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+        }
+
+        // UNLOCK: unlock all clients for eGPU hot unplug path
+        if (nv->is_external_gpu)
+            serverUnlockAllClients(&g_resServ);
+    }
+
+    gpumgrSetCurrentGpuInstance(NV_U32_MAX);
+}
+
+NV_STATUS RmGetAdapterStatus(
+    nv_state_t *pNv,
+    NvU32      *pStatus
+)
+{
+    //
+    // This status is determined in RmInitAdapter(); the glue layer
+    // requests it when the adapter failed to initialize to learn
+    // more about the error condition. This is currently limited to
+    // osVerifySystemEnvironment() failures.
+    //
+    nv_priv_t *nvp;
+
+    nvp = NV_GET_NV_PRIV(pNv);
+    if (nvp == NULL)
+    {
+        return NV_ERR_INVALID_STATE;
+    }
+
+    *pStatus = nvp->status;
+    return NV_OK;
+}
+
+static void initVendorSpecificRegistry(
+    OBJGPU *pGpu,
+    NvU16 device_id
+)
+{
+    NV_STATUS rmStatus;
+    NvU32 i;
+    NvU32 subsystem_id;
+    NvU32 subsystem_vendor_id;
+    NvU32 subsystem_device_id;
+    NvU32 vendor_id = 0;
+
+    if (!pGpu)
+        return;
+
+    if (pGpu->bIsSOC)
+        return;
+
+    rmStatus = GPU_BUS_CFG_RD32(pGpu,
+            NV_CONFIG_PCI_NV_11, &subsystem_id);
+    if (rmStatus != NV_OK)
+    {
+       NV_PRINTF(LEVEL_ERROR,
+                 "%s: Cannot read NV_CONFIG_PCI_NV_11\n", __FUNCTION__);
+       return;
+    }
+
+    subsystem_vendor_id = (subsystem_id & 0xffff);
+    subsystem_device_id = (subsystem_id >> 16);
+
+    for (i = 0; (nb_id_table[i].subsystem_vendor_id) != 0; i++)
+    {
+        if ((nb_id_table[i].subsystem_vendor_id == subsystem_vendor_id) &&
+            (nb_id_table[i].subsystem_device_id == subsystem_device_id) &&
+            (nb_id_table[i].gpu_device_id == device_id))
+        {
+            vendor_id = subsystem_vendor_id;
+            break;
+        }
+    }
+
+    if (vendor_id != 0)
+    {
+        for (i = 0; nb_reg_table[i].vendor_id != 0; i++)
+        {
+            if (nb_reg_table[i].vendor_id == vendor_id)
+            {
+                osWriteRegistryDword(pGpu, nb_reg_table[i].name,
+                        nb_reg_table[i].data);
+            }
+        }
+    }
+}
+
+static void initUnixSpecificRegistry(
+    OBJGPU *pGpu
+)
+{
+    // By default, enable GPU reset on Unix
+    osWriteRegistryDword(pGpu, "RMSecBusResetEnable", 1);
+    osWriteRegistryDword(pGpu, "RMForcePcieConfigSave", 1);
+
+}
+
+void
+osRemoveGpu(
+    NvU32 domain,
+    NvU8 bus,
+    NvU8 device
+)
+{
+    void   *handle;
+
+    handle = os_pci_init_handle(domain, bus, device, 0, NULL, NULL);
+    if (handle != NULL)
+    {
+        os_pci_remove(handle);
+    }
+}
+
+/*
+ * Check to see if this board supports GPU exclusion.
+ */
+static NvBool RmIsExcludingAllowed(
+    nv_state_t *pNv
+)
+{
+    NvU32 feature = 0;
+    NvU32 brand;
+
+    // DGX-2/HGX-2 systems pre-date the PBI call
+    if (pNv->pci_info.device_id == 0x1db8)
+        return NV_TRUE;
+
+    if (pciPbiGetFeature(pNv->handle, &feature) != NV_OK)
+        return NV_FALSE;
+
+    brand = DRF_VAL(_PBI, _EXECUTE_ROUTINE, _GET_FEATURE_EXCLUSION, feature);
+
+    return (brand == NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION_ALLOWED);
+}
+
+NV_STATUS RmExcludeAdapter(
+    nv_state_t *nv
+)
+{
+    NV_STATUS rmStatus;
+
+    if (!RmIsExcludingAllowed(nv))
+        return NV_ERR_NOT_SUPPORTED;
+
+    rmStatus = gpumgrExcludeGpuId(nv->gpu_id);
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "failed to exclude GPU: 0x%x\n", rmStatus);
+        return rmStatus;
+    }
+
+    /*
+     * GPU exclusion only happens during initialization, and therefore there
+     * is no need to stop activity on the GPU
+     */
+
+    return rmStatus;
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osmemdesc.c b/userspace/sdks/nv570/nvalloc/unix/src/osmemdesc.c
new file mode 100644
index 0000000..84730cb
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/osmemdesc.c
@@ -0,0 +1,1325 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2012-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/******************* OS Memory Descriptor APIS *****************************\
+*                                                                           *
+* This contains routines to create and destroy OS memory descriptor         *
+*                                                                           *
+****************************************************************************/
+
+#include <nv.h>                     // NV device driver interface
+#include <os/os.h>
+#include <osapi.h>
+#include <nvos.h>
+#include <rmapi/client.h>
+#include <rmapi/rs_utils.h>
+#include <gpu_mgr/gpu_mgr.h>
+
+#include "gpu/bif/kernel_bif.h"
+
+static NV_STATUS osCreateOsDescriptorFromPageArray(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void **);
+static void osDestroyOsDescriptorPageArray(PMEMORY_DESCRIPTOR);
+
+static NV_STATUS osCreateOsDescriptorFromIoMemory(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static void osDestroyOsDescriptorFromIoMemory(PMEMORY_DESCRIPTOR);
+
+static NV_STATUS osCreateOsDescriptorFromPhysAddr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static void osDestroyOsDescriptorFromPhysAddr(PMEMORY_DESCRIPTOR);
+
+static NV_STATUS osCreateOsDescriptorFromFileHandle(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static NV_STATUS osCreateOsDescriptorFromDmaBufPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static void osDestroyOsDescriptorFromDmaBuf(PMEMORY_DESCRIPTOR);
+static NV_STATUS osCreateOsDescriptorFromSgtPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
+static void osDestroyOsDescriptorFromSgt(PMEMORY_DESCRIPTOR);
+
+static NV_STATUS osCheckGpuBarsOverlapAddrRange(NvRangeU64 addrRange);
+
+NV_STATUS
+osCreateMemFromOsDescriptor
+(
+    OBJGPU             *pGpu,
+    NvP64               pDescriptor,
+    NvHandle            hClient,
+    NvU32               flags,
+    NvU64              *pLimit,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    NvU32               descriptorType,
+    RS_PRIV_LEVEL       privilegeLevel
+)
+{
+    RmClient* pClient;
+    NV_STATUS rmStatus;
+    void *pPrivate;
+
+    pClient = serverutilGetClientUnderLock(hClient);
+    if ((pDescriptor == NvP64_NULL) ||
+        (*pLimit == 0) ||
+        (pClient == NULL))
+    {
+        return NV_ERR_INVALID_PARAM_STRUCT;
+    }
+
+    //
+    // For the sake of simplicity, unmatched RM and OS page
+    // sizes are not currently supported in this path, except for
+    // aarch64.
+    //
+    // Also, the nvmap handle is sent which can be any random number so
+    // the virtual address alignment sanity check can't be done here.
+    //
+    if (!NVCPU_IS_AARCH64 &&
+        (NV_RM_PAGE_SIZE != os_page_size))
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    //
+    // The two checks below use cached privilege because they 
+    // concern the privilege level of the client, and not the
+    // privilege level of the calling context which may be 
+    // overridden to KERNEL at some internal callsites.
+    //
+
+    //
+    // The RM cannot obtain a table of physical addresses
+    // for a kernel virtual address range on all of
+    // the supported UNIX platforms. Since this path is
+    // not really compelling for kernel allocations on any
+    // of those platforms, it is not supported.
+    // For UVM, they could have pre-allocated sysmem to register
+    // with RM so we put in an exception for that case.
+    //
+    if ((rmclientGetCachedPrivilege(pClient) >= RS_PRIV_LEVEL_KERNEL) &&
+        (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR) &&
+        (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE) &&
+        (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) &&
+        (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR))
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    //
+    // NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR and
+    // NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR can only be utilized by kernel space
+    // rm-clients.
+    //
+    if ((rmclientGetCachedPrivilege(pClient) < RS_PRIV_LEVEL_KERNEL) &&
+        ((descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) ||
+         (descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR)))
+    {
+        return NV_ERR_NOT_SUPPORTED;
+    }
+
+    switch (descriptorType)
+    {
+        case NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS:
+            rmStatus = NV_ERR_NOT_SUPPORTED;
+            break;
+        case NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR:
+            if (privilegeLevel < RS_PRIV_LEVEL_KERNEL)
+            {
+                rmStatus = NV_ERR_NOT_SUPPORTED;
+                break;
+            }
+            rmStatus = osCreateOsDescriptorFromPhysAddr(pGpu, pDescriptor,
+                    hClient, flags, pLimit, ppMemDesc, &pPrivate);
+            break;
+        case NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY:
+            rmStatus = osCreateOsDescriptorFromIoMemory(pGpu, pDescriptor,
+                    hClient, flags, pLimit, ppMemDesc, &pPrivate);
+            break;
+        case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY:
+            rmStatus = osCreateOsDescriptorFromPageArray(pGpu, pDescriptor,
+                    hClient, flags, pLimit, ppMemDesc, &pPrivate);
+            break;
+        case NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE:
+            rmStatus = osCreateOsDescriptorFromFileHandle(pGpu, pDescriptor,
+                     hClient, flags, pLimit, ppMemDesc, &pPrivate);
+            break;
+        case NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR:
+            rmStatus = osCreateOsDescriptorFromDmaBufPtr(pGpu, pDescriptor,
+                     hClient, flags, pLimit, ppMemDesc, &pPrivate);
+            break;
+        case NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR:
+            rmStatus = osCreateOsDescriptorFromSgtPtr(pGpu, pDescriptor,
+                     hClient, flags, pLimit, ppMemDesc, &pPrivate);
+            break;
+        default:
+            rmStatus = NV_ERR_INVALID_ARGUMENT;
+            break;
+    }
+
+    return rmStatus;
+}
+
+static NV_STATUS
+osCreateMemdescFromPages
+(
+    OBJGPU *pGpu,
+    NvU64 size,
+    NvU32 flags,
+    NvU32 cacheType,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void  *pImportPriv,
+    void **ppPrivate
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+    MEMORY_DESCRIPTOR *pMemDesc;
+    NvU64 memdescFlags = MEMDESC_FLAGS_NONE;
+    NvU32 gpuCachedFlags;
+    NvBool bUnprotected = NV_FALSE;
+    NvU64 osPageCount;
+
+    //
+    // Align size up to os page size. This is important in
+    // order to support submemdesc mappings at native page size.
+    // Once dynamic tracking is enabled this will not be needed
+    // as all submemory will get tracked at its native page size.
+    //
+    size = NV_ALIGN_UP64(size, os_page_size);
+
+    osPageCount = size >> os_page_shift;
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
+    {
+        memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _MEMORY_PROTECTION, _UNPROTECTED, flags))
+    {
+        memdescFlags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY;
+        bUnprotected = NV_TRUE;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, _YES, flags))
+    {
+        memdescFlags |= MEMDESC_FLAGS_USER_READ_ONLY;
+        memdescFlags |= MEMDESC_FLAGS_DEVICE_READ_ONLY;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, _YES, flags))
+    {
+        memdescFlags |= MEMDESC_FLAGS_DEVICE_READ_ONLY;
+    }
+
+    rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0,
+                             NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM,
+                             cacheType, memdescFlags);
+    if (rmStatus != NV_OK)
+    {
+        return rmStatus;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
+        gpuCachedFlags = NV_MEMORY_CACHED;
+    else
+        gpuCachedFlags = NV_MEMORY_UNCACHED;
+
+    pMemDesc = *ppMemDesc;
+    rmStatus = nv_register_user_pages(NV_GET_NV_STATE(pGpu),
+            osPageCount,
+            memdescGetPteArray(pMemDesc, AT_CPU), pImportPriv,
+            ppPrivate, bUnprotected);
+    if (rmStatus != NV_OK)
+    {
+        memdescDestroy(pMemDesc);
+        return rmStatus;
+    }
+
+    memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
+    memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
+    memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        NV_ASSERT_OR_GOTO((os_page_size & (pMemDesc->pageArrayGranularity - 1ULL)) == 0, cleanup);
+        NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, os_page_size), cleanup);
+    }
+    else
+    {
+        NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, NV_RM_PAGE_SIZE), cleanup);
+    }
+
+    if (!NV_IS_ALIGNED64(memdescGetPhysAddr(pMemDesc, AT_CPU, 0), os_page_size))
+    {
+        rmStatus = NV_ERR_INVALID_ARGUMENT;
+        NV_ASSERT_OR_GOTO(0, cleanup);
+    }
+
+
+    //
+    // If the OS layer doesn't think in RM page size, we need to inflate the
+    // PTE array into RM pages.
+    //
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion.
+        NV_ASSERT_OR_ELSE(os_page_size == pMemDesc->pageArrayGranularity, rmStatus = NV_ERR_INVALID_ARGUMENT; goto cleanup);
+    }
+    else if ((NV_RM_PAGE_SIZE < os_page_size) &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+    {
+        RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU), pMemDesc->PageCount);
+    }
+
+    //
+    // memdescMapIommu() requires the OS-private data to be set on the memory
+    // descriptor, but we don't want to wire up the teardown callback just yet:
+    // that callback needs to unpin the pages, but that will already be done
+    // as part of failure handling further up the stack if memdescMapIommu()
+    // fails. So we only set up the priv-data cleanup callback once we're sure
+    // this call will succeed.
+    //
+    memdescSetMemData(pMemDesc, *ppPrivate, NULL);
+
+    rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+cleanup:
+    if (rmStatus != NV_OK)
+    {
+        if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+        {
+            // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion.
+            NV_ASSERT(os_page_size == pMemDesc->pageArrayGranularity);
+        }
+        else if ((NV_RM_PAGE_SIZE < os_page_size) &&
+                !memdescGetContiguity(pMemDesc, AT_CPU))
+        {
+            RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+                                    pMemDesc->PageCount);
+        }
+
+        nv_unregister_user_pages(NV_GET_NV_STATE(pGpu),
+                                 osPageCount,
+                                 NULL /* import_priv */, ppPrivate);
+        memdescDestroy(pMemDesc);
+        return rmStatus;
+    }
+
+    return NV_OK;
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromPageArray
+(
+    OBJGPU  *pGpu,
+    NvP64    pDescriptor,
+    NvHandle hClient,
+    NvU32    flags,
+    NvU64   *pLimit,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void **ppPrivate
+)
+{
+    NV_STATUS rmStatus;
+
+    *ppPrivate = NvP64_VALUE(pDescriptor);
+
+    //
+    // Since the only type of memory permitted in this path
+    // is anonymous user memory, certain restrictions
+    // apply for the allocation flags:
+    //
+    //  1) anonymous memory is write-back cacheable, hence
+    //     the _COHERENCY flag must match.
+    //
+    //  2) the RM has no control over the location of the
+    //     associated pages in memory and thus cannot
+    //     honor requests for contiguous memory.
+    //
+    // These restrictions are enforced here to avoid subtle
+    // bugs later on.
+    //
+    if ((!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, flags) &&
+         !FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, flags)) ||
+         FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
+    {
+        return NV_ERR_INVALID_FLAGS;
+    }
+
+    rmStatus = osCreateMemdescFromPages(pGpu, (*pLimit + 1), flags,
+                                        NV_MEMORY_CACHED, ppMemDesc,
+                                        NULL /* pImportPriv */, ppPrivate);
+    if (rmStatus != NV_OK)
+    {
+        return rmStatus;
+    }
+
+    // All is well - wire up the cleanup callback now
+    memdescSetMemData(*ppMemDesc, memdescGetMemData(*ppMemDesc),
+                      osDestroyOsDescriptorPageArray);
+
+    return NV_OK;
+}
+
+/*!
+ * @brief Checks if the given address range overlaps with the BARs for any of
+ * the GPUs.
+ */
+static NV_STATUS
+osCheckGpuBarsOverlapAddrRange
+(
+    NvRangeU64 addrRange
+)
+{
+    NvRangeU64 gpuPhysAddrRange;
+    NvRangeU64 gpuPhysFbAddrRange;
+    NvRangeU64 gpuPhysInstAddrRange;
+    NvU32 gpuInstance;
+    OBJGPU *pGpu;
+    NvU32 gpuMask;
+    NV_STATUS rmStatus;
+
+    rmStatus = gpumgrGetGpuAttachInfo(NULL, &gpuMask);
+    NV_ASSERT_OR_RETURN(rmStatus == NV_OK, rmStatus);
+
+    gpuInstance = 0;
+    while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
+    {
+        NV_INIT_RANGE(gpuPhysAddrRange, pGpu->busInfo.gpuPhysAddr,
+            pGpu->busInfo.gpuPhysAddr +  pGpu->deviceMappings[0].gpuNvLength -1);
+
+        if (NV_IS_OVERLAPPING_RANGE(gpuPhysAddrRange, addrRange))
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s(): phys range 0x%016llx-0x%016llx overlaps with GPU BAR0\n",
+                      __FUNCTION__, addrRange.min, addrRange.max);
+            return NV_ERR_INVALID_ADDRESS;
+        }
+
+        if ((pGpu->pGpuArch->bGpuArchIsZeroFb) ||
+            (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)))
+        {
+            // No local FB or BAR2 for ZERO_FB and TEGRA_SOC_NVDISPLAY.
+            continue;
+        }
+
+        NV_INIT_RANGE(gpuPhysFbAddrRange, gpumgrGetGpuPhysFbAddr(pGpu),
+            gpumgrGetGpuPhysFbAddr(pGpu) + pGpu->fbLength -1);
+
+        NV_INIT_RANGE(gpuPhysInstAddrRange, pGpu->busInfo.gpuPhysInstAddr,
+            pGpu->busInfo.gpuPhysInstAddr + pGpu->instLength -1);
+
+        if (NV_IS_OVERLAPPING_RANGE(gpuPhysFbAddrRange, addrRange) ||
+            NV_IS_OVERLAPPING_RANGE(gpuPhysInstAddrRange, addrRange))
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s(): phys range 0x%016llx-0x%016llx overlaps with FB or GPU BAR2\n",
+                      __FUNCTION__, addrRange.min, addrRange.max);
+            return NV_ERR_INVALID_ADDRESS;
+        }
+    }
+
+    return NV_OK;
+}
+
+static NvU64
+_doWarBug4040336
+(
+    OBJGPU *pGpu,
+    NvU64 addr
+)
+{
+    if (gpuIsWarBug4040336Enabled(pGpu))
+    {
+        if ((addr & 0xffffffff00000000ULL) == 0x7fff00000000ULL)
+        {
+            addr = addr & 0xffffffffULL;
+        }
+    }
+    return addr;
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromIoMemory
+(
+    OBJGPU  *pGpu,
+    NvP64    pDescriptor,
+    NvHandle hClient,
+    NvU32    flags,
+    NvU64   *pLimit,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void    **ppPrivate
+)
+{
+    NV_STATUS rmStatus;
+    NvU32 gpuCachedFlags;
+    MEMORY_DESCRIPTOR *pMemDesc;
+    NvU64 *pPteArray;
+    NvRangeU64 physAddrRange;
+    NvU64 *base = 0;
+    NvBool bAllowMmap;
+    NvU64 size;
+
+    //
+    // Unlike the page array path, this one deals exclusively
+    // with I/O memory, which is expected to be contiguous
+    // physically, and which may only be accessed with uncached
+    // transactions.
+    //
+    if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags) ||
+        !FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
+    {
+        return NV_ERR_INVALID_FLAGS;
+    }
+
+    //
+    // _PEER_MAP_OVERRIDE flag is controlled by the RM and not the client.
+    //
+    // RM will set the _PEER_MAP_OVERRIDE_REQUIRED flag itself for IO memory
+    // memory imported with RmVidHeapControl.
+    //
+    if (FLD_TEST_DRF(OS02, _FLAGS, _PEER_MAP_OVERRIDE, _REQUIRED, flags))
+    {
+        //
+        // Don't allow MMIO mappings for unprivileged users
+        // This is a temporary WAR for bug 1630288 "[PeerSync] threat related
+        // to GPU"
+        //
+        KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+        if (!pKernelBif->peerMappingOverride && !osIsAdministrator())
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s(): permission denied, allowPeermapping=%d\n",
+                      __FUNCTION__, pKernelBif->peerMappingOverride);
+            return NV_ERR_INSUFFICIENT_PERMISSIONS;
+        }
+    }
+
+    bAllowMmap = !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NEVER_MAP, flags);
+
+    base = (void *)(NvUPtr)pDescriptor;
+
+    //
+    // There is an architectural deadlock scenario involved when full-duplex P2P
+    // enabled over BAR1. See #3 in the description of bug 1571948 which explains
+    // the classic deadlock. So, make sure to error out usermode's memory
+    // registration if a memory range falls within any of the available GPU's
+    // BAR window.
+    //
+    physAddrRange.min = *base;
+    physAddrRange.max = *base + *pLimit;
+
+    rmStatus = osCheckGpuBarsOverlapAddrRange(physAddrRange);
+    if (rmStatus != NV_OK)
+    {
+        return rmStatus;
+    }
+
+    //
+    // BF3's PCIe MMIO bus address at 0x800000000000(CPU PA 0x7fff00000000) is
+    // too high for Ampere to address. As a result, BF3's bus address is
+    // moved to < 4GB. Now, the CPU PA and the bus address are no longer 1:1
+    // and needs to be adjusted.
+    //
+    *base = _doWarBug4040336(pGpu, *base);
+
+    size = NV_ALIGN_UP64(*pLimit + 1, os_page_size);
+
+    rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0,
+                             NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM,
+                             NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE);
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s(): error %d while attempting to create the MMIO mapping\n",
+                  __FUNCTION__, rmStatus);
+        return rmStatus;
+    }
+
+    pMemDesc = *ppMemDesc;
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
+        gpuCachedFlags = NV_MEMORY_CACHED;
+    else
+        gpuCachedFlags = NV_MEMORY_UNCACHED;
+
+    memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
+    memdescSetMemData(pMemDesc, NULL, NULL);
+    memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
+    memdescSetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM, NV_TRUE);
+
+    pPteArray = memdescGetPteArray(pMemDesc, AT_CPU);
+    pPteArray[0] = *base;
+
+    *ppPrivate = NULL;
+
+    if (!NV_IS_ALIGNED64(pPteArray[0], os_page_size))
+    {
+        rmStatus = NV_ERR_INVALID_ARGUMENT;
+        NV_ASSERT_OR_GOTO(0, cleanup);
+    }
+    NvU64 osPageCount;
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, os_page_size), cleanup);
+        osPageCount = pMemDesc->PageCount;
+    }
+    else
+    {
+        NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, NV_RM_PAGE_SIZE), cleanup);
+        osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);
+    }
+
+    if (bAllowMmap)
+    {
+        rmStatus = nv_register_peer_io_mem(NV_GET_NV_STATE(pGpu), pPteArray,
+                                osPageCount,
+                                ppPrivate);
+        if (rmStatus != NV_OK)
+        {
+            memdescDestroy(pMemDesc);
+            return rmStatus;
+        }
+    }
+
+    memdescSetMemData(pMemDesc, *ppPrivate, NULL);
+
+    //
+    // memdescMapIommu() requires the OS-private data to be set on the memory
+    // descriptor, but we don't want to wire up the teardown callback just yet:
+    // that callback needs to unpin the pages, but that will already be done
+    // as part of failure handling further up the stack if memdescMapIommu()
+    // fails. So we only set up the priv-data cleanup callback once we're sure
+    // this call will succeed.
+    //
+    rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+cleanup:
+    if (rmStatus != NV_OK)
+    {
+        if (*ppPrivate != NULL)
+        {
+            nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), *ppPrivate);
+        }
+        memdescDestroy(pMemDesc);
+        return rmStatus;
+    }
+
+    // All is well - wire up the cleanup callback now
+    memdescSetMemData(pMemDesc, memdescGetMemData(pMemDesc),
+        osDestroyOsDescriptorFromIoMemory);
+
+    return NV_OK;
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromPhysAddr
+(
+    OBJGPU  *pGpu,
+    NvP64    pDescriptor,
+    NvHandle hClient,
+    NvU32    flags,
+    NvU64   *pLimit,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void    **ppPrivate
+)
+{
+    NV_STATUS rmStatus;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    MEMORY_DESCRIPTOR *pMemDesc;
+    NvU64 *pPteArray;
+    NvU64  base = 0;
+    NvU32  cache_type = NV_MEMORY_CACHED;
+    NvU64  memdescFlags = MEMDESC_FLAGS_NONE;
+    NvU64 *pPhys_addrs;
+    NvU64  num_os_pages;
+    NvU32  idx;
+    NvU64  size;
+
+    // Currently only work with contiguous sysmem allocations
+    if (!FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
+    {
+        return NV_ERR_INVALID_FLAGS;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_TYPE_SYNCPOINT, _APERTURE, flags))
+    {
+        // Syncpoint memory is uncached, DMA mapping needs to skip CPU sync.
+        cache_type = NV_MEMORY_UNCACHED;
+
+        //
+        // Syncpoint memory is NISO. Don't attempt to IOMMU map if the NISO
+        // IOMMU isn't enabled.
+        //
+        if (!NV_SOC_IS_NISO_IOMMU_PRESENT(nv))
+        {
+            memdescFlags |= MEMDESC_FLAGS_SKIP_IOMMU_MAPPING;
+        }
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
+    {
+        memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
+    }
+
+    base = (NvU64)pDescriptor;
+    size = NV_ALIGN_UP64((*pLimit + 1), os_page_size);
+
+    NV_ASSERT_OR_RETURN(NV_IS_ALIGNED64(base, os_page_size), NV_ERR_INVALID_ARGUMENT);
+
+    rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0,
+                             NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM,
+                             cache_type, memdescFlags);
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s(): error %d while creating memdesc for kernel memory\n",
+                  __FUNCTION__, rmStatus);
+        return rmStatus;
+    }
+
+    pMemDesc = *ppMemDesc;
+
+    memdescSetMemData(pMemDesc, NULL, NULL);
+    memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
+
+    pPteArray = memdescGetPteArray(pMemDesc, AT_CPU);
+    pPteArray[0] = base;
+
+    num_os_pages = size >> os_page_shift;
+    pPhys_addrs  = portMemAllocNonPaged(sizeof(NvU64) * num_os_pages);
+    if (pPhys_addrs == NULL)
+        goto cleanup_memdesc;
+
+    for (idx = 0; idx < num_os_pages; idx++) 
+    {
+        pPhys_addrs[idx] =  base + (idx * os_page_size); 
+    }
+
+    *ppPrivate = NULL;
+    rmStatus = nv_register_phys_pages(nv, pPhys_addrs, num_os_pages,
+                                      memdescGetCpuCacheAttrib(pMemDesc),
+                                      ppPrivate);
+    if (rmStatus != NV_OK)
+        goto cleanup_memdesc;
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, size, os_page_size));
+    }
+    else
+    {
+        NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, size, NV_RM_PAGE_SIZE));
+    }
+
+    // If IOMMU skip flag wasn't set earlier, create IOVA mapping.
+    if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_SKIP_IOMMU_MAPPING))
+    {
+        //
+        // memdescMapIommu() requires the OS-private data to be set on the memory
+        // descriptor, but we don't want to wire up the teardown callback just yet:
+        // that callback needs to unpin the pages, but that will already be done
+        // as part of failure handling further up the stack if memdescMapIommu()
+        // fails. So we only set up the priv-data cleanup callback once we're sure
+        // this call will succeed.
+        //
+        memdescSetMemData(pMemDesc, *ppPrivate, NULL);
+
+        rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+        if (rmStatus != NV_OK)
+            goto cleanup_pages;
+    }
+
+    // All is well - wire up the cleanup callback now
+    memdescSetMemData(pMemDesc, *ppPrivate,
+        osDestroyOsDescriptorFromPhysAddr);
+
+    portMemFree(pPhys_addrs);
+
+    return NV_OK;
+
+cleanup_pages:
+    if (*ppPrivate != NULL)
+    {
+        nv_unregister_phys_pages(NV_GET_NV_STATE(pGpu), *ppPrivate);
+    }
+
+cleanup_memdesc:
+    memdescDestroy(pMemDesc);
+
+    portMemFree(pPhys_addrs);
+
+    return rmStatus;
+}
+
+static NV_STATUS
+_createMemdescFromDmaBufSgtHelper
+(
+    OBJGPU  *pGpu,
+    NvU32    flags,
+    void *pImportPriv,
+    struct sg_table *pImportSgt,
+    NvU32 size,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void **ppPrivate,
+    MEM_DATA_RELEASE_CALL_BACK *pMemDataReleaseCallback
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+    NvU32 cacheType = NV_MEMORY_UNCACHED;
+    MEMORY_DESCRIPTOR *pMemDesc;
+    NvU64 memdescFlags = MEMDESC_FLAGS_NONE;
+    NvU32 gpuCachedFlags;
+    NvBool isPeerMmio = NV_FALSE;
+    NvU64 osPageCount;
+
+    //
+    // Align size up to os page size. This is important in
+    // order to support submemdesc mappings at native page size.
+    // Once dynamic tracking is enabled this will not be needed
+    // as all submemor
+    //
+    size = NV_ALIGN_UP64(size, os_page_size);
+
+    osPageCount = size >> os_page_shift;
+
+    NV_ASSERT((pMemDataReleaseCallback == osDestroyOsDescriptorFromDmaBuf) ||
+              (pMemDataReleaseCallback == osDestroyOsDescriptorFromSgt));
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_COMBINE, flags))
+    {
+        cacheType = NV_MEMORY_WRITECOMBINED;
+    }
+    else if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags))
+    {
+        cacheType = NV_MEMORY_CACHED;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
+    {
+        gpuCachedFlags = NV_MEMORY_CACHED;
+    }
+    else
+    {
+        gpuCachedFlags = NV_MEMORY_UNCACHED;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_TYPE_SYNCPOINT, _APERTURE, flags))
+    {
+        // Syncpoint memory is uncached.
+        if ((cacheType      != NV_MEMORY_UNCACHED) ||
+            (gpuCachedFlags != NV_MEMORY_UNCACHED))
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "%s(): Error: Syncpoint memory region should be uncached!!!\n",
+                      __FUNCTION__);
+            return NV_ERR_INVALID_FLAGS;
+        }
+
+        isPeerMmio           = NV_TRUE;
+
+        NV_PRINTF(LEVEL_INFO,
+                  "%s(): Syncpoint type sgt!\n", __FUNCTION__);
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
+    {
+        memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, _YES, flags))
+    {
+        memdescFlags |= MEMDESC_FLAGS_USER_READ_ONLY;
+    }
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, _YES, flags))
+    {
+        memdescFlags |= MEMDESC_FLAGS_DEVICE_READ_ONLY;
+    }
+
+    rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0,
+                             NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM,
+                             cacheType, memdescFlags);
+    if (rmStatus != NV_OK)
+    {
+        return rmStatus;
+    }
+
+    pMemDesc = *ppMemDesc;
+
+    memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
+    memdescSetMemData(pMemDesc, NULL, NULL);
+    memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
+    memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
+
+    *ppPrivate = NULL;
+    rmStatus = nv_register_sgt(NV_GET_NV_STATE(pGpu),
+                               memdescGetPteArray(pMemDesc, AT_CPU),
+                               osPageCount,
+                               memdescGetCpuCacheAttrib(pMemDesc),
+                               ppPrivate,
+                               pImportSgt,
+                               pImportPriv,
+                               isPeerMmio);
+    if (rmStatus != NV_OK)
+    {
+        memdescDestroy(pMemDesc);
+        return rmStatus;
+    }
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, os_page_size), cleanup);
+    }
+    else
+    {
+        NV_ASSERT_OK_OR_GOTO(rmStatus, memdescSetAllocSizeFields(pMemDesc, size, NV_RM_PAGE_SIZE), cleanup);
+    }
+
+    if (!NV_IS_ALIGNED64(memdescGetPhysAddr(pMemDesc, AT_CPU, 0), os_page_size))
+    {
+        rmStatus = NV_ERR_INVALID_ARGUMENT;
+        NV_ASSERT_OR_GOTO(0, cleanup);
+    }
+
+    //
+    // If the OS layer doesn't think in RM page size, we need to inflate the
+    // PTE array into RM pages.
+    //
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion.
+        NV_ASSERT_OR_ELSE(os_page_size == pMemDesc->pageArrayGranularity, rmStatus = NV_ERR_INVALID_ARGUMENT; goto cleanup);
+    }
+    else if ((NV_RM_PAGE_SIZE < os_page_size) &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+    {
+        RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU), pMemDesc->PageCount);
+    }
+
+    memdescSetMemData(*ppMemDesc, *ppPrivate, NULL);
+
+    //
+    // memdescMapIommu() requires the OS-private data to be set on the memory
+    // descriptor, but we don't want to wire up the teardown callback just yet:
+    // that callback does teardown that will already be done as part of failure
+    // handling further up the stack if memdescMapIommu() fails. So we only
+    // setup the priv-data cleanup callback once we're sure this call will
+    // succeed.
+    //
+    rmStatus = memdescMapIommu(*ppMemDesc, pGpu->busInfo.iovaspaceId);
+cleanup:
+    if (rmStatus != NV_OK)
+    {
+        if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+        {
+            // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion.
+            NV_ASSERT(os_page_size == pMemDesc->pageArrayGranularity);
+        }
+        else if ((NV_RM_PAGE_SIZE < os_page_size) &&
+                !memdescGetContiguity(pMemDesc, AT_CPU))
+        {
+            RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+                                    pMemDesc->PageCount);
+        }
+        if (*ppPrivate != NULL)
+        {
+            nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
+                              (void **) &pImportPriv, *ppPrivate);
+        }
+        memdescDestroy(pMemDesc);
+        return rmStatus;
+    }
+
+    // All is well - wire up the cleanup callback now
+    memdescSetMemData(*ppMemDesc, *ppPrivate, pMemDataReleaseCallback);
+
+    return rmStatus;
+}
+
+static NV_STATUS
+_createMemdescFromDmaBuf
+(
+    OBJGPU  *pGpu,
+    NvU32    flags,
+    nv_dma_buf_t *pImportPriv,
+    struct sg_table *pImportSgt,
+    NvU32 size,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void **ppPrivate
+)
+{
+    NV_STATUS rmStatus =
+        _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPriv, pImportSgt,
+                                          size, ppMemDesc, ppPrivate,
+                                          osDestroyOsDescriptorFromDmaBuf);
+    if (rmStatus != NV_OK)
+    {
+        nv_dma_release_dma_buf(pImportPriv);
+    }
+
+    return rmStatus;
+}
+
+static NV_STATUS
+_createMemdescFromSgt
+(
+    OBJGPU  *pGpu,
+    NvU32    flags,
+    struct drm_gem_object *pImportPrivGem,
+    struct sg_table *pImportSgt,
+    NvU32 size,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void **ppPrivate
+)
+{
+    NV_STATUS rmStatus =
+        _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPrivGem,
+                                          pImportSgt, size, ppMemDesc,
+                                          ppPrivate,
+                                          osDestroyOsDescriptorFromSgt);
+    if (rmStatus != NV_OK)
+    {
+        nv_dma_release_sgt(pImportSgt, pImportPrivGem);
+    }
+
+    return rmStatus;
+}
+
+static nv_dma_device_t *GetDmaDeviceForImport
+(
+    nv_state_t *nv,
+    NvU32 flags
+)
+{
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags) &&
+        (nv->niso_dma_dev != NULL))
+    {
+        return nv->niso_dma_dev;
+    }
+    else
+    {
+        return nv->dma_dev;
+    }
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromFileHandle
+(
+    OBJGPU  *pGpu,
+    NvP64    pDescriptor,
+    NvHandle hClient,
+    NvU32    flags,
+    NvU64   *pLimit,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void    **ppPrivate
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_dma_device_t *dma_dev = NULL;
+    NvU32 size = 0;
+    nv_dma_buf_t *pImportPriv = NULL;
+    struct sg_table *pImportSgt = NULL;
+    NvBool bRoDeviceMap = NV_FALSE;
+    NvS32 fd;
+
+    fd = (NvS32)((NvU64)pDescriptor);
+    if ((NvU64)fd != (NvU64)pDescriptor)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s(): fd must fit within a signed 32-bit integer!\n",
+                  __FUNCTION__);
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    dma_dev = GetDmaDeviceForImport(nv, flags);
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, _YES, flags))
+    {
+        bRoDeviceMap = NV_TRUE;
+        NV_PRINTF(LEVEL_INFO,
+                  "%s(): RO DMA Mapping - flags [%x]!\n",
+                  __FUNCTION__, flags);
+    }
+
+    rmStatus = nv_dma_import_from_fd(dma_dev, fd, bRoDeviceMap, &size,
+                                     &pImportSgt, &pImportPriv);
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s(): Error (%d) while trying to import fd!\n",
+                  __FUNCTION__, rmStatus);
+        return rmStatus;
+    }
+
+    return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv,
+                                    pImportSgt,
+                                    size, ppMemDesc, ppPrivate);
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromSgtPtr
+(
+    OBJGPU  *pGpu,
+    NvP64    pDescriptor,
+    NvHandle hClient,
+    NvU32    flags,
+    NvU64   *pLimit,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void    **ppPrivate
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS *params =
+        (NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS*)((NvUPtr) pDescriptor);
+
+    struct sg_table *sgt = params->sgt;
+    struct drm_gem_object *gem = params->gem;
+
+    rmStatus = nv_dma_import_sgt(nv->dma_dev, sgt, gem);
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "%s(): Error (%d) while trying to import sgt!\n",
+                  __FUNCTION__, rmStatus);
+        return rmStatus;
+    }
+
+    return _createMemdescFromSgt(pGpu, flags, gem, sgt,
+                                 (*pLimit + 1), ppMemDesc, ppPrivate);
+}
+
+static NV_STATUS
+osCreateOsDescriptorFromDmaBufPtr
+(
+    OBJGPU  *pGpu,
+    NvP64    pDescriptor,
+    NvHandle hClient,
+    NvU32    flags,
+    NvU64   *pLimit,
+    MEMORY_DESCRIPTOR **ppMemDesc,
+    void    **ppPrivate
+)
+{
+    NV_STATUS rmStatus = NV_OK;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_dma_device_t *dma_dev = NULL;
+    NvU32 size = 0;
+    nv_dma_buf_t *pImportPriv = NULL;
+    struct sg_table *pImportSgt = NULL;
+    void *dmaBuf = (void*)((NvUPtr)pDescriptor);
+    NvBool bRoDeviceMap = NV_FALSE;
+
+    dma_dev = GetDmaDeviceForImport(nv, flags);
+
+    if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_DEVICE_READ_ONLY, _YES, flags))
+    {
+        bRoDeviceMap = NV_TRUE;
+        NV_PRINTF(LEVEL_INFO,
+                  "%s(): RO DMA Mapping - flags [%x]!\n",
+                  __FUNCTION__, flags);
+    }
+
+    rmStatus = nv_dma_import_dma_buf(dma_dev, dmaBuf, bRoDeviceMap, &size,
+                                     &pImportSgt, &pImportPriv);
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF_COND(rmStatus == NV_ERR_NOT_SUPPORTED, LEVEL_INFO, LEVEL_ERROR,
+                       "Error (%d) while trying to import dma_buf!\n", rmStatus);
+        return rmStatus;
+    }
+
+    return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv,
+                                    pImportSgt,
+                                    size, ppMemDesc, ppPrivate);
+}
+
+static void
+osDestroyOsDescriptorFromPhysAddr
+(
+    PMEMORY_DESCRIPTOR pMemDesc
+)
+{
+    OBJGPU *pGpu  = pMemDesc->pGpu;
+    void *pPrivate;
+
+    pPrivate = memdescGetMemData(pMemDesc);
+    NV_ASSERT(pPrivate != NULL);
+
+    if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_SKIP_IOMMU_MAPPING))
+        memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+
+    nv_unregister_phys_pages(NV_GET_NV_STATE(pGpu), pPrivate);
+}
+
+static void
+osDestroyOsDescriptorFromIoMemory
+(
+    PMEMORY_DESCRIPTOR pMemDesc
+)
+{
+    OBJGPU *pGpu  = pMemDesc->pGpu;
+    void *pPrivate = memdescGetMemData(pMemDesc);
+
+    if (pPrivate == NULL)
+    {
+        return;
+    }
+
+    nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), pPrivate);
+}
+
+static void
+osDestroyOsDescriptorPageArray
+(
+    MEMORY_DESCRIPTOR *pMemDesc
+)
+{
+    OBJGPU   *pGpu        = pMemDesc->pGpu;
+    NvU64     osPageCount;
+    NV_STATUS status;
+    void     *pPrivate;
+
+    pPrivate = memdescGetMemData(pMemDesc);
+
+    NV_ASSERT(pPrivate != NULL);
+
+    //
+    // TODO: Bug 1811006: Notably skip any IOMMU mapping management as the
+    // pMemDesc->pGpu might have been torn down already and the pGpu passed in
+    // doesn't necessarily have IOMMU mappings. For now just allow memdescDestroy()
+    // to clean up whatever is there (this may not work correctly either if any
+    // of the IOMMU mappings have outlasted their VASPACEs). This should
+    // be cleaned up once the fix for bug 1811006 is known.
+    //
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion.
+        NV_ASSERT_OR_RETURN_VOID(os_page_size == pMemDesc->pageArrayGranularity);
+        osPageCount = pMemDesc->PageCount;
+    }
+    else
+    {
+        osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);
+        if ((NV_RM_PAGE_SIZE < os_page_size) &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+        {
+            RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+                                    pMemDesc->PageCount);
+        }
+    }
+
+    nv_unregister_user_pages(NV_GET_NV_STATE(pGpu), osPageCount,
+                             NULL /* import_priv */, &pPrivate);
+
+    if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_FOREIGN_PAGE))
+    {
+        os_free_mem(pPrivate);
+    }
+    else
+    {
+        //
+        // We use MEMDESC_FLAGS_USER_READ_ONLY because this reflects the
+        // NVOS02_FLAGS_ALLOC_USER_READ_ONLY flag value passed into
+        // os_lock_user_pages(). That flag also results in
+        // MEMDESC_FLAGS_DEVICE_READ_ONLY being set.
+        //
+        NvBool writable = !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY);
+        NvU32 flags = DRF_NUM(_LOCK_USER_PAGES, _FLAGS, _WRITE, writable);
+        status = os_unlock_user_pages(osPageCount, pPrivate, flags);
+        NV_ASSERT(status == NV_OK);
+    }
+}
+
+static void
+osDestroyOsDescriptorFromDmaBuf
+(
+    PMEMORY_DESCRIPTOR pMemDesc
+)
+{
+    OBJGPU *pGpu = pMemDesc->pGpu;
+    void *pPrivate = memdescGetMemData(pMemDesc);
+
+    struct sg_table *pImportSgt;
+    void *pImportPriv;
+
+    /*
+     * Unmap IOMMU now or we will get a kernel crash when it is unmapped after
+     * pImportSgt is freed.
+     */
+    memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion.
+        NV_ASSERT_OR_RETURN_VOID(os_page_size == pMemDesc->pageArrayGranularity);
+    }
+    else if ((NV_RM_PAGE_SIZE < os_page_size) &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+    {
+        RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+                                pMemDesc->PageCount);
+    }
+
+    nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
+                      &pImportPriv, pPrivate);
+
+    /*
+     * pImportSgt doesn't need to be passed to nv_dma_release_dma_buf() because
+     * the DMA-BUF associated with pImportPriv already has a reference to the
+     * SGT.
+     */
+
+    nv_dma_release_dma_buf(pImportPriv);
+}
+
+static void
+osDestroyOsDescriptorFromSgt
+(
+    PMEMORY_DESCRIPTOR pMemDesc
+)
+{
+    OBJGPU *pGpu = pMemDesc->pGpu;
+    void *pPrivate = memdescGetMemData(pMemDesc);
+
+    struct sg_table *pImportSgt;
+    struct drm_gem_object *pImportPrivGem;
+
+    NV_ASSERT(pPrivate != NULL);
+
+    /*
+     * Unmap IOMMU now or we will get a kernel crash when it is unmapped after
+     * pImportSgt is freed.
+     */
+    memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
+
+    if (IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
+    {
+        // Because os_page_size == pMemDesc->pageArrayGranularity, we don't need to do any page array conversion.
+        NV_ASSERT_OR_RETURN_VOID(os_page_size == pMemDesc->pageArrayGranularity);
+    }
+    else if ((NV_RM_PAGE_SIZE < os_page_size) &&
+            !memdescGetContiguity(pMemDesc, AT_CPU))
+    {
+        RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
+                                pMemDesc->PageCount);
+    }
+
+    nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
+                      (void **) &pImportPrivGem, pPrivate);
+
+    nv_dma_release_sgt(pImportSgt, pImportPrivGem);
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osnvlink.c b/userspace/sdks/nv570/nvalloc/unix/src/osnvlink.c
new file mode 100644
index 0000000..af65f9e
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/osnvlink.c
@@ -0,0 +1,783 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include "kernel/gpu/nvlink/kernel_nvlink.h"
+#include "osapi.h"
+
+#if defined(INCLUDE_NVLINK_LIB)
+#include "nvlink.h"
+//
+// The functions in this file are a workaround for a significant design flaw
+// where RM callbacks for the nvlink library are built with the altstack
+// enabled, but the nvlink library is not built with altstack support. Whenever
+// the library calls a callback, the stack switching needs to be accounted for
+// or else we will observe corruption of data structures in the nvlink library
+// as data is pushed onto what the callback thinks is the stack. See bug
+// 1710300.
+//
+// This bug has also exposed other problems, such as the complete lack of
+// locking awareness by these callbacks (e.g., assumption that the RMAPI and
+// GPU locks are always held on entry, which is not a legitimate assumption).
+// For now, we ignore that just to unblock testing.
+//
+extern NvlStatus knvlinkCoreAddLinkCallback(struct nvlink_link *);
+extern NvlStatus knvlinkCoreRemoveLinkCallback(struct nvlink_link *);
+extern NvlStatus knvlinkCoreLockLinkCallback(struct nvlink_link *);
+extern void      knvlinkCoreUnlockLinkCallback(struct nvlink_link *);
+extern NvlStatus knvlinkCoreQueueLinkChangeCallback(struct nvlink_link_change *);
+extern NvlStatus knvlinkCoreSetDlLinkModeCallback(struct nvlink_link *, NvU64, NvU32);
+extern NvlStatus knvlinkCoreGetDlLinkModeCallback(struct nvlink_link *, NvU64 *);
+extern NvlStatus knvlinkCoreSetTlLinkModeCallback(struct nvlink_link *, NvU64, NvU32);
+extern NvlStatus knvlinkCoreGetTlLinkModeCallback(struct nvlink_link *, NvU64 *);
+extern NvlStatus knvlinkCoreGetTxSublinkModeCallback(struct nvlink_link *, NvU64 *, NvU32 *);
+extern NvlStatus knvlinkCoreSetTxSublinkModeCallback(struct nvlink_link *, NvU64, NvU32);
+extern NvlStatus knvlinkCoreGetRxSublinkModeCallback(struct nvlink_link *, NvU64 *, NvU32 *);
+extern NvlStatus knvlinkCoreSetRxSublinkModeCallback(struct nvlink_link *, NvU64, NvU32);
+extern NvlStatus knvlinkCoreReadDiscoveryTokenCallback(struct nvlink_link *, NvU64 *);
+extern NvlStatus knvlinkCoreWriteDiscoveryTokenCallback(struct nvlink_link *, NvU64);
+extern void      knvlinkCoreTrainingCompleteCallback(struct nvlink_link *);
+extern void      knvlinkCoreGetUphyLoadCallback(struct nvlink_link *, NvBool*);
+extern NvlStatus knvlinkCoreGetCciLinkModeCallback(struct nvlink_link *, NvU64 *);
+
+/*!
+ * @brief Helper to allocate an alternate stack from within core RM.
+ *
+ * This needs to be an NV_API_CALL (built to use the original stack instead
+ * of the altstack) since it is called before we switch to using the altstack.
+ */
+static NV_STATUS NV_API_CALL osNvlinkAllocAltStack(nvidia_stack_t **pSp)
+{
+    NV_STATUS status = NV_OK;
+    nvidia_stack_t *sp = NULL;
+#if defined(NVCPU_X86_64) && defined(__use_altstack__)
+    status = os_alloc_mem((void **)&sp, sizeof(nvidia_stack_t));
+    if (status == NV_OK)
+    {
+        sp->size = sizeof(sp->stack);
+        sp->top  = sp->stack + sp->size;
+    }
+#endif
+    *pSp = sp;
+    return status;
+}
+
+/*!
+ * @brief Helper to free an alternate stack from within core RM.
+ *
+ * This needs to be an NV_API_CALL (built to use the original stack instead
+ * of the altstack) since it is called after we've switched back to using the
+ * original stack.
+ */
+static void NV_API_CALL osNvlinkFreeAltStack(nvidia_stack_t *sp)
+{
+#if defined(NVCPU_X86_64) && defined(__use_altstack__)
+    os_free_mem(sp);
+#endif
+}
+
+static NV_STATUS NV_API_CALL osNvlinkGetAltStack(nvidia_stack_t **sp)
+{
+    return osNvlinkAllocAltStack(sp);
+}
+static void NV_API_CALL osNvlinkPutAltStack(nvidia_stack_t *sp)
+{
+    osNvlinkFreeAltStack(sp);
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_add_link
+(
+    struct nvlink_link *link
+)
+{
+    void *fp;
+    NvlStatus status;
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    status = knvlinkCoreAddLinkCallback(link);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_remove_link
+(
+    struct nvlink_link *link
+)
+{
+    void *fp;
+    NvlStatus status;
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    status = knvlinkCoreRemoveLinkCallback(link);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_lock_link
+(
+    struct nvlink_link *link
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreLockLinkCallback(link);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static void NV_API_CALL rm_nvlink_ops_unlock_link
+(
+    struct nvlink_link *link
+)
+{
+    void *fp;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    knvlinkCoreUnlockLinkCallback(link);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_queue_link_change
+(
+    struct nvlink_link_change *link_change
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreQueueLinkChangeCallback(link_change);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_dl_link_mode
+(
+    struct nvlink_link *link,
+    NvU64 mode,
+    NvU32 flags
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreSetDlLinkModeCallback(link, mode, flags);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_dl_link_mode
+(
+    struct nvlink_link *link,
+    NvU64 *mode
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreGetDlLinkModeCallback(link, mode);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_tl_link_mode
+(
+    struct nvlink_link *link,
+    NvU64 mode,
+    NvU32 flags
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreSetTlLinkModeCallback(link, mode, flags);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_tl_link_mode
+(
+    struct nvlink_link *link,
+    NvU64 *mode
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreGetTlLinkModeCallback(link, mode);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_tx_mode
+(
+    struct nvlink_link *link,
+    NvU64 mode,
+    NvU32 flags
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreSetTxSublinkModeCallback(link, mode, flags);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_tx_mode
+(
+    struct nvlink_link *link,
+    NvU64 *mode,
+    NvU32 *subMode
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreGetTxSublinkModeCallback(link, mode, subMode);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_rx_mode
+(
+    struct nvlink_link *link,
+    NvU64 mode,
+    NvU32 flags
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreSetRxSublinkModeCallback(link, mode, flags);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_rx_mode
+(
+    struct nvlink_link *link,
+    NvU64 *mode,
+    NvU32 *subMode
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreGetRxSublinkModeCallback(link, mode, subMode);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_set_link_rx_detect
+(
+    struct nvlink_link *link,
+    NvU32 flags
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreSetRxSublinkDetectCallback(link, flags);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_link_rx_detect
+(
+    struct nvlink_link *link
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreGetRxSublinkDetectCallback(link);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static void NV_API_CALL rm_nvlink_get_uphy_load
+(
+    struct nvlink_link *link,
+    NvBool *bUnlocked
+)
+{
+    void *fp;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    knvlinkCoreGetUphyLoadCallback(link, bUnlocked);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_read_link_discovery_token
+(
+    struct nvlink_link *link,
+    NvU64 *token
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreReadDiscoveryTokenCallback(link, token);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_write_link_discovery_token
+(
+    struct nvlink_link *link,
+    NvU64 token
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreWriteDiscoveryTokenCallback(link, token);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+
+    return status;
+}
+
+static void NV_API_CALL rm_nvlink_ops_training_complete
+(
+    struct nvlink_link *link
+)
+{
+    void *fp;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t *sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    knvlinkCoreTrainingCompleteCallback(link);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_ali_training
+(
+    struct nvlink_link *link
+)
+{
+    void *fp;
+    NvlStatus status;
+    THREAD_STATE_NODE threadState = {0};
+    nvidia_stack_t * sp;
+
+    if (osNvlinkGetAltStack(&sp) != NV_OK)
+    {
+        return NVL_ERR_GENERIC;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp, fp);
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+    status = knvlinkCoreAliTrainingCallback(link);
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    NV_EXIT_RM_RUNTIME(sp, fp);
+
+    osNvlinkPutAltStack(sp);
+    return status;
+}
+
+static NvlStatus NV_API_CALL rm_nvlink_ops_get_cci_link_mode
+(
+    struct nvlink_link *link,
+    NvU64 *mode
+)
+{
+    return NVL_SUCCESS;
+}
+
+#endif /* defined(INCLUDE_NVLINK_LIB) */
+
+const struct nvlink_link_handlers* osGetNvlinkLinkCallbacks(void)
+{
+#if defined(INCLUDE_NVLINK_LIB)
+    static const struct nvlink_link_handlers rm_nvlink_link_ops =
+    {
+        .add                        = rm_nvlink_ops_add_link,
+        .remove                     = rm_nvlink_ops_remove_link,
+        .lock                       = rm_nvlink_ops_lock_link,
+        .unlock                     = rm_nvlink_ops_unlock_link,
+        .queue_link_change          = rm_nvlink_ops_queue_link_change,
+        .set_dl_link_mode           = rm_nvlink_ops_set_dl_link_mode,
+        .get_dl_link_mode           = rm_nvlink_ops_get_dl_link_mode,
+        .set_tl_link_mode           = rm_nvlink_ops_set_tl_link_mode,
+        .get_tl_link_mode           = rm_nvlink_ops_get_tl_link_mode,
+        .set_tx_mode                = rm_nvlink_ops_set_link_tx_mode,
+        .get_tx_mode                = rm_nvlink_ops_get_link_tx_mode,
+        .set_rx_mode                = rm_nvlink_ops_set_link_rx_mode,
+        .get_rx_mode                = rm_nvlink_ops_get_link_rx_mode,
+        .set_rx_detect              = rm_nvlink_ops_set_link_rx_detect,
+        .get_rx_detect              = rm_nvlink_ops_get_link_rx_detect,
+        .write_discovery_token      = rm_nvlink_ops_write_link_discovery_token,
+        .read_discovery_token       = rm_nvlink_ops_read_link_discovery_token,
+        .training_complete          = rm_nvlink_ops_training_complete,
+        .get_uphy_load              = rm_nvlink_get_uphy_load,
+        .get_cci_link_mode          = rm_nvlink_ops_get_cci_link_mode,
+        .ali_training               = rm_nvlink_ops_ali_training,
+    };
+
+    return &rm_nvlink_link_ops;
+#else
+    return NULL;
+#endif
+}
+
+/*
+ * @brief Verif only function to get the chiplib overrides for link connection
+ * state for all NVLINKs.
+ *
+ * If chiplib overrides exist, each link can either be enabled (1) or disabled (0)
+ *
+ * @param[in]   pGpu                GPU object pointer
+ * @param[in]   maxLinks            Size of pLinkConnection array
+ * @param[out]  pLinkConnection     array of pLinkConnection values to be populated by MODS
+ *
+ * @return      NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available)
+ */
+NV_STATUS
+osGetForcedNVLinkConnection
+(
+    OBJGPU *pGpu,
+    NvU32   maxLinks,
+    NvU32   *pLinkConnection
+)
+{
+    int i, ret;
+    NV_STATUS status;
+    char path[64];
+
+    NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER);
+    NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED);
+    NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT);
+
+    for (i = 0; i < maxLinks; i++)
+    {
+        ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|NVLink%u", i);
+        NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1)));
+
+        status = gpuSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]);
+        if (status == NV_OK)
+        {
+            NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__,
+                      path, pLinkConnection[i]);
+        }
+        else
+        {
+            NV_PRINTF(LEVEL_INFO, "%s: gpuSimEscapeRead for '%s' failed (%u)\n",
+                      __FUNCTION__, path, status);
+            return NV_ERR_NOT_SUPPORTED;
+        }
+    }
+    return NV_OK;
+}
+
+/*
+ * @brief Get Platform suggested NVLink linerate
+ *
+ * NVLink will use this function to get the platform suggested linerate
+ * if available in FRU or device tree.
+ *
+ * @param[in]   pGpu                GPU object pointer
+ * @param[out]  NvU32 *             Suggested datarate
+ *
+ * @return      NV_OK or NV_ERR_NOT_SUPPORTED (platform linerate data not available)
+ */
+NV_STATUS
+osGetPlatformNvlinkLinerate
+(
+    OBJGPU *pGpu,
+    NvU32   *lineRate
+)
+{
+    // TODO: Remove this function
+    return NV_ERR_NOT_SUPPORTED;
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/osunix.c b/userspace/sdks/nv570/nvalloc/unix/src/osunix.c
new file mode 100644
index 0000000..635bf86
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/osunix.c
@@ -0,0 +1,42 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+/***************************** HW State Routines ***************************\
+*                                                                           *
+*         Fills in os specific function pointers for the Unix OS object.    *
+*                                                                           *
+\***************************************************************************/
+
+#include <os/os.h>
+#include <core/system.h>
+
+void
+osInitObjOS(OBJOS *pOS)
+{
+    OBJSYS *pSys = SYS_GET_INSTANCE();
+
+    pSys->setProperty(pSys, PDB_PROP_SYS_ONDEMAND_VBLANK_CONTROL_ENABLE_DEFAULT, NV_TRUE);
+    pOS->setProperty(pOS, PDB_PROP_OS_CACHED_MEMORY_MAPPINGS_FOR_ACPI_TABLE, NV_TRUE);
+    pOS->setProperty(pOS, PDB_PROP_OS_LIMIT_GPU_RESET, NV_TRUE);
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/power-management-tegra.c b/userspace/sdks/nv570/nvalloc/unix/src/power-management-tegra.c
new file mode 100644
index 0000000..38c91d0
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/power-management-tegra.c
@@ -0,0 +1,100 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+
+#include <nv-priv.h>
+#include <osapi.h>
+#include <core/locks.h>
+
+NV_STATUS
+RmPowerManagementTegra(
+    OBJGPU *pGpu,
+    nv_pm_action_t pmAction
+)
+{
+    //
+    // Default to NV_OK. there may cases where resman is loaded, but
+    // no devices are allocated (we're still at the console). in these
+    // cases, it's fine to let the system do whatever it wants.
+    //
+    NV_STATUS rmStatus = NV_OK;
+
+    if (pGpu)
+    {
+        nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+        nv_priv_t  *nvp = NV_GET_NV_PRIV(nv);
+
+        switch (pmAction)
+        {
+            case NV_PM_ACTION_HIBERNATE:
+                nvp->pm_state.InHibernate = NV_TRUE;
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH, NV_TRUE);
+                rmStatus = gpuStateUnload(pGpu, GPU_STATE_FLAGS_PM_HIBERNATE |
+                                                GPU_STATE_FLAGS_PRESERVING |
+                                                GPU_STATE_FLAGS_PM_TRANSITION);
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_HIBERNATE, NV_TRUE);
+                break;
+
+            case NV_PM_ACTION_STANDBY:
+                nvp->pm_state.InHibernate = NV_FALSE;
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH, NV_TRUE);
+                rmStatus = gpuStateUnload(pGpu, GPU_STATE_FLAGS_PM_SUSPEND |
+                                                GPU_STATE_FLAGS_PRESERVING |
+                                                GPU_STATE_FLAGS_PM_TRANSITION);;
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_STANDBY, NV_TRUE);
+                pGpu->bInD3Cold = NV_TRUE;
+                break;
+
+            case NV_PM_ACTION_RESUME:
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_RESUME_CODEPATH, NV_TRUE);
+
+                if (nvp->pm_state.InHibernate)
+                {
+                    rmStatus = gpuStateLoad(pGpu, GPU_STATE_FLAGS_PM_HIBERNATE |
+                                                  GPU_STATE_FLAGS_PRESERVING |
+                                                  GPU_STATE_FLAGS_PM_TRANSITION);;
+                    pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_HIBERNATE, NV_FALSE);
+                }
+                else
+                {
+                    rmStatus = gpuStateLoad(pGpu, GPU_STATE_FLAGS_PM_SUSPEND |
+                                                  GPU_STATE_FLAGS_PRESERVING |
+                                                  GPU_STATE_FLAGS_PM_TRANSITION);;
+                    pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_STANDBY, NV_FALSE);
+                    pGpu->bInD3Cold = NV_FALSE;
+                }
+
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH, NV_FALSE);
+                pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_PM_RESUME_CODEPATH, NV_FALSE);
+                break;
+
+            default:
+                rmStatus = NV_ERR_INVALID_ARGUMENT;
+                break;
+        }
+    }
+
+    return rmStatus;
+}
+
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/registry.c b/userspace/sdks/nv570/nvalloc/unix/src/registry.c
new file mode 100644
index 0000000..70bbcc5
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/registry.c
@@ -0,0 +1,712 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include <nv-priv.h>
+#include <nvos.h>
+
+#if defined(DEBUG_REGISTRY)
+#define DBG_REG_PRINTF(a, ...) \
+    NV_PRINTF(LEVEL_INFO, a, ##__VA_ARGS__)
+#else
+#define DBG_REG_PRINTF(a, ...)
+#endif
+
+static NvS32 stringCaseCompare(
+    const char *string1,
+    const char *string2
+)
+{
+    NvU8 c1, c2;
+
+    do
+    {
+        c1 = *string1, c2 = *string2;
+        if (c1 >= 'A' && c1 <= 'Z')
+            c1 += ('a' - 'A');
+        if (c2 >= 'A' && c2 <= 'Z')
+            c2 += ('a' - 'A');
+        string1++, string2++;
+    }
+    while ((c1 == c2) && (c1 != '\0'));
+
+    return (c1 - c2);
+}
+
+static nv_reg_entry_t *the_registry = NULL;
+
+static nv_reg_entry_t* regCreateNewRegistryKey(
+    nv_state_t *nv,
+    const char *regParmStr
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    nv_reg_entry_t *new_reg = NULL;
+    char *new_ParmStr = NULL;
+    NvU32 parm_size;
+
+    if (regParmStr == NULL)
+    {
+        DBG_BREAKPOINT();
+        return NULL;
+    }
+
+    new_reg = portMemAllocNonPaged(sizeof(nv_reg_entry_t));
+    if (NULL == new_reg)
+    {
+        NV_PRINTF(LEVEL_ERROR, "failed to grow registry\n");
+        return NULL;
+    }
+
+    portMemSet(new_reg, 0, sizeof(nv_reg_entry_t));
+
+    if (regParmStr != NULL)
+    {
+        parm_size = (portStringLength(regParmStr) + 1);
+        new_ParmStr = portMemAllocNonPaged(parm_size);
+        if (NULL == new_ParmStr)
+        {
+            NV_PRINTF(LEVEL_ERROR, "failed to allocate registry param string\n");
+            portMemFree(new_reg);
+            return NULL;
+        }
+
+        NV_ASSERT(parm_size <= NVOS38_MAX_REGISTRY_STRING_LENGTH);
+
+        if (portMemCopy(new_ParmStr, parm_size, regParmStr, parm_size) == NULL)
+        {
+            NV_PRINTF(LEVEL_ERROR, "failed to copy registry param string\n");
+            portMemFree(new_ParmStr);
+            portMemFree(new_reg);
+            return NULL;
+        }
+    }
+
+    new_reg->regParmStr = new_ParmStr;
+    new_reg->type       = NV_REGISTRY_ENTRY_TYPE_UNKNOWN;
+
+    if (nvp != NULL)
+    {
+        new_reg->next = nvp->pRegistry;
+        nvp->pRegistry = new_reg;
+        DBG_REG_PRINTF("local registry now at 0x%p\n", nvp->pRegistry);
+    }
+    else
+    {
+        new_reg->next = the_registry;
+        the_registry = new_reg;
+        DBG_REG_PRINTF("global registry now at 0x%p\n", the_registry);
+    }
+
+    return new_reg;
+}
+
+static NV_STATUS regFreeEntry(nv_reg_entry_t *tmp)
+{
+    portMemFree(tmp->regParmStr);
+    tmp->regParmStr = NULL;
+    {
+        portMemFree(tmp->pdata);
+        tmp->pdata = NULL;
+        tmp->len = 0;
+    }
+    portMemFree(tmp);
+
+    return NV_OK;
+}
+
+static nv_reg_entry_t* regFindRegistryEntry(
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU32       type,
+    NvBool     *bGlobalEntry
+)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    nv_reg_entry_t *tmp;
+
+    DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+    if (nvp != NULL)
+    {
+        tmp = nvp->pRegistry;
+        DBG_REG_PRINTF("   local registry at 0x%p\n", tmp);
+
+        while ((tmp != NULL) && (tmp->regParmStr != NULL))
+        {
+            DBG_REG_PRINTF("  Testing against %s\n",
+                    tmp->regParmStr);
+            if ((stringCaseCompare(tmp->regParmStr, regParmStr) == 0) &&
+                (type == tmp->type))
+            {
+                DBG_REG_PRINTF("    found a match!\n");
+                if (bGlobalEntry)
+                    *bGlobalEntry = NV_FALSE;
+                return tmp;
+            }
+            tmp = tmp->next;
+        }
+    }
+
+    tmp = the_registry;
+    DBG_REG_PRINTF("   global registry at 0x%p\n", tmp);
+
+    while ((tmp != NULL) && (tmp->regParmStr != NULL))
+    {
+        DBG_REG_PRINTF("  Testing against %s\n",
+                tmp->regParmStr);
+        if ((stringCaseCompare(tmp->regParmStr, regParmStr) == 0) &&
+            (type == tmp->type))
+        {
+            DBG_REG_PRINTF("    found a match!\n");
+            if (bGlobalEntry)
+                *bGlobalEntry = NV_TRUE;
+            return tmp;
+        }
+        tmp = tmp->next;
+    }
+
+    DBG_REG_PRINTF("  no match\n");
+    return NULL;
+}
+
+NV_STATUS RmWriteRegistryDword(
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU32       Data
+)
+{
+    nv_reg_entry_t *tmp;
+    NvBool bGlobalEntry;
+
+    if (regParmStr == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    DBG_REG_PRINTF("%s: %s -> 0x%x\n", __FUNCTION__, regParmStr, Data);
+
+    tmp = regFindRegistryEntry(nv, regParmStr,
+                               NV_REGISTRY_ENTRY_TYPE_DWORD, &bGlobalEntry);
+
+    // If we found an entry and we were looking for a global entry and
+    // found a global, or we were looking for a per-GPU entry and found a
+    // per-GPU entry
+    if (tmp != NULL &&
+        ((nv == NULL && bGlobalEntry) ||
+         (nv != NULL && !bGlobalEntry)))
+    {
+        tmp->data = Data;
+
+        if (stringCaseCompare(regParmStr, "ResmanDebugLevel") == 0)
+        {
+            os_dbg_set_level(Data);
+        }
+
+        return NV_OK;
+    }
+
+    tmp = regCreateNewRegistryKey(nv, regParmStr);
+    if (tmp == NULL)
+        return NV_ERR_GENERIC;
+
+    tmp->type = NV_REGISTRY_ENTRY_TYPE_DWORD;
+    tmp->data = Data;
+
+    return NV_OK;
+}
+
+NV_STATUS RmReadRegistryDword(
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU32      *Data
+)
+{
+    nv_reg_entry_t *tmp;
+
+    if ((regParmStr == NULL) || (Data == NULL))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+    tmp = regFindRegistryEntry(nv, regParmStr,
+                               NV_REGISTRY_ENTRY_TYPE_DWORD, NULL);
+    if (tmp == NULL)
+    {
+        tmp = regFindRegistryEntry(nv, regParmStr,
+                                   NV_REGISTRY_ENTRY_TYPE_BINARY, NULL);
+        if ((tmp != NULL) && (tmp->len >= sizeof(NvU32)))
+        {
+            *Data = *(NvU32 *)tmp->pdata;
+        }
+        else
+        {
+            DBG_REG_PRINTF("   not found\n");
+            return NV_ERR_GENERIC;
+        }
+    }
+    else
+    {
+        *Data = tmp->data;
+    }
+
+    DBG_REG_PRINTF("  found in the_registry: 0x%x\n", *Data);
+
+    return NV_OK;
+}
+
+NV_STATUS RmReadRegistryBinary(
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU8       *Data,
+    NvU32      *cbLen
+)
+{
+    nv_reg_entry_t *tmp;
+    NV_STATUS status;
+
+    if ((regParmStr == NULL) || (Data == NULL) || (cbLen == NULL))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+    tmp = regFindRegistryEntry(nv, regParmStr,
+                               NV_REGISTRY_ENTRY_TYPE_BINARY, NULL);
+    if (tmp == NULL)
+    {
+        DBG_REG_PRINTF("   not found\n");
+        return NV_ERR_GENERIC;
+    }
+
+    DBG_REG_PRINTF("   found\n");
+
+    if (*cbLen >= tmp->len)
+    {
+        portMemCopy((NvU8 *)Data, *cbLen, (NvU8 *)tmp->pdata, tmp->len);
+        *cbLen = tmp->len;
+        status = NV_OK;
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "buffer (length: %u) is too small (data length: %u)\n",
+                  *cbLen, tmp->len);
+        status = NV_ERR_GENERIC;
+    }
+
+    return status;
+}
+
+NV_STATUS RmWriteRegistryBinary(
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU8       *Data,
+    NvU32       cbLen
+)
+{
+    nv_reg_entry_t *tmp;
+    NvBool bGlobalEntry;
+
+    if ((regParmStr == NULL) || (Data == NULL))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+    tmp = regFindRegistryEntry(nv, regParmStr,
+                               NV_REGISTRY_ENTRY_TYPE_BINARY, &bGlobalEntry);
+
+    // If we found an entry and we were looking for a global entry and
+    // found a global, or we were looking for a per-GPU entry and found a
+    // per-GPU entry
+    if (tmp != NULL &&
+        ((nv == NULL && bGlobalEntry) ||
+         (nv != NULL && !bGlobalEntry)))
+    {
+        if (tmp->pdata != NULL)
+        {
+            portMemFree(tmp->pdata);
+            tmp->pdata = NULL;
+            tmp->len = 0;
+        }
+    }
+    else
+    {
+        tmp = regCreateNewRegistryKey(nv, regParmStr);
+        if (tmp == NULL)
+        {
+            NV_PRINTF(LEVEL_ERROR, "failed to create binary registry entry\n");
+            return NV_ERR_GENERIC;
+        }
+    }
+
+    tmp->pdata = portMemAllocNonPaged(cbLen);
+    if (NULL == tmp->pdata)
+    {
+        NV_PRINTF(LEVEL_ERROR, "failed to write binary registry entry\n");
+        return NV_ERR_GENERIC;
+    }
+
+    tmp->type = NV_REGISTRY_ENTRY_TYPE_BINARY;
+    tmp->len = cbLen;
+    portMemCopy((NvU8 *)tmp->pdata, tmp->len, (NvU8 *)Data, cbLen);
+
+    return NV_OK;
+}
+
+NV_STATUS RmWriteRegistryString(
+    nv_state_t *nv,
+    const char *regParmStr,
+    const char *buffer,
+    NvU32       bufferLength
+)
+{
+    nv_reg_entry_t *tmp;
+    NvBool bGlobalEntry;
+
+    if ((regParmStr == NULL) || (buffer == NULL))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+    tmp = regFindRegistryEntry(nv, regParmStr,
+                               NV_REGISTRY_ENTRY_TYPE_STRING, &bGlobalEntry);
+
+    // If we found an entry and we were looking for a global entry and
+    // found a global, or we were looking for a per-GPU entry and found a
+    // per-GPU entry
+    if (tmp != NULL &&
+        ((nv == NULL && bGlobalEntry) ||
+         (nv != NULL && !bGlobalEntry)))
+    {
+        if (tmp->pdata != NULL)
+        {
+            portMemFree(tmp->pdata);
+            tmp->len = 0;
+            tmp->pdata = NULL;
+        }
+    }
+    else
+    {
+        tmp = regCreateNewRegistryKey(nv, regParmStr);
+        if (tmp == NULL)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "failed to allocate a string registry entry!\n");
+            return NV_ERR_INSUFFICIENT_RESOURCES;
+        }
+    }
+
+    tmp->pdata = portMemAllocNonPaged(bufferLength);
+    if (tmp->pdata == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "failed to write a string registry entry!\n");
+        return NV_ERR_NO_MEMORY;
+    }
+
+    tmp->type = NV_REGISTRY_ENTRY_TYPE_STRING;
+    tmp->len = bufferLength;
+    portMemCopy((void *)tmp->pdata, tmp->len, buffer, (bufferLength - 1));
+    tmp->pdata[bufferLength-1] = '\0';
+
+    return NV_OK;
+}
+
+NV_STATUS RmReadRegistryString(
+    nv_state_t *nv,
+    const char *regParmStr,
+    NvU8       *buffer,
+    NvU32      *pBufferLength
+)
+{
+    NvU32 bufferLength;
+    nv_reg_entry_t *tmp;
+
+    if ((regParmStr == NULL) || (buffer == NULL) || (pBufferLength == NULL))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    DBG_REG_PRINTF("%s: %s\n", __FUNCTION__, regParmStr);
+
+    bufferLength = *pBufferLength;
+    *pBufferLength = 0;
+    *buffer = '\0';
+
+    tmp = regFindRegistryEntry(nv, regParmStr,
+                               NV_REGISTRY_ENTRY_TYPE_STRING, NULL);
+    if (tmp == NULL)
+    {
+        return NV_ERR_GENERIC;
+    }
+
+    if (bufferLength >= tmp->len)
+    {
+        portMemCopy((void *)buffer, bufferLength, (void *)tmp->pdata, tmp->len);
+        *pBufferLength = tmp->len;
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "buffer (length: %u) is too small (data length: %u)\n",
+                  bufferLength, tmp->len);
+        return NV_ERR_BUFFER_TOO_SMALL;
+    }
+
+    return NV_OK;
+}
+
+
+NV_STATUS RmInitRegistry(void)
+{
+    NV_STATUS rmStatus;
+
+    rmStatus = os_registry_init();
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "failed to initialize the OS registry!\n");
+    }
+
+    return rmStatus;
+}
+
+NV_STATUS RmDestroyRegistry(nv_state_t *nv)
+{
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    nv_reg_entry_t *tmp;
+
+    if (nvp != NULL)
+    {
+        tmp = nvp->pRegistry;
+        nvp->pRegistry = NULL;
+    }
+    else
+    {
+        tmp = the_registry;
+        the_registry = NULL;
+    }
+
+    while (tmp != NULL)
+    {
+        nv_reg_entry_t *entry = tmp;
+        tmp = tmp->next;
+        regFreeEntry(entry);
+    }
+
+    return NV_OK;
+}
+
+static void regCountEntriesAndSize(
+    NvU32           *pNumEntries,     // Pointer to number of entries
+    NvU32           *pSize,           // Pointer to total size
+    nv_reg_entry_t  *pRegEntry        // Pointer local or global registry
+)
+{
+    //
+    // Note that *pNumEntries and *pSize are not initialized here.  This is so
+    // we can accumulate totals of both global and local registries.
+    //
+    NvU32 numEntries = *pNumEntries;
+    NvU32 size       = *pSize;
+
+    while ((pRegEntry != NULL) && (pRegEntry->regParmStr != NULL))
+    {
+        size += portStringLength(pRegEntry->regParmStr) + 1 + pRegEntry->len;
+        numEntries++;
+        pRegEntry = pRegEntry->next;
+    }
+
+    *pNumEntries = numEntries;
+    *pSize       = size;
+}
+
+static NV_STATUS regCopyEntriesToPackedBuffer(
+    PACKED_REGISTRY_TABLE  *pRegTable,        // Pointer to packed record
+    nv_reg_entry_t         *pRegEntry,        // Pointer local or global registry
+    NvU32                  *pEntryIndex,      // Pointer to next index
+    NvU32                  *pDataOffset       // Pointer to offset of next data byte.
+)
+{
+    NvU8      *pByte      = (NvU8 *)pRegTable; // Byte version of record pointer.
+    NV_STATUS  nvStatus   = NV_OK;
+    NvU32      entryIndex = *pEntryIndex;
+    NvU32      dataOffset = *pDataOffset;
+
+    // Walk the records and copy the data.
+    while ((pRegEntry != NULL) && (pRegEntry->regParmStr != NULL))
+    {
+        PACKED_REGISTRY_ENTRY *pEntry = &pRegTable->entries[entryIndex];
+        NvU32 slen = portStringLength(pRegEntry->regParmStr) + 1;
+
+        // Sanity check the data offset and index against counted totals.
+        if ((dataOffset + slen + pRegEntry->len > pRegTable->size) ||
+            (entryIndex >= pRegTable->numEntries))
+        {
+            // Something has changed since we counted them?
+            NV_PRINTF(LEVEL_ERROR, "Registry entry record is full\n");
+            nvStatus = NV_ERR_INVALID_STATE;
+            break;
+        }
+
+        // Copy registry entry name to data blob.
+        pEntry->nameOffset = dataOffset;
+        portMemCopy(&pByte[dataOffset], slen, pRegEntry->regParmStr, slen);
+        dataOffset += slen;
+
+        switch (pRegEntry->type)
+        {
+            case NV_REGISTRY_ENTRY_TYPE_DWORD:
+                pEntry->type   = REGISTRY_TABLE_ENTRY_TYPE_DWORD;
+                pEntry->length = sizeof(NvU32);
+                pEntry->data   = pRegEntry->data;
+                break;
+
+            case NV_REGISTRY_ENTRY_TYPE_BINARY:
+            case NV_REGISTRY_ENTRY_TYPE_STRING:
+                pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_STRING;
+                if (pRegEntry->type == NV_REGISTRY_ENTRY_TYPE_BINARY)
+                    pEntry->type = REGISTRY_TABLE_ENTRY_TYPE_BINARY;
+
+                if (pRegEntry->len == 0)
+                {
+                    NV_PRINTF(LEVEL_WARNING,
+                                "Registry Key not sent to GSP-RM because it has 0 DataLength\n");
+                    break;
+                }
+                pEntry->length   = pRegEntry->len;
+                pEntry->data     = dataOffset;
+                portMemCopy(&pByte[dataOffset], pEntry->length,
+                            pRegEntry->pdata, pRegEntry->len);
+                dataOffset += pRegEntry->len;
+                break;
+
+            default:
+                // We should never get here.
+                pEntry->type   = REGISTRY_TABLE_ENTRY_TYPE_UNKNOWN;
+                pEntry->length = 0;
+                pEntry->data   = 0;
+                DBG_BREAKPOINT();
+                break;
+        }
+
+        pRegEntry = pRegEntry->next;
+        entryIndex++;
+    }
+
+    *pEntryIndex = entryIndex;
+    *pDataOffset = dataOffset;
+    return nvStatus;
+}
+
+// Package registry entries
+NV_STATUS RmPackageRegistry(
+    nv_state_t            *nv,
+    PACKED_REGISTRY_TABLE *pRegTable,
+    NvU32                 *pSize
+)
+{
+    nv_priv_t             *nvp             = NV_GET_NV_PRIV(nv);
+    nv_reg_entry_t        *pLocalRegistry  = NULL;
+    NV_STATUS              nvStatus        = NV_OK;
+    NvU32                  totalSize;
+    NvU32                  numEntries;
+
+    if (pSize == NULL)
+        return NV_ERR_INVALID_ARGUMENT;
+
+    // Use the local (per-device) registry if we have one.
+    if (nvp != NULL)
+        pLocalRegistry = nvp->pRegistry;
+
+    numEntries = 0;
+    totalSize  = NV_OFFSETOF(PACKED_REGISTRY_TABLE, entries);
+
+    // Count the number of global entries and total size.
+    regCountEntriesAndSize(&numEntries, &totalSize, the_registry);
+
+    // Count the number of local entries and total size.
+    regCountEntriesAndSize(&numEntries, &totalSize, pLocalRegistry);
+
+    // Add table record size into total size.
+    totalSize += sizeof(PACKED_REGISTRY_ENTRY) * numEntries;
+
+    //
+    // If this function is called to only compute total size of registry table,
+    // then we are done here.
+    //
+    if (pRegTable == NULL)
+    {
+        *pSize = totalSize;
+        return NV_OK;
+    }
+
+    // Return warning if there are no registry entries.
+    if (numEntries == 0)
+        return NV_WARN_NOTHING_TO_DO;
+
+    if (totalSize > *pSize)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Registry entries overflow RPC record\n");
+        return NV_ERR_BUFFER_TOO_SMALL;
+    }
+
+    // Fill in our new structure with the first pass (counting) values.
+    pRegTable->size       = totalSize;
+    *pSize                = totalSize;
+    pRegTable->numEntries = numEntries;
+
+    // Offset of first byte after the registry entry table.
+    totalSize  = NV_OFFSETOF(PACKED_REGISTRY_TABLE, entries) +
+                 (sizeof(PACKED_REGISTRY_ENTRY) * numEntries);
+
+    // Starting index in the registry entry table.
+    numEntries = 0;
+
+    // Walk the global registry and copy the data.
+    nvStatus = regCopyEntriesToPackedBuffer(pRegTable,
+        the_registry, &numEntries, &totalSize);
+
+    // Walk the local registry and copy the data.
+    if (nvStatus == NV_OK)
+    {
+        nvStatus = regCopyEntriesToPackedBuffer(pRegTable,
+            pLocalRegistry, &numEntries, &totalSize);
+    }
+
+    // Sanity check second pass against first pass.
+    if ((numEntries != pRegTable->numEntries) || (totalSize != pRegTable->size))
+    {
+        NV_PRINTF(LEVEL_ERROR, "First/second pass mismatch\n");
+        nvStatus = NV_ERR_INVALID_STATE;
+    }
+
+    return nvStatus;
+}
+
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/rm-gpu-ops.c b/userspace/sdks/nv570/nvalloc/unix/src/rm-gpu-ops.c
new file mode 100644
index 0000000..688b4c5
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/rm-gpu-ops.c
@@ -0,0 +1,1079 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include <os/os.h>
+#include <osapi.h>
+#include <core/thread_state.h>
+#include "rmapi/nv_gpu_ops.h"
+#include "gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h"
+
+NV_STATUS NV_API_CALL rm_gpu_ops_create_session(
+    nvidia_stack_t *sp,
+    struct gpuSession **session)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCreateSession(session);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_destroy_session (
+    nvidia_stack_t *sp, gpuSessionHandle session)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDestroySession(session);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_device_create (
+    nvidia_stack_t *sp,
+    nvgpuSessionHandle_t session,
+    const gpuInfo *pGpuInfo,
+    const NvProcessorUuid *gpuUuid,
+    nvgpuDeviceHandle_t *device,
+    NvBool bCreateSmcPartition)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDeviceCreate(session, pGpuInfo, gpuUuid, device, bCreateSmcPartition);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_device_destroy (
+    nvidia_stack_t *sp,
+    gpuDeviceHandle device)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDeviceDestroy(device);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_address_space_create (
+    nvidia_stack_t *sp,
+    gpuDeviceHandle device,
+    NvU64 vaBase,
+    NvU64 vaSize,
+    NvU32 enableAts,
+    gpuAddressSpaceHandle *vaSpace,
+    gpuAddressSpaceInfo *vaSpaceInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsAddressSpaceCreate(device, vaBase, vaSize, enableAts,
+                                          vaSpace, vaSpaceInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_dup_address_space(
+    nvidia_stack_t *sp,
+    gpuDeviceHandle device,
+    NvHandle hUserClient,
+    NvHandle hUserVASpace,
+    gpuAddressSpaceHandle *dupedVaspace,
+    gpuAddressSpaceInfo *vaSpaceInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDupAddressSpace(device, hUserClient, hUserVASpace,
+                                       dupedVaspace, vaSpaceInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_address_space_destroy(nvidia_stack_t *sp,
+    gpuAddressSpaceHandle vaspace)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsAddressSpaceDestroy(vaspace);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return NV_OK;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_memory_alloc_fb(
+    nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace,
+    NvLength size, NvU64 *gpuOffset, gpuAllocInfo *allocInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsMemoryAllocFb(vaspace, size, gpuOffset, allocInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_get_p2p_caps(nvidia_stack_t *sp,
+                                                gpuDeviceHandle device1,
+                                                gpuDeviceHandle device2,
+                                                getP2PCapsParams *pP2pCapsParams)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsGetP2PCaps(device1, device2, pP2pCapsParams);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_memory_alloc_sys(
+    nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace,
+    NvLength size, NvU64 *gpuOffset, gpuAllocInfo *allocInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsMemoryAllocSys(vaspace, size, gpuOffset, allocInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_pma_register_callbacks(
+    nvidia_stack_t *sp,
+    void *pPma,
+    pmaEvictPagesCb_t evictPages,
+    pmaEvictRangeCb_t evictRange,
+    void *callbackData)
+{
+    THREAD_STATE_NODE threadState;
+    NV_STATUS rmStatus;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    rmStatus = pmaRegisterEvictionCb(pPma, evictPages, evictRange, callbackData);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+void  NV_API_CALL  rm_gpu_ops_pma_unregister_callbacks(
+    nvidia_stack_t *sp,
+    void *pPma)
+{
+    THREAD_STATE_NODE threadState;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    pmaUnregisterEvictionCb(pPma);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_get_pma_object(
+    nvidia_stack_t *sp,
+    gpuDeviceHandle device,
+    void **pPma,
+    const nvgpuPmaStatistics_t *pPmaPubStats)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsGetPmaObject(device, pPma,
+                                    (const UvmPmaStatistics **)pPmaPubStats);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_pma_alloc_pages(
+    nvidia_stack_t *sp, void *pPma,
+    NvLength pageCount, NvU64 pageSize,
+    nvgpuPmaAllocationOptions_t pPmaAllocOptions,
+    NvU64 *pPages)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsPmaAllocPages(pPma, pageCount, pageSize,
+                                     pPmaAllocOptions, pPages);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_pma_pin_pages(
+    nvidia_stack_t *sp, void *pPma,
+    NvU64 *pPages, NvLength pageCount, NvU64 pageSize, NvU32 flags)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsPmaPinPages(pPma, pPages, pageCount, pageSize, flags);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_memory_cpu_map(
+    nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace,
+    NvU64 gpuOffset, NvLength length, void **cpuPtr, NvU64 pageSize)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsMemoryCpuMap(vaspace, gpuOffset, length, cpuPtr,
+                                    pageSize);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_memory_cpu_ummap(
+    nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, void* cpuPtr)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsMemoryCpuUnMap(vaspace, cpuPtr);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return NV_OK;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_tsg_allocate(nvidia_stack_t *sp,
+                                                gpuAddressSpaceHandle vaspace,
+                                                const gpuTsgAllocParams *allocParams,
+                                                gpuTsgHandle *tsg)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsTsgAllocate(vaspace, allocParams, tsg);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_tsg_destroy(nvidia_stack_t * sp,
+                                             nvgpuTsgHandle_t tsg)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsTsgDestroy(tsg);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return NV_OK;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_channel_allocate(nvidia_stack_t *sp,
+                                                    const gpuTsgHandle tsg,
+                                                    const gpuChannelAllocParams *allocParams,
+                                                    gpuChannelHandle *channel,
+                                                    gpuChannelInfo *channelInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsChannelAllocate(tsg, allocParams, channel,
+                                       channelInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_channel_destroy(nvidia_stack_t * sp,
+                                                 nvgpuChannelHandle_t channel)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsChannelDestroy(channel);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return NV_OK;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_pma_free_pages(nvidia_stack_t *sp,
+    void *pPma, NvU64 *pPages, NvLength pageCount, NvU64 pageSize, NvU32 flags)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsPmaFreePages(pPma, pPages, pageCount, pageSize, flags);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return NV_OK;
+}
+
+NV_STATUS  NV_API_CALL rm_gpu_ops_memory_free(
+    nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace, NvU64 gpuOffset)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsMemoryFree(vaspace, gpuOffset);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return NV_OK;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_query_caps(nvidia_stack_t *sp,
+                                            gpuDeviceHandle device,
+                                            gpuCaps * caps)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsQueryCaps(device, caps);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_query_ces_caps(nvidia_stack_t *sp,
+                                                gpuDeviceHandle device,
+                                                gpuCesCaps *caps)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsQueryCesCaps(device, caps);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL rm_gpu_ops_get_gpu_info(nvidia_stack_t *sp,
+                                               const NvProcessorUuid *pUuid,
+                                               const gpuClientInfo *pGpuClientInfo,
+                                               gpuInfo *pGpuInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsGetGpuInfo(pUuid, pGpuClientInfo, pGpuInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_service_device_interrupts_rm(nvidia_stack_t *sp,
+                                                              gpuDeviceHandle device)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus =  nvGpuOpsServiceDeviceInterruptsRM(device);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_set_page_directory (nvidia_stack_t *sp,
+                                         gpuAddressSpaceHandle vaSpace,
+                                         NvU64 physAddress, unsigned numEntries,
+                                         NvBool bVidMemAperture, NvU32 pasid,
+                                         NvU64 *dmaAddress)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsSetPageDirectory(vaSpace, physAddress, numEntries,
+                                        bVidMemAperture, pasid, dmaAddress);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_unset_page_directory (nvidia_stack_t *sp,
+                                                 gpuAddressSpaceHandle vaSpace)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsUnsetPageDirectory(vaSpace);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_dup_allocation(nvidia_stack_t *sp,
+                                                  gpuAddressSpaceHandle srcVaSpace,
+                                                  NvU64 srcAddress,
+                                                  gpuAddressSpaceHandle dstVaSpace,
+                                                  NvU64 dstVaAlignment,
+                                                  NvU64 *dstAddress)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDupAllocation(srcVaSpace, srcAddress, dstVaSpace, dstVaAlignment, dstAddress);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_dup_memory (nvidia_stack_t *sp,
+                                               gpuDeviceHandle device,
+                                               NvHandle hClient,
+                                               NvHandle hPhysMemory,
+                                               NvHandle *hDupMemory,
+                                               nvgpuMemoryInfo_t gpuMemoryInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDupMemory(device, hClient, hPhysMemory, hDupMemory, gpuMemoryInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_free_duped_handle (nvidia_stack_t *sp,
+                                                gpuDeviceHandle device,
+                                                NvHandle hPhysHandle)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsFreeDupedHandle(device, hPhysHandle);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_get_fb_info (nvidia_stack_t *sp,
+                                                gpuDeviceHandle device,
+                                                gpuFbInfo * fbInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsGetFbInfo(device, fbInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_get_ecc_info (nvidia_stack_t *sp,
+                                                 gpuDeviceHandle device,
+                                                 gpuEccInfo * eccInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsGetEccInfo(device, eccInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+//
+// Please see the comments for nvUvmInterfaceOwnPageFaultIntr(), in
+// nv_uvm_interface.h, for the recommended way to use this routine.
+//
+// How it works:
+//
+// The rmGpuLocksAcquire call generally saves the current GPU interrupt
+// state, then disables interrupt generation for one (or all) GPUs.
+// Likewise, the rmGpuLocksRelease call restores (re-enables) those
+// interrupts to their previous state. However, the rmGpuLocksRelease
+// call does NOT restore interrupts that RM does not own.
+//
+// This is rather hard to find in the code, so: very approximately, the
+// following sequence happens: rmGpuLocksRelease, osEnableInterrupts,
+// intrRestoreNonStall_HAL, intrEncodeIntrEn_HAL, and that last one skips
+// over any interrupts that RM does not own.
+//
+// This means that things are a bit asymmetric, because this routine
+// actually changes that ownership in between the rmGpuLocksAcquire and
+// rmGpuLocksRelease calls. So:
+//
+// -- If you call this routine with bOwnInterrupts == NV_TRUE (UVM is
+//    taking ownership from the RM), then rmGpuLocksAcquire disables all
+//    GPU interrupts. Then the ownership is taken away from RM, so the
+//    rmGpuLocksRelease call leaves the replayable page fault interrupts
+//    disabled. It is then up to UVM (the caller) to enable replayable
+//    page fault interrupts when it is ready.
+//
+// -- If you call this routine with bOwnInterrupts == NV_FALSE (UVM is
+//    returning ownership to the RM), then rmGpuLocksAcquire disables
+//    all GPU interrupts that RM owns. Then the ownership is returned to
+//    RM, so the rmGpuLocksRelease call re-enables replayable page fault
+//    interrupts. So, that implies that you need to disable replayable page
+//    fault interrupts before calling this routine, in order to hand
+//    over a GPU to RM that is not generating interrupts, until RM is
+//    ready to handle the interrupts.
+//
+NV_STATUS NV_API_CALL rm_gpu_ops_own_page_fault_intr(nvidia_stack_t *sp,
+                                                     struct gpuDevice *device,
+                                                     NvBool bOwnInterrupts)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsOwnPageFaultIntr(device, bOwnInterrupts);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_init_fault_info (nvidia_stack_t *sp,
+                                                    gpuDeviceHandle device,
+                                                    gpuFaultInfo *pFaultInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsInitFaultInfo(device, pFaultInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_destroy_fault_info (nvidia_stack_t *sp,
+                                                       gpuDeviceHandle device,
+                                                       gpuFaultInfo *pFaultInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDestroyFaultInfo(device, pFaultInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+// Functions
+//
+// - rm_gpu_ops_has_pending_non_replayable_faults
+// - rm_gpu_ops_get_non_replayable_faults
+//
+// Cannot take the GPU/RM lock because it is called during fault servicing.
+// This could produce deadlocks if the UVM bottom half gets stuck behind a
+// stalling interrupt that cannot be serviced if UVM is holding the lock.
+//
+// However, these functions can be safely called with no locks because it is
+// just accessing the given client shadow fault buffer, which is implemented
+// using a lock-free queue. There is a different client shadow fault buffer
+// per GPU: RM top-half producer, UVM top/bottom-half consumer.
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_has_pending_non_replayable_faults(nvidia_stack_t *sp,
+                                                                     gpuFaultInfo *pFaultInfo,
+                                                                     NvBool *hasPendingFaults)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsHasPendingNonReplayableFaults(pFaultInfo, hasPendingFaults);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_get_non_replayable_faults(nvidia_stack_t *sp,
+                                                             gpuFaultInfo *pFaultInfo,
+                                                             void *faultBuffer,
+                                                             NvU32 *numFaults)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsGetNonReplayableFaults(pFaultInfo, faultBuffer, numFaults);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL rm_gpu_ops_flush_replayable_fault_buffer(nvidia_stack_t *sp,
+                                                                gpuFaultInfo *pFaultInfo,
+                                                                NvBool bCopyAndFlush)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsFlushReplayableFaultBuffer(pFaultInfo, bCopyAndFlush);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL rm_gpu_ops_toggle_prefetch_faults(nvidia_stack_t *sp,
+                                                         gpuFaultInfo *pFaultInfo,
+                                                         NvBool bEnable)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsTogglePrefetchFaults(pFaultInfo, bEnable);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_access_bits_buffer_alloc(nvidia_stack_t *sp,
+                                                            gpuDeviceHandle device,
+                                                            gpuAccessBitsBufferAlloc *accessBitsInfo)
+{
+    NV_STATUS rmStatus;
+    void* fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsAccessBitsBufAlloc(device, accessBitsInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_access_bits_buffer_free(nvidia_stack_t *sp,
+                                                           gpuDeviceHandle device,
+                                                           gpuAccessBitsBufferAlloc *accessBitsInfo)
+{
+    NV_STATUS rmStatus;
+    void* fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsAccessBitsBufFree(device, accessBitsInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_access_bits_dump(nvidia_stack_t *sp,
+                                                    gpuDeviceHandle device,
+                                                    gpuAccessBitsBufferAlloc *accessBitsInfo,
+                                                    UVM_ACCESS_BITS_DUMP_MODE mode)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsAccessBitsDump(device, accessBitsInfo, mode);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_init_access_cntr_info(nvidia_stack_t *sp,
+                                                         gpuDeviceHandle device,
+                                                         gpuAccessCntrInfo *accessCntrInfo,
+                                                         NvU32 accessCntrIndex)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsInitAccessCntrInfo(device, accessCntrInfo, accessCntrIndex);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_destroy_access_cntr_info(nvidia_stack_t *sp,
+                                                            gpuDeviceHandle device,
+                                                            gpuAccessCntrInfo *accessCntrInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDestroyAccessCntrInfo(device, accessCntrInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_enable_access_cntr(nvidia_stack_t *sp,
+                                                      gpuDeviceHandle device,
+                                                      gpuAccessCntrInfo *accessCntrInfo,
+                                                      gpuAccessCntrConfig *accessCntrConfig)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsEnableAccessCntr(device, accessCntrInfo, accessCntrConfig);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_disable_access_cntr(nvidia_stack_t *sp,
+                                                       gpuDeviceHandle device,
+                                                       gpuAccessCntrInfo *accessCntrInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsDisableAccessCntr(device, accessCntrInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL  rm_gpu_ops_get_nvlink_info (nvidia_stack_t *sp,
+                                                    gpuDeviceHandle device,
+                                                    gpuNvlinkInfo *nvlinkInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsGetNvlinkInfo(device, nvlinkInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_p2p_object_create(nvidia_stack_t *sp,
+                             gpuDeviceHandle device1,
+                             gpuDeviceHandle device2,
+                             NvHandle *hP2pObject)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    rmStatus = nvGpuOpsP2pObjectCreate(device1, device2, hP2pObject);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+    return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_p2p_object_destroy(nvidia_stack_t *sp,
+                              nvgpuSessionHandle_t session,
+                              NvHandle hP2pObject)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    nvGpuOpsP2pObjectDestroy(session, hP2pObject);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+NV_STATUS  NV_API_CALL
+rm_gpu_ops_get_external_alloc_ptes(nvidia_stack_t* sp,
+                                   nvgpuAddressSpaceHandle_t vaSpace,
+                                   NvHandle hDupedMemory,
+                                   NvU64 offset,
+                                   NvU64 size,
+                                   nvgpuExternalMappingInfo_t gpuExternalMappingInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    rmStatus = nvGpuOpsGetExternalAllocPtes(vaSpace, hDupedMemory, offset, size,
+                                            gpuExternalMappingInfo);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL
+rm_gpu_ops_get_external_alloc_phys_addrs(nvidia_stack_t* sp,
+                                         nvgpuAddressSpaceHandle_t vaSpace,
+                                         NvHandle hDupedMemory,
+                                         NvU64 offset,
+                                         NvU64 size,
+                                         nvgpuExternalPhysAddrInfo_t gpuExternalPhysAddrsInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    rmStatus = nvGpuOpsGetExternalAllocPhysAddrs(vaSpace, hDupedMemory, offset, size,
+                                                 gpuExternalPhysAddrsInfo);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL
+rm_gpu_ops_retain_channel(nvidia_stack_t* sp,
+                          nvgpuAddressSpaceHandle_t vaSpace,
+                          NvHandle hClient,
+                          NvHandle hChannel,
+                          void **retainedChannel,
+                          nvgpuChannelInstanceInfo_t channelInstanceInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    rmStatus = nvGpuOpsRetainChannel(vaSpace, hClient, hChannel,
+                                     (gpuRetainedChannel **)retainedChannel,
+                                     channelInstanceInfo);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL
+rm_gpu_ops_bind_channel_resources(nvidia_stack_t* sp,
+                                  void *retainedChannel,
+                                  nvgpuChannelResourceBindParams_t channelResourceBindParams)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    rmStatus = nvGpuOpsBindChannelResources(retainedChannel,
+                                            channelResourceBindParams);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+    return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_release_channel(nvidia_stack_t *sp, void *retainedChannel)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    nvGpuOpsReleaseChannel(retainedChannel);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+void NV_API_CALL
+rm_gpu_ops_stop_channel(nvidia_stack_t * sp,
+                        void *retainedChannel,
+                        NvBool bImmediate)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsStopChannel(retainedChannel, bImmediate);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+}
+
+NV_STATUS  NV_API_CALL
+rm_gpu_ops_get_channel_resource_ptes(nvidia_stack_t* sp,
+                                     nvgpuAddressSpaceHandle_t vaSpace,
+                                     NvP64 resourceDescriptor,
+                                     NvU64 offset,
+                                     NvU64 size,
+                                     nvgpuExternalMappingInfo_t gpuExternalMappingInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp, fp);
+    rmStatus = nvGpuOpsGetChannelResourcePtes(vaSpace, resourceDescriptor,
+                                              offset, size,
+                                              gpuExternalMappingInfo);
+    NV_EXIT_RM_RUNTIME(sp, fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_report_non_replayable_fault(nvidia_stack_t *sp,
+                                       nvgpuDeviceHandle_t device,
+                                       const void *pFaultPacket)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsReportNonReplayableFault(device, pFaultPacket);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_paging_channel_allocate(nvidia_stack_t *sp,
+                                   gpuDeviceHandle device,
+                                   const gpuPagingChannelAllocParams *allocParams,
+                                   gpuPagingChannelHandle *channel,
+                                   gpuPagingChannelInfo *channelInfo)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsPagingChannelAllocate(device, allocParams, channel,
+                                             channelInfo);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_paging_channel_destroy(nvidia_stack_t *sp,
+                                  gpuPagingChannelHandle channel)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsPagingChannelDestroy(channel);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_paging_channels_map(nvidia_stack_t *sp,
+                               gpuAddressSpaceHandle srcVaSpace,
+                               NvU64 srcAddress,
+                               gpuDeviceHandle device,
+                               NvU64 *dstAddress)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsPagingChannelsMap(srcVaSpace, srcAddress, device, dstAddress);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_paging_channels_unmap(nvidia_stack_t *sp,
+                                 gpuAddressSpaceHandle srcVaSpace,
+                                 NvU64 srcAddress,
+                                 gpuDeviceHandle device)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsPagingChannelsUnmap(srcVaSpace, srcAddress, device);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL
+rm_gpu_ops_paging_channel_push_stream(nvidia_stack_t *sp,
+                                      gpuPagingChannelHandle channel,
+                                      char *methodStream,
+                                      NvU32 methodStreamSize)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsPagingChannelPushStream(channel, methodStream, methodStreamSize);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+void NV_API_CALL
+rm_gpu_ops_report_fatal_error(nvidia_stack_t *sp,
+                              NV_STATUS error)
+{
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    nvGpuOpsReportFatalError(error);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_init(nvidia_stack_t *sp,
+                                                   struct ccslContext_t **ctx,
+                                                   gpuChannelHandle channel)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCcslContextInit(ctx, channel);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_clear(nvidia_stack_t *sp,
+                                                    struct ccslContext_t *ctx)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCcslContextClear(ctx);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_rotate_key(nvidia_stack_t *sp,
+                                                 UvmCslContext *contextList[],
+                                                 NvU32 contextListCount)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCcslRotateKey(contextList, contextListCount);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_rotate_iv(nvidia_stack_t *sp,
+                                                struct ccslContext_t *ctx,
+                                                NvU8 direction)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCcslRotateIv(ctx, direction);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_encrypt_with_iv(nvidia_stack_t *sp,
+                                                      struct ccslContext_t *ctx,
+                                                      NvU32 bufferSize,
+                                                      NvU8 const *inputBuffer,
+                                                      NvU8 *encryptIv,
+                                                      NvU8 *outputBuffer,
+                                                      NvU8 *authTagData)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCcslEncryptWithIv(ctx, bufferSize, inputBuffer, encryptIv, outputBuffer, authTagData);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_encrypt(nvidia_stack_t *sp,
+                                              struct ccslContext_t *ctx,
+                                              NvU32 bufferSize,
+                                              NvU8 const *inputBuffer,
+                                              NvU8 *outputBuffer,
+                                              NvU8 *authTagData)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCcslEncrypt(ctx, bufferSize, inputBuffer, outputBuffer, authTagData);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_decrypt(nvidia_stack_t *sp,
+                                              struct ccslContext_t *ctx,
+                                              NvU32 bufferSize,
+                                              NvU8 const *inputBuffer,
+                                              NvU8 const *decryptIv,
+                                              NvU32 keyRotationId,
+                                              NvU8 *outputBuffer,
+                                              NvU8 const *addAuthData,
+                                              NvU32 addAuthDataSize,
+                                              NvU8 const *authTagData)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCcslDecrypt(ctx, bufferSize, inputBuffer, decryptIv, keyRotationId, outputBuffer,
+                                   addAuthData, addAuthDataSize, authTagData);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_sign(nvidia_stack_t *sp,
+                                           struct ccslContext_t *ctx,
+                                           NvU32 bufferSize,
+                                           NvU8 const *inputBuffer,
+                                           NvU8 *authTagData)
+
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsCcslSign(ctx, bufferSize, inputBuffer, authTagData);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_query_message_pool(nvidia_stack_t *sp,
+                                                          struct ccslContext_t *ctx,
+                                                          NvU8 direction,
+                                                          NvU64 *messageNum)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsQueryMessagePool(ctx, direction, messageNum);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_increment_iv(nvidia_stack_t *sp,
+                                                    struct ccslContext_t *ctx,
+                                                    NvU8 direction,
+                                                    NvU64 increment,
+                                                    NvU8 *iv)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsIncrementIv(ctx, direction, increment, iv);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
+
+NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_log_encryption(nvidia_stack_t *sp,
+                                                      struct ccslContext_t *ctx,
+                                                      NvU8 direction,
+                                                      NvU32 bufferSize)
+{
+    NV_STATUS rmStatus;
+    void *fp;
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    rmStatus = nvGpuOpsLogEncryption(ctx, direction, bufferSize);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return rmStatus;
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/rmobjexportimport.c b/userspace/sdks/nv570/nvalloc/unix/src/rmobjexportimport.c
new file mode 100644
index 0000000..558f5fd
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/rmobjexportimport.c
@@ -0,0 +1,650 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2016-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ * @file
+ *
+ * @brief  Provides RmExportObject, RmImportObject, RmFreeObjExportHandle and
+ *         RmGetExportObjectInfo interfaces :
+ *
+ *             These interfaces allow rm clients to export their objects into
+ *         a unique RmObjExportHandle which another rm client could
+ *         import, even if the source rm client gets destroyed.
+ *
+ *             RM's device instance may get destroyed asynchronously, in which
+ *         case exported objects residing on that device instance also get
+ *         destroyed.  This means it is not possible to import it back, but the
+ *         RmObjExportHandle into which the object had been exported still
+ *         remains valid but no other object could get it.
+ *
+ *         There are not init/fini routines, it is the responsibility of the
+ *         rest of RM's eco-system to make sure that all RmObjExportHandles get
+ *         freed during driver unload.
+ *
+ *         The api lock is expected to be held before calling into
+ *         rmobjexportimport.c; do not hold gpu or any other lock.
+ */
+
+#include "rmobjexportimport.h"
+#include "nvlimits.h"
+#include "gpu/device/device.h"
+#include "gpu/mig_mgr/kernel_mig_manager.h"
+#include "gpu/mig_mgr/gpu_instance_subscription.h"
+
+#include "containers/map.h"
+#include "rmapi/rmapi.h"
+#include "rmapi/rs_utils.h"
+
+#include "class/cl0080.h"
+#include "class/cl2080.h"
+#include <ctrl/ctrl0000/ctrl0000unix.h>
+#include <ctrl/ctrl0000/ctrl0000client.h>
+
+//
+// A reference to an RmObjExportHandle
+// generated by function RmGenerateObjExportHandle().
+//
+MAKE_MAP(RmObjExportHandleMap, NvU8);
+
+//
+// Memory allocator
+//
+PORT_MEM_ALLOCATOR *pMemAllocator;
+
+//
+// Map RmObjExportHandle -> RmObjExportHandleRef
+//
+RmObjExportHandleMap objExportHandleMap;
+
+//
+// Rm client to use to dup an object exported to RmObjExportHandle.  The minimal
+// requirement for duping is to have a device object allocated.  This rm client
+// is simply like any other external rm client and has no any special handling.
+//
+// We keep this rm client just like any other external rm client: if
+// gpu(s)/device gets powered-down/uninitialized, rm objects allocated by
+// external rm clients and located on that gpu(s)/device gets freed (the
+// os-layer does that).  In that way, code in this file doesn't need to worry
+// about freeing exported objects located on that gpu(s)/device.
+//
+NvHandle hObjExportRmClient;
+
+//
+// Tracker for device and subdevice handles. For now only one subdevice
+// (instance 0) is supported per device.
+//
+typedef struct
+{
+    NvHandle hRmDevice;
+    NvHandle hRmSubDevice;
+    NvHandle hGpuInstSub;
+} RmObjExportDevice;
+MAKE_MAP(RmObjExportDeviceMap, RmObjExportDevice);
+
+RmObjExportDeviceMap objExportDeviceMap;
+
+//
+// Usage reference counter for static object in this file like rm client used to
+// dup an exported object, memory allocator, map etc.
+//
+NvU64 objExportImportRefCount;
+
+//
+// Static functions for internal use to code in this file.
+//
+static NV_STATUS  RmRefObjExportImport    (void);
+static void       RmUnrefObjExportImport  (void);
+
+static RmObjExportHandle RmGenerateObjExportHandle  (void);
+static NV_STATUS         RmUnrefObjExportHandle     (RmObjExportHandle hObject);
+
+//
+// Free the RmObjExportHandle.
+//
+static NV_STATUS RmUnrefObjExportHandle(RmObjExportHandle hObject)
+{
+    RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+    void *pHandleRef = mapFind(&objExportHandleMap, hObject);
+    if (pHandleRef == NULL)
+    {
+        return NV_ERR_OBJECT_NOT_FOUND;
+    }
+
+    if (pRmApi->Free(pRmApi,
+                     hObjExportRmClient,
+                     (NvHandle)mapKey(&objExportHandleMap, pHandleRef)) != NV_OK)
+    {
+        NV_PRINTF(LEVEL_WARNING,
+                  "Exported object trying to free was zombie in %s\n",
+                  __FUNCTION__);
+    }
+
+    mapRemove(&objExportHandleMap, pHandleRef);
+
+    return NV_OK;
+}
+
+//
+// Generate unique RmObjExportHandle.
+//
+static RmObjExportHandle RmGenerateObjExportHandle(void)
+{
+    //
+    // The object export handle belongs to range of 0 to
+    // (MAX_OBJ_EXPORT_HANDLES - 1).
+    //
+    // Handle 0 is considered as invalid object handle, this function generates
+    // handle from range of 1 to (MAX_OBJ_EXPORT_HANDLES - 1).
+    //
+    #define MAX_OBJ_EXPORT_HANDLES 0x80000
+
+    static NvHandle hObjExportHandleNext = 1;
+
+    RmObjExportHandle hStartHandle = hObjExportHandleNext;
+    RmObjExportHandle hObject = 0;
+
+    do
+    {
+        void *pHandleRef;
+
+        hObject = hObjExportHandleNext++;
+        /* Reset hObjExportHandleNext to next valid handle */
+        if (hObjExportHandleNext == MAX_OBJ_EXPORT_HANDLES) {
+            hObjExportHandleNext = 1;
+        }
+
+        pHandleRef = mapFind(&objExportHandleMap, hObject);
+
+        if (hObject != hObjExportRmClient && pHandleRef == NULL)
+        {
+            break;
+        }
+        else
+        {
+            hObject = 0;
+        }
+
+    } while(hObjExportHandleNext != hStartHandle);
+
+    if (hObject != 0)
+    {
+        void *pHandleRef = mapInsertNew(&objExportHandleMap, hObject);
+
+        if (pHandleRef == NULL)
+        {
+            hObject = 0;
+        }
+    }
+
+    return hObject;
+}
+
+//
+// Validate that the given hObject is not one of our internally used handles.
+//
+// Note that mapFind(&objExportHandleMap, hObject) could still fail; that is the
+// caller's responsibility.
+//
+static NvBool RmValidateHandleAgainstInternalHandles(RmObjExportHandle hObject)
+{
+    RmObjExportDeviceMapIter iter;
+
+    //
+    // No external RmObjExportHandle could be valid if hObjExportRmClient has
+    // not been allocated yet, or if it is equal to any of the handles used
+    // internally by code in this file.
+    //
+    if (objExportImportRefCount == 0 || hObjExportRmClient == 0 ||
+        hObject == hObjExportRmClient)
+    {
+        return NV_FALSE;
+    }
+
+    iter = mapIterAll(&objExportDeviceMap);
+    while (mapIterNext(&iter))
+    {
+        RmObjExportDevice *pRmObjExportDevice = iter.pValue;
+
+        if (pRmObjExportDevice->hRmDevice != 0 &&
+            (hObject == pRmObjExportDevice->hRmDevice ||
+             hObject == pRmObjExportDevice->hRmSubDevice ||
+             hObject == pRmObjExportDevice->hGpuInstSub))
+        {
+            return NV_FALSE;
+        }
+    }
+
+    return NV_TRUE;
+}
+
+//
+// Increment reference count of static objects internally
+// used by code in this file.
+//
+static NV_STATUS RmRefObjExportImport(void)
+{
+    NV_STATUS rmStatus = NV_OK;
+    RM_API   *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+    if ((objExportImportRefCount++) != 0)
+    {
+        NV_ASSERT(hObjExportRmClient != 0);
+        NV_ASSERT(pMemAllocator != NULL);
+        return NV_OK;
+    }
+
+    rmStatus = pRmApi->AllocWithHandle(pRmApi,
+                                       NV01_NULL_OBJECT,
+                                       NV01_NULL_OBJECT,
+                                       NV01_NULL_OBJECT,
+                                       NV01_ROOT,
+                                       &hObjExportRmClient,
+                                       sizeof(hObjExportRmClient));
+    if (rmStatus != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Unable to alloc root in %s\n", __FUNCTION__);
+        goto failed;
+    }
+
+    pMemAllocator = portMemAllocatorCreateNonPaged();
+
+    if (pMemAllocator == NULL)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to alloc memory allocator in %s\n",
+                  __FUNCTION__);
+        goto failed;
+    }
+
+    mapInit(&objExportHandleMap, pMemAllocator);
+    mapInit(&objExportDeviceMap, pMemAllocator);
+
+    return NV_OK;
+
+failed:
+
+    RmUnrefObjExportImport();
+
+    return rmStatus;
+}
+
+//
+// Decrement reference count of static objects internally used by code in this
+// file, and free them if reference count reaches to zero.
+//
+static void RmUnrefObjExportImport(void)
+{
+    RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+    if ((--objExportImportRefCount) != 0)
+    {
+        return;
+    }
+
+    if (pMemAllocator != NULL)
+    {
+        mapDestroy(&objExportDeviceMap);
+        mapDestroy(&objExportHandleMap);
+
+        portMemAllocatorRelease(pMemAllocator);
+        pMemAllocator = NULL;
+    }
+
+    if (hObjExportRmClient != 0)
+    {
+        NV_STATUS rmStatus = pRmApi->Free(pRmApi,
+            hObjExportRmClient,
+            hObjExportRmClient);
+
+        NV_ASSERT(rmStatus == NV_OK);
+        hObjExportRmClient = 0;
+    }
+}
+
+NV_STATUS RmExportObject(NvHandle hSrcClient, NvHandle hSrcObject,
+                         RmObjExportHandle *pDstObject, NvU32 *pDeviceInstance)
+{
+    RmObjExportHandle  hDstObject;
+    NvU32              deviceInstance = NV_MAX_DEVICES;
+    NvBool             bClientAsDstParent = NV_FALSE;
+    NV_STATUS          status;
+    RM_API            *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+    MIG_INSTANCE_REF   ref = kmigmgrMakeNoMIGReference();
+    NvU64              deviceMapIdx = 0;
+    RmObjExportDevice *pObjExportDevice = NULL;
+    RsResourceRef     *pSrcResourceRef;
+    RsResourceRef     *pDeviceRef;
+
+    if (pDstObject == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    //
+    // Find the device instance on which the rm object exists.
+    //
+    NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
+        serverutilGetResourceRef(hSrcClient, hSrcObject, &pSrcResourceRef));
+
+    status = refFindAncestorOfType(pSrcResourceRef, classId(Device), &pDeviceRef);
+    if (status == NV_OK)
+    {
+        Device *pDevice = dynamicCast(pDeviceRef->pResource, Device);
+        OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice);
+        NvU64 swizzId = KMIGMGR_MAX_GPU_SWIZZID;
+
+        if (IS_MIG_IN_USE(pGpu))
+        {
+            KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
+
+            status = kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager,
+                                                     pDevice, &ref);
+            if (status == NV_OK)
+            {
+               swizzId = ref.pKernelMIGGpuInstance->swizzId;
+            }
+        }
+
+        deviceMapIdx = pDevice->deviceInst | (swizzId << 32);
+        deviceInstance = pDevice->deviceInst;
+        pObjExportDevice = mapFind(&objExportDeviceMap, deviceMapIdx);
+    }
+    else
+    {
+        bClientAsDstParent = NV_TRUE;
+    }
+
+    status = RmRefObjExportImport();
+
+    if (status != NV_OK)
+    {
+        return status;
+    }
+
+    if (!bClientAsDstParent &&
+        ((pObjExportDevice == NULL) ||
+         serverutilValidateNewResourceHandle(hObjExportRmClient,
+                pObjExportDevice->hRmDevice)))
+    {
+        //
+        // Device object has not been created or it got destroyed in the
+        // teardown path of device instance destruction; allocate a fresh device
+        // object.
+        //
+        NV0080_ALLOC_PARAMETERS params;
+        NV2080_ALLOC_PARAMETERS subdevParams;
+
+        if (pObjExportDevice == NULL)
+        {
+            pObjExportDevice = mapInsertNew(&objExportDeviceMap, deviceMapIdx);
+
+            pObjExportDevice->hRmDevice = RmGenerateObjExportHandle();
+            pObjExportDevice->hRmSubDevice = RmGenerateObjExportHandle();
+            pObjExportDevice->hGpuInstSub = NV01_NULL_OBJECT;
+
+            if (pObjExportDevice->hRmDevice == 0 ||
+                pObjExportDevice->hRmSubDevice == 0)
+            {
+                NV_PRINTF(LEVEL_ERROR, "Failed to allocate object handles in %s\n",
+                          __FUNCTION__);
+
+                mapRemove(&objExportDeviceMap, pObjExportDevice);
+
+                status = NV_ERR_NO_MEMORY;
+                goto done;
+            }
+        }
+
+        portMemSet(&params, 0, sizeof(NV0080_ALLOC_PARAMETERS));
+
+        params.deviceId = deviceInstance;
+        params.hClientShare = hObjExportRmClient;
+
+        status = pRmApi->AllocWithHandle(pRmApi,
+                                         hObjExportRmClient,
+                                         hObjExportRmClient,
+                                         pObjExportDevice->hRmDevice,
+                                         NV01_DEVICE_0,
+                                         &params,
+                                         sizeof(params));
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR, "Unable to alloc device in %s\n",
+                      __FUNCTION__);
+            goto done;
+        }
+
+        portMemSet(&subdevParams, 0, sizeof(NV2080_ALLOC_PARAMETERS));
+
+        subdevParams.subDeviceId = 0;
+
+        status = pRmApi->AllocWithHandle(pRmApi,
+                                         hObjExportRmClient,
+                                         pObjExportDevice->hRmDevice,
+                                         pObjExportDevice->hRmSubDevice,
+                                         NV20_SUBDEVICE_0,
+                                         &subdevParams,
+                                         sizeof(subdevParams));
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR, "Unable to alloc subdevice in %s\n",
+                      __FUNCTION__);
+
+            (void) pRmApi->Free(pRmApi, hObjExportRmClient,
+                                pObjExportDevice->hRmDevice);
+            goto done;
+        }
+
+        // Allocate MIG subscription, if needed
+        if ((ref.pKernelMIGGpuInstance != NULL) &&
+            (ref.pKernelMIGGpuInstance->swizzId < KMIGMGR_MAX_GPU_SWIZZID))
+        {
+            NVC637_ALLOCATION_PARAMETERS giSubAllocParams;
+
+            portMemSet(&giSubAllocParams, 0, sizeof(giSubAllocParams));
+            giSubAllocParams.swizzId = ref.pKernelMIGGpuInstance->swizzId;
+
+            pObjExportDevice->hGpuInstSub = RmGenerateObjExportHandle();
+
+            status = pRmApi->AllocWithHandle(pRmApi,
+                                             hObjExportRmClient,
+                                             pObjExportDevice->hRmSubDevice,
+                                             pObjExportDevice->hGpuInstSub,
+                                             AMPERE_SMC_PARTITION_REF,
+                                             &giSubAllocParams,
+                                             sizeof(giSubAllocParams));
+            if (status != NV_OK)
+            {
+                NV_PRINTF(LEVEL_ERROR, "Unable to alloc gpu instance subscription in %s\n",
+                          __FUNCTION__);
+
+                (void) pRmApi->Free(pRmApi, hObjExportRmClient,
+                                    pObjExportDevice->hRmDevice);
+                goto done;
+            }
+        }
+    }
+
+    hDstObject = RmGenerateObjExportHandle();
+
+    if (hDstObject == 0)
+    {
+        NV_PRINTF(LEVEL_ERROR, "Failed to allocate object handle in %s\n",
+                  __FUNCTION__);
+        status = NV_ERR_NO_MEMORY;
+        goto done;
+    }
+
+    // If duping under device handle fails, try subdevice handle.
+    status = pRmApi->DupObject(pRmApi,
+                               hObjExportRmClient,
+                               bClientAsDstParent ? hObjExportRmClient :
+                               pObjExportDevice->hRmDevice,
+                               &hDstObject,
+                               hSrcClient,
+                               hSrcObject,
+                               0 /* flags */);
+    if (status != NV_OK)
+    {
+        if (!bClientAsDstParent && (status == NV_ERR_INVALID_OBJECT_PARENT))
+        {
+            NV_PRINTF(LEVEL_INFO,
+                "pRmApi->DupObject(Dev, failed due to invalid parent in %s."
+                " Now attempting DupObject with Subdev handle.\n",
+                __FUNCTION__);
+
+            status = pRmApi->DupObject(pRmApi,
+                                       hObjExportRmClient,
+                                       pObjExportDevice->hRmSubDevice,
+                                       &hDstObject,
+                                       hSrcClient,
+                                       hSrcObject,
+                                       0 /* flags */);
+            if (status != NV_OK)
+            {
+                RmUnrefObjExportHandle(hDstObject);
+
+                NV_PRINTF(LEVEL_ERROR,
+                    "pRmApi->DupObject(Subdev, failed with error code 0x%x in %s\n",
+                    status, __FUNCTION__);
+                goto done;
+            }
+        }
+        else
+        {
+            RmUnrefObjExportHandle(hDstObject);
+
+            NV_PRINTF(LEVEL_ERROR,
+                "pRmApi->DupObject(Dev, failed with error code 0x%x in %s\n",
+                status, __FUNCTION__);
+            goto done;
+        }
+    }
+
+    if (pDeviceInstance != NULL)
+    {
+        *pDeviceInstance = deviceInstance;
+    }
+
+    *pDstObject = hDstObject;
+
+done:
+    if (status != NV_OK)
+    {
+        RmUnrefObjExportImport();
+    }
+
+    return status;
+}
+
+void RmFreeObjExportHandle(RmObjExportHandle hObject)
+{
+    if (!RmValidateHandleAgainstInternalHandles(hObject))
+    {
+        NV_PRINTF(LEVEL_ERROR, "Invalid handle to exported object in %s\n",
+                  __FUNCTION__);
+        return;
+    }
+
+    RmUnrefObjExportHandle(hObject);
+
+    RmUnrefObjExportImport();
+}
+
+NV_STATUS RmImportObject(NvHandle hDstClient, NvHandle hDstParent,
+                         NvHandle *phDstObject, RmObjExportHandle hSrcObject,
+                         NvU8 *pObjectType)
+{
+    NV_STATUS status;
+    NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS params;
+    RM_API   *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
+
+    if (!RmValidateHandleAgainstInternalHandles(hSrcObject))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    if (mapFind(&objExportHandleMap, hSrcObject) == NULL)
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    if (pObjectType != NULL)
+    {
+        params.hObject = hSrcObject;
+    //
+    // rmApiGetEffectiveAddrSpace expect mapping flags to be set as DIRECT for
+    // GPU cacheable Sysmem for Pre-Ampere chips. We are not doing any mapping
+    // here, so passing this as workaround to get the expected address space.
+    //
+        params.mapFlags = FLD_SET_DRF(OS33, _FLAGS, _MAPPING, _DIRECT, 0);
+        params.addrSpaceType = \
+                NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_INVALID;
+
+        status = pRmApi->Control(pRmApi, hObjExportRmClient, hObjExportRmClient,
+                                 NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE,
+                                 &params, sizeof(params));
+        if (status != NV_OK)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "GET_ADDR_SPACE_TYPE failed with error code 0x%x in %s\n",
+                      status, __FUNCTION__);
+            return status;
+        }
+
+        switch (params.addrSpaceType)
+        {
+            case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_SYSMEM:
+                *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_SYSMEM;
+                break;
+            case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_VIDMEM:
+                *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_VIDMEM;
+                break;
+            case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC:
+                *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC;
+                break;
+#if defined(NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC) && \
+    defined(NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC)
+            case NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC:
+                *pObjectType = NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC;
+                break;
+#endif
+            default:
+                NV_ASSERT_OK_OR_RETURN(NV_ERR_INVALID_ARGUMENT);
+        }
+    }
+
+    status = pRmApi->DupObject(pRmApi, hDstClient, hDstParent, phDstObject,
+                               hObjExportRmClient, hSrcObject,
+                               0 /* flags */);
+
+    if (status != NV_OK)
+    {
+        NV_PRINTF(LEVEL_ERROR,
+                  "pRmApi->DupObject(pRmApi,  failed with error code 0x%x in %s\n",
+                  status, __FUNCTION__);
+        return status;
+    }
+
+    return NV_OK;
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/subdevice_ctrl_os_unix.c b/userspace/sdks/nv570/nvalloc/unix/src/subdevice_ctrl_os_unix.c
new file mode 100644
index 0000000..fdbdb6b
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/subdevice_ctrl_os_unix.c
@@ -0,0 +1,97 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>                     // NV device driver interface
+#include <nv-priv.h>
+#include <os/os.h>
+#include <nvos.h>
+#include <osapi.h>
+#include "gpu/gpu.h"
+#include "gpu/gpu_resource.h"
+#include "gpu/subdevice/subdevice.h"
+#include <diagnostics/journal.h>
+#include "gpu/mem_mgr/mem_desc.h"
+#include "mem_mgr/mem.h"
+#include <nvpcie.h>
+#include <core/locks.h>
+#include "rmapi/rs_utils.h"
+#include "rmapi/client_resource.h"
+#include <class/cl0000.h>
+#include <class/cl90cd.h>
+#include <class/cl0005.h>  // NV01_EVENT
+#include <class/cl003e.h>  // NV01_MEMORY_SYSTEM
+#include <class/cl844c.h>  // G84_PERFBUFFER
+#include <ctrl/ctrl0000/ctrl0000gpu.h>
+#include <ctrl/ctrl0000/ctrl0000unix.h>
+#include <ctrl/ctrl2080/ctrl2080gpu.h>
+#include <ctrl/ctrl2080/ctrl2080unix.h>
+
+/*!
+ * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS
+ *        RmControl request. It will check if the GPU video memory will be
+ *        persistent during system suspend/resume cycle.
+ *
+ * @param[in]     pSubdevice
+ * @param[in,out] pParams
+ *
+ * @return
+ *      NV_OK   Success
+ */
+NV_STATUS
+subdeviceCtrlCmdOsUnixVidmemPersistenceStatus_IMPL
+(
+    Subdevice *pSubdevice,
+    NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS *pParams
+)
+{
+    OBJGPU     *pGpu = GPU_RES_GET_GPU(pSubdevice);
+    nv_state_t *nv   = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp  = NV_GET_NV_PRIV(nv);
+
+    pParams->bVidmemPersistent = (nv->preserve_vidmem_allocations || nvp->s0ix_pm_enabled);
+
+    return NV_OK;
+}
+
+/*!
+ * @brief Implements the NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS
+ *        RmControl request. It sets restore TGP flag which is used
+ *        to restore TGP limits when client is killed.
+ *
+ * @param[in] pSubdevice
+ * @param[in] pParams
+ *
+ * @return
+ *      NV_OK   Success
+ */
+NV_STATUS
+subdeviceCtrlCmdOsUnixUpdateTgpStatus_IMPL
+(
+    Subdevice *pSubdevice,
+    NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS *pParams
+)
+{
+    pSubdevice->bUpdateTGP = pParams->bUpdateTGP;
+
+    return NV_OK;
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/unix_console.c b/userspace/sdks/nv570/nvalloc/unix/src/unix_console.c
new file mode 100644
index 0000000..28b597b
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/unix_console.c
@@ -0,0 +1,412 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <core/locks.h>
+#include <ctrl/ctrl0080/ctrl0080unix.h>
+#include <gpu/device/device.h>
+#include <gpu/gpu.h>
+#include <nv-priv.h>
+#include <nv.h>
+#include <osapi.h>
+#include <gpu/mem_mgr/mem_mgr.h>
+#include <platform/sli/sli.h>
+
+#include <vgpu/rpc.h>
+#include "vgpu/vgpu_events.h"
+
+static NV_STATUS
+unixCallVideoBIOS
+(
+    OBJGPU *pGpu,
+    NvU32  *eax,
+    NvU32  *ebx
+)
+{
+    NV_STATUS status = NV_ERR_NOT_SUPPORTED;
+
+    if (NVCPU_IS_X86_64)
+    {
+        NvU32         eax_in = *eax;
+        NvU32         ebx_in = *ebx;
+
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI))
+        {
+            return NV_ERR_NOT_SUPPORTED;
+        }
+
+        NV_PRINTF(LEVEL_INFO, "unixCallVideoBIOS: 0x%x 0x%x, vga_satus = %d\n", *eax, *ebx, NV_PRIMARY_VGA(NV_GET_NV_STATE(pGpu)));
+
+        status = nv_vbios_call(pGpu, eax, ebx);
+
+        // this was originally changed for nt in changelist 644223
+        if (*eax != 0x4f)
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "int10h(%04x, %04x) vesa call failed! (%04x, %04x)\n",
+                      eax_in, ebx_in, *eax, *ebx);
+            status = NV_ERR_GENERIC;
+        }
+    }
+
+    return status;
+}
+
+static void
+RmSaveDisplayState
+(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t     *nv             = NV_GET_NV_STATE(pGpu);
+    nv_priv_t      *nvp            = NV_GET_NV_PRIV(nv);
+    RM_API         *pRmApi         = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    KernelDisplay  *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+    NvBool          use_vbios      = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv);
+    NvU32           eax, ebx;
+    NV_STATUS       status;
+    NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS  preUnixConsoleParams = {0};
+    NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS  postUnixConsoleParams = {0};
+
+    if (IS_VIRTUAL(pGpu) || (pKernelDisplay == NULL) || nv->client_managed_console)
+    {
+        return;
+    }
+
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI))
+    {
+        NV_PRINTF(LEVEL_INFO, "RM fallback doesn't support saving of efifb console\n");
+        return;
+    }
+
+    preUnixConsoleParams.bSave     = NV_TRUE;
+    preUnixConsoleParams.bUseVbios = use_vbios;
+
+    NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice,
+                        NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE,
+                        &preUnixConsoleParams, sizeof(preUnixConsoleParams)), return);
+
+    if (preUnixConsoleParams.bReturnEarly)
+    {
+        return;
+    }
+
+    if (use_vbios)
+    {
+        //
+        // Attempt to identify the currently set VESA mode; assume
+        // vanilla VGA text if the VBIOS call fails.
+        //
+        eax = 0x4f03;
+        ebx = 0;
+        if (NV_OK == unixCallVideoBIOS(pGpu, &eax, &ebx))
+        {
+            nvp->vga.vesaMode = (ebx & 0x3fff);
+        }
+        else
+        {
+            nvp->vga.vesaMode = 3;
+        }
+    }
+
+    postUnixConsoleParams.bSave     = NV_TRUE;
+    postUnixConsoleParams.bUseVbios = use_vbios;
+
+    NV_CHECK_OK(status, LEVEL_ERROR,
+                pRmApi->Control(pRmApi, nv->rmapi.hClient,nv->rmapi.hSubDevice,
+                                NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE,
+                                &postUnixConsoleParams, sizeof(postUnixConsoleParams)));
+}
+
+static void RmRestoreDisplayState
+(
+    OBJGPU *pGpu
+)
+{
+    nv_state_t     *nv             = NV_GET_NV_STATE(pGpu);
+    nv_priv_t      *nvp            = NV_GET_NV_PRIV(nv);
+    RM_API         *pRmApi         = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
+    NvBool          use_vbios      = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv);;
+    KernelDisplay  *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+    NV_STATUS       status;
+    NvU32           eax, ebx;
+    NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS  preUnixConsoleParams   = {0};
+    NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS  postUnixConsoleParams = {0};
+
+    //
+    // Since vGPU does all real hardware management in the host,
+    // there is nothing to do at this point in the guest OS.
+    //
+    if (IS_VIRTUAL(pGpu) || (pKernelDisplay == NULL) || nv->client_managed_console)
+    {
+        return;
+    }
+
+    //
+    // Fix up DCB index VBIOS scratch registers.
+    // The strategies employed are:
+    //
+    // SBIOS/VBIOS:
+    // Clear the DCB index, and set the previous DCB index to the original
+    // value. This allows the VBIOS (during the int10h mode-set) to
+    // determine which display to enable, and to set the head-enabled bit
+    // as needed (see bugs #264873 and #944398).
+    //
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI))
+    {
+        NV_PRINTF(LEVEL_INFO, "RM fallback doesn't support efifb console restore\n");
+        return;
+    }
+
+    preUnixConsoleParams.bUseVbios = use_vbios;
+    preUnixConsoleParams.bSave = NV_FALSE;
+
+    NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, pRmApi->Control(pRmApi, nv->rmapi.hClient,
+                        nv->rmapi.hSubDevice,
+                        NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE,
+                        &preUnixConsoleParams, sizeof(preUnixConsoleParams)), return);
+
+    if (preUnixConsoleParams.bReturnEarly)
+    {
+        return;
+    }
+
+    if (use_vbios)
+    {
+        eax = 0x4f02;
+        ebx = nvp->vga.vesaMode;
+
+        if (NV_OK == unixCallVideoBIOS(pGpu, &eax, &ebx))
+        {
+            postUnixConsoleParams.bVbiosCallSuccessful = NV_TRUE;
+        }
+    }
+
+    postUnixConsoleParams.bSave = NV_FALSE;
+    postUnixConsoleParams.bUseVbios = use_vbios;
+
+    NV_CHECK_OK(status, LEVEL_ERROR,
+                pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice,
+                                NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE,
+                                &postUnixConsoleParams, sizeof(postUnixConsoleParams)));
+}
+
+static void
+RmChangeResMode
+(
+    OBJGPU *pGpu,
+    NvBool  hires
+)
+{
+    if (hires)
+    {
+        SLI_LOOP_START(SLI_LOOP_FLAGS_NONE)
+
+        RmSaveDisplayState(pGpu);
+
+        SLI_LOOP_END
+    } 
+    else
+    {
+        SLI_LOOP_START(SLI_LOOP_FLAGS_NONE)
+
+        RmRestoreDisplayState(pGpu);
+        //
+        // vGPU:
+        //
+        // Since vGPU does all real hardware management in the host, if we
+        // are in guest OS (where IS_VIRTUAL(pGpu) is true), do an RPC to
+        // the host to trigger switch from HIRES to (LORES)VGA.
+        //
+        if (IS_VIRTUAL(pGpu))
+        {
+            NV_STATUS status = NV_OK;
+            NV_RM_RPC_SWITCH_TO_VGA(pGpu, status);
+        }
+
+        SLI_LOOP_END
+    }
+}
+
+NV_STATUS NV_API_CALL
+rm_save_low_res_mode
+(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    THREAD_STATE_NODE threadState;
+    OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    RmSaveDisplayState(pGpu);
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return NV_OK;
+}
+
+NV_STATUS
+deviceCtrlCmdOsUnixVTSwitch_IMPL
+(
+    Device *pDevice,
+    NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS *pParams
+)
+{
+    OBJGPU     *pGpu = GPU_RES_GET_GPU(pDevice);
+    nv_state_t *nv   = NV_GET_NV_STATE(pGpu);
+    NvBool      hires;
+    NvBool      bChangeResMode = NV_TRUE;
+
+    switch (pParams->cmd)
+    {
+        case NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_SAVE_VT_STATE:
+            hires = NV_TRUE;
+            break;
+
+        case NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_RESTORE_VT_STATE:
+            hires = NV_FALSE;
+            break;
+
+        case NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_CONSOLE_RESTORED:
+            bChangeResMode = NV_FALSE;
+            break;
+
+        default:
+            return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    RmUpdateGc6ConsoleRefCount(nv,
+                               pParams->cmd != NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_SAVE_VT_STATE);
+
+    if (!bChangeResMode)
+    {
+        return NV_OK;
+    }
+
+    if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FB) == NV_OK)
+    {
+        RmChangeResMode(pGpu, hires);
+        rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_INFO,"%s: Failed to acquire GPU lock",  __FUNCTION__);
+    }
+    return NV_OK;
+}
+
+NV_STATUS deviceCtrlCmdOsUnixVTGetFBInfo_IMPL
+(
+    Device *pDevice,
+    NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS *pParams
+)
+{
+    OBJGPU     *pGpu = GPU_RES_GET_GPU(pDevice);
+    NvBool bContinue = NV_TRUE;
+
+    if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FB) == NV_OK)
+    {
+        // See if the console is on one of the subdevices of this device.
+        portMemSet(pParams, 0, sizeof(*pParams));
+
+        SLI_LOOP_START(SLI_LOOP_FLAGS_NONE)
+
+            MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
+            nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+            if ((memmgrGetReservedConsoleMemDesc(pGpu, pMemoryManager) != NULL) && bContinue)
+            {
+                NvU64 baseAddr, size;
+                NvU32 width, height, depth, pitch;
+
+                // There should only be one.
+                NV_ASSERT(pParams->width == 0);
+
+                pParams->subDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
+
+                // Console is either mapped to BAR1 or BAR2 + 16 MB
+                nv_get_screen_info(nv, &baseAddr, &width, &height, &depth,
+                                   &pitch, &size);
+
+                pParams->baseAddress = baseAddr;
+                pParams->size = size;
+                pParams->width = (NvU16)width;
+                pParams->height = (NvU16)height;
+                pParams->depth = (NvU16)depth;
+                pParams->pitch = (NvU16)pitch;
+
+                if (baseAddr != 0)
+                {
+                    bContinue = NV_FALSE; 
+                }
+            }
+
+        SLI_LOOP_END
+
+        rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_INFO,"%s: Failed to acquire GPU lock",  __FUNCTION__);
+    }
+
+    return NV_OK;
+}
+
+
+void 
+dispdeviceFillVgaSavedDisplayState
+(
+    OBJGPU *pGpu,
+    NvU64   vgaAddr,
+    NvU8    vgaMemType,
+    NvBool  vgaValid,
+    NvU64   workspaceAddr,
+    NvU8    workspaceMemType,
+    NvBool  workspaceValid,
+    NvBool  baseValid,
+    NvBool  workspaceBaseValid
+)
+{
+    nv_state_t *nv        = NV_GET_NV_STATE(pGpu);
+    nv_priv_t  *nvp       = NV_GET_NV_PRIV(nv);
+    NvBool      use_vbios = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv);
+
+    if (use_vbios)
+    {
+        nvp->vga.base.addr = vgaAddr;
+        nvp->vga.base.memTarget = vgaMemType;
+        nvp->vga.base.valid = vgaValid;
+        nvp->vga.baseValid = baseValid;
+
+        nvp->vga.workspaceBase.addr = workspaceAddr;
+        nvp->vga.workspaceBase.memTarget = workspaceMemType;
+        nvp->vga.workspaceBase.valid = workspaceValid;
+        nvp->vga.workspaceBaseValid = workspaceBaseValid;
+    }
+}
+
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/unix_intr.c b/userspace/sdks/nv570/nvalloc/unix/src/unix_intr.c
new file mode 100644
index 0000000..aef36ab
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/unix_intr.c
@@ -0,0 +1,1050 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nv.h>
+#include <nv-priv.h>
+#include <osapi.h>
+#include <core/thread_state.h>
+#include <core/locks.h>
+#include <gpu_mgr/gpu_mgr.h>
+#include <gpu/gpu.h>
+#include "kernel/gpu/intr/intr.h"
+#include "gpu/bif/kernel_bif.h"
+#include "gpu/mmu/kern_gmmu.h"
+#include "gpu/disp/kern_disp.h"
+#include "gpu/disp/head/kernel_head.h"
+#include <nv_sriov_defines.h>
+
+#include "platform/sli/sli.h"
+
+//
+// Service Interrupts that are not Masked when using
+// INTR_USE_INTR_MASK_FOR_LOCKING
+// Note that everything done here is without holding the GPUs Lock.
+// Returning NV_OK means an interrupt has been serviced
+//
+static NV_STATUS _osIsrIntrMask_GpusUnlocked(OBJGPU *pGpu, THREAD_STATE_NODE *pThreadState)
+{
+    NV_STATUS rmStatus = NV_ERR_NO_INTR_PENDING;
+    Intr *pIntr = GPU_GET_INTR(pGpu);
+    MC_ENGINE_BITVECTOR intrPending, intrEngMask;
+    NvU32 vblankIntrServicedHeadMask = 0;
+    KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+    NvBool bFailedLockAcquire = NV_FALSE;
+
+    if (gpuIsGpuFullPower(pGpu) == NV_FALSE)
+    {
+        return rmStatus;
+    }
+
+    intrGetPendingLowLatencyHwDisplayIntr_HAL(pGpu, pIntr, &intrPending, pThreadState);
+
+    // Only disp interrupt remains in INTR_MASK
+    if ((pKernelDisplay != NULL) &&
+        (( pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP_LOW)) ||
+         (!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP))))
+    {
+        if (kdispAcquireLowLatencyLockConditional(&pKernelDisplay->lowLatencyLock))
+        {
+
+            if (pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_AGGRESSIVE_VBLANK_HANDLING))
+            {
+                NV_ASSERT_OK(
+                    kdispOptimizePerFrameOsCallbacks(pGpu, pKernelDisplay, NV_TRUE,
+                                                 pThreadState,
+                                                 &vblankIntrServicedHeadMask,
+                                                 &intrPending));
+
+                //
+                // Need to setup deferred vblank handling for once the
+                // GPUs Lock is released since we cleared vblank intrs.
+                //
+                if (vblankIntrServicedHeadMask)
+                {
+                    vblankIntrServicedHeadMask |= kdispGetDeferredVblankHeadMask(pKernelDisplay);
+                    kdispSetDeferredVblankHeadMask(pKernelDisplay, vblankIntrServicedHeadMask);
+                }
+            }
+            else
+            {
+                kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0,
+                                                (VBLANK_STATE_PROCESS_LOW_LATENCY |
+                                                VBLANK_STATE_PROCESS_CALLED_FROM_ISR),
+                                                pThreadState,
+                                                &vblankIntrServicedHeadMask,
+                                                &intrPending);
+            }
+
+            kdispReleaseLowLatencyLock(&pKernelDisplay->lowLatencyLock);
+        }
+        else
+        {
+            //
+            // If we lose this, then another thread is already servicing the low latency interrupts.
+            // Claim the interrupt, but go through the below disable code so that we don't immediately
+            // cause another interrupt indefinitely which could starve the other thread that already 
+            // has the lock
+            //
+            rmStatus = NV_OK;
+            bFailedLockAcquire = NV_TRUE;
+        }
+
+        //
+        // Bug 4512542: The check for INTR_MASK is currently redundant since it's
+        // already checked outside this function, but we would like to unconditionally
+        // call this function if we can service the vblank first.
+        //
+        if (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING))
+        {
+            //
+            // If an unmasked intr is still pending after the above, that means a
+            // intr other than the vblank, so we'll MSK this off
+            // for now and handle once the GPUs Lock is released.
+            //
+            intrGetPendingLowLatencyHwDisplayIntr_HAL(pGpu, pIntr, &intrPending, pThreadState);
+
+            //
+            // Only disp interrupt remains in INTR_MASK
+            // Don't re-mask it if we have an entirely separate interrupt line, unless we have lost the race above.
+            //
+            if (( pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP_LOW) && bFailedLockAcquire) ||
+                (!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP)))
+            {
+                NvU32 intrMaskFlags;
+                NvU64 oldIrql;
+
+                oldIrql = rmIntrMaskLockAcquire(pGpu);
+
+                // Skip the IntrMask update if told to by another thread.
+                intrMaskFlags = intrGetIntrMaskFlags(pIntr);
+                if ((intrMaskFlags & INTR_MASK_FLAGS_ISR_SKIP_MASK_UPDATE) == 0)
+                {
+                    if (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING))
+                    {
+                        intrGetIntrMask_HAL(pGpu, pIntr, &intrEngMask, pThreadState);
+
+                        // Only disp interrupt remains in INTR_MASK
+                        bitVectorClr(&intrEngMask, MC_ENGINE_IDX_DISP);
+
+                        if (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_TOP_EN_FOR_VBLANK_HANDLING))
+                        {
+                            intrSetDisplayInterruptEnable_HAL(pGpu, pIntr, NV_FALSE, pThreadState);
+                        }
+                        else
+                        {
+                            intrSetIntrMask_HAL(pGpu, pIntr, &intrEngMask, pThreadState);
+                        }
+                    }
+                }
+
+                rmIntrMaskLockRelease(pGpu, oldIrql);
+            }
+        }
+
+        if (vblankIntrServicedHeadMask)
+        {
+            rmStatus = NV_OK;
+        }
+    }
+
+    return rmStatus;
+}
+
+static NvBool osInterruptPending(
+    OBJGPU            *pGpu,
+    NvBool            *serviced,
+    THREAD_STATE_NODE *pThreadState
+)
+{
+    KernelDisplay   *pKernelDisplay;
+    NvBool pending, sema_release;
+    THREAD_STATE_NODE threadState;
+    Intr *pIntr = NULL;
+    MC_ENGINE_BITVECTOR intr0Pending;
+    MC_ENGINE_BITVECTOR intr1Pending;
+
+    *serviced = NV_FALSE;
+    pending = NV_FALSE;
+    sema_release = NV_TRUE;
+    NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer
+    PORT_MEM_ALLOCATOR *pIsrAllocator;
+
+    //
+    // GPU interrupt servicing ("top half")
+    //
+    // Top-level processing of GPU interrupts is performed using the
+    // steps below; although the code is straight forward, there
+    // are a few points to be aware of:
+    //
+    //  1) The GPUs lock is acquired for two reasons: to allow
+    //       looping over GPUs atomically in SLI and to sanity
+    //       check the PCI configuration space of any initialized
+    //       GPUs. If the acquisition fails, the early return
+    //       is acceptable since GPU interrupts are disabled while
+    //       the lock is held; note that returning success
+    //       in this case could interfere with the processing
+    //       of third-party device interrupts if the IRQ is shared.
+    //       Due to the above, some interrupts may be reported as
+    //       unhandled if invocations of the ISR registered with
+    //       the kernel are not serialized. This is bad, but
+    //       ignored by currently supported kernels, provided most
+    //       interrupts are handled.
+    //
+    //  2) Since acquisition of the lock disables interrupts
+    //       on all initialized GPUs, NV_PMC_INTR_EN_0 can not be
+    //       relied up on to determine whether interrupts are
+    //       expected from a given GPU. The code below is therefore
+    //       forced to rely on software state. NV_PMC_INTR_EN_0
+    //       is read only as a sanity check to guard against
+    //       invalid GPU state (lack of PCI memory access, etc.).
+    //
+    //  3) High priority interrupts (VBLANK, etc.), are serviced in
+    //       this function, service of all other interrupts is
+    //       deferred until a bottom half. If a bottom half needs
+    //       to be scheduled, release of the GPUs lock is
+    //       likewise deferred until completion of the bottom half.
+    //
+    //  4) To reduce the risk of starvation, an effort is made to
+    //       consolidate processing of interrupts pending on
+    //       all GPUs sharing a given IRQ.
+    //
+    //  5) Care is taken to ensure that the consolidated interrupt
+    //       processing is performed in the context of a GPU
+    //       that has interrupts pending. Else if additional ISR
+    //       processing via a bottom-half is required, this
+    //       bottom-half ISR might race against the GPU's shut-down
+    //       path.
+    //
+
+    pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
+    tlsIsrInit(pIsrAllocator);
+
+    //
+    // Also check if we need to acquire the GPU lock at all and get critical interrupts
+    // This should not violate (1) from above since we are not servicing the GPUs in SLI,
+    // only checking their state.
+    //
+    // To do so, two steps are required:
+    // Step 1: Check if we can service nonstall interrupts outside the GPUs lock. Otherwise
+    // we have to acquire the GPUs lock to service the nonstall interrupts anyway, and we
+    // can't get around acquiring the GPUs lock.
+    //
+    // Step 2 is inline below
+    //
+    NvBool bIsAnyStallIntrPending = NV_TRUE;
+    NvBool bIsAnyBottomHalfStallPending = NV_FALSE;
+    NvU32  isDispPendingPerGpu = 0, isDispLowLatencyPendingPerGpu = 0;
+    NvU32  isTmrPendingPerGpu     = 0;
+
+    {
+        threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+        bIsAnyStallIntrPending = NV_FALSE;
+
+        {
+            pIntr = GPU_GET_INTR(pGpu);
+            KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+
+            if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+            {
+                bIsAnyStallIntrPending = NV_TRUE;
+            }
+            else
+            if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)))
+            {
+                NvBool bCtxswLog = NV_FALSE;
+                intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
+                intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog);
+                if (!bitVectorTestAllCleared(&intr1Pending))
+                {
+                    intrServiceNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
+                    *serviced = NV_TRUE;
+                }
+
+                //
+                // Step 2: Service nonstall interrupts and check if there are any stall interrupts
+                // pending on any GPUs. If there are, then we will take the GPU lock.
+                //
+                // Bug 4223192: calling intrGetPendingStall_HAL is rather expensive,
+                // so save off the critical interrupts to be handled in the top half.
+                //
+
+                // vblank read leaf 4 + read the rest
+
+                NvU32 intrStaticReg;
+                intrGetPendingDispLowLatencyAndStaticReg_HAL(pGpu, pIntr, &intrStaticReg, &intr0Pending, &threadState);
+                intrGetRemainingPendingStall_HAL(pGpu, pIntr, intrStaticReg, &intr0Pending, &threadState);
+
+                if (!bitVectorTestAllCleared(&intr0Pending))
+                {
+                    if ((pKernelDisplay != NULL) && pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE))
+                    {
+                        if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP_LOW))
+                        {
+                            isDispLowLatencyPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu));
+
+                            // We will attempt to handle this separately from the other stall interrupts
+                            bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP_LOW);
+                        }
+                    }
+                    else
+                    {
+                        if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))
+                        {
+                            isDispPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu));
+
+                            // We will attempt to handle this separately from the other stall interrupts
+                            bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP);
+                        }
+                    }
+
+                    if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) &&
+                        !IS_VIRTUAL(pGpu) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_TMR))
+                    {
+                        isTmrPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu));
+
+                        // We will clear this in the top half
+                        bitVectorClr(&intr0Pending, MC_ENGINE_IDX_TMR);
+                    }
+
+                    if (!bitVectorTestAllCleared(&intr0Pending))
+                    {
+                        //
+                        // If any other stall interrupts are pending,
+                        // they will be handled in the bottom half.
+                        //
+                        bIsAnyBottomHalfStallPending = NV_TRUE;
+                    }
+
+                    bIsAnyStallIntrPending = NV_TRUE;
+                }
+            }
+        }
+        threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+    }
+
+    // LOCK: try to acquire GPUs lock
+    if (bIsAnyStallIntrPending &&
+        (rmDeviceGpuLocksAcquire(pGpu, GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_ISR) == NV_OK))
+    {
+        threadStateInitISRAndDeferredIntHandler(&threadState,
+            pGpu, THREAD_STATE_FLAGS_IS_ISR);
+
+        {
+            pIntr = GPU_GET_INTR(pGpu);
+            pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+
+            if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+            {
+                nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+                nv_soc_irq_type_t irq_type;
+
+                irq_type = nv_get_current_irq_type(nv);
+
+                if (irq_type == NV_SOC_IRQ_DISPLAY_TYPE)
+                {
+                    NvU32 dispIntr0Pending = 0;
+
+                    if (kdispAcquireLowLatencyLockConditional(&pKernelDisplay->lowLatencyLock))
+                    {
+                        kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0,
+                                                        (VBLANK_STATE_PROCESS_LOW_LATENCY |
+                                                        VBLANK_STATE_PROCESS_CALLED_FROM_ISR),
+                                                        &threadState,
+                                                        NULL,
+                                                        NULL);
+
+                        kdispReleaseLowLatencyLock(&pKernelDisplay->lowLatencyLock);
+
+                        *serviced = NV_TRUE;
+                    }
+
+                    kdispGetPendingInterrupts_HAL(pGpu, pKernelDisplay, &dispIntr0Pending);
+
+                    if (dispIntr0Pending != 0)
+                    {
+                        pending = NV_TRUE;
+                        sema_release = NV_FALSE;
+                    }
+                }
+            }
+            else if ((pIntr != NULL) && INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))
+            {
+                // If interrupt enable is garbage the GPU is probably in a bad state
+                if (!(intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX))
+                {
+                    //
+                    // If lockless interrupt handling was enabled, we have already cached which critical
+                    // interrupts are pending. We only set the appropriate bits in the intr0Pending bitvector
+                    // for the subsequent code to service them in the same manner as the lockless nonstall
+                    // interrupt handling disabled case. But we also clear them from intr0Pending and
+                    // for disp, check if they're still pending afterwards. We already checked whether any
+                    // other bottom half stall interrupts are pending in bIsAnyBottomHalfStallPending above.
+                    //
+                    // After all this, the combination of bIsAnyBottomHalfStallPending and intr0Pending
+                    // contains whether any stall interrupts are still pending, so check both to determine if
+                    // we need a bottom half.
+                    //
+                    bitVectorClrAll(&intr0Pending);
+
+                    if ((pKernelDisplay != NULL) && pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE))
+                    {
+                        if ((isDispLowLatencyPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
+                        {
+                            bitVectorSet(&intr0Pending, MC_ENGINE_IDX_DISP_LOW);
+                        }
+                    }
+                    else
+                    {
+                        if ((isDispPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
+                        {
+                            bitVectorSet(&intr0Pending, MC_ENGINE_IDX_DISP);
+                        }
+
+                    }
+
+                    if ((isTmrPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
+                    {
+                        bitVectorSet(&intr0Pending, MC_ENGINE_IDX_TMR);
+                    }
+                }
+
+                if ((pKernelDisplay != NULL) &&
+                    (( pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP_LOW)) ||
+                     (!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))))
+                {
+                    MC_ENGINE_BITVECTOR intrDispPending;
+                    NvBool bIsLowLatencyIntrPending;
+
+                    // If this is true, then we're here because it is true
+                    bIsLowLatencyIntrPending = pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE);
+
+                    if (kdispAcquireLowLatencyLockConditional(&pKernelDisplay->lowLatencyLock))
+                    {
+                        if (pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_AGGRESSIVE_VBLANK_HANDLING))
+                        {
+                            // Need to queue a bottom half to service the interrupts since unix otherwise won't
+                            kdispHandleAggressiveVblank(pGpu, pKernelDisplay, pThreadState, &intr0Pending);
+                        }
+                        else
+                        {
+                            kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0,
+                                                            (VBLANK_STATE_PROCESS_LOW_LATENCY |
+                                                            VBLANK_STATE_PROCESS_CALLED_FROM_ISR),
+                                                            &threadState,
+                                                            NULL,
+                                                            &intr0Pending);
+
+                        }
+
+                        *serviced = NV_TRUE;
+
+                        kdispReleaseLowLatencyLock(&pKernelDisplay->lowLatencyLock);
+                    }
+
+                    if (bIsLowLatencyIntrPending)
+                    {
+                        bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP_LOW);
+                    }
+                    else
+                    {
+                        bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP);
+                    }
+
+                    intrGetPendingLowLatencyHwDisplayIntr_HAL(pGpu, pIntr, &intrDispPending, &threadState);
+
+                    //
+                    // If deferred vblank is used (depends on PDB_PROP_KDISP_AGGRESSIVE_VBLANK_HANDLING)
+                    // and pending, we need to queue a bottom half to service the rest of the interrupts,
+                    // so signal that an interrupt is still pending
+                    //
+                    if (kdispGetDeferredVblankHeadMask(pKernelDisplay) != 0)
+                    {
+                        bitVectorSet(&intrDispPending, MC_ENGINE_IDX_DISP);
+                    }            
+
+                    bitVectorOr(&intr0Pending, &intr0Pending, &intrDispPending);
+                }
+
+                if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) &&
+                    !IS_VIRTUAL(pGpu) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_TMR))
+                {
+                    // We have to clear the top level interrupt bit here since otherwise
+                    // the bottom half will attempt to service the interrupt on the CPU
+                    // side before GSP receives the notification and services it
+                    intrClearLeafVector_HAL(pGpu, pIntr, intrGetVectorFromEngineId(pGpu, pIntr, MC_ENGINE_IDX_TMR, NV_FALSE), &threadState);
+                    bitVectorClr(&intr0Pending, MC_ENGINE_IDX_TMR);
+
+                    NV_ASSERT_OK(intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_TMR_HANDLE));
+                }
+
+                {
+                    bitVectorClrAll(&intr1Pending);
+                }
+
+                if (bIsAnyBottomHalfStallPending ||
+                    !bitVectorTestAllCleared(&intr0Pending) ||
+                    !bitVectorTestAllCleared(&intr1Pending))
+                {
+                    pending = NV_TRUE;
+                    sema_release = NV_FALSE;
+                }
+            }
+        }
+
+        if (sema_release)
+        {
+            NV_ASSERT(!pending);
+
+            // UNLOCK: release GPUs lock
+            rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+        }
+        else
+        {
+            rmDeviceGpuLockSetOwner(pGpu, GPUS_LOCK_OWNER_PENDING_DPC_REFRESH);
+        }
+
+        threadStateFreeISRAndDeferredIntHandler(&threadState,
+                pGpu, THREAD_STATE_FLAGS_IS_ISR);
+    }
+    else if (bIsAnyStallIntrPending)
+    {
+        threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+
+        {
+            pIntr = GPU_GET_INTR(pGpu);
+
+            if ((pIntr != NULL) && (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING)))
+            {
+                if (_osIsrIntrMask_GpusUnlocked(pGpu, &threadState) == NV_OK)
+                {
+                    *serviced = NV_TRUE;
+                }
+            }
+        }
+
+        threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+    }
+
+    tlsIsrDestroy(pIsrAllocator);
+    portMemAllocatorRelease(pIsrAllocator);
+
+    return pending;
+}
+
+NV_STATUS osIsr(
+    OBJGPU *pGpu
+)
+{
+    NV_STATUS status = NV_OK;
+    nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    NvBool pending = NV_FALSE;
+    NvBool serviced = NV_FALSE;
+    Intr *pIntr;
+
+    if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
+    {
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+        {
+            pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */);
+        }
+        else
+        {
+            pIntr = GPU_GET_INTR(pGpu);
+            if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))
+            {
+                KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
+                pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */);
+                kbifCheckAndRearmMSI(pGpu, pKernelBif);
+            }
+        }
+    }
+
+    if (!pending && (IS_VIRTUAL(pGpu) || !serviced))
+        status = NV_ERR_NO_INTR_PENDING;
+    else if (pending)
+        status = NV_WARN_MORE_PROCESSING_REQUIRED;
+
+    return status;
+}
+
+/*
+ * Helper function to determine when the RM SEMA/GPUS LOCK should toggle
+ * interrupts.  Based on the state of the GPU - we must add cases here as we
+ * discover them.
+ *
+ * Noteworthy special cases:
+ *
+ *   - Suspend/resume: the GPU could still be suspended and not accessible
+ *     on the bus, while passive-level threads need to grab the GPUs
+ *     lock, or other GPUs are being resumed and triggering interrupts.
+ *
+ * @param[in] pGpu  OBJGPU pointer
+ *
+ * @return NV_TRUE if the RM SEMA/GPUS LOCK should toggle interrupts, NV_FALSE
+ *         otherwise.
+ */
+NvBool osLockShouldToggleInterrupts(OBJGPU *pGpu)
+{
+    if (IS_DCE_CLIENT(pGpu))
+    {
+        return NV_TRUE;
+    }
+
+    return (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH) &&
+             gpuIsStateLoaded(pGpu));
+}
+
+void osEnableInterrupts(OBJGPU *pGpu)
+{
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+    {
+        // enable irq through os call
+        nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_TRUE);
+        return;
+    }
+    else
+    {
+        Intr *pIntr = GPU_GET_INTR(pGpu);
+        NvU32 intrEn;
+
+        if (!pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING))
+            NV_ASSERT(intrGetIntrEnFromHw_HAL(pGpu, pIntr, NULL) == INTERRUPT_TYPE_DISABLED);
+
+        intrEn = intrGetIntrEn(pIntr);
+        intrSetIntrEnInHw_HAL(pGpu, pIntr, intrEn, NULL);
+        intrSetStall_HAL(pGpu, pIntr, intrEn, NULL);
+
+        intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL);
+    }
+}
+
+void osDisableInterrupts(
+    OBJGPU *pGpu,
+    NvBool  bIsr
+)
+{
+    if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+    {
+        // disable irq through os call
+        nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_FALSE);
+        return;
+    }
+    else
+    {
+        Intr *pIntr = GPU_GET_INTR(pGpu);
+        NvU32 new_intr_en_0 = INTERRUPT_TYPE_DISABLED;
+
+        intrSetIntrEnInHw_HAL(pGpu, pIntr, new_intr_en_0, NULL);
+        intrSetStall_HAL(pGpu, pIntr, new_intr_en_0, NULL);
+
+        intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL);
+    }
+}
+
+static void RmIsrBottomHalf(
+    nv_state_t *pNv
+)
+{
+    OBJGPU    *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+    OS_THREAD_HANDLE threadId;
+    Intr *pIntr = NULL;
+    NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer
+    PORT_MEM_ALLOCATOR *pIsrAllocator;
+
+    pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
+    tlsIsrInit(pIsrAllocator);
+
+    //
+    // The owning thread changes as the ISR acquires the GPUs lock,
+    // but the bottom half releases it.  Refresh the ThreadId owner to be
+    // correct here for the bottom half context.
+    //
+    osGetCurrentThread(&threadId);
+    rmDeviceGpuLockSetOwner(pGpu, threadId);
+
+    {
+        threadStateInitISRAndDeferredIntHandler(pGpu->pDpcThreadState, pGpu,
+                                                THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER);
+    }
+
+    {
+        nv_state_t *nv = NV_GET_NV_STATE(pGpu);
+
+        KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
+        gpumgrSetCurrentGpuInstance(pGpu->gpuInstance);
+
+        pIntr = GPU_GET_INTR(pGpu);
+
+        //
+        // Call disp service incase of SOC Display,
+        // TODO : with multi interrupt handling based on irq aux interrupts are serviced by dpAuxService
+        // See JIRA task TDS-4253.
+        //
+        if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY))
+        {
+            if (nv_get_current_irq_type(nv) == NV_SOC_IRQ_DISPLAY_TYPE)
+            {
+                if (IS_DCE_CLIENT(pGpu))
+                {
+                    kdispAcquireLowLatencyLock(&pKernelDisplay->lowLatencyLock);
+
+                    kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0,
+                                                    VBLANK_STATE_PROCESS_LOW_LATENCY,
+                                                    pGpu->pDpcThreadState,
+                                                    NULL,
+                                                    NULL);
+
+                    kdispReleaseLowLatencyLock(&pKernelDisplay->lowLatencyLock);
+                }
+                else
+                {
+                }
+            }
+        }
+        else if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)))
+        {
+            intrServiceStall_HAL(pGpu, pIntr);
+        }
+
+        gpumgrSetCurrentGpuInstance(NV_U32_MAX);
+    }
+
+    // UNLOCK: release GPUs lock
+    rmDeviceGpuLocksReleaseAndThreadStateFreeDeferredIntHandlerOptimized(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+
+    tlsIsrDestroy(pIsrAllocator);
+    portMemAllocatorRelease(pIsrAllocator);
+}
+
+static void RmIsrBottomHalfUnlocked(
+    nv_state_t *pNv
+)
+{
+    OBJGPU  *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
+    Intr *pIntr;
+    THREAD_STATE_NODE threadState;
+
+    // In the GSP client scenario, the fatal fault interrupt is not shared
+    // by UVM and CPU-RM.  Instead, it is handled entirely by GSP-RM.  We
+    // therefore do not expect this function to be called.  But if it is, bail
+    // without attempting to service interrupts.
+    if (IS_GSP_CLIENT(pGpu))
+    {
+        return;
+    }
+
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    // Grab GPU lock here as this kthread-item was enqueued without grabbing GPU lock
+    if (rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DPC) == NV_OK)
+    {
+        if (FULL_GPU_SANITY_CHECK(pGpu))
+        {
+            pIntr = GPU_GET_INTR(pGpu);
+
+            if (intrGetIntrEn(pIntr) != INTERRUPT_TYPE_DISABLED)
+            {
+                MC_ENGINE_BITVECTOR intrPending;
+                intrGetPendingStall_HAL(pGpu, pIntr, &intrPending, &threadState);
+                intrServiceNonStallBottomHalf(pGpu, pIntr, &intrPending, &threadState);
+            }
+        }
+
+        rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+}
+
+NvBool NV_API_CALL rm_isr(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    NvU32      *NeedBottomHalf
+)
+{
+    NV_STATUS status;
+    nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
+    OBJGPU *pGpu;
+    NvBool retval;
+    void *fp;
+
+    if ((nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) == 0)
+    {
+        return NV_FALSE;
+    }
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    if (pGpu == NULL)
+    {
+        return NV_FALSE;
+    }
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    // call actual isr function here
+    status = isrWrapper(pGpu->testIntr, pGpu);
+
+    switch (status)
+    {
+        case NV_OK:
+            *NeedBottomHalf = NV_FALSE;
+            retval = NV_TRUE;
+            break;
+        case NV_WARN_MORE_PROCESSING_REQUIRED:
+            *NeedBottomHalf = NV_TRUE;
+            retval = NV_TRUE;
+            break;
+        case NV_ERR_NO_INTR_PENDING:
+        default:
+            *NeedBottomHalf = NV_FALSE;
+            retval = NV_FALSE;
+            break;
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return retval;
+}
+
+void NV_API_CALL rm_isr_bh(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    void    *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    RmIsrBottomHalf(pNv);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+void NV_API_CALL rm_isr_bh_unlocked(
+    nvidia_stack_t *sp,
+    nv_state_t *pNv
+)
+{
+    void    *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    RmIsrBottomHalfUnlocked(pNv);
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+}
+
+NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    NvU32 *faultsCopied
+)
+{
+    NV_STATUS status = NV_OK;
+    OBJGPU *pGpu;
+    void   *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    if (pGpu == NULL || faultsCopied == NULL)
+    {
+        status = NV_ERR_OBJECT_NOT_FOUND;
+        goto done;
+    }
+
+    if (IS_GSP_CLIENT(pGpu))
+    {
+        // Non-replayable faults are copied to the client shadow buffer by GSP-RM.
+        status = NV_OK;
+        goto done;
+    }
+    else
+    {
+        KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
+        PORT_MEM_ALLOCATOR *pIsrAllocator;
+        THREAD_STATE_NODE   threadState;
+        NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer
+
+        if (pKernelGmmu == NULL)
+        {
+            status = NV_ERR_OBJECT_NOT_FOUND;
+            goto done;
+        }
+
+        // If MMU fault buffer is not enabled, return early
+        if (!gpuIsVoltaHubIntrSupported(pGpu))
+            goto done;
+
+        pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
+        tlsIsrInit(pIsrAllocator);
+        threadStateInitISRAndDeferredIntHandler(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR);
+
+        // Copies all valid packets in RM's and client's shadow buffer
+        status  = kgmmuCopyMmuFaults_HAL(pGpu, pKernelGmmu, &threadState, faultsCopied,
+                                         NON_REPLAYABLE_FAULT_BUFFER, NV_FALSE);
+
+        threadStateFreeISRAndDeferredIntHandler(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR);
+        tlsIsrDestroy(pIsrAllocator);
+        portMemAllocatorRelease(pIsrAllocator);
+
+    }
+
+done:
+    NV_EXIT_RM_RUNTIME(sp,fp);
+
+    return status;
+}
+
+//
+// Use this call when MMU faults needs to be copied
+// outisde of RM lock.
+//
+static NV_STATUS _rm_gpu_copy_mmu_faults_unlocked(
+    OBJGPU *pGpu,
+    NvU32 *pFaultsCopied,
+    THREAD_STATE_NODE *pThreadState
+)
+{
+    KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
+
+    if (pKernelGmmu == NULL)
+    {
+        return NV_ERR_OBJECT_NOT_FOUND;
+    }
+
+    // If MMU fault buffer is not enabled, return early
+    if (!gpuIsVoltaHubIntrSupported(pGpu))
+        return NV_OK;
+
+    // Copies all valid packets in RM's and client's shadow buffer
+    return kgmmuCopyMmuFaults_HAL(pGpu, pKernelGmmu, pThreadState, pFaultsCopied,
+                                  NON_REPLAYABLE_FAULT_BUFFER, NV_FALSE);
+
+    return NV_OK;
+}
+
+//
+// Wrapper to handle calls to copy mmu faults
+//
+NV_STATUS rm_gpu_handle_mmu_faults(
+    nvidia_stack_t *sp,
+    nv_state_t *nv,
+    NvU32 *faultsCopied
+)
+{
+    NvU32 status = NV_OK;
+    OBJGPU *pGpu;
+    void *fp;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+
+    *faultsCopied = 0;
+    pGpu = NV_GET_NV_PRIV_PGPU(nv);
+    
+    if (pGpu == NULL)
+    {
+        NV_EXIT_RM_RUNTIME(sp,fp);
+        return NV_ERR_OBJECT_NOT_FOUND;
+    }
+
+    {
+        KernelGmmu *pKernelGmmu;
+        PORT_MEM_ALLOCATOR *pIsrAllocator;
+        THREAD_STATE_NODE threadState;
+        NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer
+
+        pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
+        tlsIsrInit(pIsrAllocator);
+        threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+
+        pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
+
+        if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && !IS_VIRTUAL(pGpu))
+        {
+            Intr *pIntr = GPU_GET_INTR(pGpu);
+
+            if (kgmmuIsNonReplayableFaultPending_HAL(pGpu, pKernelGmmu, &threadState))
+            {
+                // We have to clear the top level interrupt bit here since otherwise
+                // the bottom half will attempt to service the interrupt on the CPU
+                // side before GSP receives the notification and services it
+                kgmmuClearNonReplayableFaultIntr_HAL(pGpu, pKernelGmmu, &threadState);
+                status = intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_NON_REPLAYABLE_FAULT_HANDLE);
+
+            }
+        }
+        else if (IS_VIRTUAL_WITH_SRIOV(pGpu))
+        {
+            if (kgmmuIsNonReplayableFaultPending_HAL(pGpu, pKernelGmmu, &threadState))
+            {
+                status = _rm_gpu_copy_mmu_faults_unlocked(pGpu, faultsCopied, &threadState);
+            }
+        }
+        else
+        {
+            if (IS_GSP_CLIENT(pGpu))
+            {
+                // Non-replayable faults are copied to the client shadow buffer by GSP-RM.
+                status = NV_OK;
+            }
+            else
+            {
+                status = _rm_gpu_copy_mmu_faults_unlocked(pGpu, faultsCopied, &threadState);
+            }
+        }
+
+        threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
+        tlsIsrDestroy(pIsrAllocator);
+        portMemAllocatorRelease(pIsrAllocator);
+    }
+
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return status;
+}
+
+NvBool NV_API_CALL rm_is_msix_allowed(
+    nvidia_stack_t *sp,
+    nv_state_t     *nv
+)
+{
+    nv_priv_t           *pNvp = NV_GET_NV_PRIV(nv);
+    THREAD_STATE_NODE    threadState;
+    void                *fp;
+    NvBool               ret = NV_FALSE;
+
+    NV_ENTER_RM_RUNTIME(sp,fp);
+    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
+
+    if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK)
+    {
+        ret = gpumgrIsDeviceMsixAllowed(nv->regs->cpu_address,
+                                        pNvp->pmc_boot_1, pNvp->pmc_boot_42);
+        rmapiLockRelease();
+    }
+
+    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
+    NV_EXIT_RM_RUNTIME(sp,fp);
+    return ret;
+}
diff --git a/userspace/sdks/nv570/nvalloc/unix/src/vbioscall.c b/userspace/sdks/nv570/nvalloc/unix/src/vbioscall.c
new file mode 100644
index 0000000..8eeb85e
--- /dev/null
+++ b/userspace/sdks/nv570/nvalloc/unix/src/vbioscall.c
@@ -0,0 +1,457 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+/******************************************************************************
+*
+*   Description:
+*       This file provides the glue layer between the RM's OS object
+*       unixCallVideoBIOS() method and the x86 real mode emulator
+*
+******************************************************************************/
+
+
+#include <core/core.h>
+#include <nv.h>
+#include <nv-priv.h>
+#include <osapi.h>
+
+#include <os/os.h>
+
+#include <x86emu/x86emu.h>
+
+#define IO_LOG(port, val)
+
+#define NV_ROMLEY_VGA_PHYS_ADDR                 0x9d000
+#define NV_VGA_PHYS_ADDR                        0xa0000
+#define NV_VIDEO_ROM_PHYS_ADDR                  0xc0000
+#define NV_EXPANSION_VIDEO_ROM_PHYS_ADDR        0xc8000
+#define NV_SYSTEM_ROM_PHYS_ADDR                 0xf0000
+
+#define X86EMU_LOWMEM       0x600   // Interrupt vectors, BIOS data size
+#define X86EMU_STACK_ADDR   0x10000
+#define X86EMU_STACK_SIZE   0x1000
+#define X86EMU_BUF_ADDR     0x20000 // EDID buffer for VESA int10 0x4f15
+#define X86EMU_BUF_SIZE     128
+
+#define OP_HLT              0xf4    // HLT instruction - causes the emulator to return
+
+// This is where IRET from int10 will take us
+#define X86EMU_IRET_SEG     (X86EMU_STACK_ADDR >> 4)
+#define X86EMU_IRET_OFF     0
+
+#define X86EMU_DFLT_FLAGS   (F_IF)
+
+static NvBool    x86emuReady;
+static void     *x86emuStack;
+
+static struct x86emu_mem_seg {
+    NvU32           start;
+    NvU32           end;
+    void            *vaddr;
+}   x86emu_mem_map[] = {
+    {NV_ROMLEY_VGA_PHYS_ADDR, NV_VGA_PHYS_ADDR - 1},
+    {NV_VGA_PHYS_ADDR, NV_VIDEO_ROM_PHYS_ADDR - 1},                       // vga
+    {NV_VIDEO_ROM_PHYS_ADDR, NV_EXPANSION_VIDEO_ROM_PHYS_ADDR -1},        // vbios, sbios
+    {NV_EXPANSION_VIDEO_ROM_PHYS_ADDR, NV_SYSTEM_ROM_PHYS_ADDR -1},       // vbios, sbios
+    {NV_SYSTEM_ROM_PHYS_ADDR, 0xFFFFF},                                   // Motherboard BIOS
+    {X86EMU_STACK_ADDR, X86EMU_STACK_ADDR + X86EMU_STACK_SIZE - 1},       // emulator stack
+    {0,                 X86EMU_LOWMEM - 1},                               // Interrupt vectors, BIOS data
+    {X86EMU_BUF_ADDR,   X86EMU_BUF_ADDR + X86EMU_BUF_SIZE - 1},
+};
+
+enum {
+    X86EMU_SEG_ROMLEY_VGA = 0,
+    X86EMU_SEG_VGA,
+    X86EMU_SEG_ROM,
+    X86EMU_SEG_ROM_EXPANSION,
+    X86EMU_SEG_ROM_MOTHERBOARD_BIOS,
+    X86EMU_SEG_EMULATOR_STACK,
+    X86EMU_SEG_BIOS_DATA,
+    X86EMU_SEG_EDID_BUFFER,
+    X86EMU_NUM_SEGS
+};
+
+ct_assert(X86EMU_NUM_SEGS == NV_ARRAY_ELEMENTS(x86emu_mem_map));
+
+static NvU8
+x_inb(NvU16 port)
+{
+    NvU8 val;
+
+    val = os_io_read_byte(port);
+    IO_LOG(port, val);
+    return val;
+}
+
+static void
+x_outb(NvU16 port, NvU8 val)
+{
+    IO_LOG(port, val);
+    os_io_write_byte(port, val);
+}
+
+static NvU16
+x_inw(NvU16 port)
+{
+    NvU16 val;
+
+    val = os_io_read_word(port);
+    IO_LOG(port, val);
+    return val;
+}
+
+static void
+x_outw(NvU16 port, NvU16 val)
+{
+    IO_LOG(port, val);
+    os_io_write_word(port, val);
+}
+
+static u32
+x_inl(X86EMU_pioAddr port)
+{
+    NvU32 val;
+
+    val = os_io_read_dword(port);
+    IO_LOG(port, val);
+    return val;
+}
+
+static void
+x_outl(X86EMU_pioAddr port, u32 val)
+{
+    IO_LOG(port, val);
+    os_io_write_dword(port, val);
+}
+
+static void *
+Mem_addr_xlat(NvU32 addr, NvU32 sz)
+{
+    int                     i;
+    struct x86emu_mem_seg   *pseg;
+
+    for (i = 0; i < X86EMU_NUM_SEGS; ++i)
+    {
+        pseg = x86emu_mem_map + i;
+        if (pseg->vaddr != 0 && addr >= pseg->start && addr + sz - 1 <= pseg->end)
+            return (void *)((char *)pseg->vaddr + addr - pseg->start);
+    }
+
+    X86EMU_halt_sys();
+
+    return 0;
+}
+
+static NvU8
+Mem_rb(u32 addr)
+{
+    NvU8    *va = Mem_addr_xlat(addr, 1);
+
+    return va != 0 ? *va : 0;
+}
+
+static NvU16
+Mem_rw(u32 addr)
+{
+    NvU16   *va = Mem_addr_xlat(addr, 2);
+
+    return va != 0 ? *va : 0;
+}
+
+static u32
+Mem_rl(u32 addr)
+{
+    NvU32   *va = Mem_addr_xlat(addr, 4);
+
+    return va != 0 ? *va : 0;
+}
+
+static void
+Mem_wb(u32 addr, NvU8 val)
+{
+    NvU8    *va = Mem_addr_xlat(addr, 1);
+
+    if (va != 0)
+        *va = val;
+}
+
+static void
+Mem_ww(u32 addr, NvU16 val)
+{
+    NvU16   *va = Mem_addr_xlat(addr, 2);
+
+    if (va != 0)
+        *va = val;
+}
+
+static void
+Mem_wl(u32 addr, u32 val)
+{
+    NvU32   *va = Mem_addr_xlat(addr, 4);
+
+    if (va != 0)
+        *va = val;
+}
+
+static NvU16
+get_int_seg(int i)
+{
+    return Mem_rw(i * 4 + 2);
+}
+
+
+static NvU16
+get_int_off(int i)
+{
+    return Mem_rw(i * 4);
+}
+
+
+static void
+pushw(NvU16 i)
+{
+        M.x86.R_ESP -= 2;
+        Mem_ww((M.x86.R_SS << 4) + M.x86.R_ESP, i);
+}
+
+static void
+x86emu_do_int(int num)
+{
+#if 0
+    Int10Current->num = num;
+
+    if (!int_handler(Int10Current)) {
+        X86EMU_halt_sys();
+    }
+#else
+    if ((num == 0x15) && (M.x86.R_AX == 0x5f80))
+    {
+        //
+        // Handle the MXM_SYS_INFO_CALLBACK_NUM int 15h SBIOS
+        // callback: disclaim support by returning a value
+        // other than 005fh (MXM_SYS_INFO_CALLBACK_FUNC_SUPPORTED)
+        // to the caller.
+        //
+        M.x86.R_AX = 0;
+    }
+    else
+    {
+        NV_PRINTF(LEVEL_ERROR, "x86emu: int $%d (eax = %08x)\n",
+                  num, M.x86.R_EAX);
+        DBG_BREAKPOINT();
+        X86EMU_halt_sys();
+    }
+#endif
+}
+
+NV_STATUS
+RmInitX86EmuState(OBJGPU *pGpu) {
+    int                     i;
+    struct x86emu_mem_seg   *pseg;
+    X86EMU_intrFuncs        *intFuncs;
+    X86EMU_pioFuncs         pioFuncs = {
+        (&x_inb),
+        (&x_inw),
+        (&x_inl),
+        (&x_outb),
+        (&x_outw),
+        (&x_outl)
+    };
+
+    X86EMU_memFuncs         memFuncs = {
+        (&Mem_rb),
+        (&Mem_rw),
+        (&Mem_rl),
+        (&Mem_wb),
+        (&Mem_ww),
+        (&Mem_wl)
+    };
+
+    if (!NV_PRIMARY_VGA(NV_GET_NV_STATE(pGpu)))     // not the primary GPU
+        return NV_OK;
+
+    NV_ASSERT(!x86emuReady);
+
+    x86emuStack = portMemAllocNonPaged(
+                      X86EMU_STACK_SIZE + sizeof(X86EMU_intrFuncs*) * 256);
+    if (x86emuStack == NULL)
+        return NV_ERR_NO_MEMORY;
+
+    // Interrupt dispatch table
+    intFuncs = (void *)((NvU8 *)x86emuStack + X86EMU_STACK_SIZE);
+
+    // Fill virtual addresses in the memory map
+    for (i = 0; i < X86EMU_NUM_SEGS; ++i)
+    {
+        pseg = x86emu_mem_map + i;
+        switch (i)
+        {
+            case X86EMU_SEG_ROM:
+            case X86EMU_SEG_ROM_EXPANSION:
+            case X86EMU_SEG_ROM_MOTHERBOARD_BIOS:
+                nv_get_updated_emu_seg(&pseg->start, &pseg->end);
+                /* fallthrough */
+            case X86EMU_SEG_VGA:
+            case X86EMU_SEG_BIOS_DATA:
+                pseg->vaddr = os_map_kernel_space(pseg->start,
+                                                  pseg->end - pseg->start + 1,
+                                                  NV_MEMORY_CACHED);
+                if (pseg->vaddr == 0)
+                {
+                    NV_PRINTF(LEVEL_ERROR, "x86emu can't map phys addr 0x%05x\n",
+                              pseg->start);
+                    return NV_ERR_GENERIC;
+                }
+                break;
+            case X86EMU_SEG_EMULATOR_STACK:
+                pseg->vaddr = x86emuStack;
+                break;
+            default:
+                pseg->vaddr = 0;
+                break;
+        }
+    }
+    X86EMU_setupMemFuncs(&memFuncs);
+
+    M.mem_base = 0;
+    M.mem_size = 1024*1024;
+    X86EMU_setupPioFuncs(&pioFuncs);
+
+    for (i=0;i<256;i++)
+        intFuncs[i] = x86emu_do_int;
+    X86EMU_setupIntrFuncs(intFuncs);
+
+    x86emuReady = NV_TRUE;
+
+    return NV_OK;
+}
+
+void
+RmFreeX86EmuState(OBJGPU *pGpu)
+{
+    int                     i;
+    struct x86emu_mem_seg   *pseg;
+
+    if (! x86emuReady || !NV_PRIMARY_VGA(NV_GET_NV_STATE(pGpu)))     // not the primary GPU
+        return;
+
+    portMemFree(x86emuStack);
+    x86emuStack = 0;
+
+    for (i = 0; i < X86EMU_NUM_SEGS; ++i)
+    {
+        pseg = x86emu_mem_map + i;
+        switch (i)
+        {
+            case X86EMU_SEG_ROMLEY_VGA:
+            case X86EMU_SEG_VGA:
+            case X86EMU_SEG_ROM:
+            case X86EMU_SEG_ROM_EXPANSION:
+            case X86EMU_SEG_ROM_MOTHERBOARD_BIOS:
+            case X86EMU_SEG_BIOS_DATA:
+                if (pseg->vaddr != 0)
+                    os_unmap_kernel_space(pseg->vaddr, pseg->end - pseg->start + 1);
+        }
+        pseg->vaddr = 0;
+    }
+
+    x86emuReady = NV_FALSE;
+}
+
+NV_STATUS
+nv_vbios_call(
+    OBJGPU *pGpu,
+    NvU32 *eax,
+    NvU32 *ebx)
+{
+    NvU16                   seg;
+    NvU16                   off;
+    struct x86emu_mem_seg   *pseg;
+
+    if (!NV_PRIMARY_VGA(NV_GET_NV_STATE(pGpu)))
+        return NV_ERR_GENERIC;
+
+    seg = get_int_seg(0x10);
+    off = get_int_off(0x10);
+
+    pseg = &x86emu_mem_map[X86EMU_SEG_VGA];
+    if (seg < (pseg->start >> 4))
+    {
+        pseg = &x86emu_mem_map[X86EMU_SEG_ROMLEY_VGA];
+        if (seg < (pseg->start >> 4))
+        {
+            NV_PRINTF(LEVEL_ERROR,
+                      "cannot call the VBIOS. INT10 vector not in ROM: %04x:%04x\n",
+                      seg, off);
+            return NV_ERR_GENERIC;
+        }
+        else if (pseg->vaddr == NULL)
+        {
+            pseg->vaddr = os_map_kernel_space(pseg->start,
+                    (pseg->end - pseg->start + 1),
+                     NV_MEMORY_CACHED);
+            if (pseg->vaddr == NULL)
+            {
+                NV_PRINTF(LEVEL_ERROR, "x86emu can't map phys addr 0x%05x\n",
+                          pseg->start);
+                return NV_ERR_GENERIC;
+            }
+        }
+    }
+
+    // Reset the CPU
+    portMemSet(&M, 0, sizeof(M));
+
+    M.x86.R_SS = X86EMU_STACK_ADDR >> 4;
+    M.x86.R_ESP = X86EMU_STACK_SIZE;
+    M.x86.R_CS = seg;
+    M.x86.R_EIP = off;
+    M.x86.R_EFLG = X86EMU_DFLT_FLAGS;
+    M.x86.R_EAX = *eax;
+    M.x86.R_EBX = *ebx;
+    M.x86.R_ECX = 0;
+    M.x86.R_EDX = 0;
+    M.x86.R_ES = X86EMU_BUF_ADDR >> 4;
+
+    X86EMU_trace_on();
+
+    // Map ES:EDI to buffer. Used by vesa intr 4f15 - read EDID
+    pseg = &x86emu_mem_map[X86EMU_SEG_EDID_BUFFER];
+    pseg->vaddr = NULL;
+
+    // Prepare the code for IRET to jump to
+    Mem_wb((X86EMU_IRET_SEG << 4) + X86EMU_IRET_OFF, OP_HLT);
+
+    pushw(X86EMU_DFLT_FLAGS);
+    pushw(X86EMU_IRET_SEG);
+    pushw(X86EMU_IRET_OFF);
+
+    // Run the emulator
+    X86EMU_exec();
+
+    *eax = M.x86.R_EAX;
+    *ebx = M.x86.R_EBX;
+
+    return NV_OK;
+}