/* * SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: MIT * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ /************************************************************************************************************** * * Description: * UNIX-general, device-independent initialization code for * the resource manager. * * **************************************************************************************************************/ #include #include // NV device driver interface #include #include #include #include #include #include #include #include #include #include "gpu/gpu.h" #include #include "gpu/bus/kern_bus.h" #include "nverror.h" #include #include #include #include #include "kernel/gpu/intr/intr.h" #include #include #include #include #include #include #include #include "liblogdecode.h" #include #include #include #include #include #include #include #include #include #include #include "platform/nbsi/nbsi_read.h" #include "gpu_mgr/gpu_db.h" #include #include #include #include #include #include // RMCONFIG: need definition of REGISTER_ALL_HALS() #include "g_hal_register.h" typedef enum { RM_INIT_OK, /* general os errors */ RM_INIT_REG_SETUP_FAILED = 0x10, RM_INIT_SYS_ENVIRONMENT_FAILED, /* gpu errors */ RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED = 0x20, RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED, RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED, RM_INIT_GPU_PRE_INIT_FAILED, RM_INIT_GPU_STATE_INIT_FAILED, RM_INIT_GPU_LOAD_FAILED, RM_INIT_GPU_DMA_CONFIGURATION_FAILED, RM_INIT_GPU_GPUMGR_EXPANDED_VISIBILITY_FAILED, /* vbios errors */ RM_INIT_VBIOS_FAILED = 0x30, RM_INIT_VBIOS_POST_FAILED, RM_INIT_VBIOS_X86EMU_FAILED, /* scalability errors */ RM_INIT_SCALABILITY_FAILED = 0x40, /* general core rm errors */ RM_INIT_WATCHDOG_FAILED, RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED, RM_GPUDB_REGISTER_FAILED, RM_INIT_ALLOC_RMAPI_FAILED, RM_INIT_GPUINFO_WITH_RMAPI_FAILED, /* rm firmware errors */ RM_INIT_FIRMWARE_POLICY_FAILED = 0x60, RM_INIT_FIRMWARE_FETCH_FAILED, RM_INIT_FIRMWARE_INIT_FAILED, RM_INIT_MAX_FAILURES } rm_init_status; typedef rm_init_status RM_INIT_STATUS; typedef struct { RM_INIT_STATUS initStatus; NV_STATUS rmStatus; NvU32 line; } UNIX_STATUS; #define INIT_UNIX_STATUS { RM_INIT_OK, NV_OK, 0 } #define RM_INIT_SUCCESS(init) ((init) == RM_INIT_OK) #define RM_SET_ERROR(status, err) { (status).initStatus = (err); \ (status).line = __LINE__; } static inline NvU64 nv_encode_pci_info(nv_pci_info_t *pci_info) { return gpuEncodeDomainBusDevice(pci_info->domain, pci_info->bus, pci_info->slot); } static inline NvU32 nv_generate_id_from_pci_info(nv_pci_info_t *pci_info) { return gpuGenerate32BitId(pci_info->domain, pci_info->bus, pci_info->slot); } static void nv_set_probed_gpu_flags(nv_state_t *nv) { NvU32 flags = 0; if (NV_IS_SOC_DISPLAY_DEVICE(nv)) { flags |= DRF_DEF(0000, _CTRL_GPU_PROBED_ID_FLAGS, _SOC_DISPLAY, _TRUE); } gpumgrSetProbedFlags(nv->gpu_id, flags); } static inline void nv_os_map_kernel_space(nv_state_t *nv, nv_aperture_t *aperture) { NV_ASSERT(aperture->map == NULL); // let's start off assuming a standard device and map the registers // normally. It is unfortunate to hard-code the register size here, but we don't // want to fail trying to map all of a multi-devices' register space aperture->map = osMapKernelSpace(aperture->cpu_address, aperture->size, NV_MEMORY_UNCACHED, NV_PROTECT_READ_WRITE); aperture->map_u = (nv_phwreg_t)aperture->map; } // local prototypes static NV_STATUS initCoreLogic(OBJGPU *); static void initVendorSpecificRegistry(OBJGPU *, NvU16); static NV_STATUS teardownCoreLogic(OBJOS *, OBJGPU *); static void initUnixSpecificRegistry(OBJGPU *); NV_STATUS osRmInitRm(void) { OBJSYS *pSys = SYS_GET_INSTANCE(); NV_STATUS status; if (os_is_efi_enabled()) { pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_TRUE); } // have to init this before the debug subsystem, which will // try to check the value of ResmanDebugLevel RmInitRegistry(); NvU32 data = 0; if ((osReadRegistryDword(NULL, NV_REG_ENABLE_PCIE_GEN3, &data) == NV_OK) && (data != 0)) { data = DRF_DEF(_REG_STR, _RM_PCIE, _LINK_SPEED_ALLOW_GEN3, _ENABLE); osWriteRegistryDword(NULL, NV_REG_STR_RM_PCIE_LINK_SPEED, data); } if (!(osReadRegistryDword(NULL, NV_REG_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, &data) == NV_OK) || data) { pSys->setProperty(pSys, PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, NV_TRUE); } if ((osReadRegistryDword(NULL, NV_REG_ENABLE_STREAM_MEMOPS, &data) == NV_OK) && (data != 0)) { data = DRF_DEF(_REG_STR, _RM_STREAM_MEMOPS, _ENABLE, _YES); osWriteRegistryDword(NULL, NV_REG_STR_RM_STREAM_MEMOPS, data); } if ((osReadRegistryDword(NULL, NV_REG_STR_RM_FORCE_BAR_PATH, &data) == NV_OK) && (data == 1)) { // // Exposing the GPU memory as NUMA node memory requires coherent CPU // mappings over NVLink. If those mappings are disabled, also disable // NUMA-onlining of the device memory. // osWriteRegistryDword(NULL, NV_REG_ENABLE_USER_NUMA_MANAGEMENT, 0); } if ((osReadRegistryDword(NULL, NV_REG_NVLINK_DISABLE, &data) == NV_OK) && (data == 1)) { osWriteRegistryDword(NULL, NV_REG_STR_RM_NVLINK_CONTROL, 0x1); } if ((osReadRegistryDword(NULL, NV_REG_ENABLE_PCIE_RELAXED_ORDERING_MODE, &data) == NV_OK) && (data == 1)) { osWriteRegistryDword(NULL, NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING, NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING_ENABLE); } // init the debug subsystem if necessary os_dbg_init(); nvDbgInitRmMsg(NULL); // Force nvlog reinit since module params are now available NVLOG_UPDATE(); // Register all supported hals status = REGISTER_ALL_HALS(); if (status != NV_OK) { RmDestroyRegistry(NULL); return status; } rm_check_s0ix_regkey_and_platform_support(); // Setup any ThreadState defaults threadStateInitSetupFlags(THREAD_STATE_SETUP_FLAGS_ENABLED | THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED | THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED | THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED); return NV_OK; } void RmShutdownRm(void) { NV_PRINTF(LEVEL_INFO, "shutdown rm\n"); RmDestroyRegistry(NULL); // Free objects created with RmInitRm, including the system object RmDestroyRm(); } // // osAttachGpu // // This routine is used as a callback by the gpumgrAttachGpu // interface to allow os-dependent code to set up any state // before engine construction begins. // NV_STATUS osAttachGpu( OBJGPU *pGpu, void *pOsGpuInfo ) { nv_state_t *nv = (nv_state_t *)pOsGpuInfo; nv_priv_t *nvp; nvp = NV_GET_NV_PRIV(nv); nvp->pGpu = pGpu; NV_SET_NV_STATE(pGpu, (void *)nv); initUnixSpecificRegistry(pGpu); // Assign default values to Registry keys for VGX if (os_is_vgx_hyper()) { initVGXSpecificRegistry(pGpu); } return NV_OK; } NV_STATUS osDpcAttachGpu( OBJGPU *pGpu, void *pOsGpuInfo ) { return NV_OK; // Nothing to do for unix } void osDpcDetachGpu( OBJGPU *pGpu ) { return; // Nothing to do for unix } NV_STATUS osHandleGpuLost ( OBJGPU *pGpu, NvBool bEmitXid ) { nv_state_t *nv = NV_GET_NV_STATE(pGpu); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NvU32 pmc_boot_0; // Determine if we've already run the handler if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED)) { return NV_OK; } pmc_boot_0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0); if (pmc_boot_0 != nvp->pmc_boot_0) { // // This doesn't support PEX Reset and Recovery yet. // This will help to prevent accessing registers of a GPU // which has fallen off the bus. // if (bEmitXid) { nvErrorLog_va((void *)pGpu, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, "GPU has fallen off the bus."); } gpuNotifySubDeviceEvent(pGpu, NV2080_NOTIFIERS_GPU_UNAVAILABLE, NULL, 0, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, 0); NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU has fallen off the bus.\n"); if (pGpu->boardInfo != NULL && pGpu->boardInfo->serialNumber[0] != '\0') { NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU serial number is %s.\n", pGpu->boardInfo->serialNumber); } gpuSetDisconnectedProperties(pGpu); if (IS_GSP_CLIENT(pGpu)) { KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); // Notify all channels of the error so that UVM can fail gracefully if (pKernelRc != NULL) { krcRcAndNotifyAllChannels(pGpu, pKernelRc, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, NV_FALSE); } } // Initiate a crash dump immediately. RmLogGpuCrash(pGpu); // Set SURPRISE_REMOVAL flag for eGPU to help in device removal. if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_EXTERNAL_GPU)) { nv->flags |= NV_FLAG_IN_SURPRISE_REMOVAL; } DBG_BREAKPOINT(); } return NV_OK; } /*! * @brief Traverse bus topology till Gpu's root port. * If any of the intermediate bridge has TB3 supported vendorId and hotplug * capability(not necessarily same bridge), mark the Gpu as External Gpu. * * @params[in] pGpu OBJGPU pointer * @params[in] pCl OBJCL pointer * * @return NV_OK * Identified to be eGPU * @return others * Not an eGPU / error on identfying * */ NvBool RmCheckForExternalGpu ( OBJGPU *pGpu, OBJCL *pCl ) { NvU8 bus; NvU32 domain; void *handleUp; NvU8 busUp, devUp, funcUp; NvU16 vendorIdUp, deviceIdUp; NvU32 portCaps, pciCaps, slotCaps; NvU32 PCIECapPtr; RM_API *pRmApi; NV_STATUS status, rmStatus; NvBool bTb3Bridge = NV_FALSE, bSlotHotPlugSupport = NV_FALSE; NvBool iseGPUBridge = NV_FALSE; pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); domain = gpuGetDomain(pGpu); bus = gpuGetBus(pGpu); do { // Find the upstream bridge handleUp = clFindP2PBrdg(pCl, domain, bus, &busUp, &devUp, &funcUp, &vendorIdUp, &deviceIdUp); if (!handleUp) { return iseGPUBridge; } if (vendorIdUp == PCI_VENDOR_ID_INTEL) { // Check for the supported TB3(ThunderBolt 3) bridges. NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS params = { 0 }; // LOCK: acquire GPUs lock rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); if (rmStatus != NV_OK) { return iseGPUBridge; } params.pciDeviceId = deviceIdUp; status = pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice, NV2080_CTRL_CMD_INTERNAL_GET_EGPU_BRIDGE_INFO, ¶ms, sizeof(params)); // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Error 0x%08x on eGPU Approval for Bridge ID: 0x%08x\n", status, deviceIdUp); DBG_BREAKPOINT(); return iseGPUBridge; } else { // Check for the approved eGPU BUS TB3 if (params.iseGPUBridge && params.approvedBusType == NV2080_CTRL_INTERNAL_EGPU_BUS_TYPE_TB3) { bTb3Bridge = NV_TRUE; } } } if (NV_OK != clSetPortPcieCapOffset(pCl, handleUp, &PCIECapPtr)) { // PCIE bridge but no cap pointer. break; } // Get the PCIE capabilities. pciCaps = osPciReadDword(handleUp, CL_PCIE_CAP - CL_PCIE_BEGIN + PCIECapPtr); if (CL_PCIE_CAP_SLOT & pciCaps) { // Get the slot capabilities. slotCaps = osPciReadDword(handleUp, CL_PCIE_SLOT_CAP - CL_PCIE_BEGIN + PCIECapPtr); if ((CL_PCIE_SLOT_CAP_HOTPLUG_CAPABLE & slotCaps) && (CL_PCIE_SLOT_CAP_HOTPLUG_SURPRISE & slotCaps)) { bSlotHotPlugSupport = NV_TRUE; } } if (bTb3Bridge && bSlotHotPlugSupport) { iseGPUBridge = NV_TRUE; break; } bus = busUp; // Get port caps to check if PCIE bridge is the root port portCaps = osPciReadDword(handleUp, CL_PCIE_CAP - CL_PCIE_BEGIN + PCIECapPtr); } while (!CL_IS_ROOT_PORT(portCaps)); return iseGPUBridge; } static NV_STATUS RmGetFirmwareVersion ( nv_state_t *pNv, RM_API *pRmApi ) { NV2080_CTRL_GSP_GET_FEATURES_PARAMS gsp_params; NV_STATUS rmStatus = NV_OK; pNv->cached_gpu_info.firmware_version[0] = '\0'; rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hSubDevice, NV2080_CTRL_CMD_GSP_GET_FEATURES, &gsp_params, sizeof(gsp_params)); if (rmStatus != NV_OK) { if (RMCFG_FEATURE_GSP_CLIENT_RM && pNv->request_firmware) { const char *pTmpString = "N/A"; portStringCopy(pNv->cached_gpu_info.firmware_version, sizeof(pNv->cached_gpu_info.firmware_version), pTmpString, portStringLength(pTmpString) + 1); } NV_PRINTF(LEVEL_INFO, "%s: Failed to query gpu firmware version, status=0x%x\n", __FUNCTION__, rmStatus); } else { portMemCopy(pNv->cached_gpu_info.firmware_version, sizeof(pNv->cached_gpu_info.firmware_version), gsp_params.firmwareVersion, sizeof(gsp_params.firmwareVersion)); } return NV_OK; } static NV_STATUS RmGetVbiosVersion ( nv_state_t *pNv, RM_API *pRmApi ) { NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS *vbios_params; const size_t vbiosStringLen = 15; // "xx.xx.xx.xx.xx" NV_STATUS rmStatus = NV_OK; vbios_params = portMemAllocNonPaged(sizeof(*vbios_params)); if (vbios_params == NULL) { return NV_ERR_NO_MEMORY;; } portMemSet(vbios_params, 0, sizeof(*vbios_params)); vbios_params->biosInfoList[0].index = NV2080_CTRL_BIOS_INFO_INDEX_REVISION; vbios_params->biosInfoList[1].index = NV2080_CTRL_BIOS_INFO_INDEX_OEM_REVISION; vbios_params->biosInfoListSize = 2; rmStatus = pRmApi->Control(pRmApi, pNv->rmapi.hClient, pNv->rmapi.hSubDevice, NV2080_CTRL_CMD_BIOS_GET_INFO_V2, vbios_params, sizeof(*vbios_params)); if (rmStatus == NV_OK) { const NvU32 biosRevision = vbios_params->biosInfoList[0].data; const NvU32 biosOEMRevision = vbios_params->biosInfoList[1].data; os_snprintf(pNv->cached_gpu_info.vbios_version, vbiosStringLen, "%02x.%02x.%02x.%02x.%02x", (biosRevision & 0xff000000) >> 24, (biosRevision & 0x00ff0000) >> 16, (biosRevision & 0x0000ff00) >> 8, (biosRevision & 0x000000ff) >> 0, biosOEMRevision); } else { NV_PRINTF(LEVEL_INFO, "%s: Failed to query vbios version, status=0x%x\n", __FUNCTION__, rmStatus); } portMemFree(vbios_params); return NV_OK; } /* * Initialize the required GPU information by doing RMAPI control calls * and store the same in the UNIX specific data structures. */ static NV_STATUS RmInitGpuInfoWithRmApi ( OBJGPU *pGpu ) { RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); nv_state_t *nv = NV_GET_NV_STATE(pGpu); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NV2080_CTRL_GPU_GET_INFO_V2_PARAMS *pGpuInfoParams = { 0 }; NV_STATUS status; const NvU8 *uuid; // LOCK: acquire GPUs lock status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); if (status != NV_OK) { return status; } pGpuInfoParams = portMemAllocNonPaged(sizeof(*pGpuInfoParams)); if (pGpuInfoParams == NULL) { // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); return NV_ERR_NO_MEMORY; } portMemSet(pGpuInfoParams, 0, sizeof(*pGpuInfoParams)); pGpuInfoParams->gpuInfoListSize = 3; pGpuInfoParams->gpuInfoList[0].index = NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED; pGpuInfoParams->gpuInfoList[1].index = NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY; pGpuInfoParams->gpuInfoList[2].index = NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE; status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, NV2080_CTRL_CMD_GPU_GET_INFO_V2, pGpuInfoParams, sizeof(*pGpuInfoParams)); if (status == NV_OK) { nvp->b_mobile_config_enabled = (pGpuInfoParams->gpuInfoList[0].data == NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_YES); nv->dma_buf_supported = (pGpuInfoParams->gpuInfoList[1].data == NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES); } nv->coherent = (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) || pGpu->pGpuArch->bGpuArchIsZeroFb); // // If coherent GPU memory mode is NONE, then GPU memory has struct page // on coherent platforms and no struct page on non-coherent ones. // If the mode is enabled and _NUMA, then GPU memory has struct page. // If the mode is enabled and _DRIVER, then GPU memory doesn't have struct page. // Tegra iGPU also falls under struct page category, although // COHERENT_GPU_MEMORY_MODE doesn't apply there. // if (pGpuInfoParams->gpuInfoList[2].data == NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NONE) { nv->mem_has_struct_page = nv->coherent; } else { // If mode is not _NONE, we're already on a PDB_PROP_GPU_COHERENT_CPU_MAPPING platform. nv->mem_has_struct_page = pGpu->pGpuArch->bGpuArchIsZeroFb || (pGpuInfoParams->gpuInfoList[2].data == NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NUMA); } switch (pGpuInfoParams->gpuInfoList[2].data) { case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NONE: nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_NONE; break; case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NUMA: nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_NUMA; break; case NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_DRIVER: nv->coherent_gpu_mem_mode = NV_COHERENT_GPU_MEM_MODE_DRIVER; break; } portMemFree(pGpuInfoParams); if (status != NV_OK) { goto finish; } uuid = RmGetGpuUuidRaw(nv); if (uuid == NULL) { status = NV_ERR_GPU_UUID_NOT_FOUND; goto finish; } status = RmGetFirmwareVersion(nv, pRmApi); if (status != NV_OK) { goto finish; } status = RmGetVbiosVersion(nv, pRmApi); finish: // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); return status; } static void RmSetSocDispDeviceMappings( GPUATTACHARG *gpuAttachArg, nv_state_t *nv ) { gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvAddr = (GPUHWREG*) nv->regs->map; gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvPAddr = nv->regs->cpu_address; gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvLength = (NvU32) nv->regs->size; } static void RmSetSocDpauxDeviceMappings( GPUATTACHARG *gpuAttachArg, nv_state_t *nv ) { } static void RmSetSocHdacodecDeviceMappings( GPUATTACHARG *gpuAttachArg, nv_state_t *nv ) { } static void RmSetSocMipiCalDeviceMappings( GPUATTACHARG *gpuAttachArg, nv_state_t *nv ) { } static void osInitNvMapping( nv_state_t *nv, NvU32 *pDeviceReference, UNIX_STATUS *status ) { OBJGPU *pGpu; OBJSYS *pSys = SYS_GET_INSTANCE(); GPUATTACHARG *gpuAttachArg; nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NvU32 deviceInstance; NvU32 data = 0; NvU32 dispIsoStreamId; NvU32 dispNisoStreamId; NV_PRINTF(LEVEL_INFO, "osInitNvMapping:\n"); // allocate the next available gpu device number status->rmStatus = gpumgrAllocGpuInstance(pDeviceReference); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot get valid gpu instance\n"); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); return; } // RM_BASIC_LOCK_MODEL: allocate GPU lock status->rmStatus = rmGpuLockAlloc(*pDeviceReference); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** cannot allocate GPU lock\n"); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); return; } // attach default single-entry broadcast device for this gpu status->rmStatus = gpumgrCreateDevice(&deviceInstance, NVBIT(*pDeviceReference), NULL); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot attach bc gpu\n"); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(*pDeviceReference); return; } // init attach state gpuAttachArg = portMemAllocNonPaged(sizeof(GPUATTACHARG)); if (gpuAttachArg == NULL) { NV_PRINTF(LEVEL_ERROR, "*** Cannot allocate gpuAttachArg\n"); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(*pDeviceReference); return; } portMemSet(gpuAttachArg, 0, sizeof(GPUATTACHARG)); if (NV_IS_SOC_DISPLAY_DEVICE(nv)) { gpuAttachArg->socDeviceArgs.specified = NV_TRUE; RmSetSocDispDeviceMappings(gpuAttachArg, nv); RmSetSocDpauxDeviceMappings(gpuAttachArg, nv); RmSetSocHdacodecDeviceMappings(gpuAttachArg, nv); RmSetSocMipiCalDeviceMappings(gpuAttachArg, nv); gpuAttachArg->socDeviceArgs.socChipId0 = nv->disp_sw_soc_chip_id; gpuAttachArg->socDeviceArgs.iovaspaceId = nv->iovaspace_id; } else { gpuAttachArg->fbPhysAddr = nv->fb->cpu_address; gpuAttachArg->fbBaseAddr = (GPUHWREG*) 0; // not mapped gpuAttachArg->devPhysAddr = nv->regs->cpu_address; gpuAttachArg->regBaseAddr = (GPUHWREG*) nv->regs->map; gpuAttachArg->intLine = 0; // don't know yet gpuAttachArg->instPhysAddr = nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address; gpuAttachArg->instBaseAddr = (GPUHWREG*) 0; // not mapped gpuAttachArg->regLength = nv->regs->size; gpuAttachArg->fbLength = nv->fb->size; gpuAttachArg->instLength = nv->bars[NV_GPU_BAR_INDEX_IMEM].size; gpuAttachArg->iovaspaceId = nv->iovaspace_id; gpuAttachArg->cpuNumaNodeId = nv->cpu_numa_node_id; if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) { // Default - PCIe GPUs are connected via NISO IOMMU nv->iommus.iso_iommu_present = NV_FALSE; nv->iommus.niso_iommu_present = NV_TRUE; } } // // we need this to check if we are running on virtual GPU // in gpuBindHal function later. // if (!NV_IS_SOC_DISPLAY_DEVICE(nv)) { gpuAttachArg->nvDomainBusDeviceFunc = nv_encode_pci_info(&nv->pci_info); } gpuAttachArg->bRequestFwClientRm = nv->request_fw_client_rm; gpuAttachArg->pOsAttachArg = (void *)nv; // use gpu manager to attach gpu status->rmStatus = gpumgrAttachGpu(*pDeviceReference, gpuAttachArg); portMemFree(gpuAttachArg); if (status->rmStatus != NV_OK) { gpumgrDestroyDevice(deviceInstance); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED); NV_PRINTF(LEVEL_ERROR, "*** Cannot attach gpu\n"); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(*pDeviceReference); return; } nvp->flags |= NV_INIT_FLAG_GPUMGR_ATTACH; pGpu = gpumgrGetGpu(*pDeviceReference); sysInitRegistryOverrides(pSys); sysApplyLockingPolicy(pSys); pGpu->busInfo.IntLine = nv->interrupt_line; if (nv->fb != NULL) { pGpu->registerAccess.gpuFbAddr = (GPUHWREG*) nv->fb->map; pGpu->busInfo.gpuPhysFbAddr = nv->fb->cpu_address; } // set default parent gpu gpumgrSetParentGPU(pGpu, pGpu); NV_PRINTF(LEVEL_INFO, "device instance : %d\n", *pDeviceReference); NV_PRINTF(LEVEL_INFO, "NV regs using linear address : 0x%p\n", pGpu->deviceMappings[SOC_DEV_MAPPING_DISP].gpuNvAddr); NV_PRINTF(LEVEL_INFO, "NV fb using linear address : 0x%p\n", pGpu->registerAccess.gpuFbAddr); if (!os_is_vgx_hyper()) { pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE); } else { { pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE); } } if ((osReadRegistryDword(NULL, NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS, &data) == NV_OK) && data) { if (data == NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_ENABLED) { nv->preserve_vidmem_allocations = NV_TRUE; } else if (data == NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS_AUTO) { /* If the kernel supports suspend notifiers, enable video memory allocations */ nv->preserve_vidmem_allocations = os_supports_kernel_suspend_notifiers(); } if (nv->preserve_vidmem_allocations && nv_dev_needs_vidmem_preservation(nv)) { KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); memmgrSetPmaForcePersistence(pMemoryManager, NV_TRUE); // Monolithic uses Fifolite channel which doesn't support Virtual mode if (!pMemoryManager->bUseVirtualCopyOnSuspend) { pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = NV_TRUE; } } } // Check if SMMU can be enabled on PushBuffer Aperture nv_get_disp_smmu_stream_ids(nv, &dispIsoStreamId, &dispNisoStreamId); if (dispNisoStreamId != NV_U32_MAX) { pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_FALSE); } else { pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_TRUE); } if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TRIGGER_PCIE_FLR)) { nv->flags |= NV_FLAG_TRIGGER_FLR; } if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_SOC_SDM)) { nv->flags |= NV_FLAG_HAS_CONSOLE_IN_SYSMEM_CARVEOUT; } } static NV_STATUS osInitScalability( OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); // We need PCI-E chipset information very early. if (pCl != NULL) { return clInitPcie(pGpu, pCl); } else { return NV_ERR_NOT_SUPPORTED; } } static NV_STATUS osTeardownScalability( OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); return clTeardownPcie(pGpu, pCl); } static NV_STATUS RmInitNvHal( nv_state_t *nv, NvU32 deviceReference, UNIX_STATUS *status ) { OBJGPU *pGpu = gpumgrGetGpu(deviceReference); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); PORT_UNREFERENCED_VARIABLE(pGpu); nvp->flags |= NV_INIT_FLAG_HAL; nvp->flags |= NV_INIT_FLAG_HAL_COMPONENTS; return NV_OK; } static void RmDeterminePrimaryDevice(OBJGPU *pGpu) { nv_state_t *nv = NV_GET_NV_STATE(pGpu); // Skip updating nv->primary_vga while RM is recovering after GPU reset if (nv->flags & NV_FLAG_IN_RECOVERY) { return; } // // In case of Passthru, GPU will always be secondary // if (IS_PASSTHRU(pGpu)) { return; } // // In case of VIRTUAL GPU, there is no display, hence it will be secondary // if (IS_VIRTUAL(pGpu)) { return; } // // If GPU is driving any frame buffer console(vesafb, efifb etc) // mark the console as client driven and GPU as Primary. // nv->client_managed_console = rm_get_uefi_console_status(nv); pGpu->setProperty(pGpu, PDB_PROP_GPU_PRIMARY_DEVICE, (nv->client_managed_console || !!nv->primary_vga)); } static void RmSetConsolePreservationParams(OBJGPU *pGpu) { nv_state_t *nv = NV_GET_NV_STATE(pGpu); NvU64 fbBaseAddress = 0; NvU64 fbConsoleSize = 0; MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); // // PDB_PROP_GPU_PRIMARY_DEVICE should be NV_FALSE for vGPU configuration so // return early // if (os_is_vgx_hyper() || IS_VIRTUAL(pGpu)) return; if (!gpuFuseSupportsDisplay_HAL(pGpu)) { return; } // // If this is the zero FB SOC GPU with a console in system carveout, then // the console memory is not mapped onto the BAR. The kernel can directly // access the console memory, while the GPU accesses system carveout memory // through the SMMU. // The carveout is a reserved region of system memory that doesn't get used // in standard memory allocation. // // In this scenario, you don't need to reserve console memory or BAR mapping. // if (NV_HAS_CONSOLE_IN_SYSMEM_CARVEOUT(nv)) { return; } // // Check the OS layer for any video memory used by a console // driver that should be reserved. // // If the console driver is using some amount of memory, its // mapping is presumably either pointing at the start of BAR1 // (this is the mapping the EFI GOP driver provides for // efifb), or the upper 16MB of BAR2 (this is the mapping the // VBIOS provides for, e.g., vesafb). // // In the BAR2 case, there is nothing more to do: the upper // 16MB of BAR2 will remain in physical mode // (see the documentation for NV_PBUS_BAR2_BLOCK in dev_bus.ref), // and the console driver can continue to access it. // // In the BAR1 case, RM will put all of BAR1 into virtual // mode, so we need to create a BAR1 mapping for the console // driver to use. // fbConsoleSize = rm_get_uefi_console_size(nv, &fbBaseAddress); if ((fbConsoleSize > 0) && (fbBaseAddress != 0) && (pKernelBus != NULL)) { pKernelBus->bPreserveBar1ConsoleEnabled = (fbBaseAddress == nv->fb->cpu_address); } // // Case for legacy console, where OS will not give console size // and primary_vga will be set for the device. // Hence, reserving default size for VGA text modes. // if ((fbConsoleSize == 0) && nv->primary_vga) { fbConsoleSize = 0x40000; } if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) { pMemoryManager->Ram.ReservedConsoleDispMemSize = fbConsoleSize; pMemoryManager->Ram.ReservedConsoleDispMemBase = fbBaseAddress; } else { pMemoryManager->Ram.ReservedConsoleDispMemSize = NV_ALIGN_UP(fbConsoleSize, 0x10000); } } static NV_STATUS RmInitDeviceDma( nv_state_t *nv ) { if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJVMM *pVmm = SYS_GET_VMM(pSys); OBJVASPACE *pIOVAS; NV_STATUS status = vmmCreateVaspace(pVmm, IO_VASPACE_A, nv->iovaspace_id, 0, 0ULL, ~0ULL, 0ULL, 0ULL, NULL, VASPACE_FLAGS_ENABLE_VMM, &pIOVAS); if (status != NV_OK) { return status; } } return NV_OK; } static void RmTeardownDeviceDma( nv_state_t *nv ) { if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJVMM *pVmm = SYS_GET_VMM(pSys); OBJVASPACE *pIOVAS; if (NV_OK == vmmGetVaspaceFromId(pVmm, nv->iovaspace_id, IO_VASPACE_A, &pIOVAS)) { vmmDestroyVaspace(pVmm, pIOVAS); } } } static void RmEnableDeviceClks(nv_state_t *nv) { NvU32 freqKHz = 0; NvU32 i; NV_STATUS ret; for (i = TEGRASOC_WHICH_CLK_GPU_FIRST; i <= TEGRASOC_WHICH_CLK_GPU_LAST; i++) { ret = nv_get_max_freq(nv, i, &freqKHz); if (ret != NV_OK) { NV_PRINTF(LEVEL_INFO, "NVRM: Max Freq fetch failed for Clk:%d\n", i); continue; } ret = nv_enable_clk(nv, i); if (ret != NV_OK) { NV_PRINTF(LEVEL_INFO, "NVRM: Clk prepare enable failed for Clk:%d\n", i); continue; } ret = nv_set_freq(nv, i, freqKHz); if (ret != NV_OK) { NV_PRINTF(LEVEL_INFO, "NVRM: Set Freq failed for Clk:%d\n", i); } else { NV_PRINTF(LEVEL_INFO, "NVRM: Set Freq:%d for Clk:%d\n", freqKHz, i); } } } static void RmDisableDeviceClks(nv_state_t *nv) { NvU32 i; for (i = TEGRASOC_WHICH_CLK_GPU_FIRST; i <= TEGRASOC_WHICH_CLK_GPU_LAST; i++) { nv_disable_clk(nv, i); } } static void RmInitNvDevice( NvU32 deviceReference, UNIX_STATUS *status ) { // set the device context OBJGPU *pGpu = gpumgrGetGpu(deviceReference); OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); nv_state_t *nv = NV_GET_NV_STATE(pGpu); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NV_PRINTF(LEVEL_INFO, "RmInitNvDevice:\n"); NV_PRINTF(LEVEL_INFO, "device instance : 0x%08x\n", deviceReference); // initialize all engines -- calls back osInitMapping() status->rmStatus = gpumgrStatePreInitGpu(pGpu); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot pre-initialize the device\n"); RM_SET_ERROR(*status, RM_INIT_GPU_PRE_INIT_FAILED); return; } // Configure eGPU setting if (RmCheckForExternalGpu(pGpu, pCl)) { pGpu->setProperty(pGpu, PDB_PROP_GPU_IS_EXTERNAL_GPU, NV_TRUE); nv->is_external_gpu = NV_TRUE; } status->rmStatus = gpumgrStateInitGpu(pGpu); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize the device\n"); RM_SET_ERROR(*status, RM_INIT_GPU_STATE_INIT_FAILED); return; } nvp->flags |= NV_INIT_FLAG_GPU_STATE; KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); // // Make sure bifIsMSIEnabled() gets a chance to look up the OS PCI // handle for this GPU. This must happen after GPU state // initialization. // if (pKernelBif != NULL) { kbifCheckAndRearmMSI(pGpu, pKernelBif); } // Set RM's interrupt enable state to zero here so that interrupts // won't be enabled during loading Intr *pIntr = GPU_GET_INTR(pGpu); if (pIntr != NULL) { intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); } status->rmStatus = gpumgrStateLoadGpu(pGpu, GPU_STATE_DEFAULT); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot load state into the device\n"); RM_SET_ERROR(*status, RM_INIT_GPU_LOAD_FAILED); return; } nvp->flags |= NV_INIT_FLAG_GPU_STATE_LOAD; // Setup GPU scalability (void) RmInitScalability(pGpu); return; } NV_STATUS osInitMapping( OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); nv_state_t *nv = NV_GET_NV_STATE(pGpu); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NV_PRINTF(LEVEL_INFO, "osInitMapping:\n"); // Some PCI BIOSs leave the ROM mapped. This causes problems if it overlays system RAM. Just disable it. GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_12, 0); // make sure our PCI latency timer is sufficient (max it out) GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_3, DRF_DEF(_CONFIG, _PCI_NV_3, _LATENCY_TIMER, _248_CLOCKS)); if (pCl != NULL) { initCoreLogic(pGpu); } nvp->flags |= NV_INIT_FLAG_CORE_LOGIC; return NV_OK; } // end of osInitMapping() static NV_STATUS initCoreLogic( OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); return clInit(pGpu, pCl); } static NV_STATUS teardownCoreLogic( OBJOS *pOS, OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); return clTeardown(pGpu, pCl); } static void RmTeardownDpauxRegisters( nv_state_t *nv ) { } static void RmTeardownHdacodecRegisters( nv_state_t *nv ) { } static void RmTeardownMipiCalRegisters( nv_state_t *nv ) { } static NV_STATUS RmTeardownRegisters( nv_state_t *nv ) { NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Tearing down registers\n"); if (nv->regs && nv->regs->map) { osUnmapKernelSpace(nv->regs->map, nv->regs->size); nv->regs->map = 0; nv->regs->map_u = NULL; } RmTeardownDpauxRegisters(nv); RmTeardownHdacodecRegisters(nv); RmTeardownMipiCalRegisters(nv); return NV_OK; } static NV_STATUS RmSetupDpauxRegisters( nv_state_t *nv, UNIX_STATUS *status ) { return NV_OK; } static NV_STATUS RmSetupHdacodecRegisters( nv_state_t *nv, UNIX_STATUS *status ) { return NV_OK; } static NV_STATUS RmSetupMipiCalRegisters( nv_state_t *nv, UNIX_STATUS *status ) { return NV_OK; } static void RmSetupRegisters( nv_state_t *nv, UNIX_STATUS *status ) { NV_STATUS ret; NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmSetupRegisters for 0x%x:0x%x\n", nv->pci_info.vendor_id, nv->pci_info.device_id); NV_DEV_PRINTF(NV_DBG_SETUP, nv, "pci config info:\n"); NV_DEV_PRINTF(NV_DBG_SETUP, nv, " registers look like: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx, nv->regs->cpu_address, nv->regs->size); if (nv->fb != NULL) { NV_DEV_PRINTF(NV_DBG_SETUP, nv, " fb looks like: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx "\n", nv->fb->cpu_address, nv->fb->size); } nv_os_map_kernel_space(nv, nv->regs); if (nv->regs->map == NULL) { NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "Failed to map regs registers!!\n"); RM_SET_ERROR(*status, RM_INIT_REG_SETUP_FAILED); status->rmStatus = NV_ERR_OPERATING_SYSTEM; return; } NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Successfully mapped framebuffer and registers\n"); NV_DEV_PRINTF(NV_DBG_SETUP, nv, "final mappings:\n"); NV_DEV_PRINTF(NV_DBG_SETUP, nv, " regs: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx " 0x%p\n", nv->regs->cpu_address, nv->regs->size, nv->regs->map); ret = RmSetupDpauxRegisters(nv, status); if (ret != NV_OK) goto err_unmap_disp_regs; ret = RmSetupHdacodecRegisters(nv, status); if (ret != NV_OK) { RmTeardownDpauxRegisters(nv); goto err_unmap_disp_regs; } ret = RmSetupMipiCalRegisters(nv, status); if (ret != NV_OK) { RmTeardownHdacodecRegisters(nv); RmTeardownDpauxRegisters(nv); goto err_unmap_disp_regs; } return; err_unmap_disp_regs: if (nv->regs && nv->regs->map) { osUnmapKernelSpace(nv->regs->map, nv->regs->size); nv->regs->map = 0; } return; } NvBool RmInitPrivateState( nv_state_t *pNv ) { nv_priv_t *nvp; NvU32 gpuId; NvU32 socChipId0 = 0; NvU32 pmc_boot_0 = 0; NvU32 pmc_boot_1 = 0; NvU32 pmc_boot_42 = 0; NvU32 dmaAddrWidth = 0; NvU64 gpuDomainBusDevice = 0; NV_SET_NV_PRIV(pNv, NULL); if (NV_IS_SOC_DISPLAY_DEVICE(pNv)) socChipId0 = pNv->disp_sw_soc_chip_id; else { pNv->regs->map_u = os_map_kernel_space(pNv->regs->cpu_address, os_page_size, NV_MEMORY_UNCACHED); if (pNv->regs->map_u == NULL) { NV_PRINTF(LEVEL_ERROR, "failed to map GPU registers (DISABLE_INTERRUPTS).\n"); return NV_FALSE; } pmc_boot_0 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_0); pmc_boot_1 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_1); pmc_boot_42 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_42); os_unmap_kernel_space(pNv->regs->map_u, os_page_size); pNv->regs->map_u = NULL; } if (os_alloc_mem((void **)&nvp, sizeof(*nvp)) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "failed to allocate private device state.\n"); return NV_FALSE; } if (!NV_IS_SOC_DISPLAY_DEVICE(pNv)) { gpuId = nv_generate_id_from_pci_info(&pNv->pci_info); gpuDomainBusDevice = nv_encode_pci_info(&pNv->pci_info); } else { gpuId = NV0000_CTRL_GPU_INVALID_ID - 1; gpuDomainBusDevice = 0; } if (gpumgrRegisterGpuId(gpuId, gpuDomainBusDevice) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "failed to register GPU with GPU manager.\n"); os_free_mem(nvp); return NV_FALSE; } pNv->gpu_id = gpuId; nv_set_probed_gpu_flags(pNv); pNv->iovaspace_id = nv_requires_dma_remap(pNv) ? gpuId : NV_IOVA_DOMAIN_NONE; pNv->cpu_numa_node_id = NV0000_CTRL_NO_NUMA_NODE; // Get the GpuArch instance for this architecture to determine the DMA address width. GpuArch *pGpuArch = gpumgrGetGpuArch(pmc_boot_42, socChipId0, TEGRA_CHIP_TYPE_DEFAULT); if (pGpuArch == NULL) { NV_PRINTF(LEVEL_ERROR, "failed to get GpuArch for 0x%x/0x%x.\n", pmc_boot_42, socChipId0); gpumgrUnregisterGpuId(gpuId); os_free_mem(nvp); return NV_FALSE; } dmaAddrWidth = gpuarchGetDmaAddrWidth(pGpuArch); if (dmaAddrWidth == 0) { dmaAddrWidth = gpuarchGetSystemPhysAddrWidth(pGpuArch); } nv_set_dma_address_size(pNv, dmaAddrWidth); pNv->is_tegra_pci_igpu = !NV_IS_SOC_DISPLAY_DEVICE(pNv) && pGpuArch->bGpuArchIsZeroFb; // Only certain Tegra PCI iGPUs support Rail-Gating pNv->supports_tegra_igpu_rg = pNv->is_tegra_pci_igpu && pGpuArch->bGpuarchSupportsIgpuRg; pNv->sysmem_mapped_console = pGpuArch->bGpuArchIsZeroFb; kvgpumgrAttachGpu(pNv->gpu_id); os_mem_set(nvp, 0, sizeof(*nvp)); nvp->status = NV_ERR_INVALID_STATE; nvp->pmc_boot_0 = pmc_boot_0; nvp->pmc_boot_1 = pmc_boot_1; nvp->pmc_boot_42 = pmc_boot_42; nvp->db_supported = -1; NV_SET_NV_PRIV(pNv, nvp); return NV_TRUE; } void RmClearPrivateState( nv_state_t *pNv ) { nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); NvU32 status; void *pVbiosCopy = NULL; void *pRegistryCopy = NULL; NvU32 vbiosSize; nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS]; nv_dynamic_power_t dynamicPowerCopy; NvU32 x = 0; NvU32 pmc_boot_0, pmc_boot_1, pmc_boot_42; NvBool pr3_acpi_method_present = 0; int db_supported; // // Do not clear private state after GPU resets, it is used while // recovering the GPU. Only clear the pGpu pointer, which is // restored during next initialization cycle. // if (pNv->flags & NV_FLAG_IN_RECOVERY) { nvp->pGpu = NULL; } status = nvp->status; pVbiosCopy = nvp->pVbiosCopy; vbiosSize = nvp->vbiosSize; pRegistryCopy = nvp->pRegistry; dynamicPowerCopy = nvp->dynamic_power; pmc_boot_0 = nvp->pmc_boot_0; pmc_boot_1 = nvp->pmc_boot_1; pmc_boot_42 = nvp->pmc_boot_42; pr3_acpi_method_present = nvp->pr3_acpi_method_present; db_supported = nvp->db_supported; for (x = 0; x < MAX_I2C_ADAPTERS; x++) { i2c_adapters[x] = nvp->i2c_adapters[x]; } portMemSet(nvp, 0, sizeof(nv_priv_t)); nvp->status = status; nvp->pVbiosCopy = pVbiosCopy; nvp->vbiosSize = vbiosSize; nvp->pRegistry = pRegistryCopy; nvp->dynamic_power = dynamicPowerCopy; nvp->pmc_boot_0 = pmc_boot_0; nvp->pmc_boot_1 = pmc_boot_1; nvp->pmc_boot_42 = pmc_boot_42; nvp->pr3_acpi_method_present = pr3_acpi_method_present; nvp->db_supported = db_supported; for (x = 0; x < MAX_I2C_ADAPTERS; x++) { nvp->i2c_adapters[x] = i2c_adapters[x]; } nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C; } void RmFreePrivateState( nv_state_t *pNv ) { nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); gpumgrUnregisterGpuId(pNv->gpu_id); kvgpumgrDetachGpu(pNv->gpu_id); RmDestroyRegistry(pNv); if (nvp != NULL) { portMemFree(nvp->pVbiosCopy); os_free_mem(nvp); } NV_SET_NV_PRIV(pNv, NULL); } NvBool RmPartiallyInitAdapter( nv_state_t *nv ) { NV_PRINTF(LEVEL_INFO, "%s: %04x:%02x:%02x.0\n", __FUNCTION__, nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot); nv_start_rc_timer(nv); return NV_TRUE; } static NV_STATUS RmInitX86Emu( OBJGPU *pGpu ) { NV_STATUS status = NV_OK; nv_state_t *nv = NV_GET_NV_STATE(pGpu); PORT_UNREFERENCED_VARIABLE(nv); #if NVCPU_IS_X86_64 status = RmInitX86EmuState(pGpu); #else // We don't expect a "primary VGA" adapter on non-amd64 platforms NV_ASSERT(!NV_PRIMARY_VGA(nv)); #endif return status; } static NV_STATUS RmRegisterGpudb( OBJGPU *pGpu ) { NV_STATUS rmStatus; const NvU8 *pGid; nv_state_t *pNv = NV_GET_NV_STATE(pGpu); pGid = RmGetGpuUuidRaw(pNv); if (pGid == NULL) { NV_PRINTF(LEVEL_ERROR, "Failed to get UUID\n"); return NV_ERR_OPERATING_SYSTEM; } rmStatus = gpudbRegisterGpu(pGid, &pGpu->gpuClData.upstreamPort.addr, pGpu->busInfo.nvDomainBusDeviceFunc); if (rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Failed to register GPU with GPU data base\n"); } return rmStatus; } static void RmUnixFreeRmApi( nv_state_t *nv ) { if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) { RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); if (nv->rmapi.hClient != 0) { pRmApi->Free(pRmApi, nv->rmapi.hClient, nv->rmapi.hClient); } portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi)); rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); } } static NvBool RmUnixAllocRmApi( nv_state_t *nv, NvU32 deviceId ) { NV_STATUS status = NV_OK; NV0080_ALLOC_PARAMETERS deviceParams = { 0 }; NV2080_ALLOC_PARAMETERS subDeviceParams = { 0 }; RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi)); // LOCK: acquire GPUs lock status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); if (status != NV_OK) { return status; } if (pRmApi->AllocWithHandle( pRmApi, NV01_NULL_OBJECT, NV01_NULL_OBJECT, NV01_NULL_OBJECT, NV01_ROOT, &nv->rmapi.hClient, sizeof(nv->rmapi.hClient)) != NV_OK) { goto fail; } deviceParams.deviceId = deviceId; if (pRmApi->Alloc( pRmApi, nv->rmapi.hClient, nv->rmapi.hClient, &nv->rmapi.hDevice, NV01_DEVICE_0, &deviceParams, sizeof(deviceParams)) != NV_OK) { goto fail; } subDeviceParams.subDeviceId = 0; if (pRmApi->Alloc( pRmApi, nv->rmapi.hClient, nv->rmapi.hDevice, &nv->rmapi.hSubDevice, NV20_SUBDEVICE_0, &subDeviceParams, sizeof(subDeviceParams)) != NV_OK) { goto fail; } // // The NV40_I2C allocation expected to fail, if it is disabled // with RM config. // if (pRmApi->Alloc( pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, &nv->rmapi.hI2C, NV40_I2C, NULL, 0) != NV_OK) { nv->rmapi.hI2C = 0; } // // The NV04_DISPLAY_COMMON allocation expected to fail for displayless // system. nv->rmapi.hDisp value needs to be checked before doing display // related control calls. // if (pRmApi->Alloc( pRmApi, nv->rmapi.hClient, nv->rmapi.hDevice, &nv->rmapi.hDisp, NV04_DISPLAY_COMMON, NULL, 0) != NV_OK) { nv->rmapi.hDisp = 0; } // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); return NV_TRUE; fail: // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); RmUnixFreeRmApi(nv); return NV_FALSE; } static NV_STATUS RmFetchGspRmImages ( nv_state_t *nv, GSP_FIRMWARE *pGspFw, const void **gspFwHandle, const void **gspFwLogHandle, const void **gspUcodesHandle ) { nv_firmware_chip_family_t chipFamily; nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NvU32 gpuArch = (decodePmcBoot42Architecture(nvp->pmc_boot_42) << GPU_ARCH_SHIFT); NvU32 gpuImpl = DRF_VAL(_PMC, _BOOT_42, _IMPLEMENTATION, nvp->pmc_boot_42); chipFamily = nv_firmware_get_chip_family(gpuArch, gpuImpl); portMemSet(pGspFw, 0, sizeof(*pGspFw)); *gspFwHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP, chipFamily, &pGspFw->pBuf, &pGspFw->size); NvBool bFirmwareLoadSuccessful = *gspFwHandle != NULL; // // GR-3428 development: Temporarily gated by regkey so this code can be // decoupled from the build infra changes to package up ucodes.bin. // // Because loading the bindata ends up calling request_firmware() on Linux, // and that function will always print an error to dmesg if the requested // file is not found, we use this to skip trying to load it. // // Check will be removed once ucodes.bin becomes part of the driver package. // NvU32 data = 0; if ((osReadRegistryDword(NULL, "GR3428ReadUcodesBin", &data) == NV_OK) && data) { *gspUcodesHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_UCODES, chipFamily, &pGspFw->pUcodesBin, &pGspFw->ucodesBinSize); bFirmwareLoadSuccessful &= *gspUcodesHandle != NULL; } if (!bFirmwareLoadSuccessful && !nv->allow_fallback_to_monolithic_rm) { NV_PRINTF(LEVEL_ERROR, "No firmware image found\n"); return NV_ERR_NOT_SUPPORTED; } else if (bFirmwareLoadSuccessful) { #if LIBOS_LOG_DECODE_ENABLE if (nv->enable_firmware_logs) { *gspFwLogHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP_LOG, chipFamily, &pGspFw->pLogElf, &pGspFw->logElfSize); if (*gspFwLogHandle == NULL) { NV_PRINTF(LEVEL_ERROR, "Failed to load gsp_log_*.bin, no GSP-RM logs will be printed (non-fatal)\n"); } } #endif nv->request_fw_client_rm = NV_TRUE; } else { nv->request_fw_client_rm = NV_FALSE; } return NV_OK; } static void _checkP2pChipsetSupport( nv_state_t *nv ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); // Check for PCIe P2P chipset support if (!pCl->bPciePeerWriteCapable || !pCl->bPciePeerReadCapable) { nv->flags |= NV_FLAG_PCI_P2P_UNSUPPORTED_CHIPSET; } } NvBool RmInitAdapter( nv_state_t *nv ) { NvU32 devicereference = 0; UNIX_STATUS status = INIT_UNIX_STATUS; nv_priv_t *nvp; NvBool retVal = NV_FALSE; OBJSYS *pSys; OBJGPU *pGpu = NULL; OBJOS *pOS; KernelDisplay *pKernelDisplay; const void *gspFwHandle = NULL; const void *gspFwLogHandle = NULL; const void *gspUcodesHandle = NULL; GSP_FIRMWARE gspFw = {0}; PORT_UNREFERENCED_VARIABLE(gspFw); NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter\n"); nv->flags &= ~NV_FLAG_PASSTHRU; nv->flags &= ~NV_FLAG_PCI_P2P_UNSUPPORTED_CHIPSET; RmSetupRegisters(nv, &status); if (! RM_INIT_SUCCESS(status.initStatus) ) goto failed; nvp = NV_GET_NV_PRIV(nv); nvp->status = NV_ERR_OPERATING_SYSTEM; status.rmStatus = RmInitDeviceDma(nv); if (status.rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Cannot configure the device for DMA\n"); RM_SET_ERROR(status, RM_INIT_GPU_DMA_CONFIGURATION_FAILED); goto shutdown; } nvp->flags |= NV_INIT_FLAG_DMA; pSys = SYS_GET_INSTANCE(); // // Get firmware from the OS, if requested, and decide if RM will run as a // firmware client. // if (nv->request_firmware) { if (!NV_IS_SOC_DISPLAY_DEVICE(nv)) { status.rmStatus = RmFetchGspRmImages(nv, &gspFw, &gspFwHandle, &gspFwLogHandle, &gspUcodesHandle); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_INIT_FIRMWARE_FETCH_FAILED); goto shutdown; } } else { nv->request_fw_client_rm = NV_TRUE; } } // // Initialization path requires expanded GPU visibility in GPUMGR // in order to access the GPU undergoing initialization. // status.rmStatus = gpumgrThreadEnableExpandedGpuVisibility(); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_INIT_GPU_GPUMGR_EXPANDED_VISIBILITY_FAILED); goto shutdown; } // initialize the RM device register mapping osInitNvMapping(nv, &devicereference, &status); if (! RM_INIT_SUCCESS(status.initStatus) ) { switch (status.rmStatus) { case NV_ERR_NOT_SUPPORTED: nvp->status = NV_ERR_NOT_SUPPORTED; break; } NV_PRINTF(LEVEL_ERROR, "osInitNvMapping failed, bailing out of RmInitAdapter\n"); goto shutdown; } // // now we can have a pdev for the first time... // pGpu = gpumgrGetGpu(devicereference); gpumgrSetCurrentGpuInstance(pGpu->gpuInstance); pOS = SYS_GET_OS(pSys); status.rmStatus = osInitScalability(pGpu); if (status.rmStatus == NV_OK) { nvp->flags |= NV_INIT_FLAG_SCALABILITY; } else if (status.rmStatus != NV_ERR_NOT_SUPPORTED) { RM_SET_ERROR(status, RM_INIT_SCALABILITY_FAILED); goto shutdown; } // Boot GSP-RM proxy through COT command either via FSP or SEC2 if (!IS_GSP_CLIENT(pGpu) && !IS_VIRTUAL(pGpu)) { status.rmStatus = gpuBootGspRmProxy(pGpu); if (status.rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "GSP-RM proxy boot command failed.\n"); RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); goto shutdown; } } RmDeterminePrimaryDevice(pGpu); RmSetConsolePreservationParams(pGpu); RmInitAcpiMethods(pOS, pSys, pGpu); // This needs to run before GSP-RM is booted, or else it will timeout if (pGpu->getProperty(pGpu, PDB_PROP_GPU_CLKS_IN_TEGRA_SOC)) { NV_PRINTF(LEVEL_INFO, "Enable Clocks to Max\n"); RmEnableDeviceClks(nv); } // // If GSP fw RM support is enabled then start the GSP microcode // (including the task running the full instance of the RM) and // exchange the necessary initial RPC messages before continuing // with GPU initialization here. // if (IS_GSP_CLIENT(pGpu)) { status.rmStatus = kgspInitRm(pGpu, GPU_GET_KERNEL_GSP(pGpu), &gspFw); if (status.rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Cannot initialize GSP firmware RM\n"); RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); goto shutdown; } } else if (IS_DCE_CLIENT(pGpu)) { status.rmStatus = dceclientDceRmInit(pGpu, GPU_GET_DCECLIENTRM(pGpu), NV_TRUE); if (status.rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Cannot initialize DCE firmware RM\n"); RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); goto shutdown; } } else if (nv->request_fw_client_rm) { // We were expecting to enable GSP-RM but something went wrong. if (!nv->allow_fallback_to_monolithic_rm) { RM_SET_ERROR(status, RM_INIT_FIRMWARE_POLICY_FAILED); goto shutdown; } else { NV_PRINTF(LEVEL_NOTICE, "Falling back to monolithic RM\n"); } } pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); if (pKernelDisplay != NULL) { kdispSetWarPurgeSatellitesOnCoreFree(pKernelDisplay, NV_TRUE); } if (IS_PASSTHRU(pGpu)) nv->flags |= NV_FLAG_PASSTHRU; RmInitNvHal(nv, devicereference, &status); if (!RM_INIT_SUCCESS(status.initStatus)) { NV_PRINTF(LEVEL_ERROR, "RmInitNvHal() failed, bailing out of RmInitAdapter!\n"); goto shutdown; } status.rmStatus = RmInitX86Emu(pGpu); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_INIT_VBIOS_X86EMU_FAILED); NV_PRINTF(LEVEL_ERROR, "RmInitX86Emu failed, bailing out of RmInitAdapter\n"); goto shutdown; } initVendorSpecificRegistry(pGpu, nv->pci_info.device_id); if (!IS_VIRTUAL(pGpu) && !NV_IS_SOC_DISPLAY_DEVICE(nv)) { initNbsiTable(pGpu); } // finally, initialize the device RmInitNvDevice(devicereference, &status); if (! RM_INIT_SUCCESS(status.initStatus) ) { NV_PRINTF(LEVEL_ERROR, "RmInitNvDevice failed, bailing out of RmInitAdapter\n"); switch (status.rmStatus) { case NV_ERR_INSUFFICIENT_POWER: nvp->status = NV_ERR_INSUFFICIENT_POWER; NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU does not have the necessary power cables connected.\n"); break; } goto shutdown; } // // Expanded GPU visibility in GPUMGR is no longer needed once the // GPU is initialized. // gpumgrThreadDisableExpandedGpuVisibility(); // LOCK: acquire GPUs lock status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); if (status.rmStatus != NV_OK) { goto shutdown; } status.rmStatus = osVerifySystemEnvironment(pGpu); // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_INIT_SYS_ENVIRONMENT_FAILED); switch (status.rmStatus) { case NV_ERR_IRQ_NOT_FIRING: nvp->status = NV_ERR_IRQ_NOT_FIRING; break; } NV_PRINTF(LEVEL_ERROR, "osVerifySystemEnvironment failed, bailing!\n"); goto shutdown; } Intr *pIntr = GPU_GET_INTR(pGpu); if (pIntr != NULL) { intrSetIntrEn(pIntr, INTERRUPT_TYPE_HARDWARE); } // LOCK: acquire GPUs lock status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); if (status.rmStatus != NV_OK) { goto shutdown; } KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); // initialize the watchdog (disabled by default) status.rmStatus = pKernelRc != NULL ? krcWatchdogInit_HAL(pGpu, pKernelRc, NULL) : NV_ERR_NOT_SUPPORTED; if (status.rmStatus == NV_OK) { krcWatchdogDisable(pKernelRc, NULL); nvp->flags |= NV_INIT_FLAG_FIFO_WATCHDOG; } else if (status.rmStatus == NV_ERR_NOT_SUPPORTED) { NV_PRINTF(LEVEL_INFO, "krcWatchdogInit returned _NOT_SUPPORTED. For Kepler GPUs in PGOB mode, this is normal\n"); } else { RM_SET_ERROR(status, RM_INIT_WATCHDOG_FAILED); NV_PRINTF(LEVEL_ERROR, "krcWatchdogInit failed, bailing out of RmInitAdapter\n"); // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); goto shutdown; } // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); nv_start_rc_timer(nv); nvp->status = NV_OK; if (!RmUnixAllocRmApi(nv, devicereference)) { RM_SET_ERROR(status, RM_INIT_ALLOC_RMAPI_FAILED); status.rmStatus = NV_ERR_GENERIC; goto shutdown; } if (!NV_IS_SOC_DISPLAY_DEVICE(nv)) { status.rmStatus = RmInitGpuInfoWithRmApi(pGpu); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_INIT_GPUINFO_WITH_RMAPI_FAILED); goto shutdown; } } // i2c only on master device?? RmI2cAddGpuPorts(nv); nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C; // This fifo hal call will fail on pre-fermi gpus. In that case, userd // info will remain 0 and skipped by fb mmap code KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); NvU64 udaddr = 0; NvU32 udsize = 0; if (pKernelFifo != NULL) { status.rmStatus = kfifoGetUserdBar1MapInfo_HAL(pGpu, pKernelFifo, &udaddr, &udsize); if (status.rmStatus == NV_OK) { nv->ud.cpu_address = (nv->fb->cpu_address + udaddr); nv->ud.size = udsize; } else if (status.rmStatus != NV_ERR_NOT_SUPPORTED) { RM_SET_ERROR(status, RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED); NV_PRINTF(LEVEL_ERROR, "kfifoGetUserdBar1MapInfo failed, bailing out of RmInitAdapter\n"); goto shutdown; } } nv->flags &= ~NV_FLAG_IN_RECOVERY; pSys->setProperty(pSys, PDB_PROP_SYS_SYSTEM_EVENTS_SUPPORTED, NV_TRUE); RmInitPowerManagement(nv); if (!NV_IS_SOC_DISPLAY_DEVICE(nv)) { status.rmStatus = RmRegisterGpudb(pGpu); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_GPUDB_REGISTER_FAILED); goto shutdown; } } if (nvp->b_mobile_config_enabled) { NvU32 ac_plugged = 0; if (nv_acpi_get_powersource(&ac_plugged) == NV_OK) { // LOCK: acquire GPU lock if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_NONE) == NV_OK) { // // As we have already acquired the API Lock here, we are calling // RmPowerSourceChangeEvent directly instead of rm_power_source_change_event. // RmPowerSourceChangeEvent(nv, !ac_plugged); // UNLOCK: release GPU lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); } } } _checkP2pChipsetSupport(nv); NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter succeeded!\n"); retVal = NV_TRUE; goto done; shutdown: nv->flags &= ~NV_FLAG_IN_RECOVERY; gpumgrThreadDisableExpandedGpuVisibility(); // call ShutdownAdapter to undo anything we've done above RmShutdownAdapter(nv); failed: NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "RmInitAdapter failed! (0x%x:0x%x:%d)\n", status.initStatus, status.rmStatus, status.line); done: nv_put_firmware(gspFwHandle); nv_put_firmware(gspFwLogHandle); nv_put_firmware(gspUcodesHandle); gpumgrSetCurrentGpuInstance(NV_U32_MAX); return retVal; } void RmShutdownAdapter( nv_state_t *nv ) { nv_priv_t *nvp = NV_GET_NV_PRIV(nv); OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); NV_STATUS rmStatus; if ((pGpu != NULL) && (nvp->flags & NV_INIT_FLAG_GPUMGR_ATTACH)) { NvU32 gpuInstance = gpuGetInstance(pGpu); NvU32 deviceInstance = gpuGetDeviceInstance(pGpu); OBJSYS *pSys = SYS_GET_INSTANCE(); RmUnixFreeRmApi(nv); nv->ud.cpu_address = 0; nv->ud.size = 0; // // LOCK: lock all clients in case of eGPU hot unplug, which // will not wait for all existing RM clients to stop using the GPU. // if (!nv->is_external_gpu || serverLockAllClients(&g_resServ) == NV_OK) { // LOCK: acquire GPUs lock if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) { // // Shutdown path requires expanded GPU visibility in GPUMGR in order // to access the GPU undergoing shutdown which may not be fully // initialized, and to continue accessing the GPU undergoing shutdown // after state destroy. // NV_ASSERT_OK(gpumgrThreadEnableExpandedGpuVisibility()); RmDestroyPowerManagement(nv); freeNbsiTable(pGpu); OBJCL *pCl = SYS_GET_CL(pSys); OBJOS *pOS = GPU_GET_OS(pGpu); if (pCl != NULL) { if (nvp->flags & NV_INIT_FLAG_CORE_LOGIC) { teardownCoreLogic(pOS, pGpu); } else if (nvp->flags & NV_INIT_FLAG_SCALABILITY) { osTeardownScalability(pGpu); } } if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG) { krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu), NULL); nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG; } rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance)); rmapiDelPendingDevices(NVBIT(gpuInstance)); if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) { rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT); NV_ASSERT(rmStatus == NV_OK); } if (nvp->flags & NV_INIT_FLAG_GPU_STATE) { rmStatus = gpuStateDestroy(pGpu); NV_ASSERT(rmStatus == NV_OK); } if (IS_DCE_CLIENT(pGpu)) { rmStatus = dceclientDceRmInit(pGpu, GPU_GET_DCECLIENTRM(pGpu), NV_FALSE); if (rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "DCE firmware RM Shutdown failure\n"); } } if (pGpu->getProperty(pGpu, PDB_PROP_GPU_CLKS_IN_TEGRA_SOC)) { NV_PRINTF(LEVEL_INFO, "Disable Clocks\n"); RmDisableDeviceClks(nv); } //if (nvp->flags & NV_INIT_FLAG_HAL) // destroyHal(pDev); #if NVCPU_IS_X86_64 RmFreeX86EmuState(pGpu); #endif gpumgrDetachGpu(gpuInstance); gpumgrDestroyDevice(deviceInstance); // // Expanded GPU visibility in GPUMGR is no longer needed once the // GPU is removed from GPUMGR. // gpumgrThreadDisableExpandedGpuVisibility(); if (nvp->flags & NV_INIT_FLAG_DMA) { RmTeardownDeviceDma(nv); } RmClearPrivateState(nv); RmUnInitAcpiMethods(pSys); // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(deviceInstance); } // UNLOCK: unlock all clients for eGPU hot unplug path if (nv->is_external_gpu) serverUnlockAllClients(&g_resServ); } } else { RmClearPrivateState(nv); } RmTeardownRegisters(nv); } void RmPartiallyDisableAdapter( nv_state_t *nv ) { NV_PRINTF(LEVEL_INFO, "%s: RM is in SW Persistence mode\n", __FUNCTION__); nv_stop_rc_timer(nv); } void RmDisableAdapter( nv_state_t *nv ) { NV_STATUS rmStatus; OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); NvU32 gpuMask; nv_priv_t *nvp = NV_GET_NV_PRIV(nv); if (pGpu == NULL) { return; } gpumgrSetCurrentGpuInstance(pGpu->gpuInstance); // // Normally, we re-enable interrupts when we release the lock // after RmDisableAdapter(), before disabling the bottom-half ISR. // If an interrupt came in in that time window, but the bottom half // wasn't scheduled to run in time, the semaphore acquired by the // top-half would never be released afterwards (since the bottom-half // would never run). This call to intrSetIntrEn() ensures that this won't // happen, by making sure interrupts remain disabled on the GPU in question after // RmDisableAdapter(). // Intr *pIntr = GPU_GET_INTR(pGpu); if (pIntr != NULL) { intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); } if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SECONDARY_BUS_RESET) || pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_FULLCHIP_RESET)) { pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_TIMEOUT_RECOVERY, NV_TRUE); nv->flags |= NV_FLAG_IN_RECOVERY; } // // LOCK: lock all clients in case of eGPU hot unplug, which // will not wait for all existing RM clients to stop using the GPU. // if (!nv->is_external_gpu || serverLockAllClients(&g_resServ) == NV_OK) { // LOCK: acquire GPUs lock if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) { // // Free the client allocated resources. // // Note this doesn't free RM internal resource allocations. Those are // freed during gpuStateUnload. // gpuMask = gpumgrGetGpuMask(pGpu); rmapiSetDelPendingClientResourcesFromGpuMask(gpuMask); rmapiDelPendingDevices(gpuMask); nv_stop_rc_timer(nv); OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); OBJOS *pOS = SYS_GET_OS(pSys); if (pCl != NULL) { teardownCoreLogic(pOS, pGpu); nvp->flags &= ~NV_INIT_FLAG_CORE_LOGIC; } if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG) { krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu), NULL); nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG; } if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) { rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT); NV_ASSERT(rmStatus == NV_OK); nvp->flags &= ~NV_INIT_FLAG_GPU_STATE_LOAD; } // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); } // UNLOCK: unlock all clients for eGPU hot unplug path if (nv->is_external_gpu) serverUnlockAllClients(&g_resServ); } gpumgrSetCurrentGpuInstance(NV_U32_MAX); } NV_STATUS RmGetAdapterStatus( nv_state_t *pNv, NvU32 *pStatus ) { // // This status is determined in RmInitAdapter(); the glue layer // requests it when the adapter failed to initialize to learn // more about the error condition. This is currently limited to // osVerifySystemEnvironment() failures. // nv_priv_t *nvp; nvp = NV_GET_NV_PRIV(pNv); if (nvp == NULL) { return NV_ERR_INVALID_STATE; } *pStatus = nvp->status; return NV_OK; } static void initVendorSpecificRegistry( OBJGPU *pGpu, NvU16 device_id ) { NV_STATUS rmStatus; NvU32 i; NvU32 subsystem_id; NvU32 subsystem_vendor_id; NvU32 subsystem_device_id; NvU32 vendor_id = 0; if (!pGpu) return; if (pGpu->bIsSOC) return; rmStatus = GPU_BUS_CFG_RD32(pGpu, NV_CONFIG_PCI_NV_11, &subsystem_id); if (rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "%s: Cannot read NV_CONFIG_PCI_NV_11\n", __FUNCTION__); return; } subsystem_vendor_id = (subsystem_id & 0xffff); subsystem_device_id = (subsystem_id >> 16); for (i = 0; (nb_id_table[i].subsystem_vendor_id) != 0; i++) { if ((nb_id_table[i].subsystem_vendor_id == subsystem_vendor_id) && (nb_id_table[i].subsystem_device_id == subsystem_device_id) && (nb_id_table[i].gpu_device_id == device_id)) { vendor_id = subsystem_vendor_id; break; } } if (vendor_id != 0) { for (i = 0; nb_reg_table[i].vendor_id != 0; i++) { if (nb_reg_table[i].vendor_id == vendor_id) { osWriteRegistryDword(pGpu, nb_reg_table[i].name, nb_reg_table[i].data); } } } } static void initUnixSpecificRegistry( OBJGPU *pGpu ) { // By default, enable GPU reset on Unix osWriteRegistryDword(pGpu, "RMSecBusResetEnable", 1); osWriteRegistryDword(pGpu, "RMForcePcieConfigSave", 1); } void osRemoveGpu( NvU32 domain, NvU8 bus, NvU8 device ) { void *handle; handle = os_pci_init_handle(domain, bus, device, 0, NULL, NULL); if (handle != NULL) { os_pci_remove(handle); } } /* * Check to see if this board supports GPU exclusion. */ static NvBool RmIsExcludingAllowed( nv_state_t *pNv ) { NvU32 feature = 0; NvU32 brand; // DGX-2/HGX-2 systems pre-date the PBI call if (pNv->pci_info.device_id == 0x1db8) return NV_TRUE; if (pciPbiGetFeature(pNv->handle, &feature) != NV_OK) return NV_FALSE; brand = DRF_VAL(_PBI, _EXECUTE_ROUTINE, _GET_FEATURE_EXCLUSION, feature); return (brand == NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION_ALLOWED); } NV_STATUS RmExcludeAdapter( nv_state_t *nv ) { NV_STATUS rmStatus; if (!RmIsExcludingAllowed(nv)) return NV_ERR_NOT_SUPPORTED; rmStatus = gpumgrExcludeGpuId(nv->gpu_id); if (rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "failed to exclude GPU: 0x%x\n", rmStatus); return rmStatus; } /* * GPU exclusion only happens during initialization, and therefore there * is no need to stop activity on the GPU */ return rmStatus; }