/*************************************************************************/ /*! @Title Misc memory management utility functions for Linux @Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved @License Dual MIT/GPLv2 The contents of this file are subject to the MIT license as set out below. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. Alternatively, the contents of this file may be used under the terms of the GNU General Public License Version 2 ("GPL") in which case the provisions of GPL are applicable instead of those above. If you wish to allow use of your version of this file only under the terms of GPL, and not to allow others to use your version of this file under the terms of the MIT license, indicate your decision by deleting the provisions above and replace them with the notice and other provisions required by GPL as set out in the file called "GPL-COPYING" included in this distribution. If you do not delete the provisions above, a recipient may use your version of this file under the terms of either the MIT license or GPL. This License is also included in this distribution in the file called "MIT-COPYING". EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /**************************************************************************/ #include #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38)) #ifndef AUTOCONF_INCLUDED #include #endif #endif #if !defined(PVR_LINUX_MEM_AREA_POOL_MAX_PAGES) #define PVR_LINUX_MEM_AREA_POOL_MAX_PAGES 0 #endif #include #include #include #include #include #include #include #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)) #include #endif #include #include #include #if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,1,0)) #include #endif #endif #include "img_defs.h" #include "services.h" #include "servicesint.h" #include "syscommon.h" #include "mutils.h" #include "mm.h" #include "pvrmmap.h" #include "mmap.h" #include "osfunc.h" #include "pvr_debug.h" #include "proc.h" #include "mutex.h" #include "lock.h" #if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) #include "lists.h" #endif /* If there is no explicit definition * for the minimum DMM alignment size, * then set it to "0" and let ION/DMM * set the minimum value. */ #ifndef CONFIG_TILER_GRANULARITY #define CONFIG_TILER_GRANULARITY 0 #endif /* * The page pool entry count is an atomic int so that the shrinker function * can return it even when we can't take the lock that protects the page pool * list. */ static atomic_t g_sPagePoolEntryCount = ATOMIC_INIT(0); #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) typedef enum { DEBUG_MEM_ALLOC_TYPE_KMALLOC, DEBUG_MEM_ALLOC_TYPE_VMALLOC, DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, DEBUG_MEM_ALLOC_TYPE_IOREMAP, DEBUG_MEM_ALLOC_TYPE_IO, DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE, DEBUG_MEM_ALLOC_TYPE_ION, #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) DEBUG_MEM_ALLOC_TYPE_VMAP, #endif DEBUG_MEM_ALLOC_TYPE_COUNT } DEBUG_MEM_ALLOC_TYPE; typedef struct _DEBUG_MEM_ALLOC_REC { DEBUG_MEM_ALLOC_TYPE eAllocType; IMG_VOID *pvKey; /* Some unique value (private to the eAllocType) */ IMG_VOID *pvCpuVAddr; IMG_UINT32 ulCpuPAddr; IMG_VOID *pvPrivateData; IMG_UINT32 ui32Bytes; pid_t pid; IMG_CHAR *pszFileName; IMG_UINT32 ui32Line; struct _DEBUG_MEM_ALLOC_REC *psNext; struct _DEBUG_MEM_ALLOC_REC **ppsThis; } DEBUG_MEM_ALLOC_REC; static IMPLEMENT_LIST_ANY_VA_2(DEBUG_MEM_ALLOC_REC, IMG_BOOL, IMG_FALSE) static IMPLEMENT_LIST_ANY_VA(DEBUG_MEM_ALLOC_REC) static IMPLEMENT_LIST_FOR_EACH(DEBUG_MEM_ALLOC_REC) static IMPLEMENT_LIST_INSERT(DEBUG_MEM_ALLOC_REC) static IMPLEMENT_LIST_REMOVE(DEBUG_MEM_ALLOC_REC) static DEBUG_MEM_ALLOC_REC *g_MemoryRecords; static IMG_UINT32 g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_COUNT]; static IMG_UINT32 g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_COUNT]; /* vmalloc + kmalloc + alloc_pages + kmem_cache */ static IMG_UINT32 g_SysRAMWaterMark; /* Doesn't include page pool */ static IMG_UINT32 g_SysRAMHighWaterMark; /* *DOES* include page pool */ static inline IMG_UINT32 SysRAMTrueWaterMark(void) { return g_SysRAMWaterMark + PAGES_TO_BYTES(atomic_read(&g_sPagePoolEntryCount)); } /* ioremap + io */ static IMG_UINT32 g_IOMemWaterMark; static IMG_UINT32 g_IOMemHighWaterMark; static IMG_VOID DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE eAllocType, IMG_VOID *pvKey, IMG_VOID *pvCpuVAddr, IMG_UINT32 ulCpuPAddr, IMG_VOID *pvPrivateData, IMG_UINT32 ui32Bytes, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line); static IMG_VOID DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE eAllocType, IMG_VOID *pvKey, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line); static IMG_CHAR *DebugMemAllocRecordTypeToString(DEBUG_MEM_ALLOC_TYPE eAllocType); static struct proc_dir_entry *g_SeqFileMemoryRecords; static void* ProcSeqNextMemoryRecords(struct seq_file *sfile,void* el,loff_t off); static void ProcSeqShowMemoryRecords(struct seq_file *sfile,void* el); static void* ProcSeqOff2ElementMemoryRecords(struct seq_file * sfile, loff_t off); #endif #if defined(DEBUG_LINUX_MEM_AREAS) typedef struct _DEBUG_LINUX_MEM_AREA_REC { LinuxMemArea *psLinuxMemArea; IMG_UINT32 ui32Flags; pid_t pid; struct _DEBUG_LINUX_MEM_AREA_REC *psNext; struct _DEBUG_LINUX_MEM_AREA_REC **ppsThis; }DEBUG_LINUX_MEM_AREA_REC; static IMPLEMENT_LIST_ANY_VA(DEBUG_LINUX_MEM_AREA_REC) static IMPLEMENT_LIST_FOR_EACH(DEBUG_LINUX_MEM_AREA_REC) static IMPLEMENT_LIST_INSERT(DEBUG_LINUX_MEM_AREA_REC) static IMPLEMENT_LIST_REMOVE(DEBUG_LINUX_MEM_AREA_REC) static DEBUG_LINUX_MEM_AREA_REC *g_LinuxMemAreaRecords; static IMG_UINT32 g_LinuxMemAreaCount; static IMG_UINT32 g_LinuxMemAreaWaterMark; static IMG_UINT32 g_LinuxMemAreaHighWaterMark; static struct proc_dir_entry *g_SeqFileMemArea; static void* ProcSeqNextMemArea(struct seq_file *sfile,void* el,loff_t off); static void ProcSeqShowMemArea(struct seq_file *sfile,void* el); static void* ProcSeqOff2ElementMemArea(struct seq_file *sfile, loff_t off); #endif #if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) static PVRSRV_LINUX_MUTEX g_sDebugMutex; #endif #if (defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)) static void ProcSeqStartstopDebugMutex(struct seq_file *sfile,IMG_BOOL start); #endif typedef struct { /* Linkage for page pool LRU list */ struct list_head sPagePoolItem; struct page *psPage; } LinuxPagePoolEntry; static LinuxKMemCache *g_PsLinuxMemAreaCache; static LinuxKMemCache *g_PsLinuxPagePoolCache; static LIST_HEAD(g_sPagePoolList); static int g_iPagePoolMaxEntries; #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) static IMG_VOID ReservePages(IMG_VOID *pvAddress, IMG_UINT32 ui32Length); static IMG_VOID UnreservePages(IMG_VOID *pvAddress, IMG_UINT32 ui32Length); #endif static LinuxMemArea *LinuxMemAreaStructAlloc(IMG_VOID); static IMG_VOID LinuxMemAreaStructFree(LinuxMemArea *psLinuxMemArea); #if defined(DEBUG_LINUX_MEM_AREAS) static IMG_VOID DebugLinuxMemAreaRecordAdd(LinuxMemArea *psLinuxMemArea, IMG_UINT32 ui32Flags); static DEBUG_LINUX_MEM_AREA_REC *DebugLinuxMemAreaRecordFind(LinuxMemArea *psLinuxMemArea); static IMG_VOID DebugLinuxMemAreaRecordRemove(LinuxMemArea *psLinuxMemArea); #endif static inline IMG_BOOL AreaIsUncached(IMG_UINT32 ui32AreaFlags) { return (ui32AreaFlags & (PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_UNCACHED)) != 0; } static inline IMG_BOOL CanFreeToPool(LinuxMemArea *psLinuxMemArea) { return AreaIsUncached(psLinuxMemArea->ui32AreaFlags) && !psLinuxMemArea->bNeedsCacheInvalidate; } IMG_VOID * _KMallocWrapper(IMG_UINT32 ui32ByteSize, gfp_t uFlags, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { IMG_VOID *pvRet; pvRet = kmalloc(ui32ByteSize, uFlags); #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) if (pvRet) { DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_KMALLOC, pvRet, pvRet, 0, NULL, ui32ByteSize, pszFileName, ui32Line ); } #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif return pvRet; } IMG_VOID _KFreeWrapper(IMG_VOID *pvCpuVAddr, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_KMALLOC, pvCpuVAddr, pszFileName, ui32Line); #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif kfree(pvCpuVAddr); } #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) static IMG_VOID DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE eAllocType, IMG_VOID *pvKey, IMG_VOID *pvCpuVAddr, IMG_UINT32 ulCpuPAddr, IMG_VOID *pvPrivateData, IMG_UINT32 ui32Bytes, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { DEBUG_MEM_ALLOC_REC *psRecord; LinuxLockMutex(&g_sDebugMutex); psRecord = kmalloc(sizeof(DEBUG_MEM_ALLOC_REC), GFP_KERNEL); psRecord->eAllocType = eAllocType; psRecord->pvKey = pvKey; psRecord->pvCpuVAddr = pvCpuVAddr; psRecord->ulCpuPAddr = ulCpuPAddr; psRecord->pvPrivateData = pvPrivateData; psRecord->pid = OSGetCurrentProcessIDKM(); psRecord->ui32Bytes = ui32Bytes; psRecord->pszFileName = pszFileName; psRecord->ui32Line = ui32Line; List_DEBUG_MEM_ALLOC_REC_Insert(&g_MemoryRecords, psRecord); g_WaterMarkData[eAllocType] += ui32Bytes; if (g_WaterMarkData[eAllocType] > g_HighWaterMarkData[eAllocType]) { g_HighWaterMarkData[eAllocType] = g_WaterMarkData[eAllocType]; } if (eAllocType == DEBUG_MEM_ALLOC_TYPE_KMALLOC || eAllocType == DEBUG_MEM_ALLOC_TYPE_VMALLOC || eAllocType == DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES || eAllocType == DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE) { IMG_UINT32 ui32SysRAMTrueWaterMark; g_SysRAMWaterMark += ui32Bytes; ui32SysRAMTrueWaterMark = SysRAMTrueWaterMark(); if (ui32SysRAMTrueWaterMark > g_SysRAMHighWaterMark) { g_SysRAMHighWaterMark = ui32SysRAMTrueWaterMark; } } else if (eAllocType == DEBUG_MEM_ALLOC_TYPE_IOREMAP || eAllocType == DEBUG_MEM_ALLOC_TYPE_IO) { g_IOMemWaterMark += ui32Bytes; if (g_IOMemWaterMark > g_IOMemHighWaterMark) { g_IOMemHighWaterMark = g_IOMemWaterMark; } } LinuxUnLockMutex(&g_sDebugMutex); } static IMG_BOOL DebugMemAllocRecordRemove_AnyVaCb(DEBUG_MEM_ALLOC_REC *psCurrentRecord, va_list va) { DEBUG_MEM_ALLOC_TYPE eAllocType; IMG_VOID *pvKey; eAllocType = va_arg(va, DEBUG_MEM_ALLOC_TYPE); pvKey = va_arg(va, IMG_VOID*); if (psCurrentRecord->eAllocType == eAllocType && psCurrentRecord->pvKey == pvKey) { eAllocType = psCurrentRecord->eAllocType; g_WaterMarkData[eAllocType] -= psCurrentRecord->ui32Bytes; if (eAllocType == DEBUG_MEM_ALLOC_TYPE_KMALLOC || eAllocType == DEBUG_MEM_ALLOC_TYPE_VMALLOC || eAllocType == DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES || eAllocType == DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE) { g_SysRAMWaterMark -= psCurrentRecord->ui32Bytes; } else if (eAllocType == DEBUG_MEM_ALLOC_TYPE_IOREMAP || eAllocType == DEBUG_MEM_ALLOC_TYPE_IO) { g_IOMemWaterMark -= psCurrentRecord->ui32Bytes; } List_DEBUG_MEM_ALLOC_REC_Remove(psCurrentRecord); kfree(psCurrentRecord); return IMG_TRUE; } else { return IMG_FALSE; } } static IMG_VOID DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE eAllocType, IMG_VOID *pvKey, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { /* DEBUG_MEM_ALLOC_REC **ppsCurrentRecord;*/ LinuxLockMutex(&g_sDebugMutex); /* Locate the corresponding allocation entry */ if (!List_DEBUG_MEM_ALLOC_REC_IMG_BOOL_Any_va(g_MemoryRecords, DebugMemAllocRecordRemove_AnyVaCb, eAllocType, pvKey)) { PVR_DPF((PVR_DBG_ERROR, "%s: couldn't find an entry for type=%s with pvKey=%p (called from %s, line %d\n", __FUNCTION__, DebugMemAllocRecordTypeToString(eAllocType), pvKey, pszFileName, ui32Line)); } LinuxUnLockMutex(&g_sDebugMutex); } static IMG_CHAR * DebugMemAllocRecordTypeToString(DEBUG_MEM_ALLOC_TYPE eAllocType) { IMG_CHAR *apszDebugMemoryRecordTypes[] = { "KMALLOC", "VMALLOC", "ALLOC_PAGES", "IOREMAP", "IO", "KMEM_CACHE_ALLOC", #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) "VMAP" #endif }; return apszDebugMemoryRecordTypes[eAllocType]; } #endif static IMG_BOOL AllocFlagsToPGProt(pgprot_t *pPGProtFlags, IMG_UINT32 ui32AllocFlags) { pgprot_t PGProtFlags; switch (ui32AllocFlags & PVRSRV_HAP_CACHETYPE_MASK) { case PVRSRV_HAP_CACHED: PGProtFlags = PAGE_KERNEL; break; case PVRSRV_HAP_WRITECOMBINE: PGProtFlags = PGPROT_WC(PAGE_KERNEL); break; case PVRSRV_HAP_UNCACHED: PGProtFlags = PGPROT_UC(PAGE_KERNEL); break; default: PVR_DPF((PVR_DBG_ERROR, "%s: Unknown mapping flags=0x%08x", __FUNCTION__, ui32AllocFlags)); dump_stack(); return IMG_FALSE; } *pPGProtFlags = PGProtFlags; return IMG_TRUE; } IMG_VOID * _VMallocWrapper(IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AllocFlags, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { pgprot_t PGProtFlags; IMG_VOID *pvRet; if (!AllocFlagsToPGProt(&PGProtFlags, ui32AllocFlags)) { return NULL; } /* Allocate virtually contiguous pages */ pvRet = __vmalloc(ui32Bytes, GFP_KERNEL | __GFP_HIGHMEM, PGProtFlags); #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) if (pvRet) { DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_VMALLOC, pvRet, pvRet, 0, NULL, PAGE_ALIGN(ui32Bytes), pszFileName, ui32Line ); } #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif return pvRet; } IMG_VOID _VFreeWrapper(IMG_VOID *pvCpuVAddr, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_VMALLOC, pvCpuVAddr, pszFileName, ui32Line); #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif vfree(pvCpuVAddr); } #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) static IMG_VOID * _VMapWrapper(struct page **ppsPageList, IMG_UINT32 ui32NumPages, IMG_UINT32 ui32AllocFlags, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { pgprot_t PGProtFlags; IMG_VOID *pvRet; if (!AllocFlagsToPGProt(&PGProtFlags, ui32AllocFlags)) { return NULL; } pvRet = vmap(ppsPageList, ui32NumPages, GFP_KERNEL | __GFP_HIGHMEM, PGProtFlags); #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) if (pvRet) { DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_VMAP, pvRet, pvRet, 0, NULL, PAGES_TO_BYTES(ui32NumPages), pszFileName, ui32Line ); } #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif return pvRet; } #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) #define VMapWrapper(ppsPageList, ui32Bytes, ui32AllocFlags) _VMapWrapper(ppsPageList, ui32Bytes, ui32AllocFlags, __FILE__, __LINE__) #else #define VMapWrapper(ppsPageList, ui32Bytes, ui32AllocFlags) _VMapWrapper(ppsPageList, ui32Bytes, ui32AllocFlags, NULL, 0) #endif static IMG_VOID _VUnmapWrapper(IMG_VOID *pvCpuVAddr, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_VMAP, pvCpuVAddr, pszFileName, ui32Line); #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif vunmap(pvCpuVAddr); } #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) #define VUnmapWrapper(pvCpuVAddr) _VUnmapWrapper(pvCpuVAddr, __FILE__, __LINE__) #else #define VUnmapWrapper(pvCpuVAddr) _VUnmapWrapper(pvCpuVAddr, NULL, 0) #endif #endif /* defined(PVR_LINUX_MEM_AREA_USE_VMAP) */ IMG_VOID _KMemCacheFreeWrapper(LinuxKMemCache *psCache, IMG_VOID *pvObject, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE, pvObject, pszFileName, ui32Line); #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif kmem_cache_free(psCache, pvObject); } const IMG_CHAR * KMemCacheNameWrapper(LinuxKMemCache *psCache) { PVR_UNREFERENCED_PARAMETER(psCache); /* In this case kmem_cache_t is an incomplete typedef, * so we can't even de-reference to get the name member. It is also a GPL export symbol */ return ""; } static LinuxPagePoolEntry * LinuxPagePoolEntryAlloc(IMG_VOID) { return KMemCacheAllocWrapper(g_PsLinuxPagePoolCache, GFP_KERNEL); } static IMG_VOID LinuxPagePoolEntryFree(LinuxPagePoolEntry *psPagePoolEntry) { KMemCacheFreeWrapper(g_PsLinuxPagePoolCache, psPagePoolEntry); } static struct page * AllocPageFromLinux(void) { struct page *psPage; psPage = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM, 0); if (!psPage) { return NULL; } #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) /* Reserve those pages to allow them to be re-mapped to user space */ #if (LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)) SetPageReserved(psPage); #else mem_map_reserve(psPage); #endif #endif return psPage; } static IMG_VOID FreePageToLinux(struct page *psPage) { #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) #if (LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)) ClearPageReserved(psPage); #else mem_map_reserve(psPage); #endif #endif __free_pages(psPage, 0); } #if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) static DEFINE_MUTEX(g_sPagePoolMutex); static inline void PagePoolLock(void) { mutex_lock(&g_sPagePoolMutex); } static inline void PagePoolUnlock(void) { mutex_unlock(&g_sPagePoolMutex); } static inline int PagePoolTrylock(void) { return mutex_trylock(&g_sPagePoolMutex); } #else /* (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) */ static inline void PagePoolLock(void) { } static inline void PagePoolUnlock(void) { } static inline int PagePoolTrylock(void) { return 1; } #endif /* (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) */ static inline void AddEntryToPool(LinuxPagePoolEntry *psPagePoolEntry) { list_add_tail(&psPagePoolEntry->sPagePoolItem, &g_sPagePoolList); atomic_inc(&g_sPagePoolEntryCount); } static inline void RemoveEntryFromPool(LinuxPagePoolEntry *psPagePoolEntry) { list_del(&psPagePoolEntry->sPagePoolItem); atomic_dec(&g_sPagePoolEntryCount); } static inline LinuxPagePoolEntry * RemoveFirstEntryFromPool(void) { LinuxPagePoolEntry *psPagePoolEntry; if (list_empty(&g_sPagePoolList)) { PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) == 0); return NULL; } PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) > 0); psPagePoolEntry = list_first_entry(&g_sPagePoolList, LinuxPagePoolEntry, sPagePoolItem); RemoveEntryFromPool(psPagePoolEntry); return psPagePoolEntry; } static struct page * AllocPage(IMG_UINT32 ui32AreaFlags, IMG_BOOL *pbFromPagePool) { struct page *psPage = NULL; /* * Only uncached allocations can come from the page pool. * The page pool is currently used to reduce the cost of * invalidating the CPU cache when uncached memory is allocated. */ if (AreaIsUncached(ui32AreaFlags) && atomic_read(&g_sPagePoolEntryCount) != 0) { LinuxPagePoolEntry *psPagePoolEntry; PagePoolLock(); psPagePoolEntry = RemoveFirstEntryFromPool(); PagePoolUnlock(); /* List may have changed since we checked the counter */ if (psPagePoolEntry) { psPage = psPagePoolEntry->psPage; LinuxPagePoolEntryFree(psPagePoolEntry); *pbFromPagePool = IMG_TRUE; } } if (!psPage) { psPage = AllocPageFromLinux(); if (psPage) { *pbFromPagePool = IMG_FALSE; } } return psPage; } static IMG_VOID FreePage(IMG_BOOL bToPagePool, struct page *psPage) { /* Only uncached allocations can be freed to the page pool */ if (bToPagePool && atomic_read(&g_sPagePoolEntryCount) < g_iPagePoolMaxEntries) { LinuxPagePoolEntry *psPagePoolEntry = LinuxPagePoolEntryAlloc(); if (psPagePoolEntry) { psPagePoolEntry->psPage = psPage; PagePoolLock(); AddEntryToPool(psPagePoolEntry); PagePoolUnlock(); return; } } FreePageToLinux(psPage); } static IMG_VOID FreePagePool(IMG_VOID) { LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry; PagePoolLock(); #if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) PVR_DPF((PVR_DBG_MESSAGE,"%s: Freeing %d pages from pool", __FUNCTION__, atomic_read(&g_sPagePoolEntryCount))); #else PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) == 0); PVR_ASSERT(list_empty(&g_sPagePoolList)); #endif list_for_each_entry_safe(psPagePoolEntry, psTempPoolEntry, &g_sPagePoolList, sPagePoolItem) { RemoveEntryFromPool(psPagePoolEntry); FreePageToLinux(psPagePoolEntry->psPage); LinuxPagePoolEntryFree(psPagePoolEntry); } PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) == 0); PagePoolUnlock(); } #if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) #if defined(PVRSRV_NEED_PVR_ASSERT) static struct shrinker g_sShrinker; #endif static int ShrinkPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl) { unsigned long uNumToScan = psShrinkControl->nr_to_scan; PVR_ASSERT(psShrinker == &g_sShrinker); (void)psShrinker; if (uNumToScan != 0) { LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry; PVR_DPF((PVR_DBG_MESSAGE,"%s: Number to scan: %ld", __FUNCTION__, uNumToScan)); PVR_DPF((PVR_DBG_MESSAGE,"%s: Pages in pool before scan: %d", __FUNCTION__, atomic_read(&g_sPagePoolEntryCount))); if (!PagePoolTrylock()) { PVR_TRACE(("%s: Couldn't get page pool lock", __FUNCTION__)); return -1; } list_for_each_entry_safe(psPagePoolEntry, psTempPoolEntry, &g_sPagePoolList, sPagePoolItem) { RemoveEntryFromPool(psPagePoolEntry); FreePageToLinux(psPagePoolEntry->psPage); LinuxPagePoolEntryFree(psPagePoolEntry); if (--uNumToScan == 0) { break; } } if (list_empty(&g_sPagePoolList)) { PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) == 0); } PagePoolUnlock(); PVR_DPF((PVR_DBG_MESSAGE,"%s: Pages in pool after scan: %d", __FUNCTION__, atomic_read(&g_sPagePoolEntryCount))); } return atomic_read(&g_sPagePoolEntryCount); } #endif static IMG_BOOL AllocPages(IMG_UINT32 ui32AreaFlags, struct page ***pppsPageList, IMG_HANDLE *phBlockPageList, IMG_UINT32 ui32NumPages, IMG_BOOL *pbFromPagePool) { struct page **ppsPageList; IMG_HANDLE hBlockPageList; IMG_INT32 i; /* Must be signed; see "for" loop conditions */ PVRSRV_ERROR eError; IMG_BOOL bFromPagePool = IMG_FALSE; eError = OSAllocMem(0, sizeof(*ppsPageList) * ui32NumPages, (IMG_VOID **)&ppsPageList, &hBlockPageList, "Array of pages"); if (eError != PVRSRV_OK) { goto failed_page_list_alloc; } *pbFromPagePool = IMG_TRUE; for(i = 0; i < (IMG_INT32)ui32NumPages; i++) { ppsPageList[i] = AllocPage(ui32AreaFlags, &bFromPagePool); if (!ppsPageList[i]) { goto failed_alloc_pages; } *pbFromPagePool &= bFromPagePool; } *pppsPageList = ppsPageList; *phBlockPageList = hBlockPageList; #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, ppsPageList, 0, 0, NULL, PAGES_TO_BYTES(ui32NumPages), "unknown", 0 ); #endif return IMG_TRUE; failed_alloc_pages: for(i--; i >= 0; i--) { FreePage(*pbFromPagePool, ppsPageList[i]); } (IMG_VOID) OSFreeMem(0, sizeof(*ppsPageList) * ui32NumPages, ppsPageList, hBlockPageList); failed_page_list_alloc: return IMG_FALSE; } static IMG_VOID FreePages(IMG_BOOL bToPagePool, struct page **ppsPageList, IMG_HANDLE hBlockPageList, IMG_UINT32 ui32NumPages) { IMG_INT32 i; for(i = 0; i < (IMG_INT32)ui32NumPages; i++) { FreePage(bToPagePool, ppsPageList[i]); } #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, ppsPageList, __FILE__, __LINE__); #endif (IMG_VOID) OSFreeMem(0, sizeof(*ppsPageList) * ui32NumPages, ppsPageList, hBlockPageList); } LinuxMemArea * NewVMallocLinuxMemArea(IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AreaFlags) { LinuxMemArea *psLinuxMemArea = NULL; IMG_VOID *pvCpuVAddr; #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) IMG_UINT32 ui32NumPages = 0; struct page **ppsPageList = NULL; IMG_HANDLE hBlockPageList; #endif IMG_BOOL bFromPagePool = IMG_FALSE; psLinuxMemArea = LinuxMemAreaStructAlloc(); if (!psLinuxMemArea) { goto failed; } #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) ui32NumPages = RANGE_TO_PAGES(ui32Bytes); if (!AllocPages(ui32AreaFlags, &ppsPageList, &hBlockPageList, ui32NumPages, &bFromPagePool)) { goto failed; } pvCpuVAddr = VMapWrapper(ppsPageList, ui32NumPages, ui32AreaFlags); #else /* defined(PVR_LINUX_MEM_AREA_USE_VMAP) */ pvCpuVAddr = VMallocWrapper(ui32Bytes, ui32AreaFlags); if (!pvCpuVAddr) { goto failed; } /* PG_reserved was deprecated in linux-2.6.15 */ #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) /* Reserve those pages to allow them to be re-mapped to user space */ ReservePages(pvCpuVAddr, ui32Bytes); #endif #endif /* defined(PVR_LINUX_MEM_AREA_USE_VMAP) */ psLinuxMemArea->eAreaType = LINUX_MEM_AREA_VMALLOC; psLinuxMemArea->uData.sVmalloc.pvVmallocAddress = pvCpuVAddr; #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) psLinuxMemArea->uData.sVmalloc.ppsPageList = ppsPageList; psLinuxMemArea->uData.sVmalloc.hBlockPageList = hBlockPageList; #endif psLinuxMemArea->ui32ByteSize = ui32Bytes; psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); #endif /* This works around a problem where Linux will not invalidate * the cache for physical memory it frees that is direct mapped. * * As a result, cache entries remain that may be subsequently flushed * to these physical pages after they have been allocated for another * purpose. For a subsequent cached use of this memory, that is not a * problem, but if we are allocating uncached or write-combined memory, * and bypassing the cache, it can cause subsequent uncached writes to * the memory to be replaced with junk from the cache. * * If the pages are from our page cache, no cache invalidate is needed. * * This just handles the __vmalloc() case (when we have a kernel virtual * address range). The alloc_pages() path is handled in mmap.c. */ if (AreaIsUncached(ui32AreaFlags) && !bFromPagePool) { OSInvalidateCPUCacheRangeKM(psLinuxMemArea, 0, pvCpuVAddr, ui32Bytes); } return psLinuxMemArea; failed: PVR_DPF((PVR_DBG_ERROR, "%s: failed!", __FUNCTION__)); #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) if (ppsPageList) { FreePages(bFromPagePool, ppsPageList, hBlockPageList, ui32NumPages); } #endif if (psLinuxMemArea) { LinuxMemAreaStructFree(psLinuxMemArea); } return NULL; } IMG_VOID FreeVMallocLinuxMemArea(LinuxMemArea *psLinuxMemArea) { #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) IMG_UINT32 ui32NumPages; struct page **ppsPageList; IMG_HANDLE hBlockPageList; #endif PVR_ASSERT(psLinuxMemArea); PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_VMALLOC); PVR_ASSERT(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordRemove(psLinuxMemArea); #endif PVR_DPF((PVR_DBG_MESSAGE,"%s: pvCpuVAddr: %p", __FUNCTION__, psLinuxMemArea->uData.sVmalloc.pvVmallocAddress)); #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) VUnmapWrapper(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress); ui32NumPages = RANGE_TO_PAGES(psLinuxMemArea->ui32ByteSize); ppsPageList = psLinuxMemArea->uData.sVmalloc.ppsPageList; hBlockPageList = psLinuxMemArea->uData.sVmalloc.hBlockPageList; FreePages(CanFreeToPool(psLinuxMemArea), ppsPageList, hBlockPageList, ui32NumPages); #else /* PG_reserved was deprecated in linux-2.6.15 */ #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) UnreservePages(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress, psLinuxMemArea->ui32ByteSize); #endif VFreeWrapper(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress); #endif /* defined(PVR_LINUX_MEM_AREA_USE_VMAP) */ LinuxMemAreaStructFree(psLinuxMemArea); } #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) /* Reserve pages of memory in order that they're not automatically deallocated after the last user reference dies. */ static IMG_VOID ReservePages(IMG_VOID *pvAddress, IMG_UINT32 ui32Length) { IMG_VOID *pvPage; IMG_VOID *pvEnd = pvAddress + ui32Length; for(pvPage = pvAddress; pvPage < pvEnd; pvPage += PAGE_SIZE) { #if (LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)) SetPageReserved(vmalloc_to_page(pvPage)); #else mem_map_reserve(vmalloc_to_page(pvPage)); #endif } } /* Un-reserve pages of memory in order that they can be freed. */ static IMG_VOID UnreservePages(IMG_VOID *pvAddress, IMG_UINT32 ui32Length) { IMG_VOID *pvPage; IMG_VOID *pvEnd = pvAddress + ui32Length; for(pvPage = pvAddress; pvPage < pvEnd; pvPage += PAGE_SIZE) { #if (LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)) ClearPageReserved(vmalloc_to_page(pvPage)); #else mem_map_unreserve(vmalloc_to_page(pvPage)); #endif } } #endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) */ IMG_VOID * _IORemapWrapper(IMG_CPU_PHYADDR BasePAddr, IMG_UINT32 ui32Bytes, IMG_UINT32 ui32MappingFlags, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { IMG_VOID *pvIORemapCookie; switch (ui32MappingFlags & PVRSRV_HAP_CACHETYPE_MASK) { case PVRSRV_HAP_CACHED: pvIORemapCookie = (IMG_VOID *)IOREMAP(BasePAddr.uiAddr, ui32Bytes); break; case PVRSRV_HAP_WRITECOMBINE: pvIORemapCookie = (IMG_VOID *)IOREMAP_WC(BasePAddr.uiAddr, ui32Bytes); break; case PVRSRV_HAP_UNCACHED: pvIORemapCookie = (IMG_VOID *)IOREMAP_UC(BasePAddr.uiAddr, ui32Bytes); break; default: PVR_DPF((PVR_DBG_ERROR, "IORemapWrapper: unknown mapping flags")); return NULL; } #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) if (pvIORemapCookie) { DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_IOREMAP, pvIORemapCookie, pvIORemapCookie, BasePAddr.uiAddr, NULL, ui32Bytes, pszFileName, ui32Line ); } #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif return pvIORemapCookie; } IMG_VOID _IOUnmapWrapper(IMG_VOID *pvIORemapCookie, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_IOREMAP, pvIORemapCookie, pszFileName, ui32Line); #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif iounmap(pvIORemapCookie); } LinuxMemArea * NewIORemapLinuxMemArea(IMG_CPU_PHYADDR BasePAddr, IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AreaFlags) { LinuxMemArea *psLinuxMemArea; IMG_VOID *pvIORemapCookie; psLinuxMemArea = LinuxMemAreaStructAlloc(); if (!psLinuxMemArea) { return NULL; } pvIORemapCookie = IORemapWrapper(BasePAddr, ui32Bytes, ui32AreaFlags); if (!pvIORemapCookie) { LinuxMemAreaStructFree(psLinuxMemArea); return NULL; } psLinuxMemArea->eAreaType = LINUX_MEM_AREA_IOREMAP; psLinuxMemArea->uData.sIORemap.pvIORemapCookie = pvIORemapCookie; psLinuxMemArea->uData.sIORemap.CPUPhysAddr = BasePAddr; psLinuxMemArea->ui32ByteSize = ui32Bytes; psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); #endif return psLinuxMemArea; } IMG_VOID FreeIORemapLinuxMemArea(LinuxMemArea *psLinuxMemArea) { PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_IOREMAP); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordRemove(psLinuxMemArea); #endif IOUnmapWrapper(psLinuxMemArea->uData.sIORemap.pvIORemapCookie); LinuxMemAreaStructFree(psLinuxMemArea); } #if !defined(PVR_MAKE_ALL_PFNS_SPECIAL) /* * Avoid using remap_pfn_range on RAM, if possible. On x86 systems, with * PAT enabled, remap_pfn_range checks the page attributes requested by * remap_pfn_range against those of the direct kernel mapping for those * pages (if any). This is rather annoying if the pages have been obtained * with alloc_pages, where we just ask for raw pages; we don't care about * the direct mapping. This latter issue arises when device memory is * exported from one process to another. Services implements this * using memory wrapping, which ends up creating an external KV memory area. */ static IMG_BOOL TreatExternalPagesAsContiguous(IMG_SYS_PHYADDR *psSysPhysAddr, IMG_UINT32 ui32Bytes, IMG_BOOL bPhysContig) { IMG_UINT32 ui32; IMG_UINT32 ui32AddrChk; IMG_UINT32 ui32NumPages = RANGE_TO_PAGES(ui32Bytes); /* * If bPhysContig is IMG_TRUE, we must assume psSysPhysAddr points * to the address of the first page, not an array of page addresses. */ for (ui32 = 0, ui32AddrChk = psSysPhysAddr[0].uiAddr; ui32 < ui32NumPages; ui32++, ui32AddrChk = (bPhysContig) ? (ui32AddrChk + PAGE_SIZE) : psSysPhysAddr[ui32].uiAddr) { if (!pfn_valid(PHYS_TO_PFN(ui32AddrChk))) { break; } } if (ui32 == ui32NumPages) { return IMG_FALSE; } if (!bPhysContig) { for (ui32 = 0, ui32AddrChk = psSysPhysAddr[0].uiAddr; ui32 < ui32NumPages; ui32++, ui32AddrChk += PAGE_SIZE) { if (psSysPhysAddr[ui32].uiAddr != ui32AddrChk) { return IMG_FALSE; } } } return IMG_TRUE; } #endif LinuxMemArea *NewExternalKVLinuxMemArea(IMG_SYS_PHYADDR *pBasePAddr, IMG_VOID *pvCPUVAddr, IMG_UINT32 ui32Bytes, IMG_BOOL bPhysContig, IMG_UINT32 ui32AreaFlags) { LinuxMemArea *psLinuxMemArea; psLinuxMemArea = LinuxMemAreaStructAlloc(); if (!psLinuxMemArea) { return NULL; } psLinuxMemArea->eAreaType = LINUX_MEM_AREA_EXTERNAL_KV; psLinuxMemArea->uData.sExternalKV.pvExternalKV = pvCPUVAddr; psLinuxMemArea->uData.sExternalKV.bPhysContig = #if !defined(PVR_MAKE_ALL_PFNS_SPECIAL) (bPhysContig || TreatExternalPagesAsContiguous(pBasePAddr, ui32Bytes, bPhysContig)) ? IMG_TRUE : IMG_FALSE; #else bPhysContig; #endif if (psLinuxMemArea->uData.sExternalKV.bPhysContig) { psLinuxMemArea->uData.sExternalKV.uPhysAddr.SysPhysAddr = *pBasePAddr; } else { psLinuxMemArea->uData.sExternalKV.uPhysAddr.pSysPhysAddr = pBasePAddr; } psLinuxMemArea->ui32ByteSize = ui32Bytes; psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); #endif return psLinuxMemArea; } IMG_VOID FreeExternalKVLinuxMemArea(LinuxMemArea *psLinuxMemArea) { PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_EXTERNAL_KV); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordRemove(psLinuxMemArea); #endif LinuxMemAreaStructFree(psLinuxMemArea); } LinuxMemArea * NewIOLinuxMemArea(IMG_CPU_PHYADDR BasePAddr, IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AreaFlags) { LinuxMemArea *psLinuxMemArea = LinuxMemAreaStructAlloc(); if (!psLinuxMemArea) { return NULL; } /* Nothing to activly do. We just keep a record of the physical range. */ psLinuxMemArea->eAreaType = LINUX_MEM_AREA_IO; psLinuxMemArea->uData.sIO.CPUPhysAddr.uiAddr = BasePAddr.uiAddr; psLinuxMemArea->ui32ByteSize = ui32Bytes; psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_IO, (IMG_VOID *)BasePAddr.uiAddr, 0, BasePAddr.uiAddr, NULL, ui32Bytes, "unknown", 0 ); #endif #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); #endif return psLinuxMemArea; } IMG_VOID FreeIOLinuxMemArea(LinuxMemArea *psLinuxMemArea) { PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_IO); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordRemove(psLinuxMemArea); #endif #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_IO, (IMG_VOID *)psLinuxMemArea->uData.sIO.CPUPhysAddr.uiAddr, __FILE__, __LINE__); #endif /* Nothing more to do than free the LinuxMemArea struct */ LinuxMemAreaStructFree(psLinuxMemArea); } LinuxMemArea * NewAllocPagesLinuxMemArea(IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AreaFlags) { LinuxMemArea *psLinuxMemArea; IMG_UINT32 ui32NumPages; struct page **ppsPageList; IMG_HANDLE hBlockPageList; IMG_BOOL bFromPagePool; psLinuxMemArea = LinuxMemAreaStructAlloc(); if (!psLinuxMemArea) { goto failed_area_alloc; } ui32NumPages = RANGE_TO_PAGES(ui32Bytes); if (!AllocPages(ui32AreaFlags, &ppsPageList, &hBlockPageList, ui32NumPages, &bFromPagePool)) { goto failed_alloc_pages; } psLinuxMemArea->eAreaType = LINUX_MEM_AREA_ALLOC_PAGES; psLinuxMemArea->uData.sPageList.ppsPageList = ppsPageList; psLinuxMemArea->uData.sPageList.hBlockPageList = hBlockPageList; psLinuxMemArea->ui32ByteSize = ui32Bytes; psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); /* We defer the cache flush to the first user mapping of this memory */ psLinuxMemArea->bNeedsCacheInvalidate = AreaIsUncached(ui32AreaFlags) && !bFromPagePool; #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); #endif return psLinuxMemArea; failed_alloc_pages: LinuxMemAreaStructFree(psLinuxMemArea); failed_area_alloc: PVR_DPF((PVR_DBG_ERROR, "%s: failed", __FUNCTION__)); return NULL; } IMG_VOID FreeAllocPagesLinuxMemArea(LinuxMemArea *psLinuxMemArea) { IMG_UINT32 ui32NumPages; struct page **ppsPageList; IMG_HANDLE hBlockPageList; PVR_ASSERT(psLinuxMemArea); PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_ALLOC_PAGES); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordRemove(psLinuxMemArea); #endif ui32NumPages = RANGE_TO_PAGES(psLinuxMemArea->ui32ByteSize); ppsPageList = psLinuxMemArea->uData.sPageList.ppsPageList; hBlockPageList = psLinuxMemArea->uData.sPageList.hBlockPageList; FreePages(CanFreeToPool(psLinuxMemArea), ppsPageList, hBlockPageList, ui32NumPages); LinuxMemAreaStructFree(psLinuxMemArea); } #if defined(CONFIG_ION_OMAP) #include "env_perproc.h" #include #include #include extern struct ion_client *gpsIONClient; LinuxMemArea * NewIONLinuxMemArea(IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AreaFlags, IMG_PVOID pvPrivData, IMG_UINT32 ui32PrivDataLength) { const IMG_UINT32 ui32AllocDataLen = offsetof(struct omap_ion_tiler_alloc_data, handle); struct omap_ion_tiler_alloc_data asAllocData[PVRSRV_MAX_NUMBER_OF_MM_BUFFER_PLANES]; u32 *pu32PageAddrs[PVRSRV_MAX_NUMBER_OF_MM_BUFFER_PLANES] = { NULL, NULL, NULL}; IMG_UINT32 i, j, ui32NumHandlesPerFd; IMG_BYTE *pbPrivData = pvPrivData; IMG_CPU_PHYADDR *pCPUPhysAddrs; IMG_UINT32 iNumPages[PVRSRV_MAX_NUMBER_OF_MM_BUFFER_PLANES] = { 0, 0, 0}; LinuxMemArea *psLinuxMemArea; IMG_UINT32 ui32ProcID; IMG_UINT32 ui32TotalPagesSizeInBytes = 0, ui32TotalPages = 0; psLinuxMemArea = LinuxMemAreaStructAlloc(); if (!psLinuxMemArea) { PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate LinuxMemArea struct", __func__)); goto err_out; } /* Depending on the UM config, userspace might give us info for * one, two or three ION allocations. Divide the total size of data we * were given by this ui32AllocDataLen, and check it's 1 or 2. * Otherwise abort. */ BUG_ON(ui32PrivDataLength != ui32AllocDataLen && ui32PrivDataLength != ui32AllocDataLen * 2 && ui32PrivDataLength != ui32AllocDataLen * 3); /* This is bad !- change this logic to pass in the size or * use uniformed API */ ui32NumHandlesPerFd = ui32PrivDataLength / ui32AllocDataLen; ui32ProcID = OSGetCurrentProcessIDKM(); memset(asAllocData, 0x00, sizeof(asAllocData)); /* We do not care about what the first (Y) buffer offset would be, * but we do care for the UV buffers to be co-aligned with Y * This for SGX to find the UV offset solely based on the height * and stride of the YUV buffer.This is very important for OMAP4470 * and later chipsets, where SGX version is 544. 544 and later use * non-shader based YUV to RGB conversion unit that require * contiguous GPU virtual space */ for(i = 0; i < ui32NumHandlesPerFd; i++) { memcpy(&asAllocData[i], &pbPrivData[i * ui32AllocDataLen], ui32AllocDataLen); asAllocData[i].token = ui32ProcID; #ifndef SGX_DISABLE_DMM_OFFSET_BUFFER_ALLOCATIONS if(i == 0) { /* Tiler API says: * Allocate first buffer with the required alignment * and an offset of 0 ... */ asAllocData[i].out_align = CONFIG_TILER_GRANULARITY; asAllocData[i].offset = 0; } else { /* .. Then for the second buffer, use the offset from the first * buffer with alignment of PAGE_SIZE */ asAllocData[i].out_align = PAGE_SIZE; asAllocData[i].offset = asAllocData[0].offset; } #else asAllocData[i].offset = 0; asAllocData[i].out_align = PAGE_SIZE; #endif if(asAllocData[i].fmt == TILER_PIXEL_FMT_PAGE) { /* 1D DMM Buffers */ struct scatterlist *sg, *sglist; IMG_UINT32 ui32Num1dPages; asAllocData[i].handle = ion_alloc (gpsIONClient, ui32Bytes, PAGE_SIZE, (1 << OMAP_ION_HEAP_SYSTEM)); if (asAllocData[i].handle == NULL) { PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate via ion_alloc", __func__)); goto err_free; } sglist = ion_map_dma (gpsIONClient, asAllocData[i].handle); if (sglist == NULL) { PVR_DPF((PVR_DBG_ERROR, "%s: Failed to compute pages", __func__)); goto err_free; } ui32Num1dPages = (ui32Bytes >> PAGE_SHIFT); pu32PageAddrs[i] = kmalloc (sizeof(u32) * ui32Num1dPages, GFP_KERNEL); if (pu32PageAddrs[i] == NULL) { PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate page array", __func__)); goto err_free; } for_each_sg (sglist, sg, ui32Num1dPages, j) { pu32PageAddrs[i][j] = sg_phys (sg); } iNumPages[i] = ui32Num1dPages; } else /* 2D DMM Buffers */ { if (omap_ion_tiler_alloc(gpsIONClient, &asAllocData[i]) < 0) { PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate via ion_tiler", __func__)); goto err_free; } if (omap_tiler_pages(gpsIONClient, asAllocData[i].handle, &iNumPages[i], &pu32PageAddrs[i]) < 0) { PVR_DPF((PVR_DBG_ERROR, "%s: Failed to compute tiler pages", __func__)); goto err_free; } } } /* Basic sanity check on plane co-alignment */ if((ui32NumHandlesPerFd > 1) && (asAllocData[0].offset != asAllocData[1].offset)) { pr_err("%s: Y and UV offsets do not match for tiler handles " "%p,%p: %d != %d \n " "Expect issues with SGX544xx and later chipsets\n", __func__, asAllocData[0].handle, asAllocData[1].handle, (int)asAllocData[0].offset, (int)asAllocData[1].offset); } /* Assume the user-allocator has already done the tiler math and that the * number of tiler pages allocated matches any other allocation type. */ for(i = 0; i < ui32NumHandlesPerFd; i++) { ui32TotalPages += iNumPages[i]; } BUG_ON(ui32Bytes != (ui32TotalPages * PAGE_SIZE)); BUG_ON(sizeof(IMG_CPU_PHYADDR) != sizeof(int)); /* Glue the page lists together */ pCPUPhysAddrs = vmalloc(sizeof(IMG_CPU_PHYADDR) * ui32TotalPages); if (!pCPUPhysAddrs) { PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate page list", __func__)); goto err_free; } j = 0; for(i = 0; i < ui32NumHandlesPerFd; i++) { IMG_UINT32 ui32PageIndx; for(ui32PageIndx = 0; ui32PageIndx < iNumPages[i]; ui32PageIndx++) { pCPUPhysAddrs[j++].uiAddr = pu32PageAddrs[i][ui32PageIndx]; } psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i] = asAllocData[i].handle; psLinuxMemArea->uData.sIONTilerAlloc.planeOffsets[i] = ui32TotalPagesSizeInBytes + asAllocData[i].offset; /* Add the number of pages this plane consists of */ ui32TotalPagesSizeInBytes += (iNumPages[i] * PAGE_SIZE); } psLinuxMemArea->eAreaType = LINUX_MEM_AREA_ION; psLinuxMemArea->uData.sIONTilerAlloc.pCPUPhysAddrs = pCPUPhysAddrs; psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes = ui32NumHandlesPerFd; psLinuxMemArea->ui32ByteSize = ui32TotalPagesSizeInBytes; psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); /* We defer the cache flush to the first user mapping of this memory */ psLinuxMemArea->bNeedsCacheInvalidate = AreaIsUncached(ui32AreaFlags); #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_ION, asAllocData[0].handle, 0, 0, NULL, PAGE_ALIGN(ui32Bytes), "unknown", 0 ); #endif #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); #endif err_out: return psLinuxMemArea; err_free: LinuxMemAreaStructFree(psLinuxMemArea); psLinuxMemArea = IMG_NULL; goto err_out; } IMG_INT32 GetIONLinuxMemAreaInfo(LinuxMemArea *psLinuxMemArea, IMG_UINT32* pui32AddressOffsets, IMG_UINT32* ui32NumAddrOffsets) { IMG_UINT32 i; if(!ui32NumAddrOffsets) return -1; if(*ui32NumAddrOffsets < psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes) { *ui32NumAddrOffsets = psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes; return -1; } if(!pui32AddressOffsets) return -1; for(i = 0; i < psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes; i++) { if(psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i]) pui32AddressOffsets[i] = psLinuxMemArea->uData.sIONTilerAlloc.planeOffsets[i]; } *ui32NumAddrOffsets = psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes; return psLinuxMemArea->ui32ByteSize; } IMG_VOID FreeIONLinuxMemArea(LinuxMemArea *psLinuxMemArea) { IMG_UINT32 i; #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordRemove(psLinuxMemArea); #endif #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_ION, psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[0], __FILE__, __LINE__); #endif for(i = 0; i < psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes; i++) { if (!psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i]) break; ion_free(gpsIONClient, psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i]); psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i] = IMG_NULL; } /* free copy of page list, originals are freed by ion_free */ vfree(psLinuxMemArea->uData.sIONTilerAlloc.pCPUPhysAddrs); psLinuxMemArea->uData.sIONTilerAlloc.pCPUPhysAddrs = IMG_NULL; LinuxMemAreaStructFree(psLinuxMemArea); } #endif /* defined(CONFIG_ION_OMAP) */ struct page* LinuxMemAreaOffsetToPage(LinuxMemArea *psLinuxMemArea, IMG_UINT32 ui32ByteOffset) { IMG_UINT32 ui32PageIndex; IMG_CHAR *pui8Addr; switch (psLinuxMemArea->eAreaType) { case LINUX_MEM_AREA_ALLOC_PAGES: ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset); return psLinuxMemArea->uData.sPageList.ppsPageList[ui32PageIndex]; case LINUX_MEM_AREA_VMALLOC: pui8Addr = psLinuxMemArea->uData.sVmalloc.pvVmallocAddress; pui8Addr += ui32ByteOffset; return vmalloc_to_page(pui8Addr); case LINUX_MEM_AREA_SUB_ALLOC: /* PRQA S 3670 3 */ /* ignore recursive warning */ return LinuxMemAreaOffsetToPage(psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea, psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset + ui32ByteOffset); default: PVR_DPF((PVR_DBG_ERROR, "%s: Unsupported request for struct page from LinuxMemArea with type=%s", __FUNCTION__, LinuxMemAreaTypeToString(psLinuxMemArea->eAreaType))); return NULL; } } LinuxKMemCache * KMemCacheCreateWrapper(IMG_CHAR *pszName, size_t Size, size_t Align, IMG_UINT32 ui32Flags) { #if defined(DEBUG_LINUX_SLAB_ALLOCATIONS) ui32Flags |= SLAB_POISON|SLAB_RED_ZONE; #endif return kmem_cache_create(pszName, Size, Align, ui32Flags, NULL #if (LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,22)) , NULL #endif /* (LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,22) */ ); } IMG_VOID KMemCacheDestroyWrapper(LinuxKMemCache *psCache) { kmem_cache_destroy(psCache); } IMG_VOID * _KMemCacheAllocWrapper(LinuxKMemCache *psCache, #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,14)) gfp_t Flags, #else IMG_INT Flags, #endif IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) { IMG_VOID *pvRet; pvRet = kmem_cache_zalloc(psCache, Flags); #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE, pvRet, pvRet, 0, psCache, kmem_cache_size(psCache), pszFileName, ui32Line ); #else PVR_UNREFERENCED_PARAMETER(pszFileName); PVR_UNREFERENCED_PARAMETER(ui32Line); #endif return pvRet; } LinuxMemArea * NewSubLinuxMemArea(LinuxMemArea *psParentLinuxMemArea, IMG_UINT32 ui32ByteOffset, IMG_UINT32 ui32Bytes) { LinuxMemArea *psLinuxMemArea; PVR_ASSERT((ui32ByteOffset+ui32Bytes) <= psParentLinuxMemArea->ui32ByteSize); psLinuxMemArea = LinuxMemAreaStructAlloc(); if (!psLinuxMemArea) { return NULL; } psLinuxMemArea->eAreaType = LINUX_MEM_AREA_SUB_ALLOC; psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea = psParentLinuxMemArea; psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset = ui32ByteOffset; psLinuxMemArea->ui32ByteSize = ui32Bytes; psLinuxMemArea->ui32AreaFlags = psParentLinuxMemArea->ui32AreaFlags; psLinuxMemArea->bNeedsCacheInvalidate = psParentLinuxMemArea->bNeedsCacheInvalidate; INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); #if defined(DEBUG_LINUX_MEM_AREAS) { DEBUG_LINUX_MEM_AREA_REC *psParentRecord; psParentRecord = DebugLinuxMemAreaRecordFind(psParentLinuxMemArea); DebugLinuxMemAreaRecordAdd(psLinuxMemArea, psParentRecord->ui32Flags); } #endif return psLinuxMemArea; } static IMG_VOID FreeSubLinuxMemArea(LinuxMemArea *psLinuxMemArea) { PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_SUB_ALLOC); #if defined(DEBUG_LINUX_MEM_AREAS) DebugLinuxMemAreaRecordRemove(psLinuxMemArea); #endif /* Nothing more to do than free the LinuxMemArea structure */ LinuxMemAreaStructFree(psLinuxMemArea); } static LinuxMemArea * LinuxMemAreaStructAlloc(IMG_VOID) { /* debug */ #if 0 LinuxMemArea *psLinuxMemArea; psLinuxMemArea = kmem_cache_alloc(g_PsLinuxMemAreaCache, GFP_KERNEL); printk(KERN_ERR "%s: psLinuxMemArea=%p\n", __FUNCTION__, psLinuxMemArea); dump_stack(); return psLinuxMemArea; #else return KMemCacheAllocWrapper(g_PsLinuxMemAreaCache, GFP_KERNEL); #endif } static IMG_VOID LinuxMemAreaStructFree(LinuxMemArea *psLinuxMemArea) { KMemCacheFreeWrapper(g_PsLinuxMemAreaCache, psLinuxMemArea); /* debug */ //printk(KERN_ERR "%s(%p)\n", __FUNCTION__, psLinuxMemArea); } IMG_VOID LinuxMemAreaDeepFree(LinuxMemArea *psLinuxMemArea) { switch (psLinuxMemArea->eAreaType) { case LINUX_MEM_AREA_VMALLOC: FreeVMallocLinuxMemArea(psLinuxMemArea); break; case LINUX_MEM_AREA_ALLOC_PAGES: FreeAllocPagesLinuxMemArea(psLinuxMemArea); break; case LINUX_MEM_AREA_IOREMAP: FreeIORemapLinuxMemArea(psLinuxMemArea); break; case LINUX_MEM_AREA_EXTERNAL_KV: FreeExternalKVLinuxMemArea(psLinuxMemArea); break; case LINUX_MEM_AREA_IO: FreeIOLinuxMemArea(psLinuxMemArea); break; case LINUX_MEM_AREA_SUB_ALLOC: FreeSubLinuxMemArea(psLinuxMemArea); break; case LINUX_MEM_AREA_ION: FreeIONLinuxMemArea(psLinuxMemArea); break; default: PVR_DPF((PVR_DBG_ERROR, "%s: Unknown are type (%d)\n", __FUNCTION__, psLinuxMemArea->eAreaType)); break; } } #if defined(DEBUG_LINUX_MEM_AREAS) static IMG_VOID DebugLinuxMemAreaRecordAdd(LinuxMemArea *psLinuxMemArea, IMG_UINT32 ui32Flags) { DEBUG_LINUX_MEM_AREA_REC *psNewRecord; const IMG_CHAR *pi8FlagsString; LinuxLockMutex(&g_sDebugMutex); if (psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC) { g_LinuxMemAreaWaterMark += psLinuxMemArea->ui32ByteSize; if (g_LinuxMemAreaWaterMark > g_LinuxMemAreaHighWaterMark) { g_LinuxMemAreaHighWaterMark = g_LinuxMemAreaWaterMark; } } g_LinuxMemAreaCount++; /* Create a new memory allocation record */ psNewRecord = kmalloc(sizeof(DEBUG_LINUX_MEM_AREA_REC), GFP_KERNEL); if (psNewRecord) { /* Record the allocation */ psNewRecord->psLinuxMemArea = psLinuxMemArea; psNewRecord->ui32Flags = ui32Flags; psNewRecord->pid = OSGetCurrentProcessIDKM(); List_DEBUG_LINUX_MEM_AREA_REC_Insert(&g_LinuxMemAreaRecords, psNewRecord); } else { PVR_DPF((PVR_DBG_ERROR, "%s: failed to allocate linux memory area record.", __FUNCTION__)); } /* Sanity check the flags */ pi8FlagsString = HAPFlagsToString(ui32Flags); if (strstr(pi8FlagsString, "UNKNOWN")) { PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected flags (0x%08x) associated with psLinuxMemArea @ %p", __FUNCTION__, ui32Flags, psLinuxMemArea)); //dump_stack(); } LinuxUnLockMutex(&g_sDebugMutex); } static IMG_VOID* MatchLinuxMemArea_AnyVaCb(DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord, va_list va) { LinuxMemArea *psLinuxMemArea; psLinuxMemArea = va_arg(va, LinuxMemArea*); if (psCurrentRecord->psLinuxMemArea == psLinuxMemArea) { return psCurrentRecord; } else { return IMG_NULL; } } static DEBUG_LINUX_MEM_AREA_REC * DebugLinuxMemAreaRecordFind(LinuxMemArea *psLinuxMemArea) { DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord; LinuxLockMutex(&g_sDebugMutex); psCurrentRecord = List_DEBUG_LINUX_MEM_AREA_REC_Any_va(g_LinuxMemAreaRecords, MatchLinuxMemArea_AnyVaCb, psLinuxMemArea); /*exit_unlock:*/ LinuxUnLockMutex(&g_sDebugMutex); return psCurrentRecord; } static IMG_VOID DebugLinuxMemAreaRecordRemove(LinuxMemArea *psLinuxMemArea) { DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord; LinuxLockMutex(&g_sDebugMutex); if (psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC) { g_LinuxMemAreaWaterMark -= psLinuxMemArea->ui32ByteSize; } g_LinuxMemAreaCount--; /* Locate the corresponding allocation entry */ psCurrentRecord = List_DEBUG_LINUX_MEM_AREA_REC_Any_va(g_LinuxMemAreaRecords, MatchLinuxMemArea_AnyVaCb, psLinuxMemArea); if (psCurrentRecord) { /* Unlink the allocation record */ List_DEBUG_LINUX_MEM_AREA_REC_Remove(psCurrentRecord); kfree(psCurrentRecord); } else { PVR_DPF((PVR_DBG_ERROR, "%s: couldn't find an entry for psLinuxMemArea=%p\n", __FUNCTION__, psLinuxMemArea)); } LinuxUnLockMutex(&g_sDebugMutex); } #endif IMG_VOID * LinuxMemAreaToCpuVAddr(LinuxMemArea *psLinuxMemArea) { switch (psLinuxMemArea->eAreaType) { case LINUX_MEM_AREA_VMALLOC: return psLinuxMemArea->uData.sVmalloc.pvVmallocAddress; case LINUX_MEM_AREA_IOREMAP: return psLinuxMemArea->uData.sIORemap.pvIORemapCookie; case LINUX_MEM_AREA_EXTERNAL_KV: return psLinuxMemArea->uData.sExternalKV.pvExternalKV; case LINUX_MEM_AREA_SUB_ALLOC: { IMG_CHAR *pAddr = LinuxMemAreaToCpuVAddr(psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea); /* PRQA S 3670 */ /* ignore recursive warning */ if (!pAddr) { return NULL; } return pAddr + psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset; } default: return NULL; } } IMG_CPU_PHYADDR LinuxMemAreaToCpuPAddr(LinuxMemArea *psLinuxMemArea, IMG_UINT32 ui32ByteOffset) { IMG_CPU_PHYADDR CpuPAddr; CpuPAddr.uiAddr = 0; switch (psLinuxMemArea->eAreaType) { case LINUX_MEM_AREA_IOREMAP: { CpuPAddr = psLinuxMemArea->uData.sIORemap.CPUPhysAddr; CpuPAddr.uiAddr += ui32ByteOffset; break; } case LINUX_MEM_AREA_EXTERNAL_KV: { if (psLinuxMemArea->uData.sExternalKV.bPhysContig) { CpuPAddr = SysSysPAddrToCpuPAddr(psLinuxMemArea->uData.sExternalKV.uPhysAddr.SysPhysAddr); CpuPAddr.uiAddr += ui32ByteOffset; } else { IMG_UINT32 ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset); IMG_SYS_PHYADDR SysPAddr = psLinuxMemArea->uData.sExternalKV.uPhysAddr.pSysPhysAddr[ui32PageIndex]; CpuPAddr = SysSysPAddrToCpuPAddr(SysPAddr); CpuPAddr.uiAddr += ADDR_TO_PAGE_OFFSET(ui32ByteOffset); } break; } case LINUX_MEM_AREA_IO: { CpuPAddr = psLinuxMemArea->uData.sIO.CPUPhysAddr; CpuPAddr.uiAddr += ui32ByteOffset; break; } case LINUX_MEM_AREA_VMALLOC: { IMG_CHAR *pCpuVAddr; pCpuVAddr = (IMG_CHAR *)psLinuxMemArea->uData.sVmalloc.pvVmallocAddress; pCpuVAddr += ui32ByteOffset; CpuPAddr.uiAddr = VMallocToPhys(pCpuVAddr); break; } case LINUX_MEM_AREA_ION: { IMG_UINT32 ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset); CpuPAddr = psLinuxMemArea->uData.sIONTilerAlloc.pCPUPhysAddrs[ui32PageIndex]; CpuPAddr.uiAddr += ADDR_TO_PAGE_OFFSET(ui32ByteOffset); break; } case LINUX_MEM_AREA_ALLOC_PAGES: { struct page *page; IMG_UINT32 ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset); page = psLinuxMemArea->uData.sPageList.ppsPageList[ui32PageIndex]; CpuPAddr.uiAddr = page_to_phys(page); CpuPAddr.uiAddr += ADDR_TO_PAGE_OFFSET(ui32ByteOffset); break; } case LINUX_MEM_AREA_SUB_ALLOC: { CpuPAddr = OSMemHandleToCpuPAddr(psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea, psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset + ui32ByteOffset); break; } default: { PVR_DPF((PVR_DBG_ERROR, "%s: Unknown LinuxMemArea type (%d)\n", __FUNCTION__, psLinuxMemArea->eAreaType)); PVR_ASSERT(CpuPAddr.uiAddr); break; } } return CpuPAddr; } IMG_BOOL LinuxMemAreaPhysIsContig(LinuxMemArea *psLinuxMemArea) { switch (psLinuxMemArea->eAreaType) { case LINUX_MEM_AREA_IOREMAP: case LINUX_MEM_AREA_IO: return IMG_TRUE; case LINUX_MEM_AREA_EXTERNAL_KV: return psLinuxMemArea->uData.sExternalKV.bPhysContig; case LINUX_MEM_AREA_ION: case LINUX_MEM_AREA_VMALLOC: case LINUX_MEM_AREA_ALLOC_PAGES: return IMG_FALSE; case LINUX_MEM_AREA_SUB_ALLOC: /* PRQA S 3670 1 */ /* ignore recursive warning */ return LinuxMemAreaPhysIsContig(psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea); default: PVR_DPF((PVR_DBG_ERROR, "%s: Unknown LinuxMemArea type (%d)\n", __FUNCTION__, psLinuxMemArea->eAreaType)); break; } return IMG_FALSE; } const IMG_CHAR * LinuxMemAreaTypeToString(LINUX_MEM_AREA_TYPE eMemAreaType) { /* Note we explicitly check the types instead of e.g. * using the type to index an array of strings so * we remain orthogonal to enum changes */ switch (eMemAreaType) { case LINUX_MEM_AREA_IOREMAP: return "LINUX_MEM_AREA_IOREMAP"; case LINUX_MEM_AREA_EXTERNAL_KV: return "LINUX_MEM_AREA_EXTERNAL_KV"; case LINUX_MEM_AREA_IO: return "LINUX_MEM_AREA_IO"; case LINUX_MEM_AREA_VMALLOC: return "LINUX_MEM_AREA_VMALLOC"; case LINUX_MEM_AREA_SUB_ALLOC: return "LINUX_MEM_AREA_SUB_ALLOC"; case LINUX_MEM_AREA_ALLOC_PAGES: return "LINUX_MEM_AREA_ALLOC_PAGES"; case LINUX_MEM_AREA_ION: return "LINUX_MEM_AREA_ION"; default: PVR_ASSERT(0); } return ""; } #if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) static void ProcSeqStartstopDebugMutex(struct seq_file *sfile, IMG_BOOL start) { if (start) { LinuxLockMutex(&g_sDebugMutex); } else { LinuxUnLockMutex(&g_sDebugMutex); } } #endif /* defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) */ #if defined(DEBUG_LINUX_MEM_AREAS) static IMG_VOID* DecOffMemAreaRec_AnyVaCb(DEBUG_LINUX_MEM_AREA_REC *psNode, va_list va) { off_t *pOff = va_arg(va, off_t*); if (--(*pOff)) { return IMG_NULL; } else { return psNode; } } /* seq_file version of generating output, for reference check proc.c:CreateProcReadEntrySeq */ static void* ProcSeqNextMemArea(struct seq_file *sfile,void* el,loff_t off) { DEBUG_LINUX_MEM_AREA_REC *psRecord; psRecord = (DEBUG_LINUX_MEM_AREA_REC*) List_DEBUG_LINUX_MEM_AREA_REC_Any_va(g_LinuxMemAreaRecords, DecOffMemAreaRec_AnyVaCb, &off); return (void*)psRecord; } static void* ProcSeqOff2ElementMemArea(struct seq_file * sfile, loff_t off) { DEBUG_LINUX_MEM_AREA_REC *psRecord; if (!off) { return PVR_PROC_SEQ_START_TOKEN; } psRecord = (DEBUG_LINUX_MEM_AREA_REC*) List_DEBUG_LINUX_MEM_AREA_REC_Any_va(g_LinuxMemAreaRecords, DecOffMemAreaRec_AnyVaCb, &off); return (void*)psRecord; } static void ProcSeqShowMemArea(struct seq_file *sfile,void* el) { DEBUG_LINUX_MEM_AREA_REC *psRecord = (DEBUG_LINUX_MEM_AREA_REC*)el; if (el == PVR_PROC_SEQ_START_TOKEN) { #if !defined(DEBUG_LINUX_XML_PROC_FILES) seq_printf(sfile, "Number of Linux Memory Areas: %u\n" "At the current water mark these areas correspond to %u bytes (excluding SUB areas)\n" "At the highest water mark these areas corresponded to %u bytes (excluding SUB areas)\n" "\nDetails for all Linux Memory Areas:\n" "%s %-24s %s %s %-8s %-5s %s\n", g_LinuxMemAreaCount, g_LinuxMemAreaWaterMark, g_LinuxMemAreaHighWaterMark, "psLinuxMemArea", "LinuxMemType", "CpuVAddr", "CpuPAddr", "Bytes", "Pid", "Flags" ); #else seq_printf(sfile, "\n" "\t%u\n" "\t\n" /* (excluding SUB areas) */ "\t\n" /* (excluding SUB areas) */ "\n", g_LinuxMemAreaCount, g_LinuxMemAreaWaterMark, g_LinuxMemAreaHighWaterMark ); #endif return; } seq_printf(sfile, #if !defined(DEBUG_LINUX_XML_PROC_FILES) "%8p %-24s %8p %08x %-8d %-5u %08x=(%s)\n", #else "\n" "\t%8p\n" "\t%s\n" "\t%8p\n" "\t%08x\n" "\t%d\n" "\t%u\n" "\t%08x\n" "\t%s\n" "\n", #endif psRecord->psLinuxMemArea, LinuxMemAreaTypeToString(psRecord->psLinuxMemArea->eAreaType), LinuxMemAreaToCpuVAddr(psRecord->psLinuxMemArea), LinuxMemAreaToCpuPAddr(psRecord->psLinuxMemArea,0).uiAddr, psRecord->psLinuxMemArea->ui32ByteSize, psRecord->pid, psRecord->ui32Flags, HAPFlagsToString(psRecord->ui32Flags) ); } #endif /* DEBUG_LINUX_MEM_AREAS */ #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) static IMG_VOID* DecOffMemAllocRec_AnyVaCb(DEBUG_MEM_ALLOC_REC *psNode, va_list va) { off_t *pOff = va_arg(va, off_t*); if (--(*pOff)) { return IMG_NULL; } else { return psNode; } } /* seq_file version of generating output, for reference check proc.c:CreateProcReadEntrySeq */ static void* ProcSeqNextMemoryRecords(struct seq_file *sfile,void* el,loff_t off) { DEBUG_MEM_ALLOC_REC *psRecord; psRecord = (DEBUG_MEM_ALLOC_REC*) List_DEBUG_MEM_ALLOC_REC_Any_va(g_MemoryRecords, DecOffMemAllocRec_AnyVaCb, &off); #if defined(DEBUG_LINUX_XML_PROC_FILES) if (!psRecord) { seq_printf(sfile, "\n"); } #endif return (void*)psRecord; } static void* ProcSeqOff2ElementMemoryRecords(struct seq_file *sfile, loff_t off) { DEBUG_MEM_ALLOC_REC *psRecord; if (!off) { return PVR_PROC_SEQ_START_TOKEN; } psRecord = (DEBUG_MEM_ALLOC_REC*) List_DEBUG_MEM_ALLOC_REC_Any_va(g_MemoryRecords, DecOffMemAllocRec_AnyVaCb, &off); #if defined(DEBUG_LINUX_XML_PROC_FILES) if (!psRecord) { seq_printf(sfile, "\n"); } #endif return (void*)psRecord; } static void ProcSeqShowMemoryRecords(struct seq_file *sfile,void* el) { DEBUG_MEM_ALLOC_REC *psRecord = (DEBUG_MEM_ALLOC_REC*)el; if (el == PVR_PROC_SEQ_START_TOKEN) { #if !defined(DEBUG_LINUX_XML_PROC_FILES) /* NOTE: If you update this code, please also update the XML varient below * too! */ seq_printf(sfile, "%-60s: %d bytes\n", "Current Water Mark of bytes allocated via kmalloc", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]); seq_printf(sfile, "%-60s: %d bytes\n", "Highest Water Mark of bytes allocated via kmalloc", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]); seq_printf(sfile, "%-60s: %d bytes\n", "Current Water Mark of bytes allocated via vmalloc", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]); seq_printf(sfile, "%-60s: %d bytes\n", "Highest Water Mark of bytes allocated via vmalloc", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]); seq_printf(sfile, "%-60s: %d bytes\n", "Current Water Mark of bytes allocated via alloc_pages", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]); seq_printf(sfile, "%-60s: %d bytes\n", "Highest Water Mark of bytes allocated via alloc_pages", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]); seq_printf(sfile, "%-60s: %d bytes\n", "Current Water Mark of bytes allocated via ioremap", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]); seq_printf(sfile, "%-60s: %d bytes\n", "Highest Water Mark of bytes allocated via ioremap", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]); seq_printf(sfile, "%-60s: %d bytes\n", "Current Water Mark of bytes reserved for \"IO\" memory areas", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]); seq_printf(sfile, "%-60s: %d bytes\n", "Highest Water Mark of bytes allocated for \"IO\" memory areas", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]); seq_printf(sfile, "%-60s: %d bytes\n", "Current Water Mark of bytes allocated via kmem_cache_alloc", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]); seq_printf(sfile, "%-60s: %d bytes\n", "Highest Water Mark of bytes allocated via kmem_cache_alloc", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]); #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) seq_printf(sfile, "%-60s: %d bytes\n", "Current Water Mark of bytes mapped via vmap", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMAP]); seq_printf(sfile, "%-60s: %d bytes\n", "Highest Water Mark of bytes mapped via vmap", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMAP]); #endif #if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) seq_printf(sfile, "%-60s: %d pages\n", "Number of pages in page pool", atomic_read(&g_sPagePoolEntryCount)); #endif seq_printf( sfile, "\n"); seq_printf(sfile, "%-60s: %d bytes\n", "The Current Water Mark for memory allocated from system RAM", SysRAMTrueWaterMark()); seq_printf(sfile, "%-60s: %d bytes\n", "The Highest Water Mark for memory allocated from system RAM", g_SysRAMHighWaterMark); seq_printf(sfile, "%-60s: %d bytes\n", "The Current Water Mark for memory allocated from IO memory", g_IOMemWaterMark); seq_printf(sfile, "%-60s: %d bytes\n", "The Highest Water Mark for memory allocated from IO memory", g_IOMemHighWaterMark); seq_printf( sfile, "\n"); seq_printf(sfile, "Details for all known allocations:\n" "%-16s %-8s %-8s %-10s %-5s %-10s %s\n", "Type", "CpuVAddr", "CpuPAddr", "Bytes", "PID", "PrivateData", "Filename:Line"); #else /* DEBUG_LINUX_XML_PROC_FILES */ /* Note: If you want to update the description property of a watermark * ensure that the key property remains unchanged so that watermark data * logged over time from different driver revisions may remain comparable */ seq_printf(sfile, "\n\n"); seq_printf(sfile, "\n", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]); seq_printf(sfile, "\n", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]); seq_printf(sfile, "\n", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]); seq_printf(sfile, "\n", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]); seq_printf(sfile, "\n", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]); seq_printf(sfile, "\n", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]); seq_printf(sfile, "\n", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]); seq_printf(sfile, "\n", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]); seq_printf(sfile, "\n", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]); seq_printf(sfile, "\n", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]); seq_printf(sfile, "\n", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]); seq_printf(sfile, "\n", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]); #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) seq_printf(sfile, "\n", g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMAP]); seq_printf(sfile, "\n", g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMAP]); #endif seq_printf(sfile, "\n", SysRAMTrueWaterMark()); seq_printf(sfile, "\n", g_SysRAMHighWaterMark); seq_printf(sfile, "\n", g_IOMemWaterMark); seq_printf(sfile, "\n", g_IOMemHighWaterMark); #if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) seq_printf(sfile, "\n", PAGES_TO_BYTES(atomic_read(&g_sPagePoolEntryCount))); #endif seq_printf(sfile, "\n"); #endif /* DEBUG_LINUX_XML_PROC_FILES */ return; } if (psRecord->eAllocType != DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE) { seq_printf(sfile, #if !defined(DEBUG_LINUX_XML_PROC_FILES) "%-16s %-8p %08x %-10d %-5d %-10s %s:%d\n", #else "\n" "\t%s\n" "\t%-8p\n" "\t%08x\n" "\t%d\n" "\t%d\n" "\t%s\n" "\t%s\n" "\t%d\n" "\n", #endif DebugMemAllocRecordTypeToString(psRecord->eAllocType), psRecord->pvCpuVAddr, psRecord->ulCpuPAddr, psRecord->ui32Bytes, psRecord->pid, "NULL", psRecord->pszFileName, psRecord->ui32Line); } else { seq_printf(sfile, #if !defined(DEBUG_LINUX_XML_PROC_FILES) "%-16s %-8p %08x %-10d %-5d %-10s %s:%d\n", #else "\n" "\t%s\n" "\t%-8p\n" "\t%08x\n" "\t%d\n" "\t%d\n" "\t%s\n" "\t%s\n" "\t%d\n" "\n", #endif DebugMemAllocRecordTypeToString(psRecord->eAllocType), psRecord->pvCpuVAddr, psRecord->ulCpuPAddr, psRecord->ui32Bytes, psRecord->pid, KMemCacheNameWrapper(psRecord->pvPrivateData), psRecord->pszFileName, psRecord->ui32Line); } } #endif /* defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) */ #if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MMAP_AREAS) /* This could be moved somewhere more general */ const IMG_CHAR * HAPFlagsToString(IMG_UINT32 ui32Flags) { static IMG_CHAR szFlags[50]; IMG_INT32 i32Pos = 0; IMG_UINT32 ui32CacheTypeIndex, ui32MapTypeIndex; IMG_CHAR *apszCacheTypes[] = { "UNCACHED", "CACHED", "WRITECOMBINE", "UNKNOWN" }; IMG_CHAR *apszMapType[] = { "KERNEL_ONLY", "SINGLE_PROCESS", "MULTI_PROCESS", "FROM_EXISTING_PROCESS", "NO_CPU_VIRTUAL", "UNKNOWN" }; /* FIXME create an enum for the cache type that we can * cast and select so we get compiler warnings when * when this code isn't complete due to new flags */ if (ui32Flags & PVRSRV_HAP_UNCACHED) { ui32CacheTypeIndex = 0; } else if (ui32Flags & PVRSRV_HAP_CACHED) { ui32CacheTypeIndex = 1; } else if (ui32Flags & PVRSRV_HAP_WRITECOMBINE) { ui32CacheTypeIndex = 2; } else { ui32CacheTypeIndex = 3; PVR_DPF((PVR_DBG_ERROR, "%s: unknown cache type (%u)", __FUNCTION__, (ui32Flags & PVRSRV_HAP_CACHETYPE_MASK))); } /* FIXME create an enum for the map type that we can * cast and select so we get compiler warnings when * when this code isn't complete due to new flags */ if (ui32Flags & PVRSRV_HAP_KERNEL_ONLY) { ui32MapTypeIndex = 0; } else if (ui32Flags & PVRSRV_HAP_SINGLE_PROCESS) { ui32MapTypeIndex = 1; } else if (ui32Flags & PVRSRV_HAP_MULTI_PROCESS) { ui32MapTypeIndex = 2; } else if (ui32Flags & PVRSRV_HAP_FROM_EXISTING_PROCESS) { ui32MapTypeIndex = 3; } else if (ui32Flags & PVRSRV_HAP_NO_CPU_VIRTUAL) { ui32MapTypeIndex = 4; } else { ui32MapTypeIndex = 5; PVR_DPF((PVR_DBG_ERROR, "%s: unknown map type (%u)", __FUNCTION__, (ui32Flags & PVRSRV_HAP_MAPTYPE_MASK))); } i32Pos = sprintf(szFlags, "%s|", apszCacheTypes[ui32CacheTypeIndex]); if (i32Pos <= 0) { PVR_DPF((PVR_DBG_ERROR, "%s: sprintf for cache type %u failed (%d)", __FUNCTION__, ui32CacheTypeIndex, i32Pos)); szFlags[0] = 0; } else { sprintf(szFlags + i32Pos, "%s", apszMapType[ui32MapTypeIndex]); } return szFlags; } #endif #if defined(DEBUG_LINUX_MEM_AREAS) static IMG_VOID LinuxMMCleanup_MemAreas_ForEachCb(DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord) { LinuxMemArea *psLinuxMemArea; psLinuxMemArea = psCurrentRecord->psLinuxMemArea; PVR_DPF((PVR_DBG_ERROR, "%s: BUG!: Cleaning up Linux memory area (%p), type=%s, size=%d bytes", __FUNCTION__, psCurrentRecord->psLinuxMemArea, LinuxMemAreaTypeToString(psCurrentRecord->psLinuxMemArea->eAreaType), psCurrentRecord->psLinuxMemArea->ui32ByteSize)); /* Note this will also remove psCurrentRecord from g_LinuxMemAreaRecords * but that's ok since we have already got a pointer to the next area. */ LinuxMemAreaDeepFree(psLinuxMemArea); } #endif #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) static IMG_VOID LinuxMMCleanup_MemRecords_ForEachVa(DEBUG_MEM_ALLOC_REC *psCurrentRecord) { /* It's a bug if anything remains allocated at this point. We * report an error, and simply brute force free anything we find. */ PVR_DPF((PVR_DBG_ERROR, "%s: BUG!: Cleaning up memory: " "type=%s " "CpuVAddr=%p " "CpuPAddr=0x%08x, " "allocated @ file=%s,line=%d", __FUNCTION__, DebugMemAllocRecordTypeToString(psCurrentRecord->eAllocType), psCurrentRecord->pvCpuVAddr, psCurrentRecord->ulCpuPAddr, psCurrentRecord->pszFileName, psCurrentRecord->ui32Line)); switch (psCurrentRecord->eAllocType) { case DEBUG_MEM_ALLOC_TYPE_KMALLOC: KFreeWrapper(psCurrentRecord->pvCpuVAddr); break; case DEBUG_MEM_ALLOC_TYPE_IOREMAP: IOUnmapWrapper(psCurrentRecord->pvCpuVAddr); break; case DEBUG_MEM_ALLOC_TYPE_IO: /* Nothing needed except to free the record */ DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_IO, psCurrentRecord->pvKey, __FILE__, __LINE__); break; case DEBUG_MEM_ALLOC_TYPE_VMALLOC: VFreeWrapper(psCurrentRecord->pvCpuVAddr); break; case DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES: DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, psCurrentRecord->pvKey, __FILE__, __LINE__); break; case DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE: KMemCacheFreeWrapper(psCurrentRecord->pvPrivateData, psCurrentRecord->pvCpuVAddr); break; #if defined(PVR_LINUX_MEM_AREA_USE_VMAP) case DEBUG_MEM_ALLOC_TYPE_VMAP: VUnmapWrapper(psCurrentRecord->pvCpuVAddr); break; #endif default: PVR_ASSERT(0); } } #endif #if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) static struct shrinker g_sShrinker = { .shrink = ShrinkPagePool, .seeks = DEFAULT_SEEKS }; static IMG_BOOL g_bShrinkerRegistered; #endif IMG_VOID LinuxMMCleanup(IMG_VOID) { #if defined(DEBUG_LINUX_MEM_AREAS) { if (g_LinuxMemAreaCount) { PVR_DPF((PVR_DBG_ERROR, "%s: BUG!: There are %d LinuxMemArea allocation unfreed (%d bytes)", __FUNCTION__, g_LinuxMemAreaCount, g_LinuxMemAreaWaterMark)); } List_DEBUG_LINUX_MEM_AREA_REC_ForEach(g_LinuxMemAreaRecords, LinuxMMCleanup_MemAreas_ForEachCb); if (g_SeqFileMemArea) { RemoveProcEntrySeq(g_SeqFileMemArea); } } #endif #if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) if (g_bShrinkerRegistered) { unregister_shrinker(&g_sShrinker); } #endif /* * The page pool must be freed after any remaining mem areas, but before * the remaining memory resources. */ FreePagePool(); #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) { /* * It's a bug if anything remains allocated at this point. We * report an error, and simply brute force free anything we find. */ List_DEBUG_MEM_ALLOC_REC_ForEach(g_MemoryRecords, LinuxMMCleanup_MemRecords_ForEachVa); if (g_SeqFileMemoryRecords) { RemoveProcEntrySeq(g_SeqFileMemoryRecords); } } #endif if (g_PsLinuxMemAreaCache) { KMemCacheDestroyWrapper(g_PsLinuxMemAreaCache); } if (g_PsLinuxPagePoolCache) { KMemCacheDestroyWrapper(g_PsLinuxPagePoolCache); } } PVRSRV_ERROR LinuxMMInit(IMG_VOID) { #if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) LinuxInitMutex(&g_sDebugMutex); #endif #if defined(DEBUG_LINUX_MEM_AREAS) { g_SeqFileMemArea = CreateProcReadEntrySeq( "mem_areas", NULL, ProcSeqNextMemArea, ProcSeqShowMemArea, ProcSeqOff2ElementMemArea, ProcSeqStartstopDebugMutex ); if (!g_SeqFileMemArea) { goto failed; } } #endif #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) { g_SeqFileMemoryRecords = CreateProcReadEntrySeq( "meminfo", NULL, ProcSeqNextMemoryRecords, ProcSeqShowMemoryRecords, ProcSeqOff2ElementMemoryRecords, ProcSeqStartstopDebugMutex ); if (!g_SeqFileMemoryRecords) { goto failed; } } #endif g_PsLinuxMemAreaCache = KMemCacheCreateWrapper("img-mm", sizeof(LinuxMemArea), 0, 0); if (!g_PsLinuxMemAreaCache) { PVR_DPF((PVR_DBG_ERROR,"%s: failed to allocate mem area kmem_cache", __FUNCTION__)); goto failed; } #if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) g_iPagePoolMaxEntries = PVR_LINUX_MEM_AREA_POOL_MAX_PAGES; if (g_iPagePoolMaxEntries <= 0 || g_iPagePoolMaxEntries > INT_MAX/2) { g_iPagePoolMaxEntries = INT_MAX/2; PVR_TRACE(("%s: No limit set for page pool size", __FUNCTION__)); } else { PVR_TRACE(("%s: Maximum page pool size: %d", __FUNCTION__, g_iPagePoolMaxEntries)); } g_PsLinuxPagePoolCache = KMemCacheCreateWrapper("img-mm-pool", sizeof(LinuxPagePoolEntry), 0, 0); if (!g_PsLinuxPagePoolCache) { PVR_DPF((PVR_DBG_ERROR,"%s: failed to allocate page pool kmem_cache", __FUNCTION__)); goto failed; } #endif #if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) register_shrinker(&g_sShrinker); g_bShrinkerRegistered = IMG_TRUE; #endif return PVRSRV_OK; failed: LinuxMMCleanup(); return PVRSRV_ERROR_OUT_OF_MEMORY; }