diff options
Diffstat (limited to 'pvr-source/services4/srvkm/env/linux/mm.c')
| -rwxr-xr-x | pvr-source/services4/srvkm/env/linux/mm.c | 2945 |
1 files changed, 0 insertions, 2945 deletions
diff --git a/pvr-source/services4/srvkm/env/linux/mm.c b/pvr-source/services4/srvkm/env/linux/mm.c deleted file mode 100755 index 0815e46..0000000 --- a/pvr-source/services4/srvkm/env/linux/mm.c +++ /dev/null @@ -1,2945 +0,0 @@ -/*************************************************************************/ /*! -@Title Misc memory management utility functions for Linux -@Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved -@License Dual MIT/GPLv2 - -The contents of this file are subject to the MIT license as set out below. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -Alternatively, the contents of this file may be used under the terms of -the GNU General Public License Version 2 ("GPL") in which case the provisions -of GPL are applicable instead of those above. - -If you wish to allow use of your version of this file only under the terms of -GPL, and not to allow others to use your version of this file under the terms -of the MIT license, indicate your decision by deleting the provisions above -and replace them with the notice and other provisions required by GPL as set -out in the file called "GPL-COPYING" included in this distribution. If you do -not delete the provisions above, a recipient may use your version of this file -under the terms of either the MIT license or GPL. - -This License is also included in this distribution in the file called -"MIT-COPYING". - -EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS -PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ /**************************************************************************/ - -#include <linux/version.h> - -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38)) -#ifndef AUTOCONF_INCLUDED -#include <linux/config.h> -#endif -#endif - -#if !defined(PVR_LINUX_MEM_AREA_POOL_MAX_PAGES) -#define PVR_LINUX_MEM_AREA_POOL_MAX_PAGES 0 -#endif - -#include <linux/kernel.h> -#include <asm/atomic.h> -#include <linux/list.h> -#include <linux/mutex.h> -#include <linux/mm.h> -#include <linux/vmalloc.h> -#include <asm/io.h> -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)) -#include <linux/wrapper.h> -#endif -#include <linux/slab.h> -#include <linux/highmem.h> -#include <linux/sched.h> - -#if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,1,0)) -#include <linux/shrinker.h> -#endif -#endif - -#include "img_defs.h" -#include "services.h" -#include "servicesint.h" -#include "syscommon.h" -#include "mutils.h" -#include "mm.h" -#include "pvrmmap.h" -#include "mmap.h" -#include "osfunc.h" -#include "pvr_debug.h" -#include "proc.h" -#include "mutex.h" -#include "lock.h" - -#if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - #include "lists.h" -#endif - -/* If there is no explicit definition - * for the minimum DMM alignment size, - * then set it to "0" and let ION/DMM - * set the minimum value. */ -#ifndef CONFIG_TILER_GRANULARITY -#define CONFIG_TILER_GRANULARITY 0 -#endif - -/* - * The page pool entry count is an atomic int so that the shrinker function - * can return it even when we can't take the lock that protects the page pool - * list. - */ -static atomic_t g_sPagePoolEntryCount = ATOMIC_INIT(0); - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) -typedef enum { - DEBUG_MEM_ALLOC_TYPE_KMALLOC, - DEBUG_MEM_ALLOC_TYPE_VMALLOC, - DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, - DEBUG_MEM_ALLOC_TYPE_IOREMAP, - DEBUG_MEM_ALLOC_TYPE_IO, - DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE, - DEBUG_MEM_ALLOC_TYPE_ION, -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - DEBUG_MEM_ALLOC_TYPE_VMAP, -#endif - DEBUG_MEM_ALLOC_TYPE_COUNT -} DEBUG_MEM_ALLOC_TYPE; - -typedef struct _DEBUG_MEM_ALLOC_REC -{ - DEBUG_MEM_ALLOC_TYPE eAllocType; - IMG_VOID *pvKey; /* Some unique value (private to the eAllocType) */ - IMG_VOID *pvCpuVAddr; - IMG_UINT32 ulCpuPAddr; - IMG_VOID *pvPrivateData; - IMG_UINT32 ui32Bytes; - pid_t pid; - IMG_CHAR *pszFileName; - IMG_UINT32 ui32Line; - - struct _DEBUG_MEM_ALLOC_REC *psNext; - struct _DEBUG_MEM_ALLOC_REC **ppsThis; -} DEBUG_MEM_ALLOC_REC; - -static IMPLEMENT_LIST_ANY_VA_2(DEBUG_MEM_ALLOC_REC, IMG_BOOL, IMG_FALSE) -static IMPLEMENT_LIST_ANY_VA(DEBUG_MEM_ALLOC_REC) -static IMPLEMENT_LIST_FOR_EACH(DEBUG_MEM_ALLOC_REC) -static IMPLEMENT_LIST_INSERT(DEBUG_MEM_ALLOC_REC) -static IMPLEMENT_LIST_REMOVE(DEBUG_MEM_ALLOC_REC) - - -static DEBUG_MEM_ALLOC_REC *g_MemoryRecords; - -static IMG_UINT32 g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_COUNT]; -static IMG_UINT32 g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_COUNT]; - -/* vmalloc + kmalloc + alloc_pages + kmem_cache */ -static IMG_UINT32 g_SysRAMWaterMark; /* Doesn't include page pool */ -static IMG_UINT32 g_SysRAMHighWaterMark; /* *DOES* include page pool */ - -static inline IMG_UINT32 -SysRAMTrueWaterMark(void) -{ - return g_SysRAMWaterMark + PAGES_TO_BYTES(atomic_read(&g_sPagePoolEntryCount)); -} - -/* ioremap + io */ -static IMG_UINT32 g_IOMemWaterMark; -static IMG_UINT32 g_IOMemHighWaterMark; - -static IMG_VOID DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE eAllocType, - IMG_VOID *pvKey, - IMG_VOID *pvCpuVAddr, - IMG_UINT32 ulCpuPAddr, - IMG_VOID *pvPrivateData, - IMG_UINT32 ui32Bytes, - IMG_CHAR *pszFileName, - IMG_UINT32 ui32Line); - -static IMG_VOID DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE eAllocType, IMG_VOID *pvKey, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line); - -static IMG_CHAR *DebugMemAllocRecordTypeToString(DEBUG_MEM_ALLOC_TYPE eAllocType); - - -static struct proc_dir_entry *g_SeqFileMemoryRecords; -static void* ProcSeqNextMemoryRecords(struct seq_file *sfile,void* el,loff_t off); -static void ProcSeqShowMemoryRecords(struct seq_file *sfile,void* el); -static void* ProcSeqOff2ElementMemoryRecords(struct seq_file * sfile, loff_t off); - -#endif - - -#if defined(DEBUG_LINUX_MEM_AREAS) -typedef struct _DEBUG_LINUX_MEM_AREA_REC -{ - LinuxMemArea *psLinuxMemArea; - IMG_UINT32 ui32Flags; - pid_t pid; - - struct _DEBUG_LINUX_MEM_AREA_REC *psNext; - struct _DEBUG_LINUX_MEM_AREA_REC **ppsThis; -}DEBUG_LINUX_MEM_AREA_REC; - - -static IMPLEMENT_LIST_ANY_VA(DEBUG_LINUX_MEM_AREA_REC) -static IMPLEMENT_LIST_FOR_EACH(DEBUG_LINUX_MEM_AREA_REC) -static IMPLEMENT_LIST_INSERT(DEBUG_LINUX_MEM_AREA_REC) -static IMPLEMENT_LIST_REMOVE(DEBUG_LINUX_MEM_AREA_REC) - - - - -static DEBUG_LINUX_MEM_AREA_REC *g_LinuxMemAreaRecords; -static IMG_UINT32 g_LinuxMemAreaCount; -static IMG_UINT32 g_LinuxMemAreaWaterMark; -static IMG_UINT32 g_LinuxMemAreaHighWaterMark; - - -static struct proc_dir_entry *g_SeqFileMemArea; - -static void* ProcSeqNextMemArea(struct seq_file *sfile,void* el,loff_t off); -static void ProcSeqShowMemArea(struct seq_file *sfile,void* el); -static void* ProcSeqOff2ElementMemArea(struct seq_file *sfile, loff_t off); - -#endif - -#if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) -static PVRSRV_LINUX_MUTEX g_sDebugMutex; -#endif - -#if (defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)) -static void ProcSeqStartstopDebugMutex(struct seq_file *sfile,IMG_BOOL start); -#endif - -typedef struct -{ - /* Linkage for page pool LRU list */ - struct list_head sPagePoolItem; - - struct page *psPage; -} LinuxPagePoolEntry; - -static LinuxKMemCache *g_PsLinuxMemAreaCache; -static LinuxKMemCache *g_PsLinuxPagePoolCache; - -static LIST_HEAD(g_sPagePoolList); -static int g_iPagePoolMaxEntries; - -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) -static IMG_VOID ReservePages(IMG_VOID *pvAddress, IMG_UINT32 ui32Length); -static IMG_VOID UnreservePages(IMG_VOID *pvAddress, IMG_UINT32 ui32Length); -#endif - -static LinuxMemArea *LinuxMemAreaStructAlloc(IMG_VOID); -static IMG_VOID LinuxMemAreaStructFree(LinuxMemArea *psLinuxMemArea); -#if defined(DEBUG_LINUX_MEM_AREAS) -static IMG_VOID DebugLinuxMemAreaRecordAdd(LinuxMemArea *psLinuxMemArea, IMG_UINT32 ui32Flags); -static DEBUG_LINUX_MEM_AREA_REC *DebugLinuxMemAreaRecordFind(LinuxMemArea *psLinuxMemArea); -static IMG_VOID DebugLinuxMemAreaRecordRemove(LinuxMemArea *psLinuxMemArea); -#endif - - -static inline IMG_BOOL -AreaIsUncached(IMG_UINT32 ui32AreaFlags) -{ - return (ui32AreaFlags & (PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_UNCACHED)) != 0; -} - -static inline IMG_BOOL -CanFreeToPool(LinuxMemArea *psLinuxMemArea) -{ - return AreaIsUncached(psLinuxMemArea->ui32AreaFlags) && !psLinuxMemArea->bNeedsCacheInvalidate; -} - -IMG_VOID * -_KMallocWrapper(IMG_UINT32 ui32ByteSize, gfp_t uFlags, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) -{ - IMG_VOID *pvRet; - pvRet = kmalloc(ui32ByteSize, uFlags); -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - if (pvRet) - { - DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_KMALLOC, - pvRet, - pvRet, - 0, - NULL, - ui32ByteSize, - pszFileName, - ui32Line - ); - } -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - return pvRet; -} - - -IMG_VOID -_KFreeWrapper(IMG_VOID *pvCpuVAddr, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) -{ -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_KMALLOC, pvCpuVAddr, pszFileName, ui32Line); -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - kfree(pvCpuVAddr); -} - - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) -static IMG_VOID -DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE eAllocType, - IMG_VOID *pvKey, - IMG_VOID *pvCpuVAddr, - IMG_UINT32 ulCpuPAddr, - IMG_VOID *pvPrivateData, - IMG_UINT32 ui32Bytes, - IMG_CHAR *pszFileName, - IMG_UINT32 ui32Line) -{ - DEBUG_MEM_ALLOC_REC *psRecord; - - LinuxLockMutex(&g_sDebugMutex); - - psRecord = kmalloc(sizeof(DEBUG_MEM_ALLOC_REC), GFP_KERNEL); - - psRecord->eAllocType = eAllocType; - psRecord->pvKey = pvKey; - psRecord->pvCpuVAddr = pvCpuVAddr; - psRecord->ulCpuPAddr = ulCpuPAddr; - psRecord->pvPrivateData = pvPrivateData; - psRecord->pid = OSGetCurrentProcessIDKM(); - psRecord->ui32Bytes = ui32Bytes; - psRecord->pszFileName = pszFileName; - psRecord->ui32Line = ui32Line; - - List_DEBUG_MEM_ALLOC_REC_Insert(&g_MemoryRecords, psRecord); - - g_WaterMarkData[eAllocType] += ui32Bytes; - if (g_WaterMarkData[eAllocType] > g_HighWaterMarkData[eAllocType]) - { - g_HighWaterMarkData[eAllocType] = g_WaterMarkData[eAllocType]; - } - - if (eAllocType == DEBUG_MEM_ALLOC_TYPE_KMALLOC - || eAllocType == DEBUG_MEM_ALLOC_TYPE_VMALLOC - || eAllocType == DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES - || eAllocType == DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE) - { - IMG_UINT32 ui32SysRAMTrueWaterMark; - - g_SysRAMWaterMark += ui32Bytes; - ui32SysRAMTrueWaterMark = SysRAMTrueWaterMark(); - - if (ui32SysRAMTrueWaterMark > g_SysRAMHighWaterMark) - { - g_SysRAMHighWaterMark = ui32SysRAMTrueWaterMark; - } - } - else if (eAllocType == DEBUG_MEM_ALLOC_TYPE_IOREMAP - || eAllocType == DEBUG_MEM_ALLOC_TYPE_IO) - { - g_IOMemWaterMark += ui32Bytes; - if (g_IOMemWaterMark > g_IOMemHighWaterMark) - { - g_IOMemHighWaterMark = g_IOMemWaterMark; - } - } - - LinuxUnLockMutex(&g_sDebugMutex); -} - - -static IMG_BOOL DebugMemAllocRecordRemove_AnyVaCb(DEBUG_MEM_ALLOC_REC *psCurrentRecord, va_list va) -{ - DEBUG_MEM_ALLOC_TYPE eAllocType; - IMG_VOID *pvKey; - - eAllocType = va_arg(va, DEBUG_MEM_ALLOC_TYPE); - pvKey = va_arg(va, IMG_VOID*); - - if (psCurrentRecord->eAllocType == eAllocType - && psCurrentRecord->pvKey == pvKey) - { - eAllocType = psCurrentRecord->eAllocType; - g_WaterMarkData[eAllocType] -= psCurrentRecord->ui32Bytes; - - if (eAllocType == DEBUG_MEM_ALLOC_TYPE_KMALLOC - || eAllocType == DEBUG_MEM_ALLOC_TYPE_VMALLOC - || eAllocType == DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES - || eAllocType == DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE) - { - g_SysRAMWaterMark -= psCurrentRecord->ui32Bytes; - } - else if (eAllocType == DEBUG_MEM_ALLOC_TYPE_IOREMAP - || eAllocType == DEBUG_MEM_ALLOC_TYPE_IO) - { - g_IOMemWaterMark -= psCurrentRecord->ui32Bytes; - } - - List_DEBUG_MEM_ALLOC_REC_Remove(psCurrentRecord); - kfree(psCurrentRecord); - - return IMG_TRUE; - } - else - { - return IMG_FALSE; - } -} - - -static IMG_VOID -DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE eAllocType, IMG_VOID *pvKey, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) -{ -/* DEBUG_MEM_ALLOC_REC **ppsCurrentRecord;*/ - - LinuxLockMutex(&g_sDebugMutex); - - /* Locate the corresponding allocation entry */ - if (!List_DEBUG_MEM_ALLOC_REC_IMG_BOOL_Any_va(g_MemoryRecords, - DebugMemAllocRecordRemove_AnyVaCb, - eAllocType, - pvKey)) - { - PVR_DPF((PVR_DBG_ERROR, "%s: couldn't find an entry for type=%s with pvKey=%p (called from %s, line %d\n", - __FUNCTION__, DebugMemAllocRecordTypeToString(eAllocType), pvKey, - pszFileName, ui32Line)); - } - - LinuxUnLockMutex(&g_sDebugMutex); -} - - -static IMG_CHAR * -DebugMemAllocRecordTypeToString(DEBUG_MEM_ALLOC_TYPE eAllocType) -{ - IMG_CHAR *apszDebugMemoryRecordTypes[] = { - "KMALLOC", - "VMALLOC", - "ALLOC_PAGES", - "IOREMAP", - "IO", - "KMEM_CACHE_ALLOC", -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - "VMAP" -#endif - }; - return apszDebugMemoryRecordTypes[eAllocType]; -} -#endif - - -static IMG_BOOL -AllocFlagsToPGProt(pgprot_t *pPGProtFlags, IMG_UINT32 ui32AllocFlags) -{ - pgprot_t PGProtFlags; - - switch (ui32AllocFlags & PVRSRV_HAP_CACHETYPE_MASK) - { - case PVRSRV_HAP_CACHED: - PGProtFlags = PAGE_KERNEL; - break; - case PVRSRV_HAP_WRITECOMBINE: - PGProtFlags = PGPROT_WC(PAGE_KERNEL); - break; - case PVRSRV_HAP_UNCACHED: - PGProtFlags = PGPROT_UC(PAGE_KERNEL); - break; - default: - PVR_DPF((PVR_DBG_ERROR, - "%s: Unknown mapping flags=0x%08x", - __FUNCTION__, ui32AllocFlags)); - dump_stack(); - return IMG_FALSE; - } - - *pPGProtFlags = PGProtFlags; - - return IMG_TRUE; -} - -IMG_VOID * -_VMallocWrapper(IMG_UINT32 ui32Bytes, - IMG_UINT32 ui32AllocFlags, - IMG_CHAR *pszFileName, - IMG_UINT32 ui32Line) -{ - pgprot_t PGProtFlags; - IMG_VOID *pvRet; - - if (!AllocFlagsToPGProt(&PGProtFlags, ui32AllocFlags)) - { - return NULL; - } - - /* Allocate virtually contiguous pages */ - pvRet = __vmalloc(ui32Bytes, GFP_KERNEL | __GFP_HIGHMEM, PGProtFlags); - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - if (pvRet) - { - DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_VMALLOC, - pvRet, - pvRet, - 0, - NULL, - PAGE_ALIGN(ui32Bytes), - pszFileName, - ui32Line - ); - } -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - - return pvRet; -} - - -IMG_VOID -_VFreeWrapper(IMG_VOID *pvCpuVAddr, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) -{ -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_VMALLOC, pvCpuVAddr, pszFileName, ui32Line); -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - vfree(pvCpuVAddr); -} - - -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) -static IMG_VOID * -_VMapWrapper(struct page **ppsPageList, IMG_UINT32 ui32NumPages, IMG_UINT32 ui32AllocFlags, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) -{ - pgprot_t PGProtFlags; - IMG_VOID *pvRet; - - if (!AllocFlagsToPGProt(&PGProtFlags, ui32AllocFlags)) - { - return NULL; - } - - pvRet = vmap(ppsPageList, ui32NumPages, GFP_KERNEL | __GFP_HIGHMEM, PGProtFlags); - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - if (pvRet) - { - DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_VMAP, - pvRet, - pvRet, - 0, - NULL, - PAGES_TO_BYTES(ui32NumPages), - pszFileName, - ui32Line - ); - } -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - - return pvRet; -} - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) -#define VMapWrapper(ppsPageList, ui32Bytes, ui32AllocFlags) _VMapWrapper(ppsPageList, ui32Bytes, ui32AllocFlags, __FILE__, __LINE__) -#else -#define VMapWrapper(ppsPageList, ui32Bytes, ui32AllocFlags) _VMapWrapper(ppsPageList, ui32Bytes, ui32AllocFlags, NULL, 0) -#endif - - -static IMG_VOID -_VUnmapWrapper(IMG_VOID *pvCpuVAddr, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) -{ -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_VMAP, pvCpuVAddr, pszFileName, ui32Line); -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - vunmap(pvCpuVAddr); -} - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) -#define VUnmapWrapper(pvCpuVAddr) _VUnmapWrapper(pvCpuVAddr, __FILE__, __LINE__) -#else -#define VUnmapWrapper(pvCpuVAddr) _VUnmapWrapper(pvCpuVAddr, NULL, 0) -#endif - -#endif /* defined(PVR_LINUX_MEM_AREA_USE_VMAP) */ - - -IMG_VOID -_KMemCacheFreeWrapper(LinuxKMemCache *psCache, IMG_VOID *pvObject, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) -{ -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE, pvObject, pszFileName, ui32Line); -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - - kmem_cache_free(psCache, pvObject); -} - - -const IMG_CHAR * -KMemCacheNameWrapper(LinuxKMemCache *psCache) -{ - PVR_UNREFERENCED_PARAMETER(psCache); - - /* In this case kmem_cache_t is an incomplete typedef, - * so we can't even de-reference to get the name member. It is also a GPL export symbol */ - return ""; -} - - -static LinuxPagePoolEntry * -LinuxPagePoolEntryAlloc(IMG_VOID) -{ - return KMemCacheAllocWrapper(g_PsLinuxPagePoolCache, GFP_KERNEL); -} - -static IMG_VOID -LinuxPagePoolEntryFree(LinuxPagePoolEntry *psPagePoolEntry) -{ - KMemCacheFreeWrapper(g_PsLinuxPagePoolCache, psPagePoolEntry); -} - - -static struct page * -AllocPageFromLinux(void) -{ - struct page *psPage; - - psPage = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM, 0); - if (!psPage) - { - return NULL; - - } -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) - /* Reserve those pages to allow them to be re-mapped to user space */ -#if (LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)) - SetPageReserved(psPage); -#else - mem_map_reserve(psPage); -#endif -#endif - return psPage; -} - - -static IMG_VOID -FreePageToLinux(struct page *psPage) -{ -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) -#if (LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)) - ClearPageReserved(psPage); -#else - mem_map_reserve(psPage); -#endif -#endif - __free_pages(psPage, 0); -} - - -#if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) -static DEFINE_MUTEX(g_sPagePoolMutex); - -static inline void -PagePoolLock(void) -{ - mutex_lock(&g_sPagePoolMutex); -} - -static inline void -PagePoolUnlock(void) -{ - mutex_unlock(&g_sPagePoolMutex); -} - -static inline int -PagePoolTrylock(void) -{ - return mutex_trylock(&g_sPagePoolMutex); -} - -#else /* (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) */ -static inline void -PagePoolLock(void) -{ -} - -static inline void -PagePoolUnlock(void) -{ -} - -static inline int -PagePoolTrylock(void) -{ - return 1; -} -#endif /* (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) */ - - -static inline void -AddEntryToPool(LinuxPagePoolEntry *psPagePoolEntry) -{ - list_add_tail(&psPagePoolEntry->sPagePoolItem, &g_sPagePoolList); - atomic_inc(&g_sPagePoolEntryCount); -} - -static inline void -RemoveEntryFromPool(LinuxPagePoolEntry *psPagePoolEntry) -{ - list_del(&psPagePoolEntry->sPagePoolItem); - atomic_dec(&g_sPagePoolEntryCount); -} - -static inline LinuxPagePoolEntry * -RemoveFirstEntryFromPool(void) -{ - LinuxPagePoolEntry *psPagePoolEntry; - - if (list_empty(&g_sPagePoolList)) - { - PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) == 0); - - return NULL; - } - - PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) > 0); - - psPagePoolEntry = list_first_entry(&g_sPagePoolList, LinuxPagePoolEntry, sPagePoolItem); - - RemoveEntryFromPool(psPagePoolEntry); - - return psPagePoolEntry; -} - -static struct page * -AllocPage(IMG_UINT32 ui32AreaFlags, IMG_BOOL *pbFromPagePool) -{ - struct page *psPage = NULL; - - /* - * Only uncached allocations can come from the page pool. - * The page pool is currently used to reduce the cost of - * invalidating the CPU cache when uncached memory is allocated. - */ - if (AreaIsUncached(ui32AreaFlags) && atomic_read(&g_sPagePoolEntryCount) != 0) - { - LinuxPagePoolEntry *psPagePoolEntry; - - PagePoolLock(); - psPagePoolEntry = RemoveFirstEntryFromPool(); - PagePoolUnlock(); - - /* List may have changed since we checked the counter */ - if (psPagePoolEntry) - { - psPage = psPagePoolEntry->psPage; - LinuxPagePoolEntryFree(psPagePoolEntry); - *pbFromPagePool = IMG_TRUE; - } - } - - if (!psPage) - { - psPage = AllocPageFromLinux(); - if (psPage) - { - *pbFromPagePool = IMG_FALSE; - } - } - - return psPage; - -} - -static IMG_VOID -FreePage(IMG_BOOL bToPagePool, struct page *psPage) -{ - /* Only uncached allocations can be freed to the page pool */ - if (bToPagePool && atomic_read(&g_sPagePoolEntryCount) < g_iPagePoolMaxEntries) - { - LinuxPagePoolEntry *psPagePoolEntry = LinuxPagePoolEntryAlloc(); - if (psPagePoolEntry) - { - psPagePoolEntry->psPage = psPage; - - PagePoolLock(); - AddEntryToPool(psPagePoolEntry); - PagePoolUnlock(); - - return; - } - } - - FreePageToLinux(psPage); -} - -static IMG_VOID -FreePagePool(IMG_VOID) -{ - LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry; - - PagePoolLock(); - -#if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) - PVR_DPF((PVR_DBG_MESSAGE,"%s: Freeing %d pages from pool", __FUNCTION__, atomic_read(&g_sPagePoolEntryCount))); -#else - PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) == 0); - PVR_ASSERT(list_empty(&g_sPagePoolList)); -#endif - - list_for_each_entry_safe(psPagePoolEntry, psTempPoolEntry, &g_sPagePoolList, sPagePoolItem) - { - RemoveEntryFromPool(psPagePoolEntry); - - FreePageToLinux(psPagePoolEntry->psPage); - LinuxPagePoolEntryFree(psPagePoolEntry); - } - - PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) == 0); - - PagePoolUnlock(); -} - -#if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) -#if defined(PVRSRV_NEED_PVR_ASSERT) -static struct shrinker g_sShrinker; -#endif - -static int -ShrinkPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl) -{ - unsigned long uNumToScan = psShrinkControl->nr_to_scan; - - PVR_ASSERT(psShrinker == &g_sShrinker); - (void)psShrinker; - - if (uNumToScan != 0) - { - LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry; - - PVR_DPF((PVR_DBG_MESSAGE,"%s: Number to scan: %ld", __FUNCTION__, uNumToScan)); - PVR_DPF((PVR_DBG_MESSAGE,"%s: Pages in pool before scan: %d", __FUNCTION__, atomic_read(&g_sPagePoolEntryCount))); - - if (!PagePoolTrylock()) - { - PVR_TRACE(("%s: Couldn't get page pool lock", __FUNCTION__)); - return -1; - } - - list_for_each_entry_safe(psPagePoolEntry, psTempPoolEntry, &g_sPagePoolList, sPagePoolItem) - { - RemoveEntryFromPool(psPagePoolEntry); - - FreePageToLinux(psPagePoolEntry->psPage); - LinuxPagePoolEntryFree(psPagePoolEntry); - - if (--uNumToScan == 0) - { - break; - } - } - - if (list_empty(&g_sPagePoolList)) - { - PVR_ASSERT(atomic_read(&g_sPagePoolEntryCount) == 0); - } - - PagePoolUnlock(); - - PVR_DPF((PVR_DBG_MESSAGE,"%s: Pages in pool after scan: %d", __FUNCTION__, atomic_read(&g_sPagePoolEntryCount))); - } - - return atomic_read(&g_sPagePoolEntryCount); -} -#endif - -static IMG_BOOL -AllocPages(IMG_UINT32 ui32AreaFlags, struct page ***pppsPageList, IMG_HANDLE *phBlockPageList, IMG_UINT32 ui32NumPages, IMG_BOOL *pbFromPagePool) -{ - struct page **ppsPageList; - IMG_HANDLE hBlockPageList; - IMG_INT32 i; /* Must be signed; see "for" loop conditions */ - PVRSRV_ERROR eError; - IMG_BOOL bFromPagePool = IMG_FALSE; - - eError = OSAllocMem(0, sizeof(*ppsPageList) * ui32NumPages, (IMG_VOID **)&ppsPageList, &hBlockPageList, - "Array of pages"); - if (eError != PVRSRV_OK) - { - goto failed_page_list_alloc; - } - - *pbFromPagePool = IMG_TRUE; - for(i = 0; i < (IMG_INT32)ui32NumPages; i++) - { - ppsPageList[i] = AllocPage(ui32AreaFlags, &bFromPagePool); - if (!ppsPageList[i]) - { - goto failed_alloc_pages; - } - *pbFromPagePool &= bFromPagePool; - } - - *pppsPageList = ppsPageList; - *phBlockPageList = hBlockPageList; - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, - ppsPageList, - 0, - 0, - NULL, - PAGES_TO_BYTES(ui32NumPages), - "unknown", - 0 - ); -#endif - - return IMG_TRUE; - -failed_alloc_pages: - for(i--; i >= 0; i--) - { - FreePage(*pbFromPagePool, ppsPageList[i]); - } - (IMG_VOID) OSFreeMem(0, sizeof(*ppsPageList) * ui32NumPages, ppsPageList, hBlockPageList); - -failed_page_list_alloc: - return IMG_FALSE; -} - - -static IMG_VOID -FreePages(IMG_BOOL bToPagePool, struct page **ppsPageList, IMG_HANDLE hBlockPageList, IMG_UINT32 ui32NumPages) -{ - IMG_INT32 i; - - for(i = 0; i < (IMG_INT32)ui32NumPages; i++) - { - FreePage(bToPagePool, ppsPageList[i]); - } - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, ppsPageList, __FILE__, __LINE__); -#endif - - (IMG_VOID) OSFreeMem(0, sizeof(*ppsPageList) * ui32NumPages, ppsPageList, hBlockPageList); -} - - -LinuxMemArea * -NewVMallocLinuxMemArea(IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AreaFlags) -{ - LinuxMemArea *psLinuxMemArea = NULL; - IMG_VOID *pvCpuVAddr; -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - IMG_UINT32 ui32NumPages = 0; - struct page **ppsPageList = NULL; - IMG_HANDLE hBlockPageList; -#endif - IMG_BOOL bFromPagePool = IMG_FALSE; - - psLinuxMemArea = LinuxMemAreaStructAlloc(); - if (!psLinuxMemArea) - { - goto failed; - } - -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - ui32NumPages = RANGE_TO_PAGES(ui32Bytes); - - if (!AllocPages(ui32AreaFlags, &ppsPageList, &hBlockPageList, ui32NumPages, &bFromPagePool)) - { - goto failed; - } - - pvCpuVAddr = VMapWrapper(ppsPageList, ui32NumPages, ui32AreaFlags); -#else /* defined(PVR_LINUX_MEM_AREA_USE_VMAP) */ - pvCpuVAddr = VMallocWrapper(ui32Bytes, ui32AreaFlags); - if (!pvCpuVAddr) - { - goto failed; - } -/* PG_reserved was deprecated in linux-2.6.15 */ -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) - /* Reserve those pages to allow them to be re-mapped to user space */ - ReservePages(pvCpuVAddr, ui32Bytes); -#endif -#endif /* defined(PVR_LINUX_MEM_AREA_USE_VMAP) */ - - psLinuxMemArea->eAreaType = LINUX_MEM_AREA_VMALLOC; - psLinuxMemArea->uData.sVmalloc.pvVmallocAddress = pvCpuVAddr; -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - psLinuxMemArea->uData.sVmalloc.ppsPageList = ppsPageList; - psLinuxMemArea->uData.sVmalloc.hBlockPageList = hBlockPageList; -#endif - psLinuxMemArea->ui32ByteSize = ui32Bytes; - psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; - INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); -#endif - - /* This works around a problem where Linux will not invalidate - * the cache for physical memory it frees that is direct mapped. - * - * As a result, cache entries remain that may be subsequently flushed - * to these physical pages after they have been allocated for another - * purpose. For a subsequent cached use of this memory, that is not a - * problem, but if we are allocating uncached or write-combined memory, - * and bypassing the cache, it can cause subsequent uncached writes to - * the memory to be replaced with junk from the cache. - * - * If the pages are from our page cache, no cache invalidate is needed. - * - * This just handles the __vmalloc() case (when we have a kernel virtual - * address range). The alloc_pages() path is handled in mmap.c. - */ - if (AreaIsUncached(ui32AreaFlags) && !bFromPagePool) - { - OSInvalidateCPUCacheRangeKM(psLinuxMemArea, 0, pvCpuVAddr, ui32Bytes); - } - - return psLinuxMemArea; - -failed: - PVR_DPF((PVR_DBG_ERROR, "%s: failed!", __FUNCTION__)); -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - if (ppsPageList) - { - FreePages(bFromPagePool, ppsPageList, hBlockPageList, ui32NumPages); - } -#endif - if (psLinuxMemArea) - { - LinuxMemAreaStructFree(psLinuxMemArea); - } - - return NULL; -} - - -IMG_VOID -FreeVMallocLinuxMemArea(LinuxMemArea *psLinuxMemArea) -{ -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - IMG_UINT32 ui32NumPages; - struct page **ppsPageList; - IMG_HANDLE hBlockPageList; -#endif - - PVR_ASSERT(psLinuxMemArea); - PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_VMALLOC); - PVR_ASSERT(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordRemove(psLinuxMemArea); -#endif - - PVR_DPF((PVR_DBG_MESSAGE,"%s: pvCpuVAddr: %p", - __FUNCTION__, psLinuxMemArea->uData.sVmalloc.pvVmallocAddress)); - -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - VUnmapWrapper(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress); - - ui32NumPages = RANGE_TO_PAGES(psLinuxMemArea->ui32ByteSize); - ppsPageList = psLinuxMemArea->uData.sVmalloc.ppsPageList; - hBlockPageList = psLinuxMemArea->uData.sVmalloc.hBlockPageList; - - FreePages(CanFreeToPool(psLinuxMemArea), ppsPageList, hBlockPageList, ui32NumPages); -#else -/* PG_reserved was deprecated in linux-2.6.15 */ -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) - UnreservePages(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress, - psLinuxMemArea->ui32ByteSize); -#endif - - VFreeWrapper(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress); -#endif /* defined(PVR_LINUX_MEM_AREA_USE_VMAP) */ - - LinuxMemAreaStructFree(psLinuxMemArea); -} - - -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) -/* Reserve pages of memory in order that they're not automatically - deallocated after the last user reference dies. */ -static IMG_VOID -ReservePages(IMG_VOID *pvAddress, IMG_UINT32 ui32Length) -{ - IMG_VOID *pvPage; - IMG_VOID *pvEnd = pvAddress + ui32Length; - - for(pvPage = pvAddress; pvPage < pvEnd; pvPage += PAGE_SIZE) - { -#if (LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)) - SetPageReserved(vmalloc_to_page(pvPage)); -#else - mem_map_reserve(vmalloc_to_page(pvPage)); -#endif - } -} - - -/* Un-reserve pages of memory in order that they can be freed. */ -static IMG_VOID -UnreservePages(IMG_VOID *pvAddress, IMG_UINT32 ui32Length) -{ - IMG_VOID *pvPage; - IMG_VOID *pvEnd = pvAddress + ui32Length; - - for(pvPage = pvAddress; pvPage < pvEnd; pvPage += PAGE_SIZE) - { -#if (LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)) - ClearPageReserved(vmalloc_to_page(pvPage)); -#else - mem_map_unreserve(vmalloc_to_page(pvPage)); -#endif - } -} -#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) */ - - -IMG_VOID * -_IORemapWrapper(IMG_CPU_PHYADDR BasePAddr, - IMG_UINT32 ui32Bytes, - IMG_UINT32 ui32MappingFlags, - IMG_CHAR *pszFileName, - IMG_UINT32 ui32Line) -{ - IMG_VOID *pvIORemapCookie; - - switch (ui32MappingFlags & PVRSRV_HAP_CACHETYPE_MASK) - { - case PVRSRV_HAP_CACHED: - pvIORemapCookie = (IMG_VOID *)IOREMAP(BasePAddr.uiAddr, ui32Bytes); - break; - case PVRSRV_HAP_WRITECOMBINE: - pvIORemapCookie = (IMG_VOID *)IOREMAP_WC(BasePAddr.uiAddr, ui32Bytes); - break; - case PVRSRV_HAP_UNCACHED: - pvIORemapCookie = (IMG_VOID *)IOREMAP_UC(BasePAddr.uiAddr, ui32Bytes); - break; - default: - PVR_DPF((PVR_DBG_ERROR, "IORemapWrapper: unknown mapping flags")); - return NULL; - } - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - if (pvIORemapCookie) - { - DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_IOREMAP, - pvIORemapCookie, - pvIORemapCookie, - BasePAddr.uiAddr, - NULL, - ui32Bytes, - pszFileName, - ui32Line - ); - } -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - - return pvIORemapCookie; -} - - -IMG_VOID -_IOUnmapWrapper(IMG_VOID *pvIORemapCookie, IMG_CHAR *pszFileName, IMG_UINT32 ui32Line) -{ -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_IOREMAP, pvIORemapCookie, pszFileName, ui32Line); -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - iounmap(pvIORemapCookie); -} - - -LinuxMemArea * -NewIORemapLinuxMemArea(IMG_CPU_PHYADDR BasePAddr, - IMG_UINT32 ui32Bytes, - IMG_UINT32 ui32AreaFlags) -{ - LinuxMemArea *psLinuxMemArea; - IMG_VOID *pvIORemapCookie; - - psLinuxMemArea = LinuxMemAreaStructAlloc(); - if (!psLinuxMemArea) - { - return NULL; - } - - pvIORemapCookie = IORemapWrapper(BasePAddr, ui32Bytes, ui32AreaFlags); - if (!pvIORemapCookie) - { - LinuxMemAreaStructFree(psLinuxMemArea); - return NULL; - } - - psLinuxMemArea->eAreaType = LINUX_MEM_AREA_IOREMAP; - psLinuxMemArea->uData.sIORemap.pvIORemapCookie = pvIORemapCookie; - psLinuxMemArea->uData.sIORemap.CPUPhysAddr = BasePAddr; - psLinuxMemArea->ui32ByteSize = ui32Bytes; - psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; - INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); -#endif - - return psLinuxMemArea; -} - - -IMG_VOID -FreeIORemapLinuxMemArea(LinuxMemArea *psLinuxMemArea) -{ - PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_IOREMAP); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordRemove(psLinuxMemArea); -#endif - - IOUnmapWrapper(psLinuxMemArea->uData.sIORemap.pvIORemapCookie); - - LinuxMemAreaStructFree(psLinuxMemArea); -} - - -#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL) -/* - * Avoid using remap_pfn_range on RAM, if possible. On x86 systems, with - * PAT enabled, remap_pfn_range checks the page attributes requested by - * remap_pfn_range against those of the direct kernel mapping for those - * pages (if any). This is rather annoying if the pages have been obtained - * with alloc_pages, where we just ask for raw pages; we don't care about - * the direct mapping. This latter issue arises when device memory is - * exported from one process to another. Services implements this - * using memory wrapping, which ends up creating an external KV memory area. - */ -static IMG_BOOL -TreatExternalPagesAsContiguous(IMG_SYS_PHYADDR *psSysPhysAddr, IMG_UINT32 ui32Bytes, IMG_BOOL bPhysContig) -{ - IMG_UINT32 ui32; - IMG_UINT32 ui32AddrChk; - IMG_UINT32 ui32NumPages = RANGE_TO_PAGES(ui32Bytes); - - /* - * If bPhysContig is IMG_TRUE, we must assume psSysPhysAddr points - * to the address of the first page, not an array of page addresses. - */ - for (ui32 = 0, ui32AddrChk = psSysPhysAddr[0].uiAddr; - ui32 < ui32NumPages; - ui32++, ui32AddrChk = (bPhysContig) ? (ui32AddrChk + PAGE_SIZE) : psSysPhysAddr[ui32].uiAddr) - { - if (!pfn_valid(PHYS_TO_PFN(ui32AddrChk))) - { - break; - } - } - if (ui32 == ui32NumPages) - { - return IMG_FALSE; - } - - if (!bPhysContig) - { - for (ui32 = 0, ui32AddrChk = psSysPhysAddr[0].uiAddr; - ui32 < ui32NumPages; - ui32++, ui32AddrChk += PAGE_SIZE) - { - if (psSysPhysAddr[ui32].uiAddr != ui32AddrChk) - { - return IMG_FALSE; - } - } - } - - return IMG_TRUE; -} -#endif - -LinuxMemArea *NewExternalKVLinuxMemArea(IMG_SYS_PHYADDR *pBasePAddr, IMG_VOID *pvCPUVAddr, IMG_UINT32 ui32Bytes, IMG_BOOL bPhysContig, IMG_UINT32 ui32AreaFlags) -{ - LinuxMemArea *psLinuxMemArea; - - psLinuxMemArea = LinuxMemAreaStructAlloc(); - if (!psLinuxMemArea) - { - return NULL; - } - - psLinuxMemArea->eAreaType = LINUX_MEM_AREA_EXTERNAL_KV; - psLinuxMemArea->uData.sExternalKV.pvExternalKV = pvCPUVAddr; - psLinuxMemArea->uData.sExternalKV.bPhysContig = -#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL) - (bPhysContig || TreatExternalPagesAsContiguous(pBasePAddr, ui32Bytes, bPhysContig)) - ? IMG_TRUE : IMG_FALSE; -#else - bPhysContig; -#endif - if (psLinuxMemArea->uData.sExternalKV.bPhysContig) - { - psLinuxMemArea->uData.sExternalKV.uPhysAddr.SysPhysAddr = *pBasePAddr; - } - else - { - psLinuxMemArea->uData.sExternalKV.uPhysAddr.pSysPhysAddr = pBasePAddr; - } - psLinuxMemArea->ui32ByteSize = ui32Bytes; - psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; - INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); -#endif - - return psLinuxMemArea; -} - - -IMG_VOID -FreeExternalKVLinuxMemArea(LinuxMemArea *psLinuxMemArea) -{ - PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_EXTERNAL_KV); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordRemove(psLinuxMemArea); -#endif - - LinuxMemAreaStructFree(psLinuxMemArea); -} - - -LinuxMemArea * -NewIOLinuxMemArea(IMG_CPU_PHYADDR BasePAddr, - IMG_UINT32 ui32Bytes, - IMG_UINT32 ui32AreaFlags) -{ - LinuxMemArea *psLinuxMemArea = LinuxMemAreaStructAlloc(); - if (!psLinuxMemArea) - { - return NULL; - } - - /* Nothing to activly do. We just keep a record of the physical range. */ - psLinuxMemArea->eAreaType = LINUX_MEM_AREA_IO; - psLinuxMemArea->uData.sIO.CPUPhysAddr.uiAddr = BasePAddr.uiAddr; - psLinuxMemArea->ui32ByteSize = ui32Bytes; - psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; - INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_IO, - (IMG_VOID *)BasePAddr.uiAddr, - 0, - BasePAddr.uiAddr, - NULL, - ui32Bytes, - "unknown", - 0 - ); -#endif - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); -#endif - - return psLinuxMemArea; -} - - -IMG_VOID -FreeIOLinuxMemArea(LinuxMemArea *psLinuxMemArea) -{ - PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_IO); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordRemove(psLinuxMemArea); -#endif - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_IO, - (IMG_VOID *)psLinuxMemArea->uData.sIO.CPUPhysAddr.uiAddr, __FILE__, __LINE__); -#endif - - /* Nothing more to do than free the LinuxMemArea struct */ - - LinuxMemAreaStructFree(psLinuxMemArea); -} - - -LinuxMemArea * -NewAllocPagesLinuxMemArea(IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AreaFlags) -{ - LinuxMemArea *psLinuxMemArea; - IMG_UINT32 ui32NumPages; - struct page **ppsPageList; - IMG_HANDLE hBlockPageList; - IMG_BOOL bFromPagePool; - - psLinuxMemArea = LinuxMemAreaStructAlloc(); - if (!psLinuxMemArea) - { - goto failed_area_alloc; - } - - ui32NumPages = RANGE_TO_PAGES(ui32Bytes); - - if (!AllocPages(ui32AreaFlags, &ppsPageList, &hBlockPageList, ui32NumPages, &bFromPagePool)) - { - goto failed_alloc_pages; - } - - psLinuxMemArea->eAreaType = LINUX_MEM_AREA_ALLOC_PAGES; - psLinuxMemArea->uData.sPageList.ppsPageList = ppsPageList; - psLinuxMemArea->uData.sPageList.hBlockPageList = hBlockPageList; - psLinuxMemArea->ui32ByteSize = ui32Bytes; - psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; - INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); - - /* We defer the cache flush to the first user mapping of this memory */ - psLinuxMemArea->bNeedsCacheInvalidate = AreaIsUncached(ui32AreaFlags) && !bFromPagePool; - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); -#endif - - return psLinuxMemArea; - -failed_alloc_pages: - LinuxMemAreaStructFree(psLinuxMemArea); -failed_area_alloc: - PVR_DPF((PVR_DBG_ERROR, "%s: failed", __FUNCTION__)); - - return NULL; -} - - -IMG_VOID -FreeAllocPagesLinuxMemArea(LinuxMemArea *psLinuxMemArea) -{ - IMG_UINT32 ui32NumPages; - struct page **ppsPageList; - IMG_HANDLE hBlockPageList; - - PVR_ASSERT(psLinuxMemArea); - PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_ALLOC_PAGES); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordRemove(psLinuxMemArea); -#endif - - ui32NumPages = RANGE_TO_PAGES(psLinuxMemArea->ui32ByteSize); - ppsPageList = psLinuxMemArea->uData.sPageList.ppsPageList; - hBlockPageList = psLinuxMemArea->uData.sPageList.hBlockPageList; - - FreePages(CanFreeToPool(psLinuxMemArea), ppsPageList, hBlockPageList, ui32NumPages); - - LinuxMemAreaStructFree(psLinuxMemArea); -} - -#if defined(CONFIG_ION_OMAP) - -#include "env_perproc.h" - -#include <linux/ion.h> -#include <linux/omap_ion.h> -#include <linux/scatterlist.h> - -extern struct ion_client *gpsIONClient; - -LinuxMemArea * -NewIONLinuxMemArea(IMG_UINT32 ui32Bytes, IMG_UINT32 ui32AreaFlags, - IMG_PVOID pvPrivData, IMG_UINT32 ui32PrivDataLength) -{ - const IMG_UINT32 ui32AllocDataLen = - offsetof(struct omap_ion_tiler_alloc_data, handle); - struct omap_ion_tiler_alloc_data asAllocData[PVRSRV_MAX_NUMBER_OF_MM_BUFFER_PLANES]; - u32 *pu32PageAddrs[PVRSRV_MAX_NUMBER_OF_MM_BUFFER_PLANES] = { NULL, NULL, NULL}; - IMG_UINT32 i, j, ui32NumHandlesPerFd; - IMG_BYTE *pbPrivData = pvPrivData; - IMG_CPU_PHYADDR *pCPUPhysAddrs; - IMG_UINT32 iNumPages[PVRSRV_MAX_NUMBER_OF_MM_BUFFER_PLANES] = { 0, 0, 0}; - LinuxMemArea *psLinuxMemArea; - IMG_UINT32 ui32ProcID; - IMG_UINT32 ui32TotalPagesSizeInBytes = 0, ui32TotalPages = 0; - - psLinuxMemArea = LinuxMemAreaStructAlloc(); - if (!psLinuxMemArea) - { - PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate LinuxMemArea struct", __func__)); - goto err_out; - } - - /* Depending on the UM config, userspace might give us info for - * one, two or three ION allocations. Divide the total size of data we - * were given by this ui32AllocDataLen, and check it's 1 or 2. - * Otherwise abort. - */ - BUG_ON(ui32PrivDataLength != ui32AllocDataLen && - ui32PrivDataLength != ui32AllocDataLen * 2 && - ui32PrivDataLength != ui32AllocDataLen * 3); - /* This is bad !- change this logic to pass in the size or - * use uniformed API */ - ui32NumHandlesPerFd = ui32PrivDataLength / ui32AllocDataLen; - - ui32ProcID = OSGetCurrentProcessIDKM(); - - memset(asAllocData, 0x00, sizeof(asAllocData)); - - /* We do not care about what the first (Y) buffer offset would be, - * but we do care for the UV buffers to be co-aligned with Y - * This for SGX to find the UV offset solely based on the height - * and stride of the YUV buffer.This is very important for OMAP4470 - * and later chipsets, where SGX version is 544. 544 and later use - * non-shader based YUV to RGB conversion unit that require - * contiguous GPU virtual space */ - for(i = 0; i < ui32NumHandlesPerFd; i++) - { - memcpy(&asAllocData[i], &pbPrivData[i * ui32AllocDataLen], ui32AllocDataLen); - asAllocData[i].token = ui32ProcID; - -#ifndef SGX_DISABLE_DMM_OFFSET_BUFFER_ALLOCATIONS - if(i == 0) - { - /* Tiler API says: - * Allocate first buffer with the required alignment - * and an offset of 0 ... */ - asAllocData[i].out_align = CONFIG_TILER_GRANULARITY; - asAllocData[i].offset = 0; - } - else - { /* .. Then for the second buffer, use the offset from the first - * buffer with alignment of PAGE_SIZE */ - asAllocData[i].out_align = PAGE_SIZE; - asAllocData[i].offset = asAllocData[0].offset; - } -#else - asAllocData[i].offset = 0; - asAllocData[i].out_align = PAGE_SIZE; -#endif - - if(asAllocData[i].fmt == TILER_PIXEL_FMT_PAGE) - { - /* 1D DMM Buffers */ - struct scatterlist *sg, *sglist; - IMG_UINT32 ui32Num1dPages; - - asAllocData[i].handle = ion_alloc (gpsIONClient, - ui32Bytes, - PAGE_SIZE, (1 << OMAP_ION_HEAP_SYSTEM)); - - if (asAllocData[i].handle == NULL) - { - PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate via ion_alloc", - __func__)); - goto err_free; - } - - sglist = ion_map_dma (gpsIONClient, asAllocData[i].handle); - if (sglist == NULL) - { - PVR_DPF((PVR_DBG_ERROR, "%s: Failed to compute pages", - __func__)); - goto err_free; - } - - ui32Num1dPages = (ui32Bytes >> PAGE_SHIFT); - pu32PageAddrs[i] = kmalloc (sizeof(u32) * ui32Num1dPages, GFP_KERNEL); - if (pu32PageAddrs[i] == NULL) - { - PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate page array", - __func__)); - goto err_free; - } - - for_each_sg (sglist, sg, ui32Num1dPages, j) - { - pu32PageAddrs[i][j] = sg_phys (sg); - } - - iNumPages[i] = ui32Num1dPages; - } - else /* 2D DMM Buffers */ - { - if (omap_ion_tiler_alloc(gpsIONClient, &asAllocData[i]) < 0) - { - PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate via ion_tiler", - __func__)); - goto err_free; - } - - if (omap_tiler_pages(gpsIONClient, asAllocData[i].handle, &iNumPages[i], - &pu32PageAddrs[i]) < 0) - { - PVR_DPF((PVR_DBG_ERROR, "%s: Failed to compute tiler pages", - __func__)); - goto err_free; - } - } - } - - /* Basic sanity check on plane co-alignment */ - if((ui32NumHandlesPerFd > 1) && - (asAllocData[0].offset != asAllocData[1].offset)) - { - pr_err("%s: Y and UV offsets do not match for tiler handles " - "%p,%p: %d != %d \n " - "Expect issues with SGX544xx and later chipsets\n", - __func__, asAllocData[0].handle, asAllocData[1].handle, - (int)asAllocData[0].offset, (int)asAllocData[1].offset); - } - - /* Assume the user-allocator has already done the tiler math and that the - * number of tiler pages allocated matches any other allocation type. - */ - for(i = 0; i < ui32NumHandlesPerFd; i++) - { - ui32TotalPages += iNumPages[i]; - } - - BUG_ON(ui32Bytes != (ui32TotalPages * PAGE_SIZE)); - BUG_ON(sizeof(IMG_CPU_PHYADDR) != sizeof(int)); - - /* Glue the page lists together */ - pCPUPhysAddrs = vmalloc(sizeof(IMG_CPU_PHYADDR) * ui32TotalPages); - if (!pCPUPhysAddrs) - { - PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate page list", __func__)); - goto err_free; - } - - j = 0; - for(i = 0; i < ui32NumHandlesPerFd; i++) - { - IMG_UINT32 ui32PageIndx; - for(ui32PageIndx = 0; ui32PageIndx < iNumPages[i]; ui32PageIndx++) - { - pCPUPhysAddrs[j++].uiAddr = pu32PageAddrs[i][ui32PageIndx]; - } - - psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i] = - asAllocData[i].handle; - psLinuxMemArea->uData.sIONTilerAlloc.planeOffsets[i] = - ui32TotalPagesSizeInBytes + asAllocData[i].offset; - /* Add the number of pages this plane consists of */ - ui32TotalPagesSizeInBytes += (iNumPages[i] * PAGE_SIZE); - } - - psLinuxMemArea->eAreaType = LINUX_MEM_AREA_ION; - psLinuxMemArea->uData.sIONTilerAlloc.pCPUPhysAddrs = pCPUPhysAddrs; - psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes = - ui32NumHandlesPerFd; - psLinuxMemArea->ui32ByteSize = ui32TotalPagesSizeInBytes; - psLinuxMemArea->ui32AreaFlags = ui32AreaFlags; - INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); - - /* We defer the cache flush to the first user mapping of this memory */ - psLinuxMemArea->bNeedsCacheInvalidate = AreaIsUncached(ui32AreaFlags); - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_ION, - asAllocData[0].handle, - 0, - 0, - NULL, - PAGE_ALIGN(ui32Bytes), - "unknown", - 0 - ); -#endif - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags); -#endif - -err_out: - return psLinuxMemArea; - -err_free: - LinuxMemAreaStructFree(psLinuxMemArea); - psLinuxMemArea = IMG_NULL; - goto err_out; -} - -IMG_INT32 -GetIONLinuxMemAreaInfo(LinuxMemArea *psLinuxMemArea, IMG_UINT32* pui32AddressOffsets, - IMG_UINT32* ui32NumAddrOffsets) -{ - IMG_UINT32 i; - - if(!ui32NumAddrOffsets) - return -1; - - if(*ui32NumAddrOffsets < psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes) - { - *ui32NumAddrOffsets = psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes; - return -1; - } - - if(!pui32AddressOffsets) - return -1; - - for(i = 0; i < psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes; i++) - { - if(psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i]) - pui32AddressOffsets[i] = - psLinuxMemArea->uData.sIONTilerAlloc.planeOffsets[i]; - } - - *ui32NumAddrOffsets = psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes; - - return psLinuxMemArea->ui32ByteSize; -} - -IMG_VOID -FreeIONLinuxMemArea(LinuxMemArea *psLinuxMemArea) -{ - IMG_UINT32 i; - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordRemove(psLinuxMemArea); -#endif - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_ION, - psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[0], - __FILE__, __LINE__); -#endif - - for(i = 0; i < psLinuxMemArea->uData.sIONTilerAlloc.ui32NumValidPlanes; i++) - { - if (!psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i]) - break; - ion_free(gpsIONClient, psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i]); - psLinuxMemArea->uData.sIONTilerAlloc.psIONHandle[i] = IMG_NULL; - } - - /* free copy of page list, originals are freed by ion_free */ - vfree(psLinuxMemArea->uData.sIONTilerAlloc.pCPUPhysAddrs); - psLinuxMemArea->uData.sIONTilerAlloc.pCPUPhysAddrs = IMG_NULL; - - LinuxMemAreaStructFree(psLinuxMemArea); -} - -#endif /* defined(CONFIG_ION_OMAP) */ - -struct page* -LinuxMemAreaOffsetToPage(LinuxMemArea *psLinuxMemArea, - IMG_UINT32 ui32ByteOffset) -{ - IMG_UINT32 ui32PageIndex; - IMG_CHAR *pui8Addr; - - switch (psLinuxMemArea->eAreaType) - { - case LINUX_MEM_AREA_ALLOC_PAGES: - ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset); - return psLinuxMemArea->uData.sPageList.ppsPageList[ui32PageIndex]; - - case LINUX_MEM_AREA_VMALLOC: - pui8Addr = psLinuxMemArea->uData.sVmalloc.pvVmallocAddress; - pui8Addr += ui32ByteOffset; - return vmalloc_to_page(pui8Addr); - - case LINUX_MEM_AREA_SUB_ALLOC: - /* PRQA S 3670 3 */ /* ignore recursive warning */ - return LinuxMemAreaOffsetToPage(psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea, - psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset - + ui32ByteOffset); - default: - PVR_DPF((PVR_DBG_ERROR, - "%s: Unsupported request for struct page from LinuxMemArea with type=%s", - __FUNCTION__, LinuxMemAreaTypeToString(psLinuxMemArea->eAreaType))); - return NULL; - } -} - - -LinuxKMemCache * -KMemCacheCreateWrapper(IMG_CHAR *pszName, - size_t Size, - size_t Align, - IMG_UINT32 ui32Flags) -{ -#if defined(DEBUG_LINUX_SLAB_ALLOCATIONS) - ui32Flags |= SLAB_POISON|SLAB_RED_ZONE; -#endif - return kmem_cache_create(pszName, Size, Align, ui32Flags, NULL -#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,22)) - , NULL -#endif /* (LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,22) */ - ); -} - - -IMG_VOID -KMemCacheDestroyWrapper(LinuxKMemCache *psCache) -{ - kmem_cache_destroy(psCache); -} - - -IMG_VOID * -_KMemCacheAllocWrapper(LinuxKMemCache *psCache, -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,14)) - gfp_t Flags, -#else - IMG_INT Flags, -#endif - IMG_CHAR *pszFileName, - IMG_UINT32 ui32Line) -{ - IMG_VOID *pvRet; - - pvRet = kmem_cache_zalloc(psCache, Flags); - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE, - pvRet, - pvRet, - 0, - psCache, - kmem_cache_size(psCache), - pszFileName, - ui32Line - ); -#else - PVR_UNREFERENCED_PARAMETER(pszFileName); - PVR_UNREFERENCED_PARAMETER(ui32Line); -#endif - - return pvRet; -} - - -LinuxMemArea * -NewSubLinuxMemArea(LinuxMemArea *psParentLinuxMemArea, - IMG_UINT32 ui32ByteOffset, - IMG_UINT32 ui32Bytes) -{ - LinuxMemArea *psLinuxMemArea; - - PVR_ASSERT((ui32ByteOffset+ui32Bytes) <= psParentLinuxMemArea->ui32ByteSize); - - psLinuxMemArea = LinuxMemAreaStructAlloc(); - if (!psLinuxMemArea) - { - return NULL; - } - - psLinuxMemArea->eAreaType = LINUX_MEM_AREA_SUB_ALLOC; - psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea = psParentLinuxMemArea; - psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset = ui32ByteOffset; - psLinuxMemArea->ui32ByteSize = ui32Bytes; - psLinuxMemArea->ui32AreaFlags = psParentLinuxMemArea->ui32AreaFlags; - psLinuxMemArea->bNeedsCacheInvalidate = psParentLinuxMemArea->bNeedsCacheInvalidate; - INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList); - -#if defined(DEBUG_LINUX_MEM_AREAS) - { - DEBUG_LINUX_MEM_AREA_REC *psParentRecord; - psParentRecord = DebugLinuxMemAreaRecordFind(psParentLinuxMemArea); - DebugLinuxMemAreaRecordAdd(psLinuxMemArea, psParentRecord->ui32Flags); - } -#endif - - return psLinuxMemArea; -} - - -static IMG_VOID -FreeSubLinuxMemArea(LinuxMemArea *psLinuxMemArea) -{ - PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_SUB_ALLOC); - -#if defined(DEBUG_LINUX_MEM_AREAS) - DebugLinuxMemAreaRecordRemove(psLinuxMemArea); -#endif - - /* Nothing more to do than free the LinuxMemArea structure */ - - LinuxMemAreaStructFree(psLinuxMemArea); -} - - -static LinuxMemArea * -LinuxMemAreaStructAlloc(IMG_VOID) -{ -/* debug */ -#if 0 - LinuxMemArea *psLinuxMemArea; - psLinuxMemArea = kmem_cache_alloc(g_PsLinuxMemAreaCache, GFP_KERNEL); - printk(KERN_ERR "%s: psLinuxMemArea=%p\n", __FUNCTION__, psLinuxMemArea); - dump_stack(); - return psLinuxMemArea; -#else - return KMemCacheAllocWrapper(g_PsLinuxMemAreaCache, GFP_KERNEL); -#endif -} - - -static IMG_VOID -LinuxMemAreaStructFree(LinuxMemArea *psLinuxMemArea) -{ - KMemCacheFreeWrapper(g_PsLinuxMemAreaCache, psLinuxMemArea); - /* debug */ - //printk(KERN_ERR "%s(%p)\n", __FUNCTION__, psLinuxMemArea); -} - - -IMG_VOID -LinuxMemAreaDeepFree(LinuxMemArea *psLinuxMemArea) -{ - switch (psLinuxMemArea->eAreaType) - { - case LINUX_MEM_AREA_VMALLOC: - FreeVMallocLinuxMemArea(psLinuxMemArea); - break; - case LINUX_MEM_AREA_ALLOC_PAGES: - FreeAllocPagesLinuxMemArea(psLinuxMemArea); - break; - case LINUX_MEM_AREA_IOREMAP: - FreeIORemapLinuxMemArea(psLinuxMemArea); - break; - case LINUX_MEM_AREA_EXTERNAL_KV: - FreeExternalKVLinuxMemArea(psLinuxMemArea); - break; - case LINUX_MEM_AREA_IO: - FreeIOLinuxMemArea(psLinuxMemArea); - break; - case LINUX_MEM_AREA_SUB_ALLOC: - FreeSubLinuxMemArea(psLinuxMemArea); - break; - case LINUX_MEM_AREA_ION: - FreeIONLinuxMemArea(psLinuxMemArea); - break; - default: - PVR_DPF((PVR_DBG_ERROR, "%s: Unknown are type (%d)\n", - __FUNCTION__, psLinuxMemArea->eAreaType)); - break; - } -} - - -#if defined(DEBUG_LINUX_MEM_AREAS) -static IMG_VOID -DebugLinuxMemAreaRecordAdd(LinuxMemArea *psLinuxMemArea, IMG_UINT32 ui32Flags) -{ - DEBUG_LINUX_MEM_AREA_REC *psNewRecord; - const IMG_CHAR *pi8FlagsString; - - LinuxLockMutex(&g_sDebugMutex); - - if (psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC) - { - g_LinuxMemAreaWaterMark += psLinuxMemArea->ui32ByteSize; - if (g_LinuxMemAreaWaterMark > g_LinuxMemAreaHighWaterMark) - { - g_LinuxMemAreaHighWaterMark = g_LinuxMemAreaWaterMark; - } - } - g_LinuxMemAreaCount++; - - /* Create a new memory allocation record */ - psNewRecord = kmalloc(sizeof(DEBUG_LINUX_MEM_AREA_REC), GFP_KERNEL); - if (psNewRecord) - { - /* Record the allocation */ - psNewRecord->psLinuxMemArea = psLinuxMemArea; - psNewRecord->ui32Flags = ui32Flags; - psNewRecord->pid = OSGetCurrentProcessIDKM(); - - List_DEBUG_LINUX_MEM_AREA_REC_Insert(&g_LinuxMemAreaRecords, psNewRecord); - } - else - { - PVR_DPF((PVR_DBG_ERROR, - "%s: failed to allocate linux memory area record.", - __FUNCTION__)); - } - - /* Sanity check the flags */ - pi8FlagsString = HAPFlagsToString(ui32Flags); - if (strstr(pi8FlagsString, "UNKNOWN")) - { - PVR_DPF((PVR_DBG_ERROR, - "%s: Unexpected flags (0x%08x) associated with psLinuxMemArea @ %p", - __FUNCTION__, - ui32Flags, - psLinuxMemArea)); - //dump_stack(); - } - - LinuxUnLockMutex(&g_sDebugMutex); -} - - - -static IMG_VOID* MatchLinuxMemArea_AnyVaCb(DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord, - va_list va) -{ - LinuxMemArea *psLinuxMemArea; - - psLinuxMemArea = va_arg(va, LinuxMemArea*); - if (psCurrentRecord->psLinuxMemArea == psLinuxMemArea) - { - return psCurrentRecord; - } - else - { - return IMG_NULL; - } -} - - -static DEBUG_LINUX_MEM_AREA_REC * -DebugLinuxMemAreaRecordFind(LinuxMemArea *psLinuxMemArea) -{ - DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord; - - LinuxLockMutex(&g_sDebugMutex); - psCurrentRecord = List_DEBUG_LINUX_MEM_AREA_REC_Any_va(g_LinuxMemAreaRecords, - MatchLinuxMemArea_AnyVaCb, - psLinuxMemArea); - -/*exit_unlock:*/ - LinuxUnLockMutex(&g_sDebugMutex); - - return psCurrentRecord; -} - - -static IMG_VOID -DebugLinuxMemAreaRecordRemove(LinuxMemArea *psLinuxMemArea) -{ - DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord; - - LinuxLockMutex(&g_sDebugMutex); - - if (psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC) - { - g_LinuxMemAreaWaterMark -= psLinuxMemArea->ui32ByteSize; - } - g_LinuxMemAreaCount--; - - /* Locate the corresponding allocation entry */ - psCurrentRecord = List_DEBUG_LINUX_MEM_AREA_REC_Any_va(g_LinuxMemAreaRecords, - MatchLinuxMemArea_AnyVaCb, - psLinuxMemArea); - if (psCurrentRecord) - { - /* Unlink the allocation record */ - List_DEBUG_LINUX_MEM_AREA_REC_Remove(psCurrentRecord); - kfree(psCurrentRecord); - } - else - { - PVR_DPF((PVR_DBG_ERROR, "%s: couldn't find an entry for psLinuxMemArea=%p\n", - __FUNCTION__, psLinuxMemArea)); - } - - LinuxUnLockMutex(&g_sDebugMutex); -} -#endif - - -IMG_VOID * -LinuxMemAreaToCpuVAddr(LinuxMemArea *psLinuxMemArea) -{ - switch (psLinuxMemArea->eAreaType) - { - case LINUX_MEM_AREA_VMALLOC: - return psLinuxMemArea->uData.sVmalloc.pvVmallocAddress; - case LINUX_MEM_AREA_IOREMAP: - return psLinuxMemArea->uData.sIORemap.pvIORemapCookie; - case LINUX_MEM_AREA_EXTERNAL_KV: - return psLinuxMemArea->uData.sExternalKV.pvExternalKV; - case LINUX_MEM_AREA_SUB_ALLOC: - { - IMG_CHAR *pAddr = - LinuxMemAreaToCpuVAddr(psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea); /* PRQA S 3670 */ /* ignore recursive warning */ - if (!pAddr) - { - return NULL; - } - return pAddr + psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset; - } - default: - return NULL; - } -} - - -IMG_CPU_PHYADDR -LinuxMemAreaToCpuPAddr(LinuxMemArea *psLinuxMemArea, IMG_UINT32 ui32ByteOffset) -{ - IMG_CPU_PHYADDR CpuPAddr; - - CpuPAddr.uiAddr = 0; - - switch (psLinuxMemArea->eAreaType) - { - case LINUX_MEM_AREA_IOREMAP: - { - CpuPAddr = psLinuxMemArea->uData.sIORemap.CPUPhysAddr; - CpuPAddr.uiAddr += ui32ByteOffset; - break; - } - case LINUX_MEM_AREA_EXTERNAL_KV: - { - if (psLinuxMemArea->uData.sExternalKV.bPhysContig) - { - CpuPAddr = SysSysPAddrToCpuPAddr(psLinuxMemArea->uData.sExternalKV.uPhysAddr.SysPhysAddr); - CpuPAddr.uiAddr += ui32ByteOffset; - } - else - { - IMG_UINT32 ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset); - IMG_SYS_PHYADDR SysPAddr = psLinuxMemArea->uData.sExternalKV.uPhysAddr.pSysPhysAddr[ui32PageIndex]; - - CpuPAddr = SysSysPAddrToCpuPAddr(SysPAddr); - CpuPAddr.uiAddr += ADDR_TO_PAGE_OFFSET(ui32ByteOffset); - } - break; - } - case LINUX_MEM_AREA_IO: - { - CpuPAddr = psLinuxMemArea->uData.sIO.CPUPhysAddr; - CpuPAddr.uiAddr += ui32ByteOffset; - break; - } - case LINUX_MEM_AREA_VMALLOC: - { - IMG_CHAR *pCpuVAddr; - pCpuVAddr = - (IMG_CHAR *)psLinuxMemArea->uData.sVmalloc.pvVmallocAddress; - pCpuVAddr += ui32ByteOffset; - CpuPAddr.uiAddr = VMallocToPhys(pCpuVAddr); - break; - } - case LINUX_MEM_AREA_ION: - { - IMG_UINT32 ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset); - CpuPAddr = psLinuxMemArea->uData.sIONTilerAlloc.pCPUPhysAddrs[ui32PageIndex]; - CpuPAddr.uiAddr += ADDR_TO_PAGE_OFFSET(ui32ByteOffset); - break; - } - case LINUX_MEM_AREA_ALLOC_PAGES: - { - struct page *page; - IMG_UINT32 ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset); - page = psLinuxMemArea->uData.sPageList.ppsPageList[ui32PageIndex]; - CpuPAddr.uiAddr = page_to_phys(page); - CpuPAddr.uiAddr += ADDR_TO_PAGE_OFFSET(ui32ByteOffset); - break; - } - case LINUX_MEM_AREA_SUB_ALLOC: - { - CpuPAddr = - OSMemHandleToCpuPAddr(psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea, - psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset - + ui32ByteOffset); - break; - } - default: - { - PVR_DPF((PVR_DBG_ERROR, "%s: Unknown LinuxMemArea type (%d)\n", - __FUNCTION__, psLinuxMemArea->eAreaType)); - PVR_ASSERT(CpuPAddr.uiAddr); - break; - } - } - - return CpuPAddr; -} - - -IMG_BOOL -LinuxMemAreaPhysIsContig(LinuxMemArea *psLinuxMemArea) -{ - switch (psLinuxMemArea->eAreaType) - { - case LINUX_MEM_AREA_IOREMAP: - case LINUX_MEM_AREA_IO: - return IMG_TRUE; - - case LINUX_MEM_AREA_EXTERNAL_KV: - return psLinuxMemArea->uData.sExternalKV.bPhysContig; - - case LINUX_MEM_AREA_ION: - case LINUX_MEM_AREA_VMALLOC: - case LINUX_MEM_AREA_ALLOC_PAGES: - return IMG_FALSE; - - case LINUX_MEM_AREA_SUB_ALLOC: - /* PRQA S 3670 1 */ /* ignore recursive warning */ - return LinuxMemAreaPhysIsContig(psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea); - - default: - PVR_DPF((PVR_DBG_ERROR, "%s: Unknown LinuxMemArea type (%d)\n", - __FUNCTION__, psLinuxMemArea->eAreaType)); - break; - } - return IMG_FALSE; -} - - -const IMG_CHAR * -LinuxMemAreaTypeToString(LINUX_MEM_AREA_TYPE eMemAreaType) -{ - /* Note we explicitly check the types instead of e.g. - * using the type to index an array of strings so - * we remain orthogonal to enum changes */ - switch (eMemAreaType) - { - case LINUX_MEM_AREA_IOREMAP: - return "LINUX_MEM_AREA_IOREMAP"; - case LINUX_MEM_AREA_EXTERNAL_KV: - return "LINUX_MEM_AREA_EXTERNAL_KV"; - case LINUX_MEM_AREA_IO: - return "LINUX_MEM_AREA_IO"; - case LINUX_MEM_AREA_VMALLOC: - return "LINUX_MEM_AREA_VMALLOC"; - case LINUX_MEM_AREA_SUB_ALLOC: - return "LINUX_MEM_AREA_SUB_ALLOC"; - case LINUX_MEM_AREA_ALLOC_PAGES: - return "LINUX_MEM_AREA_ALLOC_PAGES"; - case LINUX_MEM_AREA_ION: - return "LINUX_MEM_AREA_ION"; - default: - PVR_ASSERT(0); - } - - return ""; -} - - -#if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) -static void ProcSeqStartstopDebugMutex(struct seq_file *sfile, IMG_BOOL start) -{ - if (start) - { - LinuxLockMutex(&g_sDebugMutex); - } - else - { - LinuxUnLockMutex(&g_sDebugMutex); - } -} -#endif /* defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) */ - -#if defined(DEBUG_LINUX_MEM_AREAS) - -static IMG_VOID* DecOffMemAreaRec_AnyVaCb(DEBUG_LINUX_MEM_AREA_REC *psNode, va_list va) -{ - off_t *pOff = va_arg(va, off_t*); - if (--(*pOff)) - { - return IMG_NULL; - } - else - { - return psNode; - } -} - -/* seq_file version of generating output, for reference check proc.c:CreateProcReadEntrySeq */ -static void* ProcSeqNextMemArea(struct seq_file *sfile,void* el,loff_t off) -{ - DEBUG_LINUX_MEM_AREA_REC *psRecord; - psRecord = (DEBUG_LINUX_MEM_AREA_REC*) - List_DEBUG_LINUX_MEM_AREA_REC_Any_va(g_LinuxMemAreaRecords, - DecOffMemAreaRec_AnyVaCb, - &off); - return (void*)psRecord; -} - -static void* ProcSeqOff2ElementMemArea(struct seq_file * sfile, loff_t off) -{ - DEBUG_LINUX_MEM_AREA_REC *psRecord; - if (!off) - { - return PVR_PROC_SEQ_START_TOKEN; - } - - psRecord = (DEBUG_LINUX_MEM_AREA_REC*) - List_DEBUG_LINUX_MEM_AREA_REC_Any_va(g_LinuxMemAreaRecords, - DecOffMemAreaRec_AnyVaCb, - &off); - return (void*)psRecord; -} - - -static void ProcSeqShowMemArea(struct seq_file *sfile,void* el) -{ - DEBUG_LINUX_MEM_AREA_REC *psRecord = (DEBUG_LINUX_MEM_AREA_REC*)el; - if (el == PVR_PROC_SEQ_START_TOKEN) - { - -#if !defined(DEBUG_LINUX_XML_PROC_FILES) - seq_printf(sfile, - "Number of Linux Memory Areas: %u\n" - "At the current water mark these areas correspond to %u bytes (excluding SUB areas)\n" - "At the highest water mark these areas corresponded to %u bytes (excluding SUB areas)\n" - "\nDetails for all Linux Memory Areas:\n" - "%s %-24s %s %s %-8s %-5s %s\n", - g_LinuxMemAreaCount, - g_LinuxMemAreaWaterMark, - g_LinuxMemAreaHighWaterMark, - "psLinuxMemArea", - "LinuxMemType", - "CpuVAddr", - "CpuPAddr", - "Bytes", - "Pid", - "Flags" - ); -#else - seq_printf(sfile, - "<mem_areas_header>\n" - "\t<count>%u</count>\n" - "\t<watermark key=\"mar0\" description=\"current\" bytes=\"%u\"/>\n" /* (excluding SUB areas) */ - "\t<watermark key=\"mar1\" description=\"high\" bytes=\"%u\"/>\n" /* (excluding SUB areas) */ - "</mem_areas_header>\n", - g_LinuxMemAreaCount, - g_LinuxMemAreaWaterMark, - g_LinuxMemAreaHighWaterMark - ); -#endif - return; - } - - seq_printf(sfile, -#if !defined(DEBUG_LINUX_XML_PROC_FILES) - "%8p %-24s %8p %08x %-8d %-5u %08x=(%s)\n", -#else - "<linux_mem_area>\n" - "\t<pointer>%8p</pointer>\n" - "\t<type>%s</type>\n" - "\t<cpu_virtual>%8p</cpu_virtual>\n" - "\t<cpu_physical>%08x</cpu_physical>\n" - "\t<bytes>%d</bytes>\n" - "\t<pid>%u</pid>\n" - "\t<flags>%08x</flags>\n" - "\t<flags_string>%s</flags_string>\n" - "</linux_mem_area>\n", -#endif - psRecord->psLinuxMemArea, - LinuxMemAreaTypeToString(psRecord->psLinuxMemArea->eAreaType), - LinuxMemAreaToCpuVAddr(psRecord->psLinuxMemArea), - LinuxMemAreaToCpuPAddr(psRecord->psLinuxMemArea,0).uiAddr, - psRecord->psLinuxMemArea->ui32ByteSize, - psRecord->pid, - psRecord->ui32Flags, - HAPFlagsToString(psRecord->ui32Flags) - ); - -} - -#endif /* DEBUG_LINUX_MEM_AREAS */ - - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - -static IMG_VOID* DecOffMemAllocRec_AnyVaCb(DEBUG_MEM_ALLOC_REC *psNode, va_list va) -{ - off_t *pOff = va_arg(va, off_t*); - if (--(*pOff)) - { - return IMG_NULL; - } - else - { - return psNode; - } -} - - -/* seq_file version of generating output, for reference check proc.c:CreateProcReadEntrySeq */ -static void* ProcSeqNextMemoryRecords(struct seq_file *sfile,void* el,loff_t off) -{ - DEBUG_MEM_ALLOC_REC *psRecord; - psRecord = (DEBUG_MEM_ALLOC_REC*) - List_DEBUG_MEM_ALLOC_REC_Any_va(g_MemoryRecords, - DecOffMemAllocRec_AnyVaCb, - &off); -#if defined(DEBUG_LINUX_XML_PROC_FILES) - if (!psRecord) - { - seq_printf(sfile, "</meminfo>\n"); - } -#endif - - return (void*)psRecord; -} - -static void* ProcSeqOff2ElementMemoryRecords(struct seq_file *sfile, loff_t off) -{ - DEBUG_MEM_ALLOC_REC *psRecord; - if (!off) - { - return PVR_PROC_SEQ_START_TOKEN; - } - - psRecord = (DEBUG_MEM_ALLOC_REC*) - List_DEBUG_MEM_ALLOC_REC_Any_va(g_MemoryRecords, - DecOffMemAllocRec_AnyVaCb, - &off); - -#if defined(DEBUG_LINUX_XML_PROC_FILES) - if (!psRecord) - { - seq_printf(sfile, "</meminfo>\n"); - } -#endif - - return (void*)psRecord; -} - -static void ProcSeqShowMemoryRecords(struct seq_file *sfile,void* el) -{ - DEBUG_MEM_ALLOC_REC *psRecord = (DEBUG_MEM_ALLOC_REC*)el; - if (el == PVR_PROC_SEQ_START_TOKEN) - { -#if !defined(DEBUG_LINUX_XML_PROC_FILES) - /* NOTE: If you update this code, please also update the XML varient below - * too! */ - - seq_printf(sfile, "%-60s: %d bytes\n", - "Current Water Mark of bytes allocated via kmalloc", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Highest Water Mark of bytes allocated via kmalloc", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Current Water Mark of bytes allocated via vmalloc", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Highest Water Mark of bytes allocated via vmalloc", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Current Water Mark of bytes allocated via alloc_pages", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Highest Water Mark of bytes allocated via alloc_pages", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Current Water Mark of bytes allocated via ioremap", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Highest Water Mark of bytes allocated via ioremap", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Current Water Mark of bytes reserved for \"IO\" memory areas", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Highest Water Mark of bytes allocated for \"IO\" memory areas", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Current Water Mark of bytes allocated via kmem_cache_alloc", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Highest Water Mark of bytes allocated via kmem_cache_alloc", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]); -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - seq_printf(sfile, "%-60s: %d bytes\n", - "Current Water Mark of bytes mapped via vmap", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMAP]); - seq_printf(sfile, "%-60s: %d bytes\n", - "Highest Water Mark of bytes mapped via vmap", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMAP]); -#endif -#if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) - seq_printf(sfile, "%-60s: %d pages\n", - "Number of pages in page pool", - atomic_read(&g_sPagePoolEntryCount)); -#endif - seq_printf( sfile, "\n"); - seq_printf(sfile, "%-60s: %d bytes\n", - "The Current Water Mark for memory allocated from system RAM", - SysRAMTrueWaterMark()); - seq_printf(sfile, "%-60s: %d bytes\n", - "The Highest Water Mark for memory allocated from system RAM", - g_SysRAMHighWaterMark); - seq_printf(sfile, "%-60s: %d bytes\n", - "The Current Water Mark for memory allocated from IO memory", - g_IOMemWaterMark); - seq_printf(sfile, "%-60s: %d bytes\n", - "The Highest Water Mark for memory allocated from IO memory", - g_IOMemHighWaterMark); - - seq_printf( sfile, "\n"); - - seq_printf(sfile, "Details for all known allocations:\n" - "%-16s %-8s %-8s %-10s %-5s %-10s %s\n", - "Type", - "CpuVAddr", - "CpuPAddr", - "Bytes", - "PID", - "PrivateData", - "Filename:Line"); - -#else /* DEBUG_LINUX_XML_PROC_FILES */ - - /* Note: If you want to update the description property of a watermark - * ensure that the key property remains unchanged so that watermark data - * logged over time from different driver revisions may remain comparable - */ - seq_printf(sfile, "<meminfo>\n<meminfo_header>\n"); - seq_printf(sfile, - "<watermark key=\"mr0\" description=\"kmalloc_current\" bytes=\"%d\"/>\n", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]); - seq_printf(sfile, - "<watermark key=\"mr1\" description=\"kmalloc_high\" bytes=\"%d\"/>\n", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]); - seq_printf(sfile, - "<watermark key=\"mr2\" description=\"vmalloc_current\" bytes=\"%d\"/>\n", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]); - seq_printf(sfile, - "<watermark key=\"mr3\" description=\"vmalloc_high\" bytes=\"%d\"/>\n", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]); - seq_printf(sfile, - "<watermark key=\"mr4\" description=\"alloc_pages_current\" bytes=\"%d\"/>\n", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]); - seq_printf(sfile, - "<watermark key=\"mr5\" description=\"alloc_pages_high\" bytes=\"%d\"/>\n", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]); - seq_printf(sfile, - "<watermark key=\"mr6\" description=\"ioremap_current\" bytes=\"%d\"/>\n", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]); - seq_printf(sfile, - "<watermark key=\"mr7\" description=\"ioremap_high\" bytes=\"%d\"/>\n", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]); - seq_printf(sfile, - "<watermark key=\"mr8\" description=\"io_current\" bytes=\"%d\"/>\n", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]); - seq_printf(sfile, - "<watermark key=\"mr9\" description=\"io_high\" bytes=\"%d\"/>\n", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]); - seq_printf(sfile, - "<watermark key=\"mr10\" description=\"kmem_cache_current\" bytes=\"%d\"/>\n", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]); - seq_printf(sfile, - "<watermark key=\"mr11\" description=\"kmem_cache_high\" bytes=\"%d\"/>\n", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]); -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - seq_printf(sfile, - "<watermark key=\"mr12\" description=\"vmap_current\" bytes=\"%d\"/>\n", - g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMAP]); - seq_printf(sfile, - "<watermark key=\"mr13\" description=\"vmap_high\" bytes=\"%d\"/>\n", - g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMAP]); -#endif - seq_printf(sfile, - "<watermark key=\"mr14\" description=\"system_ram_current\" bytes=\"%d\"/>\n", - SysRAMTrueWaterMark()); - seq_printf(sfile, - "<watermark key=\"mr15\" description=\"system_ram_high\" bytes=\"%d\"/>\n", - g_SysRAMHighWaterMark); - seq_printf(sfile, - "<watermark key=\"mr16\" description=\"system_io_current\" bytes=\"%d\"/>\n", - g_IOMemWaterMark); - seq_printf(sfile, - "<watermark key=\"mr17\" description=\"system_io_high\" bytes=\"%d\"/>\n", - g_IOMemHighWaterMark); - -#if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) - seq_printf(sfile, - "<watermark key=\"mr18\" description=\"page_pool_current\" bytes=\"%d\"/>\n", - PAGES_TO_BYTES(atomic_read(&g_sPagePoolEntryCount))); -#endif - seq_printf(sfile, "</meminfo_header>\n"); - -#endif /* DEBUG_LINUX_XML_PROC_FILES */ - return; - } - - if (psRecord->eAllocType != DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE) - { - seq_printf(sfile, -#if !defined(DEBUG_LINUX_XML_PROC_FILES) - "%-16s %-8p %08x %-10d %-5d %-10s %s:%d\n", -#else - "<allocation>\n" - "\t<type>%s</type>\n" - "\t<cpu_virtual>%-8p</cpu_virtual>\n" - "\t<cpu_physical>%08x</cpu_physical>\n" - "\t<bytes>%d</bytes>\n" - "\t<pid>%d</pid>\n" - "\t<private>%s</private>\n" - "\t<filename>%s</filename>\n" - "\t<line>%d</line>\n" - "</allocation>\n", -#endif - DebugMemAllocRecordTypeToString(psRecord->eAllocType), - psRecord->pvCpuVAddr, - psRecord->ulCpuPAddr, - psRecord->ui32Bytes, - psRecord->pid, - "NULL", - psRecord->pszFileName, - psRecord->ui32Line); - } - else - { - seq_printf(sfile, -#if !defined(DEBUG_LINUX_XML_PROC_FILES) - "%-16s %-8p %08x %-10d %-5d %-10s %s:%d\n", -#else - "<allocation>\n" - "\t<type>%s</type>\n" - "\t<cpu_virtual>%-8p</cpu_virtual>\n" - "\t<cpu_physical>%08x</cpu_physical>\n" - "\t<bytes>%d</bytes>\n" - "\t<pid>%d</pid>\n" - "\t<private>%s</private>\n" - "\t<filename>%s</filename>\n" - "\t<line>%d</line>\n" - "</allocation>\n", -#endif - DebugMemAllocRecordTypeToString(psRecord->eAllocType), - psRecord->pvCpuVAddr, - psRecord->ulCpuPAddr, - psRecord->ui32Bytes, - psRecord->pid, - KMemCacheNameWrapper(psRecord->pvPrivateData), - psRecord->pszFileName, - psRecord->ui32Line); - } -} - -#endif /* defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) */ - - -#if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MMAP_AREAS) -/* This could be moved somewhere more general */ -const IMG_CHAR * -HAPFlagsToString(IMG_UINT32 ui32Flags) -{ - static IMG_CHAR szFlags[50]; - IMG_INT32 i32Pos = 0; - IMG_UINT32 ui32CacheTypeIndex, ui32MapTypeIndex; - IMG_CHAR *apszCacheTypes[] = { - "UNCACHED", - "CACHED", - "WRITECOMBINE", - "UNKNOWN" - }; - IMG_CHAR *apszMapType[] = { - "KERNEL_ONLY", - "SINGLE_PROCESS", - "MULTI_PROCESS", - "FROM_EXISTING_PROCESS", - "NO_CPU_VIRTUAL", - "UNKNOWN" - }; - - /* FIXME create an enum for the cache type that we can - * cast and select so we get compiler warnings when - * when this code isn't complete due to new flags */ - if (ui32Flags & PVRSRV_HAP_UNCACHED) { - ui32CacheTypeIndex = 0; - } else if (ui32Flags & PVRSRV_HAP_CACHED) { - ui32CacheTypeIndex = 1; - } else if (ui32Flags & PVRSRV_HAP_WRITECOMBINE) { - ui32CacheTypeIndex = 2; - } else { - ui32CacheTypeIndex = 3; - PVR_DPF((PVR_DBG_ERROR, "%s: unknown cache type (%u)", - __FUNCTION__, (ui32Flags & PVRSRV_HAP_CACHETYPE_MASK))); - } - - /* FIXME create an enum for the map type that we can - * cast and select so we get compiler warnings when - * when this code isn't complete due to new flags */ - if (ui32Flags & PVRSRV_HAP_KERNEL_ONLY) { - ui32MapTypeIndex = 0; - } else if (ui32Flags & PVRSRV_HAP_SINGLE_PROCESS) { - ui32MapTypeIndex = 1; - } else if (ui32Flags & PVRSRV_HAP_MULTI_PROCESS) { - ui32MapTypeIndex = 2; - } else if (ui32Flags & PVRSRV_HAP_FROM_EXISTING_PROCESS) { - ui32MapTypeIndex = 3; - } else if (ui32Flags & PVRSRV_HAP_NO_CPU_VIRTUAL) { - ui32MapTypeIndex = 4; - } else { - ui32MapTypeIndex = 5; - PVR_DPF((PVR_DBG_ERROR, "%s: unknown map type (%u)", - __FUNCTION__, (ui32Flags & PVRSRV_HAP_MAPTYPE_MASK))); - } - - i32Pos = sprintf(szFlags, "%s|", apszCacheTypes[ui32CacheTypeIndex]); - if (i32Pos <= 0) - { - PVR_DPF((PVR_DBG_ERROR, "%s: sprintf for cache type %u failed (%d)", - __FUNCTION__, ui32CacheTypeIndex, i32Pos)); - szFlags[0] = 0; - } - else - { - sprintf(szFlags + i32Pos, "%s", apszMapType[ui32MapTypeIndex]); - } - - return szFlags; -} -#endif - -#if defined(DEBUG_LINUX_MEM_AREAS) -static IMG_VOID LinuxMMCleanup_MemAreas_ForEachCb(DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord) -{ - LinuxMemArea *psLinuxMemArea; - - psLinuxMemArea = psCurrentRecord->psLinuxMemArea; - PVR_DPF((PVR_DBG_ERROR, "%s: BUG!: Cleaning up Linux memory area (%p), type=%s, size=%d bytes", - __FUNCTION__, - psCurrentRecord->psLinuxMemArea, - LinuxMemAreaTypeToString(psCurrentRecord->psLinuxMemArea->eAreaType), - psCurrentRecord->psLinuxMemArea->ui32ByteSize)); - /* Note this will also remove psCurrentRecord from g_LinuxMemAreaRecords - * but that's ok since we have already got a pointer to the next area. */ - LinuxMemAreaDeepFree(psLinuxMemArea); -} -#endif - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) -static IMG_VOID LinuxMMCleanup_MemRecords_ForEachVa(DEBUG_MEM_ALLOC_REC *psCurrentRecord) - -{ - -/* It's a bug if anything remains allocated at this point. We - * report an error, and simply brute force free anything we find. */ - PVR_DPF((PVR_DBG_ERROR, "%s: BUG!: Cleaning up memory: " - "type=%s " - "CpuVAddr=%p " - "CpuPAddr=0x%08x, " - "allocated @ file=%s,line=%d", - __FUNCTION__, - DebugMemAllocRecordTypeToString(psCurrentRecord->eAllocType), - psCurrentRecord->pvCpuVAddr, - psCurrentRecord->ulCpuPAddr, - psCurrentRecord->pszFileName, - psCurrentRecord->ui32Line)); - switch (psCurrentRecord->eAllocType) - { - case DEBUG_MEM_ALLOC_TYPE_KMALLOC: - KFreeWrapper(psCurrentRecord->pvCpuVAddr); - break; - case DEBUG_MEM_ALLOC_TYPE_IOREMAP: - IOUnmapWrapper(psCurrentRecord->pvCpuVAddr); - break; - case DEBUG_MEM_ALLOC_TYPE_IO: - /* Nothing needed except to free the record */ - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_IO, psCurrentRecord->pvKey, __FILE__, __LINE__); - break; - case DEBUG_MEM_ALLOC_TYPE_VMALLOC: - VFreeWrapper(psCurrentRecord->pvCpuVAddr); - break; - case DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES: - DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, psCurrentRecord->pvKey, __FILE__, __LINE__); - break; - case DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE: - KMemCacheFreeWrapper(psCurrentRecord->pvPrivateData, psCurrentRecord->pvCpuVAddr); - break; -#if defined(PVR_LINUX_MEM_AREA_USE_VMAP) - case DEBUG_MEM_ALLOC_TYPE_VMAP: - VUnmapWrapper(psCurrentRecord->pvCpuVAddr); - break; -#endif - default: - PVR_ASSERT(0); - } -} -#endif - - -#if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) -static struct shrinker g_sShrinker = -{ - .shrink = ShrinkPagePool, - .seeks = DEFAULT_SEEKS -}; - -static IMG_BOOL g_bShrinkerRegistered; -#endif - -IMG_VOID -LinuxMMCleanup(IMG_VOID) -{ -#if defined(DEBUG_LINUX_MEM_AREAS) - { - if (g_LinuxMemAreaCount) - { - PVR_DPF((PVR_DBG_ERROR, "%s: BUG!: There are %d LinuxMemArea allocation unfreed (%d bytes)", - __FUNCTION__, g_LinuxMemAreaCount, g_LinuxMemAreaWaterMark)); - } - - List_DEBUG_LINUX_MEM_AREA_REC_ForEach(g_LinuxMemAreaRecords, LinuxMMCleanup_MemAreas_ForEachCb); - - if (g_SeqFileMemArea) - { - RemoveProcEntrySeq(g_SeqFileMemArea); - } - } -#endif - -#if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) - if (g_bShrinkerRegistered) - { - unregister_shrinker(&g_sShrinker); - } -#endif - - /* - * The page pool must be freed after any remaining mem areas, but before - * the remaining memory resources. - */ - FreePagePool(); - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - { - - /* - * It's a bug if anything remains allocated at this point. We - * report an error, and simply brute force free anything we find. - */ - List_DEBUG_MEM_ALLOC_REC_ForEach(g_MemoryRecords, LinuxMMCleanup_MemRecords_ForEachVa); - - if (g_SeqFileMemoryRecords) - { - RemoveProcEntrySeq(g_SeqFileMemoryRecords); - } - } -#endif - - if (g_PsLinuxMemAreaCache) - { - KMemCacheDestroyWrapper(g_PsLinuxMemAreaCache); - } - - if (g_PsLinuxPagePoolCache) - { - KMemCacheDestroyWrapper(g_PsLinuxPagePoolCache); - } -} - -PVRSRV_ERROR -LinuxMMInit(IMG_VOID) -{ -#if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - LinuxInitMutex(&g_sDebugMutex); -#endif - -#if defined(DEBUG_LINUX_MEM_AREAS) - { - g_SeqFileMemArea = CreateProcReadEntrySeq( - "mem_areas", - NULL, - ProcSeqNextMemArea, - ProcSeqShowMemArea, - ProcSeqOff2ElementMemArea, - ProcSeqStartstopDebugMutex - ); - if (!g_SeqFileMemArea) - { - goto failed; - } - } -#endif - - -#if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) - { - g_SeqFileMemoryRecords = CreateProcReadEntrySeq( - "meminfo", - NULL, - ProcSeqNextMemoryRecords, - ProcSeqShowMemoryRecords, - ProcSeqOff2ElementMemoryRecords, - ProcSeqStartstopDebugMutex - ); - if (!g_SeqFileMemoryRecords) - { - goto failed; - } - } -#endif - - g_PsLinuxMemAreaCache = KMemCacheCreateWrapper("img-mm", sizeof(LinuxMemArea), 0, 0); - if (!g_PsLinuxMemAreaCache) - { - PVR_DPF((PVR_DBG_ERROR,"%s: failed to allocate mem area kmem_cache", __FUNCTION__)); - goto failed; - } - -#if (PVR_LINUX_MEM_AREA_POOL_MAX_PAGES != 0) - g_iPagePoolMaxEntries = PVR_LINUX_MEM_AREA_POOL_MAX_PAGES; - if (g_iPagePoolMaxEntries <= 0 || g_iPagePoolMaxEntries > INT_MAX/2) - { - g_iPagePoolMaxEntries = INT_MAX/2; - PVR_TRACE(("%s: No limit set for page pool size", __FUNCTION__)); - } - else - { - PVR_TRACE(("%s: Maximum page pool size: %d", __FUNCTION__, g_iPagePoolMaxEntries)); - } - - g_PsLinuxPagePoolCache = KMemCacheCreateWrapper("img-mm-pool", sizeof(LinuxPagePoolEntry), 0, 0); - if (!g_PsLinuxPagePoolCache) - { - PVR_DPF((PVR_DBG_ERROR,"%s: failed to allocate page pool kmem_cache", __FUNCTION__)); - goto failed; - } -#endif - -#if defined(PVR_LINUX_MEM_AREA_POOL_ALLOW_SHRINK) - register_shrinker(&g_sShrinker); - g_bShrinkerRegistered = IMG_TRUE; -#endif - - return PVRSRV_OK; - -failed: - LinuxMMCleanup(); - return PVRSRV_ERROR_OUT_OF_MEMORY; -} - |