From 1403b1a38e8b19a4cc17e2c158e278628943a436 Mon Sep 17 00:00:00 2001 From: Pauli Nieminen Date: Thu, 1 Apr 2010 12:44:57 +0000 Subject: drm/ttm: add pool wc/uc page allocator V3 On AGP system we might allocate/free routinely uncached or wc memory, changing page from cached (wb) to uc or wc is very expensive and involves a lot of flushing. To improve performance this allocator use a pool of uc,wc pages. Pools are protected with spinlocks to allow multiple threads to allocate pages simultanously. Expensive operations are done outside of spinlock to maximize concurrency. Pools are linked lists of pages that were recently freed. mm shrink callback allows kernel to claim back pages when they are required for something else. Fixes: * set_pages_array_wb handles highmem pages so we don't have to remove them from pool. * Add count parameter to ttm_put_pages to avoid looping in free code. * Change looping from _safe to normal in pool fill error path. * Initialize sum variable and make the loop prettier in get_num_unused_pages. * Moved pages_freed reseting inside the loop in ttm_page_pool_free. * Add warning comment about spinlock context in ttm_page_pool_free. Based on Jerome Glisse's and Dave Airlie's pool allocator. Signed-off-by: Jerome Glisse Signed-off-by: Dave Airlie Signed-off-by: Pauli Nieminen Reviewed-by: Jerome Glisse Signed-off-by: Dave Airlie --- include/drm/ttm/ttm_page_alloc.h | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 include/drm/ttm/ttm_page_alloc.h (limited to 'include/drm/ttm') diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h new file mode 100644 index 0000000..043d817 --- /dev/null +++ b/include/drm/ttm/ttm_page_alloc.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) Red Hat Inc. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Jerome Glisse + */ +#ifndef TTM_PAGE_ALLOC +#define TTM_PAGE_ALLOC + +#include "ttm_bo_driver.h" +#include "ttm_memory.h" + +/** + * Get count number of pages from pool to pages list. + * + * @pages: heado of empty linked list where pages are filled. + * @flags: ttm flags for page allocation. + * @cstate: ttm caching state for the page. + * @count: number of pages to allocate. + */ +int ttm_get_pages(struct list_head *pages, + int flags, + enum ttm_caching_state cstate, + unsigned count); +/** + * Put linked list of pages to pool. + * + * @pages: list of pages to free. + * @page_count: number of pages in the list. Zero can be passed for unknown + * count. + * @flags: ttm flags for page allocation. + * @cstate: ttm caching state. + */ +void ttm_put_pages(struct list_head *pages, + unsigned page_count, + int flags, + enum ttm_caching_state cstate); +/** + * Initialize pool allocator. + * + * Pool allocator is internaly reference counted so it can be initialized + * multiple times but ttm_page_alloc_fini has to be called same number of + * times. + */ +int ttm_page_alloc_init(unsigned max_pages); +/** + * Free pool allocator. + */ +void ttm_page_alloc_fini(void); + +#endif -- cgit v1.1 From 0745866165598b067442c472911280527b08be3e Mon Sep 17 00:00:00 2001 From: Pauli Nieminen Date: Thu, 1 Apr 2010 12:44:58 +0000 Subject: drm/ttm: Add debugfs output entry to pool allocator. ttm_page_alloc_debugfs can be registered to output the state of pools. Debugfs file will output number of pages freed from the pool, number of pages in pool now and the lowes number of pages in pool since previous shrink. Signed-off-by: Pauli Nieminen Signed-off-by: Dave Airlie --- include/drm/ttm/ttm_page_alloc.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/drm/ttm') diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h index 043d817..8b091c3 100644 --- a/include/drm/ttm/ttm_page_alloc.h +++ b/include/drm/ttm/ttm_page_alloc.h @@ -67,4 +67,8 @@ int ttm_page_alloc_init(unsigned max_pages); */ void ttm_page_alloc_fini(void); +/** + * Output the state of pools to debugfs file + */ +extern int ttm_page_alloc_debugfs(struct seq_file *m, void *data); #endif -- cgit v1.1 From c96af79e3463d5d3f865625baa8bb8aa4c0944a0 Mon Sep 17 00:00:00 2001 From: Pauli Nieminen Date: Thu, 1 Apr 2010 12:45:03 +0000 Subject: drm/ttm: Add sysfs interface to control pool allocator. Sysfs interface allows user to configure pool allocator functionality and change limits for the size of pool. Signed-off-by: Pauli Nieminen Signed-off-by: Dave Airlie --- include/drm/ttm/ttm_page_alloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/drm/ttm') diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h index 8b091c3..8bb4de5 100644 --- a/include/drm/ttm/ttm_page_alloc.h +++ b/include/drm/ttm/ttm_page_alloc.h @@ -61,7 +61,7 @@ void ttm_put_pages(struct list_head *pages, * multiple times but ttm_page_alloc_fini has to be called same number of * times. */ -int ttm_page_alloc_init(unsigned max_pages); +int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages); /** * Free pool allocator. */ -- cgit v1.1 From 9d87fa2138d06ff400551800d67d522625033e35 Mon Sep 17 00:00:00 2001 From: Jerome Glisse Date: Wed, 7 Apr 2010 10:21:19 +0000 Subject: drm/ttm: split no_wait argument in 2 GPU or reserve wait There is case where we want to be able to wait only for the GPU while not waiting for other buffer to be unreserved. This patch split the no_wait argument all the way down in the whole ttm path so that upper level can decide on what to wait on or not. [airlied: squashed these 4 for bisectability reasons.] drm/radeon/kms: update to TTM no_wait splitted argument drm/nouveau: update to TTM no_wait splitted argument drm/vmwgfx: update to TTM no_wait splitted argument [vmwgfx patch: Reviewed-by: Thomas Hellstrom ] Signed-off-by: Jerome Glisse Acked-by: Thomas Hellstrom Signed-off-by: Dave Airlie --- include/drm/ttm/ttm_bo_api.h | 6 ++++-- include/drm/ttm/ttm_bo_driver.h | 29 ++++++++++++++++++----------- 2 files changed, 22 insertions(+), 13 deletions(-) (limited to 'include/drm/ttm') diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 81eb9f4..8c8005e 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -313,7 +313,8 @@ extern int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, * @bo: The buffer object. * @placement: Proposed placement for the buffer object. * @interruptible: Sleep interruptible if sleeping. - * @no_wait: Return immediately if the buffer is busy. + * @no_wait_reserve: Return immediately if other buffers are busy. + * @no_wait_gpu: Return immediately if the GPU is busy. * * Changes placement and caching policy of the buffer object * according proposed placement. @@ -325,7 +326,8 @@ extern int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, */ extern int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, - bool interruptible, bool no_wait); + bool interruptible, bool no_wait_reserve, + bool no_wait_gpu); /** * ttm_bo_unref diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index e929c27..69f70e4 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -311,7 +311,8 @@ struct ttm_bo_driver { */ int (*move) (struct ttm_buffer_object *bo, bool evict, bool interruptible, - bool no_wait, struct ttm_mem_reg *new_mem); + bool no_wait_reserve, bool no_wait_gpu, + struct ttm_mem_reg *new_mem); /** * struct ttm_bo_driver_member verify_access @@ -633,7 +634,8 @@ extern bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, * @proposed_placement: Proposed new placement for the buffer object. * @mem: A struct ttm_mem_reg. * @interruptible: Sleep interruptible when sliping. - * @no_wait: Don't sleep waiting for space to become available. + * @no_wait_reserve: Return immediately if other buffers are busy. + * @no_wait_gpu: Return immediately if the GPU is busy. * * Allocate memory space for the buffer object pointed to by @bo, using * the placement flags in @mem, potentially evicting other idle buffer objects. @@ -647,7 +649,8 @@ extern bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, extern int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, - bool interruptible, bool no_wait); + bool interruptible, + bool no_wait_reserve, bool no_wait_gpu); /** * ttm_bo_wait_for_cpu * @@ -826,7 +829,8 @@ extern void ttm_bo_unblock_reservation(struct ttm_buffer_object *bo); * * @bo: A pointer to a struct ttm_buffer_object. * @evict: 1: This is an eviction. Don't try to pipeline. - * @no_wait: Never sleep, but rather return with -EBUSY. + * @no_wait_reserve: Return immediately if other buffers are busy. + * @no_wait_gpu: Return immediately if the GPU is busy. * @new_mem: struct ttm_mem_reg indicating where to move. * * Optimized move function for a buffer object with both old and @@ -840,15 +844,16 @@ extern void ttm_bo_unblock_reservation(struct ttm_buffer_object *bo); */ extern int ttm_bo_move_ttm(struct ttm_buffer_object *bo, - bool evict, bool no_wait, - struct ttm_mem_reg *new_mem); + bool evict, bool no_wait_reserve, + bool no_wait_gpu, struct ttm_mem_reg *new_mem); /** * ttm_bo_move_memcpy * * @bo: A pointer to a struct ttm_buffer_object. * @evict: 1: This is an eviction. Don't try to pipeline. - * @no_wait: Never sleep, but rather return with -EBUSY. + * @no_wait_reserve: Return immediately if other buffers are busy. + * @no_wait_gpu: Return immediately if the GPU is busy. * @new_mem: struct ttm_mem_reg indicating where to move. * * Fallback move function for a mappable buffer object in mappable memory. @@ -862,8 +867,8 @@ extern int ttm_bo_move_ttm(struct ttm_buffer_object *bo, */ extern int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool evict, - bool no_wait, struct ttm_mem_reg *new_mem); + bool evict, bool no_wait_reserve, + bool no_wait_gpu, struct ttm_mem_reg *new_mem); /** * ttm_bo_free_old_node @@ -882,7 +887,8 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); * @sync_obj_arg: An argument to pass to the sync object idle / wait * functions. * @evict: This is an evict move. Don't return until the buffer is idle. - * @no_wait: Never sleep, but rather return with -EBUSY. + * @no_wait_reserve: Return immediately if other buffers are busy. + * @no_wait_gpu: Return immediately if the GPU is busy. * @new_mem: struct ttm_mem_reg indicating where to move. * * Accelerated move function to be called when an accelerated move @@ -896,7 +902,8 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, void *sync_obj, void *sync_obj_arg, - bool evict, bool no_wait, + bool evict, bool no_wait_reserve, + bool no_wait_gpu, struct ttm_mem_reg *new_mem); /** * ttm_io_prot -- cgit v1.1 From 82c5da6bf8b55a931b042fb531083863d26c8020 Mon Sep 17 00:00:00 2001 From: Jerome Glisse Date: Fri, 9 Apr 2010 14:39:23 +0200 Subject: drm/ttm: ttm_fault callback to allow driver to handle bo placement V6 On fault the driver is given the opportunity to perform any operation it sees fit in order to place the buffer into a CPU visible area of memory. This patch doesn't break TTM users, nouveau, vmwgfx and radeon should keep working properly. Future patch will take advantage of this infrastructure and remove the old path from TTM once driver are converted. V2 return VM_FAULT_NOPAGE if callback return -EBUSY or -ERESTARTSYS V3 balance io_mem_reserve and io_mem_free call, fault_reserve_notify is responsible to perform any necessary task for mapping to succeed V4 minor cleanup, atomic_t -> bool as member is protected by reserve mecanism from concurent access V5 the callback is now responsible for iomapping the bo and providing a virtual address this simplify TTM and will allow to get rid of TTM_MEMTYPE_FLAG_NEEDS_IOREMAP V6 use the bus addr data to decide to ioremap or this isn't needed but we don't necesarily need to ioremap in the callback but still allow driver to use static mapping Signed-off-by: Jerome Glisse Reviewed-by: Thomas Hellstrom Signed-off-by: Dave Airlie --- include/drm/ttm/ttm_bo_api.h | 23 +++++++++++++++++++++++ include/drm/ttm/ttm_bo_driver.h | 16 +++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) (limited to 'include/drm/ttm') diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 8c8005e..3e273e0 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -66,6 +66,26 @@ struct ttm_placement { const uint32_t *busy_placement; }; +/** + * struct ttm_bus_placement + * + * @addr: mapped virtual address + * @base: bus base address + * @is_iomem: is this io memory ? + * @size: size in byte + * @offset: offset from the base address + * + * Structure indicating the bus placement of an object. + */ +struct ttm_bus_placement { + void *addr; + unsigned long base; + unsigned long size; + unsigned long offset; + bool is_iomem; + bool io_reserved; +}; + /** * struct ttm_mem_reg @@ -75,6 +95,7 @@ struct ttm_placement { * @num_pages: Actual size of memory region in pages. * @page_alignment: Page alignment. * @placement: Placement flags. + * @bus: Placement on io bus accessible to the CPU * * Structure indicating the placement and space resources used by a * buffer object. @@ -87,6 +108,7 @@ struct ttm_mem_reg { uint32_t page_alignment; uint32_t mem_type; uint32_t placement; + struct ttm_bus_placement bus; }; /** @@ -274,6 +296,7 @@ struct ttm_bo_kmap_obj { ttm_bo_map_kmap = 3, ttm_bo_map_premapped = 4 | TTM_BO_MAP_IOMEM_MASK, } bo_kmap_type; + struct ttm_buffer_object *bo; }; /** diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 69f70e4..da39865 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -352,12 +352,21 @@ struct ttm_bo_driver { struct ttm_mem_reg *new_mem); /* notify the driver we are taking a fault on this BO * and have reserved it */ - void (*fault_reserve_notify)(struct ttm_buffer_object *bo); + int (*fault_reserve_notify)(struct ttm_buffer_object *bo); /** * notify the driver that we're about to swap out this bo */ void (*swap_notify) (struct ttm_buffer_object *bo); + + /** + * Driver callback on when mapping io memory (for bo_move_memcpy + * for instance). TTM will take care to call io_mem_free whenever + * the mapping is not use anymore. io_mem_reserve & io_mem_free + * are balanced. + */ + int (*io_mem_reserve)(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem); + void (*io_mem_free)(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem); }; /** @@ -685,6 +694,11 @@ extern int ttm_bo_pci_offset(struct ttm_bo_device *bdev, unsigned long *bus_offset, unsigned long *bus_size); +extern int ttm_mem_io_reserve(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem); +extern void ttm_mem_io_free(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem); + extern void ttm_bo_global_release(struct ttm_global_reference *ref); extern int ttm_bo_global_init(struct ttm_global_reference *ref); -- cgit v1.1 From 0c321c79627189204d7d0bf65ab19f5ac419abed Mon Sep 17 00:00:00 2001 From: Jerome Glisse Date: Wed, 7 Apr 2010 10:21:27 +0000 Subject: drm/ttm: remove io_ field from TTM V6 All TTM driver have been converted to new io_mem_reserve/free interface which allow driver to choose and return proper io base, offset to core TTM for ioremapping if necessary. This patch remove what is now deadcode. V2 adapt to match with change in first patch of the patchset V3 update after io_mem_reserve/io_mem_free callback balancing V4 adjust to minor cleanup V5 remove the needs ioremap flag V6 keep the ioremapping facility in TTM [airlied- squashed driver removals in here also] Signed-off-by: Jerome Glisse Reviewed-by: Thomas Hellstrom Signed-off-by: Dave Airlie --- include/drm/ttm/ttm_bo_driver.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include/drm/ttm') diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index da39865..7720b17 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -176,8 +176,6 @@ struct ttm_tt { #define TTM_MEMTYPE_FLAG_FIXED (1 << 0) /* Fixed (on-card) PCI memory */ #define TTM_MEMTYPE_FLAG_MAPPABLE (1 << 1) /* Memory mappable */ -#define TTM_MEMTYPE_FLAG_NEEDS_IOREMAP (1 << 2) /* Fixed memory needs ioremap - before kernel access. */ #define TTM_MEMTYPE_FLAG_CMA (1 << 3) /* Can't map aperture */ /** @@ -189,13 +187,6 @@ struct ttm_tt { * managed by this memory type. * @gpu_offset: If used, the GPU offset of the first managed page of * fixed memory or the first managed location in an aperture. - * @io_offset: The io_offset of the first managed page of IO memory or - * the first managed location in an aperture. For TTM_MEMTYPE_FLAG_CMA - * memory, this should be set to NULL. - * @io_size: The size of a managed IO region (fixed memory or aperture). - * @io_addr: Virtual kernel address if the io region is pre-mapped. For - * TTM_MEMTYPE_FLAG_NEEDS_IOREMAP there is no pre-mapped io map and - * @io_addr should be set to NULL. * @size: Size of the managed region. * @available_caching: A mask of available caching types, TTM_PL_FLAG_XX, * as defined in ttm_placement_common.h @@ -221,9 +212,6 @@ struct ttm_mem_type_manager { bool use_type; uint32_t flags; unsigned long gpu_offset; - unsigned long io_offset; - unsigned long io_size; - void *io_addr; uint64_t size; uint32_t available_caching; uint32_t default_caching; -- cgit v1.1 From 7c5ee5366f79f53de2a11e73953daee6d58df124 Mon Sep 17 00:00:00 2001 From: Matthew Garrett Date: Mon, 26 Apr 2010 16:00:09 -0400 Subject: ttm: Provide an API for starting and stopping the delayed workqueue We want to be able to prevent the delayed workqueue from changing state while we're reclocking, so add an API to block and unblock it. Signed-off-by: Matthew Garrett Signed-off-by: Dave Airlie --- include/drm/ttm/ttm_bo_api.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/drm/ttm') diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 3e273e0..267a86c 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -362,6 +362,23 @@ extern int ttm_bo_validate(struct ttm_buffer_object *bo, extern void ttm_bo_unref(struct ttm_buffer_object **bo); /** + * ttm_bo_lock_delayed_workqueue + * + * Prevent the delayed workqueue from running. + * Returns + * True if the workqueue was queued at the time + */ +extern int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev); + +/** + * ttm_bo_unlock_delayed_workqueue + * + * Allows the delayed workqueue to run. + */ +extern void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, + int resched); + +/** * ttm_bo_synccpu_write_grab * * @bo: The buffer object: -- cgit v1.1