Implement a policy for selecting memory types.

6 files changed, 205 insertions, 56 deletions

diff --git a/linux-core/drmP.h b/linux-core/drmP.h
index dd07a603..5834c9dc 100644
--- a/linux-core/drmP.h
+++ b/linux-core/drmP.h
@@ -673,12 +673,17 @@ typedef struct drm_mem_type_manager {
  */
 
 typedef struct drm_bo_driver{
+        const uint32_t *mem_type_prio;
+	const uint32_t *mem_busy_prio;
+        uint32_t num_mem_type_prio;
+	uint32_t num_mem_busy_prio;
 	drm_ttm_backend_t *(*create_ttm_backend_entry) 
 		(struct drm_device *dev);
 	int (*fence_type)(uint32_t flags, uint32_t *class, uint32_t *type);
 	int (*invalidate_caches)(struct drm_device *dev, uint32_t flags);
 	int (*init_mem_type)(struct drm_device *dev, uint32_t type,
 			     drm_mem_type_manager_t *man);
+        uint32_t (*evict_flags) (struct drm_device *dev, uint32_t type);
 } drm_bo_driver_t;
 
 
@@ -800,19 +805,9 @@ typedef struct drm_fence_manager{
         atomic_t count;
 } drm_fence_manager_t;
 
-
-typedef struct drm_bo_mem_region {
-	drm_mm_node_t *node;
-	uint32_t memory_type;
-        drm_ttm_t *ttm;
-	unsigned long bus_offset;
-	unsigned long num_pages;
-	uint32_t vm_flags;
-} drm_bo_mem_region_t;
-	
-
 typedef struct drm_buffer_manager{
 	struct mutex init_mutex;
+	struct mutex evict_mutex;
 	int nice_mode;
 	int initialized;
         drm_file_t *last_to_validate;
@@ -1003,6 +998,16 @@ typedef struct drm_fence_object{
 	uint32_t submitted_flush;
 } drm_fence_object_t;
 
+typedef struct drm_bo_mem_reg {
+	drm_mm_node_t *mm_node;
+	unsigned long size;
+	unsigned long num_pages;
+	uint32_t page_alignment;
+	uint32_t mem_type;
+	uint32_t flags;
+	uint32_t mask;
+} drm_bo_mem_reg_t;
+	
 
 typedef struct drm_buffer_object{
 	drm_device_t *dev;
diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c
index 9a27a4b5..fa659d04 100644
--- a/linux-core/drm_bo.c
+++ b/linux-core/drm_bo.c
@@ -56,6 +56,8 @@ static void drm_bo_destroy_locked(drm_buffer_object_t *bo);
 static int drm_bo_setup_vm_locked(drm_buffer_object_t *bo);
 static void drm_bo_takedown_vm_locked(drm_buffer_object_t *bo);
 static void drm_bo_unmap_virtual(drm_buffer_object_t *bo);
+static int drm_bo_mem_space(drm_device_t *dev, drm_bo_mem_reg_t *mem,
+			    int no_wait);
 
 #define DRM_FLAG_MASKED(_old, _new, _mask) {\
 (_old) ^= (((_old) ^ (_new)) & (_mask)); \
@@ -497,6 +499,7 @@ static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type,
 	int ret = 0;
 	drm_device_t *dev = bo->dev;
 	drm_buffer_manager_t *bm = &dev->bm;
+	drm_bo_mem_reg_t evict_mem;
 
 	/*
 	 * Someone might have modified the buffer before we took the buffer mutex.
@@ -509,22 +512,39 @@ static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type,
 
 	ret = drm_bo_wait(bo, 0, 0, no_wait);
 
-	if (ret) {
-		if (ret != -EAGAIN)
-			DRM_ERROR("Failed to expire fence before "
-				  "buffer eviction.\n");
+	if (ret && ret != -EAGAIN) {
+		DRM_ERROR("Failed to expire fence before "
+			  "buffer eviction.\n");
 		goto out;
 	}
 
-	if (mem_type == DRM_BO_MEM_TT) {
-		ret = drm_move_tt_to_local(bo, 1, force_no_move);
-		if (ret)
-			goto out;
-		mutex_lock(&dev->struct_mutex);
-		list_del_init(&bo->lru);
-		drm_bo_add_to_lru(bo, bm);
-		mutex_unlock(&dev->struct_mutex);
+	evict_mem.num_pages = bo->num_pages;
+	evict_mem.page_alignment = bo->page_alignment;
+	evict_mem.size = evict_mem.num_pages << PAGE_SHIFT;
+	evict_mem.mask = dev->driver->bo_driver->evict_flags(dev, mem_type);
+	
+	ret = drm_bo_mem_space(dev, &evict_mem, no_wait);
+
+	if (ret && ret != -EAGAIN) {
+		DRM_ERROR("Failed to find memory space for "
+			  "buffer eviction.\n");
+		goto out;
+	}
+	
+	if ((mem_type != DRM_BO_MEM_TT) &&
+	    (evict_mem.mem_type != DRM_BO_MEM_LOCAL)) {
+		ret = -EINVAL;
+		DRM_ERROR("Unsupported memory types for eviction.\n");
+		goto out;
 	}
+      
+	ret = drm_move_tt_to_local(bo, 1, force_no_move);
+	if (ret)
+		goto out;
+	mutex_lock(&dev->struct_mutex);
+	list_del_init(&bo->lru);
+	drm_bo_add_to_lru(bo, bm);
+	mutex_unlock(&dev->struct_mutex);
 
 	if (ret)
 		goto out;
@@ -535,26 +555,25 @@ static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type,
 	return ret;
 }
 
-/*
- * bo->mutex locked.
- */
 
-int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type,
-		       int no_wait)
+
+static int drm_bo_mem_force_space(drm_device_t *dev,
+				  drm_bo_mem_reg_t *mem,
+				  uint32_t mem_type,
+				  int no_wait)
 {
-	drm_device_t *dev = bo->dev;
 	drm_mm_node_t *node;
 	drm_buffer_manager_t *bm = &dev->bm;
 	drm_buffer_object_t *entry;
 	drm_mem_type_manager_t *man = &bm->man[mem_type];
-	drm_mm_t *mm = &man->manager;
 	struct list_head *lru;
-	unsigned long size = bo->num_pages;
+	unsigned long num_pages = mem->num_pages;
 	int ret;
 
 	mutex_lock(&dev->struct_mutex);
 	do {
-		node = drm_mm_search_free(mm, size, bo->page_alignment, 1);
+		node = drm_mm_search_free(&man->manager, num_pages, 
+					  mem->page_alignment, 1);
 		if (node)
 			break;
 
@@ -563,11 +582,11 @@ int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type,
 			break;
 
 		entry = list_entry(lru->next, drm_buffer_object_t, lru);
-
 		atomic_inc(&entry->usage);
 		mutex_unlock(&dev->struct_mutex);
 		mutex_lock(&entry->mutex);
-		BUG_ON(bo->flags & DRM_BO_FLAG_NO_MOVE);
+		BUG_ON(entry->flags & (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT));
+
 		ret = drm_bo_evict(entry, mem_type, no_wait, 0);
 		mutex_unlock(&entry->mutex);
 		drm_bo_usage_deref_unlocked(entry);
@@ -577,34 +596,108 @@ int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type,
 	} while (1);
 
 	if (!node) {
-		DRM_ERROR("Out of videoram / aperture space\n");
 		mutex_unlock(&dev->struct_mutex);
 		return -ENOMEM;
 	}
 
-	node = drm_mm_get_block(node, size, bo->page_alignment);
+	node = drm_mm_get_block(node, num_pages, mem->page_alignment);
 	mutex_unlock(&dev->struct_mutex);
-	BUG_ON(!node);
-	node->private = (void *)bo;
-
-	bo->mm_node = node;
-	bo->offset = node->start * PAGE_SIZE;
+	mem->mm_node = node;
+	mem->mem_type = mem_type;
+	mem->flags = drm_bo_type_flags(mem_type);
 	return 0;
 }
 
-static int drm_move_local_to_tt(drm_buffer_object_t * bo, int no_wait)
+
+static int drm_bo_mem_space(drm_device_t *dev,
+	                    drm_bo_mem_reg_t *mem,
+			    int no_wait)
 {
-	drm_device_t *dev = bo->dev;
-	int ret = 0;
+	drm_buffer_manager_t *bm= &dev->bm;
+	drm_mem_type_manager_t *man; 
 
-	if (!(bo->mm_node && (bo->flags & DRM_BO_FLAG_NO_MOVE))) {
-		BUG_ON(bo->mm_node);
-		ret = drm_bo_alloc_space(bo, DRM_BO_MEM_TT, no_wait);
-		if (ret)
-			return ret;
+	uint32_t num_prios = dev->driver->bo_driver->num_mem_type_prio;
+	const uint32_t *prios = dev->driver->bo_driver->mem_type_prio;
+	uint32_t i;
+	uint32_t mem_type = DRM_BO_MEM_LOCAL;
+	int type_found = 0;
+	int type_ok = 0;
+	int has_eagain = 0;
+	drm_mm_node_t *node = NULL;
+	int ret;
+
+	for (i=0; i<num_prios; ++i) {
+		mem_type = prios[i];
+		type_ok = drm_bo_type_flags(mem_type) & mem->mask ;
+		if (!type_ok)
+			continue;
+
+		if (mem_type == DRM_BO_MEM_LOCAL)
+			break;
+
+		man = &bm->man[mem_type];
+		mutex_lock(&dev->struct_mutex);
+		if (man->has_type && man->use_type) {
+			type_found = 1;
+			node = drm_mm_search_free(&man->manager, mem->num_pages, 
+						  mem->page_alignment, 1);
+			if (node) 
+				node = drm_mm_get_block(node, mem->num_pages, 
+							mem->page_alignment);
+		}
+		mutex_unlock(&dev->struct_mutex);
+		if (node)
+			break;
 	}
+	
+	if ((type_ok && (mem_type == DRM_BO_MEM_LOCAL)) || node) {
+		mem->mm_node = node;
+		mem->mem_type = mem_type;
+		mem->flags = drm_bo_type_flags(mem_type);
+		return 0;
+	}
+
+	if (!type_found) {
+		DRM_ERROR("Requested memory types are not supported\n");
+		return -EINVAL;
+	}
+
+	num_prios = dev->driver->bo_driver->num_mem_busy_prio;
+	prios = dev->driver->bo_driver->mem_busy_prio;
+
+	for (i=0; i<num_prios; ++i) {
+		mem_type = prios[i];
+		if (!(drm_bo_type_flags(mem_type) & mem->mask))
+			continue;
+		
+		man = &bm->man[mem_type];
+		ret = drm_bo_mem_force_space(dev, mem, mem_type, no_wait);
+		
+		if (ret == 0) 
+			return 0;
+		
+		if (ret == -EAGAIN)
+			has_eagain = 1;
+	}
+
+	ret = (has_eagain) ? -EAGAIN : -ENOMEM;
+	return ret;
+}
+
 
-	DRM_DEBUG("Flipping in to AGP 0x%08lx\n", bo->mm_node->start);
+
+
+static int drm_move_local_to_tt(drm_buffer_object_t * bo, 
+				drm_bo_mem_reg_t * mem, 
+				int no_wait)
+{
+	drm_device_t *dev = bo->dev;
+	int ret = 0;
+	
+	bo->mm_node = mem->mm_node;
+
+	DRM_DEBUG("Flipping in to AGP 0x%08lx 0x%08lx\n", 
+		  bo->mm_node->start, bo->mm_node->size);
 	
 #ifdef DRM_ODD_MM_COMPAT
 	mutex_lock(&dev->struct_mutex);
@@ -631,6 +724,7 @@ static int drm_move_local_to_tt(drm_buffer_object_t * bo, int no_wait)
 	bo->flags &= ~DRM_BO_MASK_MEM;
 	bo->flags |= DRM_BO_FLAG_MEM_TT;
 	bo->mem_type = DRM_BO_MEM_TT;
+	bo->offset = bo->mm_node->start << PAGE_SHIFT;
 
 #ifdef DRM_ODD_MM_COMPAT
 	ret = drm_bo_remap_bound(bo);
@@ -1103,14 +1197,18 @@ static void drm_buffer_user_object_unmap(drm_file_t * priv,
  * bo->mutex locked. 
  */
 
-static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_flags,
+static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_mem_flags,
 			      int no_wait, int force_no_move)
 {
+	drm_device_t *dev = bo->dev;
+	drm_buffer_manager_t *bm = &dev->bm;
 	int ret = 0;
+	drm_bo_mem_reg_t mem;
 
 	/*
 	 * Flush outstanding fences.
 	 */
+
 	drm_bo_busy(bo);
 
 	/*
@@ -1126,16 +1224,38 @@ static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_flags,
 	 */
 
 	ret = drm_bo_wait(bo, 0, 0, no_wait);
+	if (ret)
+		return ret;
 
-	if (ret == -EINTR)
-		return -EAGAIN;
+
+	mem.num_pages = bo->num_pages;
+	mem.size = mem.num_pages << PAGE_SHIFT;
+	mem.mask = new_mem_flags;
+	mem.page_alignment = bo->page_alignment;
+
+	mutex_lock(&bm->evict_mutex);
+	mutex_lock(&dev->struct_mutex);
+	list_del(&bo->lru);
+	list_add_tail(&bo->lru,&bm->unfenced);
+	DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_UNFENCED, _DRM_BO_FLAG_UNFENCED);
+	mutex_unlock(&dev->struct_mutex);
+
+	ret = drm_bo_mem_space(dev, &mem, no_wait);
+	mutex_unlock(&bm->evict_mutex);
+	
 	if (ret)
 		return ret;
 
-	if (new_flags & DRM_BO_FLAG_MEM_TT) {
-		ret = drm_move_local_to_tt(bo, no_wait);
-		if (ret)
+	if (mem.mem_type == DRM_BO_MEM_TT) {
+		ret = drm_move_local_to_tt(bo, &mem, no_wait);
+		if (ret) {
+			mutex_lock(&dev->struct_mutex);
+			list_del_init(&bo->lru);
+			drm_bo_add_to_lru(bo, bm);
+			mutex_unlock(&dev->struct_mutex);
+			DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED);
 			return ret;
+		}
 	} else {
 		drm_move_tt_to_local(bo, 0, force_no_move);
 	}
@@ -1231,6 +1351,8 @@ static int drm_buffer_object_validate(drm_buffer_object_t * bo,
 		list_add_tail(&bo->lru, &bm->unfenced);
 		mutex_unlock(&dev->struct_mutex);
 	} else {
+		DRM_FLAG_MASKED(bo->priv_flags, 0,
+				_DRM_BO_FLAG_UNFENCED);
 		mutex_lock(&dev->struct_mutex);
 		list_del_init(&bo->lru);
 		drm_bo_add_to_lru(bo, bm);
diff --git a/linux-core/drm_stub.c b/linux-core/drm_stub.c
index 60123cdc..22592324 100644
--- a/linux-core/drm_stub.c
+++ b/linux-core/drm_stub.c
@@ -67,6 +67,7 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev,
 	mutex_init(&dev->struct_mutex);
 	mutex_init(&dev->ctxlist_mutex);
 	mutex_init(&dev->bm.init_mutex);
+	mutex_init(&dev->bm.evict_mutex);
 
 	dev->pdev = pdev;
 	dev->pci_device = pdev->device;
diff --git a/linux-core/i915_buffer.c b/linux-core/i915_buffer.c
index cdbe579a..13a3e9bb 100644
--- a/linux-core/i915_buffer.c
+++ b/linux-core/i915_buffer.c
@@ -97,6 +97,17 @@ int i915_init_mem_type(drm_device_t *dev, uint32_t type,
 	return 0;
 }
 
+uint32_t i915_evict_flags(drm_device_t *dev, uint32_t type)
+{
+	switch(type) {
+	case DRM_BO_MEM_LOCAL:
+	case DRM_BO_MEM_TT:
+		return DRM_BO_FLAG_MEM_LOCAL;
+	default:
+		return DRM_BO_FLAG_MEM_TT;
+	}
+}
+
 void i915_emit_copy_blit(drm_device_t *dev,
 			 uint32_t src_offset,
 			 uint32_t dst_offset,
diff --git a/linux-core/i915_drv.c b/linux-core/i915_drv.c
index 64ce3c15..8c39c249 100644
--- a/linux-core/i915_drv.c
+++ b/linux-core/i915_drv.c
@@ -50,11 +50,20 @@ static drm_fence_driver_t i915_fence_driver = {
 };
 #endif
 #ifdef I915_HAVE_BUFFER
+
+static uint32_t i915_mem_prios[] = {DRM_BO_MEM_PRIV0, DRM_BO_MEM_TT, DRM_BO_MEM_LOCAL};
+static uint32_t i915_busy_prios[] = {DRM_BO_MEM_TT, DRM_BO_MEM_PRIV0, DRM_BO_MEM_LOCAL};
+
 static drm_bo_driver_t i915_bo_driver = {
+	.mem_type_prio = i915_mem_prios,
+	.mem_busy_prio = i915_busy_prios,
+	.num_mem_type_prio = sizeof(i915_mem_prios)/sizeof(uint32_t),
+	.num_mem_busy_prio = sizeof(i915_busy_prios)/sizeof(uint32_t),
 	.create_ttm_backend_entry = i915_create_ttm_backend_entry,
 	.fence_type = i915_fence_types,
 	.invalidate_caches = i915_invalidate_caches,
 	.init_mem_type = i915_init_mem_type,
+	.evict_flags = i915_evict_flags,
 };
 #endif
 
diff --git a/shared-core/i915_drv.h b/shared-core/i915_drv.h
index be7dd76a..55c8cf57 100644
--- a/shared-core/i915_drv.h
+++ b/shared-core/i915_drv.h
@@ -191,6 +191,7 @@ extern int i915_fence_types(uint32_t buffer_flags, uint32_t *class, uint32_t *ty
 extern int i915_invalidate_caches(drm_device_t *dev, uint32_t buffer_flags);
 extern int i915_init_mem_type(drm_device_t *dev, uint32_t type, 
 			       drm_mem_type_manager_t *man);
+extern uint32_t i915_evict_flags(drm_device_t *dev, uint32_t type);
 #endif
 
 #define I915_READ(reg)          DRM_READ32(dev_priv->mmio_map, (reg))