memory manager: Make device driver aware of different memory types.

Memory types are either fixed (on-card or pre-bound AGP) or not fixed
(dynamically bound) to an aperture. They also carry information about:

1) Whether they can be mapped cached.
2) Whether they are at all mappable.
3) Whether they need an ioremap to be accessible from kernel space.

In this way VRAM memory and, for example, pre-bound AGP appear
identical to the memory manager.

This also makes support for unmappable VRAM simple to implement.

6 files changed, 127 insertions, 50 deletions

diff --git a/linux-core/drmP.h b/linux-core/drmP.h
index 9c748e6e..c0064bb7 100644
--- a/linux-core/drmP.h
+++ b/linux-core/drmP.h
@@ -650,17 +650,30 @@ typedef struct drm_ref_object {
 
 #include "drm_ttm.h"
 
+
+typedef struct drm_mem_type_manager {
+	int has_type;
+	int use_type;
+	drm_mm_t manager;
+	struct list_head lru;
+	struct list_head pinned;
+	uint32_t flags;
+	unsigned long io_offset;                 
+	unsigned long io_size;
+	void *io_addr;
+} drm_mem_type_manager_t;
+
 /*
  * buffer object driver
  */
 
 typedef struct drm_bo_driver{
-	int cached[DRM_BO_MEM_TYPES];
-        drm_local_map_t *iomap[DRM_BO_MEM_TYPES];
 	drm_ttm_backend_t *(*create_ttm_backend_entry) 
 		(struct drm_device *dev);
 	int (*fence_type)(uint32_t flags, uint32_t *class, uint32_t *type);
 	int (*invalidate_caches)(struct drm_device *dev, uint32_t flags);
+	int (*init_mem_type)(struct drm_device *dev, uint32_t type,
+			     drm_mem_type_manager_t *man);
 } drm_bo_driver_t;
 
 
@@ -782,16 +795,18 @@ typedef struct drm_fence_manager{
         atomic_t count;
 } drm_fence_manager_t;
 
+#define _DRM_FLAG_MEMTYPE_FIXED     0x00000001   /* Fixed (on-card) PCI memory */
+#define _DRM_FLAG_MEMTYPE_MAPPABLE  0x00000002   /* Memory mappable */
+#define _DRM_FLAG_MEMTYPE_CACHED    0x00000004   /* Supports cached binding */
+#define _DRM_FLAG_NEEDS_IOREMAP     0x00000008   /* Fixed memory needs ioremap
+						    before kernel access. */
+
 typedef struct drm_buffer_manager{
 	struct mutex init_mutex;
 	int nice_mode;
 	int initialized;
         drm_file_t *last_to_validate;
-	int has_type[DRM_BO_MEM_TYPES];
-        int use_type[DRM_BO_MEM_TYPES];
-	drm_mm_t manager[DRM_BO_MEM_TYPES];
-	struct list_head lru[DRM_BO_MEM_TYPES];
-        struct list_head pinned[DRM_BO_MEM_TYPES];
+	drm_mem_type_manager_t man[DRM_BO_MEM_TYPES];
 	struct list_head unfenced;
 	struct list_head ddestroy;
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c
index 2b960c75..b72e9912 100644
--- a/linux-core/drm_bo.c
+++ b/linux-core/drm_bo.c
@@ -74,8 +74,10 @@ static void drm_bo_add_to_lru(drm_buffer_object_t * bo,
 			      drm_buffer_manager_t * bm)
 {
 	struct list_head *list;
-	bo->mem_type = 0;
+	drm_mem_type_manager_t *man;
 
+	bo->mem_type = 0;
+	
 	switch(bo->flags & DRM_BO_MASK_MEM) {
 	case DRM_BO_FLAG_MEM_TT:
 		bo->mem_type = DRM_BO_MEM_TT;
@@ -89,8 +91,10 @@ static void drm_bo_add_to_lru(drm_buffer_object_t * bo,
 	default:
 		BUG_ON(1);		
 	}
+	
+	man = &bm->man[bo->mem_type];
 	list = (bo->flags & (DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE)) ?
-		&bm->pinned[bo->mem_type] : &bm->lru[bo->mem_type];
+		&man->pinned : &man->lru;
 	list_add_tail(&bo->lru, list);
 	return;
 }
@@ -543,7 +547,8 @@ int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type,
 	drm_mm_node_t *node;
 	drm_buffer_manager_t *bm = &dev->bm;
 	drm_buffer_object_t *entry;
-	drm_mm_t *mm = &bm->manager[mem_type];
+	drm_mem_type_manager_t *man = &bm->man[mem_type];
+	drm_mm_t *mm = &man->manager;
 	struct list_head *lru;
 	unsigned long size = bo->num_pages;
 	int ret;
@@ -554,7 +559,7 @@ int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type,
 		if (node)
 			break;
 
-		lru = &bm->lru[mem_type];
+		lru = &man->lru;
 		if (lru->next == lru)
 			break;
 
@@ -638,7 +643,6 @@ static int drm_bo_new_flags(drm_device_t * dev,
 {
 	uint32_t new_flags = 0;
 	uint32_t new_props;
-	drm_bo_driver_t *driver = dev->driver->bo_driver;
 	drm_buffer_manager_t *bm = &dev->bm;
 	unsigned i;
 
@@ -647,7 +651,7 @@ static int drm_bo_new_flags(drm_device_t * dev,
 	 */
 
 	for (i = 0; i < DRM_BO_MEM_TYPES; ++i) {
-		if (!bm->use_type[i])
+		if (!bm->man[i].use_type)
 			new_mask &= ~drm_bo_type_flags(i);
 	}
 
@@ -659,14 +663,18 @@ static int drm_bo_new_flags(drm_device_t * dev,
 	}
 	if (new_mask & DRM_BO_FLAG_BIND_CACHED) {
 		if (((new_mask & DRM_BO_FLAG_MEM_TT) &&
-		     !driver->cached[DRM_BO_MEM_TT]) &&
-		    ((new_mask & DRM_BO_FLAG_MEM_VRAM)
-		     && !driver->cached[DRM_BO_MEM_VRAM])) {
+		     !(bm->man[DRM_BO_MEM_TT].flags &
+			_DRM_FLAG_MEMTYPE_CACHED) &&
+		     ((new_mask & DRM_BO_FLAG_MEM_VRAM)
+		      && !(bm->man[DRM_BO_MEM_VRAM].flags &
+			   _DRM_FLAG_MEMTYPE_CACHED)))) {
 			new_mask &= ~DRM_BO_FLAG_BIND_CACHED;
 		} else {
-			if (!driver->cached[DRM_BO_MEM_TT])
+			if (!(bm->man[DRM_BO_MEM_TT].flags &
+			      _DRM_FLAG_MEMTYPE_CACHED))
 				new_flags &= DRM_BO_FLAG_MEM_TT;
-			if (!driver->cached[DRM_BO_MEM_VRAM])
+			if (!(bm->man[DRM_BO_MEM_VRAM].flags &
+			      _DRM_FLAG_MEMTYPE_CACHED))
 				new_flags &= DRM_BO_FLAG_MEM_VRAM;
 		}
 	}
@@ -1735,6 +1743,8 @@ static int drm_bo_force_list_clean(drm_device_t * dev,
 int drm_bo_clean_mm(drm_device_t * dev, unsigned mem_type)
 {
 	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man = &bm->man[mem_type];
+	drm_mem_type_manager_t *local_man = &bm->man[DRM_BO_MEM_LOCAL];
 	int ret = -EINVAL;
 
 	if (mem_type >= DRM_BO_MEM_TYPES) {
@@ -1742,13 +1752,13 @@ int drm_bo_clean_mm(drm_device_t * dev, unsigned mem_type)
 		return ret;
 	}
 
-	if (!bm->has_type[mem_type]) {
+	if (!man->has_type) {
 		DRM_ERROR("Trying to take down uninitialized "
 			  "memory manager type\n");
 		return ret;
 	}
-	bm->use_type[mem_type] = 0;
-	bm->has_type[mem_type] = 0;
+	man->use_type = 0;
+	man->has_type = 0;
 
 	ret = 0;
 	if (mem_type > 0) {
@@ -1763,15 +1773,12 @@ int drm_bo_clean_mm(drm_device_t * dev, unsigned mem_type)
 		 * Throw out evicted no-move buffers.
 		 */
 
-		drm_bo_force_list_clean(dev, &bm->pinned[DRM_BO_MEM_LOCAL],
-					mem_type, 1, 0);
-		drm_bo_force_list_clean(dev, &bm->lru[mem_type], mem_type, 1,
-					0);
-		drm_bo_force_list_clean(dev, &bm->pinned[mem_type], mem_type, 1,
-					0);
+		drm_bo_force_list_clean(dev, &local_man->pinned, mem_type, 1, 0);
+		drm_bo_force_list_clean(dev, &man->lru, mem_type, 1, 0);
+		drm_bo_force_list_clean(dev, &man->pinned, mem_type, 1, 0);
 
-		if (drm_mm_clean(&bm->manager[mem_type])) {
-			drm_mm_takedown(&bm->manager[mem_type]);
+		if (drm_mm_clean(&man->manager)) {
+			drm_mm_takedown(&man->manager);
 		} else {
 			ret = -EBUSY;
 		}
@@ -1784,6 +1791,7 @@ static int drm_bo_lock_mm(drm_device_t * dev, unsigned mem_type)
 {
 	int ret;
 	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man = &bm->man[mem_type];
 
 	if (mem_type == 0 || mem_type >= DRM_BO_MEM_TYPES) {
 		DRM_ERROR("Illegal memory manager memory type %u,\n", mem_type);
@@ -1793,11 +1801,11 @@ static int drm_bo_lock_mm(drm_device_t * dev, unsigned mem_type)
 	ret = drm_bo_force_list_clean(dev, &bm->unfenced, mem_type, 0, 1);
 	if (ret)
 		return ret;
-	ret = drm_bo_force_list_clean(dev, &bm->lru[mem_type], mem_type, 0, 1);
+	ret = drm_bo_force_list_clean(dev, &man->lru, mem_type, 0, 1);
 	if (ret)
 		return ret;
 	ret =
-	    drm_bo_force_list_clean(dev, &bm->pinned[mem_type], mem_type, 0, 1);
+	    drm_bo_force_list_clean(dev, &man->pinned, mem_type, 0, 1);
 	return ret;
 }
 
@@ -1807,32 +1815,39 @@ static int drm_bo_init_mm(drm_device_t * dev,
 {
 	drm_buffer_manager_t *bm = &dev->bm;
 	int ret = -EINVAL;
+	drm_mem_type_manager_t *man;
 
 	if (type >= DRM_BO_MEM_TYPES) {
 		DRM_ERROR("Illegal memory type %d\n", type);
 		return ret;
 	}
-	if (bm->has_type[type]) {
+	
+	man = &bm->man[type];
+	if (man->has_type) {
 		DRM_ERROR("Memory manager already initialized for type %d\n",
 			  type);
 		return ret;
 	}
 
+	ret = dev->driver->bo_driver->init_mem_type(dev, type, man);
+	if (ret) 
+		return ret;
+
 	ret = 0;
 	if (type != DRM_BO_MEM_LOCAL) {
 		if (!p_size) {
 			DRM_ERROR("Zero size memory manager type %d\n", type);
 			return ret;
 		}
-		ret = drm_mm_init(&bm->manager[type], p_offset, p_size);
+		ret = drm_mm_init(&man->manager, p_offset, p_size);
 		if (ret)
 			return ret;
 	}
-	bm->has_type[type] = 1;
-	bm->use_type[type] = 1;
+	man->has_type = 1;
+	man->use_type = 1;
 
-	INIT_LIST_HEAD(&bm->lru[type]);
-	INIT_LIST_HEAD(&bm->pinned[type]);
+	INIT_LIST_HEAD(&man->lru);
+	INIT_LIST_HEAD(&man->pinned);
 
 	return 0;
 }
@@ -1847,6 +1862,7 @@ int drm_bo_driver_finish(drm_device_t * dev)
 	drm_buffer_manager_t *bm = &dev->bm;
 	int ret = 0;
 	unsigned i = DRM_BO_MEM_TYPES;
+	drm_mem_type_manager_t *man;
 
 	mutex_lock(&dev->bm.init_mutex);
 	mutex_lock(&dev->struct_mutex);
@@ -1856,14 +1872,15 @@ int drm_bo_driver_finish(drm_device_t * dev)
 	bm->initialized = 0;
 
 	while (i--) {
-		if (bm->has_type[i]) {
-			bm->use_type[i] = 0;
+		man = &bm->man[i];
+		if (man->has_type) {
+			man->use_type = 0;
 			if ((i != DRM_BO_MEM_LOCAL) && drm_bo_clean_mm(dev, i)) {
 				ret = -EBUSY;
 				DRM_ERROR("DRM memory manager type %d "
 					  "is not clean.\n", i);
 			}
-			bm->has_type[i] = 0;
+			man->has_type = 0;
 		}
 	}
 	mutex_unlock(&dev->struct_mutex);
@@ -1875,10 +1892,10 @@ int drm_bo_driver_finish(drm_device_t * dev)
 	if (list_empty(&bm->ddestroy)) {
 		DRM_DEBUG("Delayed destroy list was clean\n");
 	}
-	if (list_empty(&bm->lru[0])) {
+	if (list_empty(&bm->man[0].lru)) {
 		DRM_DEBUG("Swap list was clean\n");
 	}
-	if (list_empty(&bm->pinned[0])) {
+	if (list_empty(&bm->man[0].pinned)) {
 		DRM_DEBUG("NO_MOVE list was clean\n");
 	}
 	if (list_empty(&bm->unfenced)) {
diff --git a/linux-core/i915_buffer.c b/linux-core/i915_buffer.c
index c3e54468..53002301 100644
--- a/linux-core/i915_buffer.c
+++ b/linux-core/i915_buffer.c
@@ -64,3 +64,36 @@ int i915_invalidate_caches(drm_device_t * dev, uint32_t flags)
 
 	return i915_emit_mi_flush(dev, flush_cmd);
 }
+
+int i915_init_mem_type(drm_device_t *dev, uint32_t type, 
+		       drm_mem_type_manager_t *man)
+{
+	switch(type) {
+	case DRM_BO_MEM_LOCAL:
+		break;
+	case DRM_BO_MEM_TT:
+		man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE |
+			_DRM_FLAG_MEMTYPE_CACHED;
+		break;
+	case DRM_BO_MEM_PRIV0:
+		if (!(drm_core_has_AGP(dev) && dev->agp)) {
+			DRM_ERROR("AGP is not enabled for memory type %u\n", 
+				  (unsigned) type);
+			return -EINVAL;
+		}
+		man->io_offset = dev->agp->agp_info.aper_base;
+		man->io_size = dev->agp->agp_info.aper_size * 1024 * 1024;
+
+		man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE |
+			_DRM_FLAG_MEMTYPE_CACHED |
+			_DRM_FLAG_MEMTYPE_FIXED |
+			_DRM_FLAG_NEEDS_IOREMAP;
+
+		man->io_addr = NULL;
+		break;
+	default:
+		DRM_ERROR("Unsupported memory type %u\n", (unsigned) type);
+		return -EINVAL;
+	}
+	return 0;
+}
diff --git a/linux-core/i915_drv.c b/linux-core/i915_drv.c
index 2c5b43d0..64ce3c15 100644
--- a/linux-core/i915_drv.c
+++ b/linux-core/i915_drv.c
@@ -51,11 +51,10 @@ static drm_fence_driver_t i915_fence_driver = {
 #endif
 #ifdef I915_HAVE_BUFFER
 static drm_bo_driver_t i915_bo_driver = {
-        .iomap = {NULL, NULL},
-	.cached = {1, 1},
 	.create_ttm_backend_entry = i915_create_ttm_backend_entry,
 	.fence_type = i915_fence_types,
-	.invalidate_caches = i915_invalidate_caches
+	.invalidate_caches = i915_invalidate_caches,
+	.init_mem_type = i915_init_mem_type,
 };
 #endif
 
diff --git a/shared-core/drm.h b/shared-core/drm.h
index 9efb1dc4..38cca882 100644
--- a/shared-core/drm.h
+++ b/shared-core/drm.h
@@ -738,8 +738,12 @@ typedef struct drm_fence_arg {
 #define DRM_BO_FLAG_MEM_TT     0x02000000
 /* Vram memory */
 #define DRM_BO_FLAG_MEM_VRAM   0x04000000
-/* Unmappable Vram memory */
-#define DRM_BO_FLAG_MEM_VRAM_NM   0x08000000
+/* Up to the driver to define. */
+#define DRM_BO_FLAG_MEM_PRIV0  0x10000000
+#define DRM_BO_FLAG_MEM_PRIV1  0x20000000
+#define DRM_BO_FLAG_MEM_PRIV2  0x40000000
+#define DRM_BO_FLAG_MEM_PRIV3  0x80000000
+
 /* Memory flag mask */
 #define DRM_BO_MASK_MEM         0xFF000000
 
@@ -823,8 +827,13 @@ typedef struct drm_bo_arg{
 #define DRM_BO_MEM_LOCAL 0
 #define DRM_BO_MEM_TT 1
 #define DRM_BO_MEM_VRAM 2
-#define DRM_BO_MEM_VRAM_NM 3
-#define DRM_BO_MEM_TYPES 2 /* For now. */
+#define DRM_BO_MEM_PRIV0 4
+#define DRM_BO_MEM_PRIV1 5
+#define DRM_BO_MEM_PRIV2 6
+#define DRM_BO_MEM_PRIV3 7
+
+
+#define DRM_BO_MEM_TYPES 8 /* For now. */
 
 typedef union drm_mm_init_arg{
 	struct {
diff --git a/shared-core/i915_drv.h b/shared-core/i915_drv.h
index ef9f3638..be7dd76a 100644
--- a/shared-core/i915_drv.h
+++ b/shared-core/i915_drv.h
@@ -126,7 +126,9 @@ typedef struct drm_i915_private {
 	uint32_t flush_pending;
 	uint32_t saved_flush_status;
 #endif
-
+#ifdef I915_HAVE_BUFFER
+	void *agp_iomap;
+#endif
 	spinlock_t swaps_lock;
 	drm_i915_vbl_swap_t vbl_swaps;
 	unsigned int swaps_pending;
@@ -187,6 +189,8 @@ extern void i915_poke_flush(drm_device_t *dev);
 extern drm_ttm_backend_t *i915_create_ttm_backend_entry(drm_device_t *dev);
 extern int i915_fence_types(uint32_t buffer_flags, uint32_t *class, uint32_t *type);
 extern int i915_invalidate_caches(drm_device_t *dev, uint32_t buffer_flags);
+extern int i915_init_mem_type(drm_device_t *dev, uint32_t type, 
+			       drm_mem_type_manager_t *man);
 #endif
 
 #define I915_READ(reg)          DRM_READ32(dev_priv->mmio_map, (reg))