Merge branch 'ttm-vram-0-1-branch'

18 files changed, 2591 insertions, 1645 deletions

diff --git a/linux-core/Makefile.kernel b/linux-core/Makefile.kernel
index 05d6e149..c2bfb671 100644
--- a/linux-core/Makefile.kernel
+++ b/linux-core/Makefile.kernel
@@ -13,7 +13,7 @@ drm-objs    := drm_auth.o drm_bufs.o drm_context.o drm_dma.o drm_drawable.o \
 		drm_sysfs.o drm_pci.o drm_agpsupport.o drm_scatter.o \
 		drm_memory_debug.o ati_pcigart.o drm_sman.o \
 		drm_hashtab.o drm_mm.o drm_object.o drm_compat.o \
-	        drm_fence.o drm_ttm.o drm_bo.o
+	        drm_fence.o drm_ttm.o drm_bo.o drm_bo_move.o
 tdfx-objs   := tdfx_drv.o
 r128-objs   := r128_drv.o r128_cce.o r128_state.o r128_irq.o
 mga-objs    := mga_drv.o mga_dma.o mga_state.o mga_warp.o mga_irq.o
diff --git a/linux-core/drmP.h b/linux-core/drmP.h
index edc15575..9b5f5bdd 100644
--- a/linux-core/drmP.h
+++ b/linux-core/drmP.h
@@ -595,78 +595,8 @@ typedef struct ati_pcigart_info {
 	drm_local_map_t mapping;
 } drm_ati_pcigart_info;
 
-/*
- * User space objects and their references.
- */
-
-#define drm_user_object_entry(_ptr, _type, _member) container_of(_ptr, _type, _member)
-
-typedef enum {
-		drm_fence_type,
-		drm_buffer_type,
-		drm_ttm_type
-
-		/*
-		 * Add other user space object types here. 
-		 */
-
-} drm_object_type_t;
-
-
-
-
-/*
- * A user object is a structure that helps the drm give out user handles
- * to kernel internal objects and to keep track of these objects so that 
- * they can be destroyed, for example when the user space process exits.
- * Designed to be accessible using a user space 32-bit handle. 
- */
-
-typedef struct drm_user_object{
-	drm_hash_item_t hash;
-	struct list_head list;
-	drm_object_type_t type;
-        atomic_t refcount;
-        int shareable;
-        drm_file_t *owner;
-	void (*ref_struct_locked) (drm_file_t *priv, struct drm_user_object *obj, 
-				   drm_ref_t ref_action); 
-	void (*unref)(drm_file_t *priv, struct drm_user_object *obj, 
-		      drm_ref_t unref_action);
-	void (*remove)(drm_file_t *priv, struct drm_user_object *obj);
-} drm_user_object_t;
-
-/*
- * A ref object is a structure which is used to
- * keep track of references to user objects and to keep track of these
- * references so that they can be destroyed for example when the user space
- * process exits. Designed to be accessible using a pointer to the _user_ object.
- */
-
-
-typedef struct drm_ref_object {
-	drm_hash_item_t hash;
-	struct list_head list;
-	atomic_t refcount;
-	drm_ref_t unref_action;
-} drm_ref_object_t;
-
-
-#include "drm_ttm.h"
-
-/*
- * buffer object driver
- */
-
-typedef struct drm_bo_driver{
-	int cached[DRM_BO_MEM_TYPES];
-        drm_local_map_t *iomap[DRM_BO_MEM_TYPES];
-	drm_ttm_backend_t *(*create_ttm_backend_entry) 
-		(struct drm_device *dev);
-	int (*fence_type)(uint32_t flags, uint32_t *class, uint32_t *type);
-	int (*invalidate_caches)(struct drm_device *dev, uint32_t flags);
-} drm_bo_driver_t;
 
+#include "drm_objects.h"
 
 /**
  * DRM driver structure. This structure represent the common code for
@@ -755,63 +685,6 @@ typedef struct drm_head {
 } drm_head_t;
 
 
-typedef struct drm_fence_driver{
-	int no_types;
-	uint32_t wrap_diff;
-	uint32_t flush_diff;
-        uint32_t sequence_mask;
-        int lazy_capable;
-	int (*emit) (struct drm_device *dev, uint32_t flags,
-		     uint32_t *breadcrumb,
-		     uint32_t *native_type);
-	void (*poke_flush) (struct drm_device *dev);
-} drm_fence_driver_t;
-
-#define _DRM_FENCE_TYPE_EXE 0x00
-
-typedef struct drm_fence_manager{
-        int initialized;
-	rwlock_t lock;
-
-	/*
-	 * The list below should be maintained in sequence order and 
-	 * access is protected by the above spinlock.
-	 */
-
-	struct list_head ring;
-	struct list_head *fence_types[32];
-	volatile uint32_t pending_flush;
-	wait_queue_head_t fence_queue;
-	int pending_exe_flush;
-	uint32_t last_exe_flush;
-	uint32_t exe_flush_sequence;
-        atomic_t count;
-} drm_fence_manager_t;
-
-typedef struct drm_buffer_manager{
-	struct mutex init_mutex;
-	int nice_mode;
-	int initialized;
-        drm_file_t *last_to_validate;
-	int has_type[DRM_BO_MEM_TYPES];
-        int use_type[DRM_BO_MEM_TYPES];
-	drm_mm_t manager[DRM_BO_MEM_TYPES];
-	struct list_head lru[DRM_BO_MEM_TYPES];
-        struct list_head pinned[DRM_BO_MEM_TYPES];
-	struct list_head unfenced;
-	struct list_head ddestroy;
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
-        struct work_struct wq;
-#else
-        struct delayed_work wq;
-#endif
-        uint32_t fence_type;
-        unsigned long cur_pages;
-        atomic_t count;
-} drm_buffer_manager_t;
-
-
-
 /**
  * DRM device structure. This structure represent a complete card that
  * may contain multiple heads.
@@ -966,62 +839,6 @@ typedef struct drm_agp_ttm_priv {
 } drm_agp_ttm_priv;
 #endif
 
-typedef struct drm_fence_object{
-	drm_user_object_t base;
-        atomic_t usage;
-
-	/*
-	 * The below three fields are protected by the fence manager spinlock.
-	 */
-
-	struct list_head ring;
-        int class;
-        uint32_t native_type;
-	uint32_t type;
-	uint32_t signaled;
-	uint32_t sequence;
-	uint32_t flush_mask;
-	uint32_t submitted_flush;
-} drm_fence_object_t;
-
-
-typedef struct drm_buffer_object{
-	drm_device_t *dev;
-	drm_user_object_t base;
-
-    /*
-     * If there is a possibility that the usage variable is zero,
-     * then dev->struct_mutext should be locked before incrementing it.
-     */
-
-	atomic_t usage;
-	drm_ttm_object_t *ttm_object;
-        drm_ttm_t *ttm;
-	unsigned long num_pages;
-        unsigned long buffer_start;
-        drm_bo_type_t type;
-        unsigned long offset;
-        uint32_t page_alignment;
-	atomic_t mapped;
-	uint32_t flags;
-	uint32_t mask;
-        uint32_t mem_type;
-
-	drm_mm_node_t *mm_node;     /* MM node for on-card RAM */
-        struct list_head lru;
-	struct list_head ddestroy;
-
-	uint32_t fence_type;
-        uint32_t fence_class;
-	drm_fence_object_t *fence;
-        uint32_t priv_flags;
-	wait_queue_head_t event_queue;
-        struct mutex mutex;
-} drm_buffer_object_t;
-
-#define _DRM_BO_FLAG_UNFENCED 0x00000001
-#define _DRM_BO_FLAG_EVICTED  0x00000002
-
 
 static __inline__ int drm_core_check_feature(struct drm_device *dev,
 					     int feature)
@@ -1365,105 +1182,9 @@ static inline drm_mm_t *drm_get_mm(drm_mm_node_t *block)
 }
   
 
-/*
- * User space object bookkeeping (drm_object.c)
- */
-
-/*
- * Must be called with the struct_mutex held.
- */
-
-extern int drm_add_user_object(drm_file_t *priv, drm_user_object_t *item, 
-
-/*
- * Must be called with the struct_mutex held.
- */
-			       int shareable);
-extern drm_user_object_t *drm_lookup_user_object(drm_file_t *priv, uint32_t key);
-
-/*
- * Must be called with the struct_mutex held.
- * If "item" has been obtained by a call to drm_lookup_user_object. You may not
- * release the struct_mutex before calling drm_remove_ref_object.
- * This function may temporarily release the struct_mutex.
- */
 
-extern int drm_remove_user_object(drm_file_t *priv, drm_user_object_t *item);
 
-/*
- * Must be called with the struct_mutex held. May temporarily release it.
- */
-
-extern int drm_add_ref_object(drm_file_t *priv, drm_user_object_t *referenced_object,
-			      drm_ref_t ref_action);
-
-/*
- * Must be called with the struct_mutex held.
- */
-
-drm_ref_object_t *drm_lookup_ref_object(drm_file_t *priv, 
-					drm_user_object_t *referenced_object,
-					drm_ref_t ref_action);
-/*
- * Must be called with the struct_mutex held.
- * If "item" has been obtained by a call to drm_lookup_ref_object. You may not
- * release the struct_mutex before calling drm_remove_ref_object.
- * This function may temporarily release the struct_mutex.
- */
-
-extern void drm_remove_ref_object(drm_file_t *priv, drm_ref_object_t *item);
-extern int drm_user_object_ref(drm_file_t *priv, uint32_t user_token, drm_object_type_t type,
-			       drm_user_object_t **object);
-extern int drm_user_object_unref(drm_file_t *priv, uint32_t user_token, drm_object_type_t type);
-
-
-
-/*
- * fence objects (drm_fence.c)
- */
-
-extern void drm_fence_handler(drm_device_t *dev, uint32_t breadcrumb, uint32_t type);
-extern void drm_fence_manager_init(drm_device_t *dev);
-extern void drm_fence_manager_takedown(drm_device_t *dev);
-extern void drm_fence_flush_old(drm_device_t *dev, uint32_t sequence);
-extern int drm_fence_object_flush(drm_device_t * dev,
-				  volatile drm_fence_object_t * fence, 
-				  uint32_t type);
-extern int drm_fence_object_signaled(volatile drm_fence_object_t * fence, 
-				     uint32_t type);
-extern void drm_fence_usage_deref_locked(drm_device_t * dev,
-					 drm_fence_object_t * fence);
-extern void drm_fence_usage_deref_unlocked(drm_device_t * dev,
-					 drm_fence_object_t * fence);
-extern int drm_fence_object_wait(drm_device_t * dev, 
-				 volatile drm_fence_object_t * fence,
-				 int lazy, int ignore_signals, uint32_t mask);
-extern int drm_fence_object_create(drm_device_t *dev, uint32_t type,
-				   uint32_t fence_flags, 
-				   drm_fence_object_t **c_fence);
-extern int drm_fence_add_user_object(drm_file_t *priv, 
-				     drm_fence_object_t *fence,
-				     int shareable);
-
-
-
-
-
-extern int drm_fence_ioctl(DRM_IOCTL_ARGS);
-
-/*
- * buffer objects (drm_bo.c)
- */
 
-extern int drm_bo_ioctl(DRM_IOCTL_ARGS);
-extern int drm_mm_init_ioctl(DRM_IOCTL_ARGS);
-extern int drm_bo_driver_finish(drm_device_t *dev);
-extern int drm_bo_driver_init(drm_device_t *dev);
-extern int drm_fence_buffer_objects(drm_file_t * priv,
-				    struct list_head *list, 
-				    uint32_t fence_flags,
-				    drm_fence_object_t *fence,
-				    drm_fence_object_t **used_fence);
 
 extern void drm_core_ioremap(struct drm_map *map, struct drm_device *dev);
 extern void drm_core_ioremapfree(struct drm_map *map, struct drm_device *dev);
diff --git a/linux-core/drm_agpsupport.c b/linux-core/drm_agpsupport.c
index 9cdbdaf0..6b93d249 100644
--- a/linux-core/drm_agpsupport.c
+++ b/linux-core/drm_agpsupport.c
@@ -606,8 +606,8 @@ static int drm_agp_bind_ttm(drm_ttm_backend_t *backend,
 	int ret;
 
 	DRM_DEBUG("drm_agp_bind_ttm\n");
-	DRM_MASK_VAL(backend->flags, DRM_BE_FLAG_BOUND_CACHED,
-		     (cached) ? DRM_BE_FLAG_BOUND_CACHED : 0);
+	DRM_FLAG_MASKED(backend->flags, (cached) ? DRM_BE_FLAG_BOUND_CACHED : 0,
+			DRM_BE_FLAG_BOUND_CACHED);
 	mem->is_flushed = TRUE;
 	mem->type = (cached) ? agp_priv->cached_type : agp_priv->uncached_type;
 	ret = drm_agp_bind_memory(mem, offset);
@@ -620,7 +620,7 @@ static int drm_agp_bind_ttm(drm_ttm_backend_t *backend,
 static int drm_agp_unbind_ttm(drm_ttm_backend_t *backend) {
 
 	drm_agp_ttm_priv *agp_priv = (drm_agp_ttm_priv *) backend->private;
-
+	
 	DRM_DEBUG("drm_agp_unbind_ttm\n");
 	if (agp_priv->mem->is_bound)
 		return drm_agp_unbind_memory(agp_priv->mem);
@@ -710,7 +710,6 @@ drm_ttm_backend_t *drm_agp_init_ttm(struct drm_device *dev,
 	agp_priv->uncached_type = AGP_USER_MEMORY;
 	agp_priv->bridge = dev->agp->bridge;
 	agp_priv->populated = FALSE;
-	agp_be->aperture_base = dev->agp->agp_info.aper_base;
 	agp_be->private = (void *) agp_priv;
 	agp_be->needs_ub_cache_adjust = drm_agp_needs_unbind_cache_adjust;
 	agp_be->populate = drm_agp_populate;
@@ -718,10 +717,8 @@ drm_ttm_backend_t *drm_agp_init_ttm(struct drm_device *dev,
 	agp_be->bind = drm_agp_bind_ttm;
 	agp_be->unbind = drm_agp_unbind_ttm;
 	agp_be->destroy = drm_agp_destroy_ttm;
-	DRM_MASK_VAL(agp_be->flags, DRM_BE_FLAG_NEEDS_FREE,
-		     (backend == NULL) ? DRM_BE_FLAG_NEEDS_FREE : 0);
-	DRM_MASK_VAL(agp_be->flags, DRM_BE_FLAG_CBA,
-		     (dev->agp->cant_use_aperture) ? DRM_BE_FLAG_CBA : 0);
+	DRM_FLAG_MASKED(agp_be->flags, (backend == NULL) ? DRM_BE_FLAG_NEEDS_FREE : 0,
+			DRM_BE_FLAG_NEEDS_FREE);
 	agp_be->drm_map_type = _DRM_AGP;
 	return agp_be;
 }
diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c
index 2b960c75..e593258b 100644
--- a/linux-core/drm_bo.c
+++ b/linux-core/drm_bo.c
@@ -1,6 +1,6 @@
 /**************************************************************************
  * 
- * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
+ * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
  * All Rights Reserved.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -10,6 +10,10 @@
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
  * 
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
@@ -18,11 +22,6 @@
  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- * 
  * 
  **************************************************************************/
 /*
@@ -32,34 +31,28 @@
 #include "drmP.h"
 
 /*
- * Buffer object locking policy:
- * Lock dev->struct_mutex;
- * Increase usage
- * Unlock dev->struct_mutex;
- * Lock buffer->mutex;
- * Do whatever you want;
- * Unlock buffer->mutex;
- * Decrease usage. Call destruction if zero.
+ * Locking may look a bit complicated but isn't really:
+ *
+ * The buffer usage atomic_t needs to be protected by dev->struct_mutex
+ * when there is a chance that it can be zero before or after the operation.
+ * 
+ * dev->struct_mutex also protects all lists and list heads. Hash tables and hash
+ * heads.
  *
- * User object visibility ups usage just once, since it has its own 
- * refcounting.
+ * bo->mutex protects the buffer object itself excluding the usage field.
+ * bo->mutex does also protect the buffer list heads, so to manipulate those, we need
+ * both the bo->mutex and the dev->struct_mutex.
  *
- * Destruction:
- * lock dev->struct_mutex;
- * Verify that usage is zero. Otherwise unlock and continue.
- * Destroy object.
- * unlock dev->struct_mutex;
+ * Locking order is bo->mutex, dev->struct_mutex. Therefore list traversal is a bit
+ * complicated. When dev->struct_mutex is released to grab bo->mutex, the list
+ * traversal will, in general, need to be restarted.
  *
- * Mutex and spinlock locking orders:
- * 1.) Buffer mutex
- * 2.) Refer to ttm locking orders.
  */
 
-static void drm_bo_destroy_locked(drm_buffer_object_t *bo);
-
-#define DRM_FLAG_MASKED(_old, _new, _mask) {\
-(_old) ^= (((_old) ^ (_new)) & (_mask)); \
-}
+static void drm_bo_destroy_locked(drm_buffer_object_t * bo);
+static int drm_bo_setup_vm_locked(drm_buffer_object_t * bo);
+static void drm_bo_takedown_vm_locked(drm_buffer_object_t * bo);
+static void drm_bo_unmap_virtual(drm_buffer_object_t * bo);
 
 static inline uint32_t drm_bo_type_flags(unsigned type)
 {
@@ -70,76 +63,189 @@ static inline uint32_t drm_bo_type_flags(unsigned type)
  * bo locked. dev->struct_mutex locked.
  */
 
-static void drm_bo_add_to_lru(drm_buffer_object_t * bo,
-			      drm_buffer_manager_t * bm)
+void drm_bo_add_to_pinned_lru(drm_buffer_object_t * bo)
 {
-	struct list_head *list;
-	bo->mem_type = 0;
+	drm_mem_type_manager_t *man;
 
-	switch(bo->flags & DRM_BO_MASK_MEM) {
-	case DRM_BO_FLAG_MEM_TT:
-		bo->mem_type = DRM_BO_MEM_TT;
-		break;
-	case DRM_BO_FLAG_MEM_VRAM:
-		bo->mem_type = DRM_BO_MEM_VRAM;
-		break;
-	case DRM_BO_FLAG_MEM_LOCAL:
-		bo->mem_type = DRM_BO_MEM_LOCAL;
-		break;
-	default:
-		BUG_ON(1);		
+	man = &bo->dev->bm.man[bo->pinned_mem_type];
+	list_add_tail(&bo->pinned_lru, &man->pinned);
+}
+
+void drm_bo_add_to_lru(drm_buffer_object_t * bo)
+{
+	drm_mem_type_manager_t *man;
+
+	if (!(bo->mem.mask & (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT))) {
+		man = &bo->dev->bm.man[bo->mem.mem_type];
+		list_add_tail(&bo->lru, &man->lru);
+	} else {
+		INIT_LIST_HEAD(&bo->lru);
 	}
-	list = (bo->flags & (DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE)) ?
-		&bm->pinned[bo->mem_type] : &bm->lru[bo->mem_type];
-	list_add_tail(&bo->lru, list);
-	return;
+}
+
+static int drm_bo_vm_pre_move(drm_buffer_object_t * bo, int old_is_pci)
+{
+#ifdef DRM_ODD_MM_COMPAT
+	int ret;
+
+	ret = drm_bo_lock_kmm(bo);
+	if (ret)
+		return ret;
+	drm_bo_unmap_virtual(bo);
+	if (old_is_pci)
+		drm_bo_finish_unmap(bo);
+#else
+	drm_bo_unmap_virtual(bo);
+#endif
+	return 0;
+}
+
+static void drm_bo_vm_post_move(drm_buffer_object_t * bo)
+{
+#ifdef DRM_ODD_MM_COMPAT
+	int ret;
+
+	ret = drm_bo_remap_bound(bo);
+	if (ret) {
+		DRM_ERROR("Failed to remap a bound buffer object.\n"
+			  "\tThis might cause a sigbus later.\n");
+	}
+	drm_bo_unlock_kmm(bo);
+#endif
 }
 
 /*
- * bo locked.
+ * Call bo->mutex locked.
  */
 
-static int drm_move_tt_to_local(drm_buffer_object_t * bo, int evict,
-				int force_no_move)
+static int drm_bo_add_ttm(drm_buffer_object_t * bo)
 {
 	drm_device_t *dev = bo->dev;
-	int ret;
+	int ret = 0;
+	bo->ttm = NULL;
 
-	if (bo->mm_node) {
-		mutex_lock(&dev->struct_mutex);
-		if (evict)
-			ret = drm_evict_ttm(bo->ttm);
-		else
-			ret = drm_unbind_ttm(bo->ttm);
+	switch (bo->type) {
+	case drm_bo_type_dc:
+		bo->ttm = drm_ttm_init(dev, bo->mem.num_pages << PAGE_SHIFT);
+		if (!bo->ttm)
+			ret = -ENOMEM;
+		break;
+	case drm_bo_type_user:
+	case drm_bo_type_fake:
+		break;
+	default:
+		DRM_ERROR("Illegal buffer object type\n");
+		ret = -EINVAL;
+		break;
+	}
 
-		if (ret) {
-			mutex_unlock(&dev->struct_mutex);
-			if (ret == -EAGAIN)
-				schedule();
-			return ret;
-		}
+	return ret;
+}
+
+static int drm_bo_handle_move_mem(drm_buffer_object_t * bo,
+				  drm_bo_mem_reg_t * mem,
+				  int evict, int no_wait)
+{
+	drm_device_t *dev = bo->dev;
+	drm_buffer_manager_t *bm = &dev->bm;
+	int old_is_pci = drm_mem_reg_is_pci(dev, &bo->mem);
+	int new_is_pci = drm_mem_reg_is_pci(dev, mem);
+	drm_mem_type_manager_t *old_man = &bm->man[bo->mem.mem_type];
+	drm_mem_type_manager_t *new_man = &bm->man[mem->mem_type];
+	int ret = 0;
 
-		if (!(bo->flags & DRM_BO_FLAG_NO_MOVE) || force_no_move) {
-			drm_mm_put_block(bo->mm_node);
-			bo->mm_node = NULL;
+	if (old_is_pci || new_is_pci)
+		ret = drm_bo_vm_pre_move(bo, old_is_pci);
+	if (ret)
+		return ret;
+
+	/*
+	 * Create and bind a ttm if required.
+	 */
+
+	if (!(new_man->flags & _DRM_FLAG_MEMTYPE_FIXED) && (bo->ttm == NULL)) {
+		ret = drm_bo_add_ttm(bo);
+		if (ret)
+			goto out_err;
+
+		if (mem->mem_type != DRM_BO_MEM_LOCAL) {
+			ret = drm_bind_ttm(bo->ttm, new_man->flags &
+					   DRM_BO_FLAG_CACHED,
+					   mem->mm_node->start);
+			if (ret)
+				goto out_err;
 		}
-		mutex_unlock(&dev->struct_mutex);
 	}
 
-	bo->flags &= ~DRM_BO_FLAG_MEM_TT;
-	bo->flags |= DRM_BO_FLAG_MEM_LOCAL | DRM_BO_FLAG_CACHED;
+	if ((bo->mem.mem_type == DRM_BO_MEM_LOCAL) && bo->ttm == NULL) {
+
+		drm_bo_mem_reg_t *old_mem = &bo->mem;
+		uint32_t save_flags = old_mem->flags;
+		uint32_t save_mask = old_mem->mask;
+
+		*old_mem = *mem;
+		mem->mm_node = NULL;
+		old_mem->mask = save_mask;
+		DRM_FLAG_MASKED(save_flags, mem->flags, DRM_BO_MASK_MEMTYPE);
+
+	} else if (!(old_man->flags & _DRM_FLAG_MEMTYPE_FIXED) &&
+		   !(new_man->flags & _DRM_FLAG_MEMTYPE_FIXED)) {
+
+		ret = drm_bo_move_ttm(bo, evict, no_wait, mem);
+
+	} else if (dev->driver->bo_driver->move) {
+		ret = dev->driver->bo_driver->move(bo, evict, no_wait, mem);
+
+	} else {
+
+		ret = drm_bo_move_memcpy(bo, evict, no_wait, mem);
+
+	}
+
+	if (ret)
+		goto out_err;
+
+	if (old_is_pci || new_is_pci)
+		drm_bo_vm_post_move(bo);
+
+	if (bo->priv_flags & _DRM_BO_FLAG_EVICTED) {
+		ret =
+		    dev->driver->bo_driver->invalidate_caches(dev,
+							      bo->mem.flags);
+		if (ret)
+			DRM_ERROR("Can not flush read caches\n");
+	}
+
+	DRM_FLAG_MASKED(bo->priv_flags,
+			(evict) ? _DRM_BO_FLAG_EVICTED : 0,
+			_DRM_BO_FLAG_EVICTED);
+
+	if (bo->mem.mm_node)
+		bo->offset = bo->mem.mm_node->start << PAGE_SHIFT;
 
 	return 0;
-}
 
+      out_err:
+	if (old_is_pci || new_is_pci)
+		drm_bo_vm_post_move(bo);
+
+	new_man = &bm->man[bo->mem.mem_type];
+	if ((new_man->flags & _DRM_FLAG_MEMTYPE_FIXED) && bo->ttm) {
+		drm_ttm_unbind(bo->ttm);
+		drm_destroy_ttm(bo->ttm);
+		bo->ttm = NULL;
+	}
+
+	return ret;
+}
 
 /*
  * Call bo->mutex locked.
  * Wait until the buffer is idle.
  */
 
-static int drm_bo_wait(drm_buffer_object_t * bo, int lazy, int ignore_signals,
-		       int no_wait)
+int drm_bo_wait(drm_buffer_object_t * bo, int lazy, int ignore_signals,
+		int no_wait)
 {
 
 	drm_fence_object_t *fence = bo->fence;
@@ -168,14 +274,44 @@ static int drm_bo_wait(drm_buffer_object_t * bo, int lazy, int ignore_signals,
 	return 0;
 }
 
+static int drm_bo_expire_fence(drm_buffer_object_t * bo, int allow_errors)
+{
+	drm_device_t *dev = bo->dev;
+	drm_buffer_manager_t *bm = &dev->bm;
+
+	if (bo->fence) {
+		if (bm->nice_mode) {
+			unsigned long _end = jiffies + 3 * DRM_HZ;
+			int ret;
+			do {
+				ret = drm_bo_wait(bo, 0, 1, 0);
+				if (ret && allow_errors)
+					return ret;
+
+			} while (ret && !time_after_eq(jiffies, _end));
+
+			if (bo->fence) {
+				bm->nice_mode = 0;
+				DRM_ERROR("Detected GPU lockup or "
+					  "fence driver was taken down. "
+					  "Evicting buffer.\n");
+			}
+		}
+		if (bo->fence) {
+			drm_fence_usage_deref_unlocked(dev, bo->fence);
+			bo->fence = NULL;
+		}
+	}
+	return 0;
+}
+
 /*
  * Call dev->struct_mutex locked.
  * Attempts to remove all private references to a buffer by expiring its
  * fence object and removing from lru lists and memory managers.
  */
 
-
-static void drm_bo_cleanup_refs(drm_buffer_object_t *bo, int remove_all)
+static void drm_bo_cleanup_refs(drm_buffer_object_t * bo, int remove_all)
 {
 	drm_device_t *dev = bo->dev;
 	drm_buffer_manager_t *bm = &dev->bm;
@@ -186,32 +322,14 @@ static void drm_bo_cleanup_refs(drm_buffer_object_t *bo, int remove_all)
 
 	DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED);
 
-	if (bo->fence && drm_fence_object_signaled(bo->fence,
-						   bo->fence_type)) {
+	if (bo->fence && drm_fence_object_signaled(bo->fence, bo->fence_type)) {
 		drm_fence_usage_deref_locked(dev, bo->fence);
 		bo->fence = NULL;
 	}
 
-	if (bo->fence && remove_all) {
-		if (bm->nice_mode) {
-			unsigned long _end = jiffies + 3 * DRM_HZ;
-			int ret;
-			do {
-				ret = drm_bo_wait(bo, 0, 1, 0);
-			} while (ret && !time_after_eq(jiffies, _end));
+	if (bo->fence && remove_all)
+		(void)drm_bo_expire_fence(bo, 0);
 
-			if (bo->fence) {
-				bm->nice_mode = 0;
-				DRM_ERROR("Detected GPU lockup or "
-					  "fence driver was taken down. "
-					  "Evicting waiting buffers.\n");
-			}
-			if (bo->fence) {
-				drm_fence_usage_deref_unlocked(dev, bo->fence);
-				bo->fence = NULL;
-			}
-		}
-	}
 	mutex_lock(&dev->struct_mutex);
 
 	if (!atomic_dec_and_test(&bo->usage)) {
@@ -220,9 +338,16 @@ static void drm_bo_cleanup_refs(drm_buffer_object_t *bo, int remove_all)
 
 	if (!bo->fence) {
 		list_del_init(&bo->lru);
-		if (bo->mm_node) {
-			drm_mm_put_block(bo->mm_node);
-			bo->mm_node = NULL;
+		if (bo->mem.mm_node) {
+			drm_mm_put_block(bo->mem.mm_node);
+			if (bo->pinned_node == bo->mem.mm_node)
+				bo->pinned_node = NULL;
+			bo->mem.mm_node = NULL;
+		}
+		list_del_init(&bo->pinned_lru);
+		if (bo->pinned_node) {
+			drm_mm_put_block(bo->pinned_node);
+			bo->pinned_node = NULL;
 		}
 		list_del_init(&bo->ddestroy);
 		mutex_unlock(&bo->mutex);
@@ -234,50 +359,42 @@ static void drm_bo_cleanup_refs(drm_buffer_object_t *bo, int remove_all)
 		drm_fence_object_flush(dev, bo->fence, bo->fence_type);
 		list_add_tail(&bo->ddestroy, &bm->ddestroy);
 		schedule_delayed_work(&bm->wq,
-				      ((DRM_HZ / 100) <
-				       1) ? 1 : DRM_HZ / 100);
+				      ((DRM_HZ / 100) < 1) ? 1 : DRM_HZ / 100);
 	}
 
-out:
+      out:
 	mutex_unlock(&bo->mutex);
 	return;
 }
 
-
 /*
  * Verify that refcount is 0 and that there are no internal references
  * to the buffer object. Then destroy it.
  */
 
-static void drm_bo_destroy_locked(drm_buffer_object_t *bo)
+static void drm_bo_destroy_locked(drm_buffer_object_t * bo)
 {
 	drm_device_t *dev = bo->dev;
 	drm_buffer_manager_t *bm = &dev->bm;
 
-	if (list_empty(&bo->lru) && bo->mm_node == NULL && atomic_read(&bo->usage) == 0) {
-		BUG_ON(bo->fence != NULL);
-
-		if (bo->ttm) {
-			unsigned long _end = jiffies + DRM_HZ;
-			int ret;
-
-			do {
-				ret = drm_unbind_ttm(bo->ttm);
-				if (ret == -EAGAIN) {
-					mutex_unlock(&dev->struct_mutex);
-					schedule();
-					mutex_lock(&dev->struct_mutex);
-				}
-			} while (ret == -EAGAIN && !time_after_eq(jiffies, _end));
-
-			if (ret) {
-				DRM_ERROR("Couldn't unbind TTM region while destroying a buffer. "
-					  "Bad. Continuing anyway\n");
-			}
+	if (list_empty(&bo->lru) && bo->mem.mm_node == NULL &&
+	    list_empty(&bo->pinned_lru) && bo->pinned_node == NULL &&
+	    list_empty(&bo->ddestroy) && atomic_read(&bo->usage) == 0) {
+		if (bo->fence != NULL) {
+			DRM_ERROR("Fence was non-zero.\n");
+			drm_bo_cleanup_refs(bo, 0);
+			return;
 		}
 
-		if (bo->ttm_object) {
-			drm_ttm_object_deref_locked(dev, bo->ttm_object);
+#ifdef DRM_ODD_MM_COMPAT
+		BUG_ON(!list_empty(&bo->vma_list));
+		BUG_ON(!list_empty(&bo->p_mm_list));
+#endif
+
+		if (bo->ttm) {
+			drm_ttm_unbind(bo->ttm);
+			drm_destroy_ttm(bo->ttm);
+			bo->ttm = NULL;
 		}
 
 		atomic_dec(&bm->count);
@@ -297,7 +414,6 @@ static void drm_bo_destroy_locked(drm_buffer_object_t *bo)
 	return;
 }
 
-
 /*
  * Call dev->struct_mutex locked.
  */
@@ -325,7 +441,6 @@ static void drm_bo_delayed_delete(drm_device_t * dev, int remove_all)
 			atomic_dec(&nentry->usage);
 		}
 	}
-
 }
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
@@ -338,11 +453,11 @@ static void drm_bo_delayed_workqueue(struct work_struct *work)
 	drm_device_t *dev = (drm_device_t *) data;
 	drm_buffer_manager_t *bm = &dev->bm;
 #else
-	drm_buffer_manager_t *bm = container_of(work, drm_buffer_manager_t, wq.work);
+	drm_buffer_manager_t *bm =
+	    container_of(work, drm_buffer_manager_t, wq.work);
 	drm_device_t *dev = container_of(bm, drm_device_t, bm);
 #endif
 
-
 	DRM_DEBUG("Delayed delete Worker\n");
 
 	mutex_lock(&dev->struct_mutex);
@@ -358,7 +473,7 @@ static void drm_bo_delayed_workqueue(struct work_struct *work)
 	mutex_unlock(&dev->struct_mutex);
 }
 
-static void drm_bo_usage_deref_locked(drm_buffer_object_t * bo)
+void drm_bo_usage_deref_locked(drm_buffer_object_t * bo)
 {
 	if (atomic_dec_and_test(&bo->usage)) {
 		drm_bo_destroy_locked(bo);
@@ -367,8 +482,11 @@ static void drm_bo_usage_deref_locked(drm_buffer_object_t * bo)
 
 static void drm_bo_base_deref_locked(drm_file_t * priv, drm_user_object_t * uo)
 {
-	drm_bo_usage_deref_locked(drm_user_object_entry(uo, drm_buffer_object_t,
-							base));
+	drm_buffer_object_t *bo =
+	    drm_user_object_entry(uo, drm_buffer_object_t, base);
+
+	drm_bo_takedown_vm_locked(bo);
+	drm_bo_usage_deref_locked(bo);
 }
 
 static void drm_bo_usage_deref_unlocked(drm_buffer_object_t * bo)
@@ -454,6 +572,7 @@ int drm_fence_buffer_objects(drm_file_t * priv,
 	count = 0;
 	l = f_list.next;
 	while (l != &f_list) {
+		prefetch(l->next);
 		entry = list_entry(l, drm_buffer_object_t, lru);
 		atomic_inc(&entry->usage);
 		mutex_unlock(&dev->struct_mutex);
@@ -468,7 +587,7 @@ int drm_fence_buffer_objects(drm_file_t * priv,
 			DRM_FLAG_MASKED(entry->priv_flags, 0,
 					_DRM_BO_FLAG_UNFENCED);
 			DRM_WAKEUP(&entry->event_queue);
-			drm_bo_add_to_lru(entry, bm);
+			drm_bo_add_to_lru(entry);
 		}
 		mutex_unlock(&entry->mutex);
 		drm_bo_usage_deref_locked(entry);
@@ -489,11 +608,11 @@ EXPORT_SYMBOL(drm_fence_buffer_objects);
  */
 
 static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type,
-			int no_wait, int force_no_move)
+			int no_wait)
 {
 	int ret = 0;
 	drm_device_t *dev = bo->dev;
-	drm_buffer_manager_t *bm = &dev->bm;
+	drm_bo_mem_reg_t evict_mem;
 
 	/*
 	 * Someone might have modified the buffer before we took the buffer mutex.
@@ -501,70 +620,93 @@ static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type,
 
 	if (bo->priv_flags & _DRM_BO_FLAG_UNFENCED)
 		goto out;
-	if (!(bo->flags & drm_bo_type_flags(mem_type)))
+	if (bo->mem.mem_type != mem_type)
 		goto out;
 
 	ret = drm_bo_wait(bo, 0, 0, no_wait);
 
+	if (ret && ret != -EAGAIN) {
+		DRM_ERROR("Failed to expire fence before "
+			  "buffer eviction.\n");
+		goto out;
+	}
+
+	evict_mem = bo->mem;
+	evict_mem.mm_node = NULL;
+
+	if (bo->type == drm_bo_type_fake) {
+		bo->mem.mem_type = DRM_BO_MEM_LOCAL;
+		bo->mem.mm_node = NULL;
+		goto out1;
+	}
+
+	evict_mem = bo->mem;
+	evict_mem.mask = dev->driver->bo_driver->evict_flags(dev, mem_type);
+	ret = drm_bo_mem_space(bo, &evict_mem, no_wait);
+
 	if (ret) {
 		if (ret != -EAGAIN)
-			DRM_ERROR("Failed to expire fence before "
-				  "buffer eviction.\n");
+			DRM_ERROR("Failed to find memory space for "
+				  "buffer 0x%p eviction.\n", bo);
 		goto out;
 	}
 
-	if (mem_type == DRM_BO_MEM_TT) {
-		ret = drm_move_tt_to_local(bo, 1, force_no_move);
-		if (ret)
-			goto out;
-		mutex_lock(&dev->struct_mutex);
-		list_del_init(&bo->lru);
-		drm_bo_add_to_lru(bo, bm);
-		mutex_unlock(&dev->struct_mutex);
-	}
+	ret = drm_bo_handle_move_mem(bo, &evict_mem, 1, no_wait);
 
-	if (ret)
+	if (ret) {
+		if (ret != -EAGAIN)
+			DRM_ERROR("Buffer eviction failed\n");
 		goto out;
+	}
+
+      out1:
+	mutex_lock(&dev->struct_mutex);
+	if (evict_mem.mm_node) {
+		if (evict_mem.mm_node != bo->pinned_node)
+			drm_mm_put_block(evict_mem.mm_node);
+		evict_mem.mm_node = NULL;
+	}
+	list_del(&bo->lru);
+	drm_bo_add_to_lru(bo);
+	mutex_unlock(&dev->struct_mutex);
 
 	DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_EVICTED,
 			_DRM_BO_FLAG_EVICTED);
+
       out:
 	return ret;
 }
 
-/*
- * bo->mutex locked.
- */
-
-int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type,
-		       int no_wait)
+static int drm_bo_mem_force_space(drm_device_t * dev,
+				  drm_bo_mem_reg_t * mem,
+				  uint32_t mem_type, int no_wait)
 {
-	drm_device_t *dev = bo->dev;
 	drm_mm_node_t *node;
 	drm_buffer_manager_t *bm = &dev->bm;
 	drm_buffer_object_t *entry;
-	drm_mm_t *mm = &bm->manager[mem_type];
+	drm_mem_type_manager_t *man = &bm->man[mem_type];
 	struct list_head *lru;
-	unsigned long size = bo->num_pages;
+	unsigned long num_pages = mem->num_pages;
 	int ret;
 
 	mutex_lock(&dev->struct_mutex);
 	do {
-		node = drm_mm_search_free(mm, size, bo->page_alignment, 1);
+		node = drm_mm_search_free(&man->manager, num_pages,
+					  mem->page_alignment, 1);
 		if (node)
 			break;
 
-		lru = &bm->lru[mem_type];
+		lru = &man->lru;
 		if (lru->next == lru)
 			break;
 
 		entry = list_entry(lru->next, drm_buffer_object_t, lru);
-
 		atomic_inc(&entry->usage);
 		mutex_unlock(&dev->struct_mutex);
 		mutex_lock(&entry->mutex);
-		BUG_ON(bo->flags & DRM_BO_FLAG_NO_MOVE);
-		ret = drm_bo_evict(entry, mem_type, no_wait, 0);
+		BUG_ON(entry->mem.flags & (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT));
+
+		ret = drm_bo_evict(entry, mem_type, no_wait);
 		mutex_unlock(&entry->mutex);
 		drm_bo_usage_deref_unlocked(entry);
 		if (ret)
@@ -573,146 +715,163 @@ int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type,
 	} while (1);
 
 	if (!node) {
-		DRM_ERROR("Out of videoram / aperture space\n");
 		mutex_unlock(&dev->struct_mutex);
 		return -ENOMEM;
 	}
 
-	node = drm_mm_get_block(node, size, bo->page_alignment);
+	node = drm_mm_get_block(node, num_pages, mem->page_alignment);
 	mutex_unlock(&dev->struct_mutex);
-	BUG_ON(!node);
-	node->private = (void *)bo;
-
-	bo->mm_node = node;
-	bo->offset = node->start * PAGE_SIZE;
+	mem->mm_node = node;
+	mem->mem_type = mem_type;
 	return 0;
 }
 
-static int drm_move_local_to_tt(drm_buffer_object_t * bo, int no_wait)
+static int drm_bo_mt_compatible(drm_mem_type_manager_t * man,
+				uint32_t mem_type,
+				uint32_t mask, uint32_t * res_mask)
 {
-	drm_device_t *dev = bo->dev;
-	drm_ttm_backend_t *be;
-	int ret;
+	uint32_t cur_flags = drm_bo_type_flags(mem_type);
+	uint32_t flag_diff;
 
-	if (!(bo->mm_node && (bo->flags & DRM_BO_FLAG_NO_MOVE))) {
-		BUG_ON(bo->mm_node);
-		ret = drm_bo_alloc_space(bo, DRM_BO_MEM_TT, no_wait);
-		if (ret)
-			return ret;
-	}
+	if (man->flags & _DRM_FLAG_MEMTYPE_CACHED)
+		cur_flags |= DRM_BO_FLAG_CACHED;
+	if (man->flags & _DRM_FLAG_MEMTYPE_MAPPABLE)
+		cur_flags |= DRM_BO_FLAG_MAPPABLE;
+	if (man->flags & _DRM_FLAG_MEMTYPE_CSELECT)
+		DRM_FLAG_MASKED(cur_flags, mask, DRM_BO_FLAG_CACHED);
 
-	DRM_DEBUG("Flipping in to AGP 0x%08lx\n", bo->mm_node->start);
-
-	mutex_lock(&dev->struct_mutex);
-	ret = drm_bind_ttm(bo->ttm, bo->flags & DRM_BO_FLAG_BIND_CACHED,
-			   bo->mm_node->start);
-	if (ret) {
-		drm_mm_put_block(bo->mm_node);
-		bo->mm_node = NULL;
-	}
-	mutex_unlock(&dev->struct_mutex);
+	if ((cur_flags & mask & DRM_BO_MASK_MEM) == 0)
+		return 0;
 
-	if (ret) {
-		return ret;
+	if (mem_type == DRM_BO_MEM_LOCAL) {
+		*res_mask = cur_flags;
+		return 1;
 	}
 
-	be = bo->ttm->be;
-	if (be->needs_ub_cache_adjust(be))
-		bo->flags &= ~DRM_BO_FLAG_CACHED;
-	bo->flags &= ~DRM_BO_MASK_MEM;
-	bo->flags |= DRM_BO_FLAG_MEM_TT;
+	flag_diff = (mask ^ cur_flags);
+	if ((flag_diff & DRM_BO_FLAG_CACHED) &&
+	    (!(mask & DRM_BO_FLAG_CACHED) ||
+	     (mask & DRM_BO_FLAG_FORCE_CACHING)))
+		return 0;
 
-	if (bo->priv_flags & _DRM_BO_FLAG_EVICTED) {
-		ret = dev->driver->bo_driver->invalidate_caches(dev, bo->flags);
-		if (ret)
-			DRM_ERROR("Could not flush read caches\n");
-	}
-	DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_EVICTED);
+	if ((flag_diff & DRM_BO_FLAG_MAPPABLE) &&
+	    ((mask & DRM_BO_FLAG_MAPPABLE) ||
+	     (mask & DRM_BO_FLAG_FORCE_MAPPABLE)) )
+		return 0;
 
-	return 0;
+	*res_mask = cur_flags;
+	return 1;
 }
 
-static int drm_bo_new_flags(drm_device_t * dev,
-			    uint32_t flags, uint32_t new_mask, uint32_t hint,
-			    int init, uint32_t * n_flags, uint32_t * n_mask)
+int drm_bo_mem_space(drm_buffer_object_t * bo,
+		     drm_bo_mem_reg_t * mem, int no_wait)
 {
-	uint32_t new_flags = 0;
-	uint32_t new_props;
-	drm_bo_driver_t *driver = dev->driver->bo_driver;
+	drm_device_t *dev = bo->dev;
 	drm_buffer_manager_t *bm = &dev->bm;
-	unsigned i;
+	drm_mem_type_manager_t *man;
+
+	uint32_t num_prios = dev->driver->bo_driver->num_mem_type_prio;
+	const uint32_t *prios = dev->driver->bo_driver->mem_type_prio;
+	uint32_t i;
+	uint32_t mem_type = DRM_BO_MEM_LOCAL;
+	uint32_t cur_flags;
+	int type_found = 0;
+	int type_ok = 0;
+	int has_eagain = 0;
+	drm_mm_node_t *node = NULL;
+	int ret;
 
-	/*
-	 * First adjust the mask to take away nonexistant memory types. 
-	 */
+	mem->mm_node = NULL;
+	for (i = 0; i < num_prios; ++i) {
+		mem_type = prios[i];
+		man = &bm->man[mem_type];
 
-	for (i = 0; i < DRM_BO_MEM_TYPES; ++i) {
-		if (!bm->use_type[i])
-			new_mask &= ~drm_bo_type_flags(i);
-	}
+		type_ok = drm_bo_mt_compatible(man, mem_type, mem->mask,
+					       &cur_flags);
 
-	if ((new_mask & DRM_BO_FLAG_NO_EVICT) && !DRM_SUSER(DRM_CURPROC)) {
-		DRM_ERROR
-		    ("DRM_BO_FLAG_NO_EVICT is only available to priviliged "
-		     "processes\n");
-		return -EPERM;
-	}
-	if (new_mask & DRM_BO_FLAG_BIND_CACHED) {
-		if (((new_mask & DRM_BO_FLAG_MEM_TT) &&
-		     !driver->cached[DRM_BO_MEM_TT]) &&
-		    ((new_mask & DRM_BO_FLAG_MEM_VRAM)
-		     && !driver->cached[DRM_BO_MEM_VRAM])) {
-			new_mask &= ~DRM_BO_FLAG_BIND_CACHED;
-		} else {
-			if (!driver->cached[DRM_BO_MEM_TT])
-				new_flags &= DRM_BO_FLAG_MEM_TT;
-			if (!driver->cached[DRM_BO_MEM_VRAM])
-				new_flags &= DRM_BO_FLAG_MEM_VRAM;
+		if (!type_ok)
+			continue;
+
+		if (mem_type == DRM_BO_MEM_LOCAL)
+			break;
+
+		if ((mem_type == bo->pinned_mem_type) &&
+		    (bo->pinned_node != NULL)) {
+			node = bo->pinned_node;
+			break;
 		}
-	}
 
-	if ((new_mask & DRM_BO_FLAG_READ_CACHED) &&
-	    !(new_mask & DRM_BO_FLAG_BIND_CACHED)) {
-		if ((new_mask & DRM_BO_FLAG_NO_EVICT) &&
-		    !(new_mask & DRM_BO_FLAG_MEM_LOCAL)) {
-			DRM_ERROR
-			    ("Cannot read cached from a pinned VRAM / TT buffer\n");
-			return -EINVAL;
+		mutex_lock(&dev->struct_mutex);
+		if (man->has_type && man->use_type) {
+			type_found = 1;
+			node = drm_mm_search_free(&man->manager, mem->num_pages,
+						  mem->page_alignment, 1);
+			if (node)
+				node = drm_mm_get_block(node, mem->num_pages,
+							mem->page_alignment);
 		}
+		mutex_unlock(&dev->struct_mutex);
+		if (node)
+			break;
 	}
 
-	/*
-	 * Determine new memory location:
-	 */
+	if ((type_ok && (mem_type == DRM_BO_MEM_LOCAL)) || node) {
+		mem->mm_node = node;
+		mem->mem_type = mem_type;
+		mem->flags = cur_flags;
+		return 0;
+	}
 
-	if (!(flags & new_mask & DRM_BO_MASK_MEM) || init) {
+	if (!type_found)
+		return -EINVAL;
 
-		new_flags = new_mask & DRM_BO_MASK_MEM;
+	num_prios = dev->driver->bo_driver->num_mem_busy_prio;
+	prios = dev->driver->bo_driver->mem_busy_prio;
 
-		if (!new_flags) {
-			DRM_ERROR("Invalid buffer object memory flags\n");
-			return -EINVAL;
-		}
+	for (i = 0; i < num_prios; ++i) {
+		mem_type = prios[i];
+		man = &bm->man[mem_type];
 
-		if (new_flags & DRM_BO_FLAG_MEM_LOCAL) {
-			if ((hint & DRM_BO_HINT_AVOID_LOCAL) &&
-			    new_flags & (DRM_BO_FLAG_MEM_VRAM |
-					 DRM_BO_FLAG_MEM_TT)) {
-				new_flags &= ~DRM_BO_FLAG_MEM_LOCAL;
-			} else {
-				new_flags = DRM_BO_FLAG_MEM_LOCAL;
-			}
-		}
-		if (new_flags & DRM_BO_FLAG_MEM_TT) {
-			if ((new_mask & DRM_BO_FLAG_PREFER_VRAM) &&
-			    new_flags & DRM_BO_FLAG_MEM_VRAM) {
-				new_flags = DRM_BO_FLAG_MEM_VRAM;
-			} else {
-				new_flags = DRM_BO_FLAG_MEM_TT;
-			}
+		if (!drm_bo_mt_compatible(man, mem_type, mem->mask, &cur_flags))
+			continue;
+
+		ret = drm_bo_mem_force_space(dev, mem, mem_type, no_wait);
+
+		if (ret == 0) {
+			mem->flags = cur_flags;
+			return 0;
 		}
-	} else {
-		new_flags = flags & DRM_BO_MASK_MEM;
+
+		if (ret == -EAGAIN)
+			has_eagain = 1;
+	}
+
+	ret = (has_eagain) ? -EAGAIN : -ENOMEM;
+	return ret;
+}
+
+EXPORT_SYMBOL(drm_bo_mem_space);
+
+static int drm_bo_new_mask(drm_buffer_object_t * bo,
+			   uint32_t new_mask, uint32_t hint)
+{
+	uint32_t new_props;
+
+	if (bo->type == drm_bo_type_user) {
+		DRM_ERROR("User buffers are not supported yet\n");
+		return -EINVAL;
+	}
+	if (bo->type == drm_bo_type_fake &&
+	    !(new_mask & (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT))) {
+		DRM_ERROR("Fake buffers must be pinned.\n");
+		return -EINVAL;
+	}
+
+	if ((new_mask & DRM_BO_FLAG_NO_EVICT) && !DRM_SUSER(DRM_CURPROC)) {
+		DRM_ERROR
+		    ("DRM_BO_FLAG_NO_EVICT is only available to priviliged "
+		     "processes\n");
+		return -EPERM;
 	}
 
 	new_props = new_mask & (DRM_BO_FLAG_EXE | DRM_BO_FLAG_WRITE |
@@ -723,22 +882,7 @@ static int drm_bo_new_flags(drm_device_t * dev,
 		return -EINVAL;
 	}
 
-	new_flags |= new_mask & ~DRM_BO_MASK_MEM;
-
-	if (((flags ^ new_flags) & DRM_BO_FLAG_BIND_CACHED) &&
-	    (new_flags & DRM_BO_FLAG_NO_EVICT) &&
-	    (flags & (DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_MEM_VRAM))) {
-		if (!(flags & DRM_BO_FLAG_CACHED)) {
-			DRM_ERROR
-			    ("Cannot change caching policy of pinned buffer\n");
-			return -EINVAL;
-		} else {
-			new_flags &= ~DRM_BO_FLAG_CACHED;
-		}
-	}
-
-	*n_flags = new_flags;
-	*n_mask = new_mask;
+	bo->mem.mask = new_mask;
 	return 0;
 }
 
@@ -825,8 +969,8 @@ static int drm_bo_read_cached(drm_buffer_object_t * bo)
 	int ret = 0;
 
 	BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED);
-	if (bo->mm_node)
-		ret = drm_bo_evict(bo, DRM_BO_MEM_TT, 1, 0);
+	if (bo->mem.mm_node)
+		ret = drm_bo_evict(bo, DRM_BO_MEM_TT, 1);
 	return ret;
 }
 
@@ -916,21 +1060,15 @@ static void drm_bo_fill_rep_arg(drm_buffer_object_t * bo,
 				drm_bo_arg_reply_t * rep)
 {
 	rep->handle = bo->base.hash.key;
-	rep->flags = bo->flags;
-	rep->size = bo->num_pages * PAGE_SIZE;
+	rep->flags = bo->mem.flags;
+	rep->size = bo->mem.num_pages * PAGE_SIZE;
 	rep->offset = bo->offset;
-
-	if (bo->ttm_object) {
-		rep->arg_handle = bo->ttm_object->map_list.user_token;
-	} else {
-		rep->arg_handle = 0;
-	}
-
-	rep->mask = bo->mask;
+	rep->arg_handle = bo->map_list.user_token;
+	rep->mask = bo->mem.mask;
 	rep->buffer_start = bo->buffer_start;
 	rep->fence_flags = bo->fence_type;
 	rep->rep_flags = 0;
-	rep->page_alignment = bo->page_alignment;
+	rep->page_alignment = bo->mem.page_alignment;
 
 	if ((bo->priv_flags & _DRM_BO_FLAG_UNFENCED) || drm_bo_quick_busy(bo)) {
 		DRM_FLAG_MASKED(rep->rep_flags, DRM_BO_REP_BUSY,
@@ -988,14 +1126,14 @@ static int drm_buffer_object_map(drm_file_t * priv, uint32_t handle,
 			}
 
 			if ((map_flags & DRM_BO_FLAG_READ) &&
-			    (bo->flags & DRM_BO_FLAG_READ_CACHED) &&
-			    (!(bo->flags & DRM_BO_FLAG_CACHED))) {
+			    (bo->mem.flags & DRM_BO_FLAG_READ_CACHED) &&
+			    (!(bo->mem.flags & DRM_BO_FLAG_CACHED))) {
 				drm_bo_read_cached(bo);
 			}
 			break;
 		} else if ((map_flags & DRM_BO_FLAG_READ) &&
-			   (bo->flags & DRM_BO_FLAG_READ_CACHED) &&
-			   (!(bo->flags & DRM_BO_FLAG_CACHED))) {
+			   (bo->mem.flags & DRM_BO_FLAG_READ_CACHED) &&
+			   (!(bo->mem.flags & DRM_BO_FLAG_CACHED))) {
 
 			/*
 			 * We are already mapped with different flags.
@@ -1078,46 +1216,124 @@ static void drm_buffer_user_object_unmap(drm_file_t * priv,
 
 /*
  * bo->mutex locked. 
+ * Note that new_mem_flags are NOT transferred to the bo->mem.mask.
  */
 
-static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_flags,
-			      int no_wait, int force_no_move)
+int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_mem_flags,
+		       int no_wait, int move_unfenced)
 {
+	drm_device_t *dev = bo->dev;
+	drm_buffer_manager_t *bm = &dev->bm;
 	int ret = 0;
-
+	drm_bo_mem_reg_t mem;
 	/*
 	 * Flush outstanding fences.
 	 */
+
 	drm_bo_busy(bo);
 
 	/*
-	 * Make sure we're not mapped.
+	 * Wait for outstanding fences.
 	 */
 
-	ret = drm_bo_wait_unmapped(bo, no_wait);
+	ret = drm_bo_wait(bo, 0, 0, no_wait);
 	if (ret)
 		return ret;
 
+	mem.num_pages = bo->mem.num_pages;
+	mem.size = mem.num_pages << PAGE_SHIFT;
+	mem.mask = new_mem_flags;
+	mem.page_alignment = bo->mem.page_alignment;
+
+	mutex_lock(&bm->evict_mutex);
+	mutex_lock(&dev->struct_mutex);
+	list_del(&bo->lru);
+	list_add_tail(&bo->lru, &bm->unfenced);
+	DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_UNFENCED,
+			_DRM_BO_FLAG_UNFENCED);
+	mutex_unlock(&dev->struct_mutex);
+
 	/*
-	 * Wait for outstanding fences.
+	 * Determine where to move the buffer.
 	 */
+	ret = drm_bo_mem_space(bo, &mem, no_wait);
+	if (ret)
+		goto out_unlock;
 
-	ret = drm_bo_wait(bo, 0, 0, no_wait);
+	ret = drm_bo_handle_move_mem(bo, &mem, 0, no_wait);
 
-	if (ret == -EINTR)
-		return -EAGAIN;
-	if (ret)
-		return ret;
+ out_unlock:
+	if (ret || !move_unfenced) {
+		mutex_lock(&dev->struct_mutex);
+		if (mem.mm_node) {
+			if (mem.mm_node != bo->pinned_node)
+				drm_mm_put_block(mem.mm_node);
+			mem.mm_node = NULL;
+		}
+		DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED);
+		DRM_WAKEUP(&bo->event_queue);
+		list_del(&bo->lru);
+		drm_bo_add_to_lru(bo);
+		mutex_unlock(&dev->struct_mutex);
+	}
 
-	if (new_flags & DRM_BO_FLAG_MEM_TT) {
-		ret = drm_move_local_to_tt(bo, no_wait);
-		if (ret)
-			return ret;
-	} else {
-		drm_move_tt_to_local(bo, 0, force_no_move);
+	mutex_unlock(&bm->evict_mutex);
+	return ret;
+}
+
+static int drm_bo_mem_compat(drm_bo_mem_reg_t * mem)
+{
+	uint32_t flag_diff = (mem->mask ^ mem->flags);
+
+	if ((mem->mask & mem->flags & DRM_BO_MASK_MEM) == 0)
+		return 0;
+	if ((flag_diff & DRM_BO_FLAG_CACHED) &&
+	    (!(mem->mask & DRM_BO_FLAG_CACHED) ||
+	     (mem->mask & DRM_BO_FLAG_FORCE_CACHING))) {
+	  return 0;
+	}
+	if ((flag_diff & DRM_BO_FLAG_MAPPABLE) &&
+	    ((mem->mask & DRM_BO_FLAG_MAPPABLE) ||
+	     (mem->mask & DRM_BO_FLAG_FORCE_MAPPABLE)))
+		return 0;
+	return 1;
+}
+
+static int drm_bo_check_fake(drm_device_t * dev, drm_bo_mem_reg_t * mem)
+{
+	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man;
+	uint32_t num_prios = dev->driver->bo_driver->num_mem_type_prio;
+	const uint32_t *prios = dev->driver->bo_driver->mem_type_prio;
+	uint32_t i;
+	int type_ok = 0;
+	uint32_t mem_type = 0;
+	uint32_t cur_flags;
+
+	if (drm_bo_mem_compat(mem))
+		return 0;
+
+	BUG_ON(mem->mm_node);
+
+	for (i = 0; i < num_prios; ++i) {
+		mem_type = prios[i];
+		man = &bm->man[mem_type];
+		type_ok = drm_bo_mt_compatible(man, mem_type, mem->mask,
+					       &cur_flags);
+		if (type_ok)
+			break;
 	}
 
-	return 0;
+	if (type_ok) {
+		mem->mm_node = NULL;
+		mem->mem_type = mem_type;
+		mem->flags = cur_flags;
+		DRM_FLAG_MASKED(mem->flags, mem->mask, ~DRM_BO_MASK_MEMTYPE);
+		return 0;
+	}
+
+	DRM_ERROR("Illegal fake buffer flags 0x%08x\n", mem->mask);
+	return -EINVAL;
 }
 
 /*
@@ -1125,98 +1341,105 @@ static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_flags,
  */
 
 static int drm_buffer_object_validate(drm_buffer_object_t * bo,
-				      uint32_t new_flags,
 				      int move_unfenced, int no_wait)
 {
 	drm_device_t *dev = bo->dev;
 	drm_buffer_manager_t *bm = &dev->bm;
-	uint32_t flag_diff = (new_flags ^ bo->flags);
 	drm_bo_driver_t *driver = dev->driver->bo_driver;
-
 	int ret;
 
-	if (new_flags & DRM_BO_FLAG_MEM_VRAM) {
-		DRM_ERROR("Vram support not implemented yet\n");
-		return -EINVAL;
-	}
-
-	DRM_DEBUG("New flags 0x%08x, Old flags 0x%08x\n", new_flags, bo->flags);
-	ret = driver->fence_type(new_flags, &bo->fence_class, &bo->fence_type);
+	DRM_DEBUG("New flags 0x%08x, Old flags 0x%08x\n", bo->mem.mask,
+		  bo->mem.flags);
+	ret =
+	    driver->fence_type(bo->mem.mask, &bo->fence_class, &bo->fence_type);
 	if (ret) {
 		DRM_ERROR("Driver did not support given buffer permissions\n");
 		return ret;
 	}
 
+	ret = drm_bo_wait_unmapped(bo, no_wait);
+	if (ret)
+		return ret;
+
+	if (bo->type == drm_bo_type_fake) {
+		ret = drm_bo_check_fake(dev, &bo->mem);
+		if (ret)
+			return ret;
+	}
+
 	/*
-	 * Move out if we need to change caching policy.
+	 * Check whether we need to move buffer.
 	 */
 
-	if ((flag_diff & DRM_BO_FLAG_BIND_CACHED) &&
-	    !(bo->flags & DRM_BO_FLAG_MEM_LOCAL)) {
-		if (bo->flags & (DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE)) {
-			DRM_ERROR("Cannot change caching policy of "
-				  "pinned buffer.\n");
-			return -EINVAL;
-		}
-		ret = drm_bo_move_buffer(bo, DRM_BO_FLAG_MEM_LOCAL, no_wait, 0);
+	if (!drm_bo_mem_compat(&bo->mem)) {
+		ret = drm_bo_move_buffer(bo, bo->mem.mask, no_wait,
+					 move_unfenced);
 		if (ret) {
 			if (ret != -EAGAIN)
 				DRM_ERROR("Failed moving buffer.\n");
 			return ret;
 		}
 	}
-	DRM_MASK_VAL(bo->flags, DRM_BO_FLAG_BIND_CACHED, new_flags);
-	flag_diff = (new_flags ^ bo->flags);
 
 	/*
-	 * Check whether we dropped no_move policy, and in that case,
-	 * release reserved manager regions.
+	 * Pinned buffers.
 	 */
 
-	if ((flag_diff & DRM_BO_FLAG_NO_MOVE) &&
-	    !(new_flags & DRM_BO_FLAG_NO_MOVE)) {
+	if (bo->mem.mask & (DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE)) {
+		bo->pinned_mem_type = bo->mem.mem_type;
 		mutex_lock(&dev->struct_mutex);
-		if (bo->mm_node) {
-			drm_mm_put_block(bo->mm_node);
-			bo->mm_node = NULL;
+		list_del_init(&bo->pinned_lru);
+		drm_bo_add_to_pinned_lru(bo);
+
+		if (bo->pinned_node != bo->mem.mm_node) {
+			if (bo->pinned_node != NULL)
+				drm_mm_put_block(bo->pinned_node);
+			bo->pinned_node = bo->mem.mm_node;
 		}
+
 		mutex_unlock(&dev->struct_mutex);
+
+	} else if (bo->pinned_node != NULL) {
+
+		mutex_lock(&dev->struct_mutex);
+		drm_mm_put_block(bo->pinned_node);
+		list_del_init(&bo->pinned_lru);
+		bo->pinned_node = NULL;
+		mutex_unlock(&dev->struct_mutex);
+
 	}
 
 	/*
-	 * Check whether we need to move buffer.
+	 * We might need to add a TTM.
 	 */
 
-	if ((bo->type != drm_bo_type_fake) && (flag_diff & DRM_BO_MASK_MEM)) {
-		ret = drm_bo_move_buffer(bo, new_flags, no_wait, 1);
-		if (ret) {
-			if (ret != -EAGAIN)
-				DRM_ERROR("Failed moving buffer.\n");
+	if (bo->mem.mem_type == DRM_BO_MEM_LOCAL && bo->ttm == NULL) {
+		ret = drm_bo_add_ttm(bo);
+		if (ret)
 			return ret;
-		}
 	}
+	DRM_FLAG_MASKED(bo->mem.flags, bo->mem.mask, ~DRM_BO_MASK_MEMTYPE);
 
-	if (move_unfenced) {
-
-		/*
-		 * Place on unfenced list.
-		 */
+	/*
+	 * Finally, adjust lru to be sure.
+	 */
 
+	mutex_lock(&dev->struct_mutex);
+	list_del(&bo->lru);
+	if (move_unfenced) {
+		list_add_tail(&bo->lru, &bm->unfenced);
 		DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_UNFENCED,
 				_DRM_BO_FLAG_UNFENCED);
-		mutex_lock(&dev->struct_mutex);
-		list_del(&bo->lru);
-		list_add_tail(&bo->lru, &bm->unfenced);
-		mutex_unlock(&dev->struct_mutex);
 	} else {
-
-		mutex_lock(&dev->struct_mutex);
-		list_del_init(&bo->lru);
-		drm_bo_add_to_lru(bo, bm);
-		mutex_unlock(&dev->struct_mutex);
+		drm_bo_add_to_lru(bo);
+		if (bo->priv_flags & _DRM_BO_FLAG_UNFENCED) {
+			DRM_WAKEUP(&bo->event_queue);
+			DRM_FLAG_MASKED(bo->priv_flags, 0,
+					_DRM_BO_FLAG_UNFENCED);
+		}
 	}
+	mutex_unlock(&dev->struct_mutex);
 
-	bo->flags = new_flags;
 	return 0;
 }
 
@@ -1225,10 +1448,8 @@ static int drm_bo_handle_validate(drm_file_t * priv, uint32_t handle,
 				  drm_bo_arg_reply_t * rep)
 {
 	drm_buffer_object_t *bo;
-	drm_device_t *dev = priv->head->dev;
 	int ret;
 	int no_wait = hint & DRM_BO_HINT_DONT_BLOCK;
-	uint32_t new_flags;
 
 	bo = drm_lookup_buffer_object(priv, handle, 1);
 	if (!bo) {
@@ -1241,22 +1462,20 @@ static int drm_bo_handle_validate(drm_file_t * priv, uint32_t handle,
 	if (ret)
 		goto out;
 
-	ret = drm_bo_new_flags(dev, bo->flags,
-			       (flags & mask) | (bo->mask & ~mask), hint,
-			       0, &new_flags, &bo->mask);
-
+	DRM_FLAG_MASKED(flags, bo->mem.mask, ~mask);
+	ret = drm_bo_new_mask(bo, flags, hint);
 	if (ret)
 		goto out;
 
 	ret =
-	    drm_buffer_object_validate(bo, new_flags,
-				       !(hint & DRM_BO_HINT_DONT_FENCE),
+	    drm_buffer_object_validate(bo, !(hint & DRM_BO_HINT_DONT_FENCE),
 				       no_wait);
 	drm_bo_fill_rep_arg(bo, rep);
 
       out:
 
 	mutex_unlock(&bo->mutex);
+
 	drm_bo_usage_deref_unlocked(bo);
 	return ret;
 }
@@ -1307,90 +1526,6 @@ static int drm_bo_handle_wait(drm_file_t * priv, uint32_t handle,
 	return ret;
 }
 
-/*
- * Call bo->mutex locked.
- */
-
-static int drm_bo_add_ttm(drm_file_t * priv, drm_buffer_object_t * bo)
-{
-	drm_device_t *dev = bo->dev;
-	drm_ttm_object_t *to = NULL;
-	int ret = 0;
-	uint32_t ttm_flags = 0;
-
-	bo->ttm_object = NULL;
-	bo->ttm = NULL;
-
-	switch (bo->type) {
-	case drm_bo_type_dc:
-		mutex_lock(&dev->struct_mutex);
-		ret = drm_ttm_object_create(dev, bo->num_pages * PAGE_SIZE,
-					    ttm_flags, &to);
-		mutex_unlock(&dev->struct_mutex);
-		break;
-	case drm_bo_type_user:
-	case drm_bo_type_fake:
-		break;
-	default:
-		DRM_ERROR("Illegal buffer object type\n");
-		ret = -EINVAL;
-		break;
-	}
-
-	if (ret) {
-		return ret;
-	}
-
-	if (to) {
-		bo->ttm_object = to;
-		bo->ttm = drm_ttm_from_object(to);
-	}
-	return ret;
-}
-
-/*
- * Transfer a buffer object's memory and LRU status to a newly
- * created object. User-space references remains with the old
- * object. Call bo->mutex locked.
- */
-
-int drm_buffer_object_transfer(drm_buffer_object_t *bo,
-			       drm_buffer_object_t **new_obj)
-{
-	drm_buffer_object_t *fbo;
-	drm_device_t *dev = bo->dev;
-	drm_buffer_manager_t *bm = &dev->bm;
-
-	fbo = drm_ctl_calloc(1, sizeof(*bo), DRM_MEM_BUFOBJ);
-	if (!fbo)
-		return -ENOMEM;
-	
-	*fbo = *bo;
-	mutex_init(&fbo->mutex);
-	mutex_lock(&fbo->mutex);
-	mutex_lock(&dev->struct_mutex);
-
-	INIT_LIST_HEAD(&fbo->ddestroy);
-	INIT_LIST_HEAD(&fbo->lru);
-	list_splice_init(&bo->lru, &fbo->lru);
-
-	bo->mm_node = NULL;
-	bo->ttm = NULL;
-	bo->ttm_object = NULL;
-	bo->fence = NULL;
-	bo->flags = 0;
-
-	fbo->mm_node->private = (void *)fbo;
-	atomic_set(&fbo->usage, 1);
-	atomic_inc(&bm->count);
-	mutex_unlock(&dev->struct_mutex);
-	mutex_unlock(&fbo->mutex);
-
-	*new_obj = fbo;
-	return 0;
-}
-		
-
 int drm_buffer_object_create(drm_file_t * priv,
 			     unsigned long size,
 			     drm_bo_type_t type,
@@ -1404,7 +1539,6 @@ int drm_buffer_object_create(drm_file_t * priv,
 	drm_buffer_manager_t *bm = &dev->bm;
 	drm_buffer_object_t *bo;
 	int ret = 0;
-	uint32_t new_flags;
 	unsigned long num_pages;
 
 	if ((buffer_start & ~PAGE_MASK) && (type != drm_bo_type_fake)) {
@@ -1429,12 +1563,18 @@ int drm_buffer_object_create(drm_file_t * priv,
 	atomic_set(&bo->mapped, -1);
 	DRM_INIT_WAITQUEUE(&bo->event_queue);
 	INIT_LIST_HEAD(&bo->lru);
+	INIT_LIST_HEAD(&bo->pinned_lru);
 	INIT_LIST_HEAD(&bo->ddestroy);
+#ifdef DRM_ODD_MM_COMPAT
+	INIT_LIST_HEAD(&bo->p_mm_list);
+	INIT_LIST_HEAD(&bo->vma_list);
+#endif
 	bo->dev = dev;
 	bo->type = type;
-	bo->num_pages = num_pages;
-	bo->mm_node = NULL;
-	bo->page_alignment = page_alignment;
+	bo->mem.mem_type = DRM_BO_MEM_LOCAL;
+	bo->mem.num_pages = num_pages;
+	bo->mem.mm_node = NULL;
+	bo->mem.page_alignment = page_alignment;
 	if (bo->type == drm_bo_type_fake) {
 		bo->offset = buffer_start;
 		bo->buffer_start = 0;
@@ -1442,18 +1582,22 @@ int drm_buffer_object_create(drm_file_t * priv,
 		bo->buffer_start = buffer_start;
 	}
 	bo->priv_flags = 0;
-	bo->flags = DRM_BO_FLAG_MEM_LOCAL | DRM_BO_FLAG_CACHED;
+	bo->mem.flags = 0;
+	bo->mem.mask = 0;
 	atomic_inc(&bm->count);
-	ret = drm_bo_new_flags(dev, bo->flags, mask, hint,
-			       1, &new_flags, &bo->mask);
-	if (ret)
-		goto out_err;
-	ret = drm_bo_add_ttm(priv, bo);
+	ret = drm_bo_new_mask(bo, mask, hint);
+
 	if (ret)
 		goto out_err;
 
-	ret = drm_buffer_object_validate(bo, new_flags, 0,
-					 hint & DRM_BO_HINT_DONT_BLOCK);
+	if (bo->type == drm_bo_type_dc) {
+		mutex_lock(&dev->struct_mutex);
+		ret = drm_bo_setup_vm_locked(bo);
+		mutex_unlock(&dev->struct_mutex);
+		if (ret)
+			goto out_err;
+	}
+	ret = drm_buffer_object_validate(bo, 0, hint & DRM_BO_HINT_DONT_BLOCK);
 	if (ret)
 		goto out_err;
 
@@ -1463,6 +1607,7 @@ int drm_buffer_object_create(drm_file_t * priv,
 
       out_err:
 	mutex_unlock(&bo->mutex);
+
 	drm_bo_usage_deref_unlocked(bo);
 	return ret;
 }
@@ -1630,111 +1775,174 @@ int drm_bo_ioctl(DRM_IOCTL_ARGS)
 	return 0;
 }
 
+/**
+ *Clean the unfenced list and put on regular LRU.
+ *This is part of the memory manager cleanup and should only be
+ *called with the DRI lock held.
+ *Call dev->struct_sem locked.
+ */
+
+static void drm_bo_clean_unfenced(drm_device_t *dev)
+{
+	drm_buffer_manager_t *bm  = &dev->bm;
+	struct list_head *head, *list;
+	drm_buffer_object_t *entry;
+
+	head = &bm->unfenced;
+
+	list = head->next;
+	while(list != head) {
+		prefetch(list->next);
+		entry = list_entry(list, drm_buffer_object_t, lru);
+
+		atomic_inc(&entry->usage);
+		mutex_unlock(&dev->struct_mutex);
+		mutex_lock(&entry->mutex);
+		mutex_lock(&dev->struct_mutex);
+
+		list_del(&entry->lru);
+		DRM_FLAG_MASKED(entry->priv_flags, 0, _DRM_BO_FLAG_UNFENCED);
+		drm_bo_add_to_lru(entry);
+		mutex_unlock(&entry->mutex);
+		list = head->next;
+	}
+}
+
+static int drm_bo_leave_list(drm_buffer_object_t * bo,
+			     uint32_t mem_type,
+			     int free_pinned, int allow_errors)
+{
+	drm_device_t *dev = bo->dev;
+	int ret = 0;
+
+	mutex_lock(&bo->mutex);
+
+	ret = drm_bo_expire_fence(bo, allow_errors);
+	if (ret)
+		goto out;
+
+	if (free_pinned) {
+		DRM_FLAG_MASKED(bo->mem.flags, 0, DRM_BO_FLAG_NO_MOVE);
+		mutex_lock(&dev->struct_mutex);
+		list_del_init(&bo->pinned_lru);
+		if (bo->pinned_node == bo->mem.mm_node)
+			bo->pinned_node = NULL;
+		if (bo->pinned_node != NULL) {
+			drm_mm_put_block(bo->pinned_node);
+			bo->pinned_node = NULL;
+		}
+		mutex_unlock(&dev->struct_mutex);
+	}
+
+	if (bo->mem.flags & DRM_BO_FLAG_NO_EVICT) {
+		DRM_ERROR("A DRM_BO_NO_EVICT buffer present at "
+			  "cleanup. Removing flag and evicting.\n");
+		bo->mem.flags &= ~DRM_BO_FLAG_NO_EVICT;
+		bo->mem.mask &= ~DRM_BO_FLAG_NO_EVICT;
+	}
+
+	if (bo->mem.mem_type == mem_type)
+		ret = drm_bo_evict(bo, mem_type, 0);
+
+	if (ret) {
+		if (allow_errors) {
+			goto out;
+		} else {
+			ret = 0;
+			DRM_ERROR("Cleanup eviction failed\n");
+		}
+	}
+
+      out:
+	mutex_unlock(&bo->mutex);
+	return ret;
+}
+
+
+static drm_buffer_object_t *drm_bo_entry(struct list_head *list,
+					 int pinned_list)
+{
+	if (pinned_list)
+		return list_entry(list, drm_buffer_object_t, pinned_lru);
+	else
+		return list_entry(list, drm_buffer_object_t, lru);
+}
+
 /*
- * dev->struct_sem locked.
+ * dev->struct_mutex locked.
  */
 
 static int drm_bo_force_list_clean(drm_device_t * dev,
 				   struct list_head *head,
 				   unsigned mem_type,
-				   int force_no_move, int allow_errors)
+				   int free_pinned,
+				   int allow_errors,
+				   int pinned_list)
 {
-	drm_buffer_manager_t *bm = &dev->bm;
 	struct list_head *list, *next, *prev;
-	drm_buffer_object_t *entry;
+	drm_buffer_object_t *entry, *nentry;
 	int ret;
-	int clean;
+	int do_restart;
 
-      retry:
-	clean = 1;
+	/*
+	 * The list traversal is a bit odd here, because an item may
+	 * disappear from the list when we release the struct_mutex or
+	 * when we decrease the usage count. Also we're not guaranteed
+	 * to drain pinned lists, so we can't always restart.
+	 */
+
+restart:
+	nentry = NULL;
 	list_for_each_safe(list, next, head) {
 		prev = list->prev;
-		entry = list_entry(list, drm_buffer_object_t, lru);
+
+		entry = (nentry != NULL) ? nentry: drm_bo_entry(list, pinned_list);
 		atomic_inc(&entry->usage);
-		mutex_unlock(&dev->struct_mutex);
-		mutex_lock(&entry->mutex);
-		mutex_lock(&dev->struct_mutex);
+		if (nentry) {
+			atomic_dec(&nentry->usage);
+			nentry = NULL;
+		}
 
-		if (prev != list->prev || next != list->next) {
-			mutex_unlock(&entry->mutex);
-			drm_bo_usage_deref_locked(entry);
-			goto retry;
+		/*
+		 * Protect the next item from destruction, so we can check
+		 * its list pointers later on.
+		 */
+
+		if (next != head) {
+			nentry = drm_bo_entry(next, pinned_list);
+			atomic_inc(&nentry->usage);
 		}
-		if (entry->mm_node) {
-			clean = 0;
+		mutex_unlock(&dev->struct_mutex);
 
-			/*
-			 * Expire the fence.
-			 */
+		ret = drm_bo_leave_list(entry, mem_type, free_pinned,
+					allow_errors);
+		mutex_lock(&dev->struct_mutex);
 
-			mutex_unlock(&dev->struct_mutex);
-			if (entry->fence && bm->nice_mode) {
-				unsigned long _end = jiffies + 3 * DRM_HZ;
-				do {
-					ret = drm_bo_wait(entry, 0, 1, 0);
-					if (ret && allow_errors) {
-						if (ret == -EINTR)
-							ret = -EAGAIN;
-						goto out_err;
-					}
-				} while (ret && !time_after_eq(jiffies, _end));
-
-				if (entry->fence) {
-					bm->nice_mode = 0;
-					DRM_ERROR("Detected GPU hang or "
-						  "fence manager was taken down. "
-						  "Evicting waiting buffers\n");
-				}
-			}
-			if (entry->fence) {
-				drm_fence_usage_deref_unlocked(dev,
-							       entry->fence);
-				entry->fence = NULL;
-			}
+		drm_bo_usage_deref_locked(entry);
+		if (ret)
+			return ret;
 
-			DRM_MASK_VAL(entry->priv_flags, _DRM_BO_FLAG_UNFENCED,
-				     0);
+		/*
+		 * Has the next item disappeared from the list?
+		 */
 
-			if (force_no_move) {
-				DRM_MASK_VAL(entry->flags, DRM_BO_FLAG_NO_MOVE,
-					     0);
-			}
-			if (entry->flags & DRM_BO_FLAG_NO_EVICT) {
-				DRM_ERROR("A DRM_BO_NO_EVICT buffer present at "
-					  "cleanup. Removing flag and evicting.\n");
-				entry->flags &= ~DRM_BO_FLAG_NO_EVICT;
-				entry->mask &= ~DRM_BO_FLAG_NO_EVICT;
-			}
+		do_restart = ((next->prev != list) && (next->prev != prev));
 
-			ret = drm_bo_evict(entry, mem_type, 1, force_no_move);
-			if (ret) {
-				if (allow_errors) {
-					goto out_err;
-				} else {
-					DRM_ERROR("Aargh. Eviction failed.\n");
-				}
-			}
-			mutex_lock(&dev->struct_mutex);
-		}
-		mutex_unlock(&entry->mutex);
-		drm_bo_usage_deref_locked(entry);
-		if (prev != list->prev || next != list->next) {
-			goto retry;
+		if (nentry != NULL && do_restart) {
+			drm_bo_usage_deref_locked(nentry);
+			nentry = NULL;
 		}
+
+		if (do_restart)
+			goto restart;
 	}
-	if (!clean)
-		goto retry;
 	return 0;
-      out_err:
-	mutex_unlock(&entry->mutex);
-	drm_bo_usage_deref_unlocked(entry);
-	mutex_lock(&dev->struct_mutex);
-	return ret;
 }
 
 int drm_bo_clean_mm(drm_device_t * dev, unsigned mem_type)
 {
 	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man = &bm->man[mem_type];
 	int ret = -EINVAL;
 
 	if (mem_type >= DRM_BO_MEM_TYPES) {
@@ -1742,36 +1950,23 @@ int drm_bo_clean_mm(drm_device_t * dev, unsigned mem_type)
 		return ret;
 	}
 
-	if (!bm->has_type[mem_type]) {
+	if (!man->has_type) {
 		DRM_ERROR("Trying to take down uninitialized "
 			  "memory manager type\n");
 		return ret;
 	}
-	bm->use_type[mem_type] = 0;
-	bm->has_type[mem_type] = 0;
+	man->use_type = 0;
+	man->has_type = 0;
 
 	ret = 0;
 	if (mem_type > 0) {
 
-		/*
-		 * Throw out unfenced buffers.
-		 */
-
-		drm_bo_force_list_clean(dev, &bm->unfenced, mem_type, 1, 0);
+		drm_bo_clean_unfenced(dev);
+		drm_bo_force_list_clean(dev, &man->lru, mem_type, 1, 0, 0);
+		drm_bo_force_list_clean(dev, &man->pinned, mem_type, 1, 0, 1);
 
-		/*
-		 * Throw out evicted no-move buffers.
-		 */
-
-		drm_bo_force_list_clean(dev, &bm->pinned[DRM_BO_MEM_LOCAL],
-					mem_type, 1, 0);
-		drm_bo_force_list_clean(dev, &bm->lru[mem_type], mem_type, 1,
-					0);
-		drm_bo_force_list_clean(dev, &bm->pinned[mem_type], mem_type, 1,
-					0);
-
-		if (drm_mm_clean(&bm->manager[mem_type])) {
-			drm_mm_takedown(&bm->manager[mem_type]);
+		if (drm_mm_clean(&man->manager)) {
+			drm_mm_takedown(&man->manager);
 		} else {
 			ret = -EBUSY;
 		}
@@ -1780,24 +1975,29 @@ int drm_bo_clean_mm(drm_device_t * dev, unsigned mem_type)
 	return ret;
 }
 
+/**
+ *Evict all buffers of a particular mem_type, but leave memory manager
+ *regions for NO_MOVE buffers intact. New buffers cannot be added at this
+ *point since we have the hardware lock.
+ */
+
 static int drm_bo_lock_mm(drm_device_t * dev, unsigned mem_type)
 {
 	int ret;
 	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man = &bm->man[mem_type];
 
 	if (mem_type == 0 || mem_type >= DRM_BO_MEM_TYPES) {
 		DRM_ERROR("Illegal memory manager memory type %u,\n", mem_type);
 		return -EINVAL;
 	}
 
-	ret = drm_bo_force_list_clean(dev, &bm->unfenced, mem_type, 0, 1);
+	drm_bo_clean_unfenced(dev);
+	ret = drm_bo_force_list_clean(dev, &man->lru, mem_type, 0, 1, 0);
 	if (ret)
 		return ret;
-	ret = drm_bo_force_list_clean(dev, &bm->lru[mem_type], mem_type, 0, 1);
-	if (ret)
-		return ret;
-	ret =
-	    drm_bo_force_list_clean(dev, &bm->pinned[mem_type], mem_type, 0, 1);
+	ret = drm_bo_force_list_clean(dev, &man->pinned, mem_type, 0, 1, 1);
+
 	return ret;
 }
 
@@ -1807,32 +2007,39 @@ static int drm_bo_init_mm(drm_device_t * dev,
 {
 	drm_buffer_manager_t *bm = &dev->bm;
 	int ret = -EINVAL;
+	drm_mem_type_manager_t *man;
 
 	if (type >= DRM_BO_MEM_TYPES) {
 		DRM_ERROR("Illegal memory type %d\n", type);
 		return ret;
 	}
-	if (bm->has_type[type]) {
+
+	man = &bm->man[type];
+	if (man->has_type) {
 		DRM_ERROR("Memory manager already initialized for type %d\n",
 			  type);
 		return ret;
 	}
 
+	ret = dev->driver->bo_driver->init_mem_type(dev, type, man);
+	if (ret)
+		return ret;
+
 	ret = 0;
 	if (type != DRM_BO_MEM_LOCAL) {
 		if (!p_size) {
 			DRM_ERROR("Zero size memory manager type %d\n", type);
 			return ret;
 		}
-		ret = drm_mm_init(&bm->manager[type], p_offset, p_size);
+		ret = drm_mm_init(&man->manager, p_offset, p_size);
 		if (ret)
 			return ret;
 	}
-	bm->has_type[type] = 1;
-	bm->use_type[type] = 1;
+	man->has_type = 1;
+	man->use_type = 1;
 
-	INIT_LIST_HEAD(&bm->lru[type]);
-	INIT_LIST_HEAD(&bm->pinned[type]);
+	INIT_LIST_HEAD(&man->lru);
+	INIT_LIST_HEAD(&man->pinned);
 
 	return 0;
 }
@@ -1847,6 +2054,7 @@ int drm_bo_driver_finish(drm_device_t * dev)
 	drm_buffer_manager_t *bm = &dev->bm;
 	int ret = 0;
 	unsigned i = DRM_BO_MEM_TYPES;
+	drm_mem_type_manager_t *man;
 
 	mutex_lock(&dev->bm.init_mutex);
 	mutex_lock(&dev->struct_mutex);
@@ -1856,17 +2064,19 @@ int drm_bo_driver_finish(drm_device_t * dev)
 	bm->initialized = 0;
 
 	while (i--) {
-		if (bm->has_type[i]) {
-			bm->use_type[i] = 0;
+		man = &bm->man[i];
+		if (man->has_type) {
+			man->use_type = 0;
 			if ((i != DRM_BO_MEM_LOCAL) && drm_bo_clean_mm(dev, i)) {
 				ret = -EBUSY;
 				DRM_ERROR("DRM memory manager type %d "
 					  "is not clean.\n", i);
 			}
-			bm->has_type[i] = 0;
+			man->has_type = 0;
 		}
 	}
 	mutex_unlock(&dev->struct_mutex);
+
 	if (!cancel_delayed_work(&bm->wq)) {
 		flush_scheduled_work();
 	}
@@ -1875,10 +2085,10 @@ int drm_bo_driver_finish(drm_device_t * dev)
 	if (list_empty(&bm->ddestroy)) {
 		DRM_DEBUG("Delayed destroy list was clean\n");
 	}
-	if (list_empty(&bm->lru[0])) {
+	if (list_empty(&bm->man[0].lru)) {
 		DRM_DEBUG("Swap list was clean\n");
 	}
-	if (list_empty(&bm->pinned[0])) {
+	if (list_empty(&bm->man[0].pinned)) {
 		DRM_DEBUG("NO_MOVE list was clean\n");
 	}
 	if (list_empty(&bm->unfenced)) {
@@ -2006,3 +2216,144 @@ int drm_mm_init_ioctl(DRM_IOCTL_ARGS)
 	DRM_COPY_TO_USER_IOCTL((void __user *)data, arg, sizeof(arg));
 	return 0;
 }
+
+/*
+ * buffer object vm functions.
+ */
+
+int drm_mem_reg_is_pci(drm_device_t * dev, drm_bo_mem_reg_t * mem)
+{
+	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man = &bm->man[mem->mem_type];
+
+	if (!(man->flags & _DRM_FLAG_MEMTYPE_FIXED)) {
+		if (mem->mem_type == DRM_BO_MEM_LOCAL)
+			return 0;
+
+		if (man->flags & _DRM_FLAG_MEMTYPE_CMA)
+			return 0;
+
+		if (mem->flags & DRM_BO_FLAG_CACHED)
+			return 0;
+	}
+	return 1;
+}
+
+EXPORT_SYMBOL(drm_mem_reg_is_pci);
+
+/**
+ * \c Get the PCI offset for the buffer object memory.
+ *
+ * \param bo The buffer object.
+ * \param bus_base On return the base of the PCI region
+ * \param bus_offset On return the byte offset into the PCI region
+ * \param bus_size On return the byte size of the buffer object or zero if
+ *     the buffer object memory is not accessible through a PCI region.
+ * \return Failure indication.
+ * 
+ * Returns -EINVAL if the buffer object is currently not mappable.
+ * Otherwise returns zero.
+ */
+
+int drm_bo_pci_offset(drm_device_t * dev,
+		      drm_bo_mem_reg_t * mem,
+		      unsigned long *bus_base,
+		      unsigned long *bus_offset, unsigned long *bus_size)
+{
+	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man = &bm->man[mem->mem_type];
+
+	*bus_size = 0;
+	if (!(man->flags & _DRM_FLAG_MEMTYPE_MAPPABLE))
+		return -EINVAL;
+
+	if (drm_mem_reg_is_pci(dev, mem)) {
+		*bus_offset = mem->mm_node->start << PAGE_SHIFT;
+		*bus_size = mem->num_pages << PAGE_SHIFT;
+		*bus_base = man->io_offset;
+	}
+
+	return 0;
+}
+
+/**
+ * \c Kill all user-space virtual mappings of this buffer object.
+ *
+ * \param bo The buffer object.
+ *
+ * Call bo->mutex locked.
+ */
+
+void drm_bo_unmap_virtual(drm_buffer_object_t * bo)
+{
+	drm_device_t *dev = bo->dev;
+	loff_t offset = ((loff_t) bo->map_list.hash.key) << PAGE_SHIFT;
+	loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT;
+
+	unmap_mapping_range(dev->dev_mapping, offset, holelen, 1);
+}
+
+static void drm_bo_takedown_vm_locked(drm_buffer_object_t * bo)
+{
+	drm_map_list_t *list = &bo->map_list;
+	drm_local_map_t *map;
+	drm_device_t *dev = bo->dev;
+
+	if (list->user_token) {
+		drm_ht_remove_item(&dev->map_hash, &list->hash);
+		list->user_token = 0;
+	}
+	if (list->file_offset_node) {
+		drm_mm_put_block(list->file_offset_node);
+		list->file_offset_node = NULL;
+	}
+
+	map = list->map;
+	if (!map)
+		return;
+
+	drm_ctl_free(map, sizeof(*map), DRM_MEM_BUFOBJ);
+	list->map = NULL;
+	list->user_token = 0ULL;
+	drm_bo_usage_deref_locked(bo);
+}
+
+static int drm_bo_setup_vm_locked(drm_buffer_object_t * bo)
+{
+	drm_map_list_t *list = &bo->map_list;
+	drm_local_map_t *map;
+	drm_device_t *dev = bo->dev;
+
+	list->map = drm_ctl_calloc(1, sizeof(*map), DRM_MEM_BUFOBJ);
+	if (!list->map)
+		return -ENOMEM;
+
+	map = list->map;
+	map->offset = 0;
+	map->type = _DRM_TTM;
+	map->flags = _DRM_REMOVABLE;
+	map->size = bo->mem.num_pages * PAGE_SIZE;
+	atomic_inc(&bo->usage);
+	map->handle = (void *)bo;
+
+	list->file_offset_node = drm_mm_search_free(&dev->offset_manager,
+						    bo->mem.num_pages, 0, 0);
+
+	if (!list->file_offset_node) {
+		drm_bo_takedown_vm_locked(bo);
+		return -ENOMEM;
+	}
+
+	list->file_offset_node = drm_mm_get_block(list->file_offset_node,
+						  bo->mem.num_pages, 0);
+
+	list->hash.key = list->file_offset_node->start;
+	if (drm_ht_insert_item(&dev->map_hash, &list->hash)) {
+		drm_bo_takedown_vm_locked(bo);
+		return -ENOMEM;
+	}
+
+	list->user_token = ((drm_u64_t) list->hash.key) << PAGE_SHIFT;
+
+	return 0;
+}
diff --git a/linux-core/drm_bo_move.c b/linux-core/drm_bo_move.c
new file mode 100644
index 00000000..21f5f6cc
--- /dev/null
+++ b/linux-core/drm_bo_move.c
@@ -0,0 +1,410 @@
+/**************************************************************************
+ * 
+ * Copyright (c) 2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
+ * All Rights Reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
+ */
+
+#include "drmP.h"
+
+/**
+ * Free the old memory node unless it's a pinned region and we
+ * have not been requested to free also pinned regions.
+ */
+
+static void drm_bo_free_old_node(drm_buffer_object_t * bo)
+{
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+
+	if (old_mem->mm_node && (old_mem->mm_node != bo->pinned_node)) {
+		mutex_lock(&bo->dev->struct_mutex);
+		drm_mm_put_block(old_mem->mm_node);
+		old_mem->mm_node = NULL;
+		mutex_unlock(&bo->dev->struct_mutex);
+	}
+	old_mem->mm_node = NULL;
+}
+
+int drm_bo_move_ttm(drm_buffer_object_t * bo,
+		    int evict, int no_wait, drm_bo_mem_reg_t * new_mem)
+{
+	drm_ttm_t *ttm = bo->ttm;
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+	uint32_t save_flags = old_mem->flags;
+	uint32_t save_mask = old_mem->mask;
+	int ret;
+
+	if (old_mem->mem_type == DRM_BO_MEM_TT) {
+		if (evict)
+			drm_ttm_evict(ttm);
+		else
+			drm_ttm_unbind(ttm);
+
+		drm_bo_free_old_node(bo);
+		DRM_FLAG_MASKED(old_mem->flags,
+				DRM_BO_FLAG_CACHED | DRM_BO_FLAG_MAPPABLE |
+				DRM_BO_FLAG_MEM_LOCAL, DRM_BO_MASK_MEMTYPE);
+		old_mem->mem_type = DRM_BO_MEM_LOCAL;
+		save_flags = old_mem->flags;
+	}
+	if (new_mem->mem_type != DRM_BO_MEM_LOCAL) {
+		ret = drm_bind_ttm(ttm,
+				   new_mem->flags & DRM_BO_FLAG_CACHED,
+				   new_mem->mm_node->start);
+		if (ret)
+			return ret;
+	}
+
+	*old_mem = *new_mem;
+	new_mem->mm_node = NULL;
+	old_mem->mask = save_mask;
+	DRM_FLAG_MASKED(save_flags, new_mem->flags, DRM_BO_MASK_MEMTYPE);
+	return 0;
+}
+
+EXPORT_SYMBOL(drm_bo_move_ttm);
+
+/**
+ * \c Return a kernel virtual address to the buffer object PCI memory.
+ *
+ * \param bo The buffer object.
+ * \return Failure indication.
+ * 
+ * Returns -EINVAL if the buffer object is currently not mappable.
+ * Returns -ENOMEM if the ioremap operation failed.
+ * Otherwise returns zero.
+ * 
+ * After a successfull call, bo->iomap contains the virtual address, or NULL
+ * if the buffer object content is not accessible through PCI space. 
+ * Call bo->mutex locked.
+ */
+
+int drm_mem_reg_ioremap(drm_device_t * dev, drm_bo_mem_reg_t * mem,
+			void **virtual)
+{
+	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man = &bm->man[mem->mem_type];
+	unsigned long bus_offset;
+	unsigned long bus_size;
+	unsigned long bus_base;
+	int ret;
+	void *addr;
+
+	*virtual = NULL;
+	ret = drm_bo_pci_offset(dev, mem, &bus_base, &bus_offset, &bus_size);
+	if (ret || bus_size == 0)
+		return ret;
+
+	if (!(man->flags & _DRM_FLAG_NEEDS_IOREMAP))
+		addr = (void *)(((u8 *) man->io_addr) + bus_offset);
+	else {
+		addr = ioremap_nocache(bus_base + bus_offset, bus_size);
+		if (!addr)
+			return -ENOMEM;
+	}
+	*virtual = addr;
+	return 0;
+}
+
+/**
+ * \c Unmap mapping obtained using drm_bo_ioremap
+ *
+ * \param bo The buffer object.
+ *
+ * Call bo->mutex locked.
+ */
+
+void drm_mem_reg_iounmap(drm_device_t * dev, drm_bo_mem_reg_t * mem,
+			 void *virtual)
+{
+	drm_buffer_manager_t *bm;
+	drm_mem_type_manager_t *man;
+
+	bm = &dev->bm;
+	man = &bm->man[mem->mem_type];
+
+	if (virtual && (man->flags & _DRM_FLAG_NEEDS_IOREMAP)) {
+		iounmap(virtual);
+	}
+}
+
+static int drm_copy_io_page(void *dst, void *src, unsigned long page)
+{
+	uint32_t *dstP =
+	    (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
+	uint32_t *srcP =
+	    (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
+
+	int i;
+	for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
+		iowrite32(ioread32(srcP++), dstP++);
+	return 0;
+}
+
+static int drm_copy_io_ttm_page(drm_ttm_t * ttm, void *src, unsigned long page)
+{
+	struct page *d = drm_ttm_get_page(ttm, page);
+	void *dst;
+
+	if (!d)
+		return -ENOMEM;
+
+	src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
+	dst = kmap(d);
+	if (!dst)
+		return -ENOMEM;
+
+	memcpy_fromio(dst, src, PAGE_SIZE);
+	kunmap(d);
+	return 0;
+}
+
+static int drm_copy_ttm_io_page(drm_ttm_t * ttm, void *dst, unsigned long page)
+{
+	struct page *s = drm_ttm_get_page(ttm, page);
+	void *src;
+
+	if (!s)
+		return -ENOMEM;
+
+	dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
+	src = kmap(s);
+	if (!src)
+		return -ENOMEM;
+
+	memcpy_toio(dst, src, PAGE_SIZE);
+	kunmap(s);
+	return 0;
+}
+
+int drm_bo_move_memcpy(drm_buffer_object_t * bo,
+		       int evict, int no_wait, drm_bo_mem_reg_t * new_mem)
+{
+	drm_device_t *dev = bo->dev;
+	drm_mem_type_manager_t *man = &dev->bm.man[new_mem->mem_type];
+	drm_ttm_t *ttm = bo->ttm;
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+	drm_bo_mem_reg_t old_copy = *old_mem;
+	void *old_iomap;
+	void *new_iomap;
+	int ret;
+	uint32_t save_flags = old_mem->flags;
+	uint32_t save_mask = old_mem->mask;
+	unsigned long i;
+	unsigned long page;
+	unsigned long add = 0;
+	int dir;
+
+	ret = drm_mem_reg_ioremap(dev, old_mem, &old_iomap);
+	if (ret)
+		return ret;
+	ret = drm_mem_reg_ioremap(dev, new_mem, &new_iomap);
+	if (ret)
+		goto out;
+
+	if (old_iomap == NULL && new_iomap == NULL)
+		goto out2;
+	if (old_iomap == NULL && ttm == NULL)
+		goto out2;
+
+	add = 0;
+	dir = 1;
+
+	if ((old_mem->mem_type == new_mem->mem_type) &&
+	    (new_mem->mm_node->start <
+	     old_mem->mm_node->start + old_mem->mm_node->size)) {
+		dir = -1;
+		add = new_mem->num_pages - 1;
+	}
+
+	for (i = 0; i < new_mem->num_pages; ++i) {
+		page = i * dir + add;
+		if (old_iomap == NULL)
+			ret = drm_copy_ttm_io_page(ttm, new_iomap, page);
+		else if (new_iomap == NULL)
+			ret = drm_copy_io_ttm_page(ttm, old_iomap, page);
+		else
+			ret = drm_copy_io_page(new_iomap, old_iomap, page);
+		if (ret)
+			goto out1;
+	}
+	mb();
+      out2:
+	drm_bo_free_old_node(bo);
+
+	*old_mem = *new_mem;
+	new_mem->mm_node = NULL;
+	old_mem->mask = save_mask;
+	DRM_FLAG_MASKED(save_flags, new_mem->flags, DRM_BO_MASK_MEMTYPE);
+
+	if ((man->flags & _DRM_FLAG_MEMTYPE_FIXED) && (ttm != NULL)) {
+		drm_ttm_unbind(ttm);
+		drm_destroy_ttm(ttm);
+		bo->ttm = NULL;
+	}
+
+      out1:
+	drm_mem_reg_iounmap(dev, new_mem, new_iomap);
+      out:
+	drm_mem_reg_iounmap(dev, &old_copy, old_iomap);
+	return ret;
+}
+
+EXPORT_SYMBOL(drm_bo_move_memcpy);
+
+/*
+ * Transfer a buffer object's memory and LRU status to a newly
+ * created object. User-space references remains with the old
+ * object. Call bo->mutex locked.
+ */
+
+int drm_buffer_object_transfer(drm_buffer_object_t * bo,
+			       drm_buffer_object_t ** new_obj)
+{
+	drm_buffer_object_t *fbo;
+	drm_device_t *dev = bo->dev;
+	drm_buffer_manager_t *bm = &dev->bm;
+
+	fbo = drm_ctl_calloc(1, sizeof(*fbo), DRM_MEM_BUFOBJ);
+	if (!fbo)
+		return -ENOMEM;
+
+	*fbo = *bo;
+	mutex_init(&fbo->mutex);
+	mutex_lock(&fbo->mutex);
+	mutex_lock(&dev->struct_mutex);
+
+	DRM_INIT_WAITQUEUE(&bo->event_queue);
+	INIT_LIST_HEAD(&fbo->ddestroy);
+	INIT_LIST_HEAD(&fbo->lru);
+	INIT_LIST_HEAD(&fbo->pinned_lru);
+#ifdef DRM_ODD_MM_COMPAT
+	INIT_LIST_HEAD(&fbo->vma_list);
+	INIT_LIST_HEAD(&fbo->p_mm_list);
+#endif
+
+	atomic_inc(&bo->fence->usage);
+	fbo->pinned_node = NULL;
+	fbo->mem.mm_node->private = (void *)fbo;
+	atomic_set(&fbo->usage, 1);
+	atomic_inc(&bm->count);
+	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&fbo->mutex);
+
+	*new_obj = fbo;
+	return 0;
+}
+
+/*
+ * Since move is underway, we need to block signals in this function.
+ * We cannot restart until it has finished.
+ */
+
+int drm_bo_move_accel_cleanup(drm_buffer_object_t * bo,
+			      int evict,
+			      int no_wait,
+			      uint32_t fence_type,
+			      uint32_t fence_flags, drm_bo_mem_reg_t * new_mem)
+{
+	drm_device_t *dev = bo->dev;
+	drm_mem_type_manager_t *man = &dev->bm.man[new_mem->mem_type];
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+	int ret;
+	uint32_t save_flags = old_mem->flags;
+	uint32_t save_mask = old_mem->mask;
+	drm_buffer_object_t *old_obj;
+
+	if (bo->fence)
+		drm_fence_usage_deref_unlocked(dev, bo->fence);
+	ret = drm_fence_object_create(dev, fence_type,
+				      fence_flags | DRM_FENCE_FLAG_EMIT,
+				      &bo->fence);
+	if (ret)
+		return ret;
+
+#ifdef DRM_ODD_MM_COMPAT
+	/*
+	 * In this mode, we don't allow pipelining a copy blit,
+	 * since the buffer will be accessible from user space 
+	 * the moment we return and rebuild the page tables.
+	 *
+	 * With normal vm operation, page tables are rebuilt
+	 * on demand using fault(), which waits for buffer idle. 
+	 */
+	if (1)
+#else
+	if (evict || ((bo->mem.mm_node == bo->pinned_node) &&
+		      bo->mem.mm_node != NULL))
+#endif
+	{
+		ret = drm_bo_wait(bo, 0, 1, 0);
+		if (ret)
+			return ret;
+
+		drm_bo_free_old_node(bo);
+
+		if ((man->flags & _DRM_FLAG_MEMTYPE_FIXED) && (bo->ttm != NULL)) {
+			drm_ttm_unbind(bo->ttm);
+			drm_destroy_ttm(bo->ttm);
+			bo->ttm = NULL;
+		}
+	} else {
+
+		/* This should help pipeline ordinary buffer moves.
+		 *
+		 * Hang old buffer memory on a new buffer object,
+		 * and leave it to be released when the GPU
+		 * operation has completed.
+		 */
+
+		ret = drm_buffer_object_transfer(bo, &old_obj);
+
+		if (ret)
+			return ret;
+
+		if (!(man->flags & _DRM_FLAG_MEMTYPE_FIXED))
+			old_obj->ttm = NULL;
+		else
+			bo->ttm = NULL;
+
+		mutex_lock(&dev->struct_mutex);
+		list_del_init(&old_obj->lru);
+		DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED);
+		drm_bo_add_to_lru(old_obj);
+
+		drm_bo_usage_deref_locked(old_obj);
+		mutex_unlock(&dev->struct_mutex);
+
+	}
+
+	*old_mem = *new_mem;
+	new_mem->mm_node = NULL;
+	old_mem->mask = save_mask;
+	DRM_FLAG_MASKED(save_flags, new_mem->flags, DRM_BO_MASK_MEMTYPE);
+	return 0;
+}
+
+EXPORT_SYMBOL(drm_bo_move_accel_cleanup);
diff --git a/linux-core/drm_compat.c b/linux-core/drm_compat.c
index 6bb58424..3bb35997 100644
--- a/linux-core/drm_compat.c
+++ b/linux-core/drm_compat.c
@@ -79,54 +79,14 @@ pgprot_t vm_get_page_prot(unsigned long vm_flags)
 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
 
 /*
- * vm code for kernels below 2,6,15 in which version a major vm write
+ * vm code for kernels below 2.6.15 in which version a major vm write
  * occured. This implement a simple straightforward 
  * version similar to what's going to be
- * in kernel 2.6.20+?
+ * in kernel 2.6.19+
+ * Kernels below 2.6.15 use nopage whereas 2.6.19 and upwards use
+ * nopfn.
  */ 
 
-static int drm_pte_is_clear(struct vm_area_struct *vma,
-			    unsigned long addr)
-{
-	struct mm_struct *mm = vma->vm_mm;
-	int ret = 1;
-	pte_t *pte;
-	pmd_t *pmd;
-	pud_t *pud;
-	pgd_t *pgd;
-	
-
-	spin_lock(&mm->page_table_lock);
-	pgd = pgd_offset(mm, addr);
-	if (pgd_none(*pgd))
-		goto unlock;
-	pud = pud_offset(pgd, addr);
-        if (pud_none(*pud))
-		goto unlock;
-	pmd = pmd_offset(pud, addr);
-	if (pmd_none(*pmd))
-		goto unlock;
-	pte = pte_offset_map(pmd, addr);
-	if (!pte)
-		goto unlock;
-	ret = pte_none(*pte);
-	pte_unmap(pte);
- unlock:	
-	spin_unlock(&mm->page_table_lock);
-	return ret;
-}
-	
-int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 
-		  unsigned long pfn, pgprot_t pgprot)
-{
-	int ret;
-	if (!drm_pte_is_clear(vma, addr))
-		return -EBUSY;
-
-	ret = io_remap_pfn_range(vma, addr, pfn, PAGE_SIZE, pgprot);
-	return ret;
-}
-
 static struct {
 	spinlock_t lock;
 	struct page *dummy_page;
@@ -160,7 +120,7 @@ void free_nopage_retry(void)
 	}
 }
 
-struct page *drm_vm_ttm_nopage(struct vm_area_struct *vma,
+struct page *drm_bo_vm_nopage(struct vm_area_struct *vma,
 			       unsigned long address, 
 			       int *type)
 {
@@ -171,7 +131,7 @@ struct page *drm_vm_ttm_nopage(struct vm_area_struct *vma,
 
 	data.address = address;
 	data.vma = vma;
-	drm_vm_ttm_fault(vma, &data);
+	drm_bo_vm_fault(vma, &data);
 	switch (data.type) {
 	case VM_FAULT_OOM:
 		return NOPAGE_OOM;
@@ -186,10 +146,85 @@ struct page *drm_vm_ttm_nopage(struct vm_area_struct *vma,
 
 #endif
 
+#if !defined(DRM_FULL_MM_COMPAT) && \
+  ((LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) || \
+   (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)))
+
+static int drm_pte_is_clear(struct vm_area_struct *vma,
+			    unsigned long addr)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	int ret = 1;
+	pte_t *pte;
+	pmd_t *pmd;
+	pud_t *pud;
+	pgd_t *pgd;
+
+	spin_lock(&mm->page_table_lock);
+	pgd = pgd_offset(mm, addr);
+	if (pgd_none(*pgd))
+		goto unlock;
+	pud = pud_offset(pgd, addr);
+        if (pud_none(*pud))
+		goto unlock;
+	pmd = pmd_offset(pud, addr);
+	if (pmd_none(*pmd))
+		goto unlock;
+	pte = pte_offset_map(pmd, addr);
+	if (!pte)
+		goto unlock;
+	ret = pte_none(*pte);
+	pte_unmap(pte);
+ unlock:
+	spin_unlock(&mm->page_table_lock);
+	return ret;
+}
+
+int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
+		  unsigned long pfn)
+{
+	int ret;
+	if (!drm_pte_is_clear(vma, addr))
+		return -EBUSY;
+
+	ret = io_remap_pfn_range(vma, addr, pfn, PAGE_SIZE, vma->vm_page_prot);
+	return ret;
+}
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19) && !defined(DRM_FULL_MM_COMPAT))
+
+/**
+ * While waiting for the fault() handler to appear in
+ * we accomplish approximately
+ * the same wrapping it with nopfn.
+ */
+
+unsigned long drm_bo_vm_nopfn(struct vm_area_struct * vma,
+			   unsigned long address)
+{
+	struct fault_data data;
+	data.address = address;
+
+	(void) drm_bo_vm_fault(vma, &data);
+	if (data.type == VM_FAULT_OOM)
+		return NOPFN_OOM;
+	else if (data.type == VM_FAULT_SIGBUS)
+		return NOPFN_SIGBUS;
+
+	/*
+	 * pfn already set.
+	 */
+
+	return 0;
+}
+#endif
+
+
 #ifdef DRM_ODD_MM_COMPAT
 
 /*
- * VM compatibility code for 2.6.15-2.6.19(?). This code implements a complicated
+ * VM compatibility code for 2.6.15-2.6.18. This code implements a complicated
  * workaround for a single BUG statement in do_no_page in these versions. The
  * tricky thing is that we need to take the mmap_sem in exclusive mode for _all_
  * vmas mapping the ttm, before dev->struct_mutex is taken. The way we do this is to 
@@ -212,109 +247,100 @@ typedef struct vma_entry {
 } vma_entry_t;
 
 
-struct page *drm_vm_ttm_nopage(struct vm_area_struct *vma,
+struct page *drm_bo_vm_nopage(struct vm_area_struct *vma,
 			       unsigned long address, 
 			       int *type)
 {
-	drm_local_map_t *map = (drm_local_map_t *) vma->vm_private_data;
+	drm_buffer_object_t *bo = (drm_buffer_object_t *) vma->vm_private_data;
 	unsigned long page_offset;
 	struct page *page;
 	drm_ttm_t *ttm; 
-	drm_buffer_manager_t *bm;
 	drm_device_t *dev;
 
-	/*
-	 * FIXME: Check can't map aperture flag.
-	 */
+	mutex_lock(&bo->mutex);
 
 	if (type)
 		*type = VM_FAULT_MINOR;
 
-	if (!map) 
-		return NOPAGE_OOM;
-
-	if (address > vma->vm_end) 
-		return NOPAGE_SIGBUS;
+	if (address > vma->vm_end) {
+		page = NOPAGE_SIGBUS;
+		goto out_unlock;
+	}
+	
+	dev = bo->dev;
 
-	ttm = (drm_ttm_t *) map->offset;	
-	dev = ttm->dev;
-	mutex_lock(&dev->struct_mutex);
-	drm_fixup_ttm_caching(ttm);
-	BUG_ON(ttm->page_flags & DRM_TTM_PAGE_UNCACHED);
+	if (drm_mem_reg_is_pci(dev, &bo->mem)) {
+		DRM_ERROR("Invalid compat nopage.\n");
+		page = NOPAGE_SIGBUS;
+		goto out_unlock;
+	}
 
-	bm = &dev->bm;
+	ttm = bo->ttm;
+	drm_ttm_fixup_caching(ttm);
 	page_offset = (address - vma->vm_start) >> PAGE_SHIFT;
-	page = ttm->pages[page_offset];
-
+	page = drm_ttm_get_page(ttm, page_offset);
 	if (!page) {
-		if (drm_alloc_memctl(PAGE_SIZE)) {
-			page = NOPAGE_OOM;
-			goto out;
-		}
-		page = ttm->pages[page_offset] = 
-			alloc_page(GFP_KERNEL | __GFP_ZERO | GFP_DMA32);
-		if (!page) {
-		        drm_free_memctl(PAGE_SIZE);
-			page = NOPAGE_OOM;
-			goto out;
-		}
-		++bm->cur_pages;
-		SetPageLocked(page);
+		page = NOPAGE_OOM;
+		goto out_unlock;
 	}
 
 	get_page(page);
- out:
-	mutex_unlock(&dev->struct_mutex);
+out_unlock:
+	mutex_unlock(&bo->mutex);
 	return page;
 }
 
 
 
 
-int drm_ttm_map_bound(struct vm_area_struct *vma)
+int drm_bo_map_bound(struct vm_area_struct *vma)
 {
-	drm_local_map_t *map = (drm_local_map_t *)vma->vm_private_data;
-	drm_ttm_t *ttm = (drm_ttm_t *) map->offset;
+	drm_buffer_object_t *bo = (drm_buffer_object_t *)vma->vm_private_data;
 	int ret = 0;
-
-	if (ttm->page_flags & DRM_TTM_PAGE_UNCACHED) {
-		unsigned long pfn = ttm->aper_offset + 
-			(ttm->be->aperture_base >> PAGE_SHIFT);
-		pgprot_t pgprot = drm_io_prot(ttm->be->drm_map_type, vma);
-		
+	unsigned long bus_base;
+	unsigned long bus_offset;
+	unsigned long bus_size;
+	
+	ret = drm_bo_pci_offset(bo->dev, &bo->mem, &bus_base, 
+				&bus_offset, &bus_size);
+	BUG_ON(ret);
+
+	if (bus_size) {
+		drm_mem_type_manager_t *man = &dev->bm.man[bo->mem.mem_type];
+		unsigned long pfn = (bus_base + bus_offset) >> PAGE_SHIFT;
+		pgprot_t pgprot = drm_io_prot(man->drm_bus_maptype, vma);
 		ret = io_remap_pfn_range(vma, vma->vm_start, pfn,
 					 vma->vm_end - vma->vm_start,
 					 pgprot);
 	}
+
 	return ret;
 }
 	
 
-int drm_ttm_add_vma(drm_ttm_t * ttm, struct vm_area_struct *vma)
+int drm_bo_add_vma(drm_buffer_object_t * bo, struct vm_area_struct *vma)
 {
 	p_mm_entry_t *entry, *n_entry;
 	vma_entry_t *v_entry;
-	drm_local_map_t *map = (drm_local_map_t *)
-		vma->vm_private_data;
 	struct mm_struct *mm = vma->vm_mm;
 
-	v_entry = drm_ctl_alloc(sizeof(*v_entry), DRM_MEM_TTM);
+	v_entry = drm_ctl_alloc(sizeof(*v_entry), DRM_MEM_BUFOBJ);
 	if (!v_entry) {
 		DRM_ERROR("Allocation of vma pointer entry failed\n");
 		return -ENOMEM;
 	}
 	v_entry->vma = vma;
-	map->handle = (void *) v_entry;
-	list_add_tail(&v_entry->head, &ttm->vma_list);
 
-	list_for_each_entry(entry, &ttm->p_mm_list, head) {
+	list_add_tail(&v_entry->head, &bo->vma_list);
+
+	list_for_each_entry(entry, &bo->p_mm_list, head) {
 		if (mm == entry->mm) {
 			atomic_inc(&entry->refcount);
 			return 0;
 		} else if ((unsigned long)mm < (unsigned long)entry->mm) ;
 	}
 
-	n_entry = drm_ctl_alloc(sizeof(*n_entry), DRM_MEM_TTM);
+	n_entry = drm_ctl_alloc(sizeof(*n_entry), DRM_MEM_BUFOBJ);
 	if (!n_entry) {
 		DRM_ERROR("Allocation of process mm pointer entry failed\n");
 		return -ENOMEM;
@@ -328,29 +354,29 @@ int drm_ttm_add_vma(drm_ttm_t * ttm, struct vm_area_struct *vma)
 	return 0;
 }
 
-void drm_ttm_delete_vma(drm_ttm_t * ttm, struct vm_area_struct *vma)
+void drm_bo_delete_vma(drm_buffer_object_t * bo, struct vm_area_struct *vma)
 {
 	p_mm_entry_t *entry, *n;
 	vma_entry_t *v_entry, *v_n;
 	int found = 0;
 	struct mm_struct *mm = vma->vm_mm;
 
-	list_for_each_entry_safe(v_entry, v_n, &ttm->vma_list, head) {
+	list_for_each_entry_safe(v_entry, v_n, &bo->vma_list, head) {
 		if (v_entry->vma == vma) {
 			found = 1;
 			list_del(&v_entry->head);
-			drm_ctl_free(v_entry, sizeof(*v_entry), DRM_MEM_TTM);
+			drm_ctl_free(v_entry, sizeof(*v_entry), DRM_MEM_BUFOBJ);
 			break;
 		}
 	}
 	BUG_ON(!found);
 
-	list_for_each_entry_safe(entry, n, &ttm->p_mm_list, head) {
+	list_for_each_entry_safe(entry, n, &bo->p_mm_list, head) {
 		if (mm == entry->mm) {
 			if (atomic_add_negative(-1, &entry->refcount)) {
 				list_del(&entry->head);
 				BUG_ON(entry->locked);
-				drm_ctl_free(entry, sizeof(*entry), DRM_MEM_TTM);
+				drm_ctl_free(entry, sizeof(*entry), DRM_MEM_BUFOBJ);
 			}
 			return;
 		}
@@ -360,12 +386,12 @@ void drm_ttm_delete_vma(drm_ttm_t * ttm, struct vm_area_struct *vma)
 
 
 
-int drm_ttm_lock_mm(drm_ttm_t * ttm)
+int drm_bo_lock_kmm(drm_buffer_object_t * bo)
 {
 	p_mm_entry_t *entry;
 	int lock_ok = 1;
 	
-	list_for_each_entry(entry, &ttm->p_mm_list, head) {
+	list_for_each_entry(entry, &bo->p_mm_list, head) {
 		BUG_ON(entry->locked);
 		if (!down_write_trylock(&entry->mm->mmap_sem)) {
 			lock_ok = 0;
@@ -377,7 +403,7 @@ int drm_ttm_lock_mm(drm_ttm_t * ttm)
 	if (lock_ok)
 		return 0;
 
-	list_for_each_entry(entry, &ttm->p_mm_list, head) {
+	list_for_each_entry(entry, &bo->p_mm_list, head) {
 		if (!entry->locked) 
 			break;
 		up_write(&entry->mm->mmap_sem);
@@ -392,43 +418,40 @@ int drm_ttm_lock_mm(drm_ttm_t * ttm)
 	return -EAGAIN;
 }
 
-void drm_ttm_unlock_mm(drm_ttm_t * ttm)
+void drm_bo_unlock_kmm(drm_buffer_object_t * bo)
 {
 	p_mm_entry_t *entry;
 	
-	list_for_each_entry(entry, &ttm->p_mm_list, head) {
+	list_for_each_entry(entry, &bo->p_mm_list, head) {
 		BUG_ON(!entry->locked);
 		up_write(&entry->mm->mmap_sem);
 		entry->locked = 0;
 	}
 }
 
-int drm_ttm_remap_bound(drm_ttm_t *ttm) 
+int drm_bo_remap_bound(drm_buffer_object_t *bo) 
 {
 	vma_entry_t *v_entry;
 	int ret = 0;
-	
-	list_for_each_entry(v_entry, &ttm->vma_list, head) {
-		ret = drm_ttm_map_bound(v_entry->vma);
-		if (ret)
-			break;
+
+	if (drm_mem_reg_is_pci(bo->dev, &bo->mem)) {
+		list_for_each_entry(v_entry, &bo->vma_list, head) {
+			ret = drm_bo_map_bound(v_entry->vma);
+			if (ret)
+				break;
+		}
 	}
 
-	drm_ttm_unlock_mm(ttm);
 	return ret;
 }
 
-void drm_ttm_finish_unmap(drm_ttm_t *ttm)
+void drm_bo_finish_unmap(drm_buffer_object_t *bo)
 {
 	vma_entry_t *v_entry;
-	
-	if (!(ttm->page_flags & DRM_TTM_PAGE_UNCACHED))
-		return;
 
-	list_for_each_entry(v_entry, &ttm->vma_list, head) {
+	list_for_each_entry(v_entry, &bo->vma_list, head) {
 		v_entry->vma->vm_flags &= ~VM_PFNMAP; 
 	}
-	drm_ttm_unlock_mm(ttm);
 }	
 
 #endif
diff --git a/linux-core/drm_compat.h b/linux-core/drm_compat.h
index 3cb5d202..0dee3564 100644
--- a/linux-core/drm_compat.h
+++ b/linux-core/drm_compat.h
@@ -158,11 +158,14 @@ static __inline__ void *kcalloc(size_t nmemb, size_t size, int flags)
 #include <linux/mm.h>
 #include <asm/page.h>
 
-#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21)) && \
-     (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15))) 
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)) && \
+     (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15)))
 #define DRM_ODD_MM_COMPAT
 #endif
 
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21))
+#define DRM_FULL_MM_COMPAT
+#endif
 
 
 /*
@@ -200,18 +203,23 @@ extern int drm_map_page_into_agp(struct page *page);
 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
 extern struct page *get_nopage_retry(void);
 extern void free_nopage_retry(void);
-struct fault_data;
-extern struct page *drm_vm_ttm_fault(struct vm_area_struct *vma, 
-				     struct fault_data *data);
 
 #define NOPAGE_REFAULT get_nopage_retry()
 #endif
 
+#if !defined(DRM_FULL_MM_COMPAT) && \
+  ((LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) || \
+   (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)))
+
+struct fault_data;
+extern struct page *drm_bo_vm_fault(struct vm_area_struct *vma,
+				    struct fault_data *data);
 
-#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21))
+#endif
+#ifndef DRM_FULL_MM_COMPAT
 
 /*
- * Hopefully, real NOPAGE_RETRY functionality will be in 2.6.19. 
+ * Hopefully, real NOPAGE_RETRY functionality will be in 2.6.19.
  * For now, just return a dummy page that we've allocated out of 
  * static space. The page will be put by do_nopage() since we've already
  * filled out the pte.
@@ -228,17 +236,21 @@ struct fault_data {
 
 
 extern int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 
-			 unsigned long pfn, pgprot_t pgprot);
-
-extern struct page *drm_vm_ttm_nopage(struct vm_area_struct *vma,
-				      unsigned long address, 
-				      int *type);
+			 unsigned long pfn);
 
-#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19))
+extern struct page *drm_bo_vm_nopage(struct vm_area_struct *vma,
+				     unsigned long address, 
+				     int *type);
+#else
+extern unsigned long drm_bo_vm_nopfn(struct vm_area_struct *vma,
+				     unsigned long address);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)) */
+#endif /* ndef DRM_FULL_MM_COMPAT */
 
 #ifdef DRM_ODD_MM_COMPAT
 
-struct drm_ttm;
+struct drm_buffer_object;
 
 
 /*
@@ -246,14 +258,14 @@ struct drm_ttm;
  * process mm pointer to the ttm mm list. Needs the ttm mutex.
  */
 
-extern int drm_ttm_add_vma(struct drm_ttm * ttm, 
+extern int drm_bo_add_vma(struct drm_buffer_object * bo, 
 			   struct vm_area_struct *vma);
 /*
  * Delete a vma and the corresponding mm pointer from the
  * ttm lists. Needs the ttm mutex.
  */
-extern void drm_ttm_delete_vma(struct drm_ttm * ttm, 
-			       struct vm_area_struct *vma);
+extern void drm_bo_delete_vma(struct drm_buffer_object * bo, 
+			      struct vm_area_struct *vma);
 
 /*
  * Attempts to lock all relevant mmap_sems for a ttm, while
@@ -262,12 +274,12 @@ extern void drm_ttm_delete_vma(struct drm_ttm * ttm,
  * schedule() and try again.
  */
 
-extern int drm_ttm_lock_mm(struct drm_ttm * ttm);
+extern int drm_bo_lock_kmm(struct drm_buffer_object * bo);
 
 /*
  * Unlock all relevant mmap_sems for a ttm.
  */
-extern void drm_ttm_unlock_mm(struct drm_ttm * ttm);
+extern void drm_bo_unlock_kmm(struct drm_buffer_object * bo);
 
 /*
  * If the ttm was bound to the aperture, this function shall be called
@@ -277,7 +289,7 @@ extern void drm_ttm_unlock_mm(struct drm_ttm * ttm);
  * releases the mmap_sems for this ttm.
  */
 
-extern void drm_ttm_finish_unmap(struct drm_ttm *ttm);
+extern void drm_bo_finish_unmap(struct drm_buffer_object *bo);
 
 /*
  * Remap all vmas of this ttm using io_remap_pfn_range. We cannot 
@@ -286,14 +298,14 @@ extern void drm_ttm_finish_unmap(struct drm_ttm *ttm);
  * releases the mmap_sems for this ttm.
  */
 
-extern int drm_ttm_remap_bound(struct drm_ttm *ttm);
+extern int drm_bo_remap_bound(struct drm_buffer_object *bo);
 
 
 /*
  * Remap a vma for a bound ttm. Call with the ttm mutex held and
  * the relevant mmap_sem locked.
  */
-extern int drm_ttm_map_bound(struct vm_area_struct *vma);
+extern int drm_bo_map_bound(struct vm_area_struct *vma);
 
 #endif
 #endif
diff --git a/linux-core/drm_fence.c b/linux-core/drm_fence.c
index 06d48255..3ccfdcb8 100644
--- a/linux-core/drm_fence.c
+++ b/linux-core/drm_fence.c
@@ -1,6 +1,6 @@
 /**************************************************************************
  * 
- * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
+ * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
  * All Rights Reserved.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -11,6 +11,10 @@
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
@@ -19,11 +23,6 @@
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- * 
- * 
  **************************************************************************/
 /*
  * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
@@ -35,7 +34,8 @@
  * Typically called by the IRQ handler.
  */
 
-void drm_fence_handler(drm_device_t * dev, uint32_t sequence, uint32_t type)
+void drm_fence_handler(drm_device_t * dev, uint32_t class,
+		       uint32_t sequence, uint32_t type)
 {
 	int wake = 0;
 	uint32_t diff;
@@ -147,7 +147,7 @@ static void drm_fence_object_destroy(drm_file_t * priv,
 	drm_fence_usage_deref_locked(dev, fence);
 }
 
-static int fence_signaled(drm_device_t * dev, volatile
+static int fence_signaled(drm_device_t * dev,
 			  drm_fence_object_t * fence,
 			  uint32_t mask, int poke_flush)
 {
@@ -157,7 +157,7 @@ static int fence_signaled(drm_device_t * dev, volatile
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
 
 	if (poke_flush)
-		driver->poke_flush(dev);
+		driver->poke_flush(dev, fence->class);
 	read_lock_irqsave(&fm->lock, flags);
 	signaled =
 	    (fence->type & mask & fence->signaled) == (fence->type & mask);
@@ -172,13 +172,12 @@ static void drm_fence_flush_exe(drm_fence_manager_t * fm,
 	uint32_t diff;
 
 	if (!fm->pending_exe_flush) {
-		volatile struct list_head *list;
+		struct list_head *list;
 
 		/*
 		 * Last_exe_flush is invalid. Find oldest sequence.
 		 */
 
-/*		list = fm->fence_types[_DRM_FENCE_TYPE_EXE];*/
 		list = &fm->ring;
 		if (list->next == &fm->ring) {
 			return;
@@ -202,14 +201,15 @@ static void drm_fence_flush_exe(drm_fence_manager_t * fm,
 	}
 }
 
-int drm_fence_object_signaled(volatile drm_fence_object_t * fence,
+int drm_fence_object_signaled(drm_fence_object_t * fence,
 			      uint32_t type)
 {
 	return ((fence->signaled & type) == type);
 }
 
 int drm_fence_object_flush(drm_device_t * dev,
-			   volatile drm_fence_object_t * fence, uint32_t type)
+			   drm_fence_object_t * fence,
+			   uint32_t type)
 {
 	drm_fence_manager_t *fm = &dev->fm;
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
@@ -235,7 +235,7 @@ int drm_fence_object_flush(drm_device_t * dev,
 		}
 	}
 	write_unlock_irqrestore(&fm->lock, flags);
-	driver->poke_flush(dev);
+	driver->poke_flush(dev, fence->class);
 	return 0;
 }
 
@@ -274,11 +274,37 @@ void drm_fence_flush_old(drm_device_t * dev, uint32_t sequence)
 
 EXPORT_SYMBOL(drm_fence_flush_old);
 
+static int drm_fence_lazy_wait(drm_device_t *dev,
+			       drm_fence_object_t *fence,
+			       int ignore_signals, uint32_t mask)
+{
+	drm_fence_manager_t *fm = &dev->fm;
+	unsigned long _end = jiffies + 3*DRM_HZ;
+	int ret = 0;
+
+	do {
+		DRM_WAIT_ON(ret, fm->fence_queue, 3 * DRM_HZ,
+			    fence_signaled(dev, fence, mask, 1));
+		if (time_after_eq(jiffies, _end))
+			break;
+	} while (ret == -EINTR && ignore_signals);
+	if (time_after_eq(jiffies, _end) && (ret != 0))
+		ret = -EBUSY;
+	if (ret) {
+		if (ret == -EBUSY) {
+			DRM_ERROR("Fence timeout. "
+				  "GPU lockup or fence driver was "
+				  "taken down.\n");
+		}
+		return ((ret == -EINTR) ? -EAGAIN : ret);
+	}
+	return 0;
+}
+
 int drm_fence_object_wait(drm_device_t * dev,
-			  volatile drm_fence_object_t * fence,
+			  drm_fence_object_t * fence,
 			  int lazy, int ignore_signals, uint32_t mask)
 {
-	drm_fence_manager_t *fm = &dev->fm;
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
 	int ret = 0;
 	unsigned long _end;
@@ -299,46 +325,32 @@ int drm_fence_object_wait(drm_device_t * dev,
 
 	if (lazy && driver->lazy_capable) {
 
-		do {
-			DRM_WAIT_ON(ret, fm->fence_queue, 3 * DRM_HZ,
-				    fence_signaled(dev, fence, mask, 1));
-			if (time_after_eq(jiffies, _end))
-				break;
-		} while (ret == -EINTR && ignore_signals);
-		if (time_after_eq(jiffies, _end) && (ret != 0))
-			ret = -EBUSY;
-		if (ret) {
-			if (ret == -EBUSY) {
-				DRM_ERROR("Fence timeout. "
-					  "GPU lockup or fence driver was "
-					  "taken down.\n");
-			}
-			return ((ret == -EINTR) ? -EAGAIN : ret);
-		}
-	} else if ((fence->class == 0) && (mask & DRM_FENCE_TYPE_EXE) &&
-		   driver->lazy_capable) {
+		ret = drm_fence_lazy_wait(dev, fence, ignore_signals, mask);
+		if (ret)
+			return ret;
 
-		/*
-		 * We use IRQ wait for EXE fence if available to gain 
-		 * CPU in some cases.
-		 */
+	} else {
 
-		do {
-			DRM_WAIT_ON(ret, fm->fence_queue, 3 * DRM_HZ,
-				    fence_signaled(dev, fence,
-						   DRM_FENCE_TYPE_EXE, 1));
-			if (time_after_eq(jiffies, _end))
-				break;
-		} while (ret == -EINTR && ignore_signals);
-		if (time_after_eq(jiffies, _end) && (ret != 0))
-			ret = -EBUSY;
-		if (ret)
-			return ((ret == -EINTR) ? -EAGAIN : ret);
-	}
+		if (driver->has_irq(dev, fence->class,
+				    DRM_FENCE_TYPE_EXE)) {
+			ret = drm_fence_lazy_wait(dev, fence, ignore_signals,
+						  DRM_FENCE_TYPE_EXE);
+			if (ret)
+				return ret;
+		}
 
+		if (driver->has_irq(dev, fence->class,
+				    mask & ~DRM_FENCE_TYPE_EXE)) {
+			ret = drm_fence_lazy_wait(dev, fence, ignore_signals,
+						  mask);
+			if (ret)
+				return ret;
+		}
+	}
 	if (fence_signaled(dev, fence, mask, 0))
 		return 0;
 
+	DRM_ERROR("Busy wait\n");
 	/*
 	 * Avoid kernel-space busy-waits.
 	 */
@@ -368,7 +380,7 @@ int drm_fence_object_emit(drm_device_t * dev, drm_fence_object_t * fence,
 	int ret;
 
 	drm_fence_unring(dev, &fence->ring);
-	ret = driver->emit(dev, fence_flags, &sequence, &native_type);
+	ret = driver->emit(dev, fence->class, fence_flags, &sequence, &native_type);
 	if (ret)
 		return ret;
 
diff --git a/linux-core/drm_mm.c b/linux-core/drm_mm.c
index 5889ee4d..634a1782 100644
--- a/linux-core/drm_mm.c
+++ b/linux-core/drm_mm.c
@@ -217,6 +217,7 @@ void drm_mm_put_block(drm_mm_node_t * cur)
 		drm_ctl_free(cur, sizeof(*cur), DRM_MEM_MM);
 	}
 }
+EXPORT_SYMBOL(drm_mm_put_block);
 
 drm_mm_node_t *drm_mm_search_free(const drm_mm_t * mm,
 				  unsigned long size,
diff --git a/linux-core/drm_object.c b/linux-core/drm_object.c
index 0157329c..939cf0d7 100644
--- a/linux-core/drm_object.c
+++ b/linux-core/drm_object.c
@@ -1,6 +1,6 @@
 /**************************************************************************
  * 
- * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
+ * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
  * All Rights Reserved.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -10,6 +10,10 @@
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
  * 
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
@@ -18,13 +22,11 @@
  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- * 
  * 
  **************************************************************************/
+/*
+ * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
+ */
 
 #include "drmP.h"
 
diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h
new file mode 100644
index 00000000..f9b8ebd9
--- /dev/null
+++ b/linux-core/drm_objects.h
@@ -0,0 +1,469 @@
+/**************************************************************************
+ * 
+ * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
+ * All Rights Reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
+ */
+
+#ifndef _DRM_OBJECTS_H
+#define _DRM_OJBECTS_H
+#define DRM_HAS_TTM
+
+struct drm_device;
+
+/***************************************************
+ * User space objects. (drm_object.c)
+ */
+
+#define drm_user_object_entry(_ptr, _type, _member) container_of(_ptr, _type, _member)
+
+typedef enum {
+	drm_fence_type,
+	drm_buffer_type,
+	drm_ttm_type
+	    /*
+	     * Add other user space object types here.
+	     */
+} drm_object_type_t;
+
+/*
+ * A user object is a structure that helps the drm give out user handles
+ * to kernel internal objects and to keep track of these objects so that
+ * they can be destroyed, for example when the user space process exits.
+ * Designed to be accessible using a user space 32-bit handle.
+ */
+
+typedef struct drm_user_object {
+	drm_hash_item_t hash;
+	struct list_head list;
+	drm_object_type_t type;
+	atomic_t refcount;
+	int shareable;
+	drm_file_t *owner;
+	void (*ref_struct_locked) (drm_file_t * priv,
+				   struct drm_user_object * obj,
+				   drm_ref_t ref_action);
+	void (*unref) (drm_file_t * priv, struct drm_user_object * obj,
+		       drm_ref_t unref_action);
+	void (*remove) (drm_file_t * priv, struct drm_user_object * obj);
+} drm_user_object_t;
+
+/*
+ * A ref object is a structure which is used to
+ * keep track of references to user objects and to keep track of these
+ * references so that they can be destroyed for example when the user space
+ * process exits. Designed to be accessible using a pointer to the _user_ object.
+ */
+
+typedef struct drm_ref_object {
+	drm_hash_item_t hash;
+	struct list_head list;
+	atomic_t refcount;
+	drm_ref_t unref_action;
+} drm_ref_object_t;
+
+/**
+ * Must be called with the struct_mutex held.
+ */
+
+extern int drm_add_user_object(drm_file_t * priv, drm_user_object_t * item,
+			       int shareable);
+/**
+ * Must be called with the struct_mutex held.
+ */
+
+extern drm_user_object_t *drm_lookup_user_object(drm_file_t * priv,
+						 uint32_t key);
+
+/*
+ * Must be called with the struct_mutex held.
+ * If "item" has been obtained by a call to drm_lookup_user_object. You may not
+ * release the struct_mutex before calling drm_remove_ref_object.
+ * This function may temporarily release the struct_mutex.
+ */
+
+extern int drm_remove_user_object(drm_file_t * priv, drm_user_object_t * item);
+
+/*
+ * Must be called with the struct_mutex held. May temporarily release it.
+ */
+
+extern int drm_add_ref_object(drm_file_t * priv,
+			      drm_user_object_t * referenced_object,
+			      drm_ref_t ref_action);
+
+/*
+ * Must be called with the struct_mutex held.
+ */
+
+drm_ref_object_t *drm_lookup_ref_object(drm_file_t * priv,
+					drm_user_object_t * referenced_object,
+					drm_ref_t ref_action);
+/*
+ * Must be called with the struct_mutex held.
+ * If "item" has been obtained by a call to drm_lookup_ref_object. You may not
+ * release the struct_mutex before calling drm_remove_ref_object.
+ * This function may temporarily release the struct_mutex.
+ */
+
+extern void drm_remove_ref_object(drm_file_t * priv, drm_ref_object_t * item);
+extern int drm_user_object_ref(drm_file_t * priv, uint32_t user_token,
+			       drm_object_type_t type,
+			       drm_user_object_t ** object);
+extern int drm_user_object_unref(drm_file_t * priv, uint32_t user_token,
+				 drm_object_type_t type);
+
+/***************************************************
+ * Fence objects. (drm_fence.c)
+ */
+
+typedef struct drm_fence_object {
+	drm_user_object_t base;
+	atomic_t usage;
+
+	/*
+	 * The below three fields are protected by the fence manager spinlock.
+	 */
+
+	struct list_head ring;
+	int class;
+	uint32_t native_type;
+	uint32_t type;
+	uint32_t signaled;
+	uint32_t sequence;
+	uint32_t flush_mask;
+	uint32_t submitted_flush;
+} drm_fence_object_t;
+
+#define _DRM_FENCE_TYPE_EXE 0x00
+
+typedef struct drm_fence_manager {
+	int initialized;
+	rwlock_t lock;
+
+	/*
+	 * The list below should be maintained in sequence order and
+	 * access is protected by the above spinlock.
+	 */
+
+	struct list_head ring;
+	struct list_head *fence_types[32];
+	volatile uint32_t pending_flush;
+	wait_queue_head_t fence_queue;
+	int pending_exe_flush;
+	uint32_t last_exe_flush;
+	uint32_t exe_flush_sequence;
+	atomic_t count;
+} drm_fence_manager_t;
+
+typedef struct drm_fence_driver {
+	int no_types;
+	uint32_t wrap_diff;
+	uint32_t flush_diff;
+	uint32_t sequence_mask;
+	int lazy_capable;
+	int (*has_irq) (struct drm_device * dev, uint32_t class,
+			uint32_t flags);
+	int (*emit) (struct drm_device * dev, uint32_t class, uint32_t flags,
+		     uint32_t * breadcrumb, uint32_t * native_type);
+	void (*poke_flush) (struct drm_device * dev, uint32_t class);
+} drm_fence_driver_t;
+
+extern void drm_fence_handler(struct drm_device *dev, uint32_t class,
+			      uint32_t sequence, uint32_t type);
+extern void drm_fence_manager_init(struct drm_device *dev);
+extern void drm_fence_manager_takedown(struct drm_device *dev);
+extern void drm_fence_flush_old(struct drm_device *dev, uint32_t sequence);
+extern int drm_fence_object_flush(struct drm_device *dev,
+				  drm_fence_object_t * fence, uint32_t type);
+extern int drm_fence_object_signaled(drm_fence_object_t * fence, uint32_t type);
+extern void drm_fence_usage_deref_locked(struct drm_device *dev,
+					 drm_fence_object_t * fence);
+extern void drm_fence_usage_deref_unlocked(struct drm_device *dev,
+					   drm_fence_object_t * fence);
+extern int drm_fence_object_wait(struct drm_device *dev,
+				 drm_fence_object_t * fence,
+				 int lazy, int ignore_signals, uint32_t mask);
+extern int drm_fence_object_create(struct drm_device *dev, uint32_t type,
+				   uint32_t fence_flags,
+				   drm_fence_object_t ** c_fence);
+extern int drm_fence_add_user_object(drm_file_t * priv,
+				     drm_fence_object_t * fence, int shareable);
+extern int drm_fence_ioctl(DRM_IOCTL_ARGS);
+
+/**************************************************
+ *TTMs
+ */
+
+/*
+ * The ttm backend GTT interface. (In our case AGP).
+ * Any similar type of device (PCIE?)
+ * needs only to implement these functions to be usable with the "TTM" interface.
+ * The AGP backend implementation lives in drm_agpsupport.c 
+ * basically maps these calls to available functions in agpgart.
+ * Each drm device driver gets an
+ * additional function pointer that creates these types, 
+ * so that the device can choose the correct aperture.
+ * (Multiple AGP apertures, etc.) 
+ * Most device drivers will let this point to the standard AGP implementation.
+ */
+
+#define DRM_BE_FLAG_NEEDS_FREE     0x00000001
+#define DRM_BE_FLAG_BOUND_CACHED   0x00000002
+
+typedef struct drm_ttm_backend {
+	void *private;
+	uint32_t flags;
+	uint32_t drm_map_type;
+	int (*needs_ub_cache_adjust) (struct drm_ttm_backend * backend);
+	int (*populate) (struct drm_ttm_backend * backend,
+			 unsigned long num_pages, struct page ** pages);
+	void (*clear) (struct drm_ttm_backend * backend);
+	int (*bind) (struct drm_ttm_backend * backend,
+		     unsigned long offset, int cached);
+	int (*unbind) (struct drm_ttm_backend * backend);
+	void (*destroy) (struct drm_ttm_backend * backend);
+} drm_ttm_backend_t;
+
+typedef struct drm_ttm {
+	struct page **pages;
+	uint32_t page_flags;
+	unsigned long num_pages;
+	unsigned long aper_offset;
+	atomic_t vma_count;
+	struct drm_device *dev;
+	int destroy;
+	uint32_t mapping_offset;
+	drm_ttm_backend_t *be;
+	enum {
+		ttm_bound,
+		ttm_evicted,
+		ttm_unbound,
+		ttm_unpopulated,
+	} state;
+
+} drm_ttm_t;
+
+extern drm_ttm_t *drm_ttm_init(struct drm_device *dev, unsigned long size);
+extern int drm_bind_ttm(drm_ttm_t * ttm, int cached, unsigned long aper_offset);
+extern void drm_ttm_unbind(drm_ttm_t * ttm);
+extern void drm_ttm_evict(drm_ttm_t * ttm);
+extern void drm_ttm_fixup_caching(drm_ttm_t * ttm);
+extern struct page *drm_ttm_get_page(drm_ttm_t * ttm, int index);
+
+/*
+ * Destroy a ttm. The user normally calls drmRmMap or a similar IOCTL to do this, 
+ * which calls this function iff there are no vmas referencing it anymore. Otherwise it is called
+ * when the last vma exits.
+ */
+
+extern int drm_destroy_ttm(drm_ttm_t * ttm);
+
+#define DRM_FLAG_MASKED(_old, _new, _mask) {\
+(_old) ^= (((_old) ^ (_new)) & (_mask)); \
+}
+
+#define DRM_TTM_MASK_FLAGS ((1 << PAGE_SHIFT) - 1)
+#define DRM_TTM_MASK_PFN (0xFFFFFFFFU - DRM_TTM_MASK_FLAGS)
+
+/*
+ * Page flags.
+ */
+
+#define DRM_TTM_PAGE_UNCACHED 0x01
+#define DRM_TTM_PAGE_USED     0x02
+#define DRM_TTM_PAGE_BOUND    0x04
+#define DRM_TTM_PAGE_PRESENT  0x08
+#define DRM_TTM_PAGE_VMALLOC  0x10
+
+/***************************************************
+ * Buffer objects. (drm_bo.c, drm_bo_move.c)
+ */
+
+typedef struct drm_bo_mem_reg {
+	drm_mm_node_t *mm_node;
+	unsigned long size;
+	unsigned long num_pages;
+	uint32_t page_alignment;
+	uint32_t mem_type;
+	uint32_t flags;
+	uint32_t mask;
+} drm_bo_mem_reg_t;
+
+typedef struct drm_buffer_object {
+	struct drm_device *dev;
+	drm_user_object_t base;
+
+	/*
+	 * If there is a possibility that the usage variable is zero,
+	 * then dev->struct_mutext should be locked before incrementing it.
+	 */
+
+	atomic_t usage;
+	unsigned long buffer_start;
+	drm_bo_type_t type;
+	unsigned long offset;
+	atomic_t mapped;
+	drm_bo_mem_reg_t mem;
+
+	struct list_head lru;
+	struct list_head ddestroy;
+
+	uint32_t fence_type;
+	uint32_t fence_class;
+	drm_fence_object_t *fence;
+	uint32_t priv_flags;
+	wait_queue_head_t event_queue;
+	struct mutex mutex;
+
+	/* For pinned buffers */
+	drm_mm_node_t *pinned_node;
+	uint32_t pinned_mem_type;
+	struct list_head pinned_lru;
+
+	/* For vm */
+
+	drm_ttm_t *ttm;
+	drm_map_list_t map_list;
+	uint32_t memory_type;
+	unsigned long bus_offset;
+	uint32_t vm_flags;
+	void *iomap;
+
+#ifdef DRM_ODD_MM_COMPAT
+	/* dev->struct_mutex only protected. */
+	struct list_head vma_list;
+	struct list_head p_mm_list;
+#endif
+
+} drm_buffer_object_t;
+
+#define _DRM_BO_FLAG_UNFENCED 0x00000001
+#define _DRM_BO_FLAG_EVICTED  0x00000002
+
+typedef struct drm_mem_type_manager {
+	int has_type;
+	int use_type;
+	drm_mm_t manager;
+	struct list_head lru;
+	struct list_head pinned;
+	uint32_t flags;
+	uint32_t drm_bus_maptype;
+	unsigned long io_offset;
+	unsigned long io_size;
+	void *io_addr;
+} drm_mem_type_manager_t;
+
+#define _DRM_FLAG_MEMTYPE_FIXED     0x00000001	/* Fixed (on-card) PCI memory */
+#define _DRM_FLAG_MEMTYPE_MAPPABLE  0x00000002	/* Memory mappable */
+#define _DRM_FLAG_MEMTYPE_CACHED    0x00000004	/* Cached binding */
+#define _DRM_FLAG_NEEDS_IOREMAP     0x00000008	/* Fixed memory needs ioremap
+						   before kernel access. */
+#define _DRM_FLAG_MEMTYPE_CMA       0x00000010	/* Can't map aperture */
+#define _DRM_FLAG_MEMTYPE_CSELECT   0x00000020	/* Select caching */
+
+typedef struct drm_buffer_manager {
+	struct mutex init_mutex;
+	struct mutex evict_mutex;
+	int nice_mode;
+	int initialized;
+	drm_file_t *last_to_validate;
+	drm_mem_type_manager_t man[DRM_BO_MEM_TYPES];
+	struct list_head unfenced;
+	struct list_head ddestroy;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+	struct work_struct wq;
+#else
+	struct delayed_work wq;
+#endif
+	uint32_t fence_type;
+	unsigned long cur_pages;
+	atomic_t count;
+} drm_buffer_manager_t;
+
+typedef struct drm_bo_driver {
+	const uint32_t *mem_type_prio;
+	const uint32_t *mem_busy_prio;
+	uint32_t num_mem_type_prio;
+	uint32_t num_mem_busy_prio;
+	drm_ttm_backend_t *(*create_ttm_backend_entry)
+	 (struct drm_device * dev);
+	int (*fence_type) (uint32_t flags, uint32_t * class, uint32_t * type);
+	int (*invalidate_caches) (struct drm_device * dev, uint32_t flags);
+	int (*init_mem_type) (struct drm_device * dev, uint32_t type,
+			      drm_mem_type_manager_t * man);
+	 uint32_t(*evict_flags) (struct drm_device * dev, uint32_t type);
+	int (*move) (struct drm_buffer_object * bo,
+		     int evict, int no_wait, struct drm_bo_mem_reg * new_mem);
+} drm_bo_driver_t;
+
+/*
+ * buffer objects (drm_bo.c)
+ */
+
+extern int drm_bo_ioctl(DRM_IOCTL_ARGS);
+extern int drm_mm_init_ioctl(DRM_IOCTL_ARGS);
+extern int drm_bo_driver_finish(struct drm_device *dev);
+extern int drm_bo_driver_init(struct drm_device *dev);
+extern int drm_bo_pci_offset(struct drm_device *dev,
+			     drm_bo_mem_reg_t * mem,
+			     unsigned long *bus_base,
+			     unsigned long *bus_offset,
+			     unsigned long *bus_size);
+extern int drm_mem_reg_is_pci(struct drm_device *dev, drm_bo_mem_reg_t * mem);
+
+extern void drm_bo_usage_deref_locked(drm_buffer_object_t * bo);
+extern int drm_fence_buffer_objects(drm_file_t * priv,
+				    struct list_head *list,
+				    uint32_t fence_flags,
+				    drm_fence_object_t * fence,
+				    drm_fence_object_t ** used_fence);
+extern void drm_bo_add_to_lru(drm_buffer_object_t * bo);
+extern int drm_bo_wait(drm_buffer_object_t * bo, int lazy, int ignore_signals,
+		       int no_wait);
+extern int drm_bo_mem_space(drm_buffer_object_t * bo,
+			    drm_bo_mem_reg_t * mem, int no_wait);
+extern int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_mem_flags,
+			      int no_wait, int move_unfenced);
+
+/*
+ * Buffer object memory move helpers.
+ * drm_bo_move.c
+ */
+
+extern int drm_bo_move_ttm(drm_buffer_object_t * bo,
+			   int evict, int no_wait, drm_bo_mem_reg_t * new_mem);
+extern int drm_bo_move_memcpy(drm_buffer_object_t * bo,
+			      int evict,
+			      int no_wait, drm_bo_mem_reg_t * new_mem);
+extern int drm_bo_move_accel_cleanup(drm_buffer_object_t * bo,
+				     int evict,
+				     int no_wait,
+				     uint32_t fence_type,
+				     uint32_t fence_flags,
+				     drm_bo_mem_reg_t * new_mem);
+
+#endif
diff --git a/linux-core/drm_stub.c b/linux-core/drm_stub.c
index 2dc95aa2..348cd2c6 100644
--- a/linux-core/drm_stub.c
+++ b/linux-core/drm_stub.c
@@ -68,6 +68,7 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev,
 	mutex_init(&dev->struct_mutex);
 	mutex_init(&dev->ctxlist_mutex);
 	mutex_init(&dev->bm.init_mutex);
+	mutex_init(&dev->bm.evict_mutex);
 
 	dev->pdev = pdev;
 	dev->pci_device = pdev->device;
diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c
index c17c41cb..e67719e2 100644
--- a/linux-core/drm_ttm.c
+++ b/linux-core/drm_ttm.c
@@ -1,6 +1,6 @@
 /**************************************************************************
  * 
- * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
+ * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
  * All Rights Reserved.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -11,6 +11,10 @@
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
@@ -18,13 +22,11 @@
  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- * 
  * 
  **************************************************************************/
+/*
+ * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
+ */
 
 #include "drmP.h"
 
@@ -33,18 +35,17 @@ static void drm_ttm_ipi_handler(void *null)
 	flush_agp_cache();
 }
 
-static void drm_ttm_cache_flush(void) 
+static void drm_ttm_cache_flush(void)
 {
 	if (on_each_cpu(drm_ttm_ipi_handler, NULL, 1, 1) != 0)
 		DRM_ERROR("Timed out waiting for drm cache flush.\n");
 }
 
-
 /*
  * Use kmalloc if possible. Otherwise fall back to vmalloc.
  */
 
-static void ttm_alloc_pages(drm_ttm_t *ttm)
+static void ttm_alloc_pages(drm_ttm_t * ttm)
 {
 	unsigned long size = ttm->num_pages * sizeof(*ttm->pages);
 	ttm->pages = NULL;
@@ -65,7 +66,7 @@ static void ttm_alloc_pages(drm_ttm_t *ttm)
 	}
 }
 
-static void ttm_free_pages(drm_ttm_t *ttm)
+static void ttm_free_pages(drm_ttm_t * ttm)
 {
 	unsigned long size = ttm->num_pages * sizeof(*ttm->pages);
 
@@ -79,27 +80,24 @@ static void ttm_free_pages(drm_ttm_t *ttm)
 	ttm->pages = NULL;
 }
 
-/*
- * Unmap all vma pages from vmas mapping this ttm.
- */
-
-static int unmap_vma_pages(drm_ttm_t * ttm)
+static struct page *drm_ttm_alloc_page(void)
 {
-	drm_device_t *dev = ttm->dev;
-	loff_t offset = ((loff_t) ttm->mapping_offset) << PAGE_SHIFT;
-	loff_t holelen = ((loff_t) ttm->num_pages) << PAGE_SHIFT;
+	struct page *page;
 
-#ifdef DRM_ODD_MM_COMPAT
-	int ret;
-	ret = drm_ttm_lock_mm(ttm);
-	if (ret)
-		return ret;
-#endif
-	unmap_mapping_range(dev->dev_mapping, offset, holelen, 1);
-#ifdef DRM_ODD_MM_COMPAT
-	drm_ttm_finish_unmap(ttm);
+	if (drm_alloc_memctl(PAGE_SIZE)) {
+		return NULL;
+	}
+	page = alloc_page(GFP_KERNEL | __GFP_ZERO | GFP_DMA32);
+	if (!page) {
+		drm_free_memctl(PAGE_SIZE);
+		return NULL;
+	}
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15))
+	SetPageLocked(page);
+#else
+	SetPageReserved(page);
 #endif
-	return 0;
+	return page;
 }
 
 /*
@@ -116,7 +114,7 @@ static int drm_set_caching(drm_ttm_t * ttm, int noncached)
 	if ((ttm->page_flags & DRM_TTM_PAGE_UNCACHED) == noncached)
 		return 0;
 
-	if (noncached) 
+	if (noncached)
 		drm_ttm_cache_flush();
 
 	for (i = 0; i < ttm->num_pages; ++i) {
@@ -135,7 +133,7 @@ static int drm_set_caching(drm_ttm_t * ttm, int noncached)
 	if (do_tlbflush)
 		flush_agp_mappings();
 
-	DRM_MASK_VAL(ttm->page_flags, DRM_TTM_PAGE_UNCACHED, noncached);
+	DRM_FLAG_MASKED(ttm->page_flags, noncached, DRM_TTM_PAGE_UNCACHED);
 
 	return 0;
 }
@@ -154,18 +152,6 @@ int drm_destroy_ttm(drm_ttm_t * ttm)
 	if (!ttm)
 		return 0;
 
-	if (atomic_read(&ttm->vma_count) > 0) {
-		ttm->destroy = 1;
-		DRM_ERROR("VMAs are still alive. Skipping destruction.\n");
-		return -EBUSY;
-	}
-
-	DRM_DEBUG("Destroying a ttm\n");
-
-#ifdef DRM_TTM_ODD_COMPAT
-	BUG_ON(!list_empty(&ttm->vma_list));
-	BUG_ON(!list_empty(&ttm->p_mm_list));
-#endif
 	be = ttm->be;
 	if (be) {
 		be->destroy(be);
@@ -193,11 +179,6 @@ int drm_destroy_ttm(drm_ttm_t * ttm)
 					DRM_ERROR("Erroneous map count. "
 						  "Leaking page mappings.\n");
 				}
-
-				/*
-				 * End debugging.
-				 */
-
 				__free_page(*cur_page);
 				drm_free_memctl(PAGE_SIZE);
 				--bm->cur_pages;
@@ -210,37 +191,36 @@ int drm_destroy_ttm(drm_ttm_t * ttm)
 	return 0;
 }
 
+struct page *drm_ttm_get_page(drm_ttm_t * ttm, int index)
+{
+	struct page *p;
+	drm_buffer_manager_t *bm = &ttm->dev->bm;
+
+	p = ttm->pages[index];
+	if (!p) {
+		p = drm_ttm_alloc_page();
+		if (!p)
+			return NULL;
+		ttm->pages[index] = p;
+		++bm->cur_pages;
+	}
+	return p;
+}
+
 static int drm_ttm_populate(drm_ttm_t * ttm)
 {
 	struct page *page;
 	unsigned long i;
-	drm_buffer_manager_t *bm;
 	drm_ttm_backend_t *be;
 
 	if (ttm->state != ttm_unpopulated)
 		return 0;
 
-	bm = &ttm->dev->bm;
 	be = ttm->be;
 	for (i = 0; i < ttm->num_pages; ++i) {
-		page = ttm->pages[i];
-		if (!page) {
-			if (drm_alloc_memctl(PAGE_SIZE)) {
-				return -ENOMEM;
-			}
-			page = alloc_page(GFP_KERNEL | __GFP_ZERO | GFP_DMA32);
-			if (!page) {
-				drm_free_memctl(PAGE_SIZE);
-				return -ENOMEM;
-			}
-#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15))
-			SetPageLocked(page);
-#else
-			SetPageReserved(page);
-#endif
-			ttm->pages[i] = page;
-			++bm->cur_pages;
-		}
+		page = drm_ttm_get_page(ttm, i);
+		if (!page)
+			return -ENOMEM;
 	}
 	be->populate(be, ttm->num_pages, ttm->pages);
 	ttm->state = ttm_unbound;
@@ -251,7 +231,7 @@ static int drm_ttm_populate(drm_ttm_t * ttm)
  * Initialize a ttm.
  */
 
-static drm_ttm_t *drm_init_ttm(struct drm_device *dev, unsigned long size)
+drm_ttm_t *drm_ttm_init(struct drm_device * dev, unsigned long size)
 {
 	drm_bo_driver_t *bo_driver = dev->driver->bo_driver;
 	drm_ttm_t *ttm;
@@ -263,11 +243,6 @@ static drm_ttm_t *drm_init_ttm(struct drm_device *dev, unsigned long size)
 	if (!ttm)
 		return NULL;
 
-#ifdef DRM_ODD_MM_COMPAT
-	INIT_LIST_HEAD(&ttm->p_mm_list);
-	INIT_LIST_HEAD(&ttm->vma_list);
-#endif
-
 	ttm->dev = dev;
 	atomic_set(&ttm->vma_count, 0);
 
@@ -300,29 +275,20 @@ static drm_ttm_t *drm_init_ttm(struct drm_device *dev, unsigned long size)
  * Unbind a ttm region from the aperture.
  */
 
-int drm_evict_ttm(drm_ttm_t * ttm)
+void drm_ttm_evict(drm_ttm_t * ttm)
 {
 	drm_ttm_backend_t *be = ttm->be;
 	int ret;
 
-	switch (ttm->state) {
-	case ttm_bound:
-		if (be->needs_ub_cache_adjust(be)) {
-			ret = unmap_vma_pages(ttm);
-			if (ret) {
-				return ret;
-			}
-		}
-		be->unbind(be);
-		break;
-	default:
-		break;
+	if (ttm->state == ttm_bound) {
+		ret = be->unbind(be);
+		BUG_ON(ret);
 	}
+
 	ttm->state = ttm_evicted;
-	return 0;
 }
 
-void drm_fixup_ttm_caching(drm_ttm_t * ttm)
+void drm_ttm_fixup_caching(drm_ttm_t * ttm)
 {
 
 	if (ttm->state == ttm_evicted) {
@@ -334,18 +300,12 @@ void drm_fixup_ttm_caching(drm_ttm_t * ttm)
 	}
 }
 
-int drm_unbind_ttm(drm_ttm_t * ttm)
+void drm_ttm_unbind(drm_ttm_t * ttm)
 {
-	int ret = 0;
-
 	if (ttm->state == ttm_bound)
-		ret = drm_evict_ttm(ttm);
-
-	if (ret)
-		return ret;
+		drm_ttm_evict(ttm);
 
-	drm_fixup_ttm_caching(ttm);
-	return 0;
+	drm_ttm_fixup_caching(ttm);
 }
 
 int drm_bind_ttm(drm_ttm_t * ttm, int cached, unsigned long aper_offset)
@@ -364,26 +324,13 @@ int drm_bind_ttm(drm_ttm_t * ttm, int cached, unsigned long aper_offset)
 	ret = drm_ttm_populate(ttm);
 	if (ret)
 		return ret;
-	if (ttm->state == ttm_unbound && !cached) {
-		ret = unmap_vma_pages(ttm);
-		if (ret)
-			return ret;
 
+	if (ttm->state == ttm_unbound && !cached) {
 		drm_set_caching(ttm, DRM_TTM_PAGE_UNCACHED);
 	}
-#ifdef DRM_ODD_MM_COMPAT
-	else if (ttm->state == ttm_evicted && !cached) {
-		ret = drm_ttm_lock_mm(ttm);
-		if (ret)
-			return ret;
-	}
-#endif
+
 	if ((ret = be->bind(be, aper_offset, cached))) {
 		ttm->state = ttm_evicted;
-#ifdef DRM_ODD_MM_COMPAT
-		if (be->needs_ub_cache_adjust(be))
-			drm_ttm_unlock_mm(ttm);
-#endif
 		DRM_ERROR("Couldn't bind backend.\n");
 		return ret;
 	}
@@ -391,130 +338,7 @@ int drm_bind_ttm(drm_ttm_t * ttm, int cached, unsigned long aper_offset)
 	ttm->aper_offset = aper_offset;
 	ttm->state = ttm_bound;
 
-#ifdef DRM_ODD_MM_COMPAT
-	if (be->needs_ub_cache_adjust(be)) {
-		ret = drm_ttm_remap_bound(ttm);
-		if (ret)
-			return ret;
-	}
-#endif
-
 	return 0;
 }
 
-/*
- * dev->struct_mutex locked.
- */
-static void drm_ttm_object_remove(drm_device_t * dev, drm_ttm_object_t * object)
-{
-	drm_map_list_t *list = &object->map_list;
-	drm_local_map_t *map;
-
-	if (list->user_token)
-		drm_ht_remove_item(&dev->map_hash, &list->hash);
-
-	if (list->file_offset_node) {
-		drm_mm_put_block(list->file_offset_node);
-		list->file_offset_node = NULL;
-	}
-
-	map = list->map;
-
-	if (map) {
-		drm_ttm_t *ttm = (drm_ttm_t *) map->offset;
-		if (ttm) {
-			if (drm_destroy_ttm(ttm) != -EBUSY) {
-				drm_ctl_free(map, sizeof(*map), DRM_MEM_TTM);
-			}
-		} else {
-			drm_ctl_free(map, sizeof(*map), DRM_MEM_TTM);
-		}
-	}
-
-	drm_ctl_free(object, sizeof(*object), DRM_MEM_TTM);
-}
-
-void drm_ttm_object_deref_locked(drm_device_t * dev, drm_ttm_object_t * to)
-{
-	if (atomic_dec_and_test(&to->usage)) {
-		drm_ttm_object_remove(dev, to);
-	}
-}
-
-void drm_ttm_object_deref_unlocked(drm_device_t * dev, drm_ttm_object_t * to)
-{
-	if (atomic_dec_and_test(&to->usage)) {
-		mutex_lock(&dev->struct_mutex);
-		if (atomic_read(&to->usage) == 0)
-			drm_ttm_object_remove(dev, to);
-		mutex_unlock(&dev->struct_mutex);
-	}
-}
-
-/*
- * Create a ttm and add it to the drm book-keeping. 
- * dev->struct_mutex locked.
- */
-
-int drm_ttm_object_create(drm_device_t * dev, unsigned long size,
-			  uint32_t flags, drm_ttm_object_t ** ttm_object)
-{
-	drm_ttm_object_t *object;
-	drm_map_list_t *list;
-	drm_local_map_t *map;
-	drm_ttm_t *ttm;
-
-	object = drm_ctl_calloc(1, sizeof(*object), DRM_MEM_TTM);
-	if (!object)
-		return -ENOMEM;
-	object->flags = flags;
-	list = &object->map_list;
-
-	list->map = drm_ctl_calloc(1, sizeof(*map), DRM_MEM_TTM);
-	if (!list->map) {
-		drm_ttm_object_remove(dev, object);
-		return -ENOMEM;
-	}
-	map = list->map;
-
-	ttm = drm_init_ttm(dev, size);
-	if (!ttm) {
-		DRM_ERROR("Could not create ttm\n");
-		drm_ttm_object_remove(dev, object);
-		return -ENOMEM;
-	}
-
-	map->offset = (unsigned long)ttm;
-	map->type = _DRM_TTM;
-	map->flags = _DRM_REMOVABLE;
-	map->size = ttm->num_pages * PAGE_SIZE;
-	map->handle = (void *)object;
-
-	/*
-	 * Add a one-page "hole" to the block size to avoid the mm subsystem
-	 * merging vmas.
-	 * FIXME: Is this really needed?
-	 */
-
-	list->file_offset_node = drm_mm_search_free(&dev->offset_manager,
-						    ttm->num_pages + 1, 0, 0);
-	if (!list->file_offset_node) {
-		drm_ttm_object_remove(dev, object);
-		return -ENOMEM;
-	}
-	list->file_offset_node = drm_mm_get_block(list->file_offset_node,
-						  ttm->num_pages + 1, 0);
-
-	list->hash.key = list->file_offset_node->start;
-
-	if (drm_ht_insert_item(&dev->map_hash, &list->hash)) {
-		drm_ttm_object_remove(dev, object);
-		return -ENOMEM;
-	}
-
-	list->user_token = ((drm_u64_t) list->hash.key) << PAGE_SHIFT;
-	ttm->mapping_offset = list->hash.key;
-	atomic_set(&object->usage, 1);
-	*ttm_object = object;
-	return 0;
-}
+EXPORT_SYMBOL(drm_bind_ttm);
diff --git a/linux-core/drm_ttm.h b/linux-core/drm_ttm.h
deleted file mode 100644
index 796f2317..00000000
--- a/linux-core/drm_ttm.h
+++ /dev/null
@@ -1,146 +0,0 @@
-/**************************************************************************
- * 
- * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
- * All Rights Reserved.
- * 
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sub license, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- * 
- * 
- **************************************************************************/
-/*
- * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
- */
-
-#ifndef _DRM_TTM_H
-#define _DRM_TTM_H
-#define DRM_HAS_TTM
-
-/*
- * The backend GART interface. (In our case AGP). Any similar type of device (PCIE?)
- * needs only to implement these functions to be usable with the "TTM" interface.
- * The AGP backend implementation lives in drm_agpsupport.c 
- * basically maps these calls to available functions in agpgart. Each drm device driver gets an
- * additional function pointer that creates these types, 
- * so that the device can choose the correct aperture.
- * (Multiple AGP apertures, etc.) 
- * Most device drivers will let this point to the standard AGP implementation.
- */
-
-#define DRM_BE_FLAG_NEEDS_FREE     0x00000001
-#define DRM_BE_FLAG_BOUND_CACHED   0x00000002
-#define DRM_BE_FLAG_CBA            0x00000004
-
-typedef struct drm_ttm_backend {
-	unsigned long aperture_base;
-	void *private;
-	uint32_t flags;
-	uint32_t drm_map_type;
-	int (*needs_ub_cache_adjust) (struct drm_ttm_backend * backend);
-	int (*populate) (struct drm_ttm_backend * backend,
-			 unsigned long num_pages, struct page ** pages);
-	void (*clear) (struct drm_ttm_backend * backend);
-	int (*bind) (struct drm_ttm_backend * backend,
-		     unsigned long offset, int cached);
-	int (*unbind) (struct drm_ttm_backend * backend);
-	void (*destroy) (struct drm_ttm_backend * backend);
-} drm_ttm_backend_t;
-
-typedef struct drm_ttm {
-	struct page **pages;
-	uint32_t page_flags;
-	unsigned long num_pages;
-	unsigned long aper_offset;
-	atomic_t vma_count;
-	struct drm_device *dev;
-	int destroy;
-	uint32_t mapping_offset;
-	drm_ttm_backend_t *be;
-	enum {
-		ttm_bound,
-		ttm_evicted,
-		ttm_unbound,
-		ttm_unpopulated,
-	} state;
-#ifdef DRM_ODD_MM_COMPAT
-	struct list_head vma_list;
-	struct list_head p_mm_list;
-#endif
-
-} drm_ttm_t;
-
-typedef struct drm_ttm_object {
-	atomic_t usage;
-	uint32_t flags;
-	drm_map_list_t map_list;
-} drm_ttm_object_t;
-
-extern int drm_ttm_object_create(struct drm_device *dev, unsigned long size,
-				 uint32_t flags,
-				 drm_ttm_object_t ** ttm_object);
-extern void drm_ttm_object_deref_locked(struct drm_device *dev,
-					drm_ttm_object_t * to);
-extern void drm_ttm_object_deref_unlocked(struct drm_device *dev,
-					  drm_ttm_object_t * to);
-extern drm_ttm_object_t *drm_lookup_ttm_object(drm_file_t * priv,
-					       uint32_t handle,
-					       int check_owner);
-extern int drm_bind_ttm(drm_ttm_t * ttm, int cached, unsigned long aper_offset);
-
-extern int drm_unbind_ttm(drm_ttm_t * ttm);
-
-/*
- * Evict a ttm region. Keeps Aperture caching policy.
- */
-
-extern int drm_evict_ttm(drm_ttm_t * ttm);
-extern void drm_fixup_ttm_caching(drm_ttm_t * ttm);
-
-/*
- * Destroy a ttm. The user normally calls drmRmMap or a similar IOCTL to do this, 
- * which calls this function iff there are no vmas referencing it anymore. Otherwise it is called
- * when the last vma exits.
- */
-
-extern int drm_destroy_ttm(drm_ttm_t * ttm);
-extern int drm_ttm_ioctl(DRM_IOCTL_ARGS);
-
-static __inline__ drm_ttm_t *drm_ttm_from_object(drm_ttm_object_t * to)
-{
-	return (drm_ttm_t *) to->map_list.map->offset;
-}
-
-#define DRM_MASK_VAL(dest, mask, val)			\
-  (dest) = ((dest) & ~(mask)) | ((val) & (mask));
-
-#define DRM_TTM_MASK_FLAGS ((1 << PAGE_SHIFT) - 1)
-#define DRM_TTM_MASK_PFN (0xFFFFFFFFU - DRM_TTM_MASK_FLAGS)
-
-/*
- * Page flags.
- */
-
-#define DRM_TTM_PAGE_UNCACHED 0x01
-#define DRM_TTM_PAGE_USED     0x02
-#define DRM_TTM_PAGE_BOUND    0x04
-#define DRM_TTM_PAGE_PRESENT  0x08
-#define DRM_TTM_PAGE_VMALLOC  0x10
-
-#endif
diff --git a/linux-core/drm_vm.c b/linux-core/drm_vm.c
index 827a7bdb..f3b1088f 100644
--- a/linux-core/drm_vm.c
+++ b/linux-core/drm_vm.c
@@ -41,9 +41,9 @@
 
 static void drm_vm_open(struct vm_area_struct *vma);
 static void drm_vm_close(struct vm_area_struct *vma);
-static void drm_vm_ttm_close(struct vm_area_struct *vma);
-static int drm_vm_ttm_open(struct vm_area_struct *vma);
-static void drm_vm_ttm_open_wrapper(struct vm_area_struct *vma);
+static int drm_bo_mmap_locked(struct vm_area_struct *vma,
+			      struct file *filp,
+			      drm_local_map_t *map);
 
 
 pgprot_t drm_io_prot(uint32_t map_type, struct vm_area_struct *vma)
@@ -159,96 +159,6 @@ static __inline__ struct page *drm_do_vm_nopage(struct vm_area_struct *vma,
 }
 #endif				/* __OS_HAS_AGP */
 
-#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) || \
-     LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
-#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21))
-static
-#endif
-struct page *drm_vm_ttm_fault(struct vm_area_struct *vma, 
-			      struct fault_data *data)
-{
-	unsigned long address = data->address;
-	drm_local_map_t *map = (drm_local_map_t *) vma->vm_private_data;
-	unsigned long page_offset;
-	struct page *page;
-	drm_ttm_t *ttm; 
-	drm_buffer_manager_t *bm;
-	drm_device_t *dev;
-	unsigned long pfn;
-	int err;
-	pgprot_t pgprot;
-
-	if (!map) {
-		data->type = VM_FAULT_OOM;
-		return NULL;
-	}
-
-	if (address > vma->vm_end) {
-		data->type = VM_FAULT_SIGBUS;
-		return NULL;
-	}
-
-	ttm = (drm_ttm_t *) map->offset;
-	
-	dev = ttm->dev;
-
-	/*
-	 * Perhaps retry here?
-	 */
-
-	mutex_lock(&dev->struct_mutex);
-	drm_fixup_ttm_caching(ttm);
-
-	bm = &dev->bm;
-	page_offset = (address - vma->vm_start) >> PAGE_SHIFT;
-	page = ttm->pages[page_offset];
-
-	if (!page) {
-		if (drm_alloc_memctl(PAGE_SIZE)) {
-			data->type = VM_FAULT_OOM;
-			goto out;
-		}
-		page = ttm->pages[page_offset] = 
-			alloc_page(GFP_KERNEL | __GFP_ZERO | GFP_DMA32);
-		if (!page) {
-			drm_free_memctl(PAGE_SIZE);
-			data->type = VM_FAULT_OOM;
-			goto out;
-		}
-		++bm->cur_pages;
-#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15))
-		SetPageLocked(page);
-#else
-		SetPageReserved(page);
-#endif
-	}
-
-	if (ttm->page_flags & DRM_TTM_PAGE_UNCACHED) {
-
-		/*
-		 * FIXME: Check can't map aperture flag.
-		 */
-
-		pfn = ttm->aper_offset + page_offset + 
-			(ttm->be->aperture_base >> PAGE_SHIFT);
-		pgprot = drm_io_prot(ttm->be->drm_map_type, vma);
-	} else {
-		pfn = page_to_pfn(page);
-		pgprot = vma->vm_page_prot;
-	}
-	
-	err = vm_insert_pfn(vma, address, pfn, pgprot);
-
-	if (!err || err == -EBUSY) 
-		data->type = VM_FAULT_MINOR; 
-	else
-		data->type = VM_FAULT_OOM;
- out:
-	mutex_unlock(&dev->struct_mutex);
-	return NULL;
-}
-#endif
-
 /**
  * \c nopage method for shared virtual memory.
  *
@@ -508,20 +418,6 @@ static struct vm_operations_struct drm_vm_sg_ops = {
 	.close = drm_vm_close,
 };
 
-#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21))
-static struct vm_operations_struct drm_vm_ttm_ops = {
-	.nopage = drm_vm_ttm_nopage,
-	.open = drm_vm_ttm_open_wrapper,
-	.close = drm_vm_ttm_close,
-};
-#else
-static struct vm_operations_struct drm_vm_ttm_ops = {
-	.fault = drm_vm_ttm_fault,
-	.open = drm_vm_ttm_open_wrapper,
-	.close = drm_vm_ttm_close,
-};
-#endif
-
 /**
  * \c open method for shared virtual memory.
  *
@@ -530,7 +426,7 @@ static struct vm_operations_struct drm_vm_ttm_ops = {
  * Create a new drm_vma_entry structure as the \p vma private data entry and
  * add it to drm_device::vmalist.
  */
-static void drm_vm_open(struct vm_area_struct *vma)
+static void drm_vm_open_locked(struct vm_area_struct *vma)
 {
 	drm_file_t *priv = vma->vm_file->private_data;
 	drm_device_t *dev = priv->head->dev;
@@ -542,36 +438,21 @@ static void drm_vm_open(struct vm_area_struct *vma)
 
 	vma_entry = drm_ctl_alloc(sizeof(*vma_entry), DRM_MEM_VMAS);
 	if (vma_entry) {
-		mutex_lock(&dev->struct_mutex);
 		vma_entry->vma = vma;
 		vma_entry->next = dev->vmalist;
 		vma_entry->pid = current->pid;
 		dev->vmalist = vma_entry;
-		mutex_unlock(&dev->struct_mutex);
 	}
 }
 
-static int drm_vm_ttm_open(struct vm_area_struct *vma) {
-  
-	drm_local_map_t *map = (drm_local_map_t *)vma->vm_private_data;
-	drm_ttm_t *ttm;
+static void drm_vm_open(struct vm_area_struct *vma)
+{
 	drm_file_t *priv = vma->vm_file->private_data;
 	drm_device_t *dev = priv->head->dev;
 
-	drm_vm_open(vma);
 	mutex_lock(&dev->struct_mutex);
-	ttm = (drm_ttm_t *) map->offset;
-	atomic_inc(&ttm->vma_count);
-#ifdef DRM_ODD_MM_COMPAT
-	drm_ttm_add_vma(ttm, vma);
-#endif
+	drm_vm_open_locked(vma);
 	mutex_unlock(&dev->struct_mutex);
-	return 0;
-}
-
-static void drm_vm_ttm_open_wrapper(struct vm_area_struct *vma) 
-{
-	drm_vm_ttm_open(vma);
 }
 
 /**
@@ -608,34 +489,6 @@ static void drm_vm_close(struct vm_area_struct *vma)
 }
 
 
-static void drm_vm_ttm_close(struct vm_area_struct *vma)
-{
-	drm_local_map_t *map = (drm_local_map_t *) vma->vm_private_data; 
-	drm_ttm_t *ttm; 
-        drm_device_t *dev;
-	int ret;
-
-	drm_vm_close(vma); 
-	if (map) {
-		ttm = (drm_ttm_t *) map->offset;
-		dev = ttm->dev;
-		mutex_lock(&dev->struct_mutex);
-#ifdef DRM_ODD_MM_COMPAT
-		drm_ttm_delete_vma(ttm, vma);
-#endif
-		if (atomic_dec_and_test(&ttm->vma_count)) {
-			if (ttm->destroy) {
-				ret = drm_destroy_ttm(ttm);
-				BUG_ON(ret);
-				drm_ctl_free(map, sizeof(*map), DRM_MEM_TTM);
-			}
-		}
-		mutex_unlock(&dev->struct_mutex);
-	}
-	return;
-}
-
-
 /**
  * mmap DMA memory.
  *
@@ -653,7 +506,6 @@ static int drm_mmap_dma(struct file *filp, struct vm_area_struct *vma)
 	drm_device_dma_t *dma;
 	unsigned long length = vma->vm_end - vma->vm_start;
 
-	lock_kernel();
 	dev = priv->head->dev;
 	dma = dev->dma;
 	DRM_DEBUG("start = 0x%lx, end = 0x%lx, page offset = 0x%lx\n",
@@ -661,10 +513,8 @@ static int drm_mmap_dma(struct file *filp, struct vm_area_struct *vma)
 
 	/* Length must match exact page count */
 	if (!dma || (length >> PAGE_SHIFT) != dma->page_count) {
-		unlock_kernel();
 		return -EINVAL;
 	}
-	unlock_kernel();
 
 	if (!capable(CAP_SYS_ADMIN) &&
 	    (dma->flags & _DRM_DMA_USE_PCI_RO)) {
@@ -686,7 +536,7 @@ static int drm_mmap_dma(struct file *filp, struct vm_area_struct *vma)
 	vma->vm_flags |= VM_RESERVED;	/* Don't swap */
 
 	vma->vm_file = filp;	/* Needed for drm_vm_open() */
-	drm_vm_open(vma);
+	drm_vm_open_locked(vma);
 	return 0;
 }
 
@@ -719,7 +569,7 @@ EXPORT_SYMBOL(drm_core_get_reg_ofs);
  * according to the mapping type and remaps the pages. Finally sets the file
  * pointer and calls vm_open().
  */
-int drm_mmap(struct file *filp, struct vm_area_struct *vma)
+static int drm_mmap_locked(struct file *filp, struct vm_area_struct *vma)
 {
 	drm_file_t *priv = filp->private_data;
 	drm_device_t *dev = priv->head->dev;
@@ -737,6 +587,7 @@ int drm_mmap(struct file *filp, struct vm_area_struct *vma)
 	 * the AGP mapped at physical address 0
 	 * --BenH.
 	 */
+
 	if (!vma->vm_pgoff
 #if __OS_HAS_AGP
 	    && (!dev->agp
@@ -833,27 +684,254 @@ int drm_mmap(struct file *filp, struct vm_area_struct *vma)
 		vma->vm_private_data = (void *)map;
 		vma->vm_flags |= VM_RESERVED;
 		break;
-	case _DRM_TTM: {
-		vma->vm_ops = &drm_vm_ttm_ops;
-		vma->vm_private_data = (void *) map;
-		vma->vm_file = filp;
-		vma->vm_flags |= VM_RESERVED | VM_IO;
-#ifdef DRM_ODD_MM_COMPAT
-		mutex_lock(&dev->struct_mutex);
-		drm_ttm_map_bound(vma);
-		mutex_unlock(&dev->struct_mutex);
-#endif		
-		if (drm_vm_ttm_open(vma))
-		        return -EAGAIN;
-		return 0;
-	}
+	case _DRM_TTM: 
+		return drm_bo_mmap_locked(vma, filp, map);
 	default:
 		return -EINVAL;	/* This should never happen. */
 	}
 	vma->vm_flags |= VM_RESERVED;	/* Don't swap */
 
 	vma->vm_file = filp;	/* Needed for drm_vm_open() */
-	drm_vm_open(vma);
+	drm_vm_open_locked(vma);
 	return 0;
 }
+
+int drm_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	drm_file_t *priv = filp->private_data;
+	drm_device_t *dev = priv->head->dev;
+	int ret;
+
+	mutex_lock(&dev->struct_mutex);
+	ret = drm_mmap_locked(filp, vma);
+	mutex_unlock(&dev->struct_mutex);
+
+	return ret;
+}
 EXPORT_SYMBOL(drm_mmap);
+
+/**
+ * buffer object vm functions.
+ */
+
+/**
+ * \c Pagefault method for buffer objects.
+ *
+ * \param vma Virtual memory area.
+ * \param data Fault data on failure or refault.
+ * \return Always NULL as we insert pfns directly.
+ *
+ * It's important that pfns are inserted while holding the bo->mutex lock.
+ * otherwise we might race with unmap_mapping_range() which is always
+ * called with the bo->mutex lock held.
+ *
+ * It's not pretty to modify the vma->vm_page_prot variable while not
+ * holding the mm semaphore in write mode. However, we have it i read mode,
+ * so we won't be racing with any other writers, and we only actually modify
+ * it when no ptes are present so it shouldn't be a big deal.
+ */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19) ||	\
+     LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
+#ifdef DRM_FULL_MM_COMPAT
+static
+#endif
+struct page *drm_bo_vm_fault(struct vm_area_struct *vma, 
+			     struct fault_data *data)
+{
+	unsigned long address = data->address;
+	drm_buffer_object_t *bo = (drm_buffer_object_t *) vma->vm_private_data;
+	unsigned long page_offset;
+	struct page *page = NULL;
+	drm_ttm_t *ttm; 
+	drm_device_t *dev;
+	unsigned long pfn;
+	int err;
+	unsigned long bus_base;
+	unsigned long bus_offset;
+	unsigned long bus_size;
+	
+
+	mutex_lock(&bo->mutex);
+
+	err = drm_bo_wait(bo, 0, 0, 0);
+	if (err) {
+		data->type = (err == -EAGAIN) ? 
+			VM_FAULT_MINOR : VM_FAULT_SIGBUS;
+		goto out_unlock;
+	}
+	
+	
+	/*
+	 * If buffer happens to be in a non-mappable location,
+	 * move it to a mappable.
+	 */
+
+#ifdef DRM_BO_FULL_COMPAT
+	if (!(bo->mem.flags & DRM_BO_FLAG_MAPPABLE)) {
+		uint32_t new_mask = bo->mem.mask | 
+			DRM_BO_FLAG_MAPPABLE | 
+			DRM_BO_FLAG_FORCE_MAPPABLE;
+		err = drm_bo_move_buffer(bo, new_mask, 0, 0);
+		
+		if (err) {
+			data->type = (err == -EAGAIN) ? 
+				VM_FAULT_MINOR : VM_FAULT_SIGBUS;
+			goto out_unlock;
+		}
+	}
+#else
+	if (!(bo->mem.flags & DRM_BO_FLAG_MAPPABLE)) {
+		unsigned long _end = jiffies + 3*DRM_HZ;
+		uint32_t new_mask = bo->mem.mask |
+			DRM_BO_FLAG_MAPPABLE |
+			DRM_BO_FLAG_FORCE_MAPPABLE;
+
+		do {
+			err = drm_bo_move_buffer(bo, new_mask, 0, 0);
+		} while((err == -EAGAIN) && !time_after_eq(jiffies, _end));
+
+		if (err) {
+			DRM_ERROR("Timeout moving buffer to mappable location.\n");
+			data->type = VM_FAULT_SIGBUS;
+			goto out_unlock;
+		}
+	}
+#endif
+
+	if (address > vma->vm_end) {
+		data->type = VM_FAULT_SIGBUS;
+		goto out_unlock;
+	}
+
+	dev = bo->dev;
+	err = drm_bo_pci_offset(dev, &bo->mem, &bus_base, &bus_offset, 
+				&bus_size);
+
+	if (err) {
+		data->type = VM_FAULT_SIGBUS;
+		goto out_unlock;
+	}
+
+	page_offset = (address - vma->vm_start) >> PAGE_SHIFT;
+
+	if (bus_size) {
+		drm_mem_type_manager_t *man = &dev->bm.man[bo->mem.mem_type];
+
+		pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) + page_offset;
+		vma->vm_page_prot = drm_io_prot(man->drm_bus_maptype, vma);
+	} else {
+		ttm = bo->ttm;
+
+		drm_ttm_fixup_caching(ttm);
+		page = drm_ttm_get_page(ttm, page_offset);
+		if (!page) {
+			data->type = VM_FAULT_OOM;
+			goto out_unlock;
+		}
+		pfn = page_to_pfn(page);
+		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	}
+	
+	err = vm_insert_pfn(vma, address, pfn);
+
+	if (!err || err == -EBUSY) 
+		data->type = VM_FAULT_MINOR; 
+	else
+		data->type = VM_FAULT_OOM;
+out_unlock:
+	mutex_unlock(&bo->mutex);
+	return NULL;
+}
+#endif
+
+static void drm_bo_vm_open_locked(struct vm_area_struct *vma)
+{
+	drm_buffer_object_t *bo = (drm_buffer_object_t *) vma->vm_private_data;
+
+	drm_vm_open_locked(vma);
+	atomic_inc(&bo->usage);
+#ifdef DRM_ODD_MM_COMPAT
+	drm_bo_add_vma(bo, vma);
+#endif
+}
+
+/**
+ * \c vma open method for buffer objects.
+ *
+ * \param vma virtual memory area.
+ */
+
+static void drm_bo_vm_open(struct vm_area_struct *vma)
+{
+	drm_buffer_object_t *bo = (drm_buffer_object_t *) vma->vm_private_data;
+	drm_device_t *dev = bo->dev;
+
+	mutex_lock(&dev->struct_mutex);
+	drm_bo_vm_open_locked(vma);
+	mutex_unlock(&dev->struct_mutex);
+}
+
+/**
+ * \c vma close method for buffer objects.
+ *
+ * \param vma virtual memory area.
+ */
+
+static void drm_bo_vm_close(struct vm_area_struct *vma)
+{
+	drm_buffer_object_t *bo = (drm_buffer_object_t *) vma->vm_private_data;
+	drm_device_t *dev = bo->dev;
+
+	drm_vm_close(vma);
+	if (bo) {
+		mutex_lock(&dev->struct_mutex);
+#ifdef DRM_ODD_MM_COMPAT
+		drm_bo_delete_vma(bo, vma);
+#endif
+		drm_bo_usage_deref_locked(bo);
+		mutex_unlock(&dev->struct_mutex);
+	}
+	return;
+}
+
+static struct vm_operations_struct drm_bo_vm_ops = {
+#ifdef DRM_FULL_MM_COMPAT
+	.fault = drm_bo_vm_fault,
+#else
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19))
+	.nopfn = drm_bo_vm_nopfn,
+#else
+	.nopage = drm_bo_vm_nopage,
+#endif
+#endif
+	.open = drm_bo_vm_open,
+	.close = drm_bo_vm_close,
+};
+
+/**
+ * mmap buffer object memory.
+ *
+ * \param vma virtual memory area.
+ * \param filp file pointer.
+ * \param map The buffer object drm map.
+ * \return zero on success or a negative number on failure.
+ */
+
+int drm_bo_mmap_locked(struct vm_area_struct *vma,
+		       struct file *filp,
+		       drm_local_map_t *map)
+{
+	vma->vm_ops = &drm_bo_vm_ops;
+	vma->vm_private_data = map->handle;
+	vma->vm_file = filp;
+	vma->vm_flags |= VM_RESERVED | VM_IO;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19))
+	vma->vm_flags |= VM_PFNMAP;
+#endif
+	drm_bo_vm_open_locked(vma);
+#ifdef DRM_ODD_MM_COMPAT
+	drm_bo_map_bound(vma);
+#endif		
+	return 0;
+}
diff --git a/linux-core/i915_buffer.c b/linux-core/i915_buffer.c
index c3e54468..374b28df 100644
--- a/linux-core/i915_buffer.c
+++ b/linux-core/i915_buffer.c
@@ -33,7 +33,6 @@
 #include "i915_drm.h"
 #include "i915_drv.h"
 
-
 drm_ttm_backend_t *i915_create_ttm_backend_entry(drm_device_t * dev)
 {
 	return drm_agp_init_ttm(dev, NULL);
@@ -64,3 +63,173 @@ int i915_invalidate_caches(drm_device_t * dev, uint32_t flags)
 
 	return i915_emit_mi_flush(dev, flush_cmd);
 }
+
+int i915_init_mem_type(drm_device_t * dev, uint32_t type,
+		       drm_mem_type_manager_t * man)
+{
+	switch (type) {
+	case DRM_BO_MEM_LOCAL:
+		man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE |
+		    _DRM_FLAG_MEMTYPE_CACHED;
+		man->drm_bus_maptype = 0;
+		break;
+	case DRM_BO_MEM_TT:
+		if (!(drm_core_has_AGP(dev) && dev->agp)) {
+			DRM_ERROR("AGP is not enabled for memory type %u\n",
+				  (unsigned)type);
+			return -EINVAL;
+		}
+		man->io_offset = dev->agp->agp_info.aper_base;
+		man->io_size = dev->agp->agp_info.aper_size * 1024 * 1024;
+		man->io_addr = NULL;
+		man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE |
+		    _DRM_FLAG_MEMTYPE_CSELECT | _DRM_FLAG_NEEDS_IOREMAP;
+		man->drm_bus_maptype = _DRM_AGP;
+		break;
+	case DRM_BO_MEM_PRIV0:
+		if (!(drm_core_has_AGP(dev) && dev->agp)) {
+			DRM_ERROR("AGP is not enabled for memory type %u\n",
+				  (unsigned)type);
+			return -EINVAL;
+		}
+		man->io_offset = dev->agp->agp_info.aper_base;
+		man->io_size = dev->agp->agp_info.aper_size * 1024 * 1024;
+		man->io_addr = NULL;
+		man->flags =  _DRM_FLAG_MEMTYPE_MAPPABLE |
+		    _DRM_FLAG_MEMTYPE_FIXED | _DRM_FLAG_NEEDS_IOREMAP;
+		man->drm_bus_maptype = _DRM_AGP;
+		break;
+	default:
+		DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+uint32_t i915_evict_flags(drm_device_t * dev, uint32_t type)
+{
+	switch (type) {
+	case DRM_BO_MEM_LOCAL:
+	case DRM_BO_MEM_TT:
+		return DRM_BO_FLAG_MEM_LOCAL;
+	default:
+		return DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_CACHED;
+	}
+}
+
+static void i915_emit_copy_blit(drm_device_t * dev,
+				uint32_t src_offset,
+				uint32_t dst_offset,
+				uint32_t pages, int direction)
+{
+	uint32_t cur_pages;
+	uint32_t stride = PAGE_SIZE;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	RING_LOCALS;
+
+	if (!dev_priv)
+		return;
+
+	i915_kernel_lost_context(dev);
+	while (pages > 0) {
+		cur_pages = pages;
+		if (cur_pages > 2048)
+			cur_pages = 2048;
+		pages -= cur_pages;
+
+		BEGIN_LP_RING(6);
+		OUT_RING(SRC_COPY_BLT_CMD | XY_SRC_COPY_BLT_WRITE_ALPHA |
+			 XY_SRC_COPY_BLT_WRITE_RGB);
+		OUT_RING((stride & 0xffff) | (0xcc << 16) | (1 << 24) |
+			 (1 << 25) | (direction ? (1 << 30) : 0));
+		OUT_RING((cur_pages << 16) | PAGE_SIZE);
+		OUT_RING(dst_offset);
+		OUT_RING(stride & 0xffff);
+		OUT_RING(src_offset);
+		ADVANCE_LP_RING();
+	}
+	return;
+}
+
+static int i915_move_blit(drm_buffer_object_t * bo,
+			  int evict, int no_wait, drm_bo_mem_reg_t * new_mem)
+{
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+	int dir = 0;
+
+	if ((old_mem->mem_type == new_mem->mem_type) &&
+	    (new_mem->mm_node->start <
+	     old_mem->mm_node->start + old_mem->mm_node->size)) {
+		dir = 1;
+	}
+
+	i915_emit_copy_blit(bo->dev,
+			    old_mem->mm_node->start << PAGE_SHIFT,
+			    new_mem->mm_node->start << PAGE_SHIFT,
+			    new_mem->num_pages, dir);
+
+	i915_emit_mi_flush(bo->dev, MI_READ_FLUSH | MI_EXE_FLUSH);
+
+	return drm_bo_move_accel_cleanup(bo, evict, no_wait,
+					 DRM_FENCE_TYPE_EXE |
+					 DRM_I915_FENCE_TYPE_RW,
+					 DRM_I915_FENCE_FLAG_FLUSHED, new_mem);
+}
+
+/*
+ * Flip destination ttm into cached-coherent AGP, 
+ * then blit and subsequently move out again.
+ */
+
+static int i915_move_flip(drm_buffer_object_t * bo,
+			  int evict, int no_wait, drm_bo_mem_reg_t * new_mem)
+{
+	drm_device_t *dev = bo->dev;
+	drm_bo_mem_reg_t tmp_mem;
+	int ret;
+
+	tmp_mem = *new_mem;
+	tmp_mem.mm_node = NULL;
+	tmp_mem.mask = DRM_BO_FLAG_MEM_TT |
+	    DRM_BO_FLAG_CACHED | DRM_BO_FLAG_FORCE_CACHING;
+
+	ret = drm_bo_mem_space(bo, &tmp_mem, no_wait);
+	if (ret)
+		return ret;
+
+	ret = drm_bind_ttm(bo->ttm, 1, tmp_mem.mm_node->start);
+	if (ret)
+		goto out_cleanup;
+
+	ret = i915_move_blit(bo, 1, no_wait, &tmp_mem);
+	if (ret)
+		goto out_cleanup;
+
+	ret = drm_bo_move_ttm(bo, evict, no_wait, new_mem);
+out_cleanup:
+	if (tmp_mem.mm_node) {
+		mutex_lock(&dev->struct_mutex);
+		if (tmp_mem.mm_node != bo->pinned_node)
+			drm_mm_put_block(tmp_mem.mm_node);
+		tmp_mem.mm_node = NULL;
+		mutex_unlock(&dev->struct_mutex);
+	}
+	return ret;
+}
+
+int i915_move(drm_buffer_object_t * bo,
+	      int evict, int no_wait, drm_bo_mem_reg_t * new_mem)
+{
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+
+	if (old_mem->mem_type == DRM_BO_MEM_LOCAL) {
+		return drm_bo_move_memcpy(bo, evict, no_wait, new_mem);
+	} else if (new_mem->mem_type == DRM_BO_MEM_LOCAL) {
+		if (i915_move_flip(bo, evict, no_wait, new_mem))
+			return drm_bo_move_memcpy(bo, evict, no_wait, new_mem);
+	} else {
+		if (i915_move_blit(bo, evict, no_wait, new_mem))
+			return drm_bo_move_memcpy(bo, evict, no_wait, new_mem);
+	}
+	return 0;
+}
diff --git a/linux-core/i915_drv.c b/linux-core/i915_drv.c
index 2c5b43d0..b7bf883c 100644
--- a/linux-core/i915_drv.c
+++ b/linux-core/i915_drv.c
@@ -47,15 +47,25 @@ static drm_fence_driver_t i915_fence_driver = {
 	.lazy_capable = 1,
 	.emit = i915_fence_emit_sequence,
 	.poke_flush = i915_poke_flush,
+	.has_irq = i915_fence_has_irq,
 };
 #endif
 #ifdef I915_HAVE_BUFFER
+
+static uint32_t i915_mem_prios[] = {DRM_BO_MEM_PRIV0, DRM_BO_MEM_TT, DRM_BO_MEM_LOCAL};
+static uint32_t i915_busy_prios[] = {DRM_BO_MEM_TT, DRM_BO_MEM_PRIV0, DRM_BO_MEM_LOCAL};
+
 static drm_bo_driver_t i915_bo_driver = {
-        .iomap = {NULL, NULL},
-	.cached = {1, 1},
+	.mem_type_prio = i915_mem_prios,
+	.mem_busy_prio = i915_busy_prios,
+	.num_mem_type_prio = sizeof(i915_mem_prios)/sizeof(uint32_t),
+	.num_mem_busy_prio = sizeof(i915_busy_prios)/sizeof(uint32_t),
 	.create_ttm_backend_entry = i915_create_ttm_backend_entry,
 	.fence_type = i915_fence_types,
-	.invalidate_caches = i915_invalidate_caches
+	.invalidate_caches = i915_invalidate_caches,
+	.init_mem_type = i915_init_mem_type,
+	.evict_flags = i915_evict_flags,
+	.move = i915_move,
 };
 #endif
 
diff --git a/linux-core/i915_fence.c b/linux-core/i915_fence.c
index 2182604c..fa2a3d1f 100644
--- a/linux-core/i915_fence.c
+++ b/linux-core/i915_fence.c
@@ -61,7 +61,7 @@ static void i915_perform_flush(drm_device_t * dev)
 
 		diff = sequence - fm->last_exe_flush;
 		if (diff < driver->wrap_diff && diff != 0) {
-			drm_fence_handler(dev, sequence, DRM_FENCE_TYPE_EXE);
+			drm_fence_handler(dev, 0, sequence, DRM_FENCE_TYPE_EXE);
 		}
 
 		diff = sequence - fm->exe_flush_sequence;
@@ -84,7 +84,7 @@ static void i915_perform_flush(drm_device_t * dev)
 			flush_flags = dev_priv->flush_flags;
 			flush_sequence = dev_priv->flush_sequence;
 			dev_priv->flush_pending = 0;
-			drm_fence_handler(dev, flush_sequence, flush_flags);
+			drm_fence_handler(dev, 0, flush_sequence, flush_flags);
 		}
 	}
 
@@ -104,13 +104,13 @@ static void i915_perform_flush(drm_device_t * dev)
 			flush_flags = dev_priv->flush_flags;
 			flush_sequence = dev_priv->flush_sequence;
 			dev_priv->flush_pending = 0;
-			drm_fence_handler(dev, flush_sequence, flush_flags);
+			drm_fence_handler(dev, 0, flush_sequence, flush_flags);
 		}
 	}
 
 }
 
-void i915_poke_flush(drm_device_t * dev)
+void i915_poke_flush(drm_device_t * dev, uint32_t class)
 {
 	drm_fence_manager_t *fm = &dev->fm;
 	unsigned long flags;
@@ -120,7 +120,7 @@ void i915_poke_flush(drm_device_t * dev)
 	write_unlock_irqrestore(&fm->lock, flags);
 }
 
-int i915_fence_emit_sequence(drm_device_t * dev, uint32_t flags,
+int i915_fence_emit_sequence(drm_device_t * dev, uint32_t class, uint32_t flags,
 			     uint32_t * sequence, uint32_t * native_type)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
@@ -144,3 +144,15 @@ void i915_fence_handler(drm_device_t * dev)
 	i915_perform_flush(dev);
 	write_unlock(&fm->lock);
 }
+
+int i915_fence_has_irq(drm_device_t *dev, uint32_t class, uint32_t flags)
+{
+	/*
+	 * We have an irq that tells us when we have a new breadcrumb.
+	 */
+
+	if (class == 0 && flags == DRM_FENCE_TYPE_EXE)
+		return 1;
+
+	return 0;
+}