1 files changed, 410 insertions, 0 deletions
diff --git a/linux-core/drm_bo_move.c b/linux-core/drm_bo_move.c
new file mode 100644
index 00000000..21f5f6cc
--- /dev/null
+++ b/linux-core/drm_bo_move.c
@@ -0,0 +1,410 @@
+/**************************************************************************
+ * 
+ * Copyright (c) 2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
+ * All Rights Reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
+ */
+
+#include "drmP.h"
+
+/**
+ * Free the old memory node unless it's a pinned region and we
+ * have not been requested to free also pinned regions.
+ */
+
+static void drm_bo_free_old_node(drm_buffer_object_t * bo)
+{
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+
+	if (old_mem->mm_node && (old_mem->mm_node != bo->pinned_node)) {
+		mutex_lock(&bo->dev->struct_mutex);
+		drm_mm_put_block(old_mem->mm_node);
+		old_mem->mm_node = NULL;
+		mutex_unlock(&bo->dev->struct_mutex);
+	}
+	old_mem->mm_node = NULL;
+}
+
+int drm_bo_move_ttm(drm_buffer_object_t * bo,
+		    int evict, int no_wait, drm_bo_mem_reg_t * new_mem)
+{
+	drm_ttm_t *ttm = bo->ttm;
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+	uint32_t save_flags = old_mem->flags;
+	uint32_t save_mask = old_mem->mask;
+	int ret;
+
+	if (old_mem->mem_type == DRM_BO_MEM_TT) {
+		if (evict)
+			drm_ttm_evict(ttm);
+		else
+			drm_ttm_unbind(ttm);
+
+		drm_bo_free_old_node(bo);
+		DRM_FLAG_MASKED(old_mem->flags,
+				DRM_BO_FLAG_CACHED | DRM_BO_FLAG_MAPPABLE |
+				DRM_BO_FLAG_MEM_LOCAL, DRM_BO_MASK_MEMTYPE);
+		old_mem->mem_type = DRM_BO_MEM_LOCAL;
+		save_flags = old_mem->flags;
+	}
+	if (new_mem->mem_type != DRM_BO_MEM_LOCAL) {
+		ret = drm_bind_ttm(ttm,
+				   new_mem->flags & DRM_BO_FLAG_CACHED,
+				   new_mem->mm_node->start);
+		if (ret)
+			return ret;
+	}
+
+	*old_mem = *new_mem;
+	new_mem->mm_node = NULL;
+	old_mem->mask = save_mask;
+	DRM_FLAG_MASKED(save_flags, new_mem->flags, DRM_BO_MASK_MEMTYPE);
+	return 0;
+}
+
+EXPORT_SYMBOL(drm_bo_move_ttm);
+
+/**
+ * \c Return a kernel virtual address to the buffer object PCI memory.
+ *
+ * \param bo The buffer object.
+ * \return Failure indication.
+ * 
+ * Returns -EINVAL if the buffer object is currently not mappable.
+ * Returns -ENOMEM if the ioremap operation failed.
+ * Otherwise returns zero.
+ * 
+ * After a successfull call, bo->iomap contains the virtual address, or NULL
+ * if the buffer object content is not accessible through PCI space. 
+ * Call bo->mutex locked.
+ */
+
+int drm_mem_reg_ioremap(drm_device_t * dev, drm_bo_mem_reg_t * mem,
+			void **virtual)
+{
+	drm_buffer_manager_t *bm = &dev->bm;
+	drm_mem_type_manager_t *man = &bm->man[mem->mem_type];
+	unsigned long bus_offset;
+	unsigned long bus_size;
+	unsigned long bus_base;
+	int ret;
+	void *addr;
+
+	*virtual = NULL;
+	ret = drm_bo_pci_offset(dev, mem, &bus_base, &bus_offset, &bus_size);
+	if (ret || bus_size == 0)
+		return ret;
+
+	if (!(man->flags & _DRM_FLAG_NEEDS_IOREMAP))
+		addr = (void *)(((u8 *) man->io_addr) + bus_offset);
+	else {
+		addr = ioremap_nocache(bus_base + bus_offset, bus_size);
+		if (!addr)
+			return -ENOMEM;
+	}
+	*virtual = addr;
+	return 0;
+}
+
+/**
+ * \c Unmap mapping obtained using drm_bo_ioremap
+ *
+ * \param bo The buffer object.
+ *
+ * Call bo->mutex locked.
+ */
+
+void drm_mem_reg_iounmap(drm_device_t * dev, drm_bo_mem_reg_t * mem,
+			 void *virtual)
+{
+	drm_buffer_manager_t *bm;
+	drm_mem_type_manager_t *man;
+
+	bm = &dev->bm;
+	man = &bm->man[mem->mem_type];
+
+	if (virtual && (man->flags & _DRM_FLAG_NEEDS_IOREMAP)) {
+		iounmap(virtual);
+	}
+}
+
+static int drm_copy_io_page(void *dst, void *src, unsigned long page)
+{
+	uint32_t *dstP =
+	    (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
+	uint32_t *srcP =
+	    (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
+
+	int i;
+	for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
+		iowrite32(ioread32(srcP++), dstP++);
+	return 0;
+}
+
+static int drm_copy_io_ttm_page(drm_ttm_t * ttm, void *src, unsigned long page)
+{
+	struct page *d = drm_ttm_get_page(ttm, page);
+	void *dst;
+
+	if (!d)
+		return -ENOMEM;
+
+	src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
+	dst = kmap(d);
+	if (!dst)
+		return -ENOMEM;
+
+	memcpy_fromio(dst, src, PAGE_SIZE);
+	kunmap(d);
+	return 0;
+}
+
+static int drm_copy_ttm_io_page(drm_ttm_t * ttm, void *dst, unsigned long page)
+{
+	struct page *s = drm_ttm_get_page(ttm, page);
+	void *src;
+
+	if (!s)
+		return -ENOMEM;
+
+	dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
+	src = kmap(s);
+	if (!src)
+		return -ENOMEM;
+
+	memcpy_toio(dst, src, PAGE_SIZE);
+	kunmap(s);
+	return 0;
+}
+
+int drm_bo_move_memcpy(drm_buffer_object_t * bo,
+		       int evict, int no_wait, drm_bo_mem_reg_t * new_mem)
+{
+	drm_device_t *dev = bo->dev;
+	drm_mem_type_manager_t *man = &dev->bm.man[new_mem->mem_type];
+	drm_ttm_t *ttm = bo->ttm;
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+	drm_bo_mem_reg_t old_copy = *old_mem;
+	void *old_iomap;
+	void *new_iomap;
+	int ret;
+	uint32_t save_flags = old_mem->flags;
+	uint32_t save_mask = old_mem->mask;
+	unsigned long i;
+	unsigned long page;
+	unsigned long add = 0;
+	int dir;
+
+	ret = drm_mem_reg_ioremap(dev, old_mem, &old_iomap);
+	if (ret)
+		return ret;
+	ret = drm_mem_reg_ioremap(dev, new_mem, &new_iomap);
+	if (ret)
+		goto out;
+
+	if (old_iomap == NULL && new_iomap == NULL)
+		goto out2;
+	if (old_iomap == NULL && ttm == NULL)
+		goto out2;
+
+	add = 0;
+	dir = 1;
+
+	if ((old_mem->mem_type == new_mem->mem_type) &&
+	    (new_mem->mm_node->start <
+	     old_mem->mm_node->start + old_mem->mm_node->size)) {
+		dir = -1;
+		add = new_mem->num_pages - 1;
+	}
+
+	for (i = 0; i < new_mem->num_pages; ++i) {
+		page = i * dir + add;
+		if (old_iomap == NULL)
+			ret = drm_copy_ttm_io_page(ttm, new_iomap, page);
+		else if (new_iomap == NULL)
+			ret = drm_copy_io_ttm_page(ttm, old_iomap, page);
+		else
+			ret = drm_copy_io_page(new_iomap, old_iomap, page);
+		if (ret)
+			goto out1;
+	}
+	mb();
+      out2:
+	drm_bo_free_old_node(bo);
+
+	*old_mem = *new_mem;
+	new_mem->mm_node = NULL;
+	old_mem->mask = save_mask;
+	DRM_FLAG_MASKED(save_flags, new_mem->flags, DRM_BO_MASK_MEMTYPE);
+
+	if ((man->flags & _DRM_FLAG_MEMTYPE_FIXED) && (ttm != NULL)) {
+		drm_ttm_unbind(ttm);
+		drm_destroy_ttm(ttm);
+		bo->ttm = NULL;
+	}
+
+      out1:
+	drm_mem_reg_iounmap(dev, new_mem, new_iomap);
+      out:
+	drm_mem_reg_iounmap(dev, &old_copy, old_iomap);
+	return ret;
+}
+
+EXPORT_SYMBOL(drm_bo_move_memcpy);
+
+/*
+ * Transfer a buffer object's memory and LRU status to a newly
+ * created object. User-space references remains with the old
+ * object. Call bo->mutex locked.
+ */
+
+int drm_buffer_object_transfer(drm_buffer_object_t * bo,
+			       drm_buffer_object_t ** new_obj)
+{
+	drm_buffer_object_t *fbo;
+	drm_device_t *dev = bo->dev;
+	drm_buffer_manager_t *bm = &dev->bm;
+
+	fbo = drm_ctl_calloc(1, sizeof(*fbo), DRM_MEM_BUFOBJ);
+	if (!fbo)
+		return -ENOMEM;
+
+	*fbo = *bo;
+	mutex_init(&fbo->mutex);
+	mutex_lock(&fbo->mutex);
+	mutex_lock(&dev->struct_mutex);
+
+	DRM_INIT_WAITQUEUE(&bo->event_queue);
+	INIT_LIST_HEAD(&fbo->ddestroy);
+	INIT_LIST_HEAD(&fbo->lru);
+	INIT_LIST_HEAD(&fbo->pinned_lru);
+#ifdef DRM_ODD_MM_COMPAT
+	INIT_LIST_HEAD(&fbo->vma_list);
+	INIT_LIST_HEAD(&fbo->p_mm_list);
+#endif
+
+	atomic_inc(&bo->fence->usage);
+	fbo->pinned_node = NULL;
+	fbo->mem.mm_node->private = (void *)fbo;
+	atomic_set(&fbo->usage, 1);
+	atomic_inc(&bm->count);
+	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&fbo->mutex);
+
+	*new_obj = fbo;
+	return 0;
+}
+
+/*
+ * Since move is underway, we need to block signals in this function.
+ * We cannot restart until it has finished.
+ */
+
+int drm_bo_move_accel_cleanup(drm_buffer_object_t * bo,
+			      int evict,
+			      int no_wait,
+			      uint32_t fence_type,
+			      uint32_t fence_flags, drm_bo_mem_reg_t * new_mem)
+{
+	drm_device_t *dev = bo->dev;
+	drm_mem_type_manager_t *man = &dev->bm.man[new_mem->mem_type];
+	drm_bo_mem_reg_t *old_mem = &bo->mem;
+	int ret;
+	uint32_t save_flags = old_mem->flags;
+	uint32_t save_mask = old_mem->mask;
+	drm_buffer_object_t *old_obj;
+
+	if (bo->fence)
+		drm_fence_usage_deref_unlocked(dev, bo->fence);
+	ret = drm_fence_object_create(dev, fence_type,
+				      fence_flags | DRM_FENCE_FLAG_EMIT,
+				      &bo->fence);
+	if (ret)
+		return ret;
+
+#ifdef DRM_ODD_MM_COMPAT
+	/*
+	 * In this mode, we don't allow pipelining a copy blit,
+	 * since the buffer will be accessible from user space 
+	 * the moment we return and rebuild the page tables.
+	 *
+	 * With normal vm operation, page tables are rebuilt
+	 * on demand using fault(), which waits for buffer idle. 
+	 */
+	if (1)
+#else
+	if (evict || ((bo->mem.mm_node == bo->pinned_node) &&
+		      bo->mem.mm_node != NULL))
+#endif
+	{
+		ret = drm_bo_wait(bo, 0, 1, 0);
+		if (ret)
+			return ret;
+
+		drm_bo_free_old_node(bo);
+
+		if ((man->flags & _DRM_FLAG_MEMTYPE_FIXED) && (bo->ttm != NULL)) {
+			drm_ttm_unbind(bo->ttm);
+			drm_destroy_ttm(bo->ttm);
+			bo->ttm = NULL;
+		}
+	} else {
+
+		/* This should help pipeline ordinary buffer moves.
+		 *
+		 * Hang old buffer memory on a new buffer object,
+		 * and leave it to be released when the GPU
+		 * operation has completed.
+		 */
+
+		ret = drm_buffer_object_transfer(bo, &old_obj);
+
+		if (ret)
+			return ret;
+
+		if (!(man->flags & _DRM_FLAG_MEMTYPE_FIXED))
+			old_obj->ttm = NULL;
+		else
+			bo->ttm = NULL;
+
+		mutex_lock(&dev->struct_mutex);
+		list_del_init(&old_obj->lru);
+		DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED);
+		drm_bo_add_to_lru(old_obj);
+
+		drm_bo_usage_deref_locked(old_obj);
+		mutex_unlock(&dev->struct_mutex);
+
+	}
+
+	*old_mem = *new_mem;
+	new_mem->mm_node = NULL;
+	old_mem->mask = save_mask;
+	DRM_FLAG_MASKED(save_flags, new_mem->flags, DRM_BO_MASK_MEMTYPE);
+	return 0;
+}
+
+EXPORT_SYMBOL(drm_bo_move_accel_cleanup);