More post-ioctl work.

3 files changed, 315 insertions, 84 deletions

diff --git a/linux-core/Makefile.kernel b/linux-core/Makefile.kernel
index e7c280d0..defbe43b 100644
--- a/linux-core/Makefile.kernel
+++ b/linux-core/Makefile.kernel
@@ -20,7 +20,7 @@ r128-objs   := r128_drv.o r128_cce.o r128_state.o r128_irq.o
 mga-objs    := mga_drv.o mga_dma.o mga_state.o mga_warp.o mga_irq.o
 i810-objs   := i810_drv.o i810_dma.o
 i915-objs   := i915_drv.o i915_dma.o i915_irq.o i915_mem.o i915_fence.o \
-		i915_buffer.o i915_compat.o
+		i915_buffer.o i915_compat.o i915_execbuf.o
 nouveau-objs := nouveau_drv.o nouveau_state.o nouveau_fifo.o nouveau_mem.o \
 		nouveau_object.o nouveau_irq.o nouveau_notifier.o nouveau_swmthd.o \
 		nouveau_sgdma.o nouveau_dma.o nouveau_buffer.o nouveau_fence.o \
diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c
index a9d87338..58568452 100644
--- a/linux-core/drm_ttm.c
+++ b/linux-core/drm_ttm.c
@@ -298,7 +298,7 @@ int drm_ttm_populate(struct drm_ttm *ttm)
 		return 0;
 
 	be = ttm->be;
-	if (ttm->page_flags & DRM_TTM_PAGE_WRITE) {
+	if (1 || (ttm->page_flags & DRM_TTM_PAGE_WRITE)) {
 		for (i = 0; i < ttm->num_pages; ++i) {
 			page = drm_ttm_get_page(ttm, i);
 			if (!page)
diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c
index fecb5ab0..1e515a57 100644
--- a/linux-core/i915_execbuf.c
+++ b/linux-core/i915_execbuf.c
@@ -29,6 +29,18 @@
 #include "i915_drm.h"
 #include "i915_drv.h"
 
+#if DRM_DEBUG_CODE
+#define DRM_DEBUG_RELOCATION	(drm_debug != 0)
+#else
+#define DRM_DEBUG_RELOCATION	0
+#endif
+
+enum i915_buf_idle {
+	i915_reloc_unchecked,
+	i915_reloc_idle,
+	i915_reloc_busy
+};
+
 struct i915_relocatee_info {
 	struct drm_buffer_object *buf;
 	unsigned long offset;
@@ -36,8 +48,8 @@ struct i915_relocatee_info {
 	unsigned page_offset;
 	struct drm_bo_kmap_obj kmap;
 	int is_iomem;
-	int idle;
         int dst;
+	int idle;
 #ifdef DRM_KMAP_ATOMIC_PROT_PFN
 	unsigned long pfn;
 	pgprot_t pg_prot;
@@ -50,76 +62,28 @@ struct drm_i915_validate_buffer {
 	int presumed_offset_correct;
 	void __user *data;
 	int ret;
+	enum i915_buf_idle idle;
 };
 
-
-static int i915_update_relocatee(struct i915_relocatee_info *relocatee,
-				 struct drm_i915_validate_buffer *buffers,
-				 unsigned int dst,
-				 unsigned long dst_offset)
+static void i915_emit_ring_reloc(struct drm_device *dev, uint32_t offset,
+				 uint32_t value)
 {
-	int ret;
-
-	if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) {
-		i915_clear_relocatee(relocatee);
-		relocatee->dst = dst;
-		relocatee->buf = buffers[dst].buffer;
-		preempt_enable();
-		ret = mutex_lock_interruptible(&relocatee->buf->mutex);
-		preempt_disable();
-		if (unlikely(ret))
-			return -EAGAIN;
-
-		ret = drm_bo_wait(relocatee->buf, 0, 0, 0);
-		if (unlikely(ret))
-			return ret;
-
-		mutex_unlock(&relocatee->buf->mutex);
-	}
-
-	if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) {
-		DRM_ERROR("Relocation destination out of bounds.\n");
-		return -EINVAL;
-	}
-	if (unlikely(!drm_bo_same_page(relocatee->page_offset, dst_offset) ||
-		     NULL == relocatee->data_page)) {
-#ifdef DRM_KMAP_ATOMIC_PROT_PFN
-		if (NULL != relocatee->data_page) {
-			kunmap_atomic(relocatee->data_page, KM_USER0);
-			relocatee->data_page = NULL;
-		}
-		ret = drm_bo_pfn_prot(relocatee->buf, dst_offset,
-				      &relocatee->pfn,
-				      &relocatee->pg_prot);
-		if (ret) {
-			DRM_ERROR("Can't map relocation destination.\n");
-			return -EINVAL;
-		}
-		relocatee->data_page =
-			kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0,
-					    relocatee->pg_prot);
-#else
-		if (NULL != relocatee->data_page) {
-			drm_bo_kunmap(&relocatee->kmap);
-			relocatee->data_page = NULL;
-		}
-
-		ret = drm_bo_kmap(relocatee->buf, dst_offset >> PAGE_SHIFT,
-				  1, &relocatee->kmap);
-		if (ret) {
-			DRM_ERROR("Can't map relocation destination.\n");
-			return ret;
-		}
-
-		relocatee->data_page = drm_bmo_virtual(&relocatee->kmap,
-						      &relocatee->is_iomem);
-#endif
-		relocatee->page_offset = dst_offset & PAGE_MASK;
-	}
-	return 0;
+	struct drm_i915_private *dev_priv =
+		(struct drm_i915_private *) dev->dev_private;
+
+	DRM_INFO("Ring reloc.\n");
+	RING_LOCALS;
+	i915_kernel_lost_context(dev);
+	BEGIN_LP_RING(6);
+	OUT_RING((0x02 << 29) | (0x40 << 22) | (0x3 << 20) | (0x3));
+	OUT_RING((0x3 << 24) | (0xF0 << 16) | (0x40));
+	OUT_RING((0x1 << 16) | (0x4));
+	OUT_RING(offset);
+	OUT_RING(value);
+	OUT_RING(0);
+	ADVANCE_LP_RING();
 }
 
-
 static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buffers,
 					    unsigned num_buffers)
 {
@@ -332,6 +296,248 @@ out_err:
 	return ret;
 }
 
+
+static void i915_clear_relocatee(struct i915_relocatee_info *relocatee)
+{
+	if (relocatee->data_page) {
+#ifndef DRM_KMAP_ATOMIC_PROT_PFN
+		drm_bo_kunmap(&relocatee->kmap);
+#else
+		kunmap_atomic(relocatee->data_page, KM_USER0);
+#endif
+		relocatee->data_page = NULL;
+	}
+	relocatee->buf = NULL;
+	relocatee->dst = ~0;
+}
+
+
+static int i915_update_relocatee(struct i915_relocatee_info *relocatee,
+				 struct drm_i915_validate_buffer *buffers,
+				 unsigned int dst,
+				 unsigned long dst_offset)
+{
+	int ret;
+	
+	if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) {
+		i915_clear_relocatee(relocatee);
+		relocatee->dst = dst;
+		relocatee->buf = buffers[dst].buffer;
+		relocatee->idle = i915_reloc_idle; buffers[dst].idle;
+#if 0
+		if (relocatee->idle == i915_reloc_unchecked) {
+			preempt_enable();
+			ret = mutex_lock_interruptible(&relocatee->buf->mutex);
+			if (unlikely(ret))
+				return -EAGAIN;
+
+			ret = drm_bo_wait(relocatee->buf, 0, 0, 1);
+			relocatee->idle = (ret == 0) ? i915_reloc_idle : i915_reloc_busy;
+			mutex_unlock(&relocatee->buf->mutex);
+			preempt_disable();
+			buffers[dst].idle = relocatee->idle;
+		}
+#endif
+	}
+
+	if (relocatee->idle == i915_reloc_busy)
+	    return 0;
+
+	if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) {
+		DRM_ERROR("Relocation destination out of bounds.\n");
+		return -EINVAL;
+	}
+	if (unlikely(!drm_bo_same_page(relocatee->page_offset, dst_offset) ||
+		     NULL == relocatee->data_page)) {
+#ifdef DRM_KMAP_ATOMIC_PROT_PFN
+		if (NULL != relocatee->data_page) {
+			kunmap_atomic(relocatee->data_page, KM_USER0);
+			relocatee->data_page = NULL;
+		}
+		ret = drm_bo_pfn_prot(relocatee->buf, dst_offset,
+				      &relocatee->pfn,
+				      &relocatee->pg_prot);
+		if (ret) {
+			DRM_ERROR("Can't map relocation destination.\n");
+			return -EINVAL;
+		}
+		relocatee->data_page =
+			kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0,
+					    relocatee->pg_prot);
+#else
+		if (NULL != relocatee->data_page) {
+			drm_bo_kunmap(&relocatee->kmap);
+			relocatee->data_page = NULL;
+		}
+
+		ret = drm_bo_kmap(relocatee->buf, dst_offset >> PAGE_SHIFT,
+				  1, &relocatee->kmap);
+		if (ret) {
+			DRM_ERROR("Can't map relocation destination.\n");
+			return ret;
+		}
+
+		relocatee->data_page = drm_bmo_virtual(&relocatee->kmap,
+						      &relocatee->is_iomem);
+#endif
+		relocatee->page_offset = dst_offset & PAGE_MASK;
+	}
+	return 0;
+}
+
+static int i915_apply_post_reloc(uint32_t reloc[],
+				 struct drm_i915_validate_buffer *buffers,
+				 uint32_t num_buffers,
+				 struct i915_relocatee_info *relocatee)
+{
+	uint32_t reloc_buffer = reloc[2];
+	uint32_t dst_buffer = reloc[3];
+	uint32_t val;
+	uint32_t index;
+	int ret;
+
+	if (likely(buffers[reloc_buffer].presumed_offset_correct))
+		return 0;
+	if (unlikely(reloc_buffer >= num_buffers)) {
+		DRM_ERROR("Invalid reloc buffer index.\n");
+		return -EINVAL;
+	}
+	if (unlikely(dst_buffer >= num_buffers)) {
+		DRM_ERROR("Invalid dest buffer index.\n");
+		return -EINVAL;
+	}
+
+	ret = i915_update_relocatee(relocatee, buffers, dst_buffer,
+				    reloc[0]);
+	if (unlikely(ret))
+		return ret;
+
+	val = buffers[reloc_buffer].buffer->offset;
+	index = (reloc[0] - relocatee->page_offset) >> 2;
+	val = val + reloc[1];
+
+	if (relocatee->idle == i915_reloc_busy) {
+		i915_emit_ring_reloc(relocatee->buf->dev, 
+				     relocatee->buf->offset + reloc[0],
+				     val);
+		return 0;
+	}
+
+#ifdef DRM_KMAP_ATOMIC_PROT_PFN
+	relocatee->data_page[index] = val;
+#else
+	if (likely(relocatee->is_iomem))
+		iowrite32(val, relocatee->data_page + index);
+	else
+		relocatee->data_page[index] = val;
+#endif
+
+	return 0;
+}
+
+static int i915_post_relocs(struct drm_file *file_priv,
+			    uint32_t __user *new_reloc_ptr,
+			    struct drm_i915_validate_buffer *buffers,
+			    unsigned int num_buffers)
+{
+	uint32_t *reloc;
+	uint32_t reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t);
+	uint32_t header_size = I915_RELOC_HEADER * sizeof(uint32_t);
+	struct i915_relocatee_info relocatee;
+	uint32_t reloc_type;
+	uint32_t num_relocs;
+	uint32_t count;
+	int ret = 0;
+	int i;
+	int short_circuit = 1;
+	uint32_t __user *reloc_ptr;
+	uint64_t new_reloc_data;
+	uint32_t reloc_buf_size;
+	uint32_t *reloc_buf;
+
+	for (i=0; i<num_buffers; ++i) {
+		if (unlikely(!buffers[i].presumed_offset_correct)) {
+			short_circuit = 0;
+			break;
+		}
+	}
+
+	if (likely(short_circuit))
+		return 0;
+
+	memset(&relocatee, 0, sizeof(relocatee));
+
+	while(new_reloc_ptr) {
+		reloc_ptr = new_reloc_ptr;
+
+		ret = get_user(num_relocs, reloc_ptr);
+		if (unlikely(ret))
+			goto out;
+		if (unlikely(!access_ok(VERIFY_READ, reloc_ptr,
+					header_size +
+				num_relocs * reloc_stride)))
+			return -EFAULT;
+
+		ret = __get_user(reloc_type, reloc_ptr + 1);
+		if (unlikely(ret))
+			goto out;
+
+		if (unlikely(reloc_type != 1)) {
+			DRM_ERROR("Unsupported relocation type requested.\n");
+			ret = -EINVAL;
+			goto out;
+		}
+
+		ret = __get_user(new_reloc_data, reloc_ptr + 2);
+		new_reloc_ptr = (uint32_t __user *) (unsigned long)
+			new_reloc_data;
+
+		reloc_ptr += I915_RELOC_HEADER;
+
+		if (num_relocs == 0)
+			goto out;
+
+		reloc_buf_size = (num_relocs * I915_RELOC0_STRIDE) * sizeof(uint32_t);
+		reloc_buf = kmalloc(reloc_buf_size, GFP_KERNEL);
+		if (!reloc_buf) {
+			DRM_ERROR("Out of memory for reloc buffer\n");
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		if (__copy_from_user(reloc_buf, reloc_ptr, reloc_buf_size)) {
+			ret = -EFAULT;
+			goto out;
+		}
+		reloc = reloc_buf;
+		preempt_disable();
+		for (count = 0; count < num_relocs; ++count) {
+			ret = i915_apply_post_reloc(reloc, buffers,
+						    num_buffers, &relocatee);
+			if (unlikely(ret)) {
+				preempt_enable();
+				goto out;
+			}
+			reloc += I915_RELOC0_STRIDE;
+		}
+		preempt_enable();
+
+		if (reloc_buf) {
+			kfree(reloc_buf);
+			reloc_buf = NULL;
+		}
+		i915_clear_relocatee(&relocatee);
+        }
+
+out:
+	if (reloc_buf) {
+		kfree(reloc_buf);
+		reloc_buf = NULL;
+	}
+
+	return ret;
+}
+
 static int i915_check_presumed(struct drm_i915_op_arg *arg,
 			       struct drm_buffer_object *bo,
 			       uint32_t __user *data,
@@ -372,7 +578,8 @@ static int i915_check_presumed(struct drm_i915_op_arg *arg,
 int i915_validate_buffer_list(struct drm_file *file_priv,
 			      unsigned int fence_class, uint64_t data,
 			      struct drm_i915_validate_buffer *buffers,
-			      uint32_t *num_buffers)
+			      uint32_t *num_buffers,
+			      uint32_t __user **post_relocs)
 {
 	struct drm_i915_op_arg arg;
 	struct drm_bo_op_req *req = &arg.d.req;
@@ -381,6 +588,7 @@ int i915_validate_buffer_list(struct drm_file *file_priv,
 	uint32_t buf_handle;
 	uint32_t __user *reloc_user_ptr;
 	struct drm_i915_validate_buffer *item = buffers;
+	*post_relocs = NULL;
 
 	do {
 		if (buf_count >= *num_buffers) {
@@ -392,8 +600,6 @@ int i915_validate_buffer_list(struct drm_file *file_priv,
 		item->buffer = NULL;
 		item->presumed_offset_correct = 0;
 
-		buffers[buf_count].buffer = NULL;
-
 		if (copy_from_user(&arg, (void __user *)(unsigned long)data, sizeof(arg))) {
 			ret = -EFAULT;
 			goto out_err;
@@ -412,7 +618,24 @@ int i915_validate_buffer_list(struct drm_file *file_priv,
 		buf_handle = req->bo_req.handle;
 		reloc_user_ptr = (uint32_t *)(unsigned long)arg.reloc_ptr;
 
-		if (reloc_user_ptr) {
+		/*
+		 * Switch mode to post-validation relocations?
+		 */
+
+		if (unlikely((buf_count == 0) && (*post_relocs == NULL) &&
+			     (reloc_user_ptr != NULL))) {
+			uint32_t reloc_type;
+
+			ret = get_user(reloc_type, reloc_user_ptr+1);
+			if (ret)
+				goto out_err;
+
+			if (reloc_type == 1)
+				*post_relocs = reloc_user_ptr;
+
+		}
+
+		if ((*post_relocs == NULL) && (reloc_user_ptr != NULL)) {
 			ret = i915_exec_reloc(file_priv, buf_handle, reloc_user_ptr, buffers, buf_count);
 			if (ret)
 				goto out_err;
@@ -425,7 +648,6 @@ int i915_validate_buffer_list(struct drm_file *file_priv,
 					     req->bo_req.fence_class, 0,
 					     &item->rep,
 					     &item->buffer);
-
 		if (ret) {
 			DRM_ERROR("error on handle validate %d\n", ret);
 			goto out_err;
@@ -559,8 +781,8 @@ void i915_fence_or_sync(struct drm_file *file_priv,
 }
 
 
-static int i915_execbuffer(struct drm_device *dev, void *data,
-			   struct drm_file *file_priv)
+int i915_execbuffer(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
@@ -570,6 +792,7 @@ static int i915_execbuffer(struct drm_device *dev, void *data,
 	struct drm_fence_arg *fence_arg = &exec_buf->fence_arg;
 	int num_buffers;
 	int ret;
+	uint32_t __user *post_relocs;
 	struct drm_i915_validate_buffer *buffers;
 
 	if (!dev_priv->allow_batchbuffer) {
@@ -577,7 +800,6 @@ static int i915_execbuffer(struct drm_device *dev, void *data,
 		return -EINVAL;
 	}
 
-
 	if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects,
 							batch->num_cliprects *
 							sizeof(struct drm_clip_rect)))
@@ -586,7 +808,6 @@ static int i915_execbuffer(struct drm_device *dev, void *data,
 	if (exec_buf->num_buffers > dev_priv->max_validate_buffers)
 		return -EINVAL;
 
-
 	ret = drm_bo_read_lock(&dev->bm.bm_lock);
 	if (ret)
 		return ret;
@@ -613,27 +834,38 @@ static int i915_execbuffer(struct drm_device *dev, void *data,
 
 	/* validate buffer list + fixup relocations */
 	ret = i915_validate_buffer_list(file_priv, 0, exec_buf->ops_list,
-					buffers, &num_buffers);
+					buffers, &num_buffers, &post_relocs);
 	if (ret)
 		goto out_err0;
 
+	if (post_relocs) {
+		ret = i915_post_relocs(file_priv, post_relocs,
+				       buffers, num_buffers);
+		if (ret)
+			goto out_err0;
+	}
+
 	/* make sure all previous memory operations have passed */
 	DRM_MEMORYBARRIER();
-	drm_agp_chipset_flush(dev);
 
-	/* submit buffer */
-	batch->start = buffers[num_buffers-1].buffer->offset;
+	if (!post_relocs) {
+		drm_agp_chipset_flush(dev);
+		batch->start = buffers[num_buffers-1].buffer->offset;
+	} else {
+		batch->start += buffers[0].buffer->offset;
+	}
 
+#if 1
+	//	(void) i915_emit_mi_flush(dev, MI_NO_WRITE_FLUSH | MI_READ_FLUSH | MI_EXE_FLUSH);
 	DRM_DEBUG("i915 exec batchbuffer, start %x used %d cliprects %d\n",
 		  batch->start, batch->used, batch->num_cliprects);
 
 	ret = i915_dispatch_batchbuffer(dev, batch);
 	if (ret)
 		goto out_err0;
-
+#endif
 	if (sarea_priv)
 		sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
-
 	i915_fence_or_sync(file_priv, fence_arg->flags, fence_arg, NULL);
 
 out_err0:
@@ -650,4 +882,3 @@ out_err0:
 	drm_bo_read_unlock(&dev->bm.bm_lock);
 	return ret;
 }
-#endif