Merge git://proxy01.pd.intel.com:9419/git/mesa/drm into crestline

1 files changed, 155 insertions, 110 deletions

diff --git a/linux-core/drm_fence.c b/linux-core/drm_fence.c
index 06d48255..6dd04a35 100644
--- a/linux-core/drm_fence.c
+++ b/linux-core/drm_fence.c
@@ -1,6 +1,6 @@
 /**************************************************************************
  * 
- * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
+ * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
  * All Rights Reserved.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -11,6 +11,10 @@
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
@@ -19,11 +23,6 @@
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- * 
- * 
  **************************************************************************/
 /*
  * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
@@ -35,21 +34,42 @@
  * Typically called by the IRQ handler.
  */
 
-void drm_fence_handler(drm_device_t * dev, uint32_t sequence, uint32_t type)
+void drm_fence_handler(drm_device_t * dev, uint32_t class,
+		       uint32_t sequence, uint32_t type)
 {
 	int wake = 0;
 	uint32_t diff;
 	uint32_t relevant;
 	drm_fence_manager_t *fm = &dev->fm;
+	drm_fence_class_manager_t *fc = &fm->class[class];
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
-	struct list_head *list, *prev;
-	drm_fence_object_t *fence;
+	struct list_head *head;
+	drm_fence_object_t *fence, *next;
 	int found = 0;
+	int is_exe = (type & DRM_FENCE_TYPE_EXE);
+	int ge_last_exe;
+
+
+	
+	diff = (sequence - fc->exe_flush_sequence) & driver->sequence_mask;
+
+	if (fc->pending_exe_flush && is_exe && diff < driver->wrap_diff)
+		fc->pending_exe_flush = 0;
 
-	if (list_empty(&fm->ring))
+	diff = (sequence - fc->last_exe_flush) & driver->sequence_mask;
+	ge_last_exe = diff < driver->wrap_diff;
+
+	if (ge_last_exe) 
+		fc->pending_flush &= ~type;
+
+	if (is_exe && ge_last_exe) {
+		fc->last_exe_flush = sequence;
+	}
+	
+	if (list_empty(&fc->ring))
 		return;
 
-	list_for_each_entry(fence, &fm->ring, ring) {
+	list_for_each_entry(fence, &fc->ring, ring) {
 		diff = (sequence - fence->sequence) & driver->sequence_mask;
 		if (diff > driver->wrap_diff) {
 			found = 1;
@@ -57,11 +77,11 @@ void drm_fence_handler(drm_device_t * dev, uint32_t sequence, uint32_t type)
 		}
 	}
 
-	list = (found) ? fence->ring.prev : fm->ring.prev;
-	prev = list->prev;
+	head = (found) ? &fence->ring : &fc->ring;
 
-	for (; list != &fm->ring; list = prev, prev = list->prev) {
-		fence = list_entry(list, drm_fence_object_t, ring);
+	list_for_each_entry_safe_reverse(fence, next, head, ring) {
+		if (&fence->ring == &fc->ring)
+			break;
 
 		type |= fence->native_type;
 		relevant = type & fence->type;
@@ -78,7 +98,7 @@ void drm_fence_handler(drm_device_t * dev, uint32_t sequence, uint32_t type)
 		    ~(fence->signaled | fence->submitted_flush);
 
 		if (relevant) {
-			fm->pending_flush |= relevant;
+			fc->pending_flush |= relevant;
 			fence->submitted_flush = fence->flush_mask;
 		}
 
@@ -89,9 +109,9 @@ void drm_fence_handler(drm_device_t * dev, uint32_t sequence, uint32_t type)
 		}
 
 	}
-
+	
 	if (wake) {
-		DRM_WAKEUP(&fm->fence_queue);
+		DRM_WAKEUP(&fc->fence_queue);
 	}
 }
 
@@ -147,7 +167,7 @@ static void drm_fence_object_destroy(drm_file_t * priv,
 	drm_fence_usage_deref_locked(dev, fence);
 }
 
-static int fence_signaled(drm_device_t * dev, volatile
+static int fence_signaled(drm_device_t * dev,
 			  drm_fence_object_t * fence,
 			  uint32_t mask, int poke_flush)
 {
@@ -157,7 +177,7 @@ static int fence_signaled(drm_device_t * dev, volatile
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
 
 	if (poke_flush)
-		driver->poke_flush(dev);
+		driver->poke_flush(dev, fence->class);
 	read_lock_irqsave(&fm->lock, flags);
 	signaled =
 	    (fence->type & mask & fence->signaled) == (fence->type & mask);
@@ -166,52 +186,35 @@ static int fence_signaled(drm_device_t * dev, volatile
 	return signaled;
 }
 
-static void drm_fence_flush_exe(drm_fence_manager_t * fm,
+static void drm_fence_flush_exe(drm_fence_class_manager_t * fc,
 				drm_fence_driver_t * driver, uint32_t sequence)
 {
 	uint32_t diff;
 
-	if (!fm->pending_exe_flush) {
-		volatile struct list_head *list;
-
-		/*
-		 * Last_exe_flush is invalid. Find oldest sequence.
-		 */
-
-/*		list = fm->fence_types[_DRM_FENCE_TYPE_EXE];*/
-		list = &fm->ring;
-		if (list->next == &fm->ring) {
-			return;
-		} else {
-			drm_fence_object_t *fence =
-			    list_entry(list->next, drm_fence_object_t, ring);
-			fm->last_exe_flush = (fence->sequence - 1) &
-			    driver->sequence_mask;
-		}
-		diff = (sequence - fm->last_exe_flush) & driver->sequence_mask;
-		if (diff >= driver->wrap_diff)
-			return;
-		fm->exe_flush_sequence = sequence;
-		fm->pending_exe_flush = 1;
+	if (!fc->pending_exe_flush) {
+		fc->exe_flush_sequence = sequence;
+		fc->pending_exe_flush = 1;
 	} else {
 		diff =
-		    (sequence - fm->exe_flush_sequence) & driver->sequence_mask;
+		    (sequence - fc->exe_flush_sequence) & driver->sequence_mask;
 		if (diff < driver->wrap_diff) {
-			fm->exe_flush_sequence = sequence;
+			fc->exe_flush_sequence = sequence;
 		}
 	}
 }
 
-int drm_fence_object_signaled(volatile drm_fence_object_t * fence,
+int drm_fence_object_signaled(drm_fence_object_t * fence,
 			      uint32_t type)
 {
 	return ((fence->signaled & type) == type);
 }
 
 int drm_fence_object_flush(drm_device_t * dev,
-			   volatile drm_fence_object_t * fence, uint32_t type)
+			   drm_fence_object_t * fence,
+			   uint32_t type)
 {
 	drm_fence_manager_t *fm = &dev->fm;
+	drm_fence_class_manager_t *fc = &fm->class[fence->class];
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
 	unsigned long flags;
 
@@ -226,16 +229,16 @@ int drm_fence_object_flush(drm_device_t * dev,
 	if (fence->submitted_flush == fence->signaled) {
 		if ((fence->type & DRM_FENCE_TYPE_EXE) &&
 		    !(fence->submitted_flush & DRM_FENCE_TYPE_EXE)) {
-			drm_fence_flush_exe(fm, driver, fence->sequence);
+			drm_fence_flush_exe(fc, driver, fence->sequence);
 			fence->submitted_flush |= DRM_FENCE_TYPE_EXE;
 		} else {
-			fm->pending_flush |= (fence->flush_mask &
+			fc->pending_flush |= (fence->flush_mask &
 					      ~fence->submitted_flush);
 			fence->submitted_flush = fence->flush_mask;
 		}
 	}
 	write_unlock_irqrestore(&fm->lock, flags);
-	driver->poke_flush(dev);
+	driver->poke_flush(dev, fence->class);
 	return 0;
 }
 
@@ -244,24 +247,35 @@ int drm_fence_object_flush(drm_device_t * dev,
  * wrapped around and reused.
  */
 
-void drm_fence_flush_old(drm_device_t * dev, uint32_t sequence)
+void drm_fence_flush_old(drm_device_t * dev, uint32_t class, uint32_t sequence)
 {
 	drm_fence_manager_t *fm = &dev->fm;
+	drm_fence_class_manager_t *fc = &fm->class[class];
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
 	uint32_t old_sequence;
 	unsigned long flags;
 	drm_fence_object_t *fence;
 	uint32_t diff;
 
+	write_lock_irqsave(&fm->lock, flags);
+	old_sequence = (sequence - driver->flush_diff) & driver->sequence_mask;
+	diff = (old_sequence - fc->last_exe_flush) & driver->sequence_mask;
+
+	if ((diff < driver->wrap_diff) && !fc->pending_exe_flush) {
+		fc->pending_exe_flush = 1;
+		fc->exe_flush_sequence = sequence - (driver->flush_diff / 2);
+	}
+	write_unlock_irqrestore(&fm->lock, flags);
+	
 	mutex_lock(&dev->struct_mutex);
 	read_lock_irqsave(&fm->lock, flags);
-	if (fm->ring.next == &fm->ring) {
+
+	if (list_empty(&fc->ring)) {
 		read_unlock_irqrestore(&fm->lock, flags);
 		mutex_unlock(&dev->struct_mutex);
 		return;
 	}
-	old_sequence = (sequence - driver->flush_diff) & driver->sequence_mask;
-	fence = list_entry(fm->ring.next, drm_fence_object_t, ring);
+	fence = list_entry(fc->ring.next, drm_fence_object_t, ring);
 	atomic_inc(&fence->usage);
 	mutex_unlock(&dev->struct_mutex);
 	diff = (old_sequence - fence->sequence) & driver->sequence_mask;
@@ -274,11 +288,40 @@ void drm_fence_flush_old(drm_device_t * dev, uint32_t sequence)
 
 EXPORT_SYMBOL(drm_fence_flush_old);
 
+static int drm_fence_lazy_wait(drm_device_t *dev,
+			       drm_fence_object_t *fence,
+			       int ignore_signals,
+			       uint32_t mask)
+{
+	drm_fence_manager_t *fm = &dev->fm;
+	drm_fence_class_manager_t *fc = &fm->class[fence->class];
+
+	unsigned long _end = jiffies + 3*DRM_HZ;
+	int ret = 0;
+
+	do {
+		DRM_WAIT_ON(ret, fc->fence_queue, 3 * DRM_HZ,
+			    fence_signaled(dev, fence, mask, 0));
+		if (time_after_eq(jiffies, _end))
+			break;
+	} while (ret == -EINTR && ignore_signals);
+	if (time_after_eq(jiffies, _end) && (ret != 0))
+		ret = -EBUSY;
+	if (ret) {
+		if (ret == -EBUSY) {
+			DRM_ERROR("Fence timeout. "
+				  "GPU lockup or fence driver was "
+				  "taken down.\n");
+		}
+		return ((ret == -EINTR) ? -EAGAIN : ret);
+	}
+	return 0;
+}
+
 int drm_fence_object_wait(drm_device_t * dev,
-			  volatile drm_fence_object_t * fence,
+			  drm_fence_object_t * fence,
 			  int lazy, int ignore_signals, uint32_t mask)
 {
-	drm_fence_manager_t *fm = &dev->fm;
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
 	int ret = 0;
 	unsigned long _end;
@@ -299,44 +342,29 @@ int drm_fence_object_wait(drm_device_t * dev,
 
 	if (lazy && driver->lazy_capable) {
 
-		do {
-			DRM_WAIT_ON(ret, fm->fence_queue, 3 * DRM_HZ,
-				    fence_signaled(dev, fence, mask, 1));
-			if (time_after_eq(jiffies, _end))
-				break;
-		} while (ret == -EINTR && ignore_signals);
-		if (time_after_eq(jiffies, _end) && (ret != 0))
-			ret = -EBUSY;
-		if (ret) {
-			if (ret == -EBUSY) {
-				DRM_ERROR("Fence timeout. "
-					  "GPU lockup or fence driver was "
-					  "taken down.\n");
-			}
-			return ((ret == -EINTR) ? -EAGAIN : ret);
-		}
-	} else if ((fence->class == 0) && (mask & DRM_FENCE_TYPE_EXE) &&
-		   driver->lazy_capable) {
+		ret = drm_fence_lazy_wait(dev, fence, ignore_signals, mask);
+		if (ret)
+			return ret;
 
-		/*
-		 * We use IRQ wait for EXE fence if available to gain 
-		 * CPU in some cases.
-		 */
+	} else {
 
-		do {
-			DRM_WAIT_ON(ret, fm->fence_queue, 3 * DRM_HZ,
-				    fence_signaled(dev, fence,
-						   DRM_FENCE_TYPE_EXE, 1));
-			if (time_after_eq(jiffies, _end))
-				break;
-		} while (ret == -EINTR && ignore_signals);
-		if (time_after_eq(jiffies, _end) && (ret != 0))
-			ret = -EBUSY;
-		if (ret)
-			return ((ret == -EINTR) ? -EAGAIN : ret);
-	}
+		if (driver->has_irq(dev, fence->class,
+				    DRM_FENCE_TYPE_EXE)) {
+			ret = drm_fence_lazy_wait(dev, fence, ignore_signals,
+						  DRM_FENCE_TYPE_EXE);
+			if (ret)
+				return ret;
+		}
 
-	if (fence_signaled(dev, fence, mask, 0))
+		if (driver->has_irq(dev, fence->class,
+				    mask & ~DRM_FENCE_TYPE_EXE)) {
+			ret = drm_fence_lazy_wait(dev, fence, ignore_signals,
+						  mask);
+			if (ret)
+				return ret;
+		}
+	}
+	if (drm_fence_object_signaled(fence, mask))
 		return 0;
 
 	/*
@@ -358,33 +386,38 @@ int drm_fence_object_wait(drm_device_t * dev,
 }
 
 int drm_fence_object_emit(drm_device_t * dev, drm_fence_object_t * fence,
-			  uint32_t fence_flags, uint32_t type)
+			  uint32_t fence_flags, uint32_t class, uint32_t type)
 {
 	drm_fence_manager_t *fm = &dev->fm;
 	drm_fence_driver_t *driver = dev->driver->fence_driver;
+	drm_fence_class_manager_t *fc = &fm->class[fence->class];
 	unsigned long flags;
 	uint32_t sequence;
 	uint32_t native_type;
 	int ret;
 
 	drm_fence_unring(dev, &fence->ring);
-	ret = driver->emit(dev, fence_flags, &sequence, &native_type);
+	ret = driver->emit(dev, class, fence_flags, &sequence, &native_type);
 	if (ret)
 		return ret;
 
 	write_lock_irqsave(&fm->lock, flags);
+	fence->class = class;
 	fence->type = type;
 	fence->flush_mask = 0x00;
 	fence->submitted_flush = 0x00;
 	fence->signaled = 0x00;
 	fence->sequence = sequence;
 	fence->native_type = native_type;
-	list_add_tail(&fence->ring, &fm->ring);
+	if (list_empty(&fc->ring)) 
+		fc->last_exe_flush = sequence - 1;
+	list_add_tail(&fence->ring, &fc->ring);
 	write_unlock_irqrestore(&fm->lock, flags);
 	return 0;
 }
 
-static int drm_fence_object_init(drm_device_t * dev, uint32_t type,
+static int drm_fence_object_init(drm_device_t * dev, uint32_t class,
+				 uint32_t type,
 				 uint32_t fence_flags,
 				 drm_fence_object_t * fence)
 {
@@ -398,7 +431,7 @@ static int drm_fence_object_init(drm_device_t * dev, uint32_t type,
 
 	write_lock_irqsave(&fm->lock, flags);
 	INIT_LIST_HEAD(&fence->ring);
-	fence->class = 0;
+	fence->class = class;
 	fence->type = type;
 	fence->flush_mask = 0;
 	fence->submitted_flush = 0;
@@ -406,7 +439,8 @@ static int drm_fence_object_init(drm_device_t * dev, uint32_t type,
 	fence->sequence = 0;
 	write_unlock_irqrestore(&fm->lock, flags);
 	if (fence_flags & DRM_FENCE_FLAG_EMIT) {
-		ret = drm_fence_object_emit(dev, fence, fence_flags, type);
+		ret = drm_fence_object_emit(dev, fence, fence_flags,
+					    fence->class, type);
 	}
 	return ret;
 }
@@ -430,7 +464,7 @@ int drm_fence_add_user_object(drm_file_t * priv, drm_fence_object_t * fence,
 
 EXPORT_SYMBOL(drm_fence_add_user_object);
 
-int drm_fence_object_create(drm_device_t * dev, uint32_t type,
+int drm_fence_object_create(drm_device_t * dev, uint32_t class, uint32_t type,
 			    unsigned flags, drm_fence_object_t ** c_fence)
 {
 	drm_fence_object_t *fence;
@@ -440,7 +474,7 @@ int drm_fence_object_create(drm_device_t * dev, uint32_t type,
 	fence = drm_ctl_alloc(sizeof(*fence), DRM_MEM_FENCE);
 	if (!fence)
 		return -ENOMEM;
-	ret = drm_fence_object_init(dev, type, flags, fence);
+	ret = drm_fence_object_init(dev, class, type, flags, fence);
 	if (ret) {
 		drm_fence_usage_deref_unlocked(dev, fence);
 		return ret;
@@ -456,22 +490,31 @@ EXPORT_SYMBOL(drm_fence_object_create);
 void drm_fence_manager_init(drm_device_t * dev)
 {
 	drm_fence_manager_t *fm = &dev->fm;
+	drm_fence_class_manager_t *class;
 	drm_fence_driver_t *fed = dev->driver->fence_driver;
 	int i;
 
+
 	fm->lock = RW_LOCK_UNLOCKED;
 	write_lock(&fm->lock);
-	INIT_LIST_HEAD(&fm->ring);
-	fm->pending_flush = 0;
-	DRM_INIT_WAITQUEUE(&fm->fence_queue);
 	fm->initialized = 0;
-	if (fed) {
-		fm->initialized = 1;
-		atomic_set(&fm->count, 0);
-		for (i = 0; i < fed->no_types; ++i) {
-			fm->fence_types[i] = &fm->ring;
-		}
+	if (!fed)
+	    goto out_unlock;
+
+	fm->initialized = 1;
+	fm->num_classes = fed->num_classes;
+	BUG_ON(fm->num_classes > _DRM_FENCE_CLASSES);
+
+	for (i=0; i<fm->num_classes; ++i) {
+	    class = &fm->class[i];
+
+	    INIT_LIST_HEAD(&class->ring);
+	    class->pending_flush = 0;
+	    DRM_INIT_WAITQUEUE(&class->fence_queue);
 	}
+
+	atomic_set(&fm->count, 0);
+ out_unlock:
 	write_unlock(&fm->lock);
 }
 
@@ -518,7 +561,8 @@ int drm_fence_ioctl(DRM_IOCTL_ARGS)
 	case drm_fence_create:
 		if (arg.flags & DRM_FENCE_FLAG_EMIT)
 			LOCK_TEST_WITH_RETURN(dev, filp);
-		ret = drm_fence_object_create(dev, arg.type, arg.flags, &fence);
+		ret = drm_fence_object_create(dev, arg.class,
+					      arg.type, arg.flags, &fence);
 		if (ret)
 			return ret;
 		ret = drm_fence_add_user_object(priv, fence,
@@ -581,7 +625,8 @@ int drm_fence_ioctl(DRM_IOCTL_ARGS)
 		fence = drm_lookup_fence_object(priv, arg.handle);
 		if (!fence)
 			return -EINVAL;
-		ret = drm_fence_object_emit(dev, fence, arg.flags, arg.type);
+		ret = drm_fence_object_emit(dev, fence, arg.flags, arg.class,
+					    arg.type);
 		break;
 	case drm_fence_buffers:
 		if (!dev->bm.initialized) {