[intel-gem] Add DRM_IOCTL_I915_GEM_SW_FINISH to flag CPU writes

When a software fallback has completed, usermode must notify the kernel so
that any scanout buffers can be synchronized. This ioctl should be called
whenever a fallback completes to flush CPU and chipset caches.

5 files changed, 192 insertions, 104 deletions

diff --git a/libdrm/intel/intel_bufmgr_gem.c b/libdrm/intel/intel_bufmgr_gem.c
index 5a28bd14..b970eacf 100644
--- a/libdrm/intel/intel_bufmgr_gem.c
+++ b/libdrm/intel/intel_bufmgr_gem.c
@@ -109,11 +109,11 @@ struct _dri_bo_gem {
     int validate_index;
 
     /**
-     * Boolean whether set_domain to CPU is current
-     * Set when set_domain has been called
-     * Cleared when a batch has been submitted
+     * Boolean whether we've started swrast
+     * Set when the buffer has been mapped
+     * Cleared when the buffer is unmapped
      */
-    int cpu_domain_set;
+    int swrast;
 
     /** Array passed to the DRM containing relocation information. */
     struct drm_i915_gem_relocation_entry *relocs;
@@ -485,25 +485,27 @@ dri_gem_bo_map(dri_bo *bo, int write_enable)
 	    bo_gem->virtual = (void *)(uintptr_t)mmap_arg.addr_ptr;
 	}
 	bo->virtual = bo_gem->virtual;
+	bo_gem->swrast = 0;
 	bo_gem->mapped = 1;
 	DBG("bo_map: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name, bo_gem->virtual);
     }
 
-    if (!bo_gem->cpu_domain_set) {
+    if (!bo_gem->swrast) {
 	set_domain.handle = bo_gem->gem_handle;
 	set_domain.read_domains = I915_GEM_DOMAIN_CPU;
-	set_domain.write_domain = write_enable ? I915_GEM_DOMAIN_CPU : 0;
+	if (write_enable)
+	    set_domain.write_domain = I915_GEM_DOMAIN_CPU;
+	else
+	    set_domain.write_domain = 0;
 	do {
 	    ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN,
 			&set_domain);
 	} while (ret == -1 && errno == EINTR);
 	if (ret != 0) {
-	    fprintf (stderr, "%s:%d: Error setting memory domains %d (%08x %08x): %s .\n",
-		     __FILE__, __LINE__,
-		     bo_gem->gem_handle, set_domain.read_domains, set_domain.write_domain,
-		     strerror (errno));
+	    fprintf (stderr, "%s:%d: Error setting swrast %d: %s\n",
+		     __FILE__, __LINE__, bo_gem->gem_handle, strerror (errno));
 	}
-	bo_gem->cpu_domain_set = 1;
+	bo_gem->swrast = 1;
     }
 
     return 0;
@@ -512,13 +514,24 @@ dri_gem_bo_map(dri_bo *bo, int write_enable)
 static int
 dri_gem_bo_unmap(dri_bo *bo)
 {
+    dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
     dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
+    struct drm_i915_gem_sw_finish sw_finish;
+    int ret;
 
     if (bo == NULL)
 	return 0;
 
     assert(bo_gem->mapped);
 
+    if (bo_gem->swrast) {
+	sw_finish.handle = bo_gem->gem_handle;
+	do {
+	    ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_SW_FINISH,
+			&sw_finish);
+	} while (ret == -1 && errno == EINTR);
+	bo_gem->swrast = 0;
+    }
     return 0;
 }
 
@@ -744,8 +757,8 @@ dri_gem_post_submit(dri_bo *batch_buf)
 	dri_bo *bo = bufmgr_gem->exec_bos[i];
 	dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
 
-	/* Need to call set_domain on next bo_map */
-	bo_gem->cpu_domain_set = 0;
+	/* Need to call swrast on next bo_map */
+	bo_gem->swrast = 0;
 
 	/* Disconnect the buffer from the validate list */
 	bo_gem->validate_index = -1;
diff --git a/linux-core/i915_gem.c b/linux-core/i915_gem.c
index 308674d8..bd4aeaa7 100644
--- a/linux-core/i915_gem.c
+++ b/linux-core/i915_gem.c
@@ -47,6 +47,9 @@ i915_gem_set_domain(struct drm_gem_object *obj,
 		    uint32_t read_domains,
 		    uint32_t write_domain);
 
+static void
+i915_gem_clflush_object(struct drm_gem_object *obj);
+
 int
 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 		    struct drm_file *file_priv)
@@ -226,6 +229,45 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 }
 
 /**
+ * Called when user space has done writes to this buffer
+ */
+int
+i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
+{
+	struct drm_i915_gem_sw_finish *args = data;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret = 0;
+
+	if (!(dev->driver->driver_features & DRIVER_GEM))
+		return -ENODEV;
+
+	mutex_lock(&dev->struct_mutex);
+	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+	if (obj == NULL) {
+		mutex_unlock(&dev->struct_mutex);
+		return -EINVAL;
+	}
+
+#if WATCH_BUF
+	DRM_INFO("%s: sw_finish %d (%p)\n",
+		 __func__, args->handle, obj);
+#endif
+    	obj_priv = obj->driver_private;
+		
+    	/** Pinned buffers may be scanout, so flush the cache
+    	 */
+    	if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) {
+    		i915_gem_clflush_object(obj);
+    		drm_agp_chipset_flush(dev);
+    	}
+	drm_gem_object_unreference(obj);
+	mutex_unlock(&dev->struct_mutex);
+	return ret;
+}
+
+/**
  * Maps the contents of an object, returning the address it is mapped
  * into.
  *
@@ -1180,13 +1222,16 @@ i915_gem_object_set_domain(struct drm_gem_object *obj,
 			    uint32_t write_domain)
 {
 	struct drm_device		*dev = obj->dev;
+	struct drm_i915_gem_object	*obj_priv = obj->driver_private;
 	uint32_t			invalidate_domains = 0;
 	uint32_t			flush_domains = 0;
 	int				ret;
 
 #if WATCH_BUF
-	DRM_INFO("%s: object %p read %08x write %08x\n",
-		 __func__, obj, read_domains, write_domain);
+	DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
+		 __func__, obj, 
+		 obj->read_domains, read_domains, 
+		 obj->write_domain, write_domain);
 #endif
 	/*
 	 * If the object isn't moving to a new write domain,
@@ -1234,6 +1279,12 @@ i915_gem_object_set_domain(struct drm_gem_object *obj,
 	obj->read_domains = read_domains;
 	dev->invalidate_domains |= invalidate_domains;
 	dev->flush_domains |= flush_domains;
+#if WATCH_BUF
+	DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
+		 __func__,
+		 obj->read_domains, obj->write_domain,
+		 dev->invalidate_domains, dev->flush_domains);
+#endif
 	return 0;
 }
 
@@ -1899,6 +1950,14 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 		return ret;
 	}
 
+	/** XXX - flush the CPU caches for pinned objects
+	 * as the X server doesn't manage domains yet
+	 */
+	if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
+		i915_gem_clflush_object(obj);
+		drm_agp_chipset_flush(dev);
+		obj->write_domain = 0;
+	}
 	args->offset = obj_priv->gtt_offset;
 	drm_gem_object_unreference(obj);
 	mutex_unlock(&dev->struct_mutex);
@@ -2000,43 +2059,105 @@ i915_gem_set_domain(struct drm_gem_object *obj,
 {
 	struct drm_device *dev = obj->dev;
 	int ret;
+	uint32_t flush_domains;
 
 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
 
 	ret = i915_gem_object_set_domain(obj, read_domains, write_domain);
 	if (ret)
 		return ret;
-	i915_gem_dev_set_domain(obj->dev);
+	flush_domains = i915_gem_dev_set_domain(obj->dev);
+	
+	if (flush_domains & ~I915_GEM_DOMAIN_CPU)
+		(void) i915_add_request(dev, flush_domains);
 
 	return 0;
 }
 
-void
-i915_gem_lastclose(struct drm_device *dev)
+/** Unbinds all objects that are on the given buffer list. */
+static int
+i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
+{
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret;
+
+	while (!list_empty(head)) {
+		obj_priv = list_first_entry(head,
+					    struct drm_i915_gem_object,
+					    list);
+		obj = obj_priv->obj;
+
+		if (obj_priv->pin_count != 0) {
+			DRM_ERROR("Pinned object in unbind list\n");
+			mutex_unlock(&dev->struct_mutex);
+			return -EINVAL;
+		}
+
+		ret = i915_gem_object_unbind(obj);
+		if (ret != 0) {
+			DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
+				  ret);
+			mutex_unlock(&dev->struct_mutex);
+			return ret;
+		}
+	}
+
+
+	return 0;
+}
+
+static int
+i915_gem_idle(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	uint32_t seqno;
+	int ret;
 
-	mutex_lock(&dev->struct_mutex);
+	if (dev_priv->mm.suspended)
+		return 0;
 
-	/* Assume that the chip has been idled at this point. Just pull them
-	 * off the execution list and unref them.  Since this is the last
-	 * close, this is also the last ref and they'll go away.
+	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
+	 * We need to replace this with a semaphore, or something.
 	 */
+	dev_priv->mm.suspended = 1;
 
-	while (!list_empty(&dev_priv->mm.active_list)) {
-		struct drm_i915_gem_object *obj_priv;
+	i915_kernel_lost_context(dev);
 
-		obj_priv = list_first_entry(&dev_priv->mm.active_list,
-					    struct drm_i915_gem_object,
-					    list);
+	/* Flush the GPU along with all non-CPU write domains
+	 */
+	i915_gem_flush(dev, ~I915_GEM_DOMAIN_CPU, ~I915_GEM_DOMAIN_CPU);
+	seqno = i915_add_request(dev, ~I915_GEM_DOMAIN_CPU);
 
-		list_del_init(&obj_priv->list);
-		obj_priv->active = 0;
-		obj_priv->obj->write_domain = 0;
-		drm_gem_object_unreference(obj_priv->obj);
+	if (seqno == 0) {
+		mutex_unlock(&dev->struct_mutex);
+		return -ENOMEM;
+	}
+	ret = i915_wait_request(dev, seqno);
+	if (ret) {
+		mutex_unlock(&dev->struct_mutex);
+		return ret;
 	}
 
-	mutex_unlock(&dev->struct_mutex);
+	/* Active and flushing should now be empty as we've
+	 * waited for a sequence higher than any pending execbuffer
+	 */
+	BUG_ON(!list_empty(&dev_priv->mm.active_list));
+	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
+
+	/* Request should now be empty as we've also waited
+	 * for the last request in the list
+	 */
+	BUG_ON(!list_empty(&dev_priv->mm.request_list));
+
+	/* Move all buffers out of the GTT. */
+	i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
+
+	BUG_ON(!list_empty(&dev_priv->mm.active_list));
+	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
+	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
+	BUG_ON(!list_empty(&dev_priv->mm.request_list));
+	return 0;
 }
 
 static int
@@ -2136,91 +2257,33 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
 	return 0;
 }
 
-/** Unbinds all objects that are on the given buffer list. */
-static int
-i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
+int
+i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
 {
-	struct drm_gem_object *obj;
-	struct drm_i915_gem_object *obj_priv;
 	int ret;
 
-	while (!list_empty(head)) {
-		obj_priv = list_first_entry(head,
-					    struct drm_i915_gem_object,
-					    list);
-		obj = obj_priv->obj;
-
-		if (obj_priv->pin_count != 0) {
-			DRM_ERROR("Pinned object in unbind list\n");
-			mutex_unlock(&dev->struct_mutex);
-			return -EINVAL;
-		}
-
-		ret = i915_gem_object_unbind(obj);
-		if (ret != 0) {
-			DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
-				  ret);
-			mutex_unlock(&dev->struct_mutex);
-			return ret;
-		}
-	}
-
+	mutex_lock(&dev->struct_mutex);
+	ret = i915_gem_idle(dev);
+	if (ret == 0)
+		i915_gem_cleanup_ringbuffer(dev);
+	mutex_unlock(&dev->struct_mutex);
 
 	return 0;
 }
 
-int
-i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
-		       struct drm_file *file_priv)
+void
+i915_gem_lastclose(struct drm_device *dev)
 {
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	uint32_t seqno;
 	int ret;
 
 	mutex_lock(&dev->struct_mutex);
-	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
-	 * We need to replace this with a semaphore, or something.
-	 */
-	dev_priv->mm.suspended = 1;
-
-	i915_kernel_lost_context(dev);
-
-	/* Flush the GPU along with all non-CPU write domains
-	 */
-	i915_gem_flush(dev, ~I915_GEM_DOMAIN_CPU, ~I915_GEM_DOMAIN_CPU);
-	seqno = i915_add_request(dev, ~I915_GEM_DOMAIN_CPU);
-	if (seqno == 0) {
-		mutex_unlock(&dev->struct_mutex);
-		return -ENOMEM;
-	}
-	ret = i915_wait_request(dev, seqno);
-	if (ret) {
-		mutex_unlock(&dev->struct_mutex);
-		return ret;
-	}
 
-	/* Active and flushing should now be empty as we've
-	 * waited for a sequence higher than any pending execbuffer
-	 */
-	BUG_ON(!list_empty(&dev_priv->mm.active_list));
-	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
-
-	/* Request should now be empty as we've also waited
-	 * for the last request in the list
-	 */
-	BUG_ON(!list_empty(&dev_priv->mm.request_list));
-
-	/* Move all buffers out of the GTT. */
-	i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
-
-	BUG_ON(!list_empty(&dev_priv->mm.active_list));
-	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
-	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
-	BUG_ON(!list_empty(&dev_priv->mm.request_list));
+	ret = i915_gem_idle(dev);
+	if (ret)
+		DRM_ERROR("failed to idle hardware: %d\n", ret);
 
 	i915_gem_cleanup_ringbuffer(dev);
-
+	
 	mutex_unlock(&dev->struct_mutex);
-
-	return 0;
 }
diff --git a/shared-core/i915_dma.c b/shared-core/i915_dma.c
index 2db6be07..6ee3d19f 100644
--- a/shared-core/i915_dma.c
+++ b/shared-core/i915_dma.c
@@ -1216,6 +1216,7 @@ struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_I915_GEM_PWRITE, i915_gem_pwrite_ioctl, 0),
 	DRM_IOCTL_DEF(DRM_I915_GEM_MMAP, i915_gem_mmap_ioctl, 0),
 	DRM_IOCTL_DEF(DRM_I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, 0),
+	DRM_IOCTL_DEF(DRM_I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, 0),
 };
 
 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
diff --git a/shared-core/i915_drm.h b/shared-core/i915_drm.h
index f16dc99f..fde474d7 100644
--- a/shared-core/i915_drm.h
+++ b/shared-core/i915_drm.h
@@ -189,6 +189,7 @@ typedef struct drm_i915_sarea {
 #define DRM_I915_GEM_PWRITE	0x1d
 #define DRM_I915_GEM_MMAP	0x1e
 #define DRM_I915_GEM_SET_DOMAIN	0x1f
+#define DRM_I915_GEM_SW_FINISH	0x20
 
 #define DRM_IOCTL_I915_INIT		DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t)
 #define DRM_IOCTL_I915_FLUSH		DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH)
@@ -221,6 +222,7 @@ typedef struct drm_i915_sarea {
 #define DRM_IOCTL_I915_GEM_PWRITE	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PWRITE, struct drm_i915_gem_pwrite)
 #define DRM_IOCTL_I915_GEM_MMAP		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP, struct drm_i915_gem_mmap)
 #define DRM_IOCTL_I915_GEM_SET_DOMAIN	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SET_DOMAIN, struct drm_i915_gem_set_domain)
+#define DRM_IOCTL_I915_GEM_SW_FINISH	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SW_FINISH, struct drm_i915_gem_sw_finish)
 
 /* Asynchronous page flipping:
  */
@@ -501,6 +503,11 @@ struct drm_i915_gem_set_domain {
 	uint32_t write_domain;
 };
 
+struct drm_i915_gem_sw_finish {
+	/** Handle for the object */
+	uint32_t handle;
+};
+
 struct drm_i915_gem_relocation_entry {
 	/**
 	 * Handle of the buffer being pointed to by this relocation entry.
@@ -565,6 +572,8 @@ struct drm_i915_gem_relocation_entry {
 #define I915_GEM_DOMAIN_INSTRUCTION	0x00000010
 /** Vertex address cache */
 #define I915_GEM_DOMAIN_VERTEX		0x00000020
+/** GTT domain - aperture and scanout */
+#define I915_GEM_DOMAIN_GTT		0x00000040
 /** @} */
 
 struct drm_i915_gem_exec_object {
diff --git a/shared-core/i915_drv.h b/shared-core/i915_drv.h
index 334bc43f..4d8a40d4 100644
--- a/shared-core/i915_drv.h
+++ b/shared-core/i915_drv.h
@@ -477,6 +477,8 @@ int i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
+int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
+			     struct drm_file *file_priv);
 int i915_gem_execbuffer(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_pin_ioctl(struct drm_device *dev, void *data,