From f1edb7ad91d8b92057ffa02eb162e3740d05a147 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Wed, 30 Jan 2008 22:06:02 +0100 Subject: Simplify the fencing code and differentiate between flushes and waiting types. Add a "command_stream_barrier" method to the bo driver. --- linux-core/i915_fence.c | 246 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 190 insertions(+), 56 deletions(-) (limited to 'linux-core/i915_fence.c') diff --git a/linux-core/i915_fence.c b/linux-core/i915_fence.c index e3c76df6..8a2e7f13 100644 --- a/linux-core/i915_fence.c +++ b/linux-core/i915_fence.c @@ -35,97 +35,121 @@ #include "i915_drv.h" /* - * Implements an intel sync flush operation. + * Initiate a sync flush if it's not already pending. */ -static void i915_perform_flush(struct drm_device *dev) +static void i915_initiate_rwflush(struct drm_i915_private *dev_priv, + struct drm_fence_class_manager *fc) +{ + if ((fc->pending_flush & DRM_I915_FENCE_TYPE_RW) && + !dev_priv->flush_pending) { + dev_priv->flush_sequence = (uint32_t) READ_BREADCRUMB(dev_priv); + dev_priv->flush_flags = fc->pending_flush; + dev_priv->saved_flush_status = READ_HWSP(dev_priv, 0); + I915_WRITE(I915REG_INSTPM, (1 << 5) | (1 << 21)); + dev_priv->flush_pending = 1; + fc->pending_flush &= ~DRM_I915_FENCE_TYPE_RW; + } +} + +static void i915_fence_flush(struct drm_device *dev, + uint32_t fence_class) +{ + struct drm_i915_private *dev_priv = + (struct drm_i915_private *) dev->dev_private; + struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[0]; + unsigned long irq_flags; + + if (unlikely(!dev_priv)) + return; + + write_lock_irqsave(&fm->lock, irq_flags); + i915_initiate_rwflush(dev_priv, fc); + write_unlock_irqrestore(&fm->lock, irq_flags); +} + +static void i915_fence_poll(struct drm_device *dev, uint32_t fence_class, + uint32_t waiting_types) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; struct drm_fence_manager *fm = &dev->fm; struct drm_fence_class_manager *fc = &fm->fence_class[0]; - struct drm_fence_driver *driver = dev->driver->fence_driver; uint32_t flush_flags = 0; uint32_t flush_sequence = 0; uint32_t i_status; - uint32_t diff; uint32_t sequence; - int rwflush; - if (!dev_priv) + if (unlikely(!dev_priv)) return; - if (fc->pending_exe_flush) { - sequence = READ_BREADCRUMB(dev_priv); + /* + * First, report any executed sync flush: + */ + + if (dev_priv->flush_pending) { + i_status = READ_HWSP(dev_priv, 0); + if ((i_status & (1 << 12)) != + (dev_priv->saved_flush_status & (1 << 12))) { + flush_flags = dev_priv->flush_flags; + flush_sequence = dev_priv->flush_sequence; + dev_priv->flush_pending = 0; + drm_fence_handler(dev, 0, flush_sequence, flush_flags, 0); + } + } - /* - * First update fences with the current breadcrumb. - */ + /* + * Report A new breadcrumb, and adjust IRQs. + */ + + if (waiting_types & DRM_FENCE_TYPE_EXE) { + sequence = READ_BREADCRUMB(dev_priv); - diff = (sequence - fc->last_exe_flush) & BREADCRUMB_MASK; - if (diff < driver->wrap_diff && diff != 0) { - drm_fence_handler(dev, 0, sequence, + if (sequence != dev_priv->reported_sequence || + !dev_priv->reported_sequence_valid) { + drm_fence_handler(dev, 0, sequence, DRM_FENCE_TYPE_EXE, 0); + dev_priv->reported_sequence = sequence; + dev_priv->reported_sequence_valid = 1; } - if (dev_priv->fence_irq_on && !fc->pending_exe_flush) { + if (dev_priv->fence_irq_on && !(waiting_types & DRM_FENCE_TYPE_EXE)) { i915_user_irq_off(dev_priv); dev_priv->fence_irq_on = 0; - } else if (!dev_priv->fence_irq_on && fc->pending_exe_flush) { + } else if (!dev_priv->fence_irq_on && (waiting_types & DRM_FENCE_TYPE_EXE)) { i915_user_irq_on(dev_priv); dev_priv->fence_irq_on = 1; } } - if (dev_priv->flush_pending) { - i_status = READ_HWSP(dev_priv, 0); - if ((i_status & (1 << 12)) != - (dev_priv->saved_flush_status & (1 << 12))) { - flush_flags = dev_priv->flush_flags; - flush_sequence = dev_priv->flush_sequence; - dev_priv->flush_pending = 0; - drm_fence_handler(dev, 0, flush_sequence, flush_flags, 0); - } - } + /* + * There may be new RW flushes pending. Start them. + */ + + i915_initiate_rwflush(dev_priv, fc); - rwflush = fc->pending_flush & DRM_I915_FENCE_TYPE_RW; - if (rwflush && !dev_priv->flush_pending) { - dev_priv->flush_sequence = (uint32_t) READ_BREADCRUMB(dev_priv); - dev_priv->flush_flags = fc->pending_flush; - dev_priv->saved_flush_status = READ_HWSP(dev_priv, 0); - I915_WRITE(I915REG_INSTPM, (1 << 5) | (1 << 21)); - dev_priv->flush_pending = 1; - fc->pending_flush &= ~DRM_I915_FENCE_TYPE_RW; - } + /* + * And possibly, but unlikely, they finish immediately. + */ if (dev_priv->flush_pending) { i_status = READ_HWSP(dev_priv, 0); - if ((i_status & (1 << 12)) != - (dev_priv->saved_flush_status & (1 << 12))) { + if (unlikely((i_status & (1 << 12)) != + (dev_priv->saved_flush_status & (1 << 12)))) { flush_flags = dev_priv->flush_flags; flush_sequence = dev_priv->flush_sequence; dev_priv->flush_pending = 0; drm_fence_handler(dev, 0, flush_sequence, flush_flags, 0); } } - } -void i915_poke_flush(struct drm_device *dev, uint32_t class) -{ - struct drm_fence_manager *fm = &dev->fm; - unsigned long flags; - - write_lock_irqsave(&fm->lock, flags); - i915_perform_flush(dev); - write_unlock_irqrestore(&fm->lock, flags); -} - -int i915_fence_emit_sequence(struct drm_device *dev, uint32_t class, +static int i915_fence_emit_sequence(struct drm_device *dev, uint32_t class, uint32_t flags, uint32_t *sequence, uint32_t *native_type) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - if (!dev_priv) + if (unlikely(!dev_priv)) return -EINVAL; i915_emit_irq(dev); @@ -140,20 +164,130 @@ int i915_fence_emit_sequence(struct drm_device *dev, uint32_t class, void i915_fence_handler(struct drm_device *dev) { struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[0]; write_lock(&fm->lock); - i915_perform_flush(dev); + i915_fence_poll(dev, 0, fc->waiting_types); write_unlock(&fm->lock); } -int i915_fence_has_irq(struct drm_device *dev, uint32_t class, uint32_t flags) +/* + * We need a separate wait function since we need to poll for + * sync flushes. + */ + +static int i915_fence_wait(struct drm_fence_object *fence, + int lazy, int interruptible, uint32_t mask) { + struct drm_device *dev = fence->dev; + drm_i915_private_t *dev_priv = (struct drm_i915_private *) dev->dev_private; + struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[0]; + int ret; + unsigned long _end = jiffies + 3 * DRM_HZ; + + drm_fence_object_flush(fence, mask); + if (likely(interruptible)) + ret = wait_event_interruptible_timeout + (fc->fence_queue, drm_fence_object_signaled(fence, DRM_FENCE_TYPE_EXE), + 3 * DRM_HZ); + else + ret = wait_event_timeout + (fc->fence_queue, drm_fence_object_signaled(fence, DRM_FENCE_TYPE_EXE), + 3 * DRM_HZ); + + if (unlikely(ret == -ERESTARTSYS)) + return -EAGAIN; + + if (unlikely(ret == 0)) + return -EBUSY; + + if (likely(mask == DRM_FENCE_TYPE_EXE || + drm_fence_object_signaled(fence, mask))) + return 0; + /* - * We have an irq that tells us when we have a new breadcrumb. + * Remove this code snippet when fixed. HWSTAM doesn't let + * flush info through... */ - if (class == 0 && flags == DRM_FENCE_TYPE_EXE) - return 1; + if (unlikely(dev_priv && !dev_priv->irq_enabled)) { + unsigned long irq_flags; - return 0; + DRM_ERROR("X server disabled IRQs before releasing frame buffer.\n"); + msleep(100); + dev_priv->flush_pending = 0; + write_lock_irqsave(&fm->lock, irq_flags); + drm_fence_handler(dev, fence->fence_class, + fence->sequence, fence->type, 0); + write_unlock_irqrestore(&fm->lock, irq_flags); + } + + /* + * Poll for sync flush completion. + */ + + return drm_fence_wait_polling(fence, lazy, interruptible, mask, _end); +} + +static uint32_t i915_fence_needed_flush(struct drm_fence_object *fence) +{ + uint32_t flush_flags = fence->waiting_types & + ~(DRM_FENCE_TYPE_EXE | fence->signaled_types); + + if (likely(flush_flags == 0 || + ((flush_flags & ~fence->native_types) == 0) || + (fence->signaled_types != DRM_FENCE_TYPE_EXE))) + return 0; + else { + struct drm_device *dev = fence->dev; + struct drm_i915_private *dev_priv = (struct drm_i915_private *) dev->dev_private; + struct drm_fence_driver *driver = dev->driver->fence_driver; + + if (unlikely(!dev_priv)) + return 0; + + if (dev_priv->flush_pending) { + uint32_t diff = (dev_priv->flush_sequence - fence->sequence) & + driver->sequence_mask; + + if (diff < driver->wrap_diff) + return 0; + } + } + return flush_flags; +} + +/* + * In the very unlikely event that "poll" is not really called very often + * we need the following function to handle sequence wraparounds. + */ + +void i915_invalidate_reported_sequence(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = (struct drm_i915_private *) + dev->dev_private; + struct drm_fence_manager *fm = &dev->fm; + unsigned long irq_flags; + + if (unlikely(!dev_priv)) + return; + + write_lock_irqsave(&fm->lock, irq_flags); + dev_priv->reported_sequence_valid = 0; + write_unlock_irqrestore(&fm->lock, irq_flags); } + + +struct drm_fence_driver i915_fence_driver = { + .num_classes = 1, + .wrap_diff = (1U << (BREADCRUMB_BITS - 1)), + .flush_diff = (1U << (BREADCRUMB_BITS - 2)), + .sequence_mask = BREADCRUMB_MASK, + .has_irq = NULL, + .emit = i915_fence_emit_sequence, + .flush = i915_fence_flush, + .poll = i915_fence_poll, + .needed_flush = i915_fence_needed_flush, + .wait = i915_fence_wait, +}; -- cgit v1.2.3