From f1edb7ad91d8b92057ffa02eb162e3740d05a147 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Wed, 30 Jan 2008 22:06:02 +0100 Subject: Simplify the fencing code and differentiate between flushes and waiting types. Add a "command_stream_barrier" method to the bo driver. --- linux-core/drm_bo.c | 34 ++-- linux-core/drm_fence.c | 389 +++++++++++++++++++++----------------------- linux-core/drm_objects.h | 75 +++++++-- linux-core/i915_drv.c | 13 +- linux-core/i915_fence.c | 246 +++++++++++++++++++++------- linux-core/nouveau_buffer.c | 3 +- linux-core/nouveau_fence.c | 32 +--- linux-core/via_fence.c | 114 +++---------- linux-core/xgi_drv.c | 11 +- linux-core/xgi_fence.c | 55 +++---- 10 files changed, 525 insertions(+), 447 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c index df10e12b..3b180d15 100644 --- a/linux-core/drm_bo.c +++ b/linux-core/drm_bo.c @@ -287,7 +287,7 @@ int drm_bo_wait(struct drm_buffer_object *bo, int lazy, int ignore_signals, DRM_ASSERT_LOCKED(&bo->mutex); if (bo->fence) { - if (drm_fence_object_signaled(bo->fence, bo->fence_type, 0)) { + if (drm_fence_object_signaled(bo->fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(&bo->fence); return 0; } @@ -354,7 +354,7 @@ static void drm_bo_cleanup_refs(struct drm_buffer_object *bo, int remove_all) DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); if (bo->fence && drm_fence_object_signaled(bo->fence, - bo->fence_type, 0)) + bo->fence_type)) drm_fence_usage_deref_unlocked(&bo->fence); if (bo->fence && remove_all) @@ -559,7 +559,7 @@ void drm_putback_buffer_objects(struct drm_device *dev) list_del_init(&entry->lru); DRM_FLAG_MASKED(entry->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); - DRM_WAKEUP(&entry->event_queue); + wake_up_all(&entry->event_queue); /* * FIXME: Might want to put back on head of list @@ -660,7 +660,7 @@ int drm_fence_buffer_objects(struct drm_device *dev, entry->fence_type = entry->new_fence_type; DRM_FLAG_MASKED(entry->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); - DRM_WAKEUP(&entry->event_queue); + wake_up_all(&entry->event_queue); drm_bo_add_to_lru(entry); } mutex_unlock(&entry->mutex); @@ -1032,7 +1032,7 @@ static int drm_bo_quick_busy(struct drm_buffer_object *bo) BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED); if (fence) { - if (drm_fence_object_signaled(fence, bo->fence_type, 0)) { + if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(&bo->fence); return 0; } @@ -1052,12 +1052,12 @@ static int drm_bo_busy(struct drm_buffer_object *bo) BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED); if (fence) { - if (drm_fence_object_signaled(fence, bo->fence_type, 0)) { + if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(&bo->fence); return 0; } drm_fence_object_flush(fence, DRM_FENCE_TYPE_EXE); - if (drm_fence_object_signaled(fence, bo->fence_type, 0)) { + if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(&bo->fence); return 0; } @@ -1249,7 +1249,7 @@ static int drm_buffer_object_map(struct drm_file *file_priv, uint32_t handle, mutex_unlock(&dev->struct_mutex); if (ret) { if (atomic_add_negative(-1, &bo->mapped)) - DRM_WAKEUP(&bo->event_queue); + wake_up_all(&bo->event_queue); } else drm_bo_fill_rep_arg(bo, rep); @@ -1306,7 +1306,7 @@ static void drm_buffer_user_object_unmap(struct drm_file *file_priv, BUG_ON(action != _DRM_REF_TYPE1); if (atomic_add_negative(-1, &bo->mapped)) - DRM_WAKEUP(&bo->event_queue); + wake_up_all(&bo->event_queue); } /* @@ -1364,7 +1364,7 @@ out_unlock: } drm_bo_add_to_lru(bo); if (bo->priv_flags & _DRM_BO_FLAG_UNFENCED) { - DRM_WAKEUP(&bo->event_queue); + wake_up_all(&bo->event_queue); DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); } @@ -1442,13 +1442,21 @@ static int drm_buffer_object_validate(struct drm_buffer_object *bo, * We're switching command submission mechanism, * or cannot simply rely on the hardware serializing for us. * - * Wait for buffer idle. + * Insert a driver-dependant barrier or wait for buffer idle. */ if ((fence_class != bo->fence_class) || ((ftype ^ bo->fence_type) & bo->fence_type)) { - ret = drm_bo_wait(bo, 0, 0, no_wait); + ret = -EINVAL; + if (driver->command_stream_barrier) { + ret = driver->command_stream_barrier(bo, + fence_class, + ftype, + no_wait); + } + if (ret) + ret = drm_bo_wait(bo, 0, 0, no_wait); if (ret) return ret; @@ -1539,7 +1547,7 @@ static int drm_buffer_object_validate(struct drm_buffer_object *bo, } else { drm_bo_add_to_lru(bo); if (bo->priv_flags & _DRM_BO_FLAG_UNFENCED) { - DRM_WAKEUP(&bo->event_queue); + wake_up_all(&bo->event_queue); DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); } diff --git a/linux-core/drm_fence.c b/linux-core/drm_fence.c index 288b4db6..a852c63f 100644 --- a/linux-core/drm_fence.c +++ b/linux-core/drm_fence.c @@ -30,6 +30,57 @@ #include "drmP.h" + +/* + * Convenience function to be called by fence::wait methods that + * need polling. + */ + +int drm_fence_wait_polling(struct drm_fence_object *fence, int lazy, + int interruptible, uint32_t mask, + unsigned long end_jiffies) +{ + struct drm_device *dev = fence->dev; + struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class]; + uint32_t count = 0; + int ret; + + DECLARE_WAITQUEUE(entry, current); + add_wait_queue(&fc->fence_queue, &entry); + + ret = 0; + + for (;;) { + __set_current_state((interruptible) ? + TASK_INTERRUPTIBLE : + TASK_UNINTERRUPTIBLE); + if (drm_fence_object_signaled(fence, mask)) + break; + if (time_after_eq(jiffies, end_jiffies)) { + ret = -EBUSY; + break; + } + if (lazy) + schedule_timeout(1); + else if ((++count & 0x0F) == 0){ + __set_current_state(TASK_RUNNING); + schedule(); + __set_current_state((interruptible) ? + TASK_INTERRUPTIBLE : + TASK_UNINTERRUPTIBLE); + } + if (interruptible && signal_pending(current)) { + ret = -EAGAIN; + break; + } + } + __set_current_state(TASK_RUNNING); + remove_wait_queue(&fc->fence_queue, &entry); + return ret; +} +EXPORT_SYMBOL(drm_fence_wait_polling); + /* * Typically called by the IRQ handler. */ @@ -39,27 +90,14 @@ void drm_fence_handler(struct drm_device *dev, uint32_t fence_class, { int wake = 0; uint32_t diff; - uint32_t relevant; + uint32_t relevant_type; + uint32_t new_type; struct drm_fence_manager *fm = &dev->fm; struct drm_fence_class_manager *fc = &fm->fence_class[fence_class]; struct drm_fence_driver *driver = dev->driver->fence_driver; struct list_head *head; struct drm_fence_object *fence, *next; int found = 0; - int is_exe = (type & DRM_FENCE_TYPE_EXE); - int ge_last_exe; - - - diff = (sequence - fc->exe_flush_sequence) & driver->sequence_mask; - - if (fc->pending_exe_flush && is_exe && diff < driver->wrap_diff) - fc->pending_exe_flush = 0; - - diff = (sequence - fc->last_exe_flush) & driver->sequence_mask; - ge_last_exe = diff < driver->wrap_diff; - - if (is_exe && ge_last_exe) - fc->last_exe_flush = sequence; if (list_empty(&fc->ring)) return; @@ -72,7 +110,7 @@ void drm_fence_handler(struct drm_device *dev, uint32_t fence_class, } } - fc->pending_flush &= ~type; + fc->waiting_types &= ~type; head = (found) ? &fence->ring : &fc->ring; list_for_each_entry_safe_reverse(fence, next, head, ring) { @@ -81,64 +119,60 @@ void drm_fence_handler(struct drm_device *dev, uint32_t fence_class, if (error) { fence->error = error; - fence->signaled = fence->type; - fence->submitted_flush = fence->type; - fence->flush_mask = fence->type; + fence->signaled_types = fence->type; list_del_init(&fence->ring); wake = 1; break; } - if (is_exe) - type |= fence->native_type; + if (type & DRM_FENCE_TYPE_EXE) + type |= fence->native_types; - relevant = type & fence->type; + relevant_type = type & fence->type; + new_type = (fence->signaled_types | relevant_type) & + ~fence->signaled_types; - if ((fence->signaled | relevant) != fence->signaled) { - fence->signaled |= relevant; - fence->flush_mask |= relevant; - fence->submitted_flush |= relevant; + if (new_type) { + fence->signaled_types |= new_type; DRM_DEBUG("Fence 0x%08lx signaled 0x%08x\n", - fence->base.hash.key, fence->signaled); - wake = 1; - } + fence->base.hash.key, fence->signaled_types); + + if (driver->needed_flush) + fc->pending_flush |= driver->needed_flush(fence); - relevant = fence->flush_mask & - ~(fence->submitted_flush | fence->signaled); + if (new_type & fence->waiting_types) + wake = 1; + } - fc->pending_flush |= relevant; - fence->submitted_flush |= relevant; + fc->waiting_types |= fence->waiting_types & ~fence->signaled_types; - if (!(fence->type & ~fence->signaled)) { + if (!(fence->type & ~fence->signaled_types)) { DRM_DEBUG("Fence completely signaled 0x%08lx\n", fence->base.hash.key); list_del_init(&fence->ring); } - } /* - * Reinstate lost flush flags. + * Reinstate lost waiting types. */ - if ((fc->pending_flush & type) != type) { + if ((fc->waiting_types & type) != type) { head = head->prev; list_for_each_entry(fence, head, ring) { if (&fence->ring == &fc->ring) break; - diff = (fc->last_exe_flush - fence->sequence) & + diff = (fc->highest_waiting_sequence - fence->sequence) & driver->sequence_mask; if (diff > driver->wrap_diff) break; - - relevant = fence->submitted_flush & ~fence->signaled; - fc->pending_flush |= relevant; + + fc->waiting_types |= fence->waiting_types & ~fence->signaled_types; } } - if (wake) { - DRM_WAKEUP(&fc->fence_queue); - } + if (wake) + wake_up_all(&fc->fence_queue); } EXPORT_SYMBOL(drm_fence_handler); @@ -219,41 +253,28 @@ static void drm_fence_object_destroy(struct drm_file *priv, drm_fence_usage_deref_locked(&fence); } -int drm_fence_object_signaled(struct drm_fence_object *fence, - uint32_t mask, int poke_flush) +int drm_fence_object_signaled(struct drm_fence_object *fence, uint32_t mask) { unsigned long flags; int signaled; struct drm_device *dev = fence->dev; struct drm_fence_manager *fm = &dev->fm; struct drm_fence_driver *driver = dev->driver->fence_driver; - - if (poke_flush) - driver->poke_flush(dev, fence->fence_class); + + mask &= fence->type; read_lock_irqsave(&fm->lock, flags); - signaled = - (fence->type & mask & fence->signaled) == (fence->type & mask); + signaled = (mask & fence->signaled_types) == mask; read_unlock_irqrestore(&fm->lock, flags); - + if (!signaled && driver->poll) { + write_lock_irqsave(&fm->lock, flags); + driver->poll(dev, fence->fence_class, mask); + signaled = (mask & fence->signaled_types) == mask; + write_unlock_irqrestore(&fm->lock, flags); + } return signaled; } EXPORT_SYMBOL(drm_fence_object_signaled); -static void drm_fence_flush_exe(struct drm_fence_class_manager *fc, - struct drm_fence_driver *driver, - uint32_t sequence) -{ - uint32_t diff; - - if (!fc->pending_exe_flush) { - fc->exe_flush_sequence = sequence; - fc->pending_exe_flush = 1; - } else { - diff = (sequence - fc->exe_flush_sequence) & driver->sequence_mask; - if (diff < driver->wrap_diff) - fc->exe_flush_sequence = sequence; - } -} int drm_fence_object_flush(struct drm_fence_object *fence, uint32_t type) @@ -262,7 +283,10 @@ int drm_fence_object_flush(struct drm_fence_object *fence, struct drm_fence_manager *fm = &dev->fm; struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class]; struct drm_fence_driver *driver = dev->driver->fence_driver; - unsigned long flags; + unsigned long irq_flags; + uint32_t saved_pending_flush; + uint32_t diff; + int call_flush; if (type & ~fence->type) { DRM_ERROR("Flush trying to extend fence type, " @@ -270,24 +294,36 @@ int drm_fence_object_flush(struct drm_fence_object *fence, return -EINVAL; } - write_lock_irqsave(&fm->lock, flags); - fence->flush_mask |= type; - if ((fence->submitted_flush & fence->signaled) - == fence->submitted_flush) { - if ((fence->type & DRM_FENCE_TYPE_EXE) && - !(fence->submitted_flush & DRM_FENCE_TYPE_EXE)) { - drm_fence_flush_exe(fc, driver, fence->sequence); - fence->submitted_flush |= DRM_FENCE_TYPE_EXE; - } else { - fc->pending_flush |= (fence->flush_mask & - ~fence->submitted_flush); - fence->submitted_flush = fence->flush_mask; - } - } - write_unlock_irqrestore(&fm->lock, flags); - driver->poke_flush(dev, fence->fence_class); + write_lock_irqsave(&fm->lock, irq_flags); + fence->waiting_types |= type; + fc->waiting_types |= fence->waiting_types; + diff = (fence->sequence - fc->highest_waiting_sequence) & + driver->sequence_mask; + + if (diff < driver->wrap_diff) + fc->highest_waiting_sequence = fence->sequence; + + /* + * fence->waiting_types has changed. Determine whether + * we need to initiate some kind of flush as a result of this. + */ + + saved_pending_flush = fc->pending_flush; + if (driver->needed_flush) + fc->pending_flush |= driver->needed_flush(fence); + + if (driver->poll) + driver->poll(dev, fence->fence_class, fence->waiting_types); + + call_flush = fc->pending_flush; + write_unlock_irqrestore(&fm->lock, irq_flags); + + if (call_flush && driver->flush) + driver->flush(dev, fence->fence_class); + return 0; } +EXPORT_SYMBOL(drm_fence_object_flush); /* * Make sure old fence objects are signaled before their fence sequences are @@ -299,90 +335,52 @@ void drm_fence_flush_old(struct drm_device *dev, uint32_t fence_class, { struct drm_fence_manager *fm = &dev->fm; struct drm_fence_class_manager *fc = &fm->fence_class[fence_class]; - struct drm_fence_driver *driver = dev->driver->fence_driver; - uint32_t old_sequence; - unsigned long flags; struct drm_fence_object *fence; + unsigned long irq_flags; + struct drm_fence_driver *driver = dev->driver->fence_driver; + int call_flush; + uint32_t diff; - write_lock_irqsave(&fm->lock, flags); - old_sequence = (sequence - driver->flush_diff) & driver->sequence_mask; - diff = (old_sequence - fc->last_exe_flush) & driver->sequence_mask; + write_lock_irqsave(&fm->lock, irq_flags); - if ((diff < driver->wrap_diff) && !fc->pending_exe_flush) { - fc->pending_exe_flush = 1; - fc->exe_flush_sequence = sequence - (driver->flush_diff / 2); - } - write_unlock_irqrestore(&fm->lock, flags); + list_for_each_entry_reverse(fence, &fc->ring, ring) { + diff = (sequence - fence->sequence) & driver->sequence_mask; + if (diff <= driver->flush_diff) + break; + + fence->waiting_types = fence->type; + fc->waiting_types |= fence->type; - mutex_lock(&dev->struct_mutex); - read_lock_irqsave(&fm->lock, flags); + if (driver->needed_flush) + fc->pending_flush |= driver->needed_flush(fence); + } + + if (driver->poll) + driver->poll(dev, fence_class, fc->waiting_types); - if (list_empty(&fc->ring)) { - read_unlock_irqrestore(&fm->lock, flags); - mutex_unlock(&dev->struct_mutex); - return; - } - fence = drm_fence_reference_locked(list_entry(fc->ring.next, struct drm_fence_object, ring)); - mutex_unlock(&dev->struct_mutex); - diff = (old_sequence - fence->sequence) & driver->sequence_mask; - read_unlock_irqrestore(&fm->lock, flags); - if (diff < driver->wrap_diff) - drm_fence_object_flush(fence, fence->type); - drm_fence_usage_deref_unlocked(&fence); -} -EXPORT_SYMBOL(drm_fence_flush_old); + call_flush = fc->pending_flush; + write_unlock_irqrestore(&fm->lock, irq_flags); -static int drm_fence_lazy_wait(struct drm_fence_object *fence, - int ignore_signals, - uint32_t mask) -{ - struct drm_device *dev = fence->dev; - struct drm_fence_manager *fm = &dev->fm; - struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class]; - int signaled; - unsigned long _end = jiffies + 3*DRM_HZ; - int ret = 0; + if (call_flush && driver->flush) + driver->flush(dev, fence->fence_class); + + /* + * FIXME: Shold we implement a wait here for really old fences? + */ - do { - DRM_WAIT_ON(ret, fc->fence_queue, 3 * DRM_HZ, - (signaled = drm_fence_object_signaled(fence, mask, 1))); - if (signaled) - return 0; - if (time_after_eq(jiffies, _end)) - break; - } while (ret == -EINTR && ignore_signals); - if (drm_fence_object_signaled(fence, mask, 0)) - return 0; - if (time_after_eq(jiffies, _end)) - ret = -EBUSY; - if (ret) { - if (ret == -EBUSY) { - DRM_ERROR("Fence timeout. " - "GPU lockup or fence driver was " - "taken down. %d 0x%08x 0x%02x 0x%02x 0x%02x\n", - fence->fence_class, - fence->sequence, - fence->type, - mask, - fence->signaled); - DRM_ERROR("Pending exe flush %d 0x%08x\n", - fc->pending_exe_flush, - fc->exe_flush_sequence); - } - return ((ret == -EINTR) ? -EAGAIN : ret); - } - return 0; } +EXPORT_SYMBOL(drm_fence_flush_old); int drm_fence_object_wait(struct drm_fence_object *fence, int lazy, int ignore_signals, uint32_t mask) { struct drm_device *dev = fence->dev; struct drm_fence_driver *driver = dev->driver->fence_driver; + struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class]; int ret = 0; - unsigned long _end; - int signaled; + unsigned long _end = 3 * DRM_HZ; if (mask & ~fence->type) { DRM_ERROR("Wait trying to extend fence type" @@ -391,58 +389,39 @@ int drm_fence_object_wait(struct drm_fence_object *fence, return -EINVAL; } - if (drm_fence_object_signaled(fence, mask, 0)) - return 0; + if (driver->wait) + return driver->wait(fence, lazy, !ignore_signals, mask); - _end = jiffies + 3 * DRM_HZ; drm_fence_object_flush(fence, mask); + if (driver->has_irq(dev, fence->fence_class, mask)) { + if (!ignore_signals) + ret = wait_event_interruptible_timeout + (fc->fence_queue, + drm_fence_object_signaled(fence, mask), + 3 * DRM_HZ); + else + ret = wait_event_timeout + (fc->fence_queue, + drm_fence_object_signaled(fence, mask), + 3 * DRM_HZ); + + if (unlikely(ret == -ERESTARTSYS)) + return -EAGAIN; + + if (unlikely(ret == 0)) + return -EBUSY; - if (lazy && driver->lazy_capable) { - - ret = drm_fence_lazy_wait(fence, ignore_signals, mask); - if (ret) - return ret; - - } else { - - if (driver->has_irq(dev, fence->fence_class, - DRM_FENCE_TYPE_EXE)) { - ret = drm_fence_lazy_wait(fence, ignore_signals, - DRM_FENCE_TYPE_EXE); - if (ret) - return ret; - } - - if (driver->has_irq(dev, fence->fence_class, - mask & ~DRM_FENCE_TYPE_EXE)) { - ret = drm_fence_lazy_wait(fence, ignore_signals, - mask); - if (ret) - return ret; - } - } - if (drm_fence_object_signaled(fence, mask, 0)) return 0; + } - /* - * Avoid kernel-space busy-waits. - */ - if (!ignore_signals) - return -EAGAIN; - - do { - schedule(); - signaled = drm_fence_object_signaled(fence, mask, 1); - } while (!signaled && !time_after_eq(jiffies, _end)); - - if (!signaled) - return -EBUSY; - - return 0; + return drm_fence_wait_polling(fence, lazy, !ignore_signals, mask, + _end); } EXPORT_SYMBOL(drm_fence_object_wait); + + int drm_fence_object_emit(struct drm_fence_object *fence, uint32_t fence_flags, uint32_t fence_class, uint32_t type) { @@ -452,25 +431,24 @@ int drm_fence_object_emit(struct drm_fence_object *fence, uint32_t fence_flags, struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class]; unsigned long flags; uint32_t sequence; - uint32_t native_type; + uint32_t native_types; int ret; drm_fence_unring(dev, &fence->ring); ret = driver->emit(dev, fence_class, fence_flags, &sequence, - &native_type); + &native_types); if (ret) return ret; write_lock_irqsave(&fm->lock, flags); fence->fence_class = fence_class; fence->type = type; - fence->flush_mask = 0x00; - fence->submitted_flush = 0x00; - fence->signaled = 0x00; + fence->waiting_types = 0; + fence->signaled_types = 0; fence->sequence = sequence; - fence->native_type = native_type; + fence->native_types = native_types; if (list_empty(&fc->ring)) - fc->last_exe_flush = sequence - 1; + fc->highest_waiting_sequence = sequence - 1; list_add_tail(&fence->ring, &fc->ring); write_unlock_irqrestore(&fm->lock, flags); return 0; @@ -500,9 +478,8 @@ static int drm_fence_object_init(struct drm_device *dev, uint32_t fence_class, INIT_LIST_HEAD(&fence->base.list); fence->fence_class = fence_class; fence->type = type; - fence->flush_mask = 0; - fence->submitted_flush = 0; - fence->signaled = 0; + fence->signaled_types = 0; + fence->waiting_types = 0; fence->sequence = 0; fence->dev = dev; write_unlock_irqrestore(&fm->lock, flags); @@ -598,7 +575,7 @@ void drm_fence_fill_arg(struct drm_fence_object *fence, arg->handle = fence->base.hash.key; arg->fence_class = fence->fence_class; arg->type = fence->type; - arg->signaled = fence->signaled; + arg->signaled = fence->signaled_types; arg->error = fence->error; arg->sequence = fence->sequence; read_unlock_irqrestore(&fm->lock, irq_flags); diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h index a2d10b5d..c35d900f 100644 --- a/linux-core/drm_objects.h +++ b/linux-core/drm_objects.h @@ -147,12 +147,11 @@ struct drm_fence_object { struct list_head ring; int fence_class; - uint32_t native_type; + uint32_t native_types; uint32_t type; - uint32_t signaled; + uint32_t signaled_types; uint32_t sequence; - uint32_t flush_mask; - uint32_t submitted_flush; + uint32_t waiting_types; uint32_t error; }; @@ -162,10 +161,9 @@ struct drm_fence_object { struct drm_fence_class_manager { struct list_head ring; uint32_t pending_flush; + uint32_t waiting_types; wait_queue_head_t fence_queue; - int pending_exe_flush; - uint32_t last_exe_flush; - uint32_t exe_flush_sequence; + uint32_t highest_waiting_sequence; }; struct drm_fence_manager { @@ -177,19 +175,49 @@ struct drm_fence_manager { }; struct drm_fence_driver { + unsigned long *waiting_jiffies; uint32_t num_classes; uint32_t wrap_diff; uint32_t flush_diff; uint32_t sequence_mask; - int lazy_capable; + + /* + * Driver implemented functions: + * has_irq() : 1 if the hardware can update the indicated type_flags using an + * irq handler. 0 if polling is required. + * + * emit() : Emit a sequence number to the command stream. + * Return the sequence number. + * + * flush() : Make sure the flags indicated in fc->pending_flush will eventually + * signal for fc->highest_received_sequence and all preceding sequences. + * Acknowledge by clearing the flags fc->pending_flush. + * + * poll() : Call drm_fence_handler with any new information. + * + * needed_flush() : Given the current state of the fence->type flags and previusly + * executed or queued flushes, return the type_flags that need flushing. + * + * wait(): Wait for the "mask" flags to signal on a given fence, performing + * whatever's necessary to make this happen. + */ + int (*has_irq) (struct drm_device *dev, uint32_t fence_class, uint32_t flags); int (*emit) (struct drm_device *dev, uint32_t fence_class, uint32_t flags, uint32_t *breadcrumb, uint32_t *native_type); - void (*poke_flush) (struct drm_device *dev, uint32_t fence_class); + void (*flush) (struct drm_device *dev, uint32_t fence_class); + void (*poll) (struct drm_device *dev, uint32_t fence_class, + uint32_t types); + uint32_t (*needed_flush) (struct drm_fence_object *fence); + int (*wait) (struct drm_fence_object *fence, int lazy, + int interruptible, uint32_t mask); }; +extern int drm_fence_wait_polling(struct drm_fence_object *fence, int lazy, + int interruptible, uint32_t mask, + unsigned long end_jiffies); extern void drm_fence_handler(struct drm_device *dev, uint32_t fence_class, uint32_t sequence, uint32_t type, uint32_t error); @@ -200,7 +228,7 @@ extern void drm_fence_flush_old(struct drm_device *dev, uint32_t fence_class, extern int drm_fence_object_flush(struct drm_fence_object *fence, uint32_t type); extern int drm_fence_object_signaled(struct drm_fence_object *fence, - uint32_t type, int flush); + uint32_t type); extern void drm_fence_usage_deref_locked(struct drm_fence_object **fence); extern void drm_fence_usage_deref_unlocked(struct drm_fence_object **fence); extern struct drm_fence_object *drm_fence_reference_locked(struct drm_fence_object *src); @@ -576,6 +604,33 @@ struct drm_bo_driver { * ttm_cache_flush */ void (*ttm_cache_flush)(struct drm_ttm *ttm); + + /* + * command_stream_barrier + * + * @dev: The drm device. + * + * @bo: The buffer object to validate. + * + * @new_fence_class: The new fence class for the buffer object. + * + * @new_fence_type: The new fence type for the buffer object. + * + * @no_wait: whether this should give up and return -EBUSY + * if this operation would require sleeping + * + * Insert a command stream barrier that makes sure that the + * buffer is idle once the commands associated with the + * current validation are starting to execute. If an error + * condition is returned, or the function pointer is NULL, + * the drm core will force buffer idle + * during validation. + */ + + int (*command_stream_barrier) (struct drm_buffer_object *bo, + uint32_t new_fence_class, + uint32_t new_fence_type, + int no_wait); }; /* diff --git a/linux-core/i915_drv.c b/linux-core/i915_drv.c index 5f2e6adc..ccc061d3 100644 --- a/linux-core/i915_drv.c +++ b/linux-core/i915_drv.c @@ -39,17 +39,9 @@ static struct pci_device_id pciidlist[] = { }; #ifdef I915_HAVE_FENCE -static struct drm_fence_driver i915_fence_driver = { - .num_classes = 1, - .wrap_diff = (1U << (BREADCRUMB_BITS - 1)), - .flush_diff = (1U << (BREADCRUMB_BITS - 2)), - .sequence_mask = BREADCRUMB_MASK, - .lazy_capable = 1, - .emit = i915_fence_emit_sequence, - .poke_flush = i915_poke_flush, - .has_irq = i915_fence_has_irq, -}; +extern struct drm_fence_driver i915_fence_driver; #endif + #ifdef I915_HAVE_BUFFER static uint32_t i915_mem_prios[] = {DRM_BO_MEM_PRIV0, DRM_BO_MEM_TT, DRM_BO_MEM_LOCAL}; @@ -67,6 +59,7 @@ static struct drm_bo_driver i915_bo_driver = { .evict_flags = i915_evict_flags, .move = i915_move, .ttm_cache_flush = i915_flush_ttm, + .command_stream_barrier = NULL, }; #endif diff --git a/linux-core/i915_fence.c b/linux-core/i915_fence.c index e3c76df6..8a2e7f13 100644 --- a/linux-core/i915_fence.c +++ b/linux-core/i915_fence.c @@ -35,97 +35,121 @@ #include "i915_drv.h" /* - * Implements an intel sync flush operation. + * Initiate a sync flush if it's not already pending. */ -static void i915_perform_flush(struct drm_device *dev) +static void i915_initiate_rwflush(struct drm_i915_private *dev_priv, + struct drm_fence_class_manager *fc) +{ + if ((fc->pending_flush & DRM_I915_FENCE_TYPE_RW) && + !dev_priv->flush_pending) { + dev_priv->flush_sequence = (uint32_t) READ_BREADCRUMB(dev_priv); + dev_priv->flush_flags = fc->pending_flush; + dev_priv->saved_flush_status = READ_HWSP(dev_priv, 0); + I915_WRITE(I915REG_INSTPM, (1 << 5) | (1 << 21)); + dev_priv->flush_pending = 1; + fc->pending_flush &= ~DRM_I915_FENCE_TYPE_RW; + } +} + +static void i915_fence_flush(struct drm_device *dev, + uint32_t fence_class) +{ + struct drm_i915_private *dev_priv = + (struct drm_i915_private *) dev->dev_private; + struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[0]; + unsigned long irq_flags; + + if (unlikely(!dev_priv)) + return; + + write_lock_irqsave(&fm->lock, irq_flags); + i915_initiate_rwflush(dev_priv, fc); + write_unlock_irqrestore(&fm->lock, irq_flags); +} + +static void i915_fence_poll(struct drm_device *dev, uint32_t fence_class, + uint32_t waiting_types) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; struct drm_fence_manager *fm = &dev->fm; struct drm_fence_class_manager *fc = &fm->fence_class[0]; - struct drm_fence_driver *driver = dev->driver->fence_driver; uint32_t flush_flags = 0; uint32_t flush_sequence = 0; uint32_t i_status; - uint32_t diff; uint32_t sequence; - int rwflush; - if (!dev_priv) + if (unlikely(!dev_priv)) return; - if (fc->pending_exe_flush) { - sequence = READ_BREADCRUMB(dev_priv); + /* + * First, report any executed sync flush: + */ + + if (dev_priv->flush_pending) { + i_status = READ_HWSP(dev_priv, 0); + if ((i_status & (1 << 12)) != + (dev_priv->saved_flush_status & (1 << 12))) { + flush_flags = dev_priv->flush_flags; + flush_sequence = dev_priv->flush_sequence; + dev_priv->flush_pending = 0; + drm_fence_handler(dev, 0, flush_sequence, flush_flags, 0); + } + } - /* - * First update fences with the current breadcrumb. - */ + /* + * Report A new breadcrumb, and adjust IRQs. + */ + + if (waiting_types & DRM_FENCE_TYPE_EXE) { + sequence = READ_BREADCRUMB(dev_priv); - diff = (sequence - fc->last_exe_flush) & BREADCRUMB_MASK; - if (diff < driver->wrap_diff && diff != 0) { - drm_fence_handler(dev, 0, sequence, + if (sequence != dev_priv->reported_sequence || + !dev_priv->reported_sequence_valid) { + drm_fence_handler(dev, 0, sequence, DRM_FENCE_TYPE_EXE, 0); + dev_priv->reported_sequence = sequence; + dev_priv->reported_sequence_valid = 1; } - if (dev_priv->fence_irq_on && !fc->pending_exe_flush) { + if (dev_priv->fence_irq_on && !(waiting_types & DRM_FENCE_TYPE_EXE)) { i915_user_irq_off(dev_priv); dev_priv->fence_irq_on = 0; - } else if (!dev_priv->fence_irq_on && fc->pending_exe_flush) { + } else if (!dev_priv->fence_irq_on && (waiting_types & DRM_FENCE_TYPE_EXE)) { i915_user_irq_on(dev_priv); dev_priv->fence_irq_on = 1; } } - if (dev_priv->flush_pending) { - i_status = READ_HWSP(dev_priv, 0); - if ((i_status & (1 << 12)) != - (dev_priv->saved_flush_status & (1 << 12))) { - flush_flags = dev_priv->flush_flags; - flush_sequence = dev_priv->flush_sequence; - dev_priv->flush_pending = 0; - drm_fence_handler(dev, 0, flush_sequence, flush_flags, 0); - } - } + /* + * There may be new RW flushes pending. Start them. + */ + + i915_initiate_rwflush(dev_priv, fc); - rwflush = fc->pending_flush & DRM_I915_FENCE_TYPE_RW; - if (rwflush && !dev_priv->flush_pending) { - dev_priv->flush_sequence = (uint32_t) READ_BREADCRUMB(dev_priv); - dev_priv->flush_flags = fc->pending_flush; - dev_priv->saved_flush_status = READ_HWSP(dev_priv, 0); - I915_WRITE(I915REG_INSTPM, (1 << 5) | (1 << 21)); - dev_priv->flush_pending = 1; - fc->pending_flush &= ~DRM_I915_FENCE_TYPE_RW; - } + /* + * And possibly, but unlikely, they finish immediately. + */ if (dev_priv->flush_pending) { i_status = READ_HWSP(dev_priv, 0); - if ((i_status & (1 << 12)) != - (dev_priv->saved_flush_status & (1 << 12))) { + if (unlikely((i_status & (1 << 12)) != + (dev_priv->saved_flush_status & (1 << 12)))) { flush_flags = dev_priv->flush_flags; flush_sequence = dev_priv->flush_sequence; dev_priv->flush_pending = 0; drm_fence_handler(dev, 0, flush_sequence, flush_flags, 0); } } - } -void i915_poke_flush(struct drm_device *dev, uint32_t class) -{ - struct drm_fence_manager *fm = &dev->fm; - unsigned long flags; - - write_lock_irqsave(&fm->lock, flags); - i915_perform_flush(dev); - write_unlock_irqrestore(&fm->lock, flags); -} - -int i915_fence_emit_sequence(struct drm_device *dev, uint32_t class, +static int i915_fence_emit_sequence(struct drm_device *dev, uint32_t class, uint32_t flags, uint32_t *sequence, uint32_t *native_type) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - if (!dev_priv) + if (unlikely(!dev_priv)) return -EINVAL; i915_emit_irq(dev); @@ -140,20 +164,130 @@ int i915_fence_emit_sequence(struct drm_device *dev, uint32_t class, void i915_fence_handler(struct drm_device *dev) { struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[0]; write_lock(&fm->lock); - i915_perform_flush(dev); + i915_fence_poll(dev, 0, fc->waiting_types); write_unlock(&fm->lock); } -int i915_fence_has_irq(struct drm_device *dev, uint32_t class, uint32_t flags) +/* + * We need a separate wait function since we need to poll for + * sync flushes. + */ + +static int i915_fence_wait(struct drm_fence_object *fence, + int lazy, int interruptible, uint32_t mask) { + struct drm_device *dev = fence->dev; + drm_i915_private_t *dev_priv = (struct drm_i915_private *) dev->dev_private; + struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[0]; + int ret; + unsigned long _end = jiffies + 3 * DRM_HZ; + + drm_fence_object_flush(fence, mask); + if (likely(interruptible)) + ret = wait_event_interruptible_timeout + (fc->fence_queue, drm_fence_object_signaled(fence, DRM_FENCE_TYPE_EXE), + 3 * DRM_HZ); + else + ret = wait_event_timeout + (fc->fence_queue, drm_fence_object_signaled(fence, DRM_FENCE_TYPE_EXE), + 3 * DRM_HZ); + + if (unlikely(ret == -ERESTARTSYS)) + return -EAGAIN; + + if (unlikely(ret == 0)) + return -EBUSY; + + if (likely(mask == DRM_FENCE_TYPE_EXE || + drm_fence_object_signaled(fence, mask))) + return 0; + /* - * We have an irq that tells us when we have a new breadcrumb. + * Remove this code snippet when fixed. HWSTAM doesn't let + * flush info through... */ - if (class == 0 && flags == DRM_FENCE_TYPE_EXE) - return 1; + if (unlikely(dev_priv && !dev_priv->irq_enabled)) { + unsigned long irq_flags; - return 0; + DRM_ERROR("X server disabled IRQs before releasing frame buffer.\n"); + msleep(100); + dev_priv->flush_pending = 0; + write_lock_irqsave(&fm->lock, irq_flags); + drm_fence_handler(dev, fence->fence_class, + fence->sequence, fence->type, 0); + write_unlock_irqrestore(&fm->lock, irq_flags); + } + + /* + * Poll for sync flush completion. + */ + + return drm_fence_wait_polling(fence, lazy, interruptible, mask, _end); +} + +static uint32_t i915_fence_needed_flush(struct drm_fence_object *fence) +{ + uint32_t flush_flags = fence->waiting_types & + ~(DRM_FENCE_TYPE_EXE | fence->signaled_types); + + if (likely(flush_flags == 0 || + ((flush_flags & ~fence->native_types) == 0) || + (fence->signaled_types != DRM_FENCE_TYPE_EXE))) + return 0; + else { + struct drm_device *dev = fence->dev; + struct drm_i915_private *dev_priv = (struct drm_i915_private *) dev->dev_private; + struct drm_fence_driver *driver = dev->driver->fence_driver; + + if (unlikely(!dev_priv)) + return 0; + + if (dev_priv->flush_pending) { + uint32_t diff = (dev_priv->flush_sequence - fence->sequence) & + driver->sequence_mask; + + if (diff < driver->wrap_diff) + return 0; + } + } + return flush_flags; +} + +/* + * In the very unlikely event that "poll" is not really called very often + * we need the following function to handle sequence wraparounds. + */ + +void i915_invalidate_reported_sequence(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = (struct drm_i915_private *) + dev->dev_private; + struct drm_fence_manager *fm = &dev->fm; + unsigned long irq_flags; + + if (unlikely(!dev_priv)) + return; + + write_lock_irqsave(&fm->lock, irq_flags); + dev_priv->reported_sequence_valid = 0; + write_unlock_irqrestore(&fm->lock, irq_flags); } + + +struct drm_fence_driver i915_fence_driver = { + .num_classes = 1, + .wrap_diff = (1U << (BREADCRUMB_BITS - 1)), + .flush_diff = (1U << (BREADCRUMB_BITS - 2)), + .sequence_mask = BREADCRUMB_MASK, + .has_irq = NULL, + .emit = i915_fence_emit_sequence, + .flush = i915_fence_flush, + .poll = i915_fence_poll, + .needed_flush = i915_fence_needed_flush, + .wait = i915_fence_wait, +}; diff --git a/linux-core/nouveau_buffer.c b/linux-core/nouveau_buffer.c index a652bb1d..11549317 100644 --- a/linux-core/nouveau_buffer.c +++ b/linux-core/nouveau_buffer.c @@ -294,5 +294,6 @@ struct drm_bo_driver nouveau_bo_driver = { .init_mem_type = nouveau_bo_init_mem_type, .evict_flags = nouveau_bo_evict_flags, .move = nouveau_bo_move, - .ttm_cache_flush= nouveau_bo_flush_ttm + .ttm_cache_flush= nouveau_bo_flush_ttm, + .command_stream_barrier = NULL }; diff --git a/linux-core/nouveau_fence.c b/linux-core/nouveau_fence.c index 4e624a7a..59dcf7d0 100644 --- a/linux-core/nouveau_fence.c +++ b/linux-core/nouveau_fence.c @@ -75,7 +75,7 @@ nouveau_fence_emit(struct drm_device *dev, uint32_t class, uint32_t flags, } static void -nouveau_fence_perform_flush(struct drm_device *dev, uint32_t class) +nouveau_fence_poll(struct drm_device *dev, uint32_t class, uint32_t waiting_types) { struct drm_nouveau_private *dev_priv = dev->dev_private; struct drm_fence_class_manager *fc = &dev->fm.fence_class[class]; @@ -83,42 +83,26 @@ nouveau_fence_perform_flush(struct drm_device *dev, uint32_t class) uint32_t pending_types = 0; DRM_DEBUG("class=%d\n", class); - - pending_types = fc->pending_flush | - ((fc->pending_exe_flush) ? DRM_FENCE_TYPE_EXE : 0); - DRM_DEBUG("pending: 0x%08x 0x%08x\n", pending_types, - fc->pending_flush); + DRM_DEBUG("pending: 0x%08x 0x%08x\n", waiting_types, fc->waiting_types); if (pending_types) { uint32_t sequence = NV_READ(chan->ref_cnt); DRM_DEBUG("got 0x%08x\n", sequence); - drm_fence_handler(dev, class, sequence, pending_types, 0); + drm_fence_handler(dev, class, sequence, waiting_types, 0); } } -static void -nouveau_fence_poke_flush(struct drm_device *dev, uint32_t class) -{ - struct drm_fence_manager *fm = &dev->fm; - unsigned long flags; - - DRM_DEBUG("class=%d\n", class); - - write_lock_irqsave(&fm->lock, flags); - nouveau_fence_perform_flush(dev, class); - write_unlock_irqrestore(&fm->lock, flags); -} - void nouveau_fence_handler(struct drm_device *dev, int channel) { struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[channel]; DRM_DEBUG("class=%d\n", channel); write_lock(&fm->lock); - nouveau_fence_perform_flush(dev, channel); + nouveau_fence_poll(dev, channel, fc->waiting_types); write_unlock(&fm->lock); } @@ -127,8 +111,10 @@ struct drm_fence_driver nouveau_fence_driver = { .wrap_diff = (1 << 30), .flush_diff = (1 << 29), .sequence_mask = 0xffffffffU, - .lazy_capable = 1, .has_irq = nouveau_fence_has_irq, .emit = nouveau_fence_emit, - .poke_flush = nouveau_fence_poke_flush + .flush = NULL, + .poll = nouveau_fence_poll, + .needed_flush = NULL, + .wait = NULL }; diff --git a/linux-core/via_fence.c b/linux-core/via_fence.c index 9af1bf3b..3a680a32 100644 --- a/linux-core/via_fence.c +++ b/linux-core/via_fence.c @@ -38,27 +38,21 @@ * DRM_VIA_FENCE_TYPE_ACCEL guarantees that all 2D & 3D rendering is complete. */ - -static uint32_t via_perform_flush(struct drm_device *dev, uint32_t class) +static void via_fence_poll(struct drm_device *dev, uint32_t class, + uint32_t waiting_types) { drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private; - struct drm_fence_class_manager *fc = &dev->fm.fence_class[class]; - uint32_t pending_flush_types = 0; uint32_t signaled_flush_types = 0; uint32_t status; if (class != 0) - return 0; + return; - if (!dev_priv) - return 0; + if (unlikely(!dev_priv)) + return; spin_lock(&dev_priv->fence_lock); - - pending_flush_types = fc->pending_flush | - ((fc->pending_exe_flush) ? DRM_FENCE_TYPE_EXE : 0); - - if (pending_flush_types) { + if (waiting_types) { /* * Take the idlelock. This guarantees that the next time a client tries @@ -77,7 +71,7 @@ static uint32_t via_perform_flush(struct drm_device *dev, uint32_t class) * Check if AGP command reader is idle. */ - if (pending_flush_types & DRM_FENCE_TYPE_EXE) + if (waiting_types & DRM_FENCE_TYPE_EXE) if (VIA_READ(0x41C) & 0x80000000) signaled_flush_types |= DRM_FENCE_TYPE_EXE; @@ -85,7 +79,7 @@ static uint32_t via_perform_flush(struct drm_device *dev, uint32_t class) * Check VRAM command queue empty and 2D + 3D engines idle. */ - if (pending_flush_types & DRM_VIA_FENCE_TYPE_ACCEL) { + if (waiting_types & DRM_VIA_FENCE_TYPE_ACCEL) { status = VIA_READ(VIA_REG_STATUS); if ((status & VIA_VR_QUEUE_BUSY) && !(status & (VIA_CMD_RGTR_BUSY | VIA_2D_ENG_BUSY | VIA_3D_ENG_BUSY))) @@ -93,8 +87,8 @@ static uint32_t via_perform_flush(struct drm_device *dev, uint32_t class) } if (signaled_flush_types) { - pending_flush_types &= ~signaled_flush_types; - if (!pending_flush_types && dev_priv->have_idlelock) { + waiting_types &= ~signaled_flush_types; + if (!waiting_types && dev_priv->have_idlelock) { drm_idlelock_release(&dev->lock); dev_priv->have_idlelock = 0; } @@ -105,8 +99,7 @@ static uint32_t via_perform_flush(struct drm_device *dev, uint32_t class) spin_unlock(&dev_priv->fence_lock); - return fc->pending_flush | - ((fc->pending_exe_flush) ? DRM_FENCE_TYPE_EXE : 0); + return; } @@ -114,8 +107,8 @@ static uint32_t via_perform_flush(struct drm_device *dev, uint32_t class) * Emit a fence sequence. */ -int via_fence_emit_sequence(struct drm_device * dev, uint32_t class, uint32_t flags, - uint32_t * sequence, uint32_t * native_type) +static int via_fence_emit_sequence(struct drm_device * dev, uint32_t class, uint32_t flags, + uint32_t * sequence, uint32_t * native_type) { drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private; int ret = 0; @@ -149,36 +142,6 @@ int via_fence_emit_sequence(struct drm_device * dev, uint32_t class, uint32_t fl return ret; } -/** - * Manual poll (from the fence manager). - */ - -void via_poke_flush(struct drm_device * dev, uint32_t class) -{ - drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private; - struct drm_fence_manager *fm = &dev->fm; - unsigned long flags; - uint32_t pending_flush; - - if (!dev_priv) - return ; - - write_lock_irqsave(&fm->lock, flags); - pending_flush = via_perform_flush(dev, class); - if (pending_flush) - pending_flush = via_perform_flush(dev, class); - write_unlock_irqrestore(&fm->lock, flags); - - /* - * Kick the timer if there are more flushes pending. - */ - - if (pending_flush && !timer_pending(&dev_priv->fence_timer)) { - dev_priv->fence_timer.expires = jiffies + 1; - add_timer(&dev_priv->fence_timer); - } -} - /** * No irq fence expirations implemented yet. * Although both the HQV engines and PCI dmablit engines signal @@ -187,45 +150,20 @@ void via_poke_flush(struct drm_device * dev, uint32_t class) * unless the caller wanting to wait for a fence object has indicated a lazy wait. */ -int via_fence_has_irq(struct drm_device * dev, uint32_t class, - uint32_t flags) +static int via_fence_has_irq(struct drm_device * dev, uint32_t class, + uint32_t flags) { return 0; } -/** - * Regularly call the flush function. This enables lazy waits, so we can - * set lazy_capable. Lazy waits don't really care when the fence expires, - * so a timer tick delay should be fine. - */ - -void via_fence_timer(unsigned long data) -{ - struct drm_device *dev = (struct drm_device *) data; - drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private; - struct drm_fence_manager *fm = &dev->fm; - uint32_t pending_flush; - struct drm_fence_class_manager *fc = &dev->fm.fence_class[0]; - - if (!dev_priv) - return; - if (!fm->initialized) - goto out_unlock; - - via_poke_flush(dev, 0); - pending_flush = fc->pending_flush | - ((fc->pending_exe_flush) ? DRM_FENCE_TYPE_EXE : 0); - - /* - * disable timer if there are no more flushes pending. - */ - - if (!pending_flush && timer_pending(&dev_priv->fence_timer)) { - BUG_ON(dev_priv->have_idlelock); - del_timer(&dev_priv->fence_timer); - } - return; -out_unlock: - return; - -} +struct drm_fence_driver via_fence_driver = { + .num_classes = 1, + .wrap_diff = (1 << 30), + .flush_diff = (1 << 20), + .sequence_mask = 0xffffffffU, + .has_irq = via_fence_has_irq, + .emit = via_fence_emit_sequence, + .poll = via_fence_poll, + .needed_flush = NULL, + .wait = NULL +}; diff --git a/linux-core/xgi_drv.c b/linux-core/xgi_drv.c index 4f0b4ed0..f0225f89 100644 --- a/linux-core/xgi_drv.c +++ b/linux-core/xgi_drv.c @@ -37,16 +37,7 @@ static struct pci_device_id pciidlist[] = { xgi_PCI_IDS }; -static struct drm_fence_driver xgi_fence_driver = { - .num_classes = 1, - .wrap_diff = BEGIN_BEGIN_IDENTIFICATION_MASK, - .flush_diff = BEGIN_BEGIN_IDENTIFICATION_MASK - 1, - .sequence_mask = BEGIN_BEGIN_IDENTIFICATION_MASK, - .lazy_capable = 1, - .emit = xgi_fence_emit_sequence, - .poke_flush = xgi_poke_flush, - .has_irq = xgi_fence_has_irq -}; +extern struct drm_fence_driver xgi_fence_driver; int xgi_bootstrap(struct drm_device *, void *, struct drm_file *); diff --git a/linux-core/xgi_fence.c b/linux-core/xgi_fence.c index 9a75581a..63ed29ee 100644 --- a/linux-core/xgi_fence.c +++ b/linux-core/xgi_fence.c @@ -30,44 +30,37 @@ #include "xgi_misc.h" #include "xgi_cmdlist.h" -static uint32_t xgi_do_flush(struct drm_device * dev, uint32_t class) +static void xgi_fence_poll(struct drm_device * dev, uint32_t class, + uint32_t waiting_types) { struct xgi_info * info = dev->dev_private; - struct drm_fence_class_manager * fc = &dev->fm.fence_class[class]; - uint32_t pending_flush_types = 0; - uint32_t signaled_flush_types = 0; + uint32_t signaled_types = 0; if ((info == NULL) || (class != 0)) - return 0; + return; DRM_SPINLOCK(&info->fence_lock); - pending_flush_types = fc->pending_flush | - ((fc->pending_exe_flush) ? DRM_FENCE_TYPE_EXE : 0); - - if (pending_flush_types) { - if (pending_flush_types & DRM_FENCE_TYPE_EXE) { + if (waiting_types) { + if (waiting_types & DRM_FENCE_TYPE_EXE) { const u32 begin_id = le32_to_cpu(DRM_READ32(info->mmio_map, 0x2820)) & BEGIN_BEGIN_IDENTIFICATION_MASK; if (begin_id != info->complete_sequence) { info->complete_sequence = begin_id; - signaled_flush_types |= DRM_FENCE_TYPE_EXE; + signaled_types |= DRM_FENCE_TYPE_EXE; } } - if (signaled_flush_types) { + if (signaled_types) { drm_fence_handler(dev, 0, info->complete_sequence, - signaled_flush_types, 0); + signaled_types, 0); } } DRM_SPINUNLOCK(&info->fence_lock); - - return fc->pending_flush | - ((fc->pending_exe_flush) ? DRM_FENCE_TYPE_EXE : 0); } @@ -98,25 +91,13 @@ int xgi_fence_emit_sequence(struct drm_device * dev, uint32_t class, } -void xgi_poke_flush(struct drm_device * dev, uint32_t class) -{ - struct drm_fence_manager * fm = &dev->fm; - unsigned long flags; - - - write_lock_irqsave(&fm->lock, flags); - xgi_do_flush(dev, class); - write_unlock_irqrestore(&fm->lock, flags); -} - - void xgi_fence_handler(struct drm_device * dev) { struct drm_fence_manager * fm = &dev->fm; - + struct drm_fence_class_manager *fc = &fm->fence_class[0]; write_lock(&fm->lock); - xgi_do_flush(dev, 0); + xgi_fence_poll(dev, 0, fc->waiting_types); write_unlock(&fm->lock); } @@ -125,3 +106,17 @@ int xgi_fence_has_irq(struct drm_device *dev, uint32_t class, uint32_t flags) { return ((class == 0) && (flags == DRM_FENCE_TYPE_EXE)) ? 1 : 0; } + +struct drm_fence_driver xgi_fence_driver = { + .num_classes = 1, + .wrap_diff = BEGIN_BEGIN_IDENTIFICATION_MASK, + .flush_diff = BEGIN_BEGIN_IDENTIFICATION_MASK - 1, + .sequence_mask = BEGIN_BEGIN_IDENTIFICATION_MASK, + .has_irq = xgi_fence_has_irq, + .emit = xgi_fence_emit_sequence, + .flush = NULL, + .poll = xgi_fence_poll, + .needed_flush = NULL, + .wait = NULL +}; + -- cgit v1.2.3