From a7dc4d08b9b4f8fe6fcaa4c778f6dd3718d1e36a Mon Sep 17 00:00:00 2001 From: Jerome Glisse Date: Mon, 10 Mar 2008 23:35:07 +0100 Subject: rradeon_ms: rework fence code and bring radeon ms up to date --- shared-core/radeon_ms.h | 14 +-- shared-core/radeon_ms_drm.c | 22 ++-- shared-core/radeon_ms_drm.h | 23 ++-- shared-core/radeon_ms_exec.c | 2 +- shared-core/radeon_ms_fence.c | 248 ++++++++++++++++++++++++++++++++---------- shared-core/radeon_ms_irq.c | 4 +- 6 files changed, 226 insertions(+), 87 deletions(-) (limited to 'shared-core') diff --git a/shared-core/radeon_ms.h b/shared-core/radeon_ms.h index 903de97d..12c945f9 100644 --- a/shared-core/radeon_ms.h +++ b/shared-core/radeon_ms.h @@ -31,6 +31,7 @@ #define __RADEON_MS_H__ #include "radeon_ms_drv.h" +#include "amd_r3xx_fence.h" #include "radeon_ms_reg.h" #include "radeon_ms_drm.h" #include "radeon_ms_rom.h" @@ -328,6 +329,7 @@ struct drm_radeon_private { /* abstract asic specific structures */ struct radeon_ms_rom rom; struct radeon_ms_properties properties; + void *fence; }; @@ -425,15 +427,9 @@ int radeon_ms_execbuffer(struct drm_device *dev, void *data, int radeon_ms_family_init(struct drm_device *dev); /* radeon_ms_fence.c */ -int radeon_ms_fence_emit_sequence(struct drm_device *dev, uint32_t class, - uint32_t flags, uint32_t *sequence, - uint32_t *native_type); -void radeon_ms_fence_handler(struct drm_device * dev); -int radeon_ms_fence_has_irq(struct drm_device *dev, uint32_t class, - uint32_t flags); -int radeon_ms_fence_types(struct drm_buffer_object *bo, - uint32_t * class, uint32_t * type); -void radeon_ms_poke_flush(struct drm_device * dev, uint32_t class); +void r3xx_fence_handler(struct drm_device * dev); +int r3xx_fence_types(struct drm_buffer_object *bo, + uint32_t * class, uint32_t * type); /* radeon_ms_fb.c */ int radeonfb_probe(struct drm_device *dev, struct drm_crtc *crtc); diff --git a/shared-core/radeon_ms_drm.c b/shared-core/radeon_ms_drm.c index 857182ae..b9245d99 100644 --- a/shared-core/radeon_ms_drm.c +++ b/shared-core/radeon_ms_drm.c @@ -43,24 +43,13 @@ static uint32_t radeon_ms_busy_prios[] = { DRM_BO_MEM_LOCAL, }; -struct drm_fence_driver radeon_ms_fence_driver = { - .num_classes = 1, - .wrap_diff = (1 << 30), - .flush_diff = (1 << 29), - .sequence_mask = 0xffffffffU, - .lazy_capable = 1, - .emit = radeon_ms_fence_emit_sequence, - .poke_flush = radeon_ms_poke_flush, - .has_irq = radeon_ms_fence_has_irq, -}; - struct drm_bo_driver radeon_ms_bo_driver = { .mem_type_prio = radeon_ms_mem_prios, .mem_busy_prio = radeon_ms_busy_prios, .num_mem_type_prio = sizeof(radeon_ms_mem_prios)/sizeof(uint32_t), .num_mem_busy_prio = sizeof(radeon_ms_busy_prios)/sizeof(uint32_t), .create_ttm_backend_entry = radeon_ms_create_ttm_backend, - .fence_type = radeon_ms_fence_types, + .fence_type = r3xx_fence_types, .invalidate_caches = radeon_ms_invalidate_caches, .init_mem_type = radeon_ms_init_mem_type, .evict_flags = radeon_ms_evict_flags, @@ -127,6 +116,13 @@ int radeon_ms_driver_load(struct drm_device *dev, unsigned long flags) return ret; } + dev_priv->fence = drm_alloc(sizeof(struct r3xx_fence), DRM_MEM_DRIVER); + if (dev_priv->fence == NULL) { + radeon_ms_driver_unload(dev); + return -ENOMEM; + } + memset(dev_priv->fence, 0, sizeof(struct r3xx_fence)); + /* we don't want userspace to be able to map this so don't use * drm_addmap */ dev_priv->mmio.offset = drm_get_resource_start(dev, 2); @@ -309,9 +305,11 @@ int radeon_ms_driver_unload(struct drm_device *dev) drm_core_ioremapfree(&dev_priv->vram, dev); } DRM_INFO("[radeon_ms] map released\n"); + drm_free(dev_priv->fence, sizeof(struct r3xx_fence), DRM_MEM_DRIVER); drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER); dev->dev_private = NULL; + DRM_INFO("[radeon_ms] that's all the folks\n"); return 0; } diff --git a/shared-core/radeon_ms_drm.h b/shared-core/radeon_ms_drm.h index 842d5331..39c050ad 100644 --- a/shared-core/radeon_ms_drm.h +++ b/shared-core/radeon_ms_drm.h @@ -30,16 +30,23 @@ #ifndef __RADEON_MS_DRM_H__ #define __RADEON_MS_DRM_H__ -/* fence definitions */ -/* The only fence class we support */ -#define DRM_RADEON_FENCE_CLASS_ACCEL 0 -/* Fence type that guarantees read-write flush */ -#define DRM_RADEON_FENCE_TYPE_RW 2 -/* cache flushes programmed just before the fence */ -#define DRM_RADEON_FENCE_FLAG_FLUSHED 0x01000000 +/* Fence + * We have only one fence class as we submit command through th + * same fifo so there is no need to synchronize buffer btw different + * cmd stream. + * + * Set DRM_RADEON_FENCE_FLAG_FLUSHED if you want a flush with + * emission of the fence + * + * For fence type we have the native DRM EXE type and the radeon RW + * type. + */ +#define DRM_RADEON_FENCE_CLASS_ACCEL 0 +#define DRM_RADEON_FENCE_TYPE_RW 2 +#define DRM_RADEON_FENCE_FLAG_FLUSHED 0x01000000 /* radeon ms ioctl */ -#define DRM_RADEON_EXECBUFFER 0x00 +#define DRM_RADEON_EXECBUFFER 0x00 struct drm_radeon_execbuffer_arg { uint64_t next; diff --git a/shared-core/radeon_ms_exec.c b/shared-core/radeon_ms_exec.c index b2ce3cbb..8e28b19e 100644 --- a/shared-core/radeon_ms_exec.c +++ b/shared-core/radeon_ms_exec.c @@ -223,7 +223,7 @@ int radeon_ms_execbuffer(struct drm_device *dev, void *data, fence_arg->handle = fence->base.hash.key; fence_arg->fence_class = fence->fence_class; fence_arg->type = fence->type; - fence_arg->signaled = fence->signaled; + fence_arg->signaled = fence->signaled_types; fence_arg->sequence = fence->sequence; } } diff --git a/shared-core/radeon_ms_fence.c b/shared-core/radeon_ms_fence.c index 6fcf5437..162d37d6 100644 --- a/shared-core/radeon_ms_fence.c +++ b/shared-core/radeon_ms_fence.c @@ -27,103 +27,241 @@ * Jerome Glisse */ #include "radeon_ms.h" +#include "amd_r3xx_fence.h" -static void radeon_ms_fence_flush(struct drm_device *dev) +#define R3XX_FENCE_SEQUENCE_RW_FLUSH 0x80000000u + +static inline int r3xx_fence_emit_sequence(struct drm_device *dev, + struct drm_radeon_private *dev_priv, + uint32_t sequence) { - struct drm_radeon_private *dev_priv = dev->dev_private; - struct drm_fence_class_manager *fc = &dev->fm.fence_class[0]; - uint32_t pending_flush_types = 0; + struct r3xx_fence *r3xx_fence = dev_priv->fence; + uint32_t cmd[2]; + int i, r; + + if (sequence & R3XX_FENCE_SEQUENCE_RW_FLUSH) { + r3xx_fence->sequence_last_flush = + sequence & ~R3XX_FENCE_SEQUENCE_RW_FLUSH; + /* Ask flush for VERTEX & FRAGPROG pipeline + * have 3D idle */ + dev_priv->flush_cache(dev); + } + cmd[0] = CP_PACKET0(dev_priv->fence_reg, 0); + cmd[1] = sequence; + for (i = 0; i < dev_priv->usec_timeout; i++) { + r = radeon_ms_ring_emit(dev, cmd, 2); + if (!r) { + dev_priv->irq_emit(dev); + return 0; + } + } + return -EBUSY; +} + +static inline uint32_t r3xx_fence_sequence(struct r3xx_fence *r3xx_fence) +{ + r3xx_fence->sequence += 1; + if (unlikely(r3xx_fence->sequence > 0x7fffffffu)) { + r3xx_fence->sequence = 1; + } + return r3xx_fence->sequence; +} + +static inline void r3xx_fence_report(struct drm_device *dev, + struct drm_radeon_private *dev_priv, + struct r3xx_fence *r3xx_fence) +{ + uint32_t fence_types = DRM_FENCE_TYPE_EXE; uint32_t sequence; if (dev_priv == NULL) { return; } - pending_flush_types = fc->pending_flush | - ((fc->pending_exe_flush) ? - DRM_FENCE_TYPE_EXE : 0); - if (pending_flush_types) { - sequence = mmio_read(dev_priv, dev_priv->fence_reg); - drm_fence_handler(dev, 0, sequence, pending_flush_types, 0); + sequence = mmio_read(dev_priv, dev_priv->fence_reg); + if (sequence & R3XX_FENCE_SEQUENCE_RW_FLUSH) { + sequence &= ~R3XX_FENCE_SEQUENCE_RW_FLUSH; + fence_types |= DRM_RADEON_FENCE_TYPE_RW; + if (sequence == r3xx_fence->sequence_last_flush) { + r3xx_fence->sequence_last_flush = 0; + } } + /* avoid to report already reported sequence */ + if (sequence != r3xx_fence->sequence_last_reported) { + drm_fence_handler(dev, 0, sequence, fence_types, 0); + r3xx_fence->sequence_last_reported = sequence; + } +} + +static void r3xx_fence_flush(struct drm_device *dev, uint32_t class) +{ + struct drm_radeon_private *dev_priv = dev->dev_private; + struct r3xx_fence *r3xx_fence = dev_priv->fence; + uint32_t sequence; + + sequence = r3xx_fence_sequence(r3xx_fence); + sequence |= R3XX_FENCE_SEQUENCE_RW_FLUSH; + r3xx_fence_emit_sequence(dev, dev_priv, sequence); } -int radeon_ms_fence_emit_sequence(struct drm_device *dev, uint32_t class, - uint32_t flags, uint32_t *sequence, - uint32_t *native_type) +static void r3xx_fence_poll(struct drm_device *dev, uint32_t fence_class, + uint32_t waiting_types) { struct drm_radeon_private *dev_priv = dev->dev_private; - uint32_t fence_id, cmd[2], i, ret; + struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[fence_class]; + struct r3xx_fence *r3xx_fence = dev_priv->fence; + + if (unlikely(!dev_priv)) { + return; + } + /* if there is a RW flush pending then submit new sequence + * preceded by flush cmds */ + if (fc->pending_flush & DRM_RADEON_FENCE_TYPE_RW) { + r3xx_fence_flush(dev, 0); + fc->pending_flush &= ~DRM_RADEON_FENCE_TYPE_RW; + } + r3xx_fence_report(dev, dev_priv, r3xx_fence); + return; +} + +static int r3xx_fence_emit(struct drm_device *dev, uint32_t class, + uint32_t flags, uint32_t *sequence, + uint32_t *native_type) +{ + struct drm_radeon_private *dev_priv = dev->dev_private; + struct r3xx_fence *r3xx_fence = dev_priv->fence; + uint32_t tmp; if (!dev_priv || dev_priv->cp_ready != 1) { return -EINVAL; } - fence_id = (++dev_priv->fence_id_last); - if (dev_priv->fence_id_last > 0x7FFFFFFF) { - fence_id = dev_priv->fence_id_last = 1; - } - *sequence = fence_id; + *sequence = tmp = r3xx_fence_sequence(r3xx_fence); *native_type = DRM_FENCE_TYPE_EXE; if (flags & DRM_RADEON_FENCE_FLAG_FLUSHED) { *native_type |= DRM_RADEON_FENCE_TYPE_RW; - dev_priv->flush_cache(dev); + tmp |= R3XX_FENCE_SEQUENCE_RW_FLUSH; } - cmd[0] = CP_PACKET0(dev_priv->fence_reg, 0); - cmd[1] = fence_id; - for (i = 0; i < dev_priv->usec_timeout; i++) { - ret = radeon_ms_ring_emit(dev, cmd, 2); - if (!ret) { - dev_priv->irq_emit(dev); + return r3xx_fence_emit_sequence(dev, dev_priv, tmp); +} + +static int r3xx_fence_has_irq(struct drm_device *dev, + uint32_t class, uint32_t type) +{ + const uint32_t type_irq_mask = DRM_FENCE_TYPE_EXE | + DRM_RADEON_FENCE_TYPE_RW; + /* + * We have an irq for EXE & RW fence. + */ + if (class == 0 && (type & type_irq_mask)) { + return 1; + } + return 0; +} + +static uint32_t r3xx_fence_needed_flush(struct drm_fence_object *fence) +{ + struct drm_device *dev = fence->dev; + struct drm_radeon_private *dev_priv = dev->dev_private; + struct r3xx_fence *r3xx_fence = dev_priv->fence; + struct drm_fence_driver *driver = dev->driver->fence_driver; + uint32_t flush_types, diff; + + flush_types = fence->waiting_types & + ~(DRM_FENCE_TYPE_EXE | fence->signaled_types); + + if (flush_types == 0 || ((flush_types & ~fence->native_types) == 0)) { + return 0; + } + if (unlikely(dev_priv == NULL)) { + return 0; + } + if (r3xx_fence->sequence_last_flush) { + diff = (r3xx_fence->sequence_last_flush - fence->sequence) & + driver->sequence_mask; + if (diff < driver->wrap_diff) { return 0; } } - return -EBUSY; + return flush_types; } -void radeon_ms_fence_handler(struct drm_device * dev) +static int r3xx_fence_wait(struct drm_fence_object *fence, + int lazy, int interruptible, uint32_t mask) { - struct drm_radeon_private *dev_priv = dev->dev_private; + struct drm_device *dev = fence->dev; struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &fm->fence_class[0]; + int r; - if (dev_priv == NULL) { - return; + drm_fence_object_flush(fence, mask); + if (likely(interruptible)) { + r = wait_event_interruptible_timeout( + fc->fence_queue, + drm_fence_object_signaled(fence, DRM_FENCE_TYPE_EXE), + 3 * DRM_HZ); + } else { + r = wait_event_timeout( + fc->fence_queue, + drm_fence_object_signaled(fence, DRM_FENCE_TYPE_EXE), + 3 * DRM_HZ); + } + if (unlikely(r == -ERESTARTSYS)) { + return -EAGAIN; + } + if (unlikely(r == 0)) { + return -EBUSY; } - write_lock(&fm->lock); - radeon_ms_fence_flush(dev); - write_unlock(&fm->lock); -} + if (likely(mask == DRM_FENCE_TYPE_EXE || + drm_fence_object_signaled(fence, mask))) { + return 0; + } -int radeon_ms_fence_has_irq(struct drm_device *dev, uint32_t class, - uint32_t flags) -{ /* - * We have an irq that tells us when we have a new breadcrumb. + * Poll for sync flush completion. */ - if (class == 0 && flags == DRM_FENCE_TYPE_EXE) - return 1; - - return 0; + return drm_fence_wait_polling(fence, lazy, interruptible, + mask, 3 * DRM_HZ); } -int radeon_ms_fence_types(struct drm_buffer_object *bo, - uint32_t *class, uint32_t *type) +struct drm_fence_driver r3xx_fence_driver = { + .num_classes = 1, + .wrap_diff = (1 << 29), + .flush_diff = (1 << 28), + .sequence_mask = 0x7fffffffU, + .has_irq = r3xx_fence_has_irq, + .emit = r3xx_fence_emit, + .flush = r3xx_fence_flush, + .poll = r3xx_fence_poll, + .needed_flush = r3xx_fence_needed_flush, + .wait = r3xx_fence_wait, +}; + +/* this are used by the buffer object code */ +int r3xx_fence_types(struct drm_buffer_object *bo, + uint32_t *class, uint32_t *type) { *class = 0; - if (bo->mem.flags & (DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE)) - *type = 3; - else - *type = 1; + if (bo->mem.flags & (DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE)) { + *type = DRM_FENCE_TYPE_EXE | DRM_RADEON_FENCE_TYPE_RW; + } else { + *type = DRM_FENCE_TYPE_EXE; + } return 0; } -void radeon_ms_poke_flush(struct drm_device *dev, uint32_t class) +/* this are used by the irq code */ +void r3xx_fence_handler(struct drm_device * dev) { + struct drm_radeon_private *dev_priv = dev->dev_private; struct drm_fence_manager *fm = &dev->fm; - unsigned long flags; + struct drm_fence_class_manager *fc = &fm->fence_class[0]; - if (class != 0) + if (unlikely(dev_priv == NULL)) { return; - write_lock_irqsave(&fm->lock, flags); - radeon_ms_fence_flush(dev); - write_unlock_irqrestore(&fm->lock, flags); + } + + write_lock(&fm->lock); + r3xx_fence_poll(dev, 0, fc->waiting_types); + write_unlock(&fm->lock); } diff --git a/shared-core/radeon_ms_irq.c b/shared-core/radeon_ms_irq.c index 2f94118f..5c68c902 100644 --- a/shared-core/radeon_ms_irq.c +++ b/shared-core/radeon_ms_irq.c @@ -92,9 +92,9 @@ irqreturn_t radeon_ms_irq_handler(DRM_IRQ_ARGS) /* SW interrupt */ if (GEN_INT_STATUS__SW_INT & status) { - radeon_ms_fence_handler(dev); + r3xx_fence_handler(dev); } - radeon_ms_fence_handler(dev); + r3xx_fence_handler(dev); return IRQ_HANDLED; } -- cgit v1.2.3