From 40c9e6a26dd251fe2bf207bb259ba7e4a7704fbe Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Thu, 28 Feb 2008 13:47:15 +0100 Subject: Add a compat kmap_atomic_prot_pfn to do quick kernel map / unmaps of PCI- or high memory. This is substantially more efficient than drm_bo_kmap, since the mapping only lives on a single processor. Unmapping is done use kunmap_atomic(). Flushes only a single tlb() entry. Add a support utility int drm_bo_pfn_prot() that returns the pfn and desired page protection for a given bo offset. This is all intended for relocations in bound TTMS or vram. Mapping-accessing-unmapping must be atomic, either using preempt_xx() macros or a spinlock. --- linux-core/drm_bo_move.c | 33 +++++++++++++++++++++++++++++++++ linux-core/drm_compat.c | 32 ++++++++++++++++++++++++++++++++ linux-core/drm_compat.h | 5 +++++ linux-core/drm_objects.h | 4 ++++ 4 files changed, 74 insertions(+) (limited to 'linux-core') diff --git a/linux-core/drm_bo_move.c b/linux-core/drm_bo_move.c index b06a09f0..30e0f43f 100644 --- a/linux-core/drm_bo_move.c +++ b/linux-core/drm_bo_move.c @@ -595,3 +595,36 @@ void drm_bo_kunmap(struct drm_bo_kmap_obj *map) map->page = NULL; } EXPORT_SYMBOL(drm_bo_kunmap); + +int drm_bo_pfn_prot(struct drm_buffer_object *bo, + unsigned long dst_offset, + unsigned long *pfn, + pgprot_t *prot) +{ + struct drm_bo_mem_reg *mem = &bo->mem; + struct drm_device *dev = bo->dev; + unsigned long bus_offset; + unsigned long bus_size; + unsigned long bus_base; + struct drm_mem_type_manager *man = &dev->bm.man[mem->mem_type]; + int ret; + + ret = drm_bo_pci_offset(dev, mem, &bus_base, &bus_offset, + &bus_size); + if (ret) + return -EINVAL; + + if (bus_size != 0) + *pfn = (bus_base + bus_offset + dst_offset) >> PAGE_SHIFT; + else if (!bo->ttm) + return -EINVAL; + else + *pfn = page_to_pfn(drm_ttm_get_page(bo->ttm, dst_offset >> PAGE_SHIFT)); + + *prot = (mem->flags & DRM_BO_FLAG_CACHED) ? + PAGE_KERNEL : drm_kernel_io_prot(man->drm_bus_maptype); + + return 0; +} +EXPORT_SYMBOL(drm_bo_pfn_prot); + diff --git a/linux-core/drm_compat.c b/linux-core/drm_compat.c index a745a7d9..32e43a0a 100644 --- a/linux-core/drm_compat.c +++ b/linux-core/drm_compat.c @@ -729,3 +729,35 @@ void *idr_replace(struct idr *idp, void *ptr, int id) } EXPORT_SYMBOL(idr_replace); #endif + +#if defined(CONFIG_X86) + +#define drm_kmap_get_fixmap_pte(vaddr) \ + pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), vaddr), (vaddr)), (vaddr)) + +void *kmap_atomic_prot_pfn(unsigned long pfn, enum km_type type, + pgprot_t protection) +{ + enum fixed_addresses idx; + unsigned long vaddr; + static pte_t *km_pte; + static int initialized = 0; + + if (unlikely(!initialized)) { + km_pte = drm_kmap_get_fixmap_pte(__fix_to_virt(FIX_KMAP_BEGIN)); + initialized = 1; + } + + pagefault_disable(); + idx = type + KM_TYPE_NR*smp_processor_id(); + vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); + set_pte(km_pte-idx, pfn_pte(pfn, protection)); + + return (void*) vaddr; +} + +EXPORT_SYMBOL(kmap_atomic_prot_pfn); + +#endif + + diff --git a/linux-core/drm_compat.h b/linux-core/drm_compat.h index f8933e0c..39027cf4 100644 --- a/linux-core/drm_compat.h +++ b/linux-core/drm_compat.h @@ -328,4 +328,9 @@ void *idr_replace(struct idr *idp, void *ptr, int id); typedef _Bool bool; #endif +#if defined(CONFIG_X86) +#define DRM_KMAP_ATOMIC_PROT_PFN +extern void *kmap_atomic_prot_pfn(unsigned long pfn, enum km_type type, + pgprot_t protection); +#endif #endif diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h index e43e8dfd..8055afe1 100644 --- a/linux-core/drm_objects.h +++ b/linux-core/drm_objects.h @@ -738,6 +738,10 @@ static inline void *drm_bmo_virtual(struct drm_bo_kmap_obj *map, int *is_iomem) extern void drm_bo_kunmap(struct drm_bo_kmap_obj *map); extern int drm_bo_kmap(struct drm_buffer_object *bo, unsigned long start_page, unsigned long num_pages, struct drm_bo_kmap_obj *map); +extern int drm_bo_pfn_prot(struct drm_buffer_object *bo, + unsigned long dst_offset, + unsigned long *pfn, + pgprot_t *prot); /* -- cgit v1.2.3 From 28d4d02d6791c15f61b718039f1d4b907f0e31e9 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Thu, 28 Feb 2008 14:05:53 +0100 Subject: Initial commit. --- linux-core/i915_execbuf.c | 653 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 653 insertions(+) create mode 100644 linux-core/i915_execbuf.c (limited to 'linux-core') diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c new file mode 100644 index 00000000..fecb5ab0 --- /dev/null +++ b/linux-core/i915_execbuf.c @@ -0,0 +1,653 @@ +/* + * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "drmP.h" +#include "drm.h" +#include "i915_drm.h" +#include "i915_drv.h" + +struct i915_relocatee_info { + struct drm_buffer_object *buf; + unsigned long offset; + uint32_t *data_page; + unsigned page_offset; + struct drm_bo_kmap_obj kmap; + int is_iomem; + int idle; + int dst; +#ifdef DRM_KMAP_ATOMIC_PROT_PFN + unsigned long pfn; + pgprot_t pg_prot; +#endif +}; + +struct drm_i915_validate_buffer { + struct drm_buffer_object *buffer; + struct drm_bo_info_rep rep; + int presumed_offset_correct; + void __user *data; + int ret; +}; + + +static int i915_update_relocatee(struct i915_relocatee_info *relocatee, + struct drm_i915_validate_buffer *buffers, + unsigned int dst, + unsigned long dst_offset) +{ + int ret; + + if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) { + i915_clear_relocatee(relocatee); + relocatee->dst = dst; + relocatee->buf = buffers[dst].buffer; + preempt_enable(); + ret = mutex_lock_interruptible(&relocatee->buf->mutex); + preempt_disable(); + if (unlikely(ret)) + return -EAGAIN; + + ret = drm_bo_wait(relocatee->buf, 0, 0, 0); + if (unlikely(ret)) + return ret; + + mutex_unlock(&relocatee->buf->mutex); + } + + if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) { + DRM_ERROR("Relocation destination out of bounds.\n"); + return -EINVAL; + } + if (unlikely(!drm_bo_same_page(relocatee->page_offset, dst_offset) || + NULL == relocatee->data_page)) { +#ifdef DRM_KMAP_ATOMIC_PROT_PFN + if (NULL != relocatee->data_page) { + kunmap_atomic(relocatee->data_page, KM_USER0); + relocatee->data_page = NULL; + } + ret = drm_bo_pfn_prot(relocatee->buf, dst_offset, + &relocatee->pfn, + &relocatee->pg_prot); + if (ret) { + DRM_ERROR("Can't map relocation destination.\n"); + return -EINVAL; + } + relocatee->data_page = + kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, + relocatee->pg_prot); +#else + if (NULL != relocatee->data_page) { + drm_bo_kunmap(&relocatee->kmap); + relocatee->data_page = NULL; + } + + ret = drm_bo_kmap(relocatee->buf, dst_offset >> PAGE_SHIFT, + 1, &relocatee->kmap); + if (ret) { + DRM_ERROR("Can't map relocation destination.\n"); + return ret; + } + + relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, + &relocatee->is_iomem); +#endif + relocatee->page_offset = dst_offset & PAGE_MASK; + } + return 0; +} + + +static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buffers, + unsigned num_buffers) +{ + while (num_buffers--) + drm_bo_usage_deref_locked(&buffers[num_buffers].buffer); +} + +int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, + struct drm_i915_validate_buffer *buffers, + struct i915_relocatee_info *relocatee, + uint32_t *reloc) +{ + unsigned index; + unsigned long new_cmd_offset; + u32 val; + int ret, i; + int buf_index = -1; + + /* + * FIXME: O(relocs * buffers) complexity. + */ + + for (i = 0; i <= num_buffers; i++) + if (buffers[i].buffer) + if (reloc[2] == buffers[i].buffer->base.hash.key) + buf_index = i; + + if (buf_index == -1) { + DRM_ERROR("Illegal relocation buffer %08X\n", reloc[2]); + return -EINVAL; + } + + /* + * Short-circuit relocations that were correctly + * guessed by the client + */ + if (buffers[buf_index].presumed_offset_correct && !DRM_DEBUG_RELOCATION) + return 0; + + new_cmd_offset = reloc[0]; + if (!relocatee->data_page || + !drm_bo_same_page(relocatee->offset, new_cmd_offset)) { + drm_bo_kunmap(&relocatee->kmap); + relocatee->data_page = NULL; + relocatee->offset = new_cmd_offset; + + /* + * Note on buffer idle: + * Since we're applying relocations, this part of the + * buffer is obviously not used by the GPU and we don't + * need to wait for buffer idle. This is an important + * consideration for user-space buffer pools. + */ + + ret = drm_bo_kmap(relocatee->buf, new_cmd_offset >> PAGE_SHIFT, + 1, &relocatee->kmap); + if (ret) { + DRM_ERROR("Could not map command buffer to apply relocs\n %08lx", new_cmd_offset); + return ret; + } + relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, + &relocatee->is_iomem); + relocatee->page_offset = (relocatee->offset & PAGE_MASK); + } + + val = buffers[buf_index].buffer->offset; + index = (reloc[0] - relocatee->page_offset) >> 2; + + /* add in validate */ + val = val + reloc[1]; + + if (DRM_DEBUG_RELOCATION) { + if (buffers[buf_index].presumed_offset_correct && + relocatee->data_page[index] != val) { + DRM_DEBUG ("Relocation mismatch source %d target %d buffer %d user %08x kernel %08x\n", + reloc[0], reloc[1], buf_index, relocatee->data_page[index], val); + } + } + + if (relocatee->is_iomem) + iowrite32(val, relocatee->data_page + index); + else + relocatee->data_page[index] = val; + return 0; +} + +int i915_process_relocs(struct drm_file *file_priv, + uint32_t buf_handle, + uint32_t __user **reloc_user_ptr, + struct i915_relocatee_info *relocatee, + struct drm_i915_validate_buffer *buffers, + uint32_t num_buffers) +{ + int ret, reloc_stride; + uint32_t cur_offset; + uint32_t reloc_count; + uint32_t reloc_type; + uint32_t reloc_buf_size; + uint32_t *reloc_buf = NULL; + int i; + + /* do a copy from user from the user ptr */ + ret = get_user(reloc_count, *reloc_user_ptr); + if (ret) { + DRM_ERROR("Could not map relocation buffer.\n"); + goto out; + } + + ret = get_user(reloc_type, (*reloc_user_ptr)+1); + if (ret) { + DRM_ERROR("Could not map relocation buffer.\n"); + goto out; + } + + if (reloc_type != 0) { + DRM_ERROR("Unsupported relocation type requested\n"); + ret = -EINVAL; + goto out; + } + + reloc_buf_size = (I915_RELOC_HEADER + (reloc_count * I915_RELOC0_STRIDE)) * sizeof(uint32_t); + reloc_buf = kmalloc(reloc_buf_size, GFP_KERNEL); + if (!reloc_buf) { + DRM_ERROR("Out of memory for reloc buffer\n"); + ret = -ENOMEM; + goto out; + } + + if (copy_from_user(reloc_buf, *reloc_user_ptr, reloc_buf_size)) { + ret = -EFAULT; + goto out; + } + + /* get next relocate buffer handle */ + *reloc_user_ptr = (uint32_t *)*(unsigned long *)&reloc_buf[2]; + + reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); /* may be different for other types of relocs */ + + DRM_DEBUG("num relocs is %d, next is %p\n", reloc_count, *reloc_user_ptr); + + for (i = 0; i < reloc_count; i++) { + cur_offset = I915_RELOC_HEADER + (i * I915_RELOC0_STRIDE); + + ret = i915_apply_reloc(file_priv, num_buffers, buffers, + relocatee, reloc_buf + cur_offset); + if (ret) + goto out; + } + +out: + if (reloc_buf) + kfree(reloc_buf); + + if (relocatee->data_page) { + drm_bo_kunmap(&relocatee->kmap); + relocatee->data_page = NULL; + } + + return ret; +} + +static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle, + uint32_t __user *reloc_user_ptr, + struct drm_i915_validate_buffer *buffers, + uint32_t buf_count) +{ + struct drm_device *dev = file_priv->head->dev; + struct i915_relocatee_info relocatee; + int ret = 0; + int b; + + /* + * Short circuit relocations when all previous + * buffers offsets were correctly guessed by + * the client + */ + if (!DRM_DEBUG_RELOCATION) { + for (b = 0; b < buf_count; b++) + if (!buffers[b].presumed_offset_correct) + break; + + if (b == buf_count) + return 0; + } + + memset(&relocatee, 0, sizeof(relocatee)); + + mutex_lock(&dev->struct_mutex); + relocatee.buf = drm_lookup_buffer_object(file_priv, buf_handle, 1); + mutex_unlock(&dev->struct_mutex); + if (!relocatee.buf) { + DRM_DEBUG("relocatee buffer invalid %08x\n", buf_handle); + ret = -EINVAL; + goto out_err; + } + + mutex_lock (&relocatee.buf->mutex); + while (reloc_user_ptr) { + ret = i915_process_relocs(file_priv, buf_handle, &reloc_user_ptr, &relocatee, buffers, buf_count); + if (ret) { + DRM_ERROR("process relocs failed\n"); + goto out_err1; + } + } + +out_err1: + mutex_unlock (&relocatee.buf->mutex); + drm_bo_usage_deref_unlocked(&relocatee.buf); +out_err: + return ret; +} + +static int i915_check_presumed(struct drm_i915_op_arg *arg, + struct drm_buffer_object *bo, + uint32_t __user *data, + int *presumed_ok) +{ + struct drm_bo_op_req *req = &arg->d.req; + uint32_t hint_offset; + uint32_t hint = req->bo_req.hint; + + *presumed_ok = 0; + + if (!(hint & DRM_BO_HINT_PRESUMED_OFFSET)) + return 0; + if (bo->offset == req->bo_req.presumed_offset) { + *presumed_ok = 1; + return 0; + } + + /* + * We need to turn off the HINT_PRESUMED_OFFSET for this buffer in + * the user-space IOCTL argument list, since the buffer has moved, + * we're about to apply relocations and we might subsequently + * hit an -EAGAIN. In that case the argument list will be reused by + * user-space, but the presumed offset is no longer valid. + * + * Needless to say, this is a bit ugly. + */ + + hint_offset = (uint32_t *)&req->bo_req.hint - (uint32_t *)arg; + hint &= ~DRM_BO_HINT_PRESUMED_OFFSET; + return __put_user(hint, data + hint_offset); +} + + +/* + * Validate, add fence and relocate a block of bos from a userspace list + */ +int i915_validate_buffer_list(struct drm_file *file_priv, + unsigned int fence_class, uint64_t data, + struct drm_i915_validate_buffer *buffers, + uint32_t *num_buffers) +{ + struct drm_i915_op_arg arg; + struct drm_bo_op_req *req = &arg.d.req; + int ret = 0; + unsigned buf_count = 0; + uint32_t buf_handle; + uint32_t __user *reloc_user_ptr; + struct drm_i915_validate_buffer *item = buffers; + + do { + if (buf_count >= *num_buffers) { + DRM_ERROR("Buffer count exceeded %d\n.", *num_buffers); + ret = -EINVAL; + goto out_err; + } + item = buffers + buf_count; + item->buffer = NULL; + item->presumed_offset_correct = 0; + + buffers[buf_count].buffer = NULL; + + if (copy_from_user(&arg, (void __user *)(unsigned long)data, sizeof(arg))) { + ret = -EFAULT; + goto out_err; + } + + ret = 0; + if (req->op != drm_bo_validate) { + DRM_ERROR + ("Buffer object operation wasn't \"validate\".\n"); + ret = -EINVAL; + goto out_err; + } + item->ret = 0; + item->data = (void __user *) (unsigned long) data; + + buf_handle = req->bo_req.handle; + reloc_user_ptr = (uint32_t *)(unsigned long)arg.reloc_ptr; + + if (reloc_user_ptr) { + ret = i915_exec_reloc(file_priv, buf_handle, reloc_user_ptr, buffers, buf_count); + if (ret) + goto out_err; + DRM_MEMORYBARRIER(); + } + + ret = drm_bo_handle_validate(file_priv, req->bo_req.handle, + req->bo_req.flags, req->bo_req.mask, + req->bo_req.hint, + req->bo_req.fence_class, 0, + &item->rep, + &item->buffer); + + if (ret) { + DRM_ERROR("error on handle validate %d\n", ret); + goto out_err; + } + + buf_count++; + + ret = i915_check_presumed(&arg, item->buffer, + (uint32_t __user *) + (unsigned long) data, + &item->presumed_offset_correct); + if (ret) + goto out_err; + + data = arg.next; + } while (data != 0); +out_err: + *num_buffers = buf_count; + item->ret = (ret != -EAGAIN) ? ret : 0; + return ret; +} + + +/* + * Remove all buffers from the unfenced list. + * If the execbuffer operation was aborted, for example due to a signal, + * this also make sure that buffers retain their original state and + * fence pointers. + * Copy back buffer information to user-space unless we were interrupted + * by a signal. In which case the IOCTL must be rerun. + */ + +static int i915_handle_copyback(struct drm_device *dev, + struct drm_i915_validate_buffer *buffers, + unsigned int num_buffers, int ret) +{ + int err = ret; + int i; + struct drm_i915_op_arg arg; + + if (ret) + drm_putback_buffer_objects(dev); + + if (ret != -EAGAIN) { + for (i = 0; i < num_buffers; ++i) { + arg.handled = 1; + arg.d.rep.ret = buffers->ret; + arg.d.rep.bo_info = buffers->rep; + if (__copy_to_user(buffers->data, &arg, sizeof(arg))) + err = -EFAULT; + buffers++; + } + } + + return err; +} + +/* + * Create a fence object, and if that fails, pretend that everything is + * OK and just idle the GPU. + */ + +void i915_fence_or_sync(struct drm_file *file_priv, + uint32_t fence_flags, + struct drm_fence_arg *fence_arg, + struct drm_fence_object **fence_p) +{ + struct drm_device *dev = file_priv->head->dev; + int ret; + struct drm_fence_object *fence; + + ret = drm_fence_buffer_objects(dev, NULL, fence_flags, + NULL, &fence); + + if (ret) { + + /* + * Fence creation failed. + * Fall back to synchronous operation and idle the engine. + */ + + (void) i915_emit_mi_flush(dev, MI_READ_FLUSH); + (void) i915_quiescent(dev); + + if (!(fence_flags & DRM_FENCE_FLAG_NO_USER)) { + + /* + * Communicate to user-space that + * fence creation has failed and that + * the engine is idle. + */ + + fence_arg->handle = ~0; + fence_arg->error = ret; + } + + drm_putback_buffer_objects(dev); + if (fence_p) + *fence_p = NULL; + return; + } + + if (!(fence_flags & DRM_FENCE_FLAG_NO_USER)) { + + ret = drm_fence_add_user_object(file_priv, fence, + fence_flags & + DRM_FENCE_FLAG_SHAREABLE); + if (!ret) + drm_fence_fill_arg(fence, fence_arg); + else { + /* + * Fence user object creation failed. + * We must idle the engine here as well, as user- + * space expects a fence object to wait on. Since we + * have a fence object we wait for it to signal + * to indicate engine "sufficiently" idle. + */ + + (void) drm_fence_object_wait(fence, 0, 1, + fence->type); + drm_fence_usage_deref_unlocked(&fence); + fence_arg->handle = ~0; + fence_arg->error = ret; + } + } + + if (fence_p) + *fence_p = fence; + else if (fence) + drm_fence_usage_deref_unlocked(&fence); +} + + +static int i915_execbuffer(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) + dev_priv->sarea_priv; + struct drm_i915_execbuffer *exec_buf = data; + struct drm_i915_batchbuffer *batch = &exec_buf->batch; + struct drm_fence_arg *fence_arg = &exec_buf->fence_arg; + int num_buffers; + int ret; + struct drm_i915_validate_buffer *buffers; + + if (!dev_priv->allow_batchbuffer) { + DRM_ERROR("Batchbuffer ioctl disabled\n"); + return -EINVAL; + } + + + if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects, + batch->num_cliprects * + sizeof(struct drm_clip_rect))) + return -EFAULT; + + if (exec_buf->num_buffers > dev_priv->max_validate_buffers) + return -EINVAL; + + + ret = drm_bo_read_lock(&dev->bm.bm_lock); + if (ret) + return ret; + + /* + * The cmdbuf_mutex makes sure the validate-submit-fence + * operation is atomic. + */ + + ret = mutex_lock_interruptible(&dev_priv->cmdbuf_mutex); + if (ret) { + drm_bo_read_unlock(&dev->bm.bm_lock); + return -EAGAIN; + } + + num_buffers = exec_buf->num_buffers; + + buffers = drm_calloc(num_buffers, sizeof(struct drm_i915_validate_buffer), DRM_MEM_DRIVER); + if (!buffers) { + drm_bo_read_unlock(&dev->bm.bm_lock); + mutex_unlock(&dev_priv->cmdbuf_mutex); + return -ENOMEM; + } + + /* validate buffer list + fixup relocations */ + ret = i915_validate_buffer_list(file_priv, 0, exec_buf->ops_list, + buffers, &num_buffers); + if (ret) + goto out_err0; + + /* make sure all previous memory operations have passed */ + DRM_MEMORYBARRIER(); + drm_agp_chipset_flush(dev); + + /* submit buffer */ + batch->start = buffers[num_buffers-1].buffer->offset; + + DRM_DEBUG("i915 exec batchbuffer, start %x used %d cliprects %d\n", + batch->start, batch->used, batch->num_cliprects); + + ret = i915_dispatch_batchbuffer(dev, batch); + if (ret) + goto out_err0; + + if (sarea_priv) + sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); + + i915_fence_or_sync(file_priv, fence_arg->flags, fence_arg, NULL); + +out_err0: + + /* handle errors */ + ret = i915_handle_copyback(dev, buffers, num_buffers, ret); + mutex_lock(&dev->struct_mutex); + i915_dereference_buffers_locked(buffers, num_buffers); + mutex_unlock(&dev->struct_mutex); + + drm_free(buffers, (exec_buf->num_buffers * sizeof(struct drm_buffer_object *)), DRM_MEM_DRIVER); + + mutex_unlock(&dev_priv->cmdbuf_mutex); + drm_bo_read_unlock(&dev->bm.bm_lock); + return ret; +} +#endif -- cgit v1.2.3 From 2305100c0fce9ec86a22660e5fed54791cff030b Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Fri, 29 Feb 2008 13:25:55 +0100 Subject: More post-ioctl work. --- linux-core/Makefile.kernel | 2 +- linux-core/drm_ttm.c | 2 +- linux-core/i915_execbuf.c | 395 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 315 insertions(+), 84 deletions(-) (limited to 'linux-core') diff --git a/linux-core/Makefile.kernel b/linux-core/Makefile.kernel index e7c280d0..defbe43b 100644 --- a/linux-core/Makefile.kernel +++ b/linux-core/Makefile.kernel @@ -20,7 +20,7 @@ r128-objs := r128_drv.o r128_cce.o r128_state.o r128_irq.o mga-objs := mga_drv.o mga_dma.o mga_state.o mga_warp.o mga_irq.o i810-objs := i810_drv.o i810_dma.o i915-objs := i915_drv.o i915_dma.o i915_irq.o i915_mem.o i915_fence.o \ - i915_buffer.o i915_compat.o + i915_buffer.o i915_compat.o i915_execbuf.o nouveau-objs := nouveau_drv.o nouveau_state.o nouveau_fifo.o nouveau_mem.o \ nouveau_object.o nouveau_irq.o nouveau_notifier.o nouveau_swmthd.o \ nouveau_sgdma.o nouveau_dma.o nouveau_buffer.o nouveau_fence.o \ diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c index a9d87338..58568452 100644 --- a/linux-core/drm_ttm.c +++ b/linux-core/drm_ttm.c @@ -298,7 +298,7 @@ int drm_ttm_populate(struct drm_ttm *ttm) return 0; be = ttm->be; - if (ttm->page_flags & DRM_TTM_PAGE_WRITE) { + if (1 || (ttm->page_flags & DRM_TTM_PAGE_WRITE)) { for (i = 0; i < ttm->num_pages; ++i) { page = drm_ttm_get_page(ttm, i); if (!page) diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c index fecb5ab0..1e515a57 100644 --- a/linux-core/i915_execbuf.c +++ b/linux-core/i915_execbuf.c @@ -29,6 +29,18 @@ #include "i915_drm.h" #include "i915_drv.h" +#if DRM_DEBUG_CODE +#define DRM_DEBUG_RELOCATION (drm_debug != 0) +#else +#define DRM_DEBUG_RELOCATION 0 +#endif + +enum i915_buf_idle { + i915_reloc_unchecked, + i915_reloc_idle, + i915_reloc_busy +}; + struct i915_relocatee_info { struct drm_buffer_object *buf; unsigned long offset; @@ -36,8 +48,8 @@ struct i915_relocatee_info { unsigned page_offset; struct drm_bo_kmap_obj kmap; int is_iomem; - int idle; int dst; + int idle; #ifdef DRM_KMAP_ATOMIC_PROT_PFN unsigned long pfn; pgprot_t pg_prot; @@ -50,76 +62,28 @@ struct drm_i915_validate_buffer { int presumed_offset_correct; void __user *data; int ret; + enum i915_buf_idle idle; }; - -static int i915_update_relocatee(struct i915_relocatee_info *relocatee, - struct drm_i915_validate_buffer *buffers, - unsigned int dst, - unsigned long dst_offset) +static void i915_emit_ring_reloc(struct drm_device *dev, uint32_t offset, + uint32_t value) { - int ret; - - if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) { - i915_clear_relocatee(relocatee); - relocatee->dst = dst; - relocatee->buf = buffers[dst].buffer; - preempt_enable(); - ret = mutex_lock_interruptible(&relocatee->buf->mutex); - preempt_disable(); - if (unlikely(ret)) - return -EAGAIN; - - ret = drm_bo_wait(relocatee->buf, 0, 0, 0); - if (unlikely(ret)) - return ret; - - mutex_unlock(&relocatee->buf->mutex); - } - - if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) { - DRM_ERROR("Relocation destination out of bounds.\n"); - return -EINVAL; - } - if (unlikely(!drm_bo_same_page(relocatee->page_offset, dst_offset) || - NULL == relocatee->data_page)) { -#ifdef DRM_KMAP_ATOMIC_PROT_PFN - if (NULL != relocatee->data_page) { - kunmap_atomic(relocatee->data_page, KM_USER0); - relocatee->data_page = NULL; - } - ret = drm_bo_pfn_prot(relocatee->buf, dst_offset, - &relocatee->pfn, - &relocatee->pg_prot); - if (ret) { - DRM_ERROR("Can't map relocation destination.\n"); - return -EINVAL; - } - relocatee->data_page = - kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, - relocatee->pg_prot); -#else - if (NULL != relocatee->data_page) { - drm_bo_kunmap(&relocatee->kmap); - relocatee->data_page = NULL; - } - - ret = drm_bo_kmap(relocatee->buf, dst_offset >> PAGE_SHIFT, - 1, &relocatee->kmap); - if (ret) { - DRM_ERROR("Can't map relocation destination.\n"); - return ret; - } - - relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, - &relocatee->is_iomem); -#endif - relocatee->page_offset = dst_offset & PAGE_MASK; - } - return 0; + struct drm_i915_private *dev_priv = + (struct drm_i915_private *) dev->dev_private; + + DRM_INFO("Ring reloc.\n"); + RING_LOCALS; + i915_kernel_lost_context(dev); + BEGIN_LP_RING(6); + OUT_RING((0x02 << 29) | (0x40 << 22) | (0x3 << 20) | (0x3)); + OUT_RING((0x3 << 24) | (0xF0 << 16) | (0x40)); + OUT_RING((0x1 << 16) | (0x4)); + OUT_RING(offset); + OUT_RING(value); + OUT_RING(0); + ADVANCE_LP_RING(); } - static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buffers, unsigned num_buffers) { @@ -332,6 +296,248 @@ out_err: return ret; } + +static void i915_clear_relocatee(struct i915_relocatee_info *relocatee) +{ + if (relocatee->data_page) { +#ifndef DRM_KMAP_ATOMIC_PROT_PFN + drm_bo_kunmap(&relocatee->kmap); +#else + kunmap_atomic(relocatee->data_page, KM_USER0); +#endif + relocatee->data_page = NULL; + } + relocatee->buf = NULL; + relocatee->dst = ~0; +} + + +static int i915_update_relocatee(struct i915_relocatee_info *relocatee, + struct drm_i915_validate_buffer *buffers, + unsigned int dst, + unsigned long dst_offset) +{ + int ret; + + if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) { + i915_clear_relocatee(relocatee); + relocatee->dst = dst; + relocatee->buf = buffers[dst].buffer; + relocatee->idle = i915_reloc_idle; buffers[dst].idle; +#if 0 + if (relocatee->idle == i915_reloc_unchecked) { + preempt_enable(); + ret = mutex_lock_interruptible(&relocatee->buf->mutex); + if (unlikely(ret)) + return -EAGAIN; + + ret = drm_bo_wait(relocatee->buf, 0, 0, 1); + relocatee->idle = (ret == 0) ? i915_reloc_idle : i915_reloc_busy; + mutex_unlock(&relocatee->buf->mutex); + preempt_disable(); + buffers[dst].idle = relocatee->idle; + } +#endif + } + + if (relocatee->idle == i915_reloc_busy) + return 0; + + if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) { + DRM_ERROR("Relocation destination out of bounds.\n"); + return -EINVAL; + } + if (unlikely(!drm_bo_same_page(relocatee->page_offset, dst_offset) || + NULL == relocatee->data_page)) { +#ifdef DRM_KMAP_ATOMIC_PROT_PFN + if (NULL != relocatee->data_page) { + kunmap_atomic(relocatee->data_page, KM_USER0); + relocatee->data_page = NULL; + } + ret = drm_bo_pfn_prot(relocatee->buf, dst_offset, + &relocatee->pfn, + &relocatee->pg_prot); + if (ret) { + DRM_ERROR("Can't map relocation destination.\n"); + return -EINVAL; + } + relocatee->data_page = + kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, + relocatee->pg_prot); +#else + if (NULL != relocatee->data_page) { + drm_bo_kunmap(&relocatee->kmap); + relocatee->data_page = NULL; + } + + ret = drm_bo_kmap(relocatee->buf, dst_offset >> PAGE_SHIFT, + 1, &relocatee->kmap); + if (ret) { + DRM_ERROR("Can't map relocation destination.\n"); + return ret; + } + + relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, + &relocatee->is_iomem); +#endif + relocatee->page_offset = dst_offset & PAGE_MASK; + } + return 0; +} + +static int i915_apply_post_reloc(uint32_t reloc[], + struct drm_i915_validate_buffer *buffers, + uint32_t num_buffers, + struct i915_relocatee_info *relocatee) +{ + uint32_t reloc_buffer = reloc[2]; + uint32_t dst_buffer = reloc[3]; + uint32_t val; + uint32_t index; + int ret; + + if (likely(buffers[reloc_buffer].presumed_offset_correct)) + return 0; + if (unlikely(reloc_buffer >= num_buffers)) { + DRM_ERROR("Invalid reloc buffer index.\n"); + return -EINVAL; + } + if (unlikely(dst_buffer >= num_buffers)) { + DRM_ERROR("Invalid dest buffer index.\n"); + return -EINVAL; + } + + ret = i915_update_relocatee(relocatee, buffers, dst_buffer, + reloc[0]); + if (unlikely(ret)) + return ret; + + val = buffers[reloc_buffer].buffer->offset; + index = (reloc[0] - relocatee->page_offset) >> 2; + val = val + reloc[1]; + + if (relocatee->idle == i915_reloc_busy) { + i915_emit_ring_reloc(relocatee->buf->dev, + relocatee->buf->offset + reloc[0], + val); + return 0; + } + +#ifdef DRM_KMAP_ATOMIC_PROT_PFN + relocatee->data_page[index] = val; +#else + if (likely(relocatee->is_iomem)) + iowrite32(val, relocatee->data_page + index); + else + relocatee->data_page[index] = val; +#endif + + return 0; +} + +static int i915_post_relocs(struct drm_file *file_priv, + uint32_t __user *new_reloc_ptr, + struct drm_i915_validate_buffer *buffers, + unsigned int num_buffers) +{ + uint32_t *reloc; + uint32_t reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); + uint32_t header_size = I915_RELOC_HEADER * sizeof(uint32_t); + struct i915_relocatee_info relocatee; + uint32_t reloc_type; + uint32_t num_relocs; + uint32_t count; + int ret = 0; + int i; + int short_circuit = 1; + uint32_t __user *reloc_ptr; + uint64_t new_reloc_data; + uint32_t reloc_buf_size; + uint32_t *reloc_buf; + + for (i=0; i= *num_buffers) { @@ -392,8 +600,6 @@ int i915_validate_buffer_list(struct drm_file *file_priv, item->buffer = NULL; item->presumed_offset_correct = 0; - buffers[buf_count].buffer = NULL; - if (copy_from_user(&arg, (void __user *)(unsigned long)data, sizeof(arg))) { ret = -EFAULT; goto out_err; @@ -412,7 +618,24 @@ int i915_validate_buffer_list(struct drm_file *file_priv, buf_handle = req->bo_req.handle; reloc_user_ptr = (uint32_t *)(unsigned long)arg.reloc_ptr; - if (reloc_user_ptr) { + /* + * Switch mode to post-validation relocations? + */ + + if (unlikely((buf_count == 0) && (*post_relocs == NULL) && + (reloc_user_ptr != NULL))) { + uint32_t reloc_type; + + ret = get_user(reloc_type, reloc_user_ptr+1); + if (ret) + goto out_err; + + if (reloc_type == 1) + *post_relocs = reloc_user_ptr; + + } + + if ((*post_relocs == NULL) && (reloc_user_ptr != NULL)) { ret = i915_exec_reloc(file_priv, buf_handle, reloc_user_ptr, buffers, buf_count); if (ret) goto out_err; @@ -425,7 +648,6 @@ int i915_validate_buffer_list(struct drm_file *file_priv, req->bo_req.fence_class, 0, &item->rep, &item->buffer); - if (ret) { DRM_ERROR("error on handle validate %d\n", ret); goto out_err; @@ -559,8 +781,8 @@ void i915_fence_or_sync(struct drm_file *file_priv, } -static int i915_execbuffer(struct drm_device *dev, void *data, - struct drm_file *file_priv) +int i915_execbuffer(struct drm_device *dev, void *data, + struct drm_file *file_priv) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) @@ -570,6 +792,7 @@ static int i915_execbuffer(struct drm_device *dev, void *data, struct drm_fence_arg *fence_arg = &exec_buf->fence_arg; int num_buffers; int ret; + uint32_t __user *post_relocs; struct drm_i915_validate_buffer *buffers; if (!dev_priv->allow_batchbuffer) { @@ -577,7 +800,6 @@ static int i915_execbuffer(struct drm_device *dev, void *data, return -EINVAL; } - if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects, batch->num_cliprects * sizeof(struct drm_clip_rect))) @@ -586,7 +808,6 @@ static int i915_execbuffer(struct drm_device *dev, void *data, if (exec_buf->num_buffers > dev_priv->max_validate_buffers) return -EINVAL; - ret = drm_bo_read_lock(&dev->bm.bm_lock); if (ret) return ret; @@ -613,27 +834,38 @@ static int i915_execbuffer(struct drm_device *dev, void *data, /* validate buffer list + fixup relocations */ ret = i915_validate_buffer_list(file_priv, 0, exec_buf->ops_list, - buffers, &num_buffers); + buffers, &num_buffers, &post_relocs); if (ret) goto out_err0; + if (post_relocs) { + ret = i915_post_relocs(file_priv, post_relocs, + buffers, num_buffers); + if (ret) + goto out_err0; + } + /* make sure all previous memory operations have passed */ DRM_MEMORYBARRIER(); - drm_agp_chipset_flush(dev); - /* submit buffer */ - batch->start = buffers[num_buffers-1].buffer->offset; + if (!post_relocs) { + drm_agp_chipset_flush(dev); + batch->start = buffers[num_buffers-1].buffer->offset; + } else { + batch->start += buffers[0].buffer->offset; + } +#if 1 + // (void) i915_emit_mi_flush(dev, MI_NO_WRITE_FLUSH | MI_READ_FLUSH | MI_EXE_FLUSH); DRM_DEBUG("i915 exec batchbuffer, start %x used %d cliprects %d\n", batch->start, batch->used, batch->num_cliprects); ret = i915_dispatch_batchbuffer(dev, batch); if (ret) goto out_err0; - +#endif if (sarea_priv) sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); - i915_fence_or_sync(file_priv, fence_arg->flags, fence_arg, NULL); out_err0: @@ -650,4 +882,3 @@ out_err0: drm_bo_read_unlock(&dev->bm.bm_lock); return ret; } -#endif -- cgit v1.2.3 From 612c22f131a25915196e69d7ec1adb6f4ec84a60 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Fri, 29 Feb 2008 15:38:55 +0100 Subject: Working revision. --- linux-core/i915_execbuf.c | 228 ++++++++++++++++++++++++++-------------------- 1 file changed, 128 insertions(+), 100 deletions(-) (limited to 'linux-core') diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c index 1e515a57..cb0e7348 100644 --- a/linux-core/i915_execbuf.c +++ b/linux-core/i915_execbuf.c @@ -1,5 +1,5 @@ /* - * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. + * Copyright 2003-2008 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a @@ -22,6 +22,10 @@ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * + * Authors: + * Thomas Hellstrom + * Dave Airlie + * ? */ #include "drmP.h" @@ -48,8 +52,9 @@ struct i915_relocatee_info { unsigned page_offset; struct drm_bo_kmap_obj kmap; int is_iomem; - int dst; + int dst; int idle; + int performed_ring_relocs; #ifdef DRM_KMAP_ATOMIC_PROT_PFN unsigned long pfn; pgprot_t pg_prot; @@ -65,13 +70,20 @@ struct drm_i915_validate_buffer { enum i915_buf_idle idle; }; +/* + * I'd like to use MI_STORE_DATA_IMM here, but I can't make + * it work. Seems like GART writes are broken with that + * instruction. Also I'm not sure that MI_FLUSH will + * act as a memory barrier for that instruction. It will + * for this single dword 2D blit. + */ + static void i915_emit_ring_reloc(struct drm_device *dev, uint32_t offset, uint32_t value) { struct drm_i915_private *dev_priv = - (struct drm_i915_private *) dev->dev_private; + (struct drm_i915_private *)dev->dev_private; - DRM_INFO("Ring reloc.\n"); RING_LOCALS; i915_kernel_lost_context(dev); BEGIN_LP_RING(6); @@ -84,8 +96,8 @@ static void i915_emit_ring_reloc(struct drm_device *dev, uint32_t offset, ADVANCE_LP_RING(); } -static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buffers, - unsigned num_buffers) +static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer + *buffers, unsigned num_buffers) { while (num_buffers--) drm_bo_usage_deref_locked(&buffers[num_buffers].buffer); @@ -93,8 +105,7 @@ static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buf int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, struct drm_i915_validate_buffer *buffers, - struct i915_relocatee_info *relocatee, - uint32_t *reloc) + struct i915_relocatee_info *relocatee, uint32_t * reloc) { unsigned index; unsigned long new_cmd_offset; @@ -130,18 +141,19 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, relocatee->data_page = NULL; relocatee->offset = new_cmd_offset; - /* - * Note on buffer idle: - * Since we're applying relocations, this part of the - * buffer is obviously not used by the GPU and we don't - * need to wait for buffer idle. This is an important - * consideration for user-space buffer pools. - */ + if (unlikely(relocatee->idle == i915_reloc_unchecked)) { + ret = drm_bo_wait(relocatee->buf, 0, 0, 0); + if (ret) + return ret; + relocatee->idle = i915_reloc_idle; + } ret = drm_bo_kmap(relocatee->buf, new_cmd_offset >> PAGE_SHIFT, 1, &relocatee->kmap); if (ret) { - DRM_ERROR("Could not map command buffer to apply relocs\n %08lx", new_cmd_offset); + DRM_ERROR + ("Could not map command buffer to apply relocs\n %08lx", + new_cmd_offset); return ret; } relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, @@ -158,8 +170,10 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, if (DRM_DEBUG_RELOCATION) { if (buffers[buf_index].presumed_offset_correct && relocatee->data_page[index] != val) { - DRM_DEBUG ("Relocation mismatch source %d target %d buffer %d user %08x kernel %08x\n", - reloc[0], reloc[1], buf_index, relocatee->data_page[index], val); + DRM_DEBUG + ("Relocation mismatch source %d target %d buffer %d user %08x kernel %08x\n", + reloc[0], reloc[1], buf_index, + relocatee->data_page[index], val); } } @@ -172,7 +186,7 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, int i915_process_relocs(struct drm_file *file_priv, uint32_t buf_handle, - uint32_t __user **reloc_user_ptr, + uint32_t __user ** reloc_user_ptr, struct i915_relocatee_info *relocatee, struct drm_i915_validate_buffer *buffers, uint32_t num_buffers) @@ -192,7 +206,7 @@ int i915_process_relocs(struct drm_file *file_priv, goto out; } - ret = get_user(reloc_type, (*reloc_user_ptr)+1); + ret = get_user(reloc_type, (*reloc_user_ptr) + 1); if (ret) { DRM_ERROR("Could not map relocation buffer.\n"); goto out; @@ -204,7 +218,9 @@ int i915_process_relocs(struct drm_file *file_priv, goto out; } - reloc_buf_size = (I915_RELOC_HEADER + (reloc_count * I915_RELOC0_STRIDE)) * sizeof(uint32_t); + reloc_buf_size = + (I915_RELOC_HEADER + + (reloc_count * I915_RELOC0_STRIDE)) * sizeof(uint32_t); reloc_buf = kmalloc(reloc_buf_size, GFP_KERNEL); if (!reloc_buf) { DRM_ERROR("Out of memory for reloc buffer\n"); @@ -218,26 +234,27 @@ int i915_process_relocs(struct drm_file *file_priv, } /* get next relocate buffer handle */ - *reloc_user_ptr = (uint32_t *)*(unsigned long *)&reloc_buf[2]; + *reloc_user_ptr = (uint32_t *) * (unsigned long *)&reloc_buf[2]; - reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); /* may be different for other types of relocs */ + reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); /* may be different for other types of relocs */ - DRM_DEBUG("num relocs is %d, next is %p\n", reloc_count, *reloc_user_ptr); + DRM_DEBUG("num relocs is %d, next is %p\n", reloc_count, + *reloc_user_ptr); for (i = 0; i < reloc_count; i++) { cur_offset = I915_RELOC_HEADER + (i * I915_RELOC0_STRIDE); - + ret = i915_apply_reloc(file_priv, num_buffers, buffers, relocatee, reloc_buf + cur_offset); if (ret) goto out; } -out: + out: if (reloc_buf) kfree(reloc_buf); - if (relocatee->data_page) { + if (relocatee->data_page) { drm_bo_kunmap(&relocatee->kmap); relocatee->data_page = NULL; } @@ -246,7 +263,7 @@ out: } static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle, - uint32_t __user *reloc_user_ptr, + uint32_t __user * reloc_user_ptr, struct drm_i915_validate_buffer *buffers, uint32_t buf_count) { @@ -264,12 +281,13 @@ static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle, for (b = 0; b < buf_count; b++) if (!buffers[b].presumed_offset_correct) break; - + if (b == buf_count) return 0; } memset(&relocatee, 0, sizeof(relocatee)); + relocatee.idle = i915_reloc_unchecked; mutex_lock(&dev->struct_mutex); relocatee.buf = drm_lookup_buffer_object(file_priv, buf_handle, 1); @@ -280,23 +298,24 @@ static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle, goto out_err; } - mutex_lock (&relocatee.buf->mutex); + mutex_lock(&relocatee.buf->mutex); while (reloc_user_ptr) { - ret = i915_process_relocs(file_priv, buf_handle, &reloc_user_ptr, &relocatee, buffers, buf_count); + ret = + i915_process_relocs(file_priv, buf_handle, &reloc_user_ptr, + &relocatee, buffers, buf_count); if (ret) { DRM_ERROR("process relocs failed\n"); goto out_err1; } } -out_err1: - mutex_unlock (&relocatee.buf->mutex); + out_err1: + mutex_unlock(&relocatee.buf->mutex); drm_bo_usage_deref_unlocked(&relocatee.buf); -out_err: + out_err: return ret; } - static void i915_clear_relocatee(struct i915_relocatee_info *relocatee) { if (relocatee->data_page) { @@ -311,20 +330,23 @@ static void i915_clear_relocatee(struct i915_relocatee_info *relocatee) relocatee->dst = ~0; } - static int i915_update_relocatee(struct i915_relocatee_info *relocatee, struct drm_i915_validate_buffer *buffers, - unsigned int dst, - unsigned long dst_offset) + unsigned int dst, unsigned long dst_offset) { int ret; - + if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) { i915_clear_relocatee(relocatee); relocatee->dst = dst; relocatee->buf = buffers[dst].buffer; - relocatee->idle = i915_reloc_idle; buffers[dst].idle; -#if 0 + relocatee->idle = buffers[dst].idle; + + /* + * Check for buffer idle. If the buffer is busy, revert to + * ring relocations. + */ + if (relocatee->idle == i915_reloc_unchecked) { preempt_enable(); ret = mutex_lock_interruptible(&relocatee->buf->mutex); @@ -332,16 +354,20 @@ static int i915_update_relocatee(struct i915_relocatee_info *relocatee, return -EAGAIN; ret = drm_bo_wait(relocatee->buf, 0, 0, 1); - relocatee->idle = (ret == 0) ? i915_reloc_idle : i915_reloc_busy; + if (ret == 0) + relocatee->idle = i915_reloc_idle; + else { + relocatee->idle = i915_reloc_busy; + relocatee->performed_ring_relocs = 1; + } mutex_unlock(&relocatee->buf->mutex); preempt_disable(); buffers[dst].idle = relocatee->idle; } -#endif } if (relocatee->idle == i915_reloc_busy) - return 0; + return 0; if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) { DRM_ERROR("Relocation destination out of bounds.\n"); @@ -355,15 +381,14 @@ static int i915_update_relocatee(struct i915_relocatee_info *relocatee, relocatee->data_page = NULL; } ret = drm_bo_pfn_prot(relocatee->buf, dst_offset, - &relocatee->pfn, - &relocatee->pg_prot); + &relocatee->pfn, &relocatee->pg_prot); if (ret) { DRM_ERROR("Can't map relocation destination.\n"); return -EINVAL; } relocatee->data_page = - kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, - relocatee->pg_prot); + kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, + relocatee->pg_prot); #else if (NULL != relocatee->data_page) { drm_bo_kunmap(&relocatee->kmap); @@ -378,7 +403,7 @@ static int i915_update_relocatee(struct i915_relocatee_info *relocatee, } relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, - &relocatee->is_iomem); + &relocatee->is_iomem); #endif relocatee->page_offset = dst_offset & PAGE_MASK; } @@ -407,8 +432,7 @@ static int i915_apply_post_reloc(uint32_t reloc[], return -EINVAL; } - ret = i915_update_relocatee(relocatee, buffers, dst_buffer, - reloc[0]); + ret = i915_update_relocatee(relocatee, buffers, dst_buffer, reloc[0]); if (unlikely(ret)) return ret; @@ -417,12 +441,10 @@ static int i915_apply_post_reloc(uint32_t reloc[], val = val + reloc[1]; if (relocatee->idle == i915_reloc_busy) { - i915_emit_ring_reloc(relocatee->buf->dev, - relocatee->buf->offset + reloc[0], - val); + i915_emit_ring_reloc(relocatee->buf->dev, + relocatee->buf->offset + reloc[0], val); return 0; } - #ifdef DRM_KMAP_ATOMIC_PROT_PFN relocatee->data_page[index] = val; #else @@ -436,7 +458,7 @@ static int i915_apply_post_reloc(uint32_t reloc[], } static int i915_post_relocs(struct drm_file *file_priv, - uint32_t __user *new_reloc_ptr, + uint32_t __user * new_reloc_ptr, struct drm_i915_validate_buffer *buffers, unsigned int num_buffers) { @@ -455,7 +477,7 @@ static int i915_post_relocs(struct drm_file *file_priv, uint32_t reloc_buf_size; uint32_t *reloc_buf; - for (i=0; ihead->dev, 0); + + i915_clear_relocatee(&relocatee); if (reloc_buf) { kfree(reloc_buf); reloc_buf = NULL; @@ -540,8 +571,7 @@ out: static int i915_check_presumed(struct drm_i915_op_arg *arg, struct drm_buffer_object *bo, - uint32_t __user *data, - int *presumed_ok) + uint32_t __user * data, int *presumed_ok) { struct drm_bo_op_req *req = &arg->d.req; uint32_t hint_offset; @@ -566,20 +596,19 @@ static int i915_check_presumed(struct drm_i915_op_arg *arg, * Needless to say, this is a bit ugly. */ - hint_offset = (uint32_t *)&req->bo_req.hint - (uint32_t *)arg; + hint_offset = (uint32_t *) & req->bo_req.hint - (uint32_t *) arg; hint &= ~DRM_BO_HINT_PRESUMED_OFFSET; return __put_user(hint, data + hint_offset); } - /* * Validate, add fence and relocate a block of bos from a userspace list */ int i915_validate_buffer_list(struct drm_file *file_priv, unsigned int fence_class, uint64_t data, struct drm_i915_validate_buffer *buffers, - uint32_t *num_buffers, - uint32_t __user **post_relocs) + uint32_t * num_buffers, + uint32_t __user ** post_relocs) { struct drm_i915_op_arg arg; struct drm_bo_op_req *req = &arg.d.req; @@ -599,8 +628,10 @@ int i915_validate_buffer_list(struct drm_file *file_priv, item = buffers + buf_count; item->buffer = NULL; item->presumed_offset_correct = 0; + item->idle = i915_reloc_unchecked; - if (copy_from_user(&arg, (void __user *)(unsigned long)data, sizeof(arg))) { + if (copy_from_user + (&arg, (void __user *)(unsigned long)data, sizeof(arg))) { ret = -EFAULT; goto out_err; } @@ -613,10 +644,10 @@ int i915_validate_buffer_list(struct drm_file *file_priv, goto out_err; } item->ret = 0; - item->data = (void __user *) (unsigned long) data; + item->data = (void __user *)(unsigned long)data; buf_handle = req->bo_req.handle; - reloc_user_ptr = (uint32_t *)(unsigned long)arg.reloc_ptr; + reloc_user_ptr = (uint32_t *) (unsigned long)arg.reloc_ptr; /* * Switch mode to post-validation relocations? @@ -626,7 +657,7 @@ int i915_validate_buffer_list(struct drm_file *file_priv, (reloc_user_ptr != NULL))) { uint32_t reloc_type; - ret = get_user(reloc_type, reloc_user_ptr+1); + ret = get_user(reloc_type, reloc_user_ptr + 1); if (ret) goto out_err; @@ -636,18 +667,19 @@ int i915_validate_buffer_list(struct drm_file *file_priv, } if ((*post_relocs == NULL) && (reloc_user_ptr != NULL)) { - ret = i915_exec_reloc(file_priv, buf_handle, reloc_user_ptr, buffers, buf_count); + ret = + i915_exec_reloc(file_priv, buf_handle, + reloc_user_ptr, buffers, buf_count); if (ret) goto out_err; DRM_MEMORYBARRIER(); } ret = drm_bo_handle_validate(file_priv, req->bo_req.handle, - req->bo_req.flags, req->bo_req.mask, - req->bo_req.hint, + req->bo_req.flags, + req->bo_req.mask, req->bo_req.hint, req->bo_req.fence_class, 0, - &item->rep, - &item->buffer); + &item->rep, &item->buffer); if (ret) { DRM_ERROR("error on handle validate %d\n", ret); goto out_err; @@ -657,20 +689,19 @@ int i915_validate_buffer_list(struct drm_file *file_priv, ret = i915_check_presumed(&arg, item->buffer, (uint32_t __user *) - (unsigned long) data, + (unsigned long)data, &item->presumed_offset_correct); if (ret) goto out_err; data = arg.next; } while (data != 0); -out_err: + out_err: *num_buffers = buf_count; item->ret = (ret != -EAGAIN) ? ret : 0; return ret; } - /* * Remove all buffers from the unfenced list. * If the execbuffer operation was aborted, for example due to a signal, @@ -719,8 +750,7 @@ void i915_fence_or_sync(struct drm_file *file_priv, int ret; struct drm_fence_object *fence; - ret = drm_fence_buffer_objects(dev, NULL, fence_flags, - NULL, &fence); + ret = drm_fence_buffer_objects(dev, NULL, fence_flags, NULL, &fence); if (ret) { @@ -729,8 +759,8 @@ void i915_fence_or_sync(struct drm_file *file_priv, * Fall back to synchronous operation and idle the engine. */ - (void) i915_emit_mi_flush(dev, MI_READ_FLUSH); - (void) i915_quiescent(dev); + (void)i915_emit_mi_flush(dev, MI_READ_FLUSH); + (void)i915_quiescent(dev); if (!(fence_flags & DRM_FENCE_FLAG_NO_USER)) { @@ -746,7 +776,7 @@ void i915_fence_or_sync(struct drm_file *file_priv, drm_putback_buffer_objects(dev); if (fence_p) - *fence_p = NULL; + *fence_p = NULL; return; } @@ -766,8 +796,7 @@ void i915_fence_or_sync(struct drm_file *file_priv, * to indicate engine "sufficiently" idle. */ - (void) drm_fence_object_wait(fence, 0, 1, - fence->type); + (void)drm_fence_object_wait(fence, 0, 1, fence->type); drm_fence_usage_deref_unlocked(&fence); fence_arg->handle = ~0; fence_arg->error = ret; @@ -780,13 +809,12 @@ void i915_fence_or_sync(struct drm_file *file_priv, drm_fence_usage_deref_unlocked(&fence); } - int i915_execbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) - dev_priv->sarea_priv; + dev_priv->sarea_priv; struct drm_i915_execbuffer *exec_buf = data; struct drm_i915_batchbuffer *batch = &exec_buf->batch; struct drm_fence_arg *fence_arg = &exec_buf->fence_arg; @@ -802,7 +830,8 @@ int i915_execbuffer(struct drm_device *dev, void *data, if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects, batch->num_cliprects * - sizeof(struct drm_clip_rect))) + sizeof(struct + drm_clip_rect))) return -EFAULT; if (exec_buf->num_buffers > dev_priv->max_validate_buffers) @@ -825,7 +854,9 @@ int i915_execbuffer(struct drm_device *dev, void *data, num_buffers = exec_buf->num_buffers; - buffers = drm_calloc(num_buffers, sizeof(struct drm_i915_validate_buffer), DRM_MEM_DRIVER); + buffers = + drm_calloc(num_buffers, sizeof(struct drm_i915_validate_buffer), + DRM_MEM_DRIVER); if (!buffers) { drm_bo_read_unlock(&dev->bm.bm_lock); mutex_unlock(&dev_priv->cmdbuf_mutex); @@ -850,33 +881,30 @@ int i915_execbuffer(struct drm_device *dev, void *data, if (!post_relocs) { drm_agp_chipset_flush(dev); - batch->start = buffers[num_buffers-1].buffer->offset; + batch->start = buffers[num_buffers - 1].buffer->offset; } else { batch->start += buffers[0].buffer->offset; } -#if 1 - // (void) i915_emit_mi_flush(dev, MI_NO_WRITE_FLUSH | MI_READ_FLUSH | MI_EXE_FLUSH); DRM_DEBUG("i915 exec batchbuffer, start %x used %d cliprects %d\n", batch->start, batch->used, batch->num_cliprects); ret = i915_dispatch_batchbuffer(dev, batch); if (ret) goto out_err0; -#endif if (sarea_priv) sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); i915_fence_or_sync(file_priv, fence_arg->flags, fence_arg, NULL); -out_err0: - - /* handle errors */ + out_err0: ret = i915_handle_copyback(dev, buffers, num_buffers, ret); mutex_lock(&dev->struct_mutex); i915_dereference_buffers_locked(buffers, num_buffers); mutex_unlock(&dev->struct_mutex); - drm_free(buffers, (exec_buf->num_buffers * sizeof(struct drm_buffer_object *)), DRM_MEM_DRIVER); + drm_free(buffers, + (exec_buf->num_buffers * sizeof(struct drm_buffer_object *)), + DRM_MEM_DRIVER); mutex_unlock(&dev_priv->cmdbuf_mutex); drm_bo_read_unlock(&dev->bm.bm_lock); -- cgit v1.2.3 From ae1bb96a7e24362500e02cf3a86bd268c2dcc835 Mon Sep 17 00:00:00 2001 From: Mike Isely Date: Fri, 14 Mar 2008 09:53:05 +1000 Subject: drm: Fix race that can lockup the kernel The i915_vblank_swap() function schedules an automatic buffer swap upon receipt of the vertical sync interrupt. Such an operation is lengthy so it can't be allowed to happen in normal interrupt context, thus the DRM implements this by scheduling the work in a kernel softirq-scheduled tasklet. In order for the buffer swap to work safely, the DRM's central lock must be taken, via a call to drm_lock_take() located in drivers/char/drm/drm_irq.c within the function drm_locked_tasklet_func(). The lock-taking logic uses a non-interrupt-blocking spinlock to implement the manipulations needed to take the lock. This semantic would be safe if all attempts to use the spinlock only happen from process context. However this buffer swap happens from softirq context which is really a form of interrupt context. Thus we have an unsafe situation, in that drm_locked_tasklet_func() can block on a spinlock already taken by a thread in process context which will never get scheduled again because of the blocked softirq tasklet. This wedges the kernel hard. To trigger this bug, run a dual-head cloned mode configuration which uses the i915 drm, then execute an opengl application which synchronizes buffer swaps against the vertical sync interrupt. In my testing, a lockup always results after running anywhere from 5 minutes to an hour and a half. I believe dual-head is needed to really trigger the problem because then the vertical sync interrupt handling is no longer predictable (due to being interrupt-sourced from two different heads running at different speeds). This raises the probability of the tasklet trying to run while the userspace DRI is doing things to the GPU (and manipulating the DRM lock). The fix is to change the relevant spinlock semantics to be the interrupt-blocking form. After this change I am no longer able to trigger the lockup; the longest test run so far was 20 hours (test stopped after that point). Note: I have examined the places where this spinlock is being employed; all are reasonably short bounded sequences and should be suitable for interrupts being blocked without impacting overall kernel interrupt response latency. Signed-off-by: Mike Isely --- linux-core/drm_fops.c | 7 +++++-- linux-core/drm_lock.c | 35 ++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 17 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_fops.c b/linux-core/drm_fops.c index 7fe274a2..a4c76f75 100644 --- a/linux-core/drm_fops.c +++ b/linux-core/drm_fops.c @@ -373,6 +373,7 @@ int drm_release(struct inode *inode, struct file *filp) struct drm_file *file_priv = filp->private_data; struct drm_device *dev = file_priv->minor->dev; int retcode = 0; + unsigned long irqflags; lock_kernel(); @@ -403,9 +404,11 @@ int drm_release(struct inode *inode, struct file *filp) */ do{ - spin_lock(&dev->lock.spinlock); + spin_lock_irqsave(&dev->lock.spinlock, + irqflags); locked = dev->lock.idle_has_lock; - spin_unlock(&dev->lock.spinlock); + spin_unlock_irqrestore(&dev->lock.spinlock, + irqflags); if (locked) break; schedule(); diff --git a/linux-core/drm_lock.c b/linux-core/drm_lock.c index b8e4a5d9..39e1261c 100644 --- a/linux-core/drm_lock.c +++ b/linux-core/drm_lock.c @@ -53,6 +53,7 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv) DECLARE_WAITQUEUE(entry, current); struct drm_lock *lock = data; int ret = 0; + unsigned long irqflags; ++file_priv->lock_count; @@ -71,9 +72,9 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv) return -EINVAL; add_wait_queue(&dev->lock.lock_queue, &entry); - spin_lock(&dev->lock.spinlock); + spin_lock_irqsave(&dev->lock.spinlock, irqflags); dev->lock.user_waiters++; - spin_unlock(&dev->lock.spinlock); + spin_unlock_irqsave(&dev->lock.spinlock, irqflags); for (;;) { __set_current_state(TASK_INTERRUPTIBLE); if (!dev->lock.hw_lock) { @@ -95,9 +96,9 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv) break; } } - spin_lock(&dev->lock.spinlock); + spin_lock_irqsave(&dev->lock.spinlock, irqflags); dev->lock.user_waiters--; - spin_unlock(&dev->lock.spinlock); + spin_unlock_irqrestore(&dev->lock.spinlock, irqflags); __set_current_state(TASK_RUNNING); remove_wait_queue(&dev->lock.lock_queue, &entry); @@ -198,8 +199,9 @@ int drm_lock_take(struct drm_lock_data *lock_data, { unsigned int old, new, prev; volatile unsigned int *lock = &lock_data->hw_lock->lock; + unsigned long irqflags; - spin_lock(&lock_data->spinlock); + spin_lock_irqsave(&lock_data->spinlock, irqflags); do { old = *lock; if (old & _DRM_LOCK_HELD) @@ -211,7 +213,7 @@ int drm_lock_take(struct drm_lock_data *lock_data, } prev = cmpxchg(lock, old, new); } while (prev != old); - spin_unlock(&lock_data->spinlock); + spin_unlock_irqrestore(&lock_data->spinlock, irqflags); if (_DRM_LOCKING_CONTEXT(old) == context) { if (old & _DRM_LOCK_HELD) { @@ -273,15 +275,16 @@ int drm_lock_free(struct drm_lock_data *lock_data, unsigned int context) { unsigned int old, new, prev; volatile unsigned int *lock = &lock_data->hw_lock->lock; + unsigned long irqflags; - spin_lock(&lock_data->spinlock); + spin_lock_irqsave(&lock_data->spinlock, irqflags); if (lock_data->kernel_waiters != 0) { drm_lock_transfer(lock_data, 0); lock_data->idle_has_lock = 1; - spin_unlock(&lock_data->spinlock); + spin_unlock_irqrestore(&lock_data->spinlock, irqflags); return 1; } - spin_unlock(&lock_data->spinlock); + spin_unlock_irqrestore(&lock_data->spinlock, irqflags); do { old = *lock; @@ -345,19 +348,20 @@ static int drm_notifier(void *priv) void drm_idlelock_take(struct drm_lock_data *lock_data) { int ret = 0; + unsigned long irqflags; - spin_lock(&lock_data->spinlock); + spin_lock_irqsave(&lock_data->spinlock, irqflags); lock_data->kernel_waiters++; if (!lock_data->idle_has_lock) { - spin_unlock(&lock_data->spinlock); + spin_unlock_irqrestore(&lock_data->spinlock, irqflags); ret = drm_lock_take(lock_data, DRM_KERNEL_CONTEXT); - spin_lock(&lock_data->spinlock); + spin_lock_irqsave(&lock_data->spinlock, irqflags); if (ret == 1) lock_data->idle_has_lock = 1; } - spin_unlock(&lock_data->spinlock); + spin_unlock_irqrestore(&lock_data->spinlock, irq_flags); } EXPORT_SYMBOL(drm_idlelock_take); @@ -365,8 +369,9 @@ void drm_idlelock_release(struct drm_lock_data *lock_data) { unsigned int old, prev; volatile unsigned int *lock = &lock_data->hw_lock->lock; + unsigned long irqflags; - spin_lock(&lock_data->spinlock); + spin_lock_irqsave(&lock_data->spinlock, irqflags); if (--lock_data->kernel_waiters == 0) { if (lock_data->idle_has_lock) { do { @@ -377,7 +382,7 @@ void drm_idlelock_release(struct drm_lock_data *lock_data) lock_data->idle_has_lock = 0; } } - spin_unlock(&lock_data->spinlock); + spin_unlock_irqrestore(&lock_data->spinlock, irqflags); } EXPORT_SYMBOL(drm_idlelock_release); -- cgit v1.2.3 From 1ea8a470fe9103036817ae3a960522c37901bddc Mon Sep 17 00:00:00 2001 From: Alan Hourihane Date: Fri, 14 Mar 2008 00:25:42 +0000 Subject: fix build problems --- linux-core/drm_lock.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_lock.c b/linux-core/drm_lock.c index 39e1261c..db1d646b 100644 --- a/linux-core/drm_lock.c +++ b/linux-core/drm_lock.c @@ -74,7 +74,7 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv) add_wait_queue(&dev->lock.lock_queue, &entry); spin_lock_irqsave(&dev->lock.spinlock, irqflags); dev->lock.user_waiters++; - spin_unlock_irqsave(&dev->lock.spinlock, irqflags); + spin_unlock_irqrestore(&dev->lock.spinlock, irqflags); for (;;) { __set_current_state(TASK_INTERRUPTIBLE); if (!dev->lock.hw_lock) { @@ -361,7 +361,7 @@ void drm_idlelock_take(struct drm_lock_data *lock_data) if (ret == 1) lock_data->idle_has_lock = 1; } - spin_unlock_irqrestore(&lock_data->spinlock, irq_flags); + spin_unlock_irqrestore(&lock_data->spinlock, irqflags); } EXPORT_SYMBOL(drm_idlelock_take); -- cgit v1.2.3 From 76946ed83df2e39e3867538e54dc743fecb4f8e8 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Sun, 16 Mar 2008 12:56:11 +1000 Subject: drm: this u32 should be a dma_addr_t doesn't fix anything but just making it consistent --- linux-core/ati_pcigart.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'linux-core') diff --git a/linux-core/ati_pcigart.c b/linux-core/ati_pcigart.c index 93519e5f..5a424200 100644 --- a/linux-core/ati_pcigart.c +++ b/linux-core/ati_pcigart.c @@ -137,7 +137,8 @@ int drm_ati_pcigart_init(struct drm_device *dev, struct drm_ati_pcigart_info *ga struct drm_sg_mem *entry = dev->sg; void *address = NULL; unsigned long pages; - u32 *pci_gart, page_base, bus_address = 0; + u32 *pci_gart, page_base; + dma_addr_t bus_address = 0; int i, j, ret = 0; int order; int max_pages; -- cgit v1.2.3 From afa803ee40c1d06066f58a34761be58ba03badb5 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Sun, 16 Mar 2008 15:01:27 +1000 Subject: ati: fix rs690 igp gart by allocating the page table in 32-bit memory --- linux-core/ati_pcigart.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'linux-core') diff --git a/linux-core/ati_pcigart.c b/linux-core/ati_pcigart.c index 5a424200..74f91bc4 100644 --- a/linux-core/ati_pcigart.c +++ b/linux-core/ati_pcigart.c @@ -43,7 +43,7 @@ static void *drm_ati_alloc_pcigart_table(int order) DRM_DEBUG("%d order\n", order); - address = __get_free_pages(GFP_KERNEL | __GFP_COMP, + address = __get_free_pages(GFP_KERNEL | __GFP_COMP | __GFP_DMA32, order); if (address == 0UL) { return NULL; -- cgit v1.2.3 From b81d7b3b8d7ca83a9b79d2dbea22f00e78180516 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Sun, 16 Mar 2008 11:39:18 +0100 Subject: [via] Allow a little larger stride for SG DMA DownloadFromScreen. --- linux-core/via_dmablit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'linux-core') diff --git a/linux-core/via_dmablit.c b/linux-core/via_dmablit.c index a6a21782..b5f9f05f 100644 --- a/linux-core/via_dmablit.c +++ b/linux-core/via_dmablit.c @@ -618,7 +618,7 @@ via_build_sg_info(struct drm_device *dev, drm_via_sg_info_t *vsg, drm_via_dmabli * (Not a big limitation anyway.) */ - if ((xfer->mem_stride - xfer->line_length) >= PAGE_SIZE) { + if ((xfer->mem_stride - xfer->line_length) > 2*PAGE_SIZE) { DRM_ERROR("Too large system memory stride. Stride: %d, " "Length: %d\n", xfer->mem_stride, xfer->line_length); return -EINVAL; -- cgit v1.2.3 From 1f96e9a98245b18c99cc6a7e66372a076b9abf6b Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Mon, 17 Mar 2008 07:05:46 +1000 Subject: drm/pcigart: fix the pci gart to use the drm_pci wrapper. This is the correct fix for the RS690 and hopefully the dma coherent work. For now we limit everybody to a 32-bit DMA mask but it is possible for RS690 to use a 40-bit DMA mask for the GART table itself, and the PCIE cards can use 40-bits for the table entries. Signed-off-by: Dave Airlie --- linux-core/ati_pcigart.c | 101 +++++++++-------------------------------------- linux-core/drmP.h | 4 ++ linux-core/drm_compat.h | 4 ++ linux-core/xgi_pcie.c | 1 + 4 files changed, 28 insertions(+), 82 deletions(-) (limited to 'linux-core') diff --git a/linux-core/ati_pcigart.c b/linux-core/ati_pcigart.c index 74f91bc4..beaa4424 100644 --- a/linux-core/ati_pcigart.c +++ b/linux-core/ati_pcigart.c @@ -34,51 +34,23 @@ #include "drmP.h" # define ATI_PCIGART_PAGE_SIZE 4096 /**< PCI GART page size */ - -static void *drm_ati_alloc_pcigart_table(int order) +static int drm_ati_alloc_pcigart_table(struct drm_device *dev, + struct drm_ati_pcigart_info *gart_info) { - unsigned long address; - struct page *page; - int i; + gart_info->table_handle = drm_pci_alloc(dev, gart_info->table_size, + PAGE_SIZE, + gart_info->table_mask); + if (gart_info->table_handle == NULL) + return -ENOMEM; - DRM_DEBUG("%d order\n", order); - - address = __get_free_pages(GFP_KERNEL | __GFP_COMP | __GFP_DMA32, - order); - if (address == 0UL) { - return NULL; - } - - page = virt_to_page(address); - - for (i = 0; i < order; i++, page++) { -#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) - get_page(page); -#endif - SetPageReserved(page); - } - - DRM_DEBUG("returning 0x%08lx\n", address); - return (void *)address; + return 0; } -static void drm_ati_free_pcigart_table(void *address, int order) +static void drm_ati_free_pcigart_table(struct drm_device *dev, + struct drm_ati_pcigart_info *gart_info) { - struct page *page; - int i; - int num_pages = 1 << order; - DRM_DEBUG("\n"); - - page = virt_to_page((unsigned long)address); - - for (i = 0; i < num_pages; i++, page++) { -#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) - __put_page(page); -#endif - ClearPageReserved(page); - } - - free_pages((unsigned long)address, order); + drm_pci_free(dev, gart_info->table_handle); + gart_info->table_handle = NULL; } int drm_ati_pcigart_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info) @@ -86,8 +58,7 @@ int drm_ati_pcigart_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info struct drm_sg_mem *entry = dev->sg; unsigned long pages; int i; - int order; - int num_pages, max_pages; + int max_pages; /* we need to support large memory configurations */ if (!entry) { @@ -95,15 +66,7 @@ int drm_ati_pcigart_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info return 0; } - order = drm_order((gart_info->table_size + (PAGE_SIZE-1)) / PAGE_SIZE); - num_pages = 1 << order; - if (gart_info->bus_addr) { - if (gart_info->gart_table_location == DRM_ATI_GART_MAIN) { - pci_unmap_single(dev->pdev, gart_info->bus_addr, - num_pages * PAGE_SIZE, - PCI_DMA_TODEVICE); - } max_pages = (gart_info->table_size / sizeof(u32)); pages = (entry->pages <= max_pages) @@ -122,10 +85,9 @@ int drm_ati_pcigart_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info if (gart_info->gart_table_location == DRM_ATI_GART_MAIN - && gart_info->addr) { + && gart_info->table_handle) { - drm_ati_free_pcigart_table(gart_info->addr, order); - gart_info->addr = NULL; + drm_ati_free_pcigart_table(dev, gart_info); } return 1; @@ -140,9 +102,7 @@ int drm_ati_pcigart_init(struct drm_device *dev, struct drm_ati_pcigart_info *ga u32 *pci_gart, page_base; dma_addr_t bus_address = 0; int i, j, ret = 0; - int order; int max_pages; - int num_pages; if (!entry) { DRM_ERROR("no scatter/gather memory!\n"); @@ -152,31 +112,14 @@ int drm_ati_pcigart_init(struct drm_device *dev, struct drm_ati_pcigart_info *ga if (gart_info->gart_table_location == DRM_ATI_GART_MAIN) { DRM_DEBUG("PCI: no table in VRAM: using normal RAM\n"); - order = drm_order((gart_info->table_size + - (PAGE_SIZE-1)) / PAGE_SIZE); - num_pages = 1 << order; - address = drm_ati_alloc_pcigart_table(order); - if (!address) { + ret = drm_ati_alloc_pcigart_table(dev, gart_info); + if (ret) { DRM_ERROR("cannot allocate PCI GART page!\n"); goto done; } - if (!dev->pdev) { - DRM_ERROR("PCI device unknown!\n"); - goto done; - } - - bus_address = pci_map_single(dev->pdev, address, - num_pages * PAGE_SIZE, - PCI_DMA_TODEVICE); - if (bus_address == 0) { - DRM_ERROR("unable to map PCIGART pages!\n"); - order = drm_order((gart_info->table_size + - (PAGE_SIZE-1)) / PAGE_SIZE); - drm_ati_free_pcigart_table(address, order); - address = NULL; - goto done; - } + address = gart_info->table_handle->vaddr; + bus_address = gart_info->table_handle->busaddr; } else { address = gart_info->addr; bus_address = gart_info->bus_addr; @@ -225,12 +168,6 @@ int drm_ati_pcigart_init(struct drm_device *dev, struct drm_ati_pcigart_info *ga } } - if (gart_info->gart_table_location == DRM_ATI_GART_MAIN) - dma_sync_single_for_device(&dev->pdev->dev, - bus_address, - max_pages * sizeof(u32), - PCI_DMA_TODEVICE); - ret = 1; #if defined(__i386__) || defined(__x86_64__) diff --git a/linux-core/drmP.h b/linux-core/drmP.h index 2f76f3df..69d31e14 100644 --- a/linux-core/drmP.h +++ b/linux-core/drmP.h @@ -52,6 +52,7 @@ #include #include #include /* For (un)lock_kernel */ +#include #include #include #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,16) @@ -596,6 +597,9 @@ struct drm_ati_pcigart_info { int gart_reg_if; void *addr; dma_addr_t bus_addr; + dma_addr_t table_mask; + dma_addr_t member_mask; + struct drm_dma_handle *table_handle; drm_local_map_t mapping; int table_size; }; diff --git a/linux-core/drm_compat.h b/linux-core/drm_compat.h index 0613b5d2..3b1287e1 100644 --- a/linux-core/drm_compat.h +++ b/linux-core/drm_compat.h @@ -339,4 +339,8 @@ extern void *kmap_atomic_prot_pfn(unsigned long pfn, enum km_type type, #define flush_agp_mappings() do {} while(0) #endif +#ifndef DMA_BIT_MASK +#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : (1ULL<<(n)) - 1) +#endif + #endif diff --git a/linux-core/xgi_pcie.c b/linux-core/xgi_pcie.c index a7d3ea24..614616a9 100644 --- a/linux-core/xgi_pcie.c +++ b/linux-core/xgi_pcie.c @@ -86,6 +86,7 @@ int xgi_pcie_heap_init(struct xgi_info * info) return err; } + info->gart_info.table_mask = DMA_BIT_MASK(32); info->gart_info.gart_table_location = DRM_ATI_GART_MAIN; info->gart_info.gart_reg_if = DRM_ATI_GART_PCI; info->gart_info.table_size = info->dev->sg->pages * sizeof(u32); -- cgit v1.2.3 From 602800a280ecaf562427eada19b118b990ab26e1 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Mon, 17 Mar 2008 11:37:10 +0100 Subject: Evict cached_mapped relocatee before applying reloc. Fix that got left out after the intel-post-reloc merge. --- linux-core/i915_execbuf.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'linux-core') diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c index ff83d795..ae4a6121 100644 --- a/linux-core/i915_execbuf.c +++ b/linux-core/i915_execbuf.c @@ -138,6 +138,8 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, new_cmd_offset = reloc[0]; if (!relocatee->data_page || !drm_bo_same_page(relocatee->offset, new_cmd_offset)) { + struct drm_bo_mem_reg *mem = &relocatee->buf->mem; + drm_bo_kunmap(&relocatee->kmap); relocatee->data_page = NULL; relocatee->offset = new_cmd_offset; @@ -149,6 +151,10 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, relocatee->idle = I915_RELOC_IDLE; } + if (unlikely((mem->mem_type != DRM_BO_MEM_LOCAL) && + (mem->flags & DRM_BO_FLAG_CACHED_MAPPED))) + drm_bo_evict_cached(relocatee->buf); + ret = drm_bo_kmap(relocatee->buf, new_cmd_offset >> PAGE_SHIFT, 1, &relocatee->kmap); if (ret) { -- cgit v1.2.3 From a244d2905052d3263bdcc26b295558a354702b89 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 24 Mar 2008 03:22:42 +1100 Subject: nouveau: silence warning --- linux-core/Makefile.kernel | 2 +- linux-core/nouveau_bo.c | 305 ++++++++++++++++++++++++++++++++++++++++++++ linux-core/nouveau_buffer.c | 299 ------------------------------------------- 3 files changed, 306 insertions(+), 300 deletions(-) create mode 100644 linux-core/nouveau_bo.c delete mode 100644 linux-core/nouveau_buffer.c (limited to 'linux-core') diff --git a/linux-core/Makefile.kernel b/linux-core/Makefile.kernel index defbe43b..f012262d 100644 --- a/linux-core/Makefile.kernel +++ b/linux-core/Makefile.kernel @@ -23,7 +23,7 @@ i915-objs := i915_drv.o i915_dma.o i915_irq.o i915_mem.o i915_fence.o \ i915_buffer.o i915_compat.o i915_execbuf.o nouveau-objs := nouveau_drv.o nouveau_state.o nouveau_fifo.o nouveau_mem.o \ nouveau_object.o nouveau_irq.o nouveau_notifier.o nouveau_swmthd.o \ - nouveau_sgdma.o nouveau_dma.o nouveau_buffer.o nouveau_fence.o \ + nouveau_sgdma.o nouveau_dma.o nouveau_bo.o nouveau_fence.o \ nv04_timer.o \ nv04_mc.o nv40_mc.o nv50_mc.o \ nv04_fb.o nv10_fb.o nv40_fb.o \ diff --git a/linux-core/nouveau_bo.c b/linux-core/nouveau_bo.c new file mode 100644 index 00000000..7a899769 --- /dev/null +++ b/linux-core/nouveau_bo.c @@ -0,0 +1,305 @@ +/* + * Copyright 2007 Dave Airlied + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +/* + * Authors: Dave Airlied + * Ben Skeggs + * Jeremy Kolb + */ + +#include "drmP.h" +#include "nouveau_drm.h" +#include "nouveau_drv.h" +#include "nouveau_dma.h" + +static struct drm_ttm_backend * +nouveau_bo_create_ttm_backend_entry(struct drm_device * dev) +{ + struct drm_nouveau_private *dev_priv = dev->dev_private; + + switch (dev_priv->gart_info.type) { + case NOUVEAU_GART_AGP: + return drm_agp_init_ttm(dev); + case NOUVEAU_GART_SGDMA: + return nouveau_sgdma_init_ttm(dev); + default: + DRM_ERROR("Unknown GART type %d\n", dev_priv->gart_info.type); + break; + } + + return NULL; +} + +static int +nouveau_bo_fence_type(struct drm_buffer_object *bo, + uint32_t *fclass, uint32_t *type) +{ + /* When we get called, *fclass is set to the requested fence class */ + + if (bo->mem.proposed_flags & (DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE)) + *type = 3; + else + *type = 1; + return 0; + +} + +static int +nouveau_bo_invalidate_caches(struct drm_device *dev, uint64_t buffer_flags) +{ + /* We'll do this from user space. */ + return 0; +} + +static int +nouveau_bo_init_mem_type(struct drm_device *dev, uint32_t type, + struct drm_mem_type_manager *man) +{ + struct drm_nouveau_private *dev_priv = dev->dev_private; + + switch (type) { + case DRM_BO_MEM_LOCAL: + man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE | + _DRM_FLAG_MEMTYPE_CACHED; + man->drm_bus_maptype = 0; + break; + case DRM_BO_MEM_VRAM: + man->flags = _DRM_FLAG_MEMTYPE_FIXED | + _DRM_FLAG_MEMTYPE_MAPPABLE | + _DRM_FLAG_NEEDS_IOREMAP; + man->io_addr = NULL; + man->drm_bus_maptype = _DRM_FRAME_BUFFER; + man->io_offset = drm_get_resource_start(dev, 1); + man->io_size = drm_get_resource_len(dev, 1); + if (man->io_size > nouveau_mem_fb_amount(dev)) + man->io_size = nouveau_mem_fb_amount(dev); + break; + case DRM_BO_MEM_PRIV0: + /* Unmappable VRAM */ + man->flags = _DRM_FLAG_MEMTYPE_CMA; + man->drm_bus_maptype = 0; + break; + case DRM_BO_MEM_TT: + switch (dev_priv->gart_info.type) { + case NOUVEAU_GART_AGP: + man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE | + _DRM_FLAG_MEMTYPE_CSELECT | + _DRM_FLAG_NEEDS_IOREMAP; + man->drm_bus_maptype = _DRM_AGP; + break; + case NOUVEAU_GART_SGDMA: + man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE | + _DRM_FLAG_MEMTYPE_CSELECT | + _DRM_FLAG_MEMTYPE_CMA; + man->drm_bus_maptype = _DRM_SCATTER_GATHER; + break; + default: + DRM_ERROR("Unknown GART type: %d\n", + dev_priv->gart_info.type); + return -EINVAL; + } + + man->io_offset = dev_priv->gart_info.aper_base; + man->io_size = dev_priv->gart_info.aper_size; + man->io_addr = NULL; + break; + default: + DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); + return -EINVAL; + } + return 0; +} + +static uint64_t +nouveau_bo_evict_flags(struct drm_buffer_object *bo) +{ + switch (bo->mem.mem_type) { + case DRM_BO_MEM_LOCAL: + case DRM_BO_MEM_TT: + return DRM_BO_FLAG_MEM_LOCAL; + default: + return DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_CACHED; + } + return 0; +} + + +/* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access + * DRM_BO_MEM_{VRAM,PRIV0,TT} directly. + */ +static int +nouveau_bo_move_m2mf(struct drm_buffer_object *bo, int evict, int no_wait, + struct drm_bo_mem_reg *new_mem) +{ + struct drm_device *dev = bo->dev; + struct drm_nouveau_private *dev_priv = dev->dev_private; + struct nouveau_drm_channel *dchan = &dev_priv->channel; + struct drm_bo_mem_reg *old_mem = &bo->mem; + uint32_t srch, dsth, page_count; + + /* Can happen during init/takedown */ + if (!dchan->chan) + return -EINVAL; + + srch = old_mem->mem_type == DRM_BO_MEM_TT ? NvDmaTT : NvDmaFB; + dsth = new_mem->mem_type == DRM_BO_MEM_TT ? NvDmaTT : NvDmaFB; + if (srch != dchan->m2mf_dma_source || dsth != dchan->m2mf_dma_destin) { + dchan->m2mf_dma_source = srch; + dchan->m2mf_dma_destin = dsth; + + BEGIN_RING(NvSubM2MF, + NV_MEMORY_TO_MEMORY_FORMAT_SET_DMA_SOURCE, 2); + OUT_RING (dchan->m2mf_dma_source); + OUT_RING (dchan->m2mf_dma_destin); + } + + page_count = new_mem->num_pages; + while (page_count) { + int line_count = (page_count > 2047) ? 2047 : page_count; + + BEGIN_RING(NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8); + OUT_RING (old_mem->mm_node->start << PAGE_SHIFT); + OUT_RING (new_mem->mm_node->start << PAGE_SHIFT); + OUT_RING (PAGE_SIZE); /* src_pitch */ + OUT_RING (PAGE_SIZE); /* dst_pitch */ + OUT_RING (PAGE_SIZE); /* line_length */ + OUT_RING (line_count); + OUT_RING ((1<<8)|(1<<0)); + OUT_RING (0); + BEGIN_RING(NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); + OUT_RING (0); + + page_count -= line_count; + } + + return drm_bo_move_accel_cleanup(bo, evict, no_wait, dchan->chan->id, + DRM_FENCE_TYPE_EXE, 0, new_mem); +} + +/* Flip pages into the GART and move if we can. */ +static int +nouveau_bo_move_gart(struct drm_buffer_object *bo, int evict, int no_wait, + struct drm_bo_mem_reg *new_mem) +{ + struct drm_device *dev = bo->dev; + struct drm_bo_mem_reg tmp_mem; + int ret; + + tmp_mem = *new_mem; + tmp_mem.mm_node = NULL; + tmp_mem.proposed_flags = (DRM_BO_FLAG_MEM_TT | + DRM_BO_FLAG_CACHED | + DRM_BO_FLAG_FORCE_CACHING); + + ret = drm_bo_mem_space(bo, &tmp_mem, no_wait); + + if (ret) + return ret; + + ret = drm_ttm_bind (bo->ttm, &tmp_mem); + if (ret) + goto out_cleanup; + + ret = nouveau_bo_move_m2mf(bo, 1, no_wait, &tmp_mem); + if (ret) + goto out_cleanup; + + ret = drm_bo_move_ttm(bo, evict, no_wait, new_mem); + +out_cleanup: + if (tmp_mem.mm_node) { + mutex_lock(&dev->struct_mutex); + if (tmp_mem.mm_node != bo->pinned_node) + drm_mm_put_block(tmp_mem.mm_node); + tmp_mem.mm_node = NULL; + mutex_unlock(&dev->struct_mutex); + } + return ret; +} + +static int +nouveau_bo_move(struct drm_buffer_object *bo, int evict, int no_wait, + struct drm_bo_mem_reg *new_mem) +{ + struct drm_bo_mem_reg *old_mem = &bo->mem; + + if (new_mem->mem_type == DRM_BO_MEM_LOCAL) { + if (old_mem->mem_type == DRM_BO_MEM_LOCAL) + return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); +#if 0 + if (!nouveau_bo_move_to_gart(bo, evict, no_wait, new_mem)) +#endif + return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); + } + else + if (old_mem->mem_type == DRM_BO_MEM_LOCAL) { +#if 0 + if (nouveau_bo_move_to_gart(bo, evict, no_wait, new_mem)) +#endif + return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); + } + else { +// if (nouveau_bo_move_m2mf(bo, evict, no_wait, new_mem)) + return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); + } + + if (0) { + nouveau_bo_move_m2mf(bo, 0, 0, NULL); + nouveau_bo_move_gart(bo, 0, 0, NULL); + } + + return 0; +} + +static void +nouveau_bo_flush_ttm(struct drm_ttm *ttm) +{ +} + +static uint32_t nouveau_mem_prios[] = { + DRM_BO_MEM_PRIV0, + DRM_BO_MEM_VRAM, + DRM_BO_MEM_TT, + DRM_BO_MEM_LOCAL +}; +static uint32_t nouveau_busy_prios[] = { + DRM_BO_MEM_TT, + DRM_BO_MEM_PRIV0, + DRM_BO_MEM_VRAM, + DRM_BO_MEM_LOCAL +}; + +struct drm_bo_driver nouveau_bo_driver = { + .mem_type_prio = nouveau_mem_prios, + .mem_busy_prio = nouveau_busy_prios, + .num_mem_type_prio = sizeof(nouveau_mem_prios)/sizeof(uint32_t), + .num_mem_busy_prio = sizeof(nouveau_busy_prios)/sizeof(uint32_t), + .create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry, + .fence_type = nouveau_bo_fence_type, + .invalidate_caches = nouveau_bo_invalidate_caches, + .init_mem_type = nouveau_bo_init_mem_type, + .evict_flags = nouveau_bo_evict_flags, + .move = nouveau_bo_move, + .ttm_cache_flush= nouveau_bo_flush_ttm, + .command_stream_barrier = NULL +}; diff --git a/linux-core/nouveau_buffer.c b/linux-core/nouveau_buffer.c deleted file mode 100644 index 11549317..00000000 --- a/linux-core/nouveau_buffer.c +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright 2007 Dave Airlied - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -/* - * Authors: Dave Airlied - * Ben Skeggs - * Jeremy Kolb - */ - -#include "drmP.h" -#include "nouveau_drm.h" -#include "nouveau_drv.h" -#include "nouveau_dma.h" - -static struct drm_ttm_backend * -nouveau_bo_create_ttm_backend_entry(struct drm_device * dev) -{ - struct drm_nouveau_private *dev_priv = dev->dev_private; - - switch (dev_priv->gart_info.type) { - case NOUVEAU_GART_AGP: - return drm_agp_init_ttm(dev); - case NOUVEAU_GART_SGDMA: - return nouveau_sgdma_init_ttm(dev); - default: - DRM_ERROR("Unknown GART type %d\n", dev_priv->gart_info.type); - break; - } - - return NULL; -} - -static int -nouveau_bo_fence_type(struct drm_buffer_object *bo, - uint32_t *fclass, uint32_t *type) -{ - /* When we get called, *fclass is set to the requested fence class */ - - if (bo->mem.proposed_flags & (DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE)) - *type = 3; - else - *type = 1; - return 0; - -} - -static int -nouveau_bo_invalidate_caches(struct drm_device *dev, uint64_t buffer_flags) -{ - /* We'll do this from user space. */ - return 0; -} - -static int -nouveau_bo_init_mem_type(struct drm_device *dev, uint32_t type, - struct drm_mem_type_manager *man) -{ - struct drm_nouveau_private *dev_priv = dev->dev_private; - - switch (type) { - case DRM_BO_MEM_LOCAL: - man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE | - _DRM_FLAG_MEMTYPE_CACHED; - man->drm_bus_maptype = 0; - break; - case DRM_BO_MEM_VRAM: - man->flags = _DRM_FLAG_MEMTYPE_FIXED | - _DRM_FLAG_MEMTYPE_MAPPABLE | - _DRM_FLAG_NEEDS_IOREMAP; - man->io_addr = NULL; - man->drm_bus_maptype = _DRM_FRAME_BUFFER; - man->io_offset = drm_get_resource_start(dev, 1); - man->io_size = drm_get_resource_len(dev, 1); - if (man->io_size > nouveau_mem_fb_amount(dev)) - man->io_size = nouveau_mem_fb_amount(dev); - break; - case DRM_BO_MEM_PRIV0: - /* Unmappable VRAM */ - man->flags = _DRM_FLAG_MEMTYPE_CMA; - man->drm_bus_maptype = 0; - break; - case DRM_BO_MEM_TT: - switch (dev_priv->gart_info.type) { - case NOUVEAU_GART_AGP: - man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE | - _DRM_FLAG_MEMTYPE_CSELECT | - _DRM_FLAG_NEEDS_IOREMAP; - man->drm_bus_maptype = _DRM_AGP; - break; - case NOUVEAU_GART_SGDMA: - man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE | - _DRM_FLAG_MEMTYPE_CSELECT | - _DRM_FLAG_MEMTYPE_CMA; - man->drm_bus_maptype = _DRM_SCATTER_GATHER; - break; - default: - DRM_ERROR("Unknown GART type: %d\n", - dev_priv->gart_info.type); - return -EINVAL; - } - - man->io_offset = dev_priv->gart_info.aper_base; - man->io_size = dev_priv->gart_info.aper_size; - man->io_addr = NULL; - break; - default: - DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); - return -EINVAL; - } - return 0; -} - -static uint64_t -nouveau_bo_evict_flags(struct drm_buffer_object *bo) -{ - switch (bo->mem.mem_type) { - case DRM_BO_MEM_LOCAL: - case DRM_BO_MEM_TT: - return DRM_BO_FLAG_MEM_LOCAL; - default: - return DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_CACHED; - } - return 0; -} - - -/* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access - * DRM_BO_MEM_{VRAM,PRIV0,TT} directly. - */ -static int -nouveau_bo_move_m2mf(struct drm_buffer_object *bo, int evict, int no_wait, - struct drm_bo_mem_reg *new_mem) -{ - struct drm_device *dev = bo->dev; - struct drm_nouveau_private *dev_priv = dev->dev_private; - struct nouveau_drm_channel *dchan = &dev_priv->channel; - struct drm_bo_mem_reg *old_mem = &bo->mem; - uint32_t srch, dsth, page_count; - - /* Can happen during init/takedown */ - if (!dchan->chan) - return -EINVAL; - - srch = old_mem->mem_type == DRM_BO_MEM_TT ? NvDmaTT : NvDmaFB; - dsth = new_mem->mem_type == DRM_BO_MEM_TT ? NvDmaTT : NvDmaFB; - if (srch != dchan->m2mf_dma_source || dsth != dchan->m2mf_dma_destin) { - dchan->m2mf_dma_source = srch; - dchan->m2mf_dma_destin = dsth; - - BEGIN_RING(NvSubM2MF, - NV_MEMORY_TO_MEMORY_FORMAT_SET_DMA_SOURCE, 2); - OUT_RING (dchan->m2mf_dma_source); - OUT_RING (dchan->m2mf_dma_destin); - } - - page_count = new_mem->num_pages; - while (page_count) { - int line_count = (page_count > 2047) ? 2047 : page_count; - - BEGIN_RING(NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8); - OUT_RING (old_mem->mm_node->start << PAGE_SHIFT); - OUT_RING (new_mem->mm_node->start << PAGE_SHIFT); - OUT_RING (PAGE_SIZE); /* src_pitch */ - OUT_RING (PAGE_SIZE); /* dst_pitch */ - OUT_RING (PAGE_SIZE); /* line_length */ - OUT_RING (line_count); - OUT_RING ((1<<8)|(1<<0)); - OUT_RING (0); - BEGIN_RING(NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); - OUT_RING (0); - - page_count -= line_count; - } - - return drm_bo_move_accel_cleanup(bo, evict, no_wait, dchan->chan->id, - DRM_FENCE_TYPE_EXE, 0, new_mem); -} - -/* Flip pages into the GART and move if we can. */ -static int -nouveau_bo_move_gart(struct drm_buffer_object *bo, int evict, int no_wait, - struct drm_bo_mem_reg *new_mem) -{ - struct drm_device *dev = bo->dev; - struct drm_bo_mem_reg tmp_mem; - int ret; - - tmp_mem = *new_mem; - tmp_mem.mm_node = NULL; - tmp_mem.proposed_flags = (DRM_BO_FLAG_MEM_TT | - DRM_BO_FLAG_CACHED | - DRM_BO_FLAG_FORCE_CACHING); - - ret = drm_bo_mem_space(bo, &tmp_mem, no_wait); - - if (ret) - return ret; - - ret = drm_ttm_bind (bo->ttm, &tmp_mem); - if (ret) - goto out_cleanup; - - ret = nouveau_bo_move_m2mf(bo, 1, no_wait, &tmp_mem); - if (ret) - goto out_cleanup; - - ret = drm_bo_move_ttm(bo, evict, no_wait, new_mem); - -out_cleanup: - if (tmp_mem.mm_node) { - mutex_lock(&dev->struct_mutex); - if (tmp_mem.mm_node != bo->pinned_node) - drm_mm_put_block(tmp_mem.mm_node); - tmp_mem.mm_node = NULL; - mutex_unlock(&dev->struct_mutex); - } - return ret; -} - -static int -nouveau_bo_move(struct drm_buffer_object *bo, int evict, int no_wait, - struct drm_bo_mem_reg *new_mem) -{ - struct drm_bo_mem_reg *old_mem = &bo->mem; - - if (new_mem->mem_type == DRM_BO_MEM_LOCAL) { - if (old_mem->mem_type == DRM_BO_MEM_LOCAL) - return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); -#if 0 - if (!nouveau_bo_move_to_gart(bo, evict, no_wait, new_mem)) -#endif - return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); - } - else - if (old_mem->mem_type == DRM_BO_MEM_LOCAL) { -#if 0 - if (nouveau_bo_move_to_gart(bo, evict, no_wait, new_mem)) -#endif - return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); - } - else { -// if (nouveau_bo_move_m2mf(bo, evict, no_wait, new_mem)) - return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); - } - return 0; -} - -static void -nouveau_bo_flush_ttm(struct drm_ttm *ttm) -{ -} - -static uint32_t nouveau_mem_prios[] = { - DRM_BO_MEM_PRIV0, - DRM_BO_MEM_VRAM, - DRM_BO_MEM_TT, - DRM_BO_MEM_LOCAL -}; -static uint32_t nouveau_busy_prios[] = { - DRM_BO_MEM_TT, - DRM_BO_MEM_PRIV0, - DRM_BO_MEM_VRAM, - DRM_BO_MEM_LOCAL -}; - -struct drm_bo_driver nouveau_bo_driver = { - .mem_type_prio = nouveau_mem_prios, - .mem_busy_prio = nouveau_busy_prios, - .num_mem_type_prio = sizeof(nouveau_mem_prios)/sizeof(uint32_t), - .num_mem_busy_prio = sizeof(nouveau_busy_prios)/sizeof(uint32_t), - .create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry, - .fence_type = nouveau_bo_fence_type, - .invalidate_caches = nouveau_bo_invalidate_caches, - .init_mem_type = nouveau_bo_init_mem_type, - .evict_flags = nouveau_bo_evict_flags, - .move = nouveau_bo_move, - .ttm_cache_flush= nouveau_bo_flush_ttm, - .command_stream_barrier = NULL -}; -- cgit v1.2.3 From b8567bafff58cfb9d77145088fd5b8ad2e5cde6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Thu, 6 Mar 2008 17:35:56 +0100 Subject: Don't call fence::poll during irq if there are no waiters. --- linux-core/i915_fence.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'linux-core') diff --git a/linux-core/i915_fence.c b/linux-core/i915_fence.c index de64a4f2..e403be6a 100644 --- a/linux-core/i915_fence.c +++ b/linux-core/i915_fence.c @@ -162,11 +162,13 @@ static int i915_fence_emit_sequence(struct drm_device *dev, uint32_t class, void i915_fence_handler(struct drm_device *dev) { + struct drm_i915_private *dev_priv = (struct drm_i915_private *) dev->dev_private; struct drm_fence_manager *fm = &dev->fm; struct drm_fence_class_manager *fc = &fm->fence_class[0]; write_lock(&fm->lock); - i915_fence_poll(dev, 0, fc->waiting_types); + if (likely(dev_priv->fence_irq_on)) + i915_fence_poll(dev, 0, fc->waiting_types); write_unlock(&fm->lock); } -- cgit v1.2.3 From 1f4ba62567d32fdd32786273326e1aab17d5d412 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Sun, 30 Mar 2008 15:14:45 +0200 Subject: [i915] Report buffer state _after_ fence submission to user-space. This fixes a problem where the wrong bo->fence_type was reported, and also saves some memory space. [bo core] export the drm_bo_fill_rep_arg function. --- linux-core/drm_bo.c | 5 +++-- linux-core/drm_objects.h | 2 ++ linux-core/i915_execbuf.c | 10 ++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c index a94bd8a1..144935d6 100644 --- a/linux-core/drm_bo.c +++ b/linux-core/drm_bo.c @@ -1163,8 +1163,8 @@ static int drm_bo_wait_unfenced(struct drm_buffer_object *bo, int no_wait, * Bo locked. */ -static void drm_bo_fill_rep_arg(struct drm_buffer_object *bo, - struct drm_bo_info_rep *rep) +void drm_bo_fill_rep_arg(struct drm_buffer_object *bo, + struct drm_bo_info_rep *rep) { if (!rep) return; @@ -1195,6 +1195,7 @@ static void drm_bo_fill_rep_arg(struct drm_buffer_object *bo, DRM_BO_REP_BUSY); } } +EXPORT_SYMBOL(drm_bo_fill_rep_arg); /* * Wait for buffer idle and register that we've mapped the buffer. diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h index 69e3f67f..1f5d6ee2 100644 --- a/linux-core/drm_objects.h +++ b/linux-core/drm_objects.h @@ -746,6 +746,8 @@ extern int drm_bo_pfn_prot(struct drm_buffer_object *bo, unsigned long dst_offset, unsigned long *pfn, pgprot_t *prot); +extern void drm_bo_fill_rep_arg(struct drm_buffer_object *bo, + struct drm_bo_info_rep *rep); /* diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c index ae4a6121..729ee0c3 100644 --- a/linux-core/i915_execbuf.c +++ b/linux-core/i915_execbuf.c @@ -64,7 +64,6 @@ struct i915_relocatee_info { struct drm_i915_validate_buffer { struct drm_buffer_object *buffer; - struct drm_bo_info_rep rep; int presumed_offset_correct; void __user *data; int ret; @@ -686,7 +685,7 @@ int i915_validate_buffer_list(struct drm_file *file_priv, req->bo_req.flags, req->bo_req.mask, req->bo_req.hint, req->bo_req.fence_class, 0, - &item->rep, &item->buffer); + NULL, &item->buffer); if (ret) { DRM_ERROR("error on handle validate %d\n", ret); goto out_err; @@ -725,6 +724,7 @@ static int i915_handle_copyback(struct drm_device *dev, int err = ret; int i; struct drm_i915_op_arg arg; + struct drm_buffer_object *bo; if (ret) drm_putback_buffer_objects(dev); @@ -733,7 +733,10 @@ static int i915_handle_copyback(struct drm_device *dev, for (i = 0; i < num_buffers; ++i) { arg.handled = 1; arg.d.rep.ret = buffers->ret; - arg.d.rep.bo_info = buffers->rep; + bo = buffers->buffer; + mutex_lock(&bo->mutex); + drm_bo_fill_rep_arg(bo, &arg.d.rep.bo_info); + mutex_unlock(&bo->mutex); if (__copy_to_user(buffers->data, &arg, sizeof(arg))) err = -EFAULT; buffers++; @@ -780,7 +783,6 @@ void i915_fence_or_sync(struct drm_file *file_priv, fence_arg->handle = ~0; fence_arg->error = ret; } - drm_putback_buffer_objects(dev); if (fence_p) *fence_p = NULL; -- cgit v1.2.3 From 22d931f9664e1857e07ce7ab8aad760a4a22f15e Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Sun, 30 Mar 2008 21:30:43 +0200 Subject: Initialize the fence::error member. --- linux-core/drm_fence.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'linux-core') diff --git a/linux-core/drm_fence.c b/linux-core/drm_fence.c index 0ca0c408..7c78e09f 100644 --- a/linux-core/drm_fence.c +++ b/linux-core/drm_fence.c @@ -445,6 +445,7 @@ int drm_fence_object_emit(struct drm_fence_object *fence, uint32_t fence_flags, fence->type = type; fence->waiting_types = 0; fence->signaled_types = 0; + fence->error = 0; fence->sequence = sequence; fence->native_types = native_types; if (list_empty(&fc->ring)) @@ -482,6 +483,7 @@ static int drm_fence_object_init(struct drm_device *dev, uint32_t fence_class, fence->signaled_types = 0; fence->waiting_types = 0; fence->sequence = 0; + fence->error = 0; fence->dev = dev; write_unlock_irqrestore(&fm->lock, flags); if (fence_flags & DRM_FENCE_FLAG_EMIT) { -- cgit v1.2.3 From 87ae5b22e3120d205f520a99cea31743903d49a2 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Sun, 6 Apr 2008 09:33:50 +0200 Subject: Fix emergency allocation accounting. --- linux-core/drm_memory.c | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_memory.c b/linux-core/drm_memory.c index 12e01414..75f5b521 100644 --- a/linux-core/drm_memory.c +++ b/linux-core/drm_memory.c @@ -61,35 +61,39 @@ static inline size_t drm_size_align(size_t size) int drm_alloc_memctl(size_t size) { - int ret = 0; + int ret = 0; unsigned long a_size = drm_size_align(size); - unsigned long new_used = drm_memctl.cur_used + a_size; + unsigned long new_used; spin_lock(&drm_memctl.lock); - if (unlikely(new_used > drm_memctl.high_threshold)) { - if (!DRM_SUSER(DRM_CURPROC) || - (new_used + drm_memctl.emer_used > drm_memctl.emer_threshold) || - (a_size > 2*PAGE_SIZE)) { - ret = -ENOMEM; - goto out; - } - - /* - * Allow small root-only allocations, even if the - * high threshold is exceeded. - */ - - new_used -= drm_memctl.high_threshold; - drm_memctl.emer_used += new_used; - a_size -= new_used; + new_used = drm_memctl.cur_used + a_size; + if (likely(new_used < drm_memctl.high_threshold)) { + drm_memctl.cur_used = new_used; + goto out; } - drm_memctl.cur_used += a_size; + + /* + * Allow small allocations from root-only processes to + * succeed until the emergency threshold is reached. + */ + + new_used += drm_memctl.emer_used; + if (unlikely(!DRM_SUSER(DRM_CURPROC) || + (a_size > 16*PAGE_SIZE) || + (new_used > drm_memctl.emer_threshold))) { + ret = -ENOMEM; + goto out; + } + + drm_memctl.cur_used = drm_memctl.high_threshold; + drm_memctl.emer_used = new_used - drm_memctl.high_threshold; out: spin_unlock(&drm_memctl.lock); return ret; } EXPORT_SYMBOL(drm_alloc_memctl); + void drm_free_memctl(size_t size) { unsigned long a_size = drm_size_align(size); -- cgit v1.2.3 From 51a0fdcf3fef5af57938d9958efd698e96d78803 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Sun, 6 Apr 2008 09:46:29 +0200 Subject: [I915] Fix VRAM eviction. --- linux-core/i915_buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'linux-core') diff --git a/linux-core/i915_buffer.c b/linux-core/i915_buffer.c index 08067476..8d991c42 100644 --- a/linux-core/i915_buffer.c +++ b/linux-core/i915_buffer.c @@ -249,10 +249,10 @@ int i915_move(struct drm_buffer_object *bo, if (old_mem->mem_type == DRM_BO_MEM_LOCAL) { return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); } else if (new_mem->mem_type == DRM_BO_MEM_LOCAL) { - if (0) /*i915_move_flip(bo, evict, no_wait, new_mem)*/ + if (1) /*i915_move_flip(bo, evict, no_wait, new_mem)*/ return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); } else { - if (0) /*i915_move_blit(bo, evict, no_wait, new_mem)*/ + if (1) /*i915_move_blit(bo, evict, no_wait, new_mem)*/ return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); } return 0; -- cgit v1.2.3 From c3888b97f60fbbc0b1382e5a16689eecaa2f79a5 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Sun, 6 Apr 2008 10:32:02 +0200 Subject: Use clflush() when available for cache flushing. --- linux-core/drm_objects.h | 2 +- linux-core/drm_ttm.c | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h index 1f5d6ee2..9bd04fff 100644 --- a/linux-core/drm_objects.h +++ b/linux-core/drm_objects.h @@ -335,7 +335,7 @@ extern void drm_ttm_unbind(struct drm_ttm *ttm); extern void drm_ttm_evict(struct drm_ttm *ttm); extern void drm_ttm_fixup_caching(struct drm_ttm *ttm); extern struct page *drm_ttm_get_page(struct drm_ttm *ttm, int index); -extern void drm_ttm_cache_flush(void); +extern void drm_ttm_cache_flush(struct page *pages[], unsigned long num_pages); extern int drm_ttm_populate(struct drm_ttm *ttm); extern int drm_ttm_set_user(struct drm_ttm *ttm, struct task_struct *tsk, diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c index e991254f..da202a58 100644 --- a/linux-core/drm_ttm.c +++ b/linux-core/drm_ttm.c @@ -30,13 +30,48 @@ #include "drmP.h" +#if defined( CONFIG_X86 ) && (LINUX_VERSION_CODE >= (2,6,24)) +static void drm_clflush_page(struct page *page) +{ + uint8_t *page_virtual; + unsigned int i; + + if (unlikely(page == NULL)) + return; + + page_virtual = kmap_atomic(page, KM_USER0); + + for (i=0; i < PAGE_SIZE; i += boot_cpu_data.x86_clflush_size) + clflush(page_virtual + i); + + kunmap_atomic(page_virtual, KM_USER0); +} + +static void drm_ttm_cache_flush_clflush(struct page *pages[], unsigned long num_pages) +{ + unsigned long i; + + mb(); + for (i=0; i < num_pages; ++i) + drm_clflush_page(*pages++); + mb(); +} +#endif + static void drm_ttm_ipi_handler(void *null) { flush_agp_cache(); } -void drm_ttm_cache_flush(void) +void drm_ttm_cache_flush(struct page *pages[], unsigned long num_pages) { + +#if defined( CONFIG_X86 ) && (LINUX_VERSION_CODE >= (2,6,24)) + if (cpu_has_clflush) { + drm_ttm_cache_flush_clflush(pages, num_pages); + return; + } +#endif if (on_each_cpu(drm_ttm_ipi_handler, NULL, 1, 1) != 0) DRM_ERROR("Timed out waiting for drm cache flush.\n"); } @@ -114,7 +149,7 @@ static int drm_ttm_set_caching(struct drm_ttm *ttm, int noncached) return 0; if (noncached) - drm_ttm_cache_flush(); + drm_ttm_cache_flush(ttm->pages, ttm->num_pages); for (i = 0; i < ttm->num_pages; ++i) { cur_page = ttm->pages + i; -- cgit v1.2.3 From e89710bef7691e4e9d0bc7d427542bfae6ce4258 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Sun, 6 Apr 2008 11:21:22 +0200 Subject: Place highmem pages last in the ttm page array. --- linux-core/drm_objects.h | 2 ++ linux-core/drm_ttm.c | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h index 9bd04fff..c32edacd 100644 --- a/linux-core/drm_objects.h +++ b/linux-core/drm_objects.h @@ -311,6 +311,8 @@ struct drm_ttm_backend { struct drm_ttm { struct page *dummy_read_page; struct page **pages; + long first_himem_page; + long last_lomem_page; uint32_t page_flags; unsigned long num_pages; atomic_t vma_count; diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c index da202a58..c306a2c0 100644 --- a/linux-core/drm_ttm.c +++ b/linux-core/drm_ttm.c @@ -263,12 +263,16 @@ struct page *drm_ttm_get_page(struct drm_ttm *ttm, int index) struct page *p; struct drm_buffer_manager *bm = &ttm->dev->bm; - p = ttm->pages[index]; - if (!p) { + while(NULL == (p = ttm->pages[index])) { p = drm_ttm_alloc_page(); if (!p) return NULL; - ttm->pages[index] = p; + + if (PageHighMem(p)) + ttm->pages[--ttm->first_himem_page] = p; + else + ttm->pages[++ttm->last_lomem_page] = p; + ++bm->cur_pages; } return p; @@ -376,6 +380,8 @@ struct drm_ttm *drm_ttm_create(struct drm_device *dev, unsigned long size, ttm->destroy = 0; ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + ttm->first_himem_page = ttm->num_pages; + ttm->last_lomem_page = -1; ttm->page_flags = page_flags; -- cgit v1.2.3 From c12b60b5094fe97db60cd0f18fafd1720679bd38 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 7 Apr 2008 13:05:51 +1000 Subject: nouveau: enable m2mf for tt<->vram moves, fix fence_poll --- linux-core/nouveau_bo.c | 3 +-- linux-core/nouveau_fence.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'linux-core') diff --git a/linux-core/nouveau_bo.c b/linux-core/nouveau_bo.c index 7a899769..fcda3037 100644 --- a/linux-core/nouveau_bo.c +++ b/linux-core/nouveau_bo.c @@ -259,12 +259,11 @@ nouveau_bo_move(struct drm_buffer_object *bo, int evict, int no_wait, return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); } else { -// if (nouveau_bo_move_m2mf(bo, evict, no_wait, new_mem)) + if (nouveau_bo_move_m2mf(bo, evict, no_wait, new_mem)) return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); } if (0) { - nouveau_bo_move_m2mf(bo, 0, 0, NULL); nouveau_bo_move_gart(bo, 0, 0, NULL); } diff --git a/linux-core/nouveau_fence.c b/linux-core/nouveau_fence.c index 59dcf7d0..4ad51ae4 100644 --- a/linux-core/nouveau_fence.c +++ b/linux-core/nouveau_fence.c @@ -80,12 +80,11 @@ nouveau_fence_poll(struct drm_device *dev, uint32_t class, uint32_t waiting_type struct drm_nouveau_private *dev_priv = dev->dev_private; struct drm_fence_class_manager *fc = &dev->fm.fence_class[class]; struct nouveau_channel *chan = dev_priv->fifos[class]; - uint32_t pending_types = 0; DRM_DEBUG("class=%d\n", class); DRM_DEBUG("pending: 0x%08x 0x%08x\n", waiting_types, fc->waiting_types); - if (pending_types) { + if (waiting_types & DRM_FENCE_TYPE_EXE) { uint32_t sequence = NV_READ(chan->ref_cnt); DRM_DEBUG("got 0x%08x\n", sequence); -- cgit v1.2.3 From dfa9f0399223d86a6478bf93be879da476f93eda Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 7 Apr 2008 13:29:11 +1000 Subject: nouveau: enable accelerated move to sysmem --- linux-core/nouveau_bo.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) (limited to 'linux-core') diff --git a/linux-core/nouveau_bo.c b/linux-core/nouveau_bo.c index fcda3037..ab3b23a4 100644 --- a/linux-core/nouveau_bo.c +++ b/linux-core/nouveau_bo.c @@ -198,8 +198,8 @@ nouveau_bo_move_m2mf(struct drm_buffer_object *bo, int evict, int no_wait, /* Flip pages into the GART and move if we can. */ static int -nouveau_bo_move_gart(struct drm_buffer_object *bo, int evict, int no_wait, - struct drm_bo_mem_reg *new_mem) +nouveau_bo_move_flipd(struct drm_buffer_object *bo, int evict, int no_wait, + struct drm_bo_mem_reg *new_mem) { struct drm_device *dev = bo->dev; struct drm_bo_mem_reg tmp_mem; @@ -212,11 +212,10 @@ nouveau_bo_move_gart(struct drm_buffer_object *bo, int evict, int no_wait, DRM_BO_FLAG_FORCE_CACHING); ret = drm_bo_mem_space(bo, &tmp_mem, no_wait); - if (ret) return ret; - ret = drm_ttm_bind (bo->ttm, &tmp_mem); + ret = drm_ttm_bind(bo->ttm, &tmp_mem); if (ret) goto out_cleanup; @@ -234,6 +233,7 @@ out_cleanup: tmp_mem.mm_node = NULL; mutex_unlock(&dev->struct_mutex); } + return ret; } @@ -246,16 +246,12 @@ nouveau_bo_move(struct drm_buffer_object *bo, int evict, int no_wait, if (new_mem->mem_type == DRM_BO_MEM_LOCAL) { if (old_mem->mem_type == DRM_BO_MEM_LOCAL) return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); -#if 0 - if (!nouveau_bo_move_to_gart(bo, evict, no_wait, new_mem)) -#endif + if (nouveau_bo_move_flipd(bo, evict, no_wait, new_mem)) return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); } else if (old_mem->mem_type == DRM_BO_MEM_LOCAL) { -#if 0 - if (nouveau_bo_move_to_gart(bo, evict, no_wait, new_mem)) -#endif + if (1 /*nouveau_bo_move_flips(bo, evict, no_wait, new_mem)*/) return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); } else { @@ -263,10 +259,6 @@ nouveau_bo_move(struct drm_buffer_object *bo, int evict, int no_wait, return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); } - if (0) { - nouveau_bo_move_gart(bo, 0, 0, NULL); - } - return 0; } -- cgit v1.2.3 From db61f02bd7e4b9d5ac416f1ef98bac1bd4d984bc Mon Sep 17 00:00:00 2001 From: Patrice Mandin Date: Mon, 7 Apr 2008 22:24:24 +0200 Subject: Missing KERNEL_VERSION macro --- linux-core/drm_ttm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c index c306a2c0..80a8ff5d 100644 --- a/linux-core/drm_ttm.c +++ b/linux-core/drm_ttm.c @@ -30,7 +30,7 @@ #include "drmP.h" -#if defined( CONFIG_X86 ) && (LINUX_VERSION_CODE >= (2,6,24)) +#if defined( CONFIG_X86 ) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24)) static void drm_clflush_page(struct page *page) { uint8_t *page_virtual; @@ -66,7 +66,7 @@ static void drm_ttm_ipi_handler(void *null) void drm_ttm_cache_flush(struct page *pages[], unsigned long num_pages) { -#if defined( CONFIG_X86 ) && (LINUX_VERSION_CODE >= (2,6,24)) +#if defined( CONFIG_X86 ) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24)) if (cpu_has_clflush) { drm_ttm_cache_flush_clflush(pages, num_pages); return; -- cgit v1.2.3 From b986d7d2c9090fc62c1853f62886dd124e8066c1 Mon Sep 17 00:00:00 2001 From: Keith Packard Date: Thu, 27 Mar 2008 11:40:04 -0700 Subject: Save and restore dsparb and d_state regs --- linux-core/i915_drv.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'linux-core') diff --git a/linux-core/i915_drv.c b/linux-core/i915_drv.c index e18bc8d0..5a6f0adc 100644 --- a/linux-core/i915_drv.c +++ b/linux-core/i915_drv.c @@ -281,6 +281,9 @@ static int i915_suspend(struct drm_device *dev, pm_message_t state) pci_save_state(dev->pdev); pci_read_config_byte(dev->pdev, LBB, &dev_priv->saveLBB); + /* Display arbitration control */ + dev_priv->saveDSPARB = I915_READ(DSPARB); + /* Pipe & plane A info */ dev_priv->savePIPEACONF = I915_READ(PIPEACONF); dev_priv->savePIPEASRC = I915_READ(PIPEASRC); @@ -374,6 +377,7 @@ static int i915_suspend(struct drm_device *dev, pm_message_t state) dev_priv->saveVGACNTRL = I915_READ(VGACNTRL); /* Clock gating state */ + dev_priv->saveD_STATE = I915_READ(D_STATE); dev_priv->saveDSPCLK_GATE_D = I915_READ(DSPCLK_GATE_D); /* Cache mode state */ @@ -413,6 +417,8 @@ static int i915_resume(struct drm_device *dev) pci_write_config_byte(dev->pdev, LBB, dev_priv->saveLBB); + I915_WRITE(DSPARB, dev_priv->saveDSPARB); + /* Pipe & plane A info */ /* Prime the clock */ if (dev_priv->saveDPLL_A & DPLL_VCO_ENABLE) { @@ -532,6 +538,7 @@ static int i915_resume(struct drm_device *dev) udelay(150); /* Clock gating state */ + I915_WRITE (D_STATE, dev_priv->saveD_STATE); I915_WRITE (DSPCLK_GATE_D, dev_priv->saveDSPCLK_GATE_D); /* Cache mode state */ -- cgit v1.2.3 From 65dd0e68ff0e0e354925adb7d5fffeb0ffbb485c Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Fri, 11 Apr 2008 09:36:12 +0200 Subject: Fix up buffer manager locking. --- linux-core/drm_bo.c | 8 ++++---- linux-core/drm_bo_lock.c | 42 +++++++++++++++++++++++++++++------------- linux-core/drm_compat.c | 2 +- linux-core/drm_objects.h | 6 +++++- linux-core/drm_vm.c | 2 +- linux-core/i915_execbuf.c | 2 +- 6 files changed, 41 insertions(+), 21 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c index 144935d6..4ef697b5 100644 --- a/linux-core/drm_bo.c +++ b/linux-core/drm_bo.c @@ -1938,7 +1938,7 @@ int drm_bo_setstatus_ioctl(struct drm_device *dev, return -EINVAL; } - ret = drm_bo_read_lock(&dev->bm.bm_lock); + ret = drm_bo_read_lock(&dev->bm.bm_lock, 1); if (ret) return ret; @@ -2449,7 +2449,7 @@ int drm_mm_init_ioctl(struct drm_device *dev, void *data, struct drm_file *file_ return -EINVAL; } - ret = drm_bo_write_lock(&bm->bm_lock, file_priv); + ret = drm_bo_write_lock(&bm->bm_lock, 1, file_priv); if (ret) return ret; @@ -2500,7 +2500,7 @@ int drm_mm_takedown_ioctl(struct drm_device *dev, void *data, struct drm_file *f return -EINVAL; } - ret = drm_bo_write_lock(&bm->bm_lock, file_priv); + ret = drm_bo_write_lock(&bm->bm_lock, 0, file_priv); if (ret) return ret; @@ -2548,7 +2548,7 @@ int drm_mm_lock_ioctl(struct drm_device *dev, void *data, struct drm_file *file_ } if (arg->lock_flags & DRM_BO_LOCK_UNLOCK_BM) { - ret = drm_bo_write_lock(&dev->bm.bm_lock, file_priv); + ret = drm_bo_write_lock(&dev->bm.bm_lock, 1, file_priv); if (ret) return ret; } diff --git a/linux-core/drm_bo_lock.c b/linux-core/drm_bo_lock.c index 2795384e..32ebfbe2 100644 --- a/linux-core/drm_bo_lock.c +++ b/linux-core/drm_bo_lock.c @@ -49,7 +49,7 @@ * unmappable regions to mappable. It's a bug to leave kernel space with the * read lock held. * - * Both read- and write lock taking is interruptible for low signal-delivery + * Both read- and write lock taking may be interruptible for low signal-delivery * latency. The locking functions will return -EAGAIN if interrupted by a * signal. * @@ -71,14 +71,20 @@ void drm_bo_read_unlock(struct drm_bo_lock *lock) if (unlikely(atomic_add_negative(-1, &lock->readers))) BUG(); if (atomic_read(&lock->readers) == 0) - wake_up_interruptible(&lock->queue); + wake_up_all(&lock->queue); } EXPORT_SYMBOL(drm_bo_read_unlock); -int drm_bo_read_lock(struct drm_bo_lock *lock) +int drm_bo_read_lock(struct drm_bo_lock *lock, int interruptible) { while (unlikely(atomic_read(&lock->write_lock_pending) != 0)) { int ret; + + if (!interruptible) { + wait_event(lock->queue, + atomic_read(&lock->write_lock_pending) == 0); + continue; + } ret = wait_event_interruptible (lock->queue, atomic_read(&lock->write_lock_pending) == 0); if (ret) @@ -87,8 +93,14 @@ int drm_bo_read_lock(struct drm_bo_lock *lock) while (unlikely(!atomic_add_unless(&lock->readers, 1, -1))) { int ret; + + if (!interruptible) { + wait_event(lock->queue, + atomic_read(&lock->readers) != -1); + continue; + } ret = wait_event_interruptible - (lock->queue, atomic_add_unless(&lock->readers, 1, -1)); + (lock->queue, atomic_read(&lock->readers) != -1); if (ret) return -EAGAIN; } @@ -100,9 +112,7 @@ static int __drm_bo_write_unlock(struct drm_bo_lock *lock) { if (unlikely(atomic_cmpxchg(&lock->readers, -1, 0) != -1)) return -EINVAL; - if (unlikely(atomic_cmpxchg(&lock->write_lock_pending, 1, 0) != 1)) - return -EINVAL; - wake_up_interruptible(&lock->queue); + wake_up_all(&lock->queue); return 0; } @@ -116,21 +126,26 @@ static void drm_bo_write_lock_remove(struct drm_file *file_priv, BUG_ON(ret); } -int drm_bo_write_lock(struct drm_bo_lock *lock, struct drm_file *file_priv) +int drm_bo_write_lock(struct drm_bo_lock *lock, int interruptible, + struct drm_file *file_priv) { int ret = 0; struct drm_device *dev; - if (unlikely(atomic_cmpxchg(&lock->write_lock_pending, 0, 1) != 0)) - return -EINVAL; + atomic_inc(&lock->write_lock_pending); while (unlikely(atomic_cmpxchg(&lock->readers, 0, -1) != 0)) { + if (!interruptible) { + wait_event(lock->queue, + atomic_read(&lock->readers) == 0); + continue; + } ret = wait_event_interruptible - (lock->queue, atomic_cmpxchg(&lock->readers, 0, -1) == 0); + (lock->queue, atomic_read(&lock->readers) == 0); if (ret) { - atomic_set(&lock->write_lock_pending, 0); - wake_up_interruptible(&lock->queue); + atomic_dec(&lock->write_lock_pending); + wake_up_all(&lock->queue); return -EAGAIN; } } @@ -141,6 +156,7 @@ int drm_bo_write_lock(struct drm_bo_lock *lock, struct drm_file *file_priv) * while holding it. */ + atomic_dec(&lock->write_lock_pending); dev = file_priv->minor->dev; mutex_lock(&dev->struct_mutex); ret = drm_add_user_object(file_priv, &lock->base, 0); diff --git a/linux-core/drm_compat.c b/linux-core/drm_compat.c index 5dabeaea..23e5028a 100644 --- a/linux-core/drm_compat.c +++ b/linux-core/drm_compat.c @@ -213,7 +213,7 @@ static struct page *drm_bo_vm_fault(struct vm_area_struct *vma, unsigned long bus_size; dev = bo->dev; - while(drm_bo_read_lock(&dev->bm.bm_lock)); + drm_bo_read_lock(&dev->bm.bm_lock, 0); mutex_lock(&bo->mutex); diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h index c32edacd..a1f3a18d 100644 --- a/linux-core/drm_objects.h +++ b/linux-core/drm_objects.h @@ -320,6 +320,8 @@ struct drm_ttm { int destroy; uint32_t mapping_offset; struct drm_ttm_backend *be; + unsigned long highest_lomem_entry; + unsigned long lowest_himem_entry; enum { ttm_bound, ttm_evicted, @@ -798,8 +800,10 @@ extern void drm_regs_init(struct drm_reg_manager *manager, extern void drm_bo_init_lock(struct drm_bo_lock *lock); extern void drm_bo_read_unlock(struct drm_bo_lock *lock); -extern int drm_bo_read_lock(struct drm_bo_lock *lock); +extern int drm_bo_read_lock(struct drm_bo_lock *lock, + int interruptible); extern int drm_bo_write_lock(struct drm_bo_lock *lock, + int interruptible, struct drm_file *file_priv); extern int drm_bo_write_unlock(struct drm_bo_lock *lock, diff --git a/linux-core/drm_vm.c b/linux-core/drm_vm.c index ffda8284..a09bcddb 100644 --- a/linux-core/drm_vm.c +++ b/linux-core/drm_vm.c @@ -738,7 +738,7 @@ static unsigned long drm_bo_vm_nopfn(struct vm_area_struct *vma, return NOPFN_SIGBUS; dev = bo->dev; - err = drm_bo_read_lock(&dev->bm.bm_lock); + err = drm_bo_read_lock(&dev->bm.bm_lock, 1); if (err) return NOPFN_REFAULT; diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c index 729ee0c3..088a2693 100644 --- a/linux-core/i915_execbuf.c +++ b/linux-core/i915_execbuf.c @@ -845,7 +845,7 @@ int i915_execbuffer(struct drm_device *dev, void *data, if (exec_buf->num_buffers > dev_priv->max_validate_buffers) return -EINVAL; - ret = drm_bo_read_lock(&dev->bm.bm_lock); + ret = drm_bo_read_lock(&dev->bm.bm_lock, 1); if (ret) return ret; -- cgit v1.2.3 From c9b73ef6daff75df27d17260a9fc84e68f1b21b4 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Sun, 13 Apr 2008 14:49:14 +0200 Subject: Unlock the BO mutex while waiting for idle, unmapped, unfenced. Move unfenced checking into idle checking. Never time out while waiting for software events like unmapped or unfenced. --- linux-core/drm_bo.c | 511 ++++++++++++++++++++++------------------------ linux-core/drm_bo_move.c | 9 +- linux-core/drm_objects.h | 14 +- linux-core/drm_vm.c | 5 +- linux-core/i915_execbuf.c | 10 +- 5 files changed, 271 insertions(+), 278 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c index 4ef697b5..0853d748 100644 --- a/linux-core/drm_bo.c +++ b/linux-core/drm_bo.c @@ -275,30 +275,81 @@ out_err: /* * Call bo->mutex locked. - * Wait until the buffer is idle. + * Returns -EBUSY if the buffer is currently rendered to or from. 0 otherwise. */ -int drm_bo_wait(struct drm_buffer_object *bo, int lazy, int ignore_signals, - int no_wait) +static int drm_bo_busy(struct drm_buffer_object *bo, int check_unfenced) { - int ret; + struct drm_fence_object *fence = bo->fence; - DRM_ASSERT_LOCKED(&bo->mutex); + if (check_unfenced && (bo->priv_flags & _DRM_BO_FLAG_UNFENCED)) + return -EBUSY; - if (bo->fence) { - if (drm_fence_object_signaled(bo->fence, bo->fence_type)) { + if (fence) { + if (drm_fence_object_signaled(fence, bo->fence_type)) { + drm_fence_usage_deref_unlocked(&bo->fence); + return 0; + } + drm_fence_object_flush(fence, DRM_FENCE_TYPE_EXE); + if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(&bo->fence); return 0; } + return -EBUSY; + } + return 0; +} + +static int drm_bo_check_unfenced(struct drm_buffer_object *bo) +{ + int ret; + + mutex_lock(&bo->mutex); + ret = (bo->priv_flags & _DRM_BO_FLAG_UNFENCED); + mutex_unlock(&bo->mutex); + return ret; +} + + +/* + * Call bo->mutex locked. + * Wait until the buffer is idle. + */ + +int drm_bo_wait(struct drm_buffer_object *bo, int lazy, int interruptible, + int no_wait, int check_unfenced) +{ + int ret; + + DRM_ASSERT_LOCKED(&bo->mutex); + while(unlikely(drm_bo_busy(bo, check_unfenced))) { if (no_wait) return -EBUSY; - ret = drm_fence_object_wait(bo->fence, lazy, ignore_signals, - bo->fence_type); - if (ret) - return ret; + if (check_unfenced && (bo->priv_flags & _DRM_BO_FLAG_UNFENCED)) { + mutex_unlock(&bo->mutex); + wait_event(bo->event_queue, !drm_bo_check_unfenced(bo)); + mutex_lock(&bo->mutex); + bo->priv_flags |= _DRM_BO_FLAG_UNLOCKED; + } + + if (bo->fence) { + struct drm_fence_object *fence; + uint32_t fence_type = bo->fence_type; + + drm_fence_reference_unlocked(&fence, bo->fence); + mutex_unlock(&bo->mutex); + + ret = drm_fence_object_wait(fence, lazy, !interruptible, + fence_type); + + drm_fence_usage_deref_unlocked(&fence); + mutex_lock(&bo->mutex); + bo->priv_flags |= _DRM_BO_FLAG_UNLOCKED; + if (ret) + return ret; + } - drm_fence_usage_deref_unlocked(&bo->fence); } return 0; } @@ -314,7 +365,7 @@ static int drm_bo_expire_fence(struct drm_buffer_object *bo, int allow_errors) unsigned long _end = jiffies + 3 * DRM_HZ; int ret; do { - ret = drm_bo_wait(bo, 0, 1, 0); + ret = drm_bo_wait(bo, 0, 0, 0, 0); if (ret && allow_errors) return ret; @@ -690,24 +741,32 @@ static int drm_bo_evict(struct drm_buffer_object *bo, unsigned mem_type, * buffer mutex. */ - if (bo->priv_flags & _DRM_BO_FLAG_UNFENCED) - goto out; - if (bo->mem.mem_type != mem_type) - goto out; - - ret = drm_bo_wait(bo, 0, 0, no_wait); + do { + bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED; + + if (unlikely(bo->mem.flags & + (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT))) + goto out_unlock; + if (unlikely(bo->priv_flags & _DRM_BO_FLAG_UNFENCED)) + goto out_unlock; + if (unlikely(bo->mem.mem_type != mem_type)) + goto out_unlock; + ret = drm_bo_wait(bo, 0, 1, no_wait, 0); + if (ret) + goto out_unlock; - if (ret && ret != -EAGAIN) { - DRM_ERROR("Failed to expire fence before " - "buffer eviction.\n"); - goto out; - } + } while(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED); evict_mem = bo->mem; evict_mem.mm_node = NULL; evict_mem = bo->mem; evict_mem.proposed_flags = dev->driver->bo_driver->evict_flags(bo); + + mutex_lock(&dev->struct_mutex); + list_del_init(&bo->lru); + mutex_unlock(&dev->struct_mutex); + ret = drm_bo_mem_space(bo, &evict_mem, no_wait); if (ret) { @@ -725,20 +784,21 @@ static int drm_bo_evict(struct drm_buffer_object *bo, unsigned mem_type, goto out; } + DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_EVICTED, + _DRM_BO_FLAG_EVICTED); + +out: mutex_lock(&dev->struct_mutex); if (evict_mem.mm_node) { if (evict_mem.mm_node != bo->pinned_node) drm_mm_put_block(evict_mem.mm_node); evict_mem.mm_node = NULL; } - list_del(&bo->lru); drm_bo_add_to_lru(bo); + BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED); +out_unlock: mutex_unlock(&dev->struct_mutex); - DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_EVICTED, - _DRM_BO_FLAG_EVICTED); - -out: return ret; } @@ -773,8 +833,6 @@ static int drm_bo_mem_force_space(struct drm_device *dev, atomic_inc(&entry->usage); mutex_unlock(&dev->struct_mutex); mutex_lock(&entry->mutex); - BUG_ON(entry->mem.flags & (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT)); - ret = drm_bo_evict(entry, mem_type, no_wait); mutex_unlock(&entry->mutex); drm_bo_usage_deref_unlocked(&entry); @@ -1040,46 +1098,23 @@ EXPORT_SYMBOL(drm_lookup_buffer_object); /* * Call bo->mutex locked. - * Returns 1 if the buffer is currently rendered to or from. 0 otherwise. + * Returns -EBUSY if the buffer is currently rendered to or from. 0 otherwise. * Doesn't do any fence flushing as opposed to the drm_bo_busy function. */ -static int drm_bo_quick_busy(struct drm_buffer_object *bo) +static int drm_bo_quick_busy(struct drm_buffer_object *bo, int check_unfenced) { struct drm_fence_object *fence = bo->fence; - BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED); - if (fence) { - if (drm_fence_object_signaled(fence, bo->fence_type)) { - drm_fence_usage_deref_unlocked(&bo->fence); - return 0; - } - return 1; - } - return 0; -} - -/* - * Call bo->mutex locked. - * Returns 1 if the buffer is currently rendered to or from. 0 otherwise. - */ - -static int drm_bo_busy(struct drm_buffer_object *bo) -{ - struct drm_fence_object *fence = bo->fence; + if (check_unfenced && (bo->priv_flags & _DRM_BO_FLAG_UNFENCED)) + return -EBUSY; - BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED); if (fence) { if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(&bo->fence); return 0; } - drm_fence_object_flush(fence, DRM_FENCE_TYPE_EXE); - if (drm_fence_object_signaled(fence, bo->fence_type)) { - drm_fence_usage_deref_unlocked(&bo->fence); - return 0; - } - return 1; + return -EBUSY; } return 0; } @@ -1103,61 +1138,26 @@ static int drm_bo_wait_unmapped(struct drm_buffer_object *bo, int no_wait) { int ret = 0; - if ((atomic_read(&bo->mapped) >= 0) && no_wait) - return -EBUSY; - - DRM_WAIT_ON(ret, bo->event_queue, 3 * DRM_HZ, - atomic_read(&bo->mapped) == -1); + if (likely(atomic_read(&bo->mapped)) == 0) + return 0; - if (ret == -EINTR) - ret = -EAGAIN; + if (unlikely(no_wait)) + return -EBUSY; - return ret; -} + do { + mutex_unlock(&bo->mutex); + ret = wait_event_interruptible(bo->event_queue, + atomic_read(&bo->mapped) == 0); + mutex_lock(&bo->mutex); + bo->priv_flags |= _DRM_BO_FLAG_UNLOCKED; -static int drm_bo_check_unfenced(struct drm_buffer_object *bo) -{ - int ret; + if (ret == -ERESTARTSYS) + ret = -EAGAIN; + } while((ret == 0) && atomic_read(&bo->mapped) > 0); - mutex_lock(&bo->mutex); - ret = (bo->priv_flags & _DRM_BO_FLAG_UNFENCED); - mutex_unlock(&bo->mutex); return ret; } -/* - * Wait until a buffer, scheduled to be fenced moves off the unfenced list. - * Until then, we cannot really do anything with it except delete it. - */ - -static int drm_bo_wait_unfenced(struct drm_buffer_object *bo, int no_wait, - int eagain_if_wait) -{ - int ret = (bo->priv_flags & _DRM_BO_FLAG_UNFENCED); - - if (ret && no_wait) - return -EBUSY; - else if (!ret) - return 0; - - ret = 0; - mutex_unlock(&bo->mutex); - DRM_WAIT_ON (ret, bo->event_queue, 3 * DRM_HZ, - !drm_bo_check_unfenced(bo)); - mutex_lock(&bo->mutex); - if (ret == -EINTR) - return -EAGAIN; - ret = (bo->priv_flags & _DRM_BO_FLAG_UNFENCED); - if (ret) { - DRM_ERROR("Timeout waiting for buffer to become fenced\n"); - return -EBUSY; - } - if (eagain_if_wait) - return -EAGAIN; - - return 0; -} - /* * Fill in the ioctl reply argument with buffer info. * Bo locked. @@ -1190,7 +1190,7 @@ void drm_bo_fill_rep_arg(struct drm_buffer_object *bo, rep->rep_flags = 0; rep->page_alignment = bo->mem.page_alignment; - if ((bo->priv_flags & _DRM_BO_FLAG_UNFENCED) || drm_bo_quick_busy(bo)) { + if ((bo->priv_flags & _DRM_BO_FLAG_UNFENCED) || drm_bo_quick_busy(bo, 1)) { DRM_FLAG_MASKED(rep->rep_flags, DRM_BO_REP_BUSY, DRM_BO_REP_BUSY); } @@ -1221,59 +1221,27 @@ static int drm_buffer_object_map(struct drm_file *file_priv, uint32_t handle, return -EINVAL; mutex_lock(&bo->mutex); - ret = drm_bo_wait_unfenced(bo, no_wait, 0); - if (ret) - goto out; - - /* - * If this returns true, we are currently unmapped. - * We need to do this test, because unmapping can - * be done without the bo->mutex held. - */ - - while (1) { - if (atomic_inc_and_test(&bo->mapped)) { - if (no_wait && drm_bo_busy(bo)) { - atomic_dec(&bo->mapped); - ret = -EBUSY; - goto out; - } - ret = drm_bo_wait(bo, 0, 0, no_wait); - if (ret) { - atomic_dec(&bo->mapped); - goto out; - } - - if (bo->mem.flags & DRM_BO_FLAG_CACHED_MAPPED) - drm_bo_evict_cached(bo); - - break; - } else if (bo->mem.flags & DRM_BO_FLAG_CACHED_MAPPED) { + do { + bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED; - /* - * We are already mapped with different flags. - * need to wait for unmap. - */ + ret = drm_bo_wait(bo, 0, 1, no_wait, 1); - ret = drm_bo_wait_unmapped(bo, no_wait); - if (ret) - goto out; + if (bo->mem.flags & DRM_BO_FLAG_CACHED_MAPPED) + drm_bo_evict_cached(bo); - continue; - } - break; - } + } while (bo->priv_flags & _DRM_BO_FLAG_UNLOCKED); + atomic_inc(&bo->mapped); mutex_lock(&dev->struct_mutex); ret = drm_add_ref_object(file_priv, &bo->base, _DRM_REF_TYPE1); mutex_unlock(&dev->struct_mutex); if (ret) { - if (atomic_add_negative(-1, &bo->mapped)) + if (atomic_dec_and_test(&bo->mapped)) wake_up_all(&bo->event_queue); } else drm_bo_fill_rep_arg(bo, rep); -out: + mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(&bo); return ret; @@ -1325,7 +1293,7 @@ static void drm_buffer_user_object_unmap(struct drm_file *file_priv, BUG_ON(action != _DRM_REF_TYPE1); - if (atomic_add_negative(-1, &bo->mapped)) + if (atomic_dec_and_test(&bo->mapped)) wake_up_all(&bo->event_queue); } @@ -1341,19 +1309,8 @@ int drm_bo_move_buffer(struct drm_buffer_object *bo, uint64_t new_mem_flags, struct drm_buffer_manager *bm = &dev->bm; int ret = 0; struct drm_bo_mem_reg mem; - /* - * Flush outstanding fences. - */ - - drm_bo_busy(bo); - /* - * Wait for outstanding fences. - */ - - ret = drm_bo_wait(bo, 0, 0, no_wait); - if (ret) - return ret; + BUG_ON(bo->fence != NULL); mem.num_pages = bo->num_pages; mem.size = mem.num_pages << PAGE_SHIFT; @@ -1439,64 +1396,14 @@ static int drm_bo_mem_compat(struct drm_bo_mem_reg *mem) static int drm_buffer_object_validate(struct drm_buffer_object *bo, uint32_t fence_class, - int move_unfenced, int no_wait) + int move_unfenced, int no_wait, + int move_buffer) { struct drm_device *dev = bo->dev; struct drm_buffer_manager *bm = &dev->bm; - struct drm_bo_driver *driver = dev->driver->bo_driver; - uint32_t ftype; int ret; - DRM_DEBUG("Proposed flags 0x%016llx, Old flags 0x%016llx\n", - (unsigned long long) bo->mem.proposed_flags, - (unsigned long long) bo->mem.flags); - - ret = driver->fence_type(bo, &fence_class, &ftype); - - if (ret) { - DRM_ERROR("Driver did not support given buffer permissions\n"); - return ret; - } - - /* - * We're switching command submission mechanism, - * or cannot simply rely on the hardware serializing for us. - * - * Insert a driver-dependant barrier or wait for buffer idle. - */ - - if ((fence_class != bo->fence_class) || - ((ftype ^ bo->fence_type) & bo->fence_type)) { - - ret = -EINVAL; - if (driver->command_stream_barrier) { - ret = driver->command_stream_barrier(bo, - fence_class, - ftype, - no_wait); - } - if (ret) - ret = drm_bo_wait(bo, 0, 0, no_wait); - - if (ret) - return ret; - - } - - bo->new_fence_class = fence_class; - bo->new_fence_type = ftype; - - ret = drm_bo_wait_unmapped(bo, no_wait); - if (ret) { - DRM_ERROR("Timed out waiting for buffer unmap.\n"); - return ret; - } - - /* - * Check whether we need to move buffer. - */ - - if (!drm_bo_mem_compat(&bo->mem)) { + if (move_buffer) { ret = drm_bo_move_buffer(bo, bo->mem.proposed_flags, no_wait, move_unfenced); if (ret) { @@ -1580,6 +1487,82 @@ static int drm_buffer_object_validate(struct drm_buffer_object *bo, return 0; } +/* + * This function is called with bo->mutex locked, but may release it + * temporarily to wait for events. + */ + +static int drm_bo_prepare_for_validate(struct drm_buffer_object *bo, + uint64_t flags, + uint64_t mask, + uint32_t hint, + uint32_t fence_class, + int no_wait, + int *move_buffer) +{ + struct drm_device *dev = bo->dev; + struct drm_bo_driver *driver = dev->driver->bo_driver; + uint32_t ftype; + + int ret; + + DRM_DEBUG("Proposed flags 0x%016llx, Old flags 0x%016llx\n", + (unsigned long long) bo->mem.proposed_flags, + (unsigned long long) bo->mem.flags); + + ret = drm_bo_modify_proposed_flags (bo, flags, mask); + if (ret) + return ret; + + ret = drm_bo_wait_unmapped(bo, no_wait); + if (ret) + return ret; + + ret = driver->fence_type(bo, &fence_class, &ftype); + + if (ret) { + DRM_ERROR("Driver did not support given buffer permissions.\n"); + return ret; + } + + /* + * We're switching command submission mechanism, + * or cannot simply rely on the hardware serializing for us. + * Insert a driver-dependant barrier or wait for buffer idle. + */ + + if ((fence_class != bo->fence_class) || + ((ftype ^ bo->fence_type) & bo->fence_type)) { + + ret = -EINVAL; + if (driver->command_stream_barrier) { + ret = driver->command_stream_barrier(bo, + fence_class, + ftype, + no_wait); + } + if (ret && ret != -EAGAIN) + ret = drm_bo_wait(bo, 0, 1, no_wait, 1); + if (ret) + return ret; + } + + bo->new_fence_class = fence_class; + bo->new_fence_type = ftype; + + /* + * Check whether we need to move buffer. + */ + + *move_buffer = 0; + if (!drm_bo_mem_compat(&bo->mem)) { + *move_buffer = 1; + ret = drm_bo_wait(bo, 0, 1, no_wait, 1); + } + + return ret; +} + /** * drm_bo_do_validate: * @@ -1612,21 +1595,28 @@ int drm_bo_do_validate(struct drm_buffer_object *bo, { int ret; int no_wait = (hint & DRM_BO_HINT_DONT_BLOCK) != 0; + int move_buffer; mutex_lock(&bo->mutex); - ret = drm_bo_wait_unfenced(bo, no_wait, 0); - if (ret) - goto out; + do { + bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED; - ret = drm_bo_modify_proposed_flags (bo, flags, mask); - if (ret) - goto out; + ret = drm_bo_prepare_for_validate(bo, flags, mask, hint, + fence_class, no_wait, + &move_buffer); + if (ret) + goto out; + + } while(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED); ret = drm_buffer_object_validate(bo, fence_class, !(hint & DRM_BO_HINT_DONT_FENCE), - no_wait); + no_wait, + move_buffer); + + BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED); out: if (rep) drm_bo_fill_rep_arg(bo, rep); @@ -1657,22 +1647,19 @@ EXPORT_SYMBOL(drm_bo_do_validate); * fencing mechanism. At this point, there isn't any use of this * from the user mode code. * - * @use_old_fence_class: don't change fence class, pull it from the buffer object - * * @rep: To be stuffed with the reply from validation - * + * * @bp_rep: To be stuffed with the buffer object pointer * - * Perform drm_bo_do_validate on a buffer referenced by a user-space handle. - * Some permissions checking is done on the parameters, otherwise this - * is a thin wrapper. + * Perform drm_bo_do_validate on a buffer referenced by a user-space handle instead + * of a pointer to a buffer object. Optionally return a pointer to the buffer object. + * This is a convenience wrapper only. */ int drm_bo_handle_validate(struct drm_file *file_priv, uint32_t handle, uint64_t flags, uint64_t mask, uint32_t hint, uint32_t fence_class, - int use_old_fence_class, struct drm_bo_info_rep *rep, struct drm_buffer_object **bo_rep) { @@ -1687,17 +1674,9 @@ int drm_bo_handle_validate(struct drm_file *file_priv, uint32_t handle, if (!bo) return -EINVAL; - if (use_old_fence_class) - fence_class = bo->fence_class; - - /* - * Only allow creator to change shared buffer mask. - */ - if (bo->base.owner != file_priv) mask &= ~(DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE); - ret = drm_bo_do_validate(bo, flags, mask, hint, fence_class, rep); if (!ret && bo_rep) @@ -1709,6 +1688,7 @@ int drm_bo_handle_validate(struct drm_file *file_priv, uint32_t handle, } EXPORT_SYMBOL(drm_bo_handle_validate); + static int drm_bo_handle_info(struct drm_file *file_priv, uint32_t handle, struct drm_bo_info_rep *rep) { @@ -1723,8 +1703,12 @@ static int drm_bo_handle_info(struct drm_file *file_priv, uint32_t handle, return -EINVAL; mutex_lock(&bo->mutex); - if (!(bo->priv_flags & _DRM_BO_FLAG_UNFENCED)) - (void)drm_bo_busy(bo); + + /* + * FIXME: Quick busy here? + */ + + drm_bo_busy(bo, 1); drm_bo_fill_rep_arg(bo, rep); mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(&bo); @@ -1748,15 +1732,11 @@ static int drm_bo_handle_wait(struct drm_file *file_priv, uint32_t handle, return -EINVAL; mutex_lock(&bo->mutex); - ret = drm_bo_wait_unfenced(bo, no_wait, 0); - if (ret) - goto out; - ret = drm_bo_wait(bo, hint & DRM_BO_HINT_WAIT_LAZY, 0, no_wait); + ret = drm_bo_wait(bo, hint & DRM_BO_HINT_WAIT_LAZY, 1, no_wait, 1); if (ret) goto out; drm_bo_fill_rep_arg(bo, rep); - out: mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(&bo); @@ -1793,7 +1773,7 @@ int drm_buffer_object_create(struct drm_device *dev, mutex_lock(&bo->mutex); atomic_set(&bo->usage, 1); - atomic_set(&bo->mapped, -1); + atomic_set(&bo->mapped, 0); DRM_INIT_WAITQUEUE(&bo->event_queue); INIT_LIST_HEAD(&bo->lru); INIT_LIST_HEAD(&bo->pinned_lru); @@ -1835,17 +1815,18 @@ int drm_buffer_object_create(struct drm_device *dev, goto out_err; } - ret = drm_buffer_object_validate(bo, 0, 0, hint & DRM_BO_HINT_DONT_BLOCK); + mutex_unlock(&bo->mutex); + ret = drm_bo_do_validate(bo, 0, 0, hint & DRM_BO_HINT_DONT_BLOCK, + 0, NULL); if (ret) - goto out_err; + goto out_err_unlocked; - mutex_unlock(&bo->mutex); *buf_obj = bo; return 0; out_err: mutex_unlock(&bo->mutex); - +out_err_unlocked: drm_bo_usage_deref_unlocked(&bo); return ret; } @@ -1931,6 +1912,7 @@ int drm_bo_setstatus_ioctl(struct drm_device *dev, struct drm_bo_map_wait_idle_arg *arg = data; struct drm_bo_info_req *req = &arg->d.req; struct drm_bo_info_rep *rep = &arg->d.rep; + struct drm_buffer_object *bo; int ret; if (!dev->bm.initialized) { @@ -1942,24 +1924,25 @@ int drm_bo_setstatus_ioctl(struct drm_device *dev, if (ret) return ret; - /* - * validate the buffer. note that 'fence_class' will be unused - * as we pass use_old_fence_class=1 here. Note also that - * the libdrm API doesn't pass fence_class to the kernel, - * so it's a good thing it isn't used here. - */ - ret = drm_bo_handle_validate(file_priv, req->handle, - req->flags, - req->mask, - req->hint | DRM_BO_HINT_DONT_FENCE, - req->fence_class, 1, - rep, NULL); + mutex_lock(&dev->struct_mutex); + bo = drm_lookup_buffer_object(file_priv, req->handle, 1); + mutex_unlock(&dev->struct_mutex); + + if (!bo) + return -EINVAL; + + if (bo->base.owner != file_priv) + req->mask &= ~(DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE); + + ret = drm_bo_do_validate(bo, req->flags, req->mask, + req->hint | DRM_BO_HINT_DONT_FENCE, + bo->fence_class, rep); + + drm_bo_usage_deref_unlocked(&bo); (void) drm_bo_read_unlock(&dev->bm.bm_lock); - if (ret) - return ret; - return 0; + return ret; } int drm_bo_map_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) diff --git a/linux-core/drm_bo_move.c b/linux-core/drm_bo_move.c index 21673daa..bf0e1b74 100644 --- a/linux-core/drm_bo_move.c +++ b/linux-core/drm_bo_move.c @@ -356,10 +356,11 @@ int drm_bo_move_accel_cleanup(struct drm_buffer_object *bo, bo->mem.mm_node != NULL)) #endif { - ret = drm_bo_wait(bo, 0, 1, 0); - if (ret) - return ret; - + if (bo->fence) { + (void) drm_fence_object_wait(bo->fence, 0, 1, + bo->fence_type); + drm_fence_usage_deref_unlocked(&bo->fence); + } drm_bo_free_old_node(bo); if ((man->flags & _DRM_FLAG_MEMTYPE_FIXED) && (bo->ttm != NULL)) { diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h index a1f3a18d..770fbc56 100644 --- a/linux-core/drm_objects.h +++ b/linux-core/drm_objects.h @@ -517,6 +517,14 @@ struct drm_buffer_object { #define _DRM_BO_FLAG_UNFENCED 0x00000001 #define _DRM_BO_FLAG_EVICTED 0x00000002 +/* + * This flag indicates that a flag called with bo->mutex held has + * temporarily released the buffer object mutex, (usually to wait for something). + * and thus any post-lock validation needs to be rerun. + */ + +#define _DRM_BO_FLAG_UNLOCKED 0x00000004 + struct drm_mem_type_manager { int has_type; int use_type; @@ -682,8 +690,8 @@ extern int drm_buffer_object_create(struct drm_device *dev, unsigned long size, uint32_t hint, uint32_t page_alignment, unsigned long buffer_start, struct drm_buffer_object **bo); -extern int drm_bo_wait(struct drm_buffer_object *bo, int lazy, int ignore_signals, - int no_wait); +extern int drm_bo_wait(struct drm_buffer_object *bo, int lazy, int interruptible, + int no_wait, int check_unfenced); extern int drm_bo_mem_space(struct drm_buffer_object *bo, struct drm_bo_mem_reg *mem, int no_wait); extern int drm_bo_move_buffer(struct drm_buffer_object *bo, @@ -695,7 +703,7 @@ extern int drm_bo_init_mm(struct drm_device *dev, unsigned type, int kern_init); extern int drm_bo_handle_validate(struct drm_file *file_priv, uint32_t handle, uint64_t flags, uint64_t mask, uint32_t hint, - uint32_t fence_class, int use_old_fence_class, + uint32_t fence_class, struct drm_bo_info_rep *rep, struct drm_buffer_object **bo_rep); extern struct drm_buffer_object *drm_lookup_buffer_object(struct drm_file *file_priv, diff --git a/linux-core/drm_vm.c b/linux-core/drm_vm.c index a09bcddb..cabfb8f4 100644 --- a/linux-core/drm_vm.c +++ b/linux-core/drm_vm.c @@ -748,12 +748,14 @@ static unsigned long drm_bo_vm_nopfn(struct vm_area_struct *vma, return NOPFN_REFAULT; } - err = drm_bo_wait(bo, 0, 0, 0); + err = drm_bo_wait(bo, 0, 1, 0, 1); if (err) { ret = (err != -EAGAIN) ? NOPFN_SIGBUS : NOPFN_REFAULT; goto out_unlock; } + bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED; + /* * If buffer happens to be in a non-mappable location, * move it to a mappable. @@ -806,6 +808,7 @@ static unsigned long drm_bo_vm_nopfn(struct vm_area_struct *vma, goto out_unlock; } out_unlock: + BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED); mutex_unlock(&bo->mutex); drm_bo_read_unlock(&dev->bm.bm_lock); return ret; diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c index 088a2693..804f3ac1 100644 --- a/linux-core/i915_execbuf.c +++ b/linux-core/i915_execbuf.c @@ -144,7 +144,7 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, relocatee->offset = new_cmd_offset; if (unlikely(relocatee->idle == I915_RELOC_UNCHECKED)) { - ret = drm_bo_wait(relocatee->buf, 0, 0, 0); + ret = drm_bo_wait(relocatee->buf, 0, 1, 0, 0); if (ret) return ret; relocatee->idle = I915_RELOC_IDLE; @@ -355,11 +355,9 @@ static int i915_update_relocatee(struct i915_relocatee_info *relocatee, if (relocatee->idle == I915_RELOC_UNCHECKED) { preempt_enable(); - ret = mutex_lock_interruptible(&relocatee->buf->mutex); - if (unlikely(ret)) - return -EAGAIN; + mutex_lock(&relocatee->buf->mutex); - ret = drm_bo_wait(relocatee->buf, 0, 0, 1); + ret = drm_bo_wait(relocatee->buf, 0, 1, 1, 0); if (ret == 0) relocatee->idle = I915_RELOC_IDLE; else { @@ -684,7 +682,7 @@ int i915_validate_buffer_list(struct drm_file *file_priv, ret = drm_bo_handle_validate(file_priv, req->bo_req.handle, req->bo_req.flags, req->bo_req.mask, req->bo_req.hint, - req->bo_req.fence_class, 0, + req->bo_req.fence_class, NULL, &item->buffer); if (ret) { DRM_ERROR("error on handle validate %d\n", ret); -- cgit v1.2.3 From c5955c652302d66719984cb5a218cb590c74ad42 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Mon, 14 Apr 2008 12:10:50 +0200 Subject: Fix buffer object creation validation. BO lock fixes. --- linux-core/drm_bo.c | 7 +++++-- linux-core/drm_bo_lock.c | 10 ++++------ 2 files changed, 9 insertions(+), 8 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c index 0853d748..6f287532 100644 --- a/linux-core/drm_bo.c +++ b/linux-core/drm_bo.c @@ -1244,6 +1244,7 @@ static int drm_buffer_object_map(struct drm_file *file_priv, uint32_t handle, mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(&bo); + return ret; } @@ -1541,8 +1542,9 @@ static int drm_bo_prepare_for_validate(struct drm_buffer_object *bo, ftype, no_wait); } - if (ret && ret != -EAGAIN) + if (ret && ret != -EAGAIN) ret = drm_bo_wait(bo, 0, 1, no_wait, 1); + if (ret) return ret; } @@ -1622,6 +1624,7 @@ out: drm_bo_fill_rep_arg(bo, rep); mutex_unlock(&bo->mutex); + return ret; } EXPORT_SYMBOL(drm_bo_do_validate); @@ -1816,7 +1819,7 @@ int drm_buffer_object_create(struct drm_device *dev, } mutex_unlock(&bo->mutex); - ret = drm_bo_do_validate(bo, 0, 0, hint & DRM_BO_HINT_DONT_BLOCK, + ret = drm_bo_do_validate(bo, 0, 0, hint | DRM_BO_HINT_DONT_FENCE, 0, NULL); if (ret) goto out_err_unlocked; diff --git a/linux-core/drm_bo_lock.c b/linux-core/drm_bo_lock.c index 32ebfbe2..08b1c6be 100644 --- a/linux-core/drm_bo_lock.c +++ b/linux-core/drm_bo_lock.c @@ -68,9 +68,7 @@ void drm_bo_init_lock(struct drm_bo_lock *lock) void drm_bo_read_unlock(struct drm_bo_lock *lock) { - if (unlikely(atomic_add_negative(-1, &lock->readers))) - BUG(); - if (atomic_read(&lock->readers) == 0) + if (atomic_dec_and_test(&lock->readers)) wake_up_all(&lock->queue); } EXPORT_SYMBOL(drm_bo_read_unlock); @@ -79,7 +77,7 @@ int drm_bo_read_lock(struct drm_bo_lock *lock, int interruptible) { while (unlikely(atomic_read(&lock->write_lock_pending) != 0)) { int ret; - + if (!interruptible) { wait_event(lock->queue, atomic_read(&lock->write_lock_pending) == 0); @@ -93,7 +91,6 @@ int drm_bo_read_lock(struct drm_bo_lock *lock, int interruptible) while (unlikely(!atomic_add_unless(&lock->readers, 1, -1))) { int ret; - if (!interruptible) { wait_event(lock->queue, atomic_read(&lock->readers) != -1); @@ -156,7 +153,8 @@ int drm_bo_write_lock(struct drm_bo_lock *lock, int interruptible, * while holding it. */ - atomic_dec(&lock->write_lock_pending); + if (atomic_dec_and_test(&lock->write_lock_pending)) + wake_up_all(&lock->queue); dev = file_priv->minor->dev; mutex_lock(&dev->struct_mutex); ret = drm_add_user_object(file_priv, &lock->base, 0); -- cgit v1.2.3 From 1ad1bd5bd95db71500edfcea8b46421d7f3cdb15 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Mon, 14 Apr 2008 13:52:33 +0200 Subject: Fix buffer object map wait error. Add some branch prediction hints. --- linux-core/drm_bo.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c index 6f287532..88b2ee66 100644 --- a/linux-core/drm_bo.c +++ b/linux-core/drm_bo.c @@ -1225,11 +1225,13 @@ static int drm_buffer_object_map(struct drm_file *file_priv, uint32_t handle, bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED; ret = drm_bo_wait(bo, 0, 1, no_wait, 1); + if (unlikely(ret)) + goto out; if (bo->mem.flags & DRM_BO_FLAG_CACHED_MAPPED) drm_bo_evict_cached(bo); - } while (bo->priv_flags & _DRM_BO_FLAG_UNLOCKED); + } while (unlikely(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED)); atomic_inc(&bo->mapped); mutex_lock(&dev->struct_mutex); @@ -1242,6 +1244,7 @@ static int drm_buffer_object_map(struct drm_file *file_priv, uint32_t handle, } else drm_bo_fill_rep_arg(bo, rep); + out: mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(&bo); @@ -1610,7 +1613,7 @@ int drm_bo_do_validate(struct drm_buffer_object *bo, if (ret) goto out; - } while(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED); + } while(unlikely(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED)); ret = drm_buffer_object_validate(bo, fence_class, -- cgit v1.2.3 From b3967765c082c4fae1954ec70474fb428ef42c70 Mon Sep 17 00:00:00 2001 From: Pekka Paalanen Date: Sun, 20 Apr 2008 20:47:38 +0300 Subject: linux-core Makefile: add GIT_REVISION This tries to automatically fetch a git revision string and if succeeds, it #defines GIT_REVISION string macro. Packagers can override it by 'make GIT_REVISION=foo'. Update Nouveau to use GIT_REVISION, if defined, instead of DRIVER_DATE in struct drm_driver. Signed-off-by: Pekka Paalanen --- linux-core/Makefile | 5 +++++ linux-core/nouveau_drv.c | 4 ++++ 2 files changed, 9 insertions(+) (limited to 'linux-core') diff --git a/linux-core/Makefile b/linux-core/Makefile index 7f6b123e..3af6f370 100644 --- a/linux-core/Makefile +++ b/linux-core/Makefile @@ -335,6 +335,11 @@ ifneq (,$(findstring i915,$(DRM_MODULES))) CONFIG_DRM_I915 := m endif +GIT_REVISION := $(shell cd "$(DRMSRCDIR)" && git-describe --abbrev=17) +ifneq ($(GIT_REVISION),) +EXTRA_CFLAGS+=-D"GIT_REVISION=\"$(GIT_REVISION)\"" +endif + include $(DRMSRCDIR)/Makefile.kernel # Depencencies diff --git a/linux-core/nouveau_drv.c b/linux-core/nouveau_drv.c index e9623eb1..c8f57dff 100644 --- a/linux-core/nouveau_drv.c +++ b/linux-core/nouveau_drv.c @@ -86,7 +86,11 @@ static struct drm_driver driver = { .name = DRIVER_NAME, .desc = DRIVER_DESC, +#ifdef GIT_REVISION + .date = GIT_REVISION, +#else .date = DRIVER_DATE, +#endif .major = DRIVER_MAJOR, .minor = DRIVER_MINOR, .patchlevel = DRIVER_PATCHLEVEL, -- cgit v1.2.3 From 9ba3aaaa1a22663ec3d8d9d1792edf10a25d0ad7 Mon Sep 17 00:00:00 2001 From: Jakob Bornecrantz Date: Wed, 23 Apr 2008 12:43:30 +0200 Subject: Fixed unlock check on EAGAIN --- linux-core/drm_vm.c | 1 + 1 file changed, 1 insertion(+) (limited to 'linux-core') diff --git a/linux-core/drm_vm.c b/linux-core/drm_vm.c index cabfb8f4..b85b4c13 100644 --- a/linux-core/drm_vm.c +++ b/linux-core/drm_vm.c @@ -751,6 +751,7 @@ static unsigned long drm_bo_vm_nopfn(struct vm_area_struct *vma, err = drm_bo_wait(bo, 0, 1, 0, 1); if (err) { ret = (err != -EAGAIN) ? NOPFN_SIGBUS : NOPFN_REFAULT; + bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED; goto out_unlock; } -- cgit v1.2.3 From 10b9a116a7b7fe3acf0848de9e0cf40f8e1bcd75 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Wed, 23 Apr 2008 17:33:09 +0200 Subject: Don't disable IRQs, just tasklets, when taking the drm lock spinlock. --- linux-core/drm_fops.c | 7 ++----- linux-core/drm_lock.c | 35 +++++++++++++++-------------------- 2 files changed, 17 insertions(+), 25 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drm_fops.c b/linux-core/drm_fops.c index a4c76f75..3bc25f24 100644 --- a/linux-core/drm_fops.c +++ b/linux-core/drm_fops.c @@ -373,7 +373,6 @@ int drm_release(struct inode *inode, struct file *filp) struct drm_file *file_priv = filp->private_data; struct drm_device *dev = file_priv->minor->dev; int retcode = 0; - unsigned long irqflags; lock_kernel(); @@ -404,11 +403,9 @@ int drm_release(struct inode *inode, struct file *filp) */ do{ - spin_lock_irqsave(&dev->lock.spinlock, - irqflags); + spin_lock_bh(&dev->lock.spinlock); locked = dev->lock.idle_has_lock; - spin_unlock_irqrestore(&dev->lock.spinlock, - irqflags); + spin_unlock_bh(&dev->lock.spinlock); if (locked) break; schedule(); diff --git a/linux-core/drm_lock.c b/linux-core/drm_lock.c index db1d646b..d4c2da0c 100644 --- a/linux-core/drm_lock.c +++ b/linux-core/drm_lock.c @@ -53,7 +53,6 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv) DECLARE_WAITQUEUE(entry, current); struct drm_lock *lock = data; int ret = 0; - unsigned long irqflags; ++file_priv->lock_count; @@ -72,9 +71,9 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv) return -EINVAL; add_wait_queue(&dev->lock.lock_queue, &entry); - spin_lock_irqsave(&dev->lock.spinlock, irqflags); + spin_lock_bh(&dev->lock.spinlock); dev->lock.user_waiters++; - spin_unlock_irqrestore(&dev->lock.spinlock, irqflags); + spin_unlock_bh(&dev->lock.spinlock); for (;;) { __set_current_state(TASK_INTERRUPTIBLE); if (!dev->lock.hw_lock) { @@ -96,9 +95,9 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv) break; } } - spin_lock_irqsave(&dev->lock.spinlock, irqflags); + spin_lock_bh(&dev->lock.spinlock); dev->lock.user_waiters--; - spin_unlock_irqrestore(&dev->lock.spinlock, irqflags); + spin_unlock_bh(&dev->lock.spinlock); __set_current_state(TASK_RUNNING); remove_wait_queue(&dev->lock.lock_queue, &entry); @@ -199,9 +198,8 @@ int drm_lock_take(struct drm_lock_data *lock_data, { unsigned int old, new, prev; volatile unsigned int *lock = &lock_data->hw_lock->lock; - unsigned long irqflags; - spin_lock_irqsave(&lock_data->spinlock, irqflags); + spin_lock_bh(&lock_data->spinlock); do { old = *lock; if (old & _DRM_LOCK_HELD) @@ -213,7 +211,7 @@ int drm_lock_take(struct drm_lock_data *lock_data, } prev = cmpxchg(lock, old, new); } while (prev != old); - spin_unlock_irqrestore(&lock_data->spinlock, irqflags); + spin_unlock_bh(&lock_data->spinlock); if (_DRM_LOCKING_CONTEXT(old) == context) { if (old & _DRM_LOCK_HELD) { @@ -275,16 +273,15 @@ int drm_lock_free(struct drm_lock_data *lock_data, unsigned int context) { unsigned int old, new, prev; volatile unsigned int *lock = &lock_data->hw_lock->lock; - unsigned long irqflags; - spin_lock_irqsave(&lock_data->spinlock, irqflags); + spin_lock_bh(&lock_data->spinlock); if (lock_data->kernel_waiters != 0) { drm_lock_transfer(lock_data, 0); lock_data->idle_has_lock = 1; - spin_unlock_irqrestore(&lock_data->spinlock, irqflags); + spin_unlock_bh(&lock_data->spinlock); return 1; } - spin_unlock_irqrestore(&lock_data->spinlock, irqflags); + spin_unlock_bh(&lock_data->spinlock); do { old = *lock; @@ -348,20 +345,19 @@ static int drm_notifier(void *priv) void drm_idlelock_take(struct drm_lock_data *lock_data) { int ret = 0; - unsigned long irqflags; - spin_lock_irqsave(&lock_data->spinlock, irqflags); + spin_lock_bh(&lock_data->spinlock); lock_data->kernel_waiters++; if (!lock_data->idle_has_lock) { - spin_unlock_irqrestore(&lock_data->spinlock, irqflags); + spin_unlock_bh(&lock_data->spinlock); ret = drm_lock_take(lock_data, DRM_KERNEL_CONTEXT); - spin_lock_irqsave(&lock_data->spinlock, irqflags); + spin_lock_bh(&lock_data->spinlock); if (ret == 1) lock_data->idle_has_lock = 1; } - spin_unlock_irqrestore(&lock_data->spinlock, irqflags); + spin_unlock_bh(&lock_data->spinlock); } EXPORT_SYMBOL(drm_idlelock_take); @@ -369,9 +365,8 @@ void drm_idlelock_release(struct drm_lock_data *lock_data) { unsigned int old, prev; volatile unsigned int *lock = &lock_data->hw_lock->lock; - unsigned long irqflags; - spin_lock_irqsave(&lock_data->spinlock, irqflags); + spin_lock_bh(&lock_data->spinlock); if (--lock_data->kernel_waiters == 0) { if (lock_data->idle_has_lock) { do { @@ -382,7 +377,7 @@ void drm_idlelock_release(struct drm_lock_data *lock_data) lock_data->idle_has_lock = 0; } } - spin_unlock_irqrestore(&lock_data->spinlock, irqflags); + spin_unlock_bh(&lock_data->spinlock); } EXPORT_SYMBOL(drm_idlelock_release); -- cgit v1.2.3 From 7f8e4060859651993921281445ec00940c577222 Mon Sep 17 00:00:00 2001 From: Jesse Barnes Date: Sun, 27 Apr 2008 09:42:17 -0700 Subject: Use fixed sized types in new ioctls Make both crtc and the command argument 32 bits to avoid any 32-on-64 compat issues. --- linux-core/drm_irq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'linux-core') diff --git a/linux-core/drm_irq.c b/linux-core/drm_irq.c index 592ea2ad..8f27d7f3 100644 --- a/linux-core/drm_irq.c +++ b/linux-core/drm_irq.c @@ -437,7 +437,7 @@ int drm_modeset_ctl(struct drm_device *dev, void *data, int crtc, ret = 0; u32 new; - crtc = modeset->arg; + crtc = modeset->crtc; if (crtc >= dev->num_crtcs) { ret = -EINVAL; goto out; -- cgit v1.2.3