From 40c9e6a26dd251fe2bf207bb259ba7e4a7704fbe Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Thu, 28 Feb 2008 13:47:15 +0100 Subject: Add a compat kmap_atomic_prot_pfn to do quick kernel map / unmaps of PCI- or high memory. This is substantially more efficient than drm_bo_kmap, since the mapping only lives on a single processor. Unmapping is done use kunmap_atomic(). Flushes only a single tlb() entry. Add a support utility int drm_bo_pfn_prot() that returns the pfn and desired page protection for a given bo offset. This is all intended for relocations in bound TTMS or vram. Mapping-accessing-unmapping must be atomic, either using preempt_xx() macros or a spinlock. --- linux-core/drm_bo_move.c | 33 +++++++++++++++++++++++++++++++++ linux-core/drm_compat.c | 32 ++++++++++++++++++++++++++++++++ linux-core/drm_compat.h | 5 +++++ linux-core/drm_objects.h | 4 ++++ 4 files changed, 74 insertions(+) (limited to 'linux-core') diff --git a/linux-core/drm_bo_move.c b/linux-core/drm_bo_move.c index b06a09f0..30e0f43f 100644 --- a/linux-core/drm_bo_move.c +++ b/linux-core/drm_bo_move.c @@ -595,3 +595,36 @@ void drm_bo_kunmap(struct drm_bo_kmap_obj *map) map->page = NULL; } EXPORT_SYMBOL(drm_bo_kunmap); + +int drm_bo_pfn_prot(struct drm_buffer_object *bo, + unsigned long dst_offset, + unsigned long *pfn, + pgprot_t *prot) +{ + struct drm_bo_mem_reg *mem = &bo->mem; + struct drm_device *dev = bo->dev; + unsigned long bus_offset; + unsigned long bus_size; + unsigned long bus_base; + struct drm_mem_type_manager *man = &dev->bm.man[mem->mem_type]; + int ret; + + ret = drm_bo_pci_offset(dev, mem, &bus_base, &bus_offset, + &bus_size); + if (ret) + return -EINVAL; + + if (bus_size != 0) + *pfn = (bus_base + bus_offset + dst_offset) >> PAGE_SHIFT; + else if (!bo->ttm) + return -EINVAL; + else + *pfn = page_to_pfn(drm_ttm_get_page(bo->ttm, dst_offset >> PAGE_SHIFT)); + + *prot = (mem->flags & DRM_BO_FLAG_CACHED) ? + PAGE_KERNEL : drm_kernel_io_prot(man->drm_bus_maptype); + + return 0; +} +EXPORT_SYMBOL(drm_bo_pfn_prot); + diff --git a/linux-core/drm_compat.c b/linux-core/drm_compat.c index a745a7d9..32e43a0a 100644 --- a/linux-core/drm_compat.c +++ b/linux-core/drm_compat.c @@ -729,3 +729,35 @@ void *idr_replace(struct idr *idp, void *ptr, int id) } EXPORT_SYMBOL(idr_replace); #endif + +#if defined(CONFIG_X86) + +#define drm_kmap_get_fixmap_pte(vaddr) \ + pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), vaddr), (vaddr)), (vaddr)) + +void *kmap_atomic_prot_pfn(unsigned long pfn, enum km_type type, + pgprot_t protection) +{ + enum fixed_addresses idx; + unsigned long vaddr; + static pte_t *km_pte; + static int initialized = 0; + + if (unlikely(!initialized)) { + km_pte = drm_kmap_get_fixmap_pte(__fix_to_virt(FIX_KMAP_BEGIN)); + initialized = 1; + } + + pagefault_disable(); + idx = type + KM_TYPE_NR*smp_processor_id(); + vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); + set_pte(km_pte-idx, pfn_pte(pfn, protection)); + + return (void*) vaddr; +} + +EXPORT_SYMBOL(kmap_atomic_prot_pfn); + +#endif + + diff --git a/linux-core/drm_compat.h b/linux-core/drm_compat.h index f8933e0c..39027cf4 100644 --- a/linux-core/drm_compat.h +++ b/linux-core/drm_compat.h @@ -328,4 +328,9 @@ void *idr_replace(struct idr *idp, void *ptr, int id); typedef _Bool bool; #endif +#if defined(CONFIG_X86) +#define DRM_KMAP_ATOMIC_PROT_PFN +extern void *kmap_atomic_prot_pfn(unsigned long pfn, enum km_type type, + pgprot_t protection); +#endif #endif diff --git a/linux-core/drm_objects.h b/linux-core/drm_objects.h index e43e8dfd..8055afe1 100644 --- a/linux-core/drm_objects.h +++ b/linux-core/drm_objects.h @@ -738,6 +738,10 @@ static inline void *drm_bmo_virtual(struct drm_bo_kmap_obj *map, int *is_iomem) extern void drm_bo_kunmap(struct drm_bo_kmap_obj *map); extern int drm_bo_kmap(struct drm_buffer_object *bo, unsigned long start_page, unsigned long num_pages, struct drm_bo_kmap_obj *map); +extern int drm_bo_pfn_prot(struct drm_buffer_object *bo, + unsigned long dst_offset, + unsigned long *pfn, + pgprot_t *prot); /* -- cgit v1.2.3 From 28d4d02d6791c15f61b718039f1d4b907f0e31e9 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Thu, 28 Feb 2008 14:05:53 +0100 Subject: Initial commit. --- linux-core/i915_execbuf.c | 653 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 653 insertions(+) create mode 100644 linux-core/i915_execbuf.c (limited to 'linux-core') diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c new file mode 100644 index 00000000..fecb5ab0 --- /dev/null +++ b/linux-core/i915_execbuf.c @@ -0,0 +1,653 @@ +/* + * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "drmP.h" +#include "drm.h" +#include "i915_drm.h" +#include "i915_drv.h" + +struct i915_relocatee_info { + struct drm_buffer_object *buf; + unsigned long offset; + uint32_t *data_page; + unsigned page_offset; + struct drm_bo_kmap_obj kmap; + int is_iomem; + int idle; + int dst; +#ifdef DRM_KMAP_ATOMIC_PROT_PFN + unsigned long pfn; + pgprot_t pg_prot; +#endif +}; + +struct drm_i915_validate_buffer { + struct drm_buffer_object *buffer; + struct drm_bo_info_rep rep; + int presumed_offset_correct; + void __user *data; + int ret; +}; + + +static int i915_update_relocatee(struct i915_relocatee_info *relocatee, + struct drm_i915_validate_buffer *buffers, + unsigned int dst, + unsigned long dst_offset) +{ + int ret; + + if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) { + i915_clear_relocatee(relocatee); + relocatee->dst = dst; + relocatee->buf = buffers[dst].buffer; + preempt_enable(); + ret = mutex_lock_interruptible(&relocatee->buf->mutex); + preempt_disable(); + if (unlikely(ret)) + return -EAGAIN; + + ret = drm_bo_wait(relocatee->buf, 0, 0, 0); + if (unlikely(ret)) + return ret; + + mutex_unlock(&relocatee->buf->mutex); + } + + if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) { + DRM_ERROR("Relocation destination out of bounds.\n"); + return -EINVAL; + } + if (unlikely(!drm_bo_same_page(relocatee->page_offset, dst_offset) || + NULL == relocatee->data_page)) { +#ifdef DRM_KMAP_ATOMIC_PROT_PFN + if (NULL != relocatee->data_page) { + kunmap_atomic(relocatee->data_page, KM_USER0); + relocatee->data_page = NULL; + } + ret = drm_bo_pfn_prot(relocatee->buf, dst_offset, + &relocatee->pfn, + &relocatee->pg_prot); + if (ret) { + DRM_ERROR("Can't map relocation destination.\n"); + return -EINVAL; + } + relocatee->data_page = + kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, + relocatee->pg_prot); +#else + if (NULL != relocatee->data_page) { + drm_bo_kunmap(&relocatee->kmap); + relocatee->data_page = NULL; + } + + ret = drm_bo_kmap(relocatee->buf, dst_offset >> PAGE_SHIFT, + 1, &relocatee->kmap); + if (ret) { + DRM_ERROR("Can't map relocation destination.\n"); + return ret; + } + + relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, + &relocatee->is_iomem); +#endif + relocatee->page_offset = dst_offset & PAGE_MASK; + } + return 0; +} + + +static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buffers, + unsigned num_buffers) +{ + while (num_buffers--) + drm_bo_usage_deref_locked(&buffers[num_buffers].buffer); +} + +int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, + struct drm_i915_validate_buffer *buffers, + struct i915_relocatee_info *relocatee, + uint32_t *reloc) +{ + unsigned index; + unsigned long new_cmd_offset; + u32 val; + int ret, i; + int buf_index = -1; + + /* + * FIXME: O(relocs * buffers) complexity. + */ + + for (i = 0; i <= num_buffers; i++) + if (buffers[i].buffer) + if (reloc[2] == buffers[i].buffer->base.hash.key) + buf_index = i; + + if (buf_index == -1) { + DRM_ERROR("Illegal relocation buffer %08X\n", reloc[2]); + return -EINVAL; + } + + /* + * Short-circuit relocations that were correctly + * guessed by the client + */ + if (buffers[buf_index].presumed_offset_correct && !DRM_DEBUG_RELOCATION) + return 0; + + new_cmd_offset = reloc[0]; + if (!relocatee->data_page || + !drm_bo_same_page(relocatee->offset, new_cmd_offset)) { + drm_bo_kunmap(&relocatee->kmap); + relocatee->data_page = NULL; + relocatee->offset = new_cmd_offset; + + /* + * Note on buffer idle: + * Since we're applying relocations, this part of the + * buffer is obviously not used by the GPU and we don't + * need to wait for buffer idle. This is an important + * consideration for user-space buffer pools. + */ + + ret = drm_bo_kmap(relocatee->buf, new_cmd_offset >> PAGE_SHIFT, + 1, &relocatee->kmap); + if (ret) { + DRM_ERROR("Could not map command buffer to apply relocs\n %08lx", new_cmd_offset); + return ret; + } + relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, + &relocatee->is_iomem); + relocatee->page_offset = (relocatee->offset & PAGE_MASK); + } + + val = buffers[buf_index].buffer->offset; + index = (reloc[0] - relocatee->page_offset) >> 2; + + /* add in validate */ + val = val + reloc[1]; + + if (DRM_DEBUG_RELOCATION) { + if (buffers[buf_index].presumed_offset_correct && + relocatee->data_page[index] != val) { + DRM_DEBUG ("Relocation mismatch source %d target %d buffer %d user %08x kernel %08x\n", + reloc[0], reloc[1], buf_index, relocatee->data_page[index], val); + } + } + + if (relocatee->is_iomem) + iowrite32(val, relocatee->data_page + index); + else + relocatee->data_page[index] = val; + return 0; +} + +int i915_process_relocs(struct drm_file *file_priv, + uint32_t buf_handle, + uint32_t __user **reloc_user_ptr, + struct i915_relocatee_info *relocatee, + struct drm_i915_validate_buffer *buffers, + uint32_t num_buffers) +{ + int ret, reloc_stride; + uint32_t cur_offset; + uint32_t reloc_count; + uint32_t reloc_type; + uint32_t reloc_buf_size; + uint32_t *reloc_buf = NULL; + int i; + + /* do a copy from user from the user ptr */ + ret = get_user(reloc_count, *reloc_user_ptr); + if (ret) { + DRM_ERROR("Could not map relocation buffer.\n"); + goto out; + } + + ret = get_user(reloc_type, (*reloc_user_ptr)+1); + if (ret) { + DRM_ERROR("Could not map relocation buffer.\n"); + goto out; + } + + if (reloc_type != 0) { + DRM_ERROR("Unsupported relocation type requested\n"); + ret = -EINVAL; + goto out; + } + + reloc_buf_size = (I915_RELOC_HEADER + (reloc_count * I915_RELOC0_STRIDE)) * sizeof(uint32_t); + reloc_buf = kmalloc(reloc_buf_size, GFP_KERNEL); + if (!reloc_buf) { + DRM_ERROR("Out of memory for reloc buffer\n"); + ret = -ENOMEM; + goto out; + } + + if (copy_from_user(reloc_buf, *reloc_user_ptr, reloc_buf_size)) { + ret = -EFAULT; + goto out; + } + + /* get next relocate buffer handle */ + *reloc_user_ptr = (uint32_t *)*(unsigned long *)&reloc_buf[2]; + + reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); /* may be different for other types of relocs */ + + DRM_DEBUG("num relocs is %d, next is %p\n", reloc_count, *reloc_user_ptr); + + for (i = 0; i < reloc_count; i++) { + cur_offset = I915_RELOC_HEADER + (i * I915_RELOC0_STRIDE); + + ret = i915_apply_reloc(file_priv, num_buffers, buffers, + relocatee, reloc_buf + cur_offset); + if (ret) + goto out; + } + +out: + if (reloc_buf) + kfree(reloc_buf); + + if (relocatee->data_page) { + drm_bo_kunmap(&relocatee->kmap); + relocatee->data_page = NULL; + } + + return ret; +} + +static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle, + uint32_t __user *reloc_user_ptr, + struct drm_i915_validate_buffer *buffers, + uint32_t buf_count) +{ + struct drm_device *dev = file_priv->head->dev; + struct i915_relocatee_info relocatee; + int ret = 0; + int b; + + /* + * Short circuit relocations when all previous + * buffers offsets were correctly guessed by + * the client + */ + if (!DRM_DEBUG_RELOCATION) { + for (b = 0; b < buf_count; b++) + if (!buffers[b].presumed_offset_correct) + break; + + if (b == buf_count) + return 0; + } + + memset(&relocatee, 0, sizeof(relocatee)); + + mutex_lock(&dev->struct_mutex); + relocatee.buf = drm_lookup_buffer_object(file_priv, buf_handle, 1); + mutex_unlock(&dev->struct_mutex); + if (!relocatee.buf) { + DRM_DEBUG("relocatee buffer invalid %08x\n", buf_handle); + ret = -EINVAL; + goto out_err; + } + + mutex_lock (&relocatee.buf->mutex); + while (reloc_user_ptr) { + ret = i915_process_relocs(file_priv, buf_handle, &reloc_user_ptr, &relocatee, buffers, buf_count); + if (ret) { + DRM_ERROR("process relocs failed\n"); + goto out_err1; + } + } + +out_err1: + mutex_unlock (&relocatee.buf->mutex); + drm_bo_usage_deref_unlocked(&relocatee.buf); +out_err: + return ret; +} + +static int i915_check_presumed(struct drm_i915_op_arg *arg, + struct drm_buffer_object *bo, + uint32_t __user *data, + int *presumed_ok) +{ + struct drm_bo_op_req *req = &arg->d.req; + uint32_t hint_offset; + uint32_t hint = req->bo_req.hint; + + *presumed_ok = 0; + + if (!(hint & DRM_BO_HINT_PRESUMED_OFFSET)) + return 0; + if (bo->offset == req->bo_req.presumed_offset) { + *presumed_ok = 1; + return 0; + } + + /* + * We need to turn off the HINT_PRESUMED_OFFSET for this buffer in + * the user-space IOCTL argument list, since the buffer has moved, + * we're about to apply relocations and we might subsequently + * hit an -EAGAIN. In that case the argument list will be reused by + * user-space, but the presumed offset is no longer valid. + * + * Needless to say, this is a bit ugly. + */ + + hint_offset = (uint32_t *)&req->bo_req.hint - (uint32_t *)arg; + hint &= ~DRM_BO_HINT_PRESUMED_OFFSET; + return __put_user(hint, data + hint_offset); +} + + +/* + * Validate, add fence and relocate a block of bos from a userspace list + */ +int i915_validate_buffer_list(struct drm_file *file_priv, + unsigned int fence_class, uint64_t data, + struct drm_i915_validate_buffer *buffers, + uint32_t *num_buffers) +{ + struct drm_i915_op_arg arg; + struct drm_bo_op_req *req = &arg.d.req; + int ret = 0; + unsigned buf_count = 0; + uint32_t buf_handle; + uint32_t __user *reloc_user_ptr; + struct drm_i915_validate_buffer *item = buffers; + + do { + if (buf_count >= *num_buffers) { + DRM_ERROR("Buffer count exceeded %d\n.", *num_buffers); + ret = -EINVAL; + goto out_err; + } + item = buffers + buf_count; + item->buffer = NULL; + item->presumed_offset_correct = 0; + + buffers[buf_count].buffer = NULL; + + if (copy_from_user(&arg, (void __user *)(unsigned long)data, sizeof(arg))) { + ret = -EFAULT; + goto out_err; + } + + ret = 0; + if (req->op != drm_bo_validate) { + DRM_ERROR + ("Buffer object operation wasn't \"validate\".\n"); + ret = -EINVAL; + goto out_err; + } + item->ret = 0; + item->data = (void __user *) (unsigned long) data; + + buf_handle = req->bo_req.handle; + reloc_user_ptr = (uint32_t *)(unsigned long)arg.reloc_ptr; + + if (reloc_user_ptr) { + ret = i915_exec_reloc(file_priv, buf_handle, reloc_user_ptr, buffers, buf_count); + if (ret) + goto out_err; + DRM_MEMORYBARRIER(); + } + + ret = drm_bo_handle_validate(file_priv, req->bo_req.handle, + req->bo_req.flags, req->bo_req.mask, + req->bo_req.hint, + req->bo_req.fence_class, 0, + &item->rep, + &item->buffer); + + if (ret) { + DRM_ERROR("error on handle validate %d\n", ret); + goto out_err; + } + + buf_count++; + + ret = i915_check_presumed(&arg, item->buffer, + (uint32_t __user *) + (unsigned long) data, + &item->presumed_offset_correct); + if (ret) + goto out_err; + + data = arg.next; + } while (data != 0); +out_err: + *num_buffers = buf_count; + item->ret = (ret != -EAGAIN) ? ret : 0; + return ret; +} + + +/* + * Remove all buffers from the unfenced list. + * If the execbuffer operation was aborted, for example due to a signal, + * this also make sure that buffers retain their original state and + * fence pointers. + * Copy back buffer information to user-space unless we were interrupted + * by a signal. In which case the IOCTL must be rerun. + */ + +static int i915_handle_copyback(struct drm_device *dev, + struct drm_i915_validate_buffer *buffers, + unsigned int num_buffers, int ret) +{ + int err = ret; + int i; + struct drm_i915_op_arg arg; + + if (ret) + drm_putback_buffer_objects(dev); + + if (ret != -EAGAIN) { + for (i = 0; i < num_buffers; ++i) { + arg.handled = 1; + arg.d.rep.ret = buffers->ret; + arg.d.rep.bo_info = buffers->rep; + if (__copy_to_user(buffers->data, &arg, sizeof(arg))) + err = -EFAULT; + buffers++; + } + } + + return err; +} + +/* + * Create a fence object, and if that fails, pretend that everything is + * OK and just idle the GPU. + */ + +void i915_fence_or_sync(struct drm_file *file_priv, + uint32_t fence_flags, + struct drm_fence_arg *fence_arg, + struct drm_fence_object **fence_p) +{ + struct drm_device *dev = file_priv->head->dev; + int ret; + struct drm_fence_object *fence; + + ret = drm_fence_buffer_objects(dev, NULL, fence_flags, + NULL, &fence); + + if (ret) { + + /* + * Fence creation failed. + * Fall back to synchronous operation and idle the engine. + */ + + (void) i915_emit_mi_flush(dev, MI_READ_FLUSH); + (void) i915_quiescent(dev); + + if (!(fence_flags & DRM_FENCE_FLAG_NO_USER)) { + + /* + * Communicate to user-space that + * fence creation has failed and that + * the engine is idle. + */ + + fence_arg->handle = ~0; + fence_arg->error = ret; + } + + drm_putback_buffer_objects(dev); + if (fence_p) + *fence_p = NULL; + return; + } + + if (!(fence_flags & DRM_FENCE_FLAG_NO_USER)) { + + ret = drm_fence_add_user_object(file_priv, fence, + fence_flags & + DRM_FENCE_FLAG_SHAREABLE); + if (!ret) + drm_fence_fill_arg(fence, fence_arg); + else { + /* + * Fence user object creation failed. + * We must idle the engine here as well, as user- + * space expects a fence object to wait on. Since we + * have a fence object we wait for it to signal + * to indicate engine "sufficiently" idle. + */ + + (void) drm_fence_object_wait(fence, 0, 1, + fence->type); + drm_fence_usage_deref_unlocked(&fence); + fence_arg->handle = ~0; + fence_arg->error = ret; + } + } + + if (fence_p) + *fence_p = fence; + else if (fence) + drm_fence_usage_deref_unlocked(&fence); +} + + +static int i915_execbuffer(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) + dev_priv->sarea_priv; + struct drm_i915_execbuffer *exec_buf = data; + struct drm_i915_batchbuffer *batch = &exec_buf->batch; + struct drm_fence_arg *fence_arg = &exec_buf->fence_arg; + int num_buffers; + int ret; + struct drm_i915_validate_buffer *buffers; + + if (!dev_priv->allow_batchbuffer) { + DRM_ERROR("Batchbuffer ioctl disabled\n"); + return -EINVAL; + } + + + if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects, + batch->num_cliprects * + sizeof(struct drm_clip_rect))) + return -EFAULT; + + if (exec_buf->num_buffers > dev_priv->max_validate_buffers) + return -EINVAL; + + + ret = drm_bo_read_lock(&dev->bm.bm_lock); + if (ret) + return ret; + + /* + * The cmdbuf_mutex makes sure the validate-submit-fence + * operation is atomic. + */ + + ret = mutex_lock_interruptible(&dev_priv->cmdbuf_mutex); + if (ret) { + drm_bo_read_unlock(&dev->bm.bm_lock); + return -EAGAIN; + } + + num_buffers = exec_buf->num_buffers; + + buffers = drm_calloc(num_buffers, sizeof(struct drm_i915_validate_buffer), DRM_MEM_DRIVER); + if (!buffers) { + drm_bo_read_unlock(&dev->bm.bm_lock); + mutex_unlock(&dev_priv->cmdbuf_mutex); + return -ENOMEM; + } + + /* validate buffer list + fixup relocations */ + ret = i915_validate_buffer_list(file_priv, 0, exec_buf->ops_list, + buffers, &num_buffers); + if (ret) + goto out_err0; + + /* make sure all previous memory operations have passed */ + DRM_MEMORYBARRIER(); + drm_agp_chipset_flush(dev); + + /* submit buffer */ + batch->start = buffers[num_buffers-1].buffer->offset; + + DRM_DEBUG("i915 exec batchbuffer, start %x used %d cliprects %d\n", + batch->start, batch->used, batch->num_cliprects); + + ret = i915_dispatch_batchbuffer(dev, batch); + if (ret) + goto out_err0; + + if (sarea_priv) + sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); + + i915_fence_or_sync(file_priv, fence_arg->flags, fence_arg, NULL); + +out_err0: + + /* handle errors */ + ret = i915_handle_copyback(dev, buffers, num_buffers, ret); + mutex_lock(&dev->struct_mutex); + i915_dereference_buffers_locked(buffers, num_buffers); + mutex_unlock(&dev->struct_mutex); + + drm_free(buffers, (exec_buf->num_buffers * sizeof(struct drm_buffer_object *)), DRM_MEM_DRIVER); + + mutex_unlock(&dev_priv->cmdbuf_mutex); + drm_bo_read_unlock(&dev->bm.bm_lock); + return ret; +} +#endif -- cgit v1.2.3 From 2305100c0fce9ec86a22660e5fed54791cff030b Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Fri, 29 Feb 2008 13:25:55 +0100 Subject: More post-ioctl work. --- linux-core/Makefile.kernel | 2 +- linux-core/drm_ttm.c | 2 +- linux-core/i915_execbuf.c | 395 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 315 insertions(+), 84 deletions(-) (limited to 'linux-core') diff --git a/linux-core/Makefile.kernel b/linux-core/Makefile.kernel index e7c280d0..defbe43b 100644 --- a/linux-core/Makefile.kernel +++ b/linux-core/Makefile.kernel @@ -20,7 +20,7 @@ r128-objs := r128_drv.o r128_cce.o r128_state.o r128_irq.o mga-objs := mga_drv.o mga_dma.o mga_state.o mga_warp.o mga_irq.o i810-objs := i810_drv.o i810_dma.o i915-objs := i915_drv.o i915_dma.o i915_irq.o i915_mem.o i915_fence.o \ - i915_buffer.o i915_compat.o + i915_buffer.o i915_compat.o i915_execbuf.o nouveau-objs := nouveau_drv.o nouveau_state.o nouveau_fifo.o nouveau_mem.o \ nouveau_object.o nouveau_irq.o nouveau_notifier.o nouveau_swmthd.o \ nouveau_sgdma.o nouveau_dma.o nouveau_buffer.o nouveau_fence.o \ diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c index a9d87338..58568452 100644 --- a/linux-core/drm_ttm.c +++ b/linux-core/drm_ttm.c @@ -298,7 +298,7 @@ int drm_ttm_populate(struct drm_ttm *ttm) return 0; be = ttm->be; - if (ttm->page_flags & DRM_TTM_PAGE_WRITE) { + if (1 || (ttm->page_flags & DRM_TTM_PAGE_WRITE)) { for (i = 0; i < ttm->num_pages; ++i) { page = drm_ttm_get_page(ttm, i); if (!page) diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c index fecb5ab0..1e515a57 100644 --- a/linux-core/i915_execbuf.c +++ b/linux-core/i915_execbuf.c @@ -29,6 +29,18 @@ #include "i915_drm.h" #include "i915_drv.h" +#if DRM_DEBUG_CODE +#define DRM_DEBUG_RELOCATION (drm_debug != 0) +#else +#define DRM_DEBUG_RELOCATION 0 +#endif + +enum i915_buf_idle { + i915_reloc_unchecked, + i915_reloc_idle, + i915_reloc_busy +}; + struct i915_relocatee_info { struct drm_buffer_object *buf; unsigned long offset; @@ -36,8 +48,8 @@ struct i915_relocatee_info { unsigned page_offset; struct drm_bo_kmap_obj kmap; int is_iomem; - int idle; int dst; + int idle; #ifdef DRM_KMAP_ATOMIC_PROT_PFN unsigned long pfn; pgprot_t pg_prot; @@ -50,76 +62,28 @@ struct drm_i915_validate_buffer { int presumed_offset_correct; void __user *data; int ret; + enum i915_buf_idle idle; }; - -static int i915_update_relocatee(struct i915_relocatee_info *relocatee, - struct drm_i915_validate_buffer *buffers, - unsigned int dst, - unsigned long dst_offset) +static void i915_emit_ring_reloc(struct drm_device *dev, uint32_t offset, + uint32_t value) { - int ret; - - if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) { - i915_clear_relocatee(relocatee); - relocatee->dst = dst; - relocatee->buf = buffers[dst].buffer; - preempt_enable(); - ret = mutex_lock_interruptible(&relocatee->buf->mutex); - preempt_disable(); - if (unlikely(ret)) - return -EAGAIN; - - ret = drm_bo_wait(relocatee->buf, 0, 0, 0); - if (unlikely(ret)) - return ret; - - mutex_unlock(&relocatee->buf->mutex); - } - - if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) { - DRM_ERROR("Relocation destination out of bounds.\n"); - return -EINVAL; - } - if (unlikely(!drm_bo_same_page(relocatee->page_offset, dst_offset) || - NULL == relocatee->data_page)) { -#ifdef DRM_KMAP_ATOMIC_PROT_PFN - if (NULL != relocatee->data_page) { - kunmap_atomic(relocatee->data_page, KM_USER0); - relocatee->data_page = NULL; - } - ret = drm_bo_pfn_prot(relocatee->buf, dst_offset, - &relocatee->pfn, - &relocatee->pg_prot); - if (ret) { - DRM_ERROR("Can't map relocation destination.\n"); - return -EINVAL; - } - relocatee->data_page = - kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, - relocatee->pg_prot); -#else - if (NULL != relocatee->data_page) { - drm_bo_kunmap(&relocatee->kmap); - relocatee->data_page = NULL; - } - - ret = drm_bo_kmap(relocatee->buf, dst_offset >> PAGE_SHIFT, - 1, &relocatee->kmap); - if (ret) { - DRM_ERROR("Can't map relocation destination.\n"); - return ret; - } - - relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, - &relocatee->is_iomem); -#endif - relocatee->page_offset = dst_offset & PAGE_MASK; - } - return 0; + struct drm_i915_private *dev_priv = + (struct drm_i915_private *) dev->dev_private; + + DRM_INFO("Ring reloc.\n"); + RING_LOCALS; + i915_kernel_lost_context(dev); + BEGIN_LP_RING(6); + OUT_RING((0x02 << 29) | (0x40 << 22) | (0x3 << 20) | (0x3)); + OUT_RING((0x3 << 24) | (0xF0 << 16) | (0x40)); + OUT_RING((0x1 << 16) | (0x4)); + OUT_RING(offset); + OUT_RING(value); + OUT_RING(0); + ADVANCE_LP_RING(); } - static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buffers, unsigned num_buffers) { @@ -332,6 +296,248 @@ out_err: return ret; } + +static void i915_clear_relocatee(struct i915_relocatee_info *relocatee) +{ + if (relocatee->data_page) { +#ifndef DRM_KMAP_ATOMIC_PROT_PFN + drm_bo_kunmap(&relocatee->kmap); +#else + kunmap_atomic(relocatee->data_page, KM_USER0); +#endif + relocatee->data_page = NULL; + } + relocatee->buf = NULL; + relocatee->dst = ~0; +} + + +static int i915_update_relocatee(struct i915_relocatee_info *relocatee, + struct drm_i915_validate_buffer *buffers, + unsigned int dst, + unsigned long dst_offset) +{ + int ret; + + if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) { + i915_clear_relocatee(relocatee); + relocatee->dst = dst; + relocatee->buf = buffers[dst].buffer; + relocatee->idle = i915_reloc_idle; buffers[dst].idle; +#if 0 + if (relocatee->idle == i915_reloc_unchecked) { + preempt_enable(); + ret = mutex_lock_interruptible(&relocatee->buf->mutex); + if (unlikely(ret)) + return -EAGAIN; + + ret = drm_bo_wait(relocatee->buf, 0, 0, 1); + relocatee->idle = (ret == 0) ? i915_reloc_idle : i915_reloc_busy; + mutex_unlock(&relocatee->buf->mutex); + preempt_disable(); + buffers[dst].idle = relocatee->idle; + } +#endif + } + + if (relocatee->idle == i915_reloc_busy) + return 0; + + if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) { + DRM_ERROR("Relocation destination out of bounds.\n"); + return -EINVAL; + } + if (unlikely(!drm_bo_same_page(relocatee->page_offset, dst_offset) || + NULL == relocatee->data_page)) { +#ifdef DRM_KMAP_ATOMIC_PROT_PFN + if (NULL != relocatee->data_page) { + kunmap_atomic(relocatee->data_page, KM_USER0); + relocatee->data_page = NULL; + } + ret = drm_bo_pfn_prot(relocatee->buf, dst_offset, + &relocatee->pfn, + &relocatee->pg_prot); + if (ret) { + DRM_ERROR("Can't map relocation destination.\n"); + return -EINVAL; + } + relocatee->data_page = + kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, + relocatee->pg_prot); +#else + if (NULL != relocatee->data_page) { + drm_bo_kunmap(&relocatee->kmap); + relocatee->data_page = NULL; + } + + ret = drm_bo_kmap(relocatee->buf, dst_offset >> PAGE_SHIFT, + 1, &relocatee->kmap); + if (ret) { + DRM_ERROR("Can't map relocation destination.\n"); + return ret; + } + + relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, + &relocatee->is_iomem); +#endif + relocatee->page_offset = dst_offset & PAGE_MASK; + } + return 0; +} + +static int i915_apply_post_reloc(uint32_t reloc[], + struct drm_i915_validate_buffer *buffers, + uint32_t num_buffers, + struct i915_relocatee_info *relocatee) +{ + uint32_t reloc_buffer = reloc[2]; + uint32_t dst_buffer = reloc[3]; + uint32_t val; + uint32_t index; + int ret; + + if (likely(buffers[reloc_buffer].presumed_offset_correct)) + return 0; + if (unlikely(reloc_buffer >= num_buffers)) { + DRM_ERROR("Invalid reloc buffer index.\n"); + return -EINVAL; + } + if (unlikely(dst_buffer >= num_buffers)) { + DRM_ERROR("Invalid dest buffer index.\n"); + return -EINVAL; + } + + ret = i915_update_relocatee(relocatee, buffers, dst_buffer, + reloc[0]); + if (unlikely(ret)) + return ret; + + val = buffers[reloc_buffer].buffer->offset; + index = (reloc[0] - relocatee->page_offset) >> 2; + val = val + reloc[1]; + + if (relocatee->idle == i915_reloc_busy) { + i915_emit_ring_reloc(relocatee->buf->dev, + relocatee->buf->offset + reloc[0], + val); + return 0; + } + +#ifdef DRM_KMAP_ATOMIC_PROT_PFN + relocatee->data_page[index] = val; +#else + if (likely(relocatee->is_iomem)) + iowrite32(val, relocatee->data_page + index); + else + relocatee->data_page[index] = val; +#endif + + return 0; +} + +static int i915_post_relocs(struct drm_file *file_priv, + uint32_t __user *new_reloc_ptr, + struct drm_i915_validate_buffer *buffers, + unsigned int num_buffers) +{ + uint32_t *reloc; + uint32_t reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); + uint32_t header_size = I915_RELOC_HEADER * sizeof(uint32_t); + struct i915_relocatee_info relocatee; + uint32_t reloc_type; + uint32_t num_relocs; + uint32_t count; + int ret = 0; + int i; + int short_circuit = 1; + uint32_t __user *reloc_ptr; + uint64_t new_reloc_data; + uint32_t reloc_buf_size; + uint32_t *reloc_buf; + + for (i=0; i= *num_buffers) { @@ -392,8 +600,6 @@ int i915_validate_buffer_list(struct drm_file *file_priv, item->buffer = NULL; item->presumed_offset_correct = 0; - buffers[buf_count].buffer = NULL; - if (copy_from_user(&arg, (void __user *)(unsigned long)data, sizeof(arg))) { ret = -EFAULT; goto out_err; @@ -412,7 +618,24 @@ int i915_validate_buffer_list(struct drm_file *file_priv, buf_handle = req->bo_req.handle; reloc_user_ptr = (uint32_t *)(unsigned long)arg.reloc_ptr; - if (reloc_user_ptr) { + /* + * Switch mode to post-validation relocations? + */ + + if (unlikely((buf_count == 0) && (*post_relocs == NULL) && + (reloc_user_ptr != NULL))) { + uint32_t reloc_type; + + ret = get_user(reloc_type, reloc_user_ptr+1); + if (ret) + goto out_err; + + if (reloc_type == 1) + *post_relocs = reloc_user_ptr; + + } + + if ((*post_relocs == NULL) && (reloc_user_ptr != NULL)) { ret = i915_exec_reloc(file_priv, buf_handle, reloc_user_ptr, buffers, buf_count); if (ret) goto out_err; @@ -425,7 +648,6 @@ int i915_validate_buffer_list(struct drm_file *file_priv, req->bo_req.fence_class, 0, &item->rep, &item->buffer); - if (ret) { DRM_ERROR("error on handle validate %d\n", ret); goto out_err; @@ -559,8 +781,8 @@ void i915_fence_or_sync(struct drm_file *file_priv, } -static int i915_execbuffer(struct drm_device *dev, void *data, - struct drm_file *file_priv) +int i915_execbuffer(struct drm_device *dev, void *data, + struct drm_file *file_priv) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) @@ -570,6 +792,7 @@ static int i915_execbuffer(struct drm_device *dev, void *data, struct drm_fence_arg *fence_arg = &exec_buf->fence_arg; int num_buffers; int ret; + uint32_t __user *post_relocs; struct drm_i915_validate_buffer *buffers; if (!dev_priv->allow_batchbuffer) { @@ -577,7 +800,6 @@ static int i915_execbuffer(struct drm_device *dev, void *data, return -EINVAL; } - if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects, batch->num_cliprects * sizeof(struct drm_clip_rect))) @@ -586,7 +808,6 @@ static int i915_execbuffer(struct drm_device *dev, void *data, if (exec_buf->num_buffers > dev_priv->max_validate_buffers) return -EINVAL; - ret = drm_bo_read_lock(&dev->bm.bm_lock); if (ret) return ret; @@ -613,27 +834,38 @@ static int i915_execbuffer(struct drm_device *dev, void *data, /* validate buffer list + fixup relocations */ ret = i915_validate_buffer_list(file_priv, 0, exec_buf->ops_list, - buffers, &num_buffers); + buffers, &num_buffers, &post_relocs); if (ret) goto out_err0; + if (post_relocs) { + ret = i915_post_relocs(file_priv, post_relocs, + buffers, num_buffers); + if (ret) + goto out_err0; + } + /* make sure all previous memory operations have passed */ DRM_MEMORYBARRIER(); - drm_agp_chipset_flush(dev); - /* submit buffer */ - batch->start = buffers[num_buffers-1].buffer->offset; + if (!post_relocs) { + drm_agp_chipset_flush(dev); + batch->start = buffers[num_buffers-1].buffer->offset; + } else { + batch->start += buffers[0].buffer->offset; + } +#if 1 + // (void) i915_emit_mi_flush(dev, MI_NO_WRITE_FLUSH | MI_READ_FLUSH | MI_EXE_FLUSH); DRM_DEBUG("i915 exec batchbuffer, start %x used %d cliprects %d\n", batch->start, batch->used, batch->num_cliprects); ret = i915_dispatch_batchbuffer(dev, batch); if (ret) goto out_err0; - +#endif if (sarea_priv) sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); - i915_fence_or_sync(file_priv, fence_arg->flags, fence_arg, NULL); out_err0: @@ -650,4 +882,3 @@ out_err0: drm_bo_read_unlock(&dev->bm.bm_lock); return ret; } -#endif -- cgit v1.2.3 From 612c22f131a25915196e69d7ec1adb6f4ec84a60 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Fri, 29 Feb 2008 15:38:55 +0100 Subject: Working revision. --- linux-core/i915_execbuf.c | 228 ++++++++++++++++++++++++++-------------------- 1 file changed, 128 insertions(+), 100 deletions(-) (limited to 'linux-core') diff --git a/linux-core/i915_execbuf.c b/linux-core/i915_execbuf.c index 1e515a57..cb0e7348 100644 --- a/linux-core/i915_execbuf.c +++ b/linux-core/i915_execbuf.c @@ -1,5 +1,5 @@ /* - * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. + * Copyright 2003-2008 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a @@ -22,6 +22,10 @@ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * + * Authors: + * Thomas Hellstrom + * Dave Airlie + * ? */ #include "drmP.h" @@ -48,8 +52,9 @@ struct i915_relocatee_info { unsigned page_offset; struct drm_bo_kmap_obj kmap; int is_iomem; - int dst; + int dst; int idle; + int performed_ring_relocs; #ifdef DRM_KMAP_ATOMIC_PROT_PFN unsigned long pfn; pgprot_t pg_prot; @@ -65,13 +70,20 @@ struct drm_i915_validate_buffer { enum i915_buf_idle idle; }; +/* + * I'd like to use MI_STORE_DATA_IMM here, but I can't make + * it work. Seems like GART writes are broken with that + * instruction. Also I'm not sure that MI_FLUSH will + * act as a memory barrier for that instruction. It will + * for this single dword 2D blit. + */ + static void i915_emit_ring_reloc(struct drm_device *dev, uint32_t offset, uint32_t value) { struct drm_i915_private *dev_priv = - (struct drm_i915_private *) dev->dev_private; + (struct drm_i915_private *)dev->dev_private; - DRM_INFO("Ring reloc.\n"); RING_LOCALS; i915_kernel_lost_context(dev); BEGIN_LP_RING(6); @@ -84,8 +96,8 @@ static void i915_emit_ring_reloc(struct drm_device *dev, uint32_t offset, ADVANCE_LP_RING(); } -static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buffers, - unsigned num_buffers) +static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer + *buffers, unsigned num_buffers) { while (num_buffers--) drm_bo_usage_deref_locked(&buffers[num_buffers].buffer); @@ -93,8 +105,7 @@ static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buf int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, struct drm_i915_validate_buffer *buffers, - struct i915_relocatee_info *relocatee, - uint32_t *reloc) + struct i915_relocatee_info *relocatee, uint32_t * reloc) { unsigned index; unsigned long new_cmd_offset; @@ -130,18 +141,19 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, relocatee->data_page = NULL; relocatee->offset = new_cmd_offset; - /* - * Note on buffer idle: - * Since we're applying relocations, this part of the - * buffer is obviously not used by the GPU and we don't - * need to wait for buffer idle. This is an important - * consideration for user-space buffer pools. - */ + if (unlikely(relocatee->idle == i915_reloc_unchecked)) { + ret = drm_bo_wait(relocatee->buf, 0, 0, 0); + if (ret) + return ret; + relocatee->idle = i915_reloc_idle; + } ret = drm_bo_kmap(relocatee->buf, new_cmd_offset >> PAGE_SHIFT, 1, &relocatee->kmap); if (ret) { - DRM_ERROR("Could not map command buffer to apply relocs\n %08lx", new_cmd_offset); + DRM_ERROR + ("Could not map command buffer to apply relocs\n %08lx", + new_cmd_offset); return ret; } relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, @@ -158,8 +170,10 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, if (DRM_DEBUG_RELOCATION) { if (buffers[buf_index].presumed_offset_correct && relocatee->data_page[index] != val) { - DRM_DEBUG ("Relocation mismatch source %d target %d buffer %d user %08x kernel %08x\n", - reloc[0], reloc[1], buf_index, relocatee->data_page[index], val); + DRM_DEBUG + ("Relocation mismatch source %d target %d buffer %d user %08x kernel %08x\n", + reloc[0], reloc[1], buf_index, + relocatee->data_page[index], val); } } @@ -172,7 +186,7 @@ int i915_apply_reloc(struct drm_file *file_priv, int num_buffers, int i915_process_relocs(struct drm_file *file_priv, uint32_t buf_handle, - uint32_t __user **reloc_user_ptr, + uint32_t __user ** reloc_user_ptr, struct i915_relocatee_info *relocatee, struct drm_i915_validate_buffer *buffers, uint32_t num_buffers) @@ -192,7 +206,7 @@ int i915_process_relocs(struct drm_file *file_priv, goto out; } - ret = get_user(reloc_type, (*reloc_user_ptr)+1); + ret = get_user(reloc_type, (*reloc_user_ptr) + 1); if (ret) { DRM_ERROR("Could not map relocation buffer.\n"); goto out; @@ -204,7 +218,9 @@ int i915_process_relocs(struct drm_file *file_priv, goto out; } - reloc_buf_size = (I915_RELOC_HEADER + (reloc_count * I915_RELOC0_STRIDE)) * sizeof(uint32_t); + reloc_buf_size = + (I915_RELOC_HEADER + + (reloc_count * I915_RELOC0_STRIDE)) * sizeof(uint32_t); reloc_buf = kmalloc(reloc_buf_size, GFP_KERNEL); if (!reloc_buf) { DRM_ERROR("Out of memory for reloc buffer\n"); @@ -218,26 +234,27 @@ int i915_process_relocs(struct drm_file *file_priv, } /* get next relocate buffer handle */ - *reloc_user_ptr = (uint32_t *)*(unsigned long *)&reloc_buf[2]; + *reloc_user_ptr = (uint32_t *) * (unsigned long *)&reloc_buf[2]; - reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); /* may be different for other types of relocs */ + reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); /* may be different for other types of relocs */ - DRM_DEBUG("num relocs is %d, next is %p\n", reloc_count, *reloc_user_ptr); + DRM_DEBUG("num relocs is %d, next is %p\n", reloc_count, + *reloc_user_ptr); for (i = 0; i < reloc_count; i++) { cur_offset = I915_RELOC_HEADER + (i * I915_RELOC0_STRIDE); - + ret = i915_apply_reloc(file_priv, num_buffers, buffers, relocatee, reloc_buf + cur_offset); if (ret) goto out; } -out: + out: if (reloc_buf) kfree(reloc_buf); - if (relocatee->data_page) { + if (relocatee->data_page) { drm_bo_kunmap(&relocatee->kmap); relocatee->data_page = NULL; } @@ -246,7 +263,7 @@ out: } static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle, - uint32_t __user *reloc_user_ptr, + uint32_t __user * reloc_user_ptr, struct drm_i915_validate_buffer *buffers, uint32_t buf_count) { @@ -264,12 +281,13 @@ static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle, for (b = 0; b < buf_count; b++) if (!buffers[b].presumed_offset_correct) break; - + if (b == buf_count) return 0; } memset(&relocatee, 0, sizeof(relocatee)); + relocatee.idle = i915_reloc_unchecked; mutex_lock(&dev->struct_mutex); relocatee.buf = drm_lookup_buffer_object(file_priv, buf_handle, 1); @@ -280,23 +298,24 @@ static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle, goto out_err; } - mutex_lock (&relocatee.buf->mutex); + mutex_lock(&relocatee.buf->mutex); while (reloc_user_ptr) { - ret = i915_process_relocs(file_priv, buf_handle, &reloc_user_ptr, &relocatee, buffers, buf_count); + ret = + i915_process_relocs(file_priv, buf_handle, &reloc_user_ptr, + &relocatee, buffers, buf_count); if (ret) { DRM_ERROR("process relocs failed\n"); goto out_err1; } } -out_err1: - mutex_unlock (&relocatee.buf->mutex); + out_err1: + mutex_unlock(&relocatee.buf->mutex); drm_bo_usage_deref_unlocked(&relocatee.buf); -out_err: + out_err: return ret; } - static void i915_clear_relocatee(struct i915_relocatee_info *relocatee) { if (relocatee->data_page) { @@ -311,20 +330,23 @@ static void i915_clear_relocatee(struct i915_relocatee_info *relocatee) relocatee->dst = ~0; } - static int i915_update_relocatee(struct i915_relocatee_info *relocatee, struct drm_i915_validate_buffer *buffers, - unsigned int dst, - unsigned long dst_offset) + unsigned int dst, unsigned long dst_offset) { int ret; - + if (unlikely(dst != relocatee->dst || NULL == relocatee->buf)) { i915_clear_relocatee(relocatee); relocatee->dst = dst; relocatee->buf = buffers[dst].buffer; - relocatee->idle = i915_reloc_idle; buffers[dst].idle; -#if 0 + relocatee->idle = buffers[dst].idle; + + /* + * Check for buffer idle. If the buffer is busy, revert to + * ring relocations. + */ + if (relocatee->idle == i915_reloc_unchecked) { preempt_enable(); ret = mutex_lock_interruptible(&relocatee->buf->mutex); @@ -332,16 +354,20 @@ static int i915_update_relocatee(struct i915_relocatee_info *relocatee, return -EAGAIN; ret = drm_bo_wait(relocatee->buf, 0, 0, 1); - relocatee->idle = (ret == 0) ? i915_reloc_idle : i915_reloc_busy; + if (ret == 0) + relocatee->idle = i915_reloc_idle; + else { + relocatee->idle = i915_reloc_busy; + relocatee->performed_ring_relocs = 1; + } mutex_unlock(&relocatee->buf->mutex); preempt_disable(); buffers[dst].idle = relocatee->idle; } -#endif } if (relocatee->idle == i915_reloc_busy) - return 0; + return 0; if (unlikely(dst_offset > relocatee->buf->num_pages * PAGE_SIZE)) { DRM_ERROR("Relocation destination out of bounds.\n"); @@ -355,15 +381,14 @@ static int i915_update_relocatee(struct i915_relocatee_info *relocatee, relocatee->data_page = NULL; } ret = drm_bo_pfn_prot(relocatee->buf, dst_offset, - &relocatee->pfn, - &relocatee->pg_prot); + &relocatee->pfn, &relocatee->pg_prot); if (ret) { DRM_ERROR("Can't map relocation destination.\n"); return -EINVAL; } relocatee->data_page = - kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, - relocatee->pg_prot); + kmap_atomic_prot_pfn(relocatee->pfn, KM_USER0, + relocatee->pg_prot); #else if (NULL != relocatee->data_page) { drm_bo_kunmap(&relocatee->kmap); @@ -378,7 +403,7 @@ static int i915_update_relocatee(struct i915_relocatee_info *relocatee, } relocatee->data_page = drm_bmo_virtual(&relocatee->kmap, - &relocatee->is_iomem); + &relocatee->is_iomem); #endif relocatee->page_offset = dst_offset & PAGE_MASK; } @@ -407,8 +432,7 @@ static int i915_apply_post_reloc(uint32_t reloc[], return -EINVAL; } - ret = i915_update_relocatee(relocatee, buffers, dst_buffer, - reloc[0]); + ret = i915_update_relocatee(relocatee, buffers, dst_buffer, reloc[0]); if (unlikely(ret)) return ret; @@ -417,12 +441,10 @@ static int i915_apply_post_reloc(uint32_t reloc[], val = val + reloc[1]; if (relocatee->idle == i915_reloc_busy) { - i915_emit_ring_reloc(relocatee->buf->dev, - relocatee->buf->offset + reloc[0], - val); + i915_emit_ring_reloc(relocatee->buf->dev, + relocatee->buf->offset + reloc[0], val); return 0; } - #ifdef DRM_KMAP_ATOMIC_PROT_PFN relocatee->data_page[index] = val; #else @@ -436,7 +458,7 @@ static int i915_apply_post_reloc(uint32_t reloc[], } static int i915_post_relocs(struct drm_file *file_priv, - uint32_t __user *new_reloc_ptr, + uint32_t __user * new_reloc_ptr, struct drm_i915_validate_buffer *buffers, unsigned int num_buffers) { @@ -455,7 +477,7 @@ static int i915_post_relocs(struct drm_file *file_priv, uint32_t reloc_buf_size; uint32_t *reloc_buf; - for (i=0; ihead->dev, 0); + + i915_clear_relocatee(&relocatee); if (reloc_buf) { kfree(reloc_buf); reloc_buf = NULL; @@ -540,8 +571,7 @@ out: static int i915_check_presumed(struct drm_i915_op_arg *arg, struct drm_buffer_object *bo, - uint32_t __user *data, - int *presumed_ok) + uint32_t __user * data, int *presumed_ok) { struct drm_bo_op_req *req = &arg->d.req; uint32_t hint_offset; @@ -566,20 +596,19 @@ static int i915_check_presumed(struct drm_i915_op_arg *arg, * Needless to say, this is a bit ugly. */ - hint_offset = (uint32_t *)&req->bo_req.hint - (uint32_t *)arg; + hint_offset = (uint32_t *) & req->bo_req.hint - (uint32_t *) arg; hint &= ~DRM_BO_HINT_PRESUMED_OFFSET; return __put_user(hint, data + hint_offset); } - /* * Validate, add fence and relocate a block of bos from a userspace list */ int i915_validate_buffer_list(struct drm_file *file_priv, unsigned int fence_class, uint64_t data, struct drm_i915_validate_buffer *buffers, - uint32_t *num_buffers, - uint32_t __user **post_relocs) + uint32_t * num_buffers, + uint32_t __user ** post_relocs) { struct drm_i915_op_arg arg; struct drm_bo_op_req *req = &arg.d.req; @@ -599,8 +628,10 @@ int i915_validate_buffer_list(struct drm_file *file_priv, item = buffers + buf_count; item->buffer = NULL; item->presumed_offset_correct = 0; + item->idle = i915_reloc_unchecked; - if (copy_from_user(&arg, (void __user *)(unsigned long)data, sizeof(arg))) { + if (copy_from_user + (&arg, (void __user *)(unsigned long)data, sizeof(arg))) { ret = -EFAULT; goto out_err; } @@ -613,10 +644,10 @@ int i915_validate_buffer_list(struct drm_file *file_priv, goto out_err; } item->ret = 0; - item->data = (void __user *) (unsigned long) data; + item->data = (void __user *)(unsigned long)data; buf_handle = req->bo_req.handle; - reloc_user_ptr = (uint32_t *)(unsigned long)arg.reloc_ptr; + reloc_user_ptr = (uint32_t *) (unsigned long)arg.reloc_ptr; /* * Switch mode to post-validation relocations? @@ -626,7 +657,7 @@ int i915_validate_buffer_list(struct drm_file *file_priv, (reloc_user_ptr != NULL))) { uint32_t reloc_type; - ret = get_user(reloc_type, reloc_user_ptr+1); + ret = get_user(reloc_type, reloc_user_ptr + 1); if (ret) goto out_err; @@ -636,18 +667,19 @@ int i915_validate_buffer_list(struct drm_file *file_priv, } if ((*post_relocs == NULL) && (reloc_user_ptr != NULL)) { - ret = i915_exec_reloc(file_priv, buf_handle, reloc_user_ptr, buffers, buf_count); + ret = + i915_exec_reloc(file_priv, buf_handle, + reloc_user_ptr, buffers, buf_count); if (ret) goto out_err; DRM_MEMORYBARRIER(); } ret = drm_bo_handle_validate(file_priv, req->bo_req.handle, - req->bo_req.flags, req->bo_req.mask, - req->bo_req.hint, + req->bo_req.flags, + req->bo_req.mask, req->bo_req.hint, req->bo_req.fence_class, 0, - &item->rep, - &item->buffer); + &item->rep, &item->buffer); if (ret) { DRM_ERROR("error on handle validate %d\n", ret); goto out_err; @@ -657,20 +689,19 @@ int i915_validate_buffer_list(struct drm_file *file_priv, ret = i915_check_presumed(&arg, item->buffer, (uint32_t __user *) - (unsigned long) data, + (unsigned long)data, &item->presumed_offset_correct); if (ret) goto out_err; data = arg.next; } while (data != 0); -out_err: + out_err: *num_buffers = buf_count; item->ret = (ret != -EAGAIN) ? ret : 0; return ret; } - /* * Remove all buffers from the unfenced list. * If the execbuffer operation was aborted, for example due to a signal, @@ -719,8 +750,7 @@ void i915_fence_or_sync(struct drm_file *file_priv, int ret; struct drm_fence_object *fence; - ret = drm_fence_buffer_objects(dev, NULL, fence_flags, - NULL, &fence); + ret = drm_fence_buffer_objects(dev, NULL, fence_flags, NULL, &fence); if (ret) { @@ -729,8 +759,8 @@ void i915_fence_or_sync(struct drm_file *file_priv, * Fall back to synchronous operation and idle the engine. */ - (void) i915_emit_mi_flush(dev, MI_READ_FLUSH); - (void) i915_quiescent(dev); + (void)i915_emit_mi_flush(dev, MI_READ_FLUSH); + (void)i915_quiescent(dev); if (!(fence_flags & DRM_FENCE_FLAG_NO_USER)) { @@ -746,7 +776,7 @@ void i915_fence_or_sync(struct drm_file *file_priv, drm_putback_buffer_objects(dev); if (fence_p) - *fence_p = NULL; + *fence_p = NULL; return; } @@ -766,8 +796,7 @@ void i915_fence_or_sync(struct drm_file *file_priv, * to indicate engine "sufficiently" idle. */ - (void) drm_fence_object_wait(fence, 0, 1, - fence->type); + (void)drm_fence_object_wait(fence, 0, 1, fence->type); drm_fence_usage_deref_unlocked(&fence); fence_arg->handle = ~0; fence_arg->error = ret; @@ -780,13 +809,12 @@ void i915_fence_or_sync(struct drm_file *file_priv, drm_fence_usage_deref_unlocked(&fence); } - int i915_execbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) - dev_priv->sarea_priv; + dev_priv->sarea_priv; struct drm_i915_execbuffer *exec_buf = data; struct drm_i915_batchbuffer *batch = &exec_buf->batch; struct drm_fence_arg *fence_arg = &exec_buf->fence_arg; @@ -802,7 +830,8 @@ int i915_execbuffer(struct drm_device *dev, void *data, if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects, batch->num_cliprects * - sizeof(struct drm_clip_rect))) + sizeof(struct + drm_clip_rect))) return -EFAULT; if (exec_buf->num_buffers > dev_priv->max_validate_buffers) @@ -825,7 +854,9 @@ int i915_execbuffer(struct drm_device *dev, void *data, num_buffers = exec_buf->num_buffers; - buffers = drm_calloc(num_buffers, sizeof(struct drm_i915_validate_buffer), DRM_MEM_DRIVER); + buffers = + drm_calloc(num_buffers, sizeof(struct drm_i915_validate_buffer), + DRM_MEM_DRIVER); if (!buffers) { drm_bo_read_unlock(&dev->bm.bm_lock); mutex_unlock(&dev_priv->cmdbuf_mutex); @@ -850,33 +881,30 @@ int i915_execbuffer(struct drm_device *dev, void *data, if (!post_relocs) { drm_agp_chipset_flush(dev); - batch->start = buffers[num_buffers-1].buffer->offset; + batch->start = buffers[num_buffers - 1].buffer->offset; } else { batch->start += buffers[0].buffer->offset; } -#if 1 - // (void) i915_emit_mi_flush(dev, MI_NO_WRITE_FLUSH | MI_READ_FLUSH | MI_EXE_FLUSH); DRM_DEBUG("i915 exec batchbuffer, start %x used %d cliprects %d\n", batch->start, batch->used, batch->num_cliprects); ret = i915_dispatch_batchbuffer(dev, batch); if (ret) goto out_err0; -#endif if (sarea_priv) sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); i915_fence_or_sync(file_priv, fence_arg->flags, fence_arg, NULL); -out_err0: - - /* handle errors */ + out_err0: ret = i915_handle_copyback(dev, buffers, num_buffers, ret); mutex_lock(&dev->struct_mutex); i915_dereference_buffers_locked(buffers, num_buffers); mutex_unlock(&dev->struct_mutex); - drm_free(buffers, (exec_buf->num_buffers * sizeof(struct drm_buffer_object *)), DRM_MEM_DRIVER); + drm_free(buffers, + (exec_buf->num_buffers * sizeof(struct drm_buffer_object *)), + DRM_MEM_DRIVER); mutex_unlock(&dev_priv->cmdbuf_mutex); drm_bo_read_unlock(&dev->bm.bm_lock); -- cgit v1.2.3