/************************************************************************** * * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * **************************************************************************/ /* * Authors: Thomas Hellström */ #include "drmP.h" /* * Locking may look a bit complicated but isn't really: * * The buffer usage atomic_t needs to be protected by dev->struct_mutex * when there is a chance that it can be zero before or after the operation. * * dev->struct_mutex also protects all lists and list heads. Hash tables and hash * heads. * * bo->mutex protects the buffer object itself excluding the usage field. * bo->mutex does also protect the buffer list heads, so to manipulate those, we need * both the bo->mutex and the dev->struct_mutex. * * Locking order is bo->mutex, dev->struct_mutex. Therefore list traversal is a bit * complicated. When dev->struct_mutex is released to grab bo->mutex, the list * traversal will, in general, need to be restarted. * */ static void drm_bo_destroy_locked(drm_buffer_object_t *bo); static int drm_bo_setup_vm_locked(drm_buffer_object_t *bo); static void drm_bo_takedown_vm_locked(drm_buffer_object_t *bo); static void drm_bo_unmap_virtual(drm_buffer_object_t *bo); static int drm_bo_mem_space(drm_device_t *dev, drm_bo_mem_reg_t *mem, int no_wait); #define DRM_FLAG_MASKED(_old, _new, _mask) {\ (_old) ^= (((_old) ^ (_new)) & (_mask)); \ } static inline uint32_t drm_bo_type_flags(unsigned type) { return (1 << (24 + type)); } /* * bo locked. dev->struct_mutex locked. */ static void drm_bo_add_to_lru(drm_buffer_object_t * bo, drm_buffer_manager_t * bm) { struct list_head *list; drm_mem_type_manager_t *man; man = &bm->man[bo->mem.mem_type]; list = (bo->mem.flags & (DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE)) ? &man->pinned : &man->lru; list_add_tail(&bo->lru, list); return; } static int drm_bo_vm_pre_move(drm_buffer_object_t *bo, int old_is_pci) { #ifdef DRM_ODD_MM_COMPAT int ret; ret = drm_bo_lock_kmm(bo); if (ret) { if (ret == -EAGAIN) schedule(); return ret; } drm_bo_unmap_virtual(bo); if (old_is_pci) drm_bo_finish_unmap(bo); #else drm_bo_unmap_virtual(bo); #endif return 0; } static void drm_bo_vm_post_move(drm_buffer_object_t *bo) { #ifdef DRM_ODD_MM_COMPAT int ret; ret = drm_bo_remap_bound(bo); if (ret) { DRM_ERROR("Failed to remap a bound buffer object.\n" "\tThis might cause a sigbus later.\n"); } drm_bo_unlock_kmm(bo); #endif } /* * Call bo->mutex locked. */ static int drm_bo_add_ttm(drm_buffer_object_t * bo) { drm_device_t *dev = bo->dev; int ret = 0; bo->ttm = NULL; switch (bo->type) { case drm_bo_type_dc: bo->ttm = drm_ttm_init(dev, bo->mem.num_pages << PAGE_SHIFT); if (!bo->ttm) ret = -ENOMEM; break; case drm_bo_type_user: case drm_bo_type_fake: break; default: DRM_ERROR("Illegal buffer object type\n"); ret = -EINVAL; break; } return ret; } static int drm_bo_handle_move_mem(drm_buffer_object_t *bo, drm_bo_mem_reg_t *mem, int evict, int no_wait) { drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; int old_is_pci = drm_mem_reg_is_pci(dev, &bo->mem); int new_is_pci = drm_mem_reg_is_pci(dev, mem); drm_mem_type_manager_t *old_man = &bm->man[bo->mem.mem_type]; drm_mem_type_manager_t *new_man = &bm->man[mem->mem_type]; int ret = 0; if (old_is_pci || new_is_pci) ret = drm_bo_vm_pre_move(bo, old_is_pci); if (ret) return ret; if ((!(old_man->flags & _DRM_FLAG_MEMTYPE_FIXED) || !(new_man->flags & _DRM_FLAG_MEMTYPE_FIXED)) && (bo->ttm == NULL)) ret = drm_bo_add_ttm(bo); if (ret) return ret; if (!(old_man->flags & _DRM_FLAG_MEMTYPE_FIXED) && !(new_man->flags & _DRM_FLAG_MEMTYPE_FIXED)) { ret = drm_bo_move_ttm(dev, bo->ttm, evict, no_wait, &bo->mem, mem); } else if (dev->driver->bo_driver->move) { ret = dev->driver->bo_driver->move(dev, bo->ttm, evict, no_wait, &bo->mem, mem); } else { ret = -EINVAL; DRM_ERROR("Unsupported function\n"); #if 0 ret = drm_bo_move_memcpy(dev, bo->ttm, evict, no_wait, &bo->mem, mem); ret = 0; #endif } if (old_is_pci || new_is_pci) drm_bo_vm_post_move(bo); if (ret) return ret; if (bo->priv_flags & _DRM_BO_FLAG_EVICTED) { ret = dev->driver->bo_driver->invalidate_caches(dev, bo->mem.flags); if (ret) DRM_ERROR("Can not flush read caches\n"); } DRM_FLAG_MASKED(bo->priv_flags, (evict) ? _DRM_BO_FLAG_EVICTED : 0, _DRM_BO_FLAG_EVICTED); if (bo->mem.mm_node) bo->offset = bo->mem.mm_node->start << PAGE_SHIFT; return 0; } /* * Call bo->mutex locked. * Wait until the buffer is idle. */ static int drm_bo_wait(drm_buffer_object_t * bo, int lazy, int ignore_signals, int no_wait) { drm_fence_object_t *fence = bo->fence; int ret; if (fence) { drm_device_t *dev = bo->dev; if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(dev, fence); bo->fence = NULL; return 0; } if (no_wait) { return -EBUSY; } ret = drm_fence_object_wait(dev, fence, lazy, ignore_signals, bo->fence_type); if (ret) return ret; drm_fence_usage_deref_unlocked(dev, fence); bo->fence = NULL; } return 0; } /* * Call dev->struct_mutex locked. * Attempts to remove all private references to a buffer by expiring its * fence object and removing from lru lists and memory managers. */ static void drm_bo_cleanup_refs(drm_buffer_object_t *bo, int remove_all) { drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; atomic_inc(&bo->usage); mutex_unlock(&dev->struct_mutex); mutex_lock(&bo->mutex); DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); if (bo->fence && drm_fence_object_signaled(bo->fence, bo->fence_type)) { drm_fence_usage_deref_locked(dev, bo->fence); bo->fence = NULL; } if (bo->fence && remove_all) { if (bm->nice_mode) { unsigned long _end = jiffies + 3 * DRM_HZ; int ret; do { ret = drm_bo_wait(bo, 0, 1, 0); } while (ret && !time_after_eq(jiffies, _end)); if (bo->fence) { bm->nice_mode = 0; DRM_ERROR("Detected GPU lockup or " "fence driver was taken down. " "Evicting waiting buffers.\n"); } if (bo->fence) { drm_fence_usage_deref_unlocked(dev, bo->fence); bo->fence = NULL; } } } mutex_lock(&dev->struct_mutex); if (!atomic_dec_and_test(&bo->usage)) { goto out; } if (!bo->fence) { list_del_init(&bo->lru); if (bo->mem.mm_node) { drm_mm_put_block(bo->mem.mm_node); bo->mem.mm_node = NULL; } list_del_init(&bo->ddestroy); mutex_unlock(&bo->mutex); drm_bo_destroy_locked(bo); return; } if (list_empty(&bo->ddestroy)) { drm_fence_object_flush(dev, bo->fence, bo->fence_type); list_add_tail(&bo->ddestroy, &bm->ddestroy); schedule_delayed_work(&bm->wq, ((DRM_HZ / 100) < 1) ? 1 : DRM_HZ / 100); } out: mutex_unlock(&bo->mutex); return; } /* * Verify that refcount is 0 and that there are no internal references * to the buffer object. Then destroy it. */ static void drm_bo_destroy_locked(drm_buffer_object_t *bo) { drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; if (list_empty(&bo->lru) && bo->mem.mm_node == NULL && atomic_read(&bo->usage) == 0) { BUG_ON(bo->fence != NULL); #ifdef DRM_ODD_MM_COMPAT BUG_ON(!list_empty(&bo->vma_list)); BUG_ON(!list_empty(&bo->p_mm_list)); #endif if (bo->ttm) { drm_ttm_unbind(bo->ttm); drm_destroy_ttm(bo->ttm); bo->ttm = NULL; } atomic_dec(&bm->count); drm_ctl_free(bo, sizeof(*bo), DRM_MEM_BUFOBJ); return; } /* * Some stuff is still trying to reference the buffer object. * Get rid of those references. */ drm_bo_cleanup_refs(bo, 0); return; } /* * Call dev->struct_mutex locked. */ static void drm_bo_delayed_delete(drm_device_t * dev, int remove_all) { drm_buffer_manager_t *bm = &dev->bm; drm_buffer_object_t *entry, *nentry; struct list_head *list, *next; list_for_each_safe(list, next, &bm->ddestroy) { entry = list_entry(list, drm_buffer_object_t, ddestroy); nentry = NULL; if (next != &bm->ddestroy) { nentry = list_entry(next, drm_buffer_object_t, ddestroy); atomic_inc(&nentry->usage); } drm_bo_cleanup_refs(entry, remove_all); if (nentry) { atomic_dec(&nentry->usage); } } } #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) static void drm_bo_delayed_workqueue(void *data) #else static void drm_bo_delayed_workqueue(struct work_struct *work) #endif { #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) drm_device_t *dev = (drm_device_t *) data; drm_buffer_manager_t *bm = &dev->bm; #else drm_buffer_manager_t *bm = container_of(work, drm_buffer_manager_t, wq.work); drm_device_t *dev = container_of(bm, drm_device_t, bm); #endif DRM_DEBUG("Delayed delete Worker\n"); mutex_lock(&dev->struct_mutex); if (!bm->initialized) { mutex_unlock(&dev->struct_mutex); return; } drm_bo_delayed_delete(dev, 0); if (bm->initialized && !list_empty(&bm->ddestroy)) { schedule_delayed_work(&bm->wq, ((DRM_HZ / 100) < 1) ? 1 : DRM_HZ / 100); } mutex_unlock(&dev->struct_mutex); } void drm_bo_usage_deref_locked(drm_buffer_object_t * bo) { if (atomic_dec_and_test(&bo->usage)) { drm_bo_destroy_locked(bo); } } static void drm_bo_base_deref_locked(drm_file_t * priv, drm_user_object_t * uo) { drm_buffer_object_t *bo = drm_user_object_entry(uo, drm_buffer_object_t, base); drm_bo_takedown_vm_locked(bo); drm_bo_usage_deref_locked(bo); } static void drm_bo_usage_deref_unlocked(drm_buffer_object_t * bo) { drm_device_t *dev = bo->dev; if (atomic_dec_and_test(&bo->usage)) { mutex_lock(&dev->struct_mutex); if (atomic_read(&bo->usage) == 0) drm_bo_destroy_locked(bo); mutex_unlock(&dev->struct_mutex); } } /* * Note. The caller has to register (if applicable) * and deregister fence object usage. */ int drm_fence_buffer_objects(drm_file_t * priv, struct list_head *list, uint32_t fence_flags, drm_fence_object_t * fence, drm_fence_object_t ** used_fence) { drm_device_t *dev = priv->head->dev; drm_buffer_manager_t *bm = &dev->bm; drm_buffer_object_t *entry; uint32_t fence_type = 0; int count = 0; int ret = 0; struct list_head *l; LIST_HEAD(f_list); mutex_lock(&dev->struct_mutex); if (!list) list = &bm->unfenced; list_for_each_entry(entry, list, lru) { BUG_ON(!(entry->priv_flags & _DRM_BO_FLAG_UNFENCED)); fence_type |= entry->fence_type; if (entry->fence_class != 0) { DRM_ERROR("Fence class %d is not implemented yet.\n", entry->fence_class); ret = -EINVAL; goto out; } count++; } if (!count) { ret = -EINVAL; goto out; } /* * Transfer to a local list before we release the dev->struct_mutex; * This is so we don't get any new unfenced objects while fencing * the ones we already have.. */ list_splice_init(list, &f_list); if (fence) { if ((fence_type & fence->type) != fence_type) { DRM_ERROR("Given fence doesn't match buffers " "on unfenced list.\n"); ret = -EINVAL; goto out; } } else { mutex_unlock(&dev->struct_mutex); ret = drm_fence_object_create(dev, fence_type, fence_flags | DRM_FENCE_FLAG_EMIT, &fence); mutex_lock(&dev->struct_mutex); if (ret) goto out; } count = 0; l = f_list.next; while (l != &f_list) { entry = list_entry(l, drm_buffer_object_t, lru); atomic_inc(&entry->usage); mutex_unlock(&dev->struct_mutex); mutex_lock(&entry->mutex); mutex_lock(&dev->struct_mutex); list_del_init(l); if (entry->priv_flags & _DRM_BO_FLAG_UNFENCED) { count++; if (entry->fence) drm_fence_usage_deref_locked(dev, entry->fence); entry->fence = fence; DRM_FLAG_MASKED(entry->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); DRM_WAKEUP(&entry->event_queue); drm_bo_add_to_lru(entry, bm); } mutex_unlock(&entry->mutex); drm_bo_usage_deref_locked(entry); l = f_list.next; } atomic_add(count, &fence->usage); DRM_DEBUG("Fenced %d buffers\n", count); out: mutex_unlock(&dev->struct_mutex); *used_fence = fence; return ret; } EXPORT_SYMBOL(drm_fence_buffer_objects); /* * bo->mutex locked */ static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type, int no_wait, int force_no_move) { int ret = 0; drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; drm_bo_mem_reg_t evict_mem; /* * Someone might have modified the buffer before we took the buffer mutex. < */ if (bo->priv_flags & _DRM_BO_FLAG_UNFENCED) goto out; if (bo->mem.mem_type != mem_type) ret = drm_bo_wait(bo, 0, 0, no_wait); if (ret && ret != -EAGAIN) { DRM_ERROR("Failed to expire fence before " "buffer eviction.\n"); goto out; } evict_mem = bo->mem; evict_mem.mask = dev->driver->bo_driver->evict_flags(dev, mem_type); ret = drm_bo_mem_space(dev, &evict_mem, no_wait); if (ret) { if (ret != -EAGAIN) DRM_ERROR("Failed to find memory space for " "buffer eviction.\n"); goto out; } ret = drm_bo_handle_move_mem(bo, &evict_mem, 1, no_wait); if (ret) { if (ret != -EAGAIN) DRM_ERROR("Buffer eviction failed\n"); goto out; } mutex_lock(&dev->struct_mutex); if (evict_mem.mm_node) { drm_mm_put_block(evict_mem.mm_node); evict_mem.mm_node = NULL; } list_del(&bo->lru); drm_bo_add_to_lru(bo, bm); mutex_unlock(&dev->struct_mutex); DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_EVICTED, _DRM_BO_FLAG_EVICTED); out: return ret; } static int drm_bo_mem_force_space(drm_device_t *dev, drm_bo_mem_reg_t *mem, uint32_t mem_type, int no_wait) { drm_mm_node_t *node; drm_buffer_manager_t *bm = &dev->bm; drm_buffer_object_t *entry; drm_mem_type_manager_t *man = &bm->man[mem_type]; struct list_head *lru; unsigned long num_pages = mem->num_pages; int ret; mutex_lock(&dev->struct_mutex); do { node = drm_mm_search_free(&man->manager, num_pages, mem->page_alignment, 1); if (node) break; lru = &man->lru; if (lru->next == lru) break; entry = list_entry(lru->next, drm_buffer_object_t, lru); atomic_inc(&entry->usage); mutex_unlock(&dev->struct_mutex); mutex_lock(&entry->mutex); BUG_ON(entry->mem.flags & (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT)); ret = drm_bo_evict(entry, mem_type, no_wait, 0); mutex_unlock(&entry->mutex); drm_bo_usage_deref_unlocked(entry); if (ret) return ret; mutex_lock(&dev->struct_mutex); } while (1); if (!node) { mutex_unlock(&dev->struct_mutex); return -ENOMEM; } node = drm_mm_get_block(node, num_pages, mem->page_alignment); mutex_unlock(&dev->struct_mutex); mem->mm_node = node; mem->mem_type = mem_type; mem->flags = drm_bo_type_flags(mem_type); return 0; } static int drm_bo_mem_space(drm_device_t *dev, drm_bo_mem_reg_t *mem, int no_wait) { drm_buffer_manager_t *bm= &dev->bm; drm_mem_type_manager_t *man; uint32_t num_prios = dev->driver->bo_driver->num_mem_type_prio; const uint32_t *prios = dev->driver->bo_driver->mem_type_prio; uint32_t i; uint32_t mem_type = DRM_BO_MEM_LOCAL; int type_found = 0; int type_ok = 0; int has_eagain = 0; drm_mm_node_t *node = NULL; int ret; for (i=0; imask ; if (!type_ok) continue; if (mem_type == DRM_BO_MEM_LOCAL) break; man = &bm->man[mem_type]; mutex_lock(&dev->struct_mutex); if (man->has_type && man->use_type) { type_found = 1; node = drm_mm_search_free(&man->manager, mem->num_pages, mem->page_alignment, 1); if (node) node = drm_mm_get_block(node, mem->num_pages, mem->page_alignment); } mutex_unlock(&dev->struct_mutex); if (node) break; } if ((type_ok && (mem_type == DRM_BO_MEM_LOCAL)) || node) { mem->mm_node = node; mem->mem_type = mem_type; mem->flags = drm_bo_type_flags(mem_type); return 0; } if (!type_found) { DRM_ERROR("Requested memory types are not supported\n"); return -EINVAL; } num_prios = dev->driver->bo_driver->num_mem_busy_prio; prios = dev->driver->bo_driver->mem_busy_prio; for (i=0; imask)) continue; man = &bm->man[mem_type]; ret = drm_bo_mem_force_space(dev, mem, mem_type, no_wait); if (ret == 0) return 0; if (ret == -EAGAIN) has_eagain = 1; } ret = (has_eagain) ? -EAGAIN : -ENOMEM; return ret; } static int drm_bo_new_flags(drm_device_t * dev, uint32_t flags, uint32_t new_mask, uint32_t hint, int init, uint32_t * n_flags, uint32_t * n_mask) { uint32_t new_flags = 0; uint32_t new_props; drm_buffer_manager_t *bm = &dev->bm; unsigned i; /* * First adjust the mask to take away nonexistant memory types. */ for (i = 0; i < DRM_BO_MEM_TYPES; ++i) { if (!bm->man[i].use_type) new_mask &= ~drm_bo_type_flags(i); } if ((new_mask & DRM_BO_FLAG_NO_EVICT) && !DRM_SUSER(DRM_CURPROC)) { DRM_ERROR ("DRM_BO_FLAG_NO_EVICT is only available to priviliged " "processes\n"); return -EPERM; } if (new_mask & DRM_BO_FLAG_BIND_CACHED) { if (((new_mask & DRM_BO_FLAG_MEM_TT) && !(bm->man[DRM_BO_MEM_TT].flags & _DRM_FLAG_MEMTYPE_CACHED) && ((new_mask & DRM_BO_FLAG_MEM_VRAM) && !(bm->man[DRM_BO_MEM_VRAM].flags & _DRM_FLAG_MEMTYPE_CACHED)))) { new_mask &= ~DRM_BO_FLAG_BIND_CACHED; } else { if (!(bm->man[DRM_BO_MEM_TT].flags & _DRM_FLAG_MEMTYPE_CACHED)) new_flags &= DRM_BO_FLAG_MEM_TT; if (!(bm->man[DRM_BO_MEM_VRAM].flags & _DRM_FLAG_MEMTYPE_CACHED)) new_flags &= DRM_BO_FLAG_MEM_VRAM; } } if ((new_mask & DRM_BO_FLAG_READ_CACHED) && !(new_mask & DRM_BO_FLAG_BIND_CACHED)) { if ((new_mask & DRM_BO_FLAG_NO_EVICT) && !(new_mask & DRM_BO_FLAG_MEM_LOCAL)) { DRM_ERROR ("Cannot read cached from a pinned VRAM / TT buffer\n"); return -EINVAL; } } /* * Determine new memory location: */ if (!(flags & new_mask & DRM_BO_MASK_MEM) || init) { new_flags = new_mask & DRM_BO_MASK_MEM; if (!new_flags) { DRM_ERROR("Invalid buffer object memory flags\n"); return -EINVAL; } if (new_flags & DRM_BO_FLAG_MEM_LOCAL) { if ((hint & DRM_BO_HINT_AVOID_LOCAL) && new_flags & (DRM_BO_FLAG_MEM_VRAM | DRM_BO_FLAG_MEM_TT)) { new_flags &= ~DRM_BO_FLAG_MEM_LOCAL; } else { new_flags = DRM_BO_FLAG_MEM_LOCAL; } } if (new_flags & DRM_BO_FLAG_MEM_TT) { if ((new_mask & DRM_BO_FLAG_PREFER_VRAM) && new_flags & DRM_BO_FLAG_MEM_VRAM) { new_flags = DRM_BO_FLAG_MEM_VRAM; } else { new_flags = DRM_BO_FLAG_MEM_TT; } } } else { new_flags = flags & DRM_BO_MASK_MEM; } new_props = new_mask & (DRM_BO_FLAG_EXE | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_READ); if (!new_props) { DRM_ERROR("Invalid buffer object rwx properties\n"); return -EINVAL; } new_flags |= new_mask & ~DRM_BO_MASK_MEM; if (((flags ^ new_flags) & DRM_BO_FLAG_BIND_CACHED) && (new_flags & DRM_BO_FLAG_NO_EVICT) && (flags & (DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_MEM_VRAM))) { if (!(flags & DRM_BO_FLAG_CACHED)) { DRM_ERROR ("Cannot change caching policy of pinned buffer\n"); return -EINVAL; } else { new_flags &= ~DRM_BO_FLAG_CACHED; } } *n_flags = new_flags; *n_mask = new_mask; return 0; } /* * Call dev->struct_mutex locked. */ drm_buffer_object_t *drm_lookup_buffer_object(drm_file_t * priv, uint32_t handle, int check_owner) { drm_user_object_t *uo; drm_buffer_object_t *bo; uo = drm_lookup_user_object(priv, handle); if (!uo || (uo->type != drm_buffer_type)) { DRM_ERROR("Could not find buffer object 0x%08x\n", handle); return NULL; } if (check_owner && priv != uo->owner) { if (!drm_lookup_ref_object(priv, uo, _DRM_REF_USE)) return NULL; } bo = drm_user_object_entry(uo, drm_buffer_object_t, base); atomic_inc(&bo->usage); return bo; } /* * Call bo->mutex locked. * Returns 1 if the buffer is currently rendered to or from. 0 otherwise. * Doesn't do any fence flushing as opposed to the drm_bo_busy function. */ static int drm_bo_quick_busy(drm_buffer_object_t * bo) { drm_fence_object_t *fence = bo->fence; BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED); if (fence) { drm_device_t *dev = bo->dev; if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(dev, fence); bo->fence = NULL; return 0; } return 1; } return 0; } /* * Call bo->mutex locked. * Returns 1 if the buffer is currently rendered to or from. 0 otherwise. */ static int drm_bo_busy(drm_buffer_object_t * bo) { drm_fence_object_t *fence = bo->fence; BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED); if (fence) { drm_device_t *dev = bo->dev; if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(dev, fence); bo->fence = NULL; return 0; } drm_fence_object_flush(dev, fence, DRM_FENCE_TYPE_EXE); if (drm_fence_object_signaled(fence, bo->fence_type)) { drm_fence_usage_deref_unlocked(dev, fence); bo->fence = NULL; return 0; } return 1; } return 0; } static int drm_bo_read_cached(drm_buffer_object_t * bo) { int ret = 0; BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED); if (bo->mem.mm_node) ret = drm_bo_evict(bo, DRM_BO_MEM_TT, 1, 0); return ret; } /* * Wait until a buffer is unmapped. */ static int drm_bo_wait_unmapped(drm_buffer_object_t * bo, int no_wait) { int ret = 0; if ((atomic_read(&bo->mapped) >= 0) && no_wait) return -EBUSY; DRM_WAIT_ON(ret, bo->event_queue, 3 * DRM_HZ, atomic_read(&bo->mapped) == -1); if (ret == -EINTR) ret = -EAGAIN; return ret; } static int drm_bo_check_unfenced(drm_buffer_object_t * bo) { int ret; mutex_lock(&bo->mutex); ret = (bo->priv_flags & _DRM_BO_FLAG_UNFENCED); mutex_unlock(&bo->mutex); return ret; } /* * Wait until a buffer, scheduled to be fenced moves off the unfenced list. * Until then, we cannot really do anything with it except delete it. * The unfenced list is a PITA, and the operations * 1) validating * 2) submitting commands * 3) fencing * Should really be an atomic operation. * We now "solve" this problem by keeping * the buffer "unfenced" after validating, but before fencing. */ static int drm_bo_wait_unfenced(drm_buffer_object_t * bo, int no_wait, int eagain_if_wait) { int ret = (bo->priv_flags & _DRM_BO_FLAG_UNFENCED); unsigned long _end = jiffies + 3 * DRM_HZ; if (ret && no_wait) return -EBUSY; else if (!ret) return 0; do { mutex_unlock(&bo->mutex); DRM_WAIT_ON(ret, bo->event_queue, 3 * DRM_HZ, !drm_bo_check_unfenced(bo)); mutex_lock(&bo->mutex); if (ret == -EINTR) return -EAGAIN; if (ret) { DRM_ERROR ("Error waiting for buffer to become fenced\n"); return ret; } ret = (bo->priv_flags & _DRM_BO_FLAG_UNFENCED); } while (ret && !time_after_eq(jiffies, _end)); if (ret) { DRM_ERROR("Timeout waiting for buffer to become fenced\n"); return ret; } if (eagain_if_wait) return -EAGAIN; return 0; } /* * Fill in the ioctl reply argument with buffer info. * Bo locked. */ static void drm_bo_fill_rep_arg(drm_buffer_object_t * bo, drm_bo_arg_reply_t * rep) { rep->handle = bo->base.hash.key; rep->flags = bo->mem.flags; rep->size = bo->mem.num_pages * PAGE_SIZE; rep->offset = bo->offset; rep->arg_handle = bo->map_list.user_token; rep->mask = bo->mem.mask; rep->buffer_start = bo->buffer_start; rep->fence_flags = bo->fence_type; rep->rep_flags = 0; rep->page_alignment = bo->mem.page_alignment; if ((bo->priv_flags & _DRM_BO_FLAG_UNFENCED) || drm_bo_quick_busy(bo)) { DRM_FLAG_MASKED(rep->rep_flags, DRM_BO_REP_BUSY, DRM_BO_REP_BUSY); } } /* * Wait for buffer idle and register that we've mapped the buffer. * Mapping is registered as a drm_ref_object with type _DRM_REF_TYPE1, * so that if the client dies, the mapping is automatically * unregistered. */ static int drm_buffer_object_map(drm_file_t * priv, uint32_t handle, uint32_t map_flags, unsigned hint, drm_bo_arg_reply_t * rep) { drm_buffer_object_t *bo; drm_device_t *dev = priv->head->dev; int ret = 0; int no_wait = hint & DRM_BO_HINT_DONT_BLOCK; mutex_lock(&dev->struct_mutex); bo = drm_lookup_buffer_object(priv, handle, 1); mutex_unlock(&dev->struct_mutex); if (!bo) return -EINVAL; mutex_lock(&bo->mutex); if (!(hint & DRM_BO_HINT_ALLOW_UNFENCED_MAP)) { ret = drm_bo_wait_unfenced(bo, no_wait, 0); if (ret) goto out; } /* * If this returns true, we are currently unmapped. * We need to do this test, because unmapping can * be done without the bo->mutex held. */ while (1) { if (atomic_inc_and_test(&bo->mapped)) { if (no_wait && drm_bo_busy(bo)) { atomic_dec(&bo->mapped); ret = -EBUSY; goto out; } ret = drm_bo_wait(bo, 0, 0, no_wait); if (ret) { atomic_dec(&bo->mapped); goto out; } if ((map_flags & DRM_BO_FLAG_READ) && (bo->mem.flags & DRM_BO_FLAG_READ_CACHED) && (!(bo->mem.flags & DRM_BO_FLAG_CACHED))) { drm_bo_read_cached(bo); } break; } else if ((map_flags & DRM_BO_FLAG_READ) && (bo->mem.flags & DRM_BO_FLAG_READ_CACHED) && (!(bo->mem.flags & DRM_BO_FLAG_CACHED))) { /* * We are already mapped with different flags. * need to wait for unmap. */ ret = drm_bo_wait_unmapped(bo, no_wait); if (ret) goto out; continue; } break; } mutex_lock(&dev->struct_mutex); ret = drm_add_ref_object(priv, &bo->base, _DRM_REF_TYPE1); mutex_unlock(&dev->struct_mutex); if (ret) { if (atomic_add_negative(-1, &bo->mapped)) DRM_WAKEUP(&bo->event_queue); } else drm_bo_fill_rep_arg(bo, rep); out: mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(bo); return ret; } static int drm_buffer_object_unmap(drm_file_t * priv, uint32_t handle) { drm_device_t *dev = priv->head->dev; drm_buffer_object_t *bo; drm_ref_object_t *ro; int ret = 0; mutex_lock(&dev->struct_mutex); bo = drm_lookup_buffer_object(priv, handle, 1); if (!bo) { ret = -EINVAL; goto out; } ro = drm_lookup_ref_object(priv, &bo->base, _DRM_REF_TYPE1); if (!ro) { ret = -EINVAL; goto out; } drm_remove_ref_object(priv, ro); drm_bo_usage_deref_locked(bo); out: mutex_unlock(&dev->struct_mutex); return ret; } /* * Call struct-sem locked. */ static void drm_buffer_user_object_unmap(drm_file_t * priv, drm_user_object_t * uo, drm_ref_t action) { drm_buffer_object_t *bo = drm_user_object_entry(uo, drm_buffer_object_t, base); /* * We DON'T want to take the bo->lock here, because we want to * hold it when we wait for unmapped buffer. */ BUG_ON(action != _DRM_REF_TYPE1); if (atomic_add_negative(-1, &bo->mapped)) DRM_WAKEUP(&bo->event_queue); } /* * bo->mutex locked. */ static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_mem_flags, int no_wait, int force_no_move) { drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; int ret = 0; drm_bo_mem_reg_t mem; /* * Flush outstanding fences. */ drm_bo_busy(bo); /* * Make sure we're not mapped. */ ret = drm_bo_wait_unmapped(bo, no_wait); if (ret) return ret; /* * Wait for outstanding fences. */ ret = drm_bo_wait(bo, 0, 0, no_wait); if (ret) return ret; mem.num_pages = bo->mem.num_pages; mem.size = mem.num_pages << PAGE_SHIFT; mem.mask = new_mem_flags; mem.page_alignment = bo->mem.page_alignment; mutex_lock(&bm->evict_mutex); mutex_lock(&dev->struct_mutex); list_del(&bo->lru); list_add_tail(&bo->lru,&bm->unfenced); DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_UNFENCED, _DRM_BO_FLAG_UNFENCED); mutex_unlock(&dev->struct_mutex); /* * Determine where to move the buffer. */ ret = drm_bo_mem_space(dev, &mem, no_wait); mutex_unlock(&bm->evict_mutex); if (ret) return ret; ret = drm_bo_handle_move_mem(bo, &mem, 0, no_wait); if (ret) { mutex_lock(&dev->struct_mutex); if (mem.mm_node) { drm_mm_put_block(mem.mm_node); mem.mm_node = NULL; } DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); list_del_init(&bo->lru); drm_bo_add_to_lru(bo, bm); mutex_unlock(&dev->struct_mutex); return ret; } return 0; } /* * bo locked. */ static int drm_buffer_object_validate(drm_buffer_object_t * bo, uint32_t new_flags, int move_unfenced, int no_wait) { drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; uint32_t flag_diff = (new_flags ^ bo->mem.flags); drm_bo_driver_t *driver = dev->driver->bo_driver; int ret; DRM_DEBUG("New flags 0x%08x, Old flags 0x%08x\n", new_flags, bo->mem.flags); ret = driver->fence_type(new_flags, &bo->fence_class, &bo->fence_type); if (ret) { DRM_ERROR("Driver did not support given buffer permissions\n"); return ret; } /* * Move out if we need to change caching policy. */ if ((flag_diff & DRM_BO_FLAG_BIND_CACHED) && !(bo->mem.flags & DRM_BO_FLAG_MEM_LOCAL)) { if (bo->mem.flags & (DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE)) { DRM_ERROR("Cannot change caching policy of " "pinned buffer.\n"); return -EINVAL; } ret = drm_bo_move_buffer(bo, DRM_BO_FLAG_MEM_LOCAL, no_wait, 0); if (ret) { if (ret != -EAGAIN) DRM_ERROR("Failed moving buffer.\n"); return ret; } } DRM_MASK_VAL(bo->mem.flags, DRM_BO_FLAG_BIND_CACHED, new_flags); flag_diff = (new_flags ^ bo->mem.flags); /* * Check whether we dropped no_move policy, and in that case, * release reserved manager regions. */ if ((flag_diff & DRM_BO_FLAG_NO_MOVE) && !(new_flags & DRM_BO_FLAG_NO_MOVE)) { mutex_lock(&dev->struct_mutex); if (bo->mem.mm_node) { drm_mm_put_block(bo->mem.mm_node); bo->mem.mm_node = NULL; } mutex_unlock(&dev->struct_mutex); } /* * Check whether we need to move buffer. */ if ((bo->type != drm_bo_type_fake) && (flag_diff & DRM_BO_MASK_MEM)) { ret = drm_bo_move_buffer(bo, new_flags, no_wait, 1); if (ret) { if (ret != -EAGAIN) DRM_ERROR("Failed moving buffer.\n"); return ret; } } if (bo->mem.mem_type == DRM_BO_MEM_LOCAL && bo->ttm == NULL) { ret = drm_bo_add_ttm(bo); if (ret) return ret; } if (move_unfenced) { /* * Place on unfenced list. */ DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_UNFENCED, _DRM_BO_FLAG_UNFENCED); mutex_lock(&dev->struct_mutex); list_del(&bo->lru); list_add_tail(&bo->lru, &bm->unfenced); mutex_unlock(&dev->struct_mutex); } else { DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); mutex_lock(&dev->struct_mutex); list_del_init(&bo->lru); drm_bo_add_to_lru(bo, bm); mutex_unlock(&dev->struct_mutex); } bo->mem.flags = new_flags; return 0; } static int drm_bo_handle_validate(drm_file_t * priv, uint32_t handle, uint32_t flags, uint32_t mask, uint32_t hint, drm_bo_arg_reply_t * rep) { drm_buffer_object_t *bo; drm_device_t *dev = priv->head->dev; int ret; int no_wait = hint & DRM_BO_HINT_DONT_BLOCK; uint32_t new_flags; bo = drm_lookup_buffer_object(priv, handle, 1); if (!bo) { return -EINVAL; } mutex_lock(&bo->mutex); ret = drm_bo_wait_unfenced(bo, no_wait, 0); if (ret) goto out; ret = drm_bo_new_flags(dev, bo->mem.flags, (flags & mask) | (bo->mem.mask & ~mask), hint, 0, &new_flags, &bo->mem.mask); if (ret) goto out; ret = drm_buffer_object_validate(bo, new_flags, !(hint & DRM_BO_HINT_DONT_FENCE), no_wait); drm_bo_fill_rep_arg(bo, rep); out: mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(bo); return ret; } static int drm_bo_handle_info(drm_file_t * priv, uint32_t handle, drm_bo_arg_reply_t * rep) { drm_buffer_object_t *bo; bo = drm_lookup_buffer_object(priv, handle, 1); if (!bo) { return -EINVAL; } mutex_lock(&bo->mutex); if (!(bo->priv_flags & _DRM_BO_FLAG_UNFENCED)) (void)drm_bo_busy(bo); drm_bo_fill_rep_arg(bo, rep); mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(bo); return 0; } static int drm_bo_handle_wait(drm_file_t * priv, uint32_t handle, uint32_t hint, drm_bo_arg_reply_t * rep) { drm_buffer_object_t *bo; int no_wait = hint & DRM_BO_HINT_DONT_BLOCK; int ret; bo = drm_lookup_buffer_object(priv, handle, 1); if (!bo) { return -EINVAL; } mutex_lock(&bo->mutex); ret = drm_bo_wait_unfenced(bo, no_wait, 0); if (ret) goto out; ret = drm_bo_wait(bo, hint & DRM_BO_HINT_WAIT_LAZY, 0, no_wait); if (ret) goto out; drm_bo_fill_rep_arg(bo, rep); out: mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(bo); return ret; } /* * Transfer a buffer object's memory and LRU status to a newly * created object. User-space references remains with the old * object. Call bo->mutex locked. */ int drm_buffer_object_transfer(drm_buffer_object_t *bo, drm_buffer_object_t **new_obj) { drm_buffer_object_t *fbo; drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; fbo = drm_ctl_calloc(1, sizeof(*bo), DRM_MEM_BUFOBJ); if (!fbo) return -ENOMEM; *fbo = *bo; mutex_init(&fbo->mutex); mutex_lock(&fbo->mutex); mutex_lock(&dev->struct_mutex); INIT_LIST_HEAD(&fbo->ddestroy); INIT_LIST_HEAD(&fbo->lru); list_splice_init(&bo->lru, &fbo->lru); bo->mem.mm_node = NULL; bo->ttm = NULL; bo->fence = NULL; bo->mem.flags = 0; fbo->mem.mm_node->private = (void *)fbo; atomic_set(&fbo->usage, 1); atomic_inc(&bm->count); mutex_unlock(&dev->struct_mutex); mutex_unlock(&fbo->mutex); *new_obj = fbo; return 0; } int drm_buffer_object_create(drm_file_t * priv, unsigned long size, drm_bo_type_t type, uint32_t mask, uint32_t hint, uint32_t page_alignment, unsigned long buffer_start, drm_buffer_object_t ** buf_obj) { drm_device_t *dev = priv->head->dev; drm_buffer_manager_t *bm = &dev->bm; drm_buffer_object_t *bo; int ret = 0; uint32_t new_flags; unsigned long num_pages; if ((buffer_start & ~PAGE_MASK) && (type != drm_bo_type_fake)) { DRM_ERROR("Invalid buffer object start.\n"); return -EINVAL; } num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; if (num_pages == 0) { DRM_ERROR("Illegal buffer object size.\n"); return -EINVAL; } bo = drm_ctl_calloc(1, sizeof(*bo), DRM_MEM_BUFOBJ); if (!bo) return -ENOMEM; mutex_init(&bo->mutex); mutex_lock(&bo->mutex); atomic_set(&bo->usage, 1); atomic_set(&bo->mapped, -1); DRM_INIT_WAITQUEUE(&bo->event_queue); INIT_LIST_HEAD(&bo->lru); INIT_LIST_HEAD(&bo->ddestroy); #ifdef DRM_ODD_MM_COMPAT INIT_LIST_HEAD(&bo->p_mm_list); INIT_LIST_HEAD(&bo->vma_list); #endif bo->dev = dev; bo->type = type; bo->mem.num_pages = num_pages; bo->mem.mm_node = NULL; bo->mem.page_alignment = page_alignment; if (bo->type == drm_bo_type_fake) { bo->offset = buffer_start; bo->buffer_start = 0; } else { bo->buffer_start = buffer_start; } bo->priv_flags = 0; bo->mem.flags = DRM_BO_FLAG_MEM_LOCAL | DRM_BO_FLAG_CACHED; atomic_inc(&bm->count); ret = drm_bo_new_flags(dev, bo->mem.flags, mask, hint, 1, &new_flags, &bo->mem.mask); if (ret) goto out_err; if (bo->type == drm_bo_type_dc) { mutex_lock(&dev->struct_mutex); ret = drm_bo_setup_vm_locked(bo); mutex_unlock(&dev->struct_mutex); if (ret) goto out_err; } ret = drm_buffer_object_validate(bo, new_flags, 0, hint & DRM_BO_HINT_DONT_BLOCK); if (ret) goto out_err; mutex_unlock(&bo->mutex); *buf_obj = bo; return 0; out_err: mutex_unlock(&bo->mutex); drm_bo_usage_deref_unlocked(bo); return ret; } static int drm_bo_add_user_object(drm_file_t * priv, drm_buffer_object_t * bo, int shareable) { drm_device_t *dev = priv->head->dev; int ret; mutex_lock(&dev->struct_mutex); ret = drm_add_user_object(priv, &bo->base, shareable); if (ret) goto out; bo->base.remove = drm_bo_base_deref_locked; bo->base.type = drm_buffer_type; bo->base.ref_struct_locked = NULL; bo->base.unref = drm_buffer_user_object_unmap; out: mutex_unlock(&dev->struct_mutex); return ret; } static int drm_bo_lock_test(drm_device_t * dev, struct file *filp) { LOCK_TEST_WITH_RETURN(dev, filp); return 0; } int drm_bo_ioctl(DRM_IOCTL_ARGS) { DRM_DEVICE; drm_bo_arg_t arg; drm_bo_arg_request_t *req = &arg.d.req; drm_bo_arg_reply_t rep; unsigned long next; drm_user_object_t *uo; drm_buffer_object_t *entry; if (!dev->bm.initialized) { DRM_ERROR("Buffer object manager is not initialized.\n"); return -EINVAL; } do { DRM_COPY_FROM_USER_IOCTL(arg, (void __user *)data, sizeof(arg)); if (arg.handled) { data = arg.next; continue; } rep.ret = 0; switch (req->op) { case drm_bo_create: rep.ret = drm_buffer_object_create(priv, req->size, req->type, req->mask, req->hint, req->page_alignment, req->buffer_start, &entry); if (rep.ret) break; rep.ret = drm_bo_add_user_object(priv, entry, req-> mask & DRM_BO_FLAG_SHAREABLE); if (rep.ret) drm_bo_usage_deref_unlocked(entry); if (rep.ret) break; mutex_lock(&entry->mutex); drm_bo_fill_rep_arg(entry, &rep); mutex_unlock(&entry->mutex); break; case drm_bo_unmap: rep.ret = drm_buffer_object_unmap(priv, req->handle); break; case drm_bo_map: rep.ret = drm_buffer_object_map(priv, req->handle, req->mask, req->hint, &rep); break; case drm_bo_destroy: mutex_lock(&dev->struct_mutex); uo = drm_lookup_user_object(priv, req->handle); if (!uo || (uo->type != drm_buffer_type) || uo->owner != priv) { mutex_unlock(&dev->struct_mutex); rep.ret = -EINVAL; break; } rep.ret = drm_remove_user_object(priv, uo); mutex_unlock(&dev->struct_mutex); break; case drm_bo_reference: rep.ret = drm_user_object_ref(priv, req->handle, drm_buffer_type, &uo); if (rep.ret) break; mutex_lock(&dev->struct_mutex); uo = drm_lookup_user_object(priv, req->handle); entry = drm_user_object_entry(uo, drm_buffer_object_t, base); atomic_dec(&entry->usage); mutex_unlock(&dev->struct_mutex); mutex_lock(&entry->mutex); drm_bo_fill_rep_arg(entry, &rep); mutex_unlock(&entry->mutex); break; case drm_bo_unreference: rep.ret = drm_user_object_unref(priv, req->handle, drm_buffer_type); break; case drm_bo_validate: rep.ret = drm_bo_lock_test(dev, filp); if (rep.ret) break; rep.ret = drm_bo_handle_validate(priv, req->handle, req->mask, req->arg_handle, req->hint, &rep); break; case drm_bo_fence: rep.ret = drm_bo_lock_test(dev, filp); if (rep.ret) break; /**/ break; case drm_bo_info: rep.ret = drm_bo_handle_info(priv, req->handle, &rep); break; case drm_bo_wait_idle: rep.ret = drm_bo_handle_wait(priv, req->handle, req->hint, &rep); break; case drm_bo_ref_fence: rep.ret = -EINVAL; DRM_ERROR("Function is not implemented yet.\n"); default: rep.ret = -EINVAL; } next = arg.next; /* * A signal interrupted us. Make sure the ioctl is restartable. */ if (rep.ret == -EAGAIN) return -EAGAIN; arg.handled = 1; arg.d.rep = rep; DRM_COPY_TO_USER_IOCTL((void __user *)data, arg, sizeof(arg)); data = next; } while (data); return 0; } /* * dev->struct_sem locked. */ static int drm_bo_force_list_clean(drm_device_t * dev, struct list_head *head, unsigned mem_type, int force_no_move, int allow_errors) { drm_buffer_manager_t *bm = &dev->bm; struct list_head *list, *next, *prev; drm_buffer_object_t *entry; int ret; int clean; retry: clean = 1; list_for_each_safe(list, next, head) { prev = list->prev; entry = list_entry(list, drm_buffer_object_t, lru); atomic_inc(&entry->usage); mutex_unlock(&dev->struct_mutex); mutex_lock(&entry->mutex); mutex_lock(&dev->struct_mutex); if (prev != list->prev || next != list->next) { mutex_unlock(&entry->mutex); drm_bo_usage_deref_locked(entry); goto retry; } if (entry->mem.mm_node) { clean = 0; /* * Expire the fence. */ mutex_unlock(&dev->struct_mutex); if (entry->fence && bm->nice_mode) { unsigned long _end = jiffies + 3 * DRM_HZ; do { ret = drm_bo_wait(entry, 0, 1, 0); if (ret && allow_errors) { if (ret == -EINTR) ret = -EAGAIN; goto out_err; } } while (ret && !time_after_eq(jiffies, _end)); if (entry->fence) { bm->nice_mode = 0; DRM_ERROR("Detected GPU hang or " "fence manager was taken down. " "Evicting waiting buffers\n"); } } if (entry->fence) { drm_fence_usage_deref_unlocked(dev, entry->fence); entry->fence = NULL; } DRM_MASK_VAL(entry->priv_flags, _DRM_BO_FLAG_UNFENCED, 0); if (force_no_move) { DRM_MASK_VAL(entry->mem.flags, DRM_BO_FLAG_NO_MOVE, 0); } if (entry->mem.flags & DRM_BO_FLAG_NO_EVICT) { DRM_ERROR("A DRM_BO_NO_EVICT buffer present at " "cleanup. Removing flag and evicting.\n"); entry->mem.flags &= ~DRM_BO_FLAG_NO_EVICT; entry->mem.mask &= ~DRM_BO_FLAG_NO_EVICT; } ret = drm_bo_evict(entry, mem_type, 1, force_no_move); if (ret) { if (allow_errors) { goto out_err; } else { DRM_ERROR("Aargh. Eviction failed.\n"); } } mutex_lock(&dev->struct_mutex); } mutex_unlock(&entry->mutex); drm_bo_usage_deref_locked(entry); if (prev != list->prev || next != list->next) { goto retry; } } if (!clean) goto retry; return 0; out_err: mutex_unlock(&entry->mutex); drm_bo_usage_deref_unlocked(entry); mutex_lock(&dev->struct_mutex); return ret; } int drm_bo_clean_mm(drm_device_t * dev, unsigned mem_type) { drm_buffer_manager_t *bm = &dev->bm; drm_mem_type_manager_t *man = &bm->man[mem_type]; drm_mem_type_manager_t *local_man = &bm->man[DRM_BO_MEM_LOCAL]; int ret = -EINVAL; if (mem_type >= DRM_BO_MEM_TYPES) { DRM_ERROR("Illegal memory type %d\n", mem_type); return ret; } if (!man->has_type) { DRM_ERROR("Trying to take down uninitialized " "memory manager type\n"); return ret; } man->use_type = 0; man->has_type = 0; ret = 0; if (mem_type > 0) { /* * Throw out unfenced buffers. */ drm_bo_force_list_clean(dev, &bm->unfenced, mem_type, 1, 0); /* * Throw out evicted no-move buffers. */ drm_bo_force_list_clean(dev, &local_man->pinned, mem_type, 1, 0); drm_bo_force_list_clean(dev, &man->lru, mem_type, 1, 0); drm_bo_force_list_clean(dev, &man->pinned, mem_type, 1, 0); if (drm_mm_clean(&man->manager)) { drm_mm_takedown(&man->manager); } else { ret = -EBUSY; } } return ret; } static int drm_bo_lock_mm(drm_device_t * dev, unsigned mem_type) { int ret; drm_buffer_manager_t *bm = &dev->bm; drm_mem_type_manager_t *man = &bm->man[mem_type]; if (mem_type == 0 || mem_type >= DRM_BO_MEM_TYPES) { DRM_ERROR("Illegal memory manager memory type %u,\n", mem_type); return -EINVAL; } ret = drm_bo_force_list_clean(dev, &bm->unfenced, mem_type, 0, 1); if (ret) return ret; ret = drm_bo_force_list_clean(dev, &man->lru, mem_type, 0, 1); if (ret) return ret; ret = drm_bo_force_list_clean(dev, &man->pinned, mem_type, 0, 1); return ret; } static int drm_bo_init_mm(drm_device_t * dev, unsigned type, unsigned long p_offset, unsigned long p_size) { drm_buffer_manager_t *bm = &dev->bm; int ret = -EINVAL; drm_mem_type_manager_t *man; if (type >= DRM_BO_MEM_TYPES) { DRM_ERROR("Illegal memory type %d\n", type); return ret; } man = &bm->man[type]; if (man->has_type) { DRM_ERROR("Memory manager already initialized for type %d\n", type); return ret; } ret = dev->driver->bo_driver->init_mem_type(dev, type, man); if (ret) return ret; ret = 0; if (type != DRM_BO_MEM_LOCAL) { if (!p_size) { DRM_ERROR("Zero size memory manager type %d\n", type); return ret; } ret = drm_mm_init(&man->manager, p_offset, p_size); if (ret) return ret; } man->has_type = 1; man->use_type = 1; INIT_LIST_HEAD(&man->lru); INIT_LIST_HEAD(&man->pinned); return 0; } /* * This is called from lastclose, so we don't need to bother about * any clients still running when we set the initialized flag to zero. */ int drm_bo_driver_finish(drm_device_t * dev) { drm_buffer_manager_t *bm = &dev->bm; int ret = 0; unsigned i = DRM_BO_MEM_TYPES; drm_mem_type_manager_t *man; mutex_lock(&dev->bm.init_mutex); mutex_lock(&dev->struct_mutex); if (!bm->initialized) goto out; bm->initialized = 0; while (i--) { man = &bm->man[i]; if (man->has_type) { man->use_type = 0; if ((i != DRM_BO_MEM_LOCAL) && drm_bo_clean_mm(dev, i)) { ret = -EBUSY; DRM_ERROR("DRM memory manager type %d " "is not clean.\n", i); } man->has_type = 0; } } mutex_unlock(&dev->struct_mutex); if (!cancel_delayed_work(&bm->wq)) { flush_scheduled_work(); } mutex_lock(&dev->struct_mutex); drm_bo_delayed_delete(dev, 1); if (list_empty(&bm->ddestroy)) { DRM_DEBUG("Delayed destroy list was clean\n"); } if (list_empty(&bm->man[0].lru)) { DRM_DEBUG("Swap list was clean\n"); } if (list_empty(&bm->man[0].pinned)) { DRM_DEBUG("NO_MOVE list was clean\n"); } if (list_empty(&bm->unfenced)) { DRM_DEBUG("Unfenced list was clean\n"); } out: mutex_unlock(&dev->struct_mutex); mutex_unlock(&dev->bm.init_mutex); return ret; } int drm_bo_driver_init(drm_device_t * dev) { drm_bo_driver_t *driver = dev->driver->bo_driver; drm_buffer_manager_t *bm = &dev->bm; int ret = -EINVAL; mutex_lock(&dev->bm.init_mutex); mutex_lock(&dev->struct_mutex); if (!driver) goto out_unlock; /* * Initialize the system memory buffer type. * Other types need to be driver / IOCTL initialized. */ ret = drm_bo_init_mm(dev, 0, 0, 0); if (ret) goto out_unlock; #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) INIT_WORK(&bm->wq, &drm_bo_delayed_workqueue, dev); #else INIT_DELAYED_WORK(&bm->wq, drm_bo_delayed_workqueue); #endif bm->initialized = 1; bm->nice_mode = 1; atomic_set(&bm->count, 0); bm->cur_pages = 0; INIT_LIST_HEAD(&bm->unfenced); INIT_LIST_HEAD(&bm->ddestroy); out_unlock: mutex_unlock(&dev->struct_mutex); mutex_unlock(&dev->bm.init_mutex); return ret; } EXPORT_SYMBOL(drm_bo_driver_init); int drm_mm_init_ioctl(DRM_IOCTL_ARGS) { DRM_DEVICE; int ret = 0; drm_mm_init_arg_t arg; drm_buffer_manager_t *bm = &dev->bm; drm_bo_driver_t *driver = dev->driver->bo_driver; if (!driver) { DRM_ERROR("Buffer objects are not supported by this driver\n"); return -EINVAL; } DRM_COPY_FROM_USER_IOCTL(arg, (void __user *)data, sizeof(arg)); switch (arg.req.op) { case mm_init: ret = -EINVAL; mutex_lock(&dev->bm.init_mutex); mutex_lock(&dev->struct_mutex); if (!bm->initialized) { DRM_ERROR("DRM memory manager was not initialized.\n"); break; } if (arg.req.mem_type == 0) { DRM_ERROR ("System memory buffers already initialized.\n"); break; } ret = drm_bo_init_mm(dev, arg.req.mem_type, arg.req.p_offset, arg.req.p_size); break; case mm_takedown: LOCK_TEST_WITH_RETURN(dev, filp); mutex_lock(&dev->bm.init_mutex); mutex_lock(&dev->struct_mutex); ret = -EINVAL; if (!bm->initialized) { DRM_ERROR("DRM memory manager was not initialized\n"); break; } if (arg.req.mem_type == 0) { DRM_ERROR("No takedown for System memory buffers.\n"); break; } ret = 0; if (drm_bo_clean_mm(dev, arg.req.mem_type)) { DRM_ERROR("Memory manager type %d not clean. " "Delaying takedown\n", arg.req.mem_type); } break; case mm_lock: LOCK_TEST_WITH_RETURN(dev, filp); mutex_lock(&dev->bm.init_mutex); mutex_lock(&dev->struct_mutex); ret = drm_bo_lock_mm(dev, arg.req.mem_type); break; case mm_unlock: LOCK_TEST_WITH_RETURN(dev, filp); mutex_lock(&dev->bm.init_mutex); mutex_lock(&dev->struct_mutex); ret = 0; break; default: DRM_ERROR("Function not implemented yet\n"); return -EINVAL; } mutex_unlock(&dev->struct_mutex); mutex_unlock(&dev->bm.init_mutex); if (ret) return ret; DRM_COPY_TO_USER_IOCTL((void __user *)data, arg, sizeof(arg)); return 0; } /* * buffer object vm functions. */ int drm_mem_reg_is_pci(drm_device_t *dev, drm_bo_mem_reg_t *mem) { drm_buffer_manager_t *bm = &dev->bm; drm_mem_type_manager_t *man = &bm->man[mem->mem_type]; if (!(man->flags & _DRM_FLAG_MEMTYPE_FIXED)) { if (mem->mem_type == DRM_BO_MEM_LOCAL) return 0; if (man->flags & _DRM_FLAG_MEMTYPE_CMA) return 0; if ((mem->mask & DRM_BO_FLAG_BIND_CACHED) && (man->flags & _DRM_FLAG_MEMTYPE_CACHED)) return 0; } return 1; } EXPORT_SYMBOL(drm_mem_reg_is_pci); /** * \c Get the PCI offset for the buffer object memory. * * \param bo The buffer object. * \param bus_base On return the base of the PCI region * \param bus_offset On return the byte offset into the PCI region * \param bus_size On return the byte size of the buffer object or zero if * the buffer object memory is not accessible through a PCI region. * \return Failure indication. * * Returns -EINVAL if the buffer object is currently not mappable. * Otherwise returns zero. */ int drm_bo_pci_offset(drm_device_t *dev, drm_bo_mem_reg_t *mem, unsigned long *bus_base, unsigned long *bus_offset, unsigned long *bus_size) { drm_buffer_manager_t *bm = &dev->bm; drm_mem_type_manager_t *man = &bm->man[mem->mem_type]; *bus_size = 0; if (!(man->flags & _DRM_FLAG_MEMTYPE_MAPPABLE)) return -EINVAL; if (drm_mem_reg_is_pci(dev, mem)) { *bus_offset = mem->mm_node->start << PAGE_SHIFT; *bus_size = mem->num_pages << PAGE_SHIFT; *bus_base = man->io_offset; } return 0; } /** * \c Return a kernel virtual address to the buffer object PCI memory. * * \param bo The buffer object. * \return Failure indication. * * Returns -EINVAL if the buffer object is currently not mappable. * Returns -ENOMEM if the ioremap operation failed. * Otherwise returns zero. * * After a successfull call, bo->iomap contains the virtual address, or NULL * if the buffer object content is not accessible through PCI space. * Call bo->mutex locked. */ #if 0 int drm_mem_reg_ioremap(drm_bo_mem_reg_t *mem) { drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; drm_mem_type_manager_t *man = &bm->man[bo->mem.mem_type]; unsigned long bus_offset; unsigned long bus_size; unsigned long bus_base; int ret; BUG_ON(bo->iomap); ret = drm_bo_pci_offset(bo, &bus_base, &bus_offset, &bus_size); if (ret || bus_size == 0) return ret; if (!(man->flags & _DRM_FLAG_NEEDS_IOREMAP)) bo->iomap = (void *) (((u8 *)man->io_addr) + bus_offset); else { bo->iomap = ioremap_nocache(bus_base + bus_offset, bus_size); if (bo->iomap) return -ENOMEM; } return 0; } /** * \c Unmap mapping obtained using drm_bo_ioremap * * \param bo The buffer object. * * Call bo->mutex locked. */ void drm_bo_iounmap(drm_buffer_object_t *bo) { drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm; drm_mem_type_manager_t *man; bm = &dev->bm; man = &bm->man[bo->mem.mem_type]; if (bo->iomap && (man->flags & _DRM_FLAG_NEEDS_IOREMAP)) iounmap(bo->iomap); bo->iomap = NULL; } #endif /** * \c Kill all user-space virtual mappings of this buffer object. * * \param bo The buffer object. * * Call bo->mutex locked. */ void drm_bo_unmap_virtual(drm_buffer_object_t *bo) { drm_device_t *dev = bo->dev; loff_t offset = ((loff_t) bo->map_list.hash.key) << PAGE_SHIFT; loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT; unmap_mapping_range(dev->dev_mapping, offset, holelen, 1); } static void drm_bo_takedown_vm_locked(drm_buffer_object_t *bo) { drm_map_list_t *list = &bo->map_list; drm_local_map_t *map; drm_device_t *dev = bo->dev; if (list->user_token) { drm_ht_remove_item(&dev->map_hash, &list->hash); list->user_token = 0; } if (list->file_offset_node) { drm_mm_put_block(list->file_offset_node); list->file_offset_node = NULL; } map = list->map; if (!map) return; drm_ctl_free(map, sizeof(*map), DRM_MEM_BUFOBJ); list->map = NULL; list->user_token = 0ULL; drm_bo_usage_deref_locked(bo); } static int drm_bo_setup_vm_locked(drm_buffer_object_t *bo) { drm_map_list_t *list = &bo->map_list; drm_local_map_t *map; drm_device_t *dev = bo->dev; list->map = drm_ctl_calloc(1, sizeof(*map), DRM_MEM_BUFOBJ); if (!list->map) return -ENOMEM; map = list->map; map->offset = 0; map->type = _DRM_TTM; map->flags = _DRM_REMOVABLE; map->size = bo->mem.num_pages * PAGE_SIZE; atomic_inc(&bo->usage); map->handle = (void *) bo; list->file_offset_node = drm_mm_search_free(&dev->offset_manager, bo->mem.num_pages, 0, 0); if (!list->file_offset_node) { drm_bo_takedown_vm_locked(bo); return -ENOMEM; } list->file_offset_node = drm_mm_get_block(list->file_offset_node, bo->mem.num_pages, 0); list->hash.key = list->file_offset_node->start; if (drm_ht_insert_item(&dev->map_hash, &list->hash)) { drm_bo_takedown_vm_locked(bo); return -ENOMEM; } list->user_token = ((drm_u64_t) list->hash.key) << PAGE_SHIFT; return 0; }