diff options
| -rw-r--r-- | linux-core/i915_gem.c | 175 | ||||
| -rw-r--r-- | shared-core/i915_dma.c | 2 | ||||
| -rw-r--r-- | shared-core/i915_drv.h | 10 | ||||
| -rw-r--r-- | shared-core/i915_irq.c | 2 |
4 files changed, 143 insertions, 46 deletions
diff --git a/linux-core/i915_gem.c b/linux-core/i915_gem.c index 4534dae2..d630b0b2 100644 --- a/linux-core/i915_gem.c +++ b/linux-core/i915_gem.c @@ -74,6 +74,31 @@ i915_gem_object_free_page_list(struct drm_gem_object *obj) } /** + * Ensures that all rendering to the object has completed and the object is + * safe to unbind from the GTT. + */ +static int +i915_gem_object_wait_rendering(struct drm_gem_object *obj) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_gem_object *obj_priv = obj->driver_private; + int ret; + + /* If there is rendering queued on the buffer being evicted, wait for + * it. + */ + if (obj_priv->last_rendering_cookie != 0) { + ret = i915_wait_irq(dev, obj_priv->last_rendering_cookie); + if (ret != 0) + return ret; + /* Clear it now that we know it's passed. */ + obj_priv->last_rendering_cookie = 0; + } + + return 0; +} + +/** * Unbinds an object from the GTT aperture. */ static void @@ -88,6 +113,8 @@ i915_gem_object_unbind(struct drm_gem_object *obj) if (obj_priv->gtt_space == NULL) return; + i915_gem_object_wait_rendering(obj); + if (obj_priv->agp_mem != NULL) { drm_unbind_agp(obj_priv->agp_mem); drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE); @@ -97,8 +124,10 @@ i915_gem_object_unbind(struct drm_gem_object *obj) drm_memrange_put_block(obj_priv->gtt_space); obj_priv->gtt_space = NULL; + list_del_init(&obj_priv->gtt_lru_entry); } +#if 0 static void i915_gem_dump_page (struct page *page, uint32_t start, uint32_t end, uint32_t bias, uint32_t mark) { @@ -139,6 +168,39 @@ i915_gem_dump_object (struct drm_gem_object *obj, int len, const char *where, ui } } } +#endif + +static int +i915_gem_evict_something(struct drm_device *dev) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_gem_object *obj; + struct drm_i915_gem_object *obj_priv; + int ret; + + /* Find the LRU buffer. */ + BUG_ON(!list_empty(&dev_priv->mm.gtt_lru)); + obj_priv = list_entry(dev_priv->mm.gtt_lru.prev, + struct drm_i915_gem_object, + gtt_lru_entry); + obj = obj_priv->obj; + + /* Only unpinned buffers should be on this list. */ + BUG_ON(obj_priv->pin_count != 0); + + /* Do this separately from the wait_rendering in + * i915_gem_object_unbind() because we want to catch interrupts and + * return. + */ + ret = i915_gem_object_wait_rendering(obj); + if (ret != 0) + return ret; + + /* Wait on the rendering and unbind the buffer. */ + i915_gem_object_unbind(obj); + + return 0; +} /** * Finds free space in the GTT aperture and binds the object there. @@ -150,7 +212,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = obj->driver_private; struct drm_memrange_node *free_space; - int page_count, i; + int page_count, i, ret; if (alignment == 0) alignment = PAGE_SIZE; @@ -159,18 +221,31 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) return -EINVAL; } + search_free: free_space = drm_memrange_search_free(&dev_priv->mm.gtt_space, obj->size, alignment, 0); - if (free_space == NULL) - return -ENOMEM; - obj_priv->gtt_space = drm_memrange_get_block(free_space, - obj->size, - alignment); - if (obj_priv->gtt_space == NULL) - return -ENOMEM; - obj_priv->gtt_space->private = obj; - obj_priv->gtt_offset = obj_priv->gtt_space->start; + if (free_space != NULL) { + obj_priv->gtt_space = + drm_memrange_get_block(free_space, obj->size, + alignment); + if (obj_priv->gtt_space != NULL) { + obj_priv->gtt_space->private = obj; + obj_priv->gtt_offset = obj_priv->gtt_space->start; + } + } + if (obj_priv->gtt_space == NULL) { + /* If the gtt is empty and we're still having trouble + * fitting our object in, we're out of memory. + */ + if (list_empty(&dev_priv->mm.gtt_lru)) + return -ENOMEM; + + ret = i915_gem_evict_something(dev); + if (ret != 0) + return ret; + goto search_free; + } #if WATCH_BUF DRM_INFO ("Binding object of size %d at 0x%08x\n", obj->size, obj_priv->gtt_offset); @@ -229,6 +304,7 @@ i915_gem_reloc_and_validate_object(struct drm_gem_object *obj, struct drm_i915_gem_validate_entry *entry) { struct drm_device *dev = obj->dev; + drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_relocation_entry reloc; struct drm_i915_gem_relocation_entry __user *relocs; struct drm_i915_gem_object *obj_priv = obj->driver_private; @@ -244,6 +320,12 @@ i915_gem_reloc_and_validate_object(struct drm_gem_object *obj, } entry->buffer_offset = obj_priv->gtt_offset; + if (obj_priv->pin_count == 0) { + /* Move our buffer to the head of the LRU. */ + list_del_init(&obj_priv->gtt_lru_entry); + list_add(&obj_priv->gtt_lru_entry, &dev_priv->mm.gtt_lru); + } + relocs = (struct drm_i915_gem_relocation_entry __user *) (uintptr_t) entry->relocs_ptr; /* Apply the relocations, using the GTT aperture to avoid cache * flushing requirements. @@ -331,8 +413,8 @@ i915_gem_reloc_and_validate_object(struct drm_gem_object *obj, return 0; } -static int -i915_gem_sync(struct drm_device *dev) +static void +i915_gem_flush(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; RING_LOCALS; @@ -341,8 +423,6 @@ i915_gem_sync(struct drm_device *dev) OUT_RING(CMD_MI_FLUSH | MI_READ_FLUSH | MI_EXE_FLUSH); OUT_RING(0); /* noop */ ADVANCE_LP_RING(); - - return i915_quiescent(dev); } static int @@ -412,6 +492,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, struct drm_gem_object **object_list; int ret, i; uint64_t exec_offset; + uint32_t cookie; LOCK_TEST_WITH_RETURN(dev, file_priv); @@ -421,14 +502,6 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, #endif i915_kernel_lost_context(dev); - /* Big hammer: flush and idle the hardware so we can map things in/out. - */ - ret = i915_gem_sync(dev); - if (ret != 0) { - DRM_ERROR ("i915_gem_sync failed %d\n", ret); - return ret; - } - /* Copy in the validate list from userland */ validate_list = drm_calloc(sizeof(*validate_list), args->buffer_count, DRM_MEM_DRIVER); @@ -468,6 +541,21 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, } } + for (i = 0; i < args->buffer_count; i++) { + struct drm_gem_object *obj = object_list[i]; + struct drm_i915_gem_object *obj_priv = obj->driver_private; + + if (obj_priv->gtt_space == NULL) { + /* We evicted the buffer in the process of validating + * our set of buffers in. We could try to recover by + * kicking them everything out and trying again from + * the start. + */ + ret = -ENOMEM; + goto err; + } + } + exec_offset = validate_list[args->buffer_count - 1].buffer_offset; /* make sure all previous memory operations have passed */ @@ -487,6 +575,25 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, goto err; } + /* Flush the rendering. We want this flush to go away, which will + * require intelligent cache management. + */ + i915_gem_flush(dev); + + /* Get a cookie representing the flush of the current buffer, which we + * can wait on. We would like to mitigate these interrupts, likely by + * only flushing occasionally (so that we have *some* interrupts + * representing completion of buffers that we can wait on when trying + * to clear up gtt space). + */ + cookie = i915_emit_irq(dev); + for (i = 0; i < args->buffer_count; i++) { + struct drm_gem_object *obj = object_list[i]; + struct drm_i915_gem_object *obj_priv = obj->driver_private; + + obj_priv->last_rendering_cookie = cookie; + } + /* Copy the new buffer offsets back to the user's validate list. */ ret = copy_to_user((struct drm_i915_relocation_entry __user*)(uintptr_t) args->buffers_ptr, @@ -495,28 +602,6 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, if (ret) DRM_ERROR ("failed to copy %d validate entries back to user (%d)\n", args->buffer_count, ret); - - /* Clean up and return */ - ret = i915_gem_sync(dev); - if (ret) - { - drm_i915_private_t *dev_priv = dev->dev_private; - uint32_t acthd = I915_READ(IS_I965G(dev) ? I965REG_ACTHD : I915REG_ACTHD); - DRM_ERROR ("failed to sync %d acthd %08x\n", ret, acthd); - i915_gem_dump_object (object_list[args->buffer_count - 1], - args->batch_len, - __FUNCTION__, - acthd); - } - - /* Evict all the buffers we moved in, leaving room for the next guy. */ - for (i = 0; i < args->buffer_count; i++) { - struct drm_gem_object *obj = object_list[i]; - struct drm_i915_gem_object *obj_priv = obj->driver_private; - - if (obj_priv->pin_count == 0) - i915_gem_object_unbind(obj); - } err: if (object_list != NULL) { for (i = 0; i < args->buffer_count; i++) @@ -595,7 +680,7 @@ int i915_gem_init_object(struct drm_gem_object *obj) return -ENOMEM; obj->driver_private = obj_priv; - + INIT_LIST_HEAD(&obj_priv->gtt_lru_entry); return 0; } diff --git a/shared-core/i915_dma.c b/shared-core/i915_dma.c index d881a231..0601b89c 100644 --- a/shared-core/i915_dma.c +++ b/shared-core/i915_dma.c @@ -1051,6 +1051,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) ret = drm_addmap(dev, base, size, _DRM_REGISTERS, _DRM_KERNEL | _DRM_DRIVER, &dev_priv->mmio_map); + INIT_LIST_HEAD(&dev_priv->mm.gtt_lru); + #ifdef __linux__ #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25) intel_init_chipset_flush_compat(dev); diff --git a/shared-core/i915_drv.h b/shared-core/i915_drv.h index 6c06ff5c..bd6fa188 100644 --- a/shared-core/i915_drv.h +++ b/shared-core/i915_drv.h @@ -244,6 +244,8 @@ typedef struct drm_i915_private { struct { struct drm_memrange gtt_space; + /** LRU List of unpinned objects in the GTT. */ + struct list_head gtt_lru; } mm; } drm_i915_private_t; @@ -256,8 +258,12 @@ enum intel_chip_family { /** driver private structure attached to each drm_gem_object */ struct drm_i915_gem_object { + struct drm_gem_object *obj; + /** Current space allocated to this object in the GTT, if any. */ struct drm_memrange_node *gtt_space; + /** This object's place on the GTT LRU list */ + struct list_head gtt_lru_entry; /** AGP memory structure for our GTT binding. */ DRM_AGP_MEM *agp_mem; @@ -276,6 +282,9 @@ struct drm_i915_gem_object { /** How many users have pinned this object in GTT space */ int pin_count; + + /** Breadcrumb of last rendering to the buffer. */ + uint32_t last_rendering_cookie; }; extern struct drm_ioctl_desc i915_ioctls[]; @@ -318,6 +327,7 @@ extern int i915_vblank_pipe_set(struct drm_device *dev, void *data, extern int i915_vblank_pipe_get(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int i915_emit_irq(struct drm_device * dev); +extern int i915_wait_irq(struct drm_device * dev, int irq_nr); extern int i915_enable_vblank(struct drm_device *dev, int crtc); extern void i915_disable_vblank(struct drm_device *dev, int crtc); extern u32 i915_get_vblank_counter(struct drm_device *dev, int crtc); diff --git a/shared-core/i915_irq.c b/shared-core/i915_irq.c index 785008d4..b17e2408 100644 --- a/shared-core/i915_irq.c +++ b/shared-core/i915_irq.c @@ -545,7 +545,7 @@ void i915_user_irq_off(drm_i915_private_t *dev_priv) } -static int i915_wait_irq(struct drm_device * dev, int irq_nr) +int i915_wait_irq(struct drm_device * dev, int irq_nr) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; int ret = 0; |