diff options
| -rw-r--r-- | linux-core/Makefile.kernel | 2 | ||||
| -rw-r--r-- | linux-core/ati_pcigart.c | 195 | ||||
| -rw-r--r-- | linux-core/drmP.h | 12 | ||||
| -rw-r--r-- | linux-core/radeon_buffer.c | 263 | ||||
| -rw-r--r-- | linux-core/radeon_drv.c | 39 | ||||
| -rw-r--r-- | linux-core/radeon_fence.c | 125 | ||||
| -rw-r--r-- | shared-core/radeon_cp.c | 25 | ||||
| -rw-r--r-- | shared-core/radeon_drm.h | 9 | ||||
| -rw-r--r-- | shared-core/radeon_drv.h | 51 | ||||
| -rw-r--r-- | shared-core/radeon_irq.c | 20 | ||||
| -rw-r--r-- | tests/ttmtest/src/ttmtest.c | 4 |
11 files changed, 719 insertions, 26 deletions
diff --git a/linux-core/Makefile.kernel b/linux-core/Makefile.kernel index 457e3f3c..0ed5471a 100644 --- a/linux-core/Makefile.kernel +++ b/linux-core/Makefile.kernel @@ -32,7 +32,7 @@ nouveau-objs := nouveau_drv.o nouveau_state.o nouveau_fifo.o nouveau_mem.o \ nv04_graph.o nv10_graph.o nv20_graph.o \ nv40_graph.o nv50_graph.o \ nv04_instmem.o nv50_instmem.o -radeon-objs := radeon_drv.o radeon_cp.o radeon_state.o radeon_mem.o radeon_irq.o r300_cmdbuf.o +radeon-objs := radeon_drv.o radeon_cp.o radeon_state.o radeon_mem.o radeon_irq.o r300_cmdbuf.o radeon_fence.o radeon_buffer.o sis-objs := sis_drv.o sis_mm.o ffb-objs := ffb_drv.o ffb_context.o savage-objs := savage_drv.o savage_bci.o savage_state.o diff --git a/linux-core/ati_pcigart.c b/linux-core/ati_pcigart.c index 7241c2a8..64e0e758 100644 --- a/linux-core/ati_pcigart.c +++ b/linux-core/ati_pcigart.c @@ -35,6 +35,45 @@ # define ATI_PCIGART_PAGE_SIZE 4096 /**< PCI GART page size */ +static __inline__ void insert_page_into_table(struct ati_pcigart_info *info, u32 page_base, u32 *pci_gart) +{ + switch(info->gart_reg_if) { + case DRM_ATI_GART_IGP: + *pci_gart = cpu_to_le32((page_base) | 0xc); + break; + case DRM_ATI_GART_PCIE: + *pci_gart = cpu_to_le32((page_base >> 8) | 0xc); + break; + default: + case DRM_ATI_GART_PCI: + *pci_gart = cpu_to_le32(page_base); + break; + } +} + +static __inline__ u32 get_page_base_from_table(struct ati_pcigart_info *info, u32 *pci_gart) +{ + u32 retval; + switch(info->gart_reg_if) { + case DRM_ATI_GART_IGP: + retval = *pci_gart; + retval &= ~0xc; + break; + case DRM_ATI_GART_PCIE: + retval = *pci_gart; + retval &= ~0xc; + retval <<= 8; + break; + default: + case DRM_ATI_GART_PCI: + retval = *pci_gart; + break; + } + return retval; +} + + + static void *drm_ati_alloc_pcigart_table(int order) { unsigned long address; @@ -207,18 +246,7 @@ int drm_ati_pcigart_init(struct drm_device *dev, struct ati_pcigart_info *gart_i page_base = (u32) entry->busaddr[i]; for (j = 0; j < (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE); j++) { - switch(gart_info->gart_reg_if) { - case DRM_ATI_GART_IGP: - *pci_gart = cpu_to_le32((page_base) | 0xc); - break; - case DRM_ATI_GART_PCIE: - *pci_gart = cpu_to_le32((page_base >> 8) | 0xc); - break; - default: - case DRM_ATI_GART_PCI: - *pci_gart = cpu_to_le32(page_base); - break; - } + insert_page_into_table(gart_info, page_base, pci_gart); pci_gart++; page_base += ATI_PCIGART_PAGE_SIZE; } @@ -238,3 +266,146 @@ int drm_ati_pcigart_init(struct drm_device *dev, struct ati_pcigart_info *gart_i return ret; } EXPORT_SYMBOL(drm_ati_pcigart_init); + +static int ati_pcigart_needs_unbind_cache_adjust(drm_ttm_backend_t *backend) +{ + return ((backend->flags & DRM_BE_FLAG_BOUND_CACHED) ? 0 : 1); +} + +static int ati_pcigart_populate(drm_ttm_backend_t *backend, + unsigned long num_pages, + struct page **pages) +{ + ati_pcigart_ttm_backend_t *atipci_be = + container_of(backend, ati_pcigart_ttm_backend_t, backend); + + DRM_ERROR("%ld\n", num_pages); + atipci_be->pages = pages; + atipci_be->num_pages = num_pages; + atipci_be->populated = 1; + return 0; +} + +static int ati_pcigart_bind_ttm(struct drm_ttm_backend *backend, + struct drm_bo_mem_reg *bo_mem) +{ + ati_pcigart_ttm_backend_t *atipci_be = + container_of(backend, ati_pcigart_ttm_backend_t, backend); + off_t j; + int i; + struct ati_pcigart_info *info = atipci_be->gart_info; + u32 *pci_gart; + u32 page_base; + unsigned long offset = bo_mem->mm_node->start; + pci_gart = info->addr; + + DRM_ERROR("Offset is %08lX\n", bo_mem->mm_node->start); + j = offset; + while (j < (offset + atipci_be->num_pages)) { + if (get_page_base_from_table(info, pci_gart+j)) + return -EBUSY; + j++; + } + + for (i = 0, j = offset; i < atipci_be->num_pages; i++, j++) { + struct page *cur_page = atipci_be->pages[i]; + /* write value */ + page_base = page_to_phys(cur_page); + insert_page_into_table(info, page_base, pci_gart + j); + } + +#if defined(__i386__) || defined(__x86_64__) + wbinvd(); +#else + mb(); +#endif + + atipci_be->gart_flush_fn(atipci_be->dev); + + atipci_be->bound = 1; + atipci_be->offset = offset; + /* need to traverse table and add entries */ + DRM_DEBUG("\n"); + return 0; +} + +static int ati_pcigart_unbind_ttm(drm_ttm_backend_t *backend) +{ + ati_pcigart_ttm_backend_t *atipci_be = + container_of(backend, ati_pcigart_ttm_backend_t, backend); + struct ati_pcigart_info *info = atipci_be->gart_info; + unsigned long offset = atipci_be->offset; + int i; + off_t j; + u32 *pci_gart = info->addr; + + DRM_DEBUG("\n"); + + if (atipci_be->bound != 1) + return -EINVAL; + + for (i = 0, j = offset; i < atipci_be->num_pages; i++, j++) { + *(pci_gart + j) = 0; + } + atipci_be->gart_flush_fn(atipci_be->dev); + atipci_be->bound = 0; + atipci_be->offset = 0; + return 0; +} + +static void ati_pcigart_clear_ttm(drm_ttm_backend_t *backend) +{ + ati_pcigart_ttm_backend_t *atipci_be = + container_of(backend, ati_pcigart_ttm_backend_t, backend); + + DRM_DEBUG("\n"); + if (atipci_be->pages) { + backend->func->unbind(backend); + atipci_be->pages = NULL; + + } + atipci_be->num_pages = 0; +} + +static void ati_pcigart_destroy_ttm(drm_ttm_backend_t *backend) +{ + ati_pcigart_ttm_backend_t *atipci_be; + if (backend) { + DRM_DEBUG("\n"); + atipci_be = container_of(backend, ati_pcigart_ttm_backend_t, backend); + if (atipci_be) { + if (atipci_be->pages) { + backend->func->clear(backend); + } + drm_ctl_free(atipci_be, sizeof(*atipci_be), DRM_MEM_TTM); + } + } +} + +static struct drm_ttm_backend_func ati_pcigart_ttm_backend = +{ + .needs_ub_cache_adjust = ati_pcigart_needs_unbind_cache_adjust, + .populate = ati_pcigart_populate, + .clear = ati_pcigart_clear_ttm, + .bind = ati_pcigart_bind_ttm, + .unbind = ati_pcigart_unbind_ttm, + .destroy = ati_pcigart_destroy_ttm, +}; + +struct drm_ttm_backend *ati_pcigart_init_ttm(struct drm_device *dev, struct ati_pcigart_info *info, void (*gart_flush_fn)(struct drm_device *dev)) +{ + ati_pcigart_ttm_backend_t *atipci_be; + + atipci_be = drm_ctl_calloc(1, sizeof (*atipci_be), DRM_MEM_TTM); + if (!atipci_be) + return NULL; + + atipci_be->populated = 0; + atipci_be->backend.func = &ati_pcigart_ttm_backend; + atipci_be->gart_info = info; + atipci_be->gart_flush_fn = gart_flush_fn; + atipci_be->dev = dev; + + return &atipci_be->backend; +} +EXPORT_SYMBOL(ati_pcigart_init_ttm); diff --git a/linux-core/drmP.h b/linux-core/drmP.h index 0237f593..2d1f6db8 100644 --- a/linux-core/drmP.h +++ b/linux-core/drmP.h @@ -849,6 +849,17 @@ struct drm_agp_ttm_backend { }; #endif +typedef struct ati_pcigart_ttm_backend { + struct drm_ttm_backend backend; + int populated; + void (*gart_flush_fn)(struct drm_device *dev); + struct ati_pcigart_info *gart_info; + unsigned long offset; + struct page **pages; + int num_pages; + int bound; + struct drm_device *dev; +} ati_pcigart_ttm_backend_t; static __inline__ int drm_core_check_feature(struct drm_device *dev, int feature) @@ -1165,6 +1176,7 @@ extern int drm_sg_free(struct drm_device *dev, void *data, /* ATI PCIGART support (ati_pcigart.h) */ extern int drm_ati_pcigart_init(struct drm_device *dev, struct ati_pcigart_info *gart_info); extern int drm_ati_pcigart_cleanup(struct drm_device *dev, struct ati_pcigart_info *gart_info); +extern struct drm_ttm_backend *ati_pcigart_init_ttm(struct drm_device *dev, struct ati_pcigart_info *info, void (*gart_flush_fn)(struct drm_device *dev)); extern drm_dma_handle_t *drm_pci_alloc(struct drm_device *dev, size_t size, size_t align, dma_addr_t maxaddr); diff --git a/linux-core/radeon_buffer.c b/linux-core/radeon_buffer.c new file mode 100644 index 00000000..8e2b20b1 --- /dev/null +++ b/linux-core/radeon_buffer.c @@ -0,0 +1,263 @@ +/************************************************************************** + * + * Copyright 2007 Dave Airlie + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * + **************************************************************************/ +/* + * Authors: Dave Airlie <airlied@linux.ie> + */ + +#include "drmP.h" +#include "radeon_drm.h" +#include "radeon_drv.h" + +struct drm_ttm_backend *radeon_create_ttm_backend_entry(struct drm_device * dev) +{ + drm_radeon_private_t *dev_priv = dev->dev_private; + + if(dev_priv->flags & RADEON_IS_AGP) + return drm_agp_init_ttm(dev); + else + return ati_pcigart_init_ttm(dev, &dev_priv->gart_info, radeon_gart_flush); +} + +int radeon_fence_types(struct drm_buffer_object *bo, uint32_t * class, uint32_t * type) +{ + *class = 0; + if (bo->mem.flags & (DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE)) + *type = 3; + else + *type = 1; + return 0; +} + +int radeon_invalidate_caches(struct drm_device * dev, uint64_t flags) +{ + drm_radeon_private_t *dev_priv = dev->dev_private; + RING_LOCALS; + + BEGIN_RING(4); + RADEON_FLUSH_CACHE(); + RADEON_FLUSH_ZCACHE(); + ADVANCE_RING(); + return 0; +} + +uint32_t radeon_evict_mask(struct drm_buffer_object *bo) +{ + switch (bo->mem.mem_type) { + case DRM_BO_MEM_LOCAL: + case DRM_BO_MEM_TT: + return DRM_BO_FLAG_MEM_LOCAL; + case DRM_BO_MEM_VRAM: + if (bo->mem.num_pages > 128) + return DRM_BO_MEM_TT; + else + return DRM_BO_MEM_LOCAL; + default: + return DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_CACHED; + } +} + +int radeon_init_mem_type(struct drm_device * dev, uint32_t type, + struct drm_mem_type_manager * man) +{ + drm_radeon_private_t *dev_priv = dev->dev_private; + + switch (type) { + case DRM_BO_MEM_LOCAL: + man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE | + _DRM_FLAG_MEMTYPE_CACHED; + man->drm_bus_maptype = 0; + break; + case DRM_BO_MEM_VRAM: + man->flags = _DRM_FLAG_MEMTYPE_FIXED | _DRM_FLAG_MEMTYPE_MAPPABLE | _DRM_FLAG_NEEDS_IOREMAP; + man->io_addr = NULL; + man->drm_bus_maptype = _DRM_FRAME_BUFFER; + man->io_offset = drm_get_resource_start(dev, 0); + man->io_size = drm_get_resource_len(dev, 0); + break; + case DRM_BO_MEM_TT: + if (dev_priv->flags & RADEON_IS_AGP) { + if (!(drm_core_has_AGP(dev) && dev->agp)) { + DRM_ERROR("AGP is not enabled for memory type %u\n", + (unsigned)type); + return -EINVAL; + } + man->io_offset = dev->agp->agp_info.aper_base; + man->io_size = dev->agp->agp_info.aper_size * 1024 * 1024; + man->io_addr = NULL; + man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE | + _DRM_FLAG_MEMTYPE_CSELECT | _DRM_FLAG_NEEDS_IOREMAP; + man->drm_bus_maptype = _DRM_AGP; + } else { + man->io_offset = dev_priv->gart_vm_start; + man->io_size = dev_priv->gart_size; + man->io_addr = NULL; + man->flags = _DRM_FLAG_MEMTYPE_CSELECT | _DRM_FLAG_MEMTYPE_MAPPABLE | _DRM_FLAG_MEMTYPE_CMA; + man->drm_bus_maptype = _DRM_SCATTER_GATHER; + } + break; + default: + DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); + return -EINVAL; + } + return 0; +} + +static void radeon_emit_copy_blit(struct drm_device * dev, + uint32_t src_offset, + uint32_t dst_offset, + uint32_t pages, int direction) +{ + uint32_t cur_pages; + uint32_t stride = PAGE_SIZE; + drm_radeon_private_t *dev_priv = dev->dev_private; + uint32_t format, height; + RING_LOCALS; + + if (!dev_priv) + return; + + /* 32-bit copy format */ + format = RADEON_COLOR_FORMAT_ARGB8888; + + /* radeon limited to 16k stride */ + stride &= 0x3fff; + while(pages > 0) { + cur_pages = pages; + if (cur_pages > 2048) + cur_pages = 2048; + pages -= cur_pages; + + /* needs verification */ + BEGIN_RING(7); + OUT_RING(CP_PACKET3(RADEON_CNTL_BITBLT_MULTI, 5)); + OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL | + RADEON_GMC_DST_PITCH_OFFSET_CNTL | + RADEON_GMC_BRUSH_NONE | + (format << 8) | + RADEON_GMC_SRC_DATATYPE_COLOR | + RADEON_ROP3_S | + RADEON_DP_SRC_SOURCE_MEMORY | + RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS); + if (direction) { + OUT_RING((stride << 22) | (src_offset >> 10)); + OUT_RING((stride << 22) | (dst_offset >> 10)); + } else { + OUT_RING((stride << 22) | (dst_offset >> 10)); + OUT_RING((stride << 22) | (src_offset >> 10)); + } + OUT_RING(0); + OUT_RING(pages); /* x - y */ + OUT_RING((stride << 16) | cur_pages); + ADVANCE_RING(); + } + + BEGIN_RING(2); + RADEON_WAIT_UNTIL_2D_IDLE(); + ADVANCE_RING(); + + return; +} + +static int radeon_move_blit(struct drm_buffer_object * bo, + int evict, int no_wait, struct drm_bo_mem_reg *new_mem) +{ + struct drm_bo_mem_reg *old_mem = &bo->mem; + int dir = 0; + + if ((old_mem->mem_type == new_mem->mem_type) && + (new_mem->mm_node->start < + old_mem->mm_node->start + old_mem->mm_node->size)) { + dir = 1; + } + + radeon_emit_copy_blit(bo->dev, + old_mem->mm_node->start << PAGE_SHIFT, + new_mem->mm_node->start << PAGE_SHIFT, + new_mem->num_pages, dir); + + + return drm_bo_move_accel_cleanup(bo, evict, no_wait, 0, + DRM_FENCE_TYPE_EXE | + DRM_RADEON_FENCE_TYPE_RW, + DRM_RADEON_FENCE_FLAG_FLUSHED, new_mem); +} + +static int radeon_move_flip(struct drm_buffer_object * bo, + int evict, int no_wait, struct drm_bo_mem_reg * new_mem) +{ + struct drm_device *dev = bo->dev; + struct drm_bo_mem_reg tmp_mem; + int ret; + + tmp_mem = *new_mem; + tmp_mem.mm_node = NULL; + tmp_mem.mask = DRM_BO_FLAG_MEM_TT | + DRM_BO_FLAG_CACHED | DRM_BO_FLAG_FORCE_CACHING; + + ret = drm_bo_mem_space(bo, &tmp_mem, no_wait); + if (ret) + return ret; + + ret = drm_bind_ttm(bo->ttm, &tmp_mem); + if (ret) + goto out_cleanup; + + ret = radeon_move_blit(bo, 1, no_wait, &tmp_mem); + if (ret) + goto out_cleanup; + + ret = drm_bo_move_ttm(bo, evict, no_wait, new_mem); +out_cleanup: + if (tmp_mem.mm_node) { + mutex_lock(&dev->struct_mutex); + if (tmp_mem.mm_node != bo->pinned_node) + drm_mm_put_block(tmp_mem.mm_node); + tmp_mem.mm_node = NULL; + mutex_unlock(&dev->struct_mutex); + } + return ret; +} + +int radeon_move(struct drm_buffer_object * bo, + int evict, int no_wait, struct drm_bo_mem_reg * new_mem) +{ + struct drm_bo_mem_reg *old_mem = &bo->mem; + + DRM_DEBUG("\n"); + if (old_mem->mem_type == DRM_BO_MEM_LOCAL) { + return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); + } else if (new_mem->mem_type == DRM_BO_MEM_LOCAL) { + if (radeon_move_flip(bo, evict, no_wait, new_mem)) + return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); + } else { + if (radeon_move_blit(bo, evict, no_wait, new_mem)) + return drm_bo_move_memcpy(bo, evict, no_wait, new_mem); + } + return 0; +} + diff --git a/linux-core/radeon_drv.c b/linux-core/radeon_drv.c index 327a6a97..107d71a1 100644 --- a/linux-core/radeon_drv.c +++ b/linux-core/radeon_drv.c @@ -56,6 +56,38 @@ static struct pci_device_id pciidlist[] = { radeon_PCI_IDS }; + +#ifdef RADEON_HAVE_FENCE +static struct drm_fence_driver radeon_fence_driver = { + .num_classes = 1, + .wrap_diff = (1 << 30), + .flush_diff = (1 << 29), + .sequence_mask = 0xffffffffU, + .lazy_capable = 1, + .emit = radeon_fence_emit_sequence, + .poke_flush = radeon_poke_flush, + .has_irq = radeon_fence_has_irq, +}; +#endif +#ifdef RADEON_HAVE_BUFFER + +static uint32_t radeon_mem_prios[] = {DRM_BO_MEM_VRAM, DRM_BO_MEM_TT, DRM_BO_MEM_LOCAL}; +static uint32_t radeon_busy_prios[] = {DRM_BO_MEM_TT, DRM_BO_MEM_VRAM, DRM_BO_MEM_LOCAL}; + +static struct drm_bo_driver radeon_bo_driver = { + .mem_type_prio = radeon_mem_prios, + .mem_busy_prio = radeon_busy_prios, + .num_mem_type_prio = sizeof(radeon_mem_prios)/sizeof(uint32_t), + .num_mem_busy_prio = sizeof(radeon_busy_prios)/sizeof(uint32_t), + .create_ttm_backend_entry = radeon_create_ttm_backend_entry, + .fence_type = radeon_fence_types, + .invalidate_caches = radeon_invalidate_caches, + .init_mem_type = radeon_init_mem_type, + .evict_mask = radeon_evict_mask, + .move = radeon_move, +}; +#endif + static int probe(struct pci_dev *pdev, const struct pci_device_id *ent); static struct drm_driver driver = { .driver_features = @@ -101,6 +133,13 @@ static struct drm_driver driver = { .remove = __devexit_p(drm_cleanup_pci), }, +#ifdef RADEON_HAVE_FENCE + .fence_driver = &radeon_fence_driver, +#endif +#ifdef RADEON_HAVE_BUFFER + .bo_driver = &radeon_bo_driver, +#endif + .name = DRIVER_NAME, .desc = DRIVER_DESC, .date = DRIVER_DATE, diff --git a/linux-core/radeon_fence.c b/linux-core/radeon_fence.c new file mode 100644 index 00000000..682f0bee --- /dev/null +++ b/linux-core/radeon_fence.c @@ -0,0 +1,125 @@ +/************************************************************************** + * + * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * + **************************************************************************/ +/* + * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com> + */ + +#include "drmP.h" +#include "drm.h" +#include "radeon_drm.h" +#include "radeon_drv.h" + +/* + * Implements an intel sync flush operation. + */ + +static void radeon_perform_flush(struct drm_device * dev) +{ + drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private; + struct drm_fence_manager *fm = &dev->fm; + struct drm_fence_class_manager *fc = &dev->fm.fence_class[0]; + struct drm_fence_driver *driver = dev->driver->fence_driver; + uint32_t pending_flush_types = 0; + uint32_t sequence; + + if (!dev_priv) + return; + + pending_flush_types = fc->pending_flush | + ((fc->pending_exe_flush) ? DRM_FENCE_TYPE_EXE : 0); + + if (pending_flush_types) { + sequence = READ_BREADCRUMB(dev_priv); + + drm_fence_handler(dev, 0, sequence, pending_flush_types, 0); + } + + return; +} + +void radeon_poke_flush(struct drm_device * dev, uint32_t class) +{ + struct drm_fence_manager *fm = &dev->fm; + unsigned long flags; + + if (class != 0) + return; + + write_lock_irqsave(&fm->lock, flags); + radeon_perform_flush(dev); + write_unlock_irqrestore(&fm->lock, flags); +} + +int radeon_fence_emit_sequence(struct drm_device *dev, uint32_t class, + uint32_t flags, uint32_t *sequence, + uint32_t *native_type) +{ + drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private; + RING_LOCALS; + + if (!dev_priv) + return -EINVAL; + + *native_type = DRM_FENCE_TYPE_EXE; + if (flags & DRM_RADEON_FENCE_FLAG_FLUSHED) { + *native_type |= DRM_RADEON_FENCE_TYPE_RW; + + BEGIN_RING(4); + + RADEON_FLUSH_CACHE(); + RADEON_FLUSH_ZCACHE(); + ADVANCE_RING(); + } + + radeon_emit_irq(dev); + *sequence = (uint32_t) dev_priv->counter; + + + return 0; +} + +void radeon_fence_handler(struct drm_device * dev) +{ + struct drm_fence_manager *fm = &dev->fm; + + write_lock(&fm->lock); + radeon_perform_flush(dev); + write_unlock(&fm->lock); +} + +int radeon_fence_has_irq(struct drm_device *dev, uint32_t class, uint32_t flags) +{ + /* + * We have an irq that tells us when we have a new breadcrumb. + */ + + if (class == 0 && flags == DRM_FENCE_TYPE_EXE) + return 1; + + return 0; +} diff --git a/shared-core/radeon_cp.c b/shared-core/radeon_cp.c index 06861381..006d04f3 100644 --- a/shared-core/radeon_cp.c +++ b/shared-core/radeon_cp.c @@ -1390,6 +1390,28 @@ static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on) } } +void radeon_gart_flush(struct drm_device *dev) +{ + drm_radeon_private_t *dev_priv = dev->dev_private; + + if (dev_priv->flags & RADEON_IS_IGPGART) { + RADEON_READ_IGPGART(dev_priv, RADEON_IGPGART_FLUSH); + RADEON_WRITE_IGPGART(RADEON_IGPGART_FLUSH, 0x1); + RADEON_READ_IGPGART(dev_priv, RADEON_IGPGART_FLUSH); + RADEON_WRITE_IGPGART(RADEON_IGPGART_FLUSH, 0x0); + } else if (dev_priv->flags & RADEON_IS_PCIE) { + u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL); + tmp |= RADEON_PCIE_TX_GART_INVALIDATE_TLB; + RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp); + tmp &= ~RADEON_PCIE_TX_GART_INVALIDATE_TLB; + RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp); + } else { + + + } + +} + static int radeon_do_init_cp(struct drm_device * dev, drm_radeon_init_t * init) { drm_radeon_private_t *dev_priv = dev->dev_private; @@ -2301,6 +2323,9 @@ int radeon_driver_firstopen(struct drm_device *dev) if (ret != 0) return ret; +#ifdef RADEON_HAVE_BUFFER + drm_bo_driver_init(dev); +#endif return 0; } diff --git a/shared-core/radeon_drm.h b/shared-core/radeon_drm.h index b0ef702b..913b23c5 100644 --- a/shared-core/radeon_drm.h +++ b/shared-core/radeon_drm.h @@ -434,8 +434,17 @@ typedef struct { int pfCurrentPage; /* which buffer is being displayed? */ int crtc2_base; /* CRTC2 frame offset */ int tiling_enabled; /* set by drm, read by 2d + 3d clients */ + + unsigned int last_fence; } drm_radeon_sarea_t; +/* The only fence class we support */ +#define DRM_RADEON_FENCE_CLASS_ACCEL 0 +/* Fence type that guarantees read-write flush */ +#define DRM_RADEON_FENCE_TYPE_RW 2 +/* cache flushes programmed just before the fence */ +#define DRM_RADEON_FENCE_FLAG_FLUSHED 0x01000000 + /* WARNING: If you change any of these defines, make sure to change the * defines in the Xserver file (xf86drmRadeon.h) * diff --git a/shared-core/radeon_drv.h b/shared-core/radeon_drv.h index 006559df..f74fe933 100644 --- a/shared-core/radeon_drv.h +++ b/shared-core/radeon_drv.h @@ -104,6 +104,11 @@ #define DRIVER_MINOR 28 #define DRIVER_PATCHLEVEL 0 +#if defined(__linux__) +#define RADEON_HAVE_FENCE +#define RADEON_HAVE_BUFFER +#endif + /* * Radeon chip families */ @@ -284,8 +289,9 @@ typedef struct drm_radeon_private { struct mem_block *fb_heap; /* SW interrupt */ - wait_queue_head_t swi_queue; - atomic_t swi_emitted; + wait_queue_head_t irq_queue; + int counter; + int vblank_crtc; uint32_t irq_enable_reg; int irq_enabled; @@ -344,6 +350,7 @@ extern int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file extern int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern void radeon_gart_flush(struct drm_device *dev); extern void radeon_freelist_reset(struct drm_device * dev); extern struct drm_buf *radeon_freelist_get(struct drm_device * dev); @@ -362,6 +369,7 @@ extern void radeon_mem_release(struct drm_file *file_priv, /* radeon_irq.c */ extern int radeon_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_emit_irq(struct drm_device * dev); extern void radeon_do_release(struct drm_device * dev); extern int radeon_driver_vblank_wait(struct drm_device * dev, @@ -395,6 +403,30 @@ extern int r300_do_cp_cmdbuf(struct drm_device *dev, struct drm_file *file_priv, drm_radeon_kcmd_buffer_t* cmdbuf); + +#ifdef RADEON_HAVE_FENCE +/* i915_fence.c */ + + +extern void radeon_fence_handler(struct drm_device *dev); +extern int radeon_fence_emit_sequence(struct drm_device *dev, uint32_t class, + uint32_t flags, uint32_t *sequence, + uint32_t *native_type); +extern void radeon_poke_flush(struct drm_device *dev, uint32_t class); +extern int radeon_fence_has_irq(struct drm_device *dev, uint32_t class, uint32_t flags); +#endif + +#ifdef RADEON_HAVE_BUFFER +/* radeon_buffer.c */ +extern drm_ttm_backend_t *radeon_create_ttm_backend_entry(struct drm_device *dev); +extern int radeon_fence_types(struct drm_buffer_object *bo, uint32_t *class, uint32_t *type); +extern int radeon_invalidate_caches(struct drm_device *dev, uint64_t buffer_flags); +extern uint32_t radeon_evict_mask(struct drm_buffer_object *bo); +extern int radeon_init_mem_type(struct drm_device * dev, uint32_t type, + struct drm_mem_type_manager * man); +extern int radeon_move(struct drm_buffer_object * bo, + int evict, int no_wait, struct drm_bo_mem_reg * new_mem); +#endif /* Flags for stats.boxes */ #define RADEON_BOX_DMA_IDLE 0x1 @@ -1266,4 +1298,19 @@ do { \ write &= mask; \ } while (0) +/* Breadcrumb - swi irq */ +#define READ_BREADCRUMB(dev_priv) RADEON_READ(RADEON_LAST_SWI_REG) + +static inline int radeon_update_breadcrumb(struct drm_device *dev) +{ + drm_radeon_private_t *dev_priv = dev->dev_private; + + dev_priv->sarea_priv->last_fence = ++dev_priv->counter; + + if (dev_priv->counter > 0x7FFFFFFFUL) + dev_priv->sarea_priv->last_fence = dev_priv->counter = 1; + + return dev_priv->counter; +} + #endif /* __RADEON_DRV_H__ */ diff --git a/shared-core/radeon_irq.c b/shared-core/radeon_irq.c index 1ece6399..3162e266 100644 --- a/shared-core/radeon_irq.c +++ b/shared-core/radeon_irq.c @@ -82,7 +82,10 @@ irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS) /* SW interrupt */ if (stat & RADEON_SW_INT_TEST) { - DRM_WAKEUP(&dev_priv->swi_queue); + DRM_WAKEUP(&dev_priv->irq_queue); +#ifdef RADEON_HAVE_FENCE + radeon_fence_handler(dev); +#endif } /* VBLANK interrupt */ @@ -109,14 +112,13 @@ irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS) return IRQ_HANDLED; } -static int radeon_emit_irq(struct drm_device * dev) +int radeon_emit_irq(struct drm_device * dev) { drm_radeon_private_t *dev_priv = dev->dev_private; unsigned int ret; RING_LOCALS; - atomic_inc(&dev_priv->swi_emitted); - ret = atomic_read(&dev_priv->swi_emitted); + ret = radeon_update_breadcrumb(dev); BEGIN_RING(4); OUT_RING_REG(RADEON_LAST_SWI_REG, ret); @@ -133,13 +135,13 @@ static int radeon_wait_irq(struct drm_device * dev, int swi_nr) (drm_radeon_private_t *) dev->dev_private; int ret = 0; - if (RADEON_READ(RADEON_LAST_SWI_REG) >= swi_nr) + if (READ_BREADCRUMB(dev_priv) >= swi_nr) return 0; dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE; - DRM_WAIT_ON(ret, dev_priv->swi_queue, 3 * DRM_HZ, - RADEON_READ(RADEON_LAST_SWI_REG) >= swi_nr); + DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ, + READ_BREADCRUMB(dev_priv) >= swi_nr); return ret; } @@ -271,8 +273,8 @@ void radeon_driver_irq_postinstall(struct drm_device * dev) drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private; - atomic_set(&dev_priv->swi_emitted, 0); - DRM_INIT_WAITQUEUE(&dev_priv->swi_queue); + dev_priv->counter = 0; + DRM_INIT_WAITQUEUE(&dev_priv->irq_queue); radeon_enable_interrupt(dev); } diff --git a/tests/ttmtest/src/ttmtest.c b/tests/ttmtest/src/ttmtest.c index 606fb0cb..052947b1 100644 --- a/tests/ttmtest/src/ttmtest.c +++ b/tests/ttmtest/src/ttmtest.c @@ -176,7 +176,7 @@ benchmarkBuffer(TinyDRIContext * ctx, unsigned long size, /* * Test system memory objects. */ - + BM_CKFATAL(drmGetLock(ctx->drmFD, ctx->hwContext, 0)); oldTime = fastrdtsc(); BM_CKFATAL(drmBOCreate(ctx->drmFD, 0, size, 0, NULL, drm_bo_type_dc, @@ -216,7 +216,7 @@ benchmarkBuffer(TinyDRIContext * ctx, unsigned long size, * Test TT bound buffer objects. */ - BM_CKFATAL(drmGetLock(ctx->drmFD, ctx->hwContext, 0)); + // BM_CKFATAL(drmGetLock(ctx->drmFD, ctx->hwContext, 0)); oldTime = fastrdtsc(); BM_CKFATAL(drmBOValidate(ctx->drmFD, &buf, DRM_BO_FLAG_MEM_TT, DRM_BO_MASK_MEM, DRM_BO_HINT_DONT_FENCE)); |