From c58574c60505a699e19e1ed59e1b441be2594e53 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Tue, 10 Oct 2006 10:37:26 +0200 Subject: Use a nopage-based approach to fault in pfns. --- linux-core/drmP.h | 1 + linux-core/drm_compat.c | 79 ++++++++++++++++++++++ linux-core/drm_compat.h | 27 +++++++- linux-core/drm_drv.c | 3 + linux-core/drm_stub.c | 9 +-- linux-core/drm_ttm.c | 175 ++---------------------------------------------- linux-core/drm_ttm.h | 2 - linux-core/drm_vm.c | 48 +++++++++---- 8 files changed, 155 insertions(+), 189 deletions(-) diff --git a/linux-core/drmP.h b/linux-core/drmP.h index 089059c8..bc57bd5c 100644 --- a/linux-core/drmP.h +++ b/linux-core/drmP.h @@ -875,6 +875,7 @@ typedef struct drm_device { drm_mm_t offset_manager; /**< User token manager */ drm_open_hash_t object_hash; /**< User token hash table for objects */ struct address_space *dev_mapping; /**< For unmap_mapping_range() */ + struct page *ttm_dummy_page; /** \name Context handle management */ /*@{ */ diff --git a/linux-core/drm_compat.c b/linux-core/drm_compat.c index 81a2bd84..2b449e90 100644 --- a/linux-core/drm_compat.c +++ b/linux-core/drm_compat.c @@ -62,3 +62,82 @@ pgprot_t vm_get_page_prot(unsigned long vm_flags) return protection_map[vm_flags & 0x0F]; #endif }; + +int drm_pte_is_clear(struct vm_area_struct *vma, + unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + int ret = 1; + pte_t *pte; + pmd_t *pmd; + pud_t *pud; + pgd_t *pgd; + + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) + spin_lock(&mm->page_table_lock); +#else + spinlock_t ptl; +#endif + + pgd = pgd_offset(mm, addr); + if (pgd_none(*pgd)) + goto unlock; + pud = pud_offset(pgd, addr); + if (pud_none(*pud)) + goto unlock; + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) + goto unlock; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) + pte = pte_offset_map(pmd, addr); +#else + pte = pte_offset_map_lock(mm, pmd, addr, &ptl); +#endif + if (!pte) + goto unlock; + ret = pte_none(*pte); +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) + pte_unmap(pte); + unlock: + spin_unlock(&mm->page_table_lock); +#else + pte_unmap_unlock(pte, ptl); + unlock: +#endif + return ret; +} + + +static struct { + spinlock_t lock; + struct page *dummy_page; + atomic_t present; +} drm_np_retry = +{SPIN_LOCK_UNLOCKED, NOPAGE_OOM, ATOMIC_INIT(0)}; + +struct page * get_nopage_retry(void) +{ + if (atomic_read(&drm_np_retry.present) == 0) { + struct page *page = alloc_page(GFP_KERNEL); + if (!page) + return NOPAGE_OOM; + spin_lock(&drm_np_retry.lock); + drm_np_retry.dummy_page = page; + atomic_set(&drm_np_retry.present,1); + spin_unlock(&drm_np_retry.lock); + } + get_page(drm_np_retry.dummy_page); + return drm_np_retry.dummy_page; +} + +void free_nopage_retry(void) +{ + if (atomic_read(&drm_np_retry.present) == 1) { + spin_lock(&drm_np_retry.lock); + __free_page(drm_np_retry.dummy_page); + drm_np_retry.dummy_page = NULL; + atomic_set(&drm_np_retry.present, 0); + spin_unlock(&drm_np_retry.lock); + } +} diff --git a/linux-core/drm_compat.h b/linux-core/drm_compat.h index cf84a70b..784b9a7d 100644 --- a/linux-core/drm_compat.h +++ b/linux-core/drm_compat.h @@ -252,7 +252,9 @@ extern pgprot_t vm_get_page_prot(unsigned long vm_flags); * that are not in the kernel linear map. */ -#define drm_alloc_gatt_pages(order) virt_to_page(alloc_gatt_pages(order)) +#define drm_alloc_gatt_pages(order) ({ \ + void *_virt = alloc_gatt_pages(order); \ + ((_virt) ? virt_to_page(_virt) : NULL);}) #define drm_free_gatt_pages(pages, order) free_gatt_pages(page_address(pages), order) #if defined(CONFIG_X86) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)) @@ -268,4 +270,27 @@ extern int drm_map_page_into_agp(struct page *page); #define unmap_page_from_agp drm_unmap_page_from_agp #endif +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)) + +/* + * Hopefully, real NOPAGE_RETRY functionality will be in 2.6.19. + * For now, just return a dummy page that we've allocated out of + * static space. The page will be put by do_nopage() since we've already + * filled out the pte. + */ +extern struct page * get_nopage_retry(void); +extern void free_nopage_retry(void); + +#define NOPAGE_RETRY get_nopage_retry() + +#endif + +/* + * Is the PTE for this address really clear so that we can use + * io_remap_pfn_range? + */ + +int drm_pte_is_clear(struct vm_area_struct *vma, + unsigned long addr); + #endif diff --git a/linux-core/drm_drv.c b/linux-core/drm_drv.c index 4cbe035f..11228363 100644 --- a/linux-core/drm_drv.c +++ b/linux-core/drm_drv.c @@ -434,6 +434,9 @@ void drm_exit(struct drm_driver *driver) } } else pci_unregister_driver(&driver->pci_driver); +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)) + free_nopage_retry(); +#endif DRM_INFO("Module unloaded\n"); } EXPORT_SYMBOL(drm_exit); diff --git a/linux-core/drm_stub.c b/linux-core/drm_stub.c index 1b406fef..e3db07dc 100644 --- a/linux-core/drm_stub.c +++ b/linux-core/drm_stub.c @@ -79,10 +79,6 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev, #endif dev->irq = pdev->irq; - dev->maplist = drm_calloc(1, sizeof(*dev->maplist), DRM_MEM_MAPS); - if (dev->maplist == NULL) - return -ENOMEM; - INIT_LIST_HEAD(&dev->maplist->head); if (drm_ht_create(&dev->map_hash, DRM_MAP_HASH_ORDER)) { drm_free(dev->maplist, sizeof(*dev->maplist), DRM_MEM_MAPS); return -ENOMEM; @@ -101,6 +97,11 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev, return -ENOMEM; } + dev->maplist = drm_calloc(1, sizeof(*dev->maplist), DRM_MEM_MAPS); + if (dev->maplist == NULL) + return -ENOMEM; + INIT_LIST_HEAD(&dev->maplist->head); + /* the DRM has 6 counters */ dev->counters = 6; dev->types[0] = _DRM_STAT_LOCK; diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c index ed50da90..51e28ac4 100644 --- a/linux-core/drm_ttm.c +++ b/linux-core/drm_ttm.c @@ -71,89 +71,6 @@ static void ttm_free(void *pointer, unsigned long size, int type) } } - -/* - * We may be manipulating other processes page tables, so for each TTM, keep track of - * which mm_structs are currently mapping the ttm so that we can take the appropriate - * locks when we modify their page tables. A typical application is when we evict another - * process' buffers. - */ - -int drm_ttm_add_mm_to_list(drm_ttm_t * ttm, struct mm_struct *mm) -{ - p_mm_entry_t *entry, *n_entry; - - list_for_each_entry(entry, &ttm->p_mm_list, head) { - if (mm == entry->mm) { - atomic_inc(&entry->refcount); - return 0; - } else if ((unsigned long)mm < (unsigned long)entry->mm) ; - } - - n_entry = drm_alloc(sizeof(*n_entry), DRM_MEM_TTM); - if (!entry) { - DRM_ERROR("Allocation of process mm pointer entry failed\n"); - return -ENOMEM; - } - INIT_LIST_HEAD(&n_entry->head); - n_entry->mm = mm; - atomic_set(&n_entry->refcount, 0); - atomic_inc(&ttm->shared_count); - ttm->mm_list_seq++; - - list_add_tail(&n_entry->head, &entry->head); - - return 0; -} - -void drm_ttm_delete_mm(drm_ttm_t * ttm, struct mm_struct *mm) -{ - p_mm_entry_t *entry, *n; - list_for_each_entry_safe(entry, n, &ttm->p_mm_list, head) { - if (mm == entry->mm) { - if (atomic_add_negative(-1, &entry->refcount)) { - list_del(&entry->head); - drm_free(entry, sizeof(*entry), DRM_MEM_TTM); - atomic_dec(&ttm->shared_count); - ttm->mm_list_seq++; - } - return; - } - } - BUG_ON(1); -} - -static void drm_ttm_unlock_mm(drm_ttm_t * ttm) -{ - p_mm_entry_t *entry; - - list_for_each_entry(entry, &ttm->p_mm_list, head) { - up_write(&entry->mm->mmap_sem); - } -} - -static int ioremap_vmas(drm_ttm_t * ttm, unsigned long page_offset, - unsigned long num_pages, unsigned long aper_offset) -{ - struct list_head *list; - int ret = 0; - - list_for_each(list, &ttm->vma_list->head) { - drm_ttm_vma_list_t *entry = - list_entry(list, drm_ttm_vma_list_t, head); - - ret = io_remap_pfn_range(entry->vma, - entry->vma->vm_start + - (page_offset << PAGE_SHIFT), - (ttm->aperture_base >> PAGE_SHIFT) + - aper_offset, num_pages << PAGE_SHIFT, - drm_io_prot(_DRM_AGP, entry->vma)); - if (ret) - break; - } - return ret; -} - /* * Unmap all vma pages from vmas mapping this ttm. */ @@ -216,17 +133,15 @@ int drm_destroy_ttm(drm_ttm_t * ttm) do_tlbflush = 1; } if (*cur_page) { - ClearPageLocked(*cur_page); - - /* - * Debugging code. Remove if the error message never - * shows up. - */ - + unlock_page(*cur_page); if (page_count(*cur_page) != 1) { DRM_ERROR("Erroneous page count. " "Leaking pages.\n"); } + if (page_mapped(*cur_page)) { + DRM_ERROR("Erroneous map count. " + "Leaking page mappings.\n"); + } /* * End debugging. @@ -334,72 +249,6 @@ static drm_ttm_t *drm_init_ttm(struct drm_device *dev, unsigned long size) return ttm; } -/* - * Lock the mmap_sems for processes that are mapping this ttm. - * This looks a bit clumsy, since we need to maintain the correct - * locking order - * mm->mmap_sem - * dev->struct_sem; - * and while we release dev->struct_sem to lock the mmap_sems, - * the mmap_sem list may have been updated. We need to revalidate - * it after relocking dev->struc_sem. - */ - -static int drm_ttm_lock_mmap_sem(drm_ttm_t * ttm) -{ - struct mm_struct **mm_list = NULL, **mm_list_p; - uint32_t list_seq; - uint32_t cur_count, shared_count; - p_mm_entry_t *entry; - unsigned i; - - cur_count = 0; - list_seq = ttm->mm_list_seq; - shared_count = atomic_read(&ttm->shared_count); - - do { - if (shared_count > cur_count) { - if (mm_list) - drm_free(mm_list, sizeof(*mm_list) * cur_count, - DRM_MEM_TTM); - cur_count = shared_count + 10; - mm_list = - drm_alloc(sizeof(*mm_list) * cur_count, - DRM_MEM_TTM); - if (!mm_list) - return -ENOMEM; - } - - mm_list_p = mm_list; - list_for_each_entry(entry, &ttm->p_mm_list, head) { - *mm_list_p++ = entry->mm; - } - - mutex_unlock(&ttm->dev->struct_mutex); - mm_list_p = mm_list; - for (i = 0; i < shared_count; ++i, ++mm_list_p) { - down_write(&((*mm_list_p)->mmap_sem)); - } - - mutex_lock(&ttm->dev->struct_mutex); - - if (list_seq != ttm->mm_list_seq) { - mm_list_p = mm_list; - for (i = 0; i < shared_count; ++i, ++mm_list_p) { - up_write(&((*mm_list_p)->mmap_sem)); - } - - } - shared_count = atomic_read(&ttm->shared_count); - - } while (list_seq != ttm->mm_list_seq); - - if (mm_list) - drm_free(mm_list, sizeof(*mm_list) * cur_count, DRM_MEM_TTM); - - return 0; -} - /* * Change caching policy for the linear kernel map * for range of pages in a ttm. @@ -449,15 +298,11 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry) { drm_ttm_backend_t *be = entry->be; drm_ttm_t *ttm = entry->owner; - int ret; if (be) { switch (entry->state) { case ttm_bound: if (ttm && be->needs_cache_adjust(be)) { - ret = drm_ttm_lock_mmap_sem(ttm); - if (ret) - return ret; unmap_vma_pages(ttm, entry->page_offset, entry->num_pages); } @@ -465,7 +310,6 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry) if (ttm && be->needs_cache_adjust(be)) { drm_set_caching(ttm, entry->page_offset, entry->num_pages, 0); - drm_ttm_unlock_mm(ttm); } break; default: @@ -613,7 +457,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region, ttm = region->owner; if (ttm && be->needs_cache_adjust(be)) { - ret = drm_ttm_lock_mmap_sem(ttm); if (ret) return ret; @@ -626,8 +469,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region, } if ((ret = be->bind(be, aper_offset))) { - if (ttm && be->needs_cache_adjust(be)) - drm_ttm_unlock_mm(ttm); drm_unbind_ttm_region(region); DRM_ERROR("Couldn't bind backend.\n"); return ret; @@ -640,12 +481,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region, cur_page_flag++; } - if (ttm && be->needs_cache_adjust(be)) { - ioremap_vmas(ttm, region->page_offset, region->num_pages, - aper_offset); - drm_ttm_unlock_mm(ttm); - } - region->state = ttm_bound; return 0; } diff --git a/linux-core/drm_ttm.h b/linux-core/drm_ttm.h index 53afe792..fcac06b5 100644 --- a/linux-core/drm_ttm.h +++ b/linux-core/drm_ttm.h @@ -164,8 +164,6 @@ int drm_rebind_ttm_region(drm_ttm_backend_list_t * entry, extern int drm_destroy_ttm(drm_ttm_t * ttm); extern void drm_user_destroy_region(drm_ttm_backend_list_t * entry); -extern int drm_ttm_add_mm_to_list(drm_ttm_t * ttm, struct mm_struct *mm); -extern void drm_ttm_delete_mm(drm_ttm_t * ttm, struct mm_struct *mm); extern int drm_ttm_ioctl(DRM_IOCTL_ARGS); static __inline__ drm_ttm_t *drm_ttm_from_object(drm_ttm_object_t * to) diff --git a/linux-core/drm_vm.c b/linux-core/drm_vm.c index 4755684e..5fbbaadd 100644 --- a/linux-core/drm_vm.c +++ b/linux-core/drm_vm.c @@ -237,7 +237,7 @@ static int drm_ttm_remap_bound_pfn(struct vm_area_struct *vma, } if (ret) { - DRM_ERROR("Map returned %c\n", ret); + DRM_ERROR("Map returned %c\n", ret); } return ret; } @@ -254,6 +254,7 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma, pgprot_t default_prot; uint32_t page_flags; drm_buffer_manager_t *bm; + drm_device_t *dev; if (address > vma->vm_end) return NOPAGE_SIGBUS; /* Disallow mremap */ @@ -262,7 +263,11 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma, map = (drm_map_t *) entry->map; ttm = (drm_ttm_t *) map->offset; - bm = &ttm->dev->bm; + + dev = ttm->dev; + mutex_lock(&dev->struct_mutex); + + bm = &dev->bm; page_offset = (address - vma->vm_start) >> PAGE_SHIFT; page = ttm->pages[page_offset]; @@ -270,22 +275,43 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma, if (!page) { if (bm->cur_pages >= bm->max_pages) { - DRM_ERROR("Maximum locked page count exceeded\n"); - return NOPAGE_OOM; + DRM_ERROR("Maximum locked page count exceeded\n"); + page = NOPAGE_OOM; + goto out; } ++bm->cur_pages; page = ttm->pages[page_offset] = drm_alloc_gatt_pages(0); + if (page) { + SetPageLocked(page); + } else { + page = NOPAGE_OOM; + } } - if (!page) - return NOPAGE_OOM; - SetPageLocked(page); - get_page(page); + if (page_flags & DRM_TTM_PAGE_UNCACHED) { - default_prot = vm_get_page_prot(vma->vm_flags); + /* + * This makes sure we don't race with another + * drm_ttm_remap_bound_pfn(); + */ - BUG_ON(page_flags & DRM_TTM_PAGE_UNCACHED); + if (!drm_pte_is_clear(vma, address)) { + page = NOPAGE_RETRY; + goto out1; + } + + drm_ttm_remap_bound_pfn(vma, address, PAGE_SIZE); + page = NOPAGE_RETRY; + goto out1; + } + get_page(page); + + out1: + default_prot = vm_get_page_prot(vma->vm_flags); vma->vm_page_prot = default_prot; + + out: + mutex_unlock(&dev->struct_mutex); return page; } @@ -645,7 +671,6 @@ static int drm_vm_ttm_open(struct vm_area_struct *vma) { *entry = *tmp_vma; map = (drm_map_t *) entry->map; ttm = (drm_ttm_t *) map->offset; - ret = drm_ttm_add_mm_to_list(ttm, vma->vm_mm); if (!ret) { atomic_inc(&ttm->vma_count); INIT_LIST_HEAD(&entry->head); @@ -717,7 +742,6 @@ static void drm_vm_ttm_close(struct vm_area_struct *vma) ttm = (drm_ttm_t *) map->offset; dev = ttm->dev; mutex_lock(&dev->struct_mutex); - drm_ttm_delete_mm(ttm, vma->vm_mm); list_del(&ttm_vma->head); drm_free(ttm_vma, sizeof(*ttm_vma), DRM_MEM_VMAS); if (atomic_dec_and_test(&ttm->vma_count)) { -- cgit v1.2.3