From c58574c60505a699e19e1ed59e1b441be2594e53 Mon Sep 17 00:00:00 2001
From: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
Date: Tue, 10 Oct 2006 10:37:26 +0200
Subject: Use a nopage-based approach to fault in pfns.
---
linux-core/drmP.h | 1 +
linux-core/drm_compat.c | 79 ++++++++++++++++++++++
linux-core/drm_compat.h | 27 +++++++-
linux-core/drm_drv.c | 3 +
linux-core/drm_stub.c | 9 +--
linux-core/drm_ttm.c | 175 ++----------------------------------------------
linux-core/drm_ttm.h | 2 -
linux-core/drm_vm.c | 48 +++++++++----
8 files changed, 155 insertions(+), 189 deletions(-)
diff --git a/linux-core/drmP.h b/linux-core/drmP.h
index 089059c8..bc57bd5c 100644
--- a/linux-core/drmP.h
+++ b/linux-core/drmP.h
@@ -875,6 +875,7 @@ typedef struct drm_device {
drm_mm_t offset_manager; /**< User token manager */
drm_open_hash_t object_hash; /**< User token hash table for objects */
struct address_space *dev_mapping; /**< For unmap_mapping_range() */
+ struct page *ttm_dummy_page;
/** \name Context handle management */
/*@{ */
diff --git a/linux-core/drm_compat.c b/linux-core/drm_compat.c
index 81a2bd84..2b449e90 100644
--- a/linux-core/drm_compat.c
+++ b/linux-core/drm_compat.c
@@ -62,3 +62,82 @@ pgprot_t vm_get_page_prot(unsigned long vm_flags)
return protection_map[vm_flags & 0x0F];
#endif
};
+
+int drm_pte_is_clear(struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ int ret = 1;
+ pte_t *pte;
+ pmd_t *pmd;
+ pud_t *pud;
+ pgd_t *pgd;
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
+ spin_lock(&mm->page_table_lock);
+#else
+ spinlock_t ptl;
+#endif
+
+ pgd = pgd_offset(mm, addr);
+ if (pgd_none(*pgd))
+ goto unlock;
+ pud = pud_offset(pgd, addr);
+ if (pud_none(*pud))
+ goto unlock;
+ pmd = pmd_offset(pud, addr);
+ if (pmd_none(*pmd))
+ goto unlock;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
+ pte = pte_offset_map(pmd, addr);
+#else
+ pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+#endif
+ if (!pte)
+ goto unlock;
+ ret = pte_none(*pte);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
+ pte_unmap(pte);
+ unlock:
+ spin_unlock(&mm->page_table_lock);
+#else
+ pte_unmap_unlock(pte, ptl);
+ unlock:
+#endif
+ return ret;
+}
+
+
+static struct {
+ spinlock_t lock;
+ struct page *dummy_page;
+ atomic_t present;
+} drm_np_retry =
+{SPIN_LOCK_UNLOCKED, NOPAGE_OOM, ATOMIC_INIT(0)};
+
+struct page * get_nopage_retry(void)
+{
+ if (atomic_read(&drm_np_retry.present) == 0) {
+ struct page *page = alloc_page(GFP_KERNEL);
+ if (!page)
+ return NOPAGE_OOM;
+ spin_lock(&drm_np_retry.lock);
+ drm_np_retry.dummy_page = page;
+ atomic_set(&drm_np_retry.present,1);
+ spin_unlock(&drm_np_retry.lock);
+ }
+ get_page(drm_np_retry.dummy_page);
+ return drm_np_retry.dummy_page;
+}
+
+void free_nopage_retry(void)
+{
+ if (atomic_read(&drm_np_retry.present) == 1) {
+ spin_lock(&drm_np_retry.lock);
+ __free_page(drm_np_retry.dummy_page);
+ drm_np_retry.dummy_page = NULL;
+ atomic_set(&drm_np_retry.present, 0);
+ spin_unlock(&drm_np_retry.lock);
+ }
+}
diff --git a/linux-core/drm_compat.h b/linux-core/drm_compat.h
index cf84a70b..784b9a7d 100644
--- a/linux-core/drm_compat.h
+++ b/linux-core/drm_compat.h
@@ -252,7 +252,9 @@ extern pgprot_t vm_get_page_prot(unsigned long vm_flags);
* that are not in the kernel linear map.
*/
-#define drm_alloc_gatt_pages(order) virt_to_page(alloc_gatt_pages(order))
+#define drm_alloc_gatt_pages(order) ({ \
+ void *_virt = alloc_gatt_pages(order); \
+ ((_virt) ? virt_to_page(_virt) : NULL);})
#define drm_free_gatt_pages(pages, order) free_gatt_pages(page_address(pages), order)
#if defined(CONFIG_X86) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
@@ -268,4 +270,27 @@ extern int drm_map_page_into_agp(struct page *page);
#define unmap_page_from_agp drm_unmap_page_from_agp
#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19))
+
+/*
+ * Hopefully, real NOPAGE_RETRY functionality will be in 2.6.19.
+ * For now, just return a dummy page that we've allocated out of
+ * static space. The page will be put by do_nopage() since we've already
+ * filled out the pte.
+ */
+extern struct page * get_nopage_retry(void);
+extern void free_nopage_retry(void);
+
+#define NOPAGE_RETRY get_nopage_retry()
+
+#endif
+
+/*
+ * Is the PTE for this address really clear so that we can use
+ * io_remap_pfn_range?
+ */
+
+int drm_pte_is_clear(struct vm_area_struct *vma,
+ unsigned long addr);
+
#endif
diff --git a/linux-core/drm_drv.c b/linux-core/drm_drv.c
index 4cbe035f..11228363 100644
--- a/linux-core/drm_drv.c
+++ b/linux-core/drm_drv.c
@@ -434,6 +434,9 @@ void drm_exit(struct drm_driver *driver)
}
} else
pci_unregister_driver(&driver->pci_driver);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19))
+ free_nopage_retry();
+#endif
DRM_INFO("Module unloaded\n");
}
EXPORT_SYMBOL(drm_exit);
diff --git a/linux-core/drm_stub.c b/linux-core/drm_stub.c
index 1b406fef..e3db07dc 100644
--- a/linux-core/drm_stub.c
+++ b/linux-core/drm_stub.c
@@ -79,10 +79,6 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev,
#endif
dev->irq = pdev->irq;
- dev->maplist = drm_calloc(1, sizeof(*dev->maplist), DRM_MEM_MAPS);
- if (dev->maplist == NULL)
- return -ENOMEM;
- INIT_LIST_HEAD(&dev->maplist->head);
if (drm_ht_create(&dev->map_hash, DRM_MAP_HASH_ORDER)) {
drm_free(dev->maplist, sizeof(*dev->maplist), DRM_MEM_MAPS);
return -ENOMEM;
@@ -101,6 +97,11 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev,
return -ENOMEM;
}
+ dev->maplist = drm_calloc(1, sizeof(*dev->maplist), DRM_MEM_MAPS);
+ if (dev->maplist == NULL)
+ return -ENOMEM;
+ INIT_LIST_HEAD(&dev->maplist->head);
+
/* the DRM has 6 counters */
dev->counters = 6;
dev->types[0] = _DRM_STAT_LOCK;
diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c
index ed50da90..51e28ac4 100644
--- a/linux-core/drm_ttm.c
+++ b/linux-core/drm_ttm.c
@@ -71,89 +71,6 @@ static void ttm_free(void *pointer, unsigned long size, int type)
}
}
-
-/*
- * We may be manipulating other processes page tables, so for each TTM, keep track of
- * which mm_structs are currently mapping the ttm so that we can take the appropriate
- * locks when we modify their page tables. A typical application is when we evict another
- * process' buffers.
- */
-
-int drm_ttm_add_mm_to_list(drm_ttm_t * ttm, struct mm_struct *mm)
-{
- p_mm_entry_t *entry, *n_entry;
-
- list_for_each_entry(entry, &ttm->p_mm_list, head) {
- if (mm == entry->mm) {
- atomic_inc(&entry->refcount);
- return 0;
- } else if ((unsigned long)mm < (unsigned long)entry->mm) ;
- }
-
- n_entry = drm_alloc(sizeof(*n_entry), DRM_MEM_TTM);
- if (!entry) {
- DRM_ERROR("Allocation of process mm pointer entry failed\n");
- return -ENOMEM;
- }
- INIT_LIST_HEAD(&n_entry->head);
- n_entry->mm = mm;
- atomic_set(&n_entry->refcount, 0);
- atomic_inc(&ttm->shared_count);
- ttm->mm_list_seq++;
-
- list_add_tail(&n_entry->head, &entry->head);
-
- return 0;
-}
-
-void drm_ttm_delete_mm(drm_ttm_t * ttm, struct mm_struct *mm)
-{
- p_mm_entry_t *entry, *n;
- list_for_each_entry_safe(entry, n, &ttm->p_mm_list, head) {
- if (mm == entry->mm) {
- if (atomic_add_negative(-1, &entry->refcount)) {
- list_del(&entry->head);
- drm_free(entry, sizeof(*entry), DRM_MEM_TTM);
- atomic_dec(&ttm->shared_count);
- ttm->mm_list_seq++;
- }
- return;
- }
- }
- BUG_ON(1);
-}
-
-static void drm_ttm_unlock_mm(drm_ttm_t * ttm)
-{
- p_mm_entry_t *entry;
-
- list_for_each_entry(entry, &ttm->p_mm_list, head) {
- up_write(&entry->mm->mmap_sem);
- }
-}
-
-static int ioremap_vmas(drm_ttm_t * ttm, unsigned long page_offset,
- unsigned long num_pages, unsigned long aper_offset)
-{
- struct list_head *list;
- int ret = 0;
-
- list_for_each(list, &ttm->vma_list->head) {
- drm_ttm_vma_list_t *entry =
- list_entry(list, drm_ttm_vma_list_t, head);
-
- ret = io_remap_pfn_range(entry->vma,
- entry->vma->vm_start +
- (page_offset << PAGE_SHIFT),
- (ttm->aperture_base >> PAGE_SHIFT) +
- aper_offset, num_pages << PAGE_SHIFT,
- drm_io_prot(_DRM_AGP, entry->vma));
- if (ret)
- break;
- }
- return ret;
-}
-
/*
* Unmap all vma pages from vmas mapping this ttm.
*/
@@ -216,17 +133,15 @@ int drm_destroy_ttm(drm_ttm_t * ttm)
do_tlbflush = 1;
}
if (*cur_page) {
- ClearPageLocked(*cur_page);
-
- /*
- * Debugging code. Remove if the error message never
- * shows up.
- */
-
+ unlock_page(*cur_page);
if (page_count(*cur_page) != 1) {
DRM_ERROR("Erroneous page count. "
"Leaking pages.\n");
}
+ if (page_mapped(*cur_page)) {
+ DRM_ERROR("Erroneous map count. "
+ "Leaking page mappings.\n");
+ }
/*
* End debugging.
@@ -334,72 +249,6 @@ static drm_ttm_t *drm_init_ttm(struct drm_device *dev, unsigned long size)
return ttm;
}
-/*
- * Lock the mmap_sems for processes that are mapping this ttm.
- * This looks a bit clumsy, since we need to maintain the correct
- * locking order
- * mm->mmap_sem
- * dev->struct_sem;
- * and while we release dev->struct_sem to lock the mmap_sems,
- * the mmap_sem list may have been updated. We need to revalidate
- * it after relocking dev->struc_sem.
- */
-
-static int drm_ttm_lock_mmap_sem(drm_ttm_t * ttm)
-{
- struct mm_struct **mm_list = NULL, **mm_list_p;
- uint32_t list_seq;
- uint32_t cur_count, shared_count;
- p_mm_entry_t *entry;
- unsigned i;
-
- cur_count = 0;
- list_seq = ttm->mm_list_seq;
- shared_count = atomic_read(&ttm->shared_count);
-
- do {
- if (shared_count > cur_count) {
- if (mm_list)
- drm_free(mm_list, sizeof(*mm_list) * cur_count,
- DRM_MEM_TTM);
- cur_count = shared_count + 10;
- mm_list =
- drm_alloc(sizeof(*mm_list) * cur_count,
- DRM_MEM_TTM);
- if (!mm_list)
- return -ENOMEM;
- }
-
- mm_list_p = mm_list;
- list_for_each_entry(entry, &ttm->p_mm_list, head) {
- *mm_list_p++ = entry->mm;
- }
-
- mutex_unlock(&ttm->dev->struct_mutex);
- mm_list_p = mm_list;
- for (i = 0; i < shared_count; ++i, ++mm_list_p) {
- down_write(&((*mm_list_p)->mmap_sem));
- }
-
- mutex_lock(&ttm->dev->struct_mutex);
-
- if (list_seq != ttm->mm_list_seq) {
- mm_list_p = mm_list;
- for (i = 0; i < shared_count; ++i, ++mm_list_p) {
- up_write(&((*mm_list_p)->mmap_sem));
- }
-
- }
- shared_count = atomic_read(&ttm->shared_count);
-
- } while (list_seq != ttm->mm_list_seq);
-
- if (mm_list)
- drm_free(mm_list, sizeof(*mm_list) * cur_count, DRM_MEM_TTM);
-
- return 0;
-}
-
/*
* Change caching policy for the linear kernel map
* for range of pages in a ttm.
@@ -449,15 +298,11 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry)
{
drm_ttm_backend_t *be = entry->be;
drm_ttm_t *ttm = entry->owner;
- int ret;
if (be) {
switch (entry->state) {
case ttm_bound:
if (ttm && be->needs_cache_adjust(be)) {
- ret = drm_ttm_lock_mmap_sem(ttm);
- if (ret)
- return ret;
unmap_vma_pages(ttm, entry->page_offset,
entry->num_pages);
}
@@ -465,7 +310,6 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry)
if (ttm && be->needs_cache_adjust(be)) {
drm_set_caching(ttm, entry->page_offset,
entry->num_pages, 0);
- drm_ttm_unlock_mm(ttm);
}
break;
default:
@@ -613,7 +457,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
ttm = region->owner;
if (ttm && be->needs_cache_adjust(be)) {
- ret = drm_ttm_lock_mmap_sem(ttm);
if (ret)
return ret;
@@ -626,8 +469,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
}
if ((ret = be->bind(be, aper_offset))) {
- if (ttm && be->needs_cache_adjust(be))
- drm_ttm_unlock_mm(ttm);
drm_unbind_ttm_region(region);
DRM_ERROR("Couldn't bind backend.\n");
return ret;
@@ -640,12 +481,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
cur_page_flag++;
}
- if (ttm && be->needs_cache_adjust(be)) {
- ioremap_vmas(ttm, region->page_offset, region->num_pages,
- aper_offset);
- drm_ttm_unlock_mm(ttm);
- }
-
region->state = ttm_bound;
return 0;
}
diff --git a/linux-core/drm_ttm.h b/linux-core/drm_ttm.h
index 53afe792..fcac06b5 100644
--- a/linux-core/drm_ttm.h
+++ b/linux-core/drm_ttm.h
@@ -164,8 +164,6 @@ int drm_rebind_ttm_region(drm_ttm_backend_list_t * entry,
extern int drm_destroy_ttm(drm_ttm_t * ttm);
extern void drm_user_destroy_region(drm_ttm_backend_list_t * entry);
-extern int drm_ttm_add_mm_to_list(drm_ttm_t * ttm, struct mm_struct *mm);
-extern void drm_ttm_delete_mm(drm_ttm_t * ttm, struct mm_struct *mm);
extern int drm_ttm_ioctl(DRM_IOCTL_ARGS);
static __inline__ drm_ttm_t *drm_ttm_from_object(drm_ttm_object_t * to)
diff --git a/linux-core/drm_vm.c b/linux-core/drm_vm.c
index 4755684e..5fbbaadd 100644
--- a/linux-core/drm_vm.c
+++ b/linux-core/drm_vm.c
@@ -237,7 +237,7 @@ static int drm_ttm_remap_bound_pfn(struct vm_area_struct *vma,
}
if (ret) {
- DRM_ERROR("Map returned %c\n", ret);
+ DRM_ERROR("Map returned %c\n", ret);
}
return ret;
}
@@ -254,6 +254,7 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
pgprot_t default_prot;
uint32_t page_flags;
drm_buffer_manager_t *bm;
+ drm_device_t *dev;
if (address > vma->vm_end)
return NOPAGE_SIGBUS; /* Disallow mremap */
@@ -262,7 +263,11 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
map = (drm_map_t *) entry->map;
ttm = (drm_ttm_t *) map->offset;
- bm = &ttm->dev->bm;
+
+ dev = ttm->dev;
+ mutex_lock(&dev->struct_mutex);
+
+ bm = &dev->bm;
page_offset = (address - vma->vm_start) >> PAGE_SHIFT;
page = ttm->pages[page_offset];
@@ -270,22 +275,43 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
if (!page) {
if (bm->cur_pages >= bm->max_pages) {
- DRM_ERROR("Maximum locked page count exceeded\n");
- return NOPAGE_OOM;
+ DRM_ERROR("Maximum locked page count exceeded\n");
+ page = NOPAGE_OOM;
+ goto out;
}
++bm->cur_pages;
page = ttm->pages[page_offset] = drm_alloc_gatt_pages(0);
+ if (page) {
+ SetPageLocked(page);
+ } else {
+ page = NOPAGE_OOM;
+ }
}
- if (!page)
- return NOPAGE_OOM;
- SetPageLocked(page);
- get_page(page);
+ if (page_flags & DRM_TTM_PAGE_UNCACHED) {
- default_prot = vm_get_page_prot(vma->vm_flags);
+ /*
+ * This makes sure we don't race with another
+ * drm_ttm_remap_bound_pfn();
+ */
- BUG_ON(page_flags & DRM_TTM_PAGE_UNCACHED);
+ if (!drm_pte_is_clear(vma, address)) {
+ page = NOPAGE_RETRY;
+ goto out1;
+ }
+
+ drm_ttm_remap_bound_pfn(vma, address, PAGE_SIZE);
+ page = NOPAGE_RETRY;
+ goto out1;
+ }
+ get_page(page);
+
+ out1:
+ default_prot = vm_get_page_prot(vma->vm_flags);
vma->vm_page_prot = default_prot;
+
+ out:
+ mutex_unlock(&dev->struct_mutex);
return page;
}
@@ -645,7 +671,6 @@ static int drm_vm_ttm_open(struct vm_area_struct *vma) {
*entry = *tmp_vma;
map = (drm_map_t *) entry->map;
ttm = (drm_ttm_t *) map->offset;
- ret = drm_ttm_add_mm_to_list(ttm, vma->vm_mm);
if (!ret) {
atomic_inc(&ttm->vma_count);
INIT_LIST_HEAD(&entry->head);
@@ -717,7 +742,6 @@ static void drm_vm_ttm_close(struct vm_area_struct *vma)
ttm = (drm_ttm_t *) map->offset;
dev = ttm->dev;
mutex_lock(&dev->struct_mutex);
- drm_ttm_delete_mm(ttm, vma->vm_mm);
list_del(&ttm_vma->head);
drm_free(ttm_vma, sizeof(*ttm_vma), DRM_MEM_VMAS);
if (atomic_dec_and_test(&ttm->vma_count)) {
--
cgit v1.2.3