Use a nopage-based approach to fault in pfns.

8 files changed, 155 insertions, 189 deletions

diff --git a/linux-core/drmP.h b/linux-core/drmP.h
index 089059c8..bc57bd5c 100644
--- a/linux-core/drmP.h
+++ b/linux-core/drmP.h
@@ -875,6 +875,7 @@ typedef struct drm_device {
         drm_mm_t offset_manager;        /**< User token manager */
         drm_open_hash_t object_hash;    /**< User token hash table for objects */
         struct address_space *dev_mapping;  /**< For unmap_mapping_range() */
+        struct page *ttm_dummy_page;
 
 	/** \name Context handle management */
 	/*@{ */
diff --git a/linux-core/drm_compat.c b/linux-core/drm_compat.c
index 81a2bd84..2b449e90 100644
--- a/linux-core/drm_compat.c
+++ b/linux-core/drm_compat.c
@@ -62,3 +62,82 @@ pgprot_t vm_get_page_prot(unsigned long vm_flags)
 	return protection_map[vm_flags & 0x0F];
 #endif
 };
+
+int drm_pte_is_clear(struct vm_area_struct *vma,
+		     unsigned long addr)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	int ret = 1;
+	pte_t *pte;
+	pmd_t *pmd;
+	pud_t *pud;
+	pgd_t *pgd;
+	
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
+	spin_lock(&mm->page_table_lock);
+#else
+	spinlock_t ptl;
+#endif
+	
+	pgd = pgd_offset(mm, addr);
+	if (pgd_none(*pgd))
+		goto unlock;
+	pud = pud_offset(pgd, addr);
+        if (pud_none(*pud))
+		goto unlock;
+	pmd = pmd_offset(pud, addr);
+	if (pmd_none(*pmd))
+		goto unlock;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
+	pte = pte_offset_map(pmd, addr);
+#else 
+	pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+#endif
+	if (!pte)
+		goto unlock;
+	ret = pte_none(*pte);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
+	pte_unmap(pte);
+ unlock:	
+	spin_unlock(&mm->page_table_lock);
+#else
+	pte_unmap_unlock(pte, ptl);
+ unlock:
+#endif
+	return ret;
+}
+	
+
+static struct {
+	spinlock_t lock;
+	struct page *dummy_page;
+	atomic_t present;
+} drm_np_retry = 
+{SPIN_LOCK_UNLOCKED, NOPAGE_OOM, ATOMIC_INIT(0)};
+
+struct page * get_nopage_retry(void)
+{
+	if (atomic_read(&drm_np_retry.present) == 0) {
+		struct page *page = alloc_page(GFP_KERNEL);
+		if (!page)
+			return NOPAGE_OOM;
+		spin_lock(&drm_np_retry.lock);
+		drm_np_retry.dummy_page = page;
+		atomic_set(&drm_np_retry.present,1);
+		spin_unlock(&drm_np_retry.lock);
+	}
+	get_page(drm_np_retry.dummy_page);
+	return drm_np_retry.dummy_page;
+}
+
+void free_nopage_retry(void)
+{
+	if (atomic_read(&drm_np_retry.present) == 1) {
+		spin_lock(&drm_np_retry.lock);
+		__free_page(drm_np_retry.dummy_page);
+		drm_np_retry.dummy_page = NULL;
+		atomic_set(&drm_np_retry.present, 0);
+		spin_unlock(&drm_np_retry.lock);
+	}
+}
diff --git a/linux-core/drm_compat.h b/linux-core/drm_compat.h
index cf84a70b..784b9a7d 100644
--- a/linux-core/drm_compat.h
+++ b/linux-core/drm_compat.h
@@ -252,7 +252,9 @@ extern pgprot_t vm_get_page_prot(unsigned long vm_flags);
  * that are not in the kernel linear map.
  */
 
-#define drm_alloc_gatt_pages(order) virt_to_page(alloc_gatt_pages(order))
+#define drm_alloc_gatt_pages(order) ({					\
+			void *_virt = alloc_gatt_pages(order);		\
+			((_virt) ? virt_to_page(_virt) : NULL);})
 #define drm_free_gatt_pages(pages, order) free_gatt_pages(page_address(pages), order) 
 
 #if defined(CONFIG_X86) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
@@ -268,4 +270,27 @@ extern int drm_map_page_into_agp(struct page *page);
 #define unmap_page_from_agp drm_unmap_page_from_agp
 #endif
 
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19))
+
+/*
+ * Hopefully, real NOPAGE_RETRY functionality will be in 2.6.19. 
+ * For now, just return a dummy page that we've allocated out of 
+ * static space. The page will be put by do_nopage() since we've already
+ * filled out the pte.
+ */
+extern struct page * get_nopage_retry(void);
+extern void free_nopage_retry(void);
+
+#define NOPAGE_RETRY get_nopage_retry()
+
+#endif
+
+/*
+ * Is the PTE for this address really clear so that we can use 
+ * io_remap_pfn_range?
+ */
+
+int drm_pte_is_clear(struct vm_area_struct *vma,
+		     unsigned long addr);
+
 #endif
diff --git a/linux-core/drm_drv.c b/linux-core/drm_drv.c
index 4cbe035f..11228363 100644
--- a/linux-core/drm_drv.c
+++ b/linux-core/drm_drv.c
@@ -434,6 +434,9 @@ void drm_exit(struct drm_driver *driver)
 		}
 	} else
 		pci_unregister_driver(&driver->pci_driver);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19))
+	free_nopage_retry();
+#endif
 	DRM_INFO("Module unloaded\n");
 }
 EXPORT_SYMBOL(drm_exit);
diff --git a/linux-core/drm_stub.c b/linux-core/drm_stub.c
index 1b406fef..e3db07dc 100644
--- a/linux-core/drm_stub.c
+++ b/linux-core/drm_stub.c
@@ -79,10 +79,6 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev,
 #endif
 	dev->irq = pdev->irq;
 
-	dev->maplist = drm_calloc(1, sizeof(*dev->maplist), DRM_MEM_MAPS);
-	if (dev->maplist == NULL)
-		return -ENOMEM;
-	INIT_LIST_HEAD(&dev->maplist->head);
 	if (drm_ht_create(&dev->map_hash, DRM_MAP_HASH_ORDER)) {
 		drm_free(dev->maplist, sizeof(*dev->maplist), DRM_MEM_MAPS);
 		return -ENOMEM;
@@ -101,6 +97,11 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev,
 		return -ENOMEM;
 	}
 
+	dev->maplist = drm_calloc(1, sizeof(*dev->maplist), DRM_MEM_MAPS);
+	if (dev->maplist == NULL)
+		return -ENOMEM;
+	INIT_LIST_HEAD(&dev->maplist->head);
+
 	/* the DRM has 6 counters */
 	dev->counters = 6;
 	dev->types[0] = _DRM_STAT_LOCK;
diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c
index ed50da90..51e28ac4 100644
--- a/linux-core/drm_ttm.c
+++ b/linux-core/drm_ttm.c
@@ -71,89 +71,6 @@ static void ttm_free(void *pointer, unsigned long size, int type)
 	}
 }		
 
-
-/*
- * We may be manipulating other processes page tables, so for each TTM, keep track of 
- * which mm_structs are currently mapping the ttm so that we can take the appropriate
- * locks when we modify their page tables. A typical application is when we evict another
- * process' buffers.
- */
-
-int drm_ttm_add_mm_to_list(drm_ttm_t * ttm, struct mm_struct *mm)
-{
-	p_mm_entry_t *entry, *n_entry;
-
-	list_for_each_entry(entry, &ttm->p_mm_list, head) {
-		if (mm == entry->mm) {
-			atomic_inc(&entry->refcount);
-			return 0;
-		} else if ((unsigned long)mm < (unsigned long)entry->mm) ;
-	}
-
-	n_entry = drm_alloc(sizeof(*n_entry), DRM_MEM_TTM);
-	if (!entry) {
-		DRM_ERROR("Allocation of process mm pointer entry failed\n");
-		return -ENOMEM;
-	}
-	INIT_LIST_HEAD(&n_entry->head);
-	n_entry->mm = mm;
-	atomic_set(&n_entry->refcount, 0);
-	atomic_inc(&ttm->shared_count);
-	ttm->mm_list_seq++;
-
-	list_add_tail(&n_entry->head, &entry->head);
-
-	return 0;
-}
-
-void drm_ttm_delete_mm(drm_ttm_t * ttm, struct mm_struct *mm)
-{
-	p_mm_entry_t *entry, *n;
-	list_for_each_entry_safe(entry, n, &ttm->p_mm_list, head) {
-		if (mm == entry->mm) {
-			if (atomic_add_negative(-1, &entry->refcount)) {
-				list_del(&entry->head);
-				drm_free(entry, sizeof(*entry), DRM_MEM_TTM);
-				atomic_dec(&ttm->shared_count);
-				ttm->mm_list_seq++;
-			}
-			return;
-		}
-	}
-	BUG_ON(1);
-}
-
-static void drm_ttm_unlock_mm(drm_ttm_t * ttm)
-{
-	p_mm_entry_t *entry;
-
-	list_for_each_entry(entry, &ttm->p_mm_list, head) {
-		up_write(&entry->mm->mmap_sem);
-	}
-}
-
-static int ioremap_vmas(drm_ttm_t * ttm, unsigned long page_offset,
-			unsigned long num_pages, unsigned long aper_offset)
-{
-	struct list_head *list;
-	int ret = 0;
-
-	list_for_each(list, &ttm->vma_list->head) {
-		drm_ttm_vma_list_t *entry =
-		    list_entry(list, drm_ttm_vma_list_t, head);
-
-		ret = io_remap_pfn_range(entry->vma,
-					 entry->vma->vm_start +
-					 (page_offset << PAGE_SHIFT),
-					 (ttm->aperture_base >> PAGE_SHIFT) +
-					 aper_offset, num_pages << PAGE_SHIFT,
-					 drm_io_prot(_DRM_AGP, entry->vma));
-		if (ret)
-			break;
-	}
-	return ret;
-}
-
 /*
  * Unmap all vma pages from vmas mapping this ttm.
  */
@@ -216,17 +133,15 @@ int drm_destroy_ttm(drm_ttm_t * ttm)
 				do_tlbflush = 1;
 			}
 			if (*cur_page) {
-				ClearPageLocked(*cur_page);
-
-				/*
-				 * Debugging code. Remove if the error message never
-				 * shows up.
-				 */
-
+				unlock_page(*cur_page);
 				if (page_count(*cur_page) != 1) {
 					DRM_ERROR("Erroneous page count. "
 						  "Leaking pages.\n");
 				}
+				if (page_mapped(*cur_page)) {
+					DRM_ERROR("Erroneous map count. "
+						  "Leaking page mappings.\n");
+				}
 
 				/*
 				 * End debugging.
@@ -335,72 +250,6 @@ static drm_ttm_t *drm_init_ttm(struct drm_device *dev, unsigned long size)
 }
 
 /*
- * Lock the mmap_sems for processes that are mapping this ttm. 
- * This looks a bit clumsy, since we need to maintain the correct
- * locking order
- * mm->mmap_sem
- * dev->struct_sem;
- * and while we release dev->struct_sem to lock the mmap_sems, 
- * the mmap_sem list may have been updated. We need to revalidate
- * it after relocking dev->struc_sem.
- */
-
-static int drm_ttm_lock_mmap_sem(drm_ttm_t * ttm)
-{
-	struct mm_struct **mm_list = NULL, **mm_list_p;
-	uint32_t list_seq;
-	uint32_t cur_count, shared_count;
-	p_mm_entry_t *entry;
-	unsigned i;
-
-	cur_count = 0;
-	list_seq = ttm->mm_list_seq;
-	shared_count = atomic_read(&ttm->shared_count);
-
-	do {
-		if (shared_count > cur_count) {
-			if (mm_list)
-				drm_free(mm_list, sizeof(*mm_list) * cur_count,
-					 DRM_MEM_TTM);
-			cur_count = shared_count + 10;
-			mm_list =
-			    drm_alloc(sizeof(*mm_list) * cur_count,
-				      DRM_MEM_TTM);
-			if (!mm_list)
-				return -ENOMEM;
-		}
-
-		mm_list_p = mm_list;
-		list_for_each_entry(entry, &ttm->p_mm_list, head) {
-			*mm_list_p++ = entry->mm;
-		}
-
-		mutex_unlock(&ttm->dev->struct_mutex);
-		mm_list_p = mm_list;
-		for (i = 0; i < shared_count; ++i, ++mm_list_p) {
-			down_write(&((*mm_list_p)->mmap_sem));
-		}
-
-		mutex_lock(&ttm->dev->struct_mutex);
-
-		if (list_seq != ttm->mm_list_seq) {
-			mm_list_p = mm_list;
-			for (i = 0; i < shared_count; ++i, ++mm_list_p) {
-				up_write(&((*mm_list_p)->mmap_sem));
-			}
-
-		}
-		shared_count = atomic_read(&ttm->shared_count);
-
-	} while (list_seq != ttm->mm_list_seq);
-
-	if (mm_list)
-		drm_free(mm_list, sizeof(*mm_list) * cur_count, DRM_MEM_TTM);
-
-	return 0;
-}
-
-/*
  * Change caching policy for the linear kernel map 
  * for range of pages in a ttm.
  */
@@ -449,15 +298,11 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry)
 {
 	drm_ttm_backend_t *be = entry->be;
 	drm_ttm_t *ttm = entry->owner;
-	int ret;
 
 	if (be) {
 		switch (entry->state) {
 		case ttm_bound:
 			if (ttm && be->needs_cache_adjust(be)) {
-				ret = drm_ttm_lock_mmap_sem(ttm);
-				if (ret)
-					return ret;
 				unmap_vma_pages(ttm, entry->page_offset,
 						entry->num_pages);
 			}
@@ -465,7 +310,6 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry)
 			if (ttm && be->needs_cache_adjust(be)) {
 				drm_set_caching(ttm, entry->page_offset,
 						entry->num_pages, 0);
-				drm_ttm_unlock_mm(ttm);
 			}
 			break;
 		default:
@@ -613,7 +457,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
 	ttm = region->owner;
 
 	if (ttm && be->needs_cache_adjust(be)) {
-		ret = drm_ttm_lock_mmap_sem(ttm);
 		if (ret)
 			return ret;
 
@@ -626,8 +469,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
 	}
 
 	if ((ret = be->bind(be, aper_offset))) {
-		if (ttm && be->needs_cache_adjust(be))
-			drm_ttm_unlock_mm(ttm);
 		drm_unbind_ttm_region(region);
 		DRM_ERROR("Couldn't bind backend.\n");
 		return ret;
@@ -640,12 +481,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
 		cur_page_flag++;
 	}
 
-	if (ttm && be->needs_cache_adjust(be)) {
-		ioremap_vmas(ttm, region->page_offset, region->num_pages,
-			     aper_offset);
-		drm_ttm_unlock_mm(ttm);
-	}
-
 	region->state = ttm_bound;
 	return 0;
 }
diff --git a/linux-core/drm_ttm.h b/linux-core/drm_ttm.h
index 53afe792..fcac06b5 100644
--- a/linux-core/drm_ttm.h
+++ b/linux-core/drm_ttm.h
@@ -164,8 +164,6 @@ int drm_rebind_ttm_region(drm_ttm_backend_list_t * entry,
 
 extern int drm_destroy_ttm(drm_ttm_t * ttm);
 extern void drm_user_destroy_region(drm_ttm_backend_list_t * entry);
-extern int drm_ttm_add_mm_to_list(drm_ttm_t * ttm, struct mm_struct *mm);
-extern void drm_ttm_delete_mm(drm_ttm_t * ttm, struct mm_struct *mm);
 extern int drm_ttm_ioctl(DRM_IOCTL_ARGS);
 
 static __inline__ drm_ttm_t *drm_ttm_from_object(drm_ttm_object_t * to)
diff --git a/linux-core/drm_vm.c b/linux-core/drm_vm.c
index 4755684e..5fbbaadd 100644
--- a/linux-core/drm_vm.c
+++ b/linux-core/drm_vm.c
@@ -237,7 +237,7 @@ static int drm_ttm_remap_bound_pfn(struct vm_area_struct *vma,
 	}
 
 	if (ret) {
-	  DRM_ERROR("Map returned %c\n", ret);
+		DRM_ERROR("Map returned %c\n", ret);
 	}
 	return ret;
 }
@@ -254,6 +254,7 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
 	pgprot_t default_prot;
 	uint32_t page_flags;
 	drm_buffer_manager_t *bm;
+	drm_device_t *dev;
 
 	if (address > vma->vm_end)
 		return NOPAGE_SIGBUS;	/* Disallow mremap */
@@ -262,7 +263,11 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
 
 	map = (drm_map_t *) entry->map;
 	ttm = (drm_ttm_t *) map->offset;
-	bm = &ttm->dev->bm;
+	
+	dev = ttm->dev;
+	mutex_lock(&dev->struct_mutex);
+
+	bm = &dev->bm;
 	page_offset = (address - vma->vm_start) >> PAGE_SHIFT;
 	page = ttm->pages[page_offset];
 
@@ -270,22 +275,43 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
 
 	if (!page) {
 		if (bm->cur_pages >= bm->max_pages) {
-	 		DRM_ERROR("Maximum locked page count exceeded\n");
-			return NOPAGE_OOM;
+	 		DRM_ERROR("Maximum locked page count exceeded\n"); 
+			page = NOPAGE_OOM;
+			goto out;
 		}
 		++bm->cur_pages;
 		page = ttm->pages[page_offset] = drm_alloc_gatt_pages(0);
+		if (page) {
+			SetPageLocked(page);
+		} else {
+			page = NOPAGE_OOM;
+		}
 	}
-	if (!page) 
-		return NOPAGE_OOM;
 
-	SetPageLocked(page);
-	get_page(page);
+	if (page_flags & DRM_TTM_PAGE_UNCACHED) {
 
-	default_prot = vm_get_page_prot(vma->vm_flags);
+		/*
+		 * This makes sure we don't race with another 
+		 * drm_ttm_remap_bound_pfn();
+		 */
 
-	BUG_ON(page_flags & DRM_TTM_PAGE_UNCACHED);
+		if (!drm_pte_is_clear(vma, address)) {
+			page = NOPAGE_RETRY;
+			goto out1;
+		}
+		       		
+		drm_ttm_remap_bound_pfn(vma, address, PAGE_SIZE);
+		page = NOPAGE_RETRY;
+		goto out1;
+	}
+	get_page(page);
+	
+ out1:
+	default_prot = vm_get_page_prot(vma->vm_flags);	    
 	vma->vm_page_prot = default_prot;
+
+ out:
+	mutex_unlock(&dev->struct_mutex);
 	return page;
 }
 
@@ -645,7 +671,6 @@ static int drm_vm_ttm_open(struct vm_area_struct *vma) {
 	        *entry = *tmp_vma;
 		map = (drm_map_t *) entry->map;
 		ttm = (drm_ttm_t *) map->offset;
-		ret = drm_ttm_add_mm_to_list(ttm, vma->vm_mm);
 		if (!ret) {
 			atomic_inc(&ttm->vma_count);
 			INIT_LIST_HEAD(&entry->head);
@@ -717,7 +742,6 @@ static void drm_vm_ttm_close(struct vm_area_struct *vma)
 		ttm = (drm_ttm_t *) map->offset;
 		dev = ttm->dev;
 		mutex_lock(&dev->struct_mutex);
-		drm_ttm_delete_mm(ttm, vma->vm_mm);
 		list_del(&ttm_vma->head);
 		drm_free(ttm_vma, sizeof(*ttm_vma), DRM_MEM_VMAS);
 		if (atomic_dec_and_test(&ttm->vma_count)) {