Adds support for PCI cards to MGA DRM

This patch adds serveral new ioctls and a new query to get_param query to
    support PCI MGA cards.
Two ioctls were added to implement interrupt based waiting. With this
    change, the client-side driver no longer needs to map the primary DMA
    region or the MMIO region. Previously, end-of-frame waiting was done by
    busy waiting in the client-side driver until one of the MMIO registers
    (the current DMA pointer) matched a pointer to the end of primary DMA
    space. By using interrupts, the busy waiting and the extra mappings are
    removed.
A third ioctl was added to bootstrap DMA. This ioctl, which is used by the
    X-server, moves a *LOT* of code from the X-server into the kernel. This
    allows the kernel to do whatever needs to be done to setup DMA buffers.
    The entire process and the locations of the buffers are hidden from
    user-mode.
Additionally, a get_param query was added to differentiate between G4x0
    cards and G550 cards. A gap was left in the numbering sequence so that,
    if needed, G450 cards could be distinguished from G400 cards. According
    to Ville Syrjälä, the G4x0 cards and the G550 cards handle
    anisotropic filtering differently. This seems the most compatible way
    to let the client-side driver know which card it's own. Doing this very
    small change now eliminates the need to bump the DRM minor version
    twice.
http://marc.theaimsgroup.com/?l=dri-devel&m=106625815319773&w=2
A number of ioctl handlers in linux-core were also modified so that they
    could be called in-kernel. In these cases, the in-kernel callable
    version kept the existing name (e.g., drm_agp_acquire) and the ioctl
    handler added _ioctl to the name (e.g., drm_agp_acquire_ioctl).
This patch also replaces the drm_agp_do_release function with
    drm_agp_release. drm_agp_release (drm_core_agp_release in the previous
    patch) is very similar to drm_agp_do_release, and I saw no reason to
    have both.
This commit *breaks the build* on BSD. Eric said that he would make the
    required updates to the BSD side soon.
Xorg bug: 3259 Reviewed by: Eric Anholt

7 files changed, 786 insertions, 53 deletions

diff --git a/shared-core/drm_pciids.txt b/shared-core/drm_pciids.txt
index 82cbdd46..aa975eed 100644
--- a/shared-core/drm_pciids.txt
+++ b/shared-core/drm_pciids.txt
@@ -110,9 +110,10 @@
 0x1002 0x5452 0 "ATI Rage 128 Pro Ultra TR (AGP)"
 
 [mga]
+0x102b 0x0520 MGA_CARD_TYPE_G200 "Matrox G200 (PCI)"
 0x102b 0x0521 MGA_CARD_TYPE_G200 "Matrox G200 (AGP)"
 0x102b 0x0525 MGA_CARD_TYPE_G400 "Matrox G400/G450 (AGP)"
-0x102b 0x2527 MGA_CARD_TYPE_G400 "Matrox G550 (AGP)"
+0x102b 0x2527 MGA_CARD_TYPE_G550 "Matrox G550 (AGP)"
 
 [mach64]
 0x1002 0x4749 0 "3D Rage Pro"
diff --git a/shared-core/mga_dma.c b/shared-core/mga_dma.c
index 4c0be4cc..998886d6 100644
--- a/shared-core/mga_dma.c
+++ b/shared-core/mga_dma.c
@@ -23,18 +23,21 @@
  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Rickard E. (Rik) Faith <faith@valinux.com>
- *    Jeff Hartmann <jhartmann@valinux.com>
- *    Keith Whitwell <keith@tungstengraphics.com>
- *
- * Rewritten by:
- *    Gareth Hughes <gareth@valinux.com>
+ */
+
+/**
+ * \file mga_dma.c
+ * DMA support for MGA G200 / G400.
+ * 
+ * \author Rickard E. (Rik) Faith <faith@valinux.com>
+ * \author Jeff Hartmann <jhartmann@valinux.com>
+ * \author Keith Whitwell <keith@tungstengraphics.com>
+ * \author Gareth Hughes <gareth@valinux.com>
  */
 
 #include "drmP.h"
 #include "drm.h"
+#include "drm_sarea.h"
 #include "mga_drm.h"
 #include "mga_drv.h"
 
@@ -148,7 +151,7 @@ void mga_do_dma_flush(drm_mga_private_t * dev_priv)
 	DRM_DEBUG("  space = 0x%06x\n", primary->space);
 
 	mga_flush_write_combine();
-	MGA_WRITE(MGA_PRIMEND, tail | MGA_PAGPXFER);
+	MGA_WRITE(MGA_PRIMEND, tail | dev_priv->dma_access);
 
 	DRM_DEBUG("done.\n");
 }
@@ -188,7 +191,7 @@ void mga_do_dma_wrap_start(drm_mga_private_t * dev_priv)
 	DRM_DEBUG("  space = 0x%06x\n", primary->space);
 
 	mga_flush_write_combine();
-	MGA_WRITE(MGA_PRIMEND, tail | MGA_PAGPXFER);
+	MGA_WRITE(MGA_PRIMEND, tail | dev_priv->dma_access);
 
 	set_bit(0, &primary->wrapped);
 	DRM_DEBUG("done.\n");
@@ -405,6 +408,405 @@ int mga_driver_preinit(drm_device_t *dev, unsigned long flags)
 	return 0;
 }
 
+/**
+ * Bootstrap the driver for AGP DMA.
+ * 
+ * \todo
+ * Investigate whether there is any benifit to storing the WARP microcode in
+ * AGP memory.  If not, the microcode may as well always be put in PCI
+ * memory.
+ *
+ * \todo
+ * This routine needs to set dma_bs->agp_mode to the mode actually configured
+ * in the hardware.  Looking just at the Linux AGP driver code, I don't see
+ * an easy way to determine this.
+ *
+ * \sa mga_do_dma_bootstrap, mga_do_pci_dma_bootstrap
+ */
+static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
+				    drm_mga_dma_bootstrap_t * dma_bs)
+{
+	drm_mga_private_t * const dev_priv = (drm_mga_private_t *) dev->dev_private;
+	const unsigned int warp_size = mga_warp_microcode_size(dev_priv);
+	int err;
+	unsigned  offset;
+	const unsigned secondary_size = dma_bs->secondary_bin_count
+		* dma_bs->secondary_bin_size;
+	const unsigned agp_size = (dma_bs->agp_size << 20);
+	drm_buf_desc_t req;
+	drm_agp_mode_t mode;
+	drm_agp_info_t info;
+
+	
+	/* Acquire AGP. */
+	err = drm_agp_acquire(dev);
+	if (err) {
+		DRM_ERROR("Unable to acquire AGP\n");
+		return err;
+	}
+
+	err = drm_agp_info(dev, &info);
+	if (err) {
+		DRM_ERROR("Unable to get AGP info\n");
+		return err;
+	}
+
+	mode.mode = (info.mode & ~0x07) | dma_bs->agp_mode;
+	err = drm_agp_enable(dev, mode);
+	if (err) {
+		DRM_ERROR("Unable to enable AGP (mode = 0x%lx)\n", mode.mode);
+		return err;
+	}
+
+
+	/* In addition to the usual AGP mode configuration, the G200 AGP cards
+	 * need to have the AGP mode "manually" set.
+	 */
+
+	if (dev_priv->chipset == MGA_CARD_TYPE_G200) {
+		if (mode.mode & 0x02) {
+			MGA_WRITE(MGA_AGP_PLL, MGA_AGP2XPLL_ENABLE);
+		}
+		else {
+			MGA_WRITE(MGA_AGP_PLL, MGA_AGP2XPLL_DISABLE);
+		}
+	}
+
+
+	/* Allocate and bind AGP memory. */
+	dev_priv->agp_pages = agp_size / PAGE_SIZE;
+	dev_priv->agp_mem = drm_alloc_agp( dev_priv->agp_pages, 0 );
+	if (dev_priv->agp_mem == NULL) {
+		dev_priv->agp_pages = 0;
+		DRM_ERROR("Unable to allocate %uMB AGP memory\n",
+			  dma_bs->agp_size);
+		return DRM_ERR(ENOMEM);
+	}
+		
+	err = drm_bind_agp( dev_priv->agp_mem, 0 );
+	if (err) {
+		DRM_ERROR("Unable to bind AGP memory\n");
+		return err;
+	}
+
+	offset = 0;
+	err = drm_addmap( dev, offset, warp_size,
+			  _DRM_AGP, _DRM_READ_ONLY, & dev_priv->warp );
+	if (err) {
+		DRM_ERROR("Unable to map WARP microcode\n");
+		return err;
+	}
+
+	offset += warp_size;
+	err = drm_addmap( dev, offset, dma_bs->primary_size,
+			  _DRM_AGP, _DRM_READ_ONLY, & dev_priv->primary );
+	if (err) {
+		DRM_ERROR("Unable to map primary DMA region\n");
+		return err;
+	}
+
+	offset += dma_bs->primary_size;
+	err = drm_addmap( dev, offset, secondary_size,
+			  _DRM_AGP, 0, & dev->agp_buffer_map );
+	if (err) {
+		DRM_ERROR("Unable to map secondary DMA region\n");
+		return err;
+	}
+
+	(void) memset( &req, 0, sizeof(req) );
+	req.count = dma_bs->secondary_bin_count;
+	req.size = dma_bs->secondary_bin_size;
+	req.flags = _DRM_AGP_BUFFER;
+	req.agp_start = offset;
+
+	err = drm_addbufs_agp( dev, & req );
+	if (err) {
+		DRM_ERROR("Unable to add secondary DMA buffers\n");
+		return err;
+	}
+
+	offset += secondary_size;
+	err = drm_addmap( dev, offset, agp_size - offset,
+			  _DRM_AGP, 0, & dev_priv->agp_textures );
+	if (err) {
+		DRM_ERROR("Unable to map AGP texture region\n");
+		return err;
+	}
+
+	drm_core_ioremap(dev_priv->warp, dev);
+	drm_core_ioremap(dev_priv->primary, dev);
+	drm_core_ioremap(dev->agp_buffer_map, dev);
+
+	if (!dev_priv->warp->handle ||
+	    !dev_priv->primary->handle || !dev->agp_buffer_map->handle) {
+		DRM_ERROR("failed to ioremap agp regions! (%p, %p, %p)\n",
+			  dev_priv->warp->handle, dev_priv->primary->handle,
+			  dev->agp_buffer_map->handle);
+		return DRM_ERR(ENOMEM);
+	}
+
+	dev_priv->dma_access = MGA_PAGPXFER;
+	dev_priv->wagp_enable = MGA_WAGP_ENABLE;
+
+	DRM_INFO("Initialized card for AGP DMA.\n");
+	return 0;
+}
+
+/**
+ * Create a "fake" drm_map_t for a pre-mapped range of PCI consistent memory.
+ *
+ * Unlike \c drm_addmap, this function just creates a \c drm_map_t wrapper for
+ * a block of PCI consistent memory.  \c drm_addmap, basically, converts a bus
+ * address to a virtual address.  However, \c drm_pci_alloc gives both the bus
+ * address and the virtual address for the memory region.  Not only is there
+ * no need to map it again, but mapping it again will cause problems.
+ * 
+ * \param dmah  DRM DMA handle returned by \c drm_pci_alloc.
+ * \param map_ptr  Location to store a pointer to the \c drm_map_t.
+ * 
+ * \returns
+ * On success, zero is returned.  Otherwise and error code suitable for
+ * returning from an ioctl is returned.
+ */
+static int mga_fake_addmap(drm_dma_handle_t * dmah, drm_map_t ** map_ptr)
+{
+	drm_map_t * map;
+
+
+	map = drm_alloc(sizeof(drm_map_t), DRM_MEM_DRIVER);
+	if (map == NULL) {
+		return DRM_ERR(ENOMEM);
+	}
+
+	map->offset = dmah->busaddr;
+	map->size = dmah->size;
+	map->type = _DRM_CONSISTENT;
+	map->flags = _DRM_READ_ONLY;
+	map->handle = dmah->vaddr;
+	map->mtrr = 0;
+	
+	*map_ptr = map;
+	
+	return 0;
+}
+
+/**
+ * Bootstrap the driver for PCI DMA.
+ * 
+ * \todo
+ * The algorithm for decreasing the size of the primary DMA buffer could be
+ * better.  The size should be rounded up to the nearest page size, then
+ * decrease the request size by a single page each pass through the loop.
+ *
+ * \todo
+ * Determine whether the maximum address passed to drm_pci_alloc is correct.
+ * The same goes for drm_addbufs_pci.
+ * 
+ * \sa mga_do_dma_bootstrap, mga_do_agp_dma_bootstrap
+ */
+static int mga_do_pci_dma_bootstrap(drm_device_t * dev,
+				    drm_mga_dma_bootstrap_t * dma_bs)
+{
+	drm_mga_private_t * const dev_priv = (drm_mga_private_t *) dev->dev_private;
+	const unsigned int warp_size = mga_warp_microcode_size(dev_priv);
+	unsigned int primary_size;
+	unsigned int bin_count;
+	int err;
+	drm_buf_desc_t req;
+
+	
+	if (dev->dma == NULL) {
+		DRM_ERROR("dev->dma is NULL\n");
+		return DRM_ERR(EFAULT);
+	}
+
+
+	/* The WARP microcode base address must be 256-byte aligned.
+	 */
+	dev_priv->warp_dmah = drm_pci_alloc(dev, warp_size, 0x100, 0x7fffffff);
+	err = mga_fake_addmap(dev_priv->warp_dmah, & dev_priv->warp);
+	if (err) {
+		DRM_ERROR("Unable to map WARP microcode\n");
+		return err;
+	}
+
+
+	/* Other than the bottom two bits being used to encode other
+	 * information, there don't appear to be any restrictions on the
+	 * alignment of the primary or secondary DMA buffers.
+	 */
+
+	dev_priv->primary_dmah = NULL;
+	for ( primary_size = dma_bs->primary_size
+	      ; primary_size != 0
+	      ; primary_size >>= 1 ) {
+		dev_priv->primary_dmah = drm_pci_alloc(dev, primary_size,
+					     0x04, 0x7fffffff);
+		if (dev_priv->primary_dmah != NULL) {
+			break;
+		}
+	}
+
+	if (dev_priv->primary_dmah == NULL) {
+		DRM_ERROR("Unable to allocate primary DMA region\n");
+		return DRM_ERR(ENOMEM);
+	}
+
+	if (dev_priv->primary_dmah->size != dma_bs->primary_size) {
+		DRM_INFO("Primary DMA buffer size reduced from %u to %u.\n",
+			 dma_bs->primary_size, 
+			 (unsigned) dev_priv->primary_dmah->size);
+		dma_bs->primary_size = dev_priv->primary_dmah->size;
+	}
+
+	err = mga_fake_addmap(dev_priv->primary_dmah, & dev_priv->primary);
+	if (err) {
+		DRM_ERROR("Unable to map primary DMA region\n");
+		return err;
+	}
+
+
+	for ( bin_count = dma_bs->secondary_bin_count
+	      ; bin_count > 0 
+	      ; bin_count-- ) {
+		(void) memset( &req, 0, sizeof(req) );
+		req.count = bin_count;
+		req.size = dma_bs->secondary_bin_size;
+
+		err = drm_addbufs_pci( dev, & req );
+		if (!err) {
+			break;
+		}
+	}
+	
+	if (bin_count == 0) {
+		DRM_ERROR("Unable to add secondary DMA buffers\n");
+		return err;
+	}
+
+	if (bin_count != dma_bs->secondary_bin_count) {
+		DRM_INFO("Secondary PCI DMA buffer bin count reduced from %u "
+			 "to %u.\n", dma_bs->secondary_bin_count, bin_count);
+
+		dma_bs->secondary_bin_count = bin_count;
+	}
+
+	dev_priv->dma_access = 0;
+	dev_priv->wagp_enable = 0;
+
+	dma_bs->agp_mode = 0;
+
+	DRM_INFO("Initialized card for PCI DMA.\n");
+	return 0;
+}
+
+
+static int mga_do_dma_bootstrap(drm_device_t * dev,
+				drm_mga_dma_bootstrap_t * dma_bs)
+{
+	const int is_agp = (dma_bs->agp_mode != 0) && drm_device_is_agp(dev);
+	int err;
+	drm_mga_private_t * const dev_priv =
+		(drm_mga_private_t *) dev->dev_private;
+
+
+	dev_priv->used_new_dma_init = 1;
+
+	/* The first steps are the same for both PCI and AGP based DMA.  Map
+	 * the cards MMIO registers and map a status page.
+	 */
+	err = drm_addmap( dev, dev_priv->mmio_base, dev_priv->mmio_size,
+			  _DRM_REGISTERS, _DRM_READ_ONLY, & dev_priv->mmio );
+	if (err) {
+		DRM_ERROR("Unable to map MMIO region\n");
+		return err;
+	}
+
+
+	err = drm_addmap( dev, 0, SAREA_MAX, _DRM_SHM,
+			  _DRM_READ_ONLY | _DRM_LOCKED | _DRM_KERNEL,
+			  & dev_priv->status );
+	if (err) {
+		DRM_ERROR("Unable to map status region\n");
+		return err;
+	}
+
+
+	/* The DMA initialization procedure is slightly different for PCI and
+	 * AGP cards.  AGP cards just allocate a large block of AGP memory and
+	 * carve off portions of it for internal uses.  The remaining memory
+	 * is returned to user-mode to be used for AGP textures.
+	 */
+
+	if (is_agp) {
+		err = mga_do_agp_dma_bootstrap(dev, dma_bs);
+	}
+	
+	/* If we attempted to initialize the card for AGP DMA but failed,
+	 * clean-up any mess that may have been created.
+	 */
+
+	if (err) {
+		mga_do_cleanup_dma(dev);
+	}
+
+
+	/* Not only do we want to try and initialized PCI cards for PCI DMA,
+	 * but we also try to initialized AGP cards that could not be
+	 * initialized for AGP DMA.  This covers the case where we have an AGP
+	 * card in a system with an unsupported AGP chipset.  In that case the
+	 * card will be detected as AGP, but we won't be able to allocate any
+	 * AGP memory, etc.
+	 */
+
+	if (!is_agp || err) {
+		err = mga_do_pci_dma_bootstrap(dev, dma_bs);
+	}
+
+
+	return err;
+}
+
+int mga_dma_bootstrap(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_mga_dma_bootstrap_t bootstrap;
+	int err;
+
+
+	DRM_COPY_FROM_USER_IOCTL(bootstrap,
+				 (drm_mga_dma_bootstrap_t __user *) data,
+				 sizeof(bootstrap));
+
+	err = mga_do_dma_bootstrap(dev, & bootstrap);
+	if (! err) {
+		static const int modes[] = { 0, 1, 2, 2, 4, 4, 4, 4 };
+		const drm_mga_private_t * const dev_priv = 
+			(drm_mga_private_t *) dev->dev_private;
+
+		if (dev_priv->agp_textures != NULL) {
+			bootstrap.texture_handle = dev_priv->agp_textures->offset;
+			bootstrap.texture_size = dev_priv->agp_textures->size;
+		}
+		else {
+			bootstrap.texture_handle = 0;
+			bootstrap.texture_size = 0;
+		}
+
+		bootstrap.agp_mode = modes[ bootstrap.agp_mode & 0x07 ];
+		if (DRM_COPY_TO_USER( (void __user *) data, & bootstrap,
+				     sizeof(bootstrap))) {
+			err = DRM_ERR(EFAULT);
+		}
+	}
+	else {
+		mga_do_cleanup_dma(dev);
+	}
+
+	return err;
+}
+
+
 static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 {
 	drm_mga_private_t *dev_priv;
@@ -443,42 +845,47 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 		return DRM_ERR(EINVAL);
 	}
 
-	dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
-	if (!dev_priv->mmio) {
-		DRM_ERROR("failed to find mmio region!\n");
-		return DRM_ERR(EINVAL);
-	}
-	dev_priv->status = drm_core_findmap(dev, init->status_offset);
-	if (!dev_priv->status) {
-		DRM_ERROR("failed to find status page!\n");
-		return DRM_ERR(EINVAL);
-	}
-	dev_priv->warp = drm_core_findmap(dev, init->warp_offset);
-	if (!dev_priv->warp) {
-		DRM_ERROR("failed to find warp microcode region!\n");
-		return DRM_ERR(EINVAL);
-	}
-	dev_priv->primary = drm_core_findmap(dev, init->primary_offset);
-	if (!dev_priv->primary) {
-		DRM_ERROR("failed to find primary dma region!\n");
-		return DRM_ERR(EINVAL);
-	}
-	dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
-	if (!dev->agp_buffer_map) {
-		DRM_ERROR("failed to find dma buffer region!\n");
-		return DRM_ERR(EINVAL);
+	if (! dev_priv->used_new_dma_init) {
+		dev_priv->status = drm_core_findmap(dev, init->status_offset);
+		if (!dev_priv->status) {
+			DRM_ERROR("failed to find status page!\n");
+			return DRM_ERR(EINVAL);
+		}
+		dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
+		if (!dev_priv->mmio) {
+			DRM_ERROR("failed to find mmio region!\n");
+			return DRM_ERR(EINVAL);
+		}
+		dev_priv->warp = drm_core_findmap(dev, init->warp_offset);
+		if (!dev_priv->warp) {
+			DRM_ERROR("failed to find warp microcode region!\n");
+			return DRM_ERR(EINVAL);
+		}
+		dev_priv->primary = drm_core_findmap(dev, init->primary_offset);
+		if (!dev_priv->primary) {
+			DRM_ERROR("failed to find primary dma region!\n");
+			return DRM_ERR(EINVAL);
+		}
+		dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
+		if (!dev->agp_buffer_map) {
+			DRM_ERROR("failed to find dma buffer region!\n");
+			return DRM_ERR(EINVAL);
+		}
+
+		drm_core_ioremap(dev_priv->warp, dev);
+		drm_core_ioremap(dev_priv->primary, dev);
+		drm_core_ioremap(dev->agp_buffer_map, dev);
 	}
 
 	dev_priv->sarea_priv =
 	    (drm_mga_sarea_t *) ((u8 *) dev_priv->sarea->handle +
 				 init->sarea_priv_offset);
 
-	drm_core_ioremap(dev_priv->warp, dev);
-	drm_core_ioremap(dev_priv->primary, dev);
-	drm_core_ioremap(dev->agp_buffer_map, dev);
-
 	if (!dev_priv->warp->handle ||
-	    !dev_priv->primary->handle || !dev->agp_buffer_map->handle) {
+	    !dev_priv->primary->handle ||
+	    ((dev_priv->dma_access != 0) &&
+	     ((dev->agp_buffer_map == NULL) ||
+	      (dev->agp_buffer_map->handle == NULL)))) {
 		DRM_ERROR("failed to ioremap agp regions!\n");
 		return DRM_ERR(ENOMEM);
 	}
@@ -538,6 +945,7 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 
 static int mga_do_cleanup_dma(drm_device_t * dev)
 {
+	int err = 0;
 	DRM_DEBUG("\n");
 
 	/* Make sure interrupts are disabled here because the uninstall ioctl
@@ -550,13 +958,73 @@ static int mga_do_cleanup_dma(drm_device_t * dev)
 	if (dev->dev_private) {
 		drm_mga_private_t *dev_priv = dev->dev_private;
 
-		if (dev_priv->warp != NULL)
+		if ((dev_priv->warp != NULL) 
+		    && (dev_priv->mmio->type != _DRM_CONSISTENT))
 			drm_core_ioremapfree(dev_priv->warp, dev);
-		if (dev_priv->primary != NULL)
+
+		if ((dev_priv->primary != NULL) 
+		    && (dev_priv->primary->type != _DRM_CONSISTENT))
 			drm_core_ioremapfree(dev_priv->primary, dev);
+
 		if (dev->agp_buffer_map != NULL)
 			drm_core_ioremapfree(dev->agp_buffer_map, dev);
 
+		if (dev_priv->used_new_dma_init) {
+			if (dev_priv->warp != NULL) {
+				drm_rmmap(dev, (void *) dev_priv->warp->offset);
+			}
+
+			if (dev_priv->primary != NULL) {
+				if (dev_priv->primary->type != _DRM_CONSISTENT) {
+					drm_rmmap(dev, (void *) dev_priv->primary->offset);
+				}
+				else {
+					drm_free(dev_priv->primary, sizeof(drm_map_t), DRM_MEM_DRIVER);
+				}
+			}
+
+			if (dev_priv->warp_dmah != NULL) {
+				drm_pci_free(dev, dev_priv->warp_dmah);
+				dev_priv->warp_dmah = NULL;
+			}
+
+			if (dev_priv->primary_dmah != NULL) {
+				drm_pci_free(dev, dev_priv->primary_dmah);
+				dev_priv->primary_dmah = NULL;
+			}
+
+			if (dev_priv->mmio != NULL) {
+				drm_rmmap(dev, (void *) dev_priv->mmio->offset);
+			}
+
+			if (dev_priv->status != NULL) {
+				drm_rmmap(dev, (void *) dev_priv->status->offset);
+			}
+
+			if (dev_priv->agp_mem != NULL) {
+				if (dev->agp_buffer_map != NULL) {
+					drm_rmmap(dev, (void *) dev->agp_buffer_map->offset);
+				}
+
+				if (dev_priv->agp_textures != NULL) {
+					drm_rmmap(dev, (void *) dev_priv->agp_textures->offset);
+					dev_priv->agp_textures = NULL;
+				}
+
+				drm_unbind_agp(dev_priv->agp_mem);
+
+				drm_free_agp(dev_priv->agp_mem, dev_priv->agp_pages);
+				dev_priv->agp_pages = 0;
+				dev_priv->agp_mem = NULL;
+			}
+
+			if ((dev->agp != NULL) && dev->agp->acquired) {
+				err = drm_agp_release(dev);
+			}
+
+			dev_priv->used_new_dma_init = 0;
+		}
+
 		dev_priv->warp = NULL;
 		dev_priv->primary = NULL;
 		dev_priv->mmio = NULL;
diff --git a/shared-core/mga_drm.h b/shared-core/mga_drm.h
index c3811fea..5b8f1564 100644
--- a/shared-core/mga_drm.h
+++ b/shared-core/mga_drm.h
@@ -73,6 +73,8 @@
 
 #define MGA_CARD_TYPE_G200	1
 #define MGA_CARD_TYPE_G400	2
+#define MGA_CARD_TYPE_G450	3       /* not currently used */
+#define MGA_CARD_TYPE_G550	4
 
 #define MGA_FRONT		0x1
 #define MGA_BACK		0x2
@@ -237,6 +239,14 @@ typedef struct _drm_mga_sarea {
 #define DRM_MGA_BLIT     0x08
 #define DRM_MGA_GETPARAM 0x09
 
+/* 3.2:
+ * ioctls for operating on fences.
+ */
+#define DRM_MGA_SET_FENCE      0x0a
+#define DRM_MGA_WAIT_FENCE     0x0b
+#define DRM_MGA_DMA_BOOTSTRAP  0x0c
+
+
 #define DRM_IOCTL_MGA_INIT     DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_INIT, drm_mga_init_t)
 #define DRM_IOCTL_MGA_FLUSH    DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_FLUSH, drm_lock_t)
 #define DRM_IOCTL_MGA_RESET    DRM_IO(  DRM_COMMAND_BASE + DRM_MGA_RESET)
@@ -247,6 +257,9 @@ typedef struct _drm_mga_sarea {
 #define DRM_IOCTL_MGA_ILOAD    DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_ILOAD, drm_mga_iload_t)
 #define DRM_IOCTL_MGA_BLIT     DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_BLIT, drm_mga_blit_t)
 #define DRM_IOCTL_MGA_GETPARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_MGA_GETPARAM, drm_mga_getparam_t)
+#define DRM_IOCTL_MGA_SET_FENCE     DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_SET_FENCE, uint32_t)
+#define DRM_IOCTL_MGA_WAIT_FENCE    DRM_IOWR(DRM_COMMAND_BASE + DRM_MGA_WAIT_FENCE, uint32_t)
+#define DRM_IOCTL_MGA_DMA_BOOTSTRAP DRM_IOWR(DRM_COMMAND_BASE + DRM_MGA_DMA_BOOTSTRAP, drm_mga_dma_bootstrap_t)
 
 typedef struct _drm_mga_warp_index {
 	int installed;
@@ -285,6 +298,74 @@ typedef struct drm_mga_init {
 	unsigned long buffers_offset;
 } drm_mga_init_t;
 
+
+typedef struct drm_mga_dma_bootstrap {
+	/**
+	 * \name AGP texture region
+	 * 
+	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, these fields will
+	 * be filled in with the actual AGP texture settings.
+	 * 
+	 * \warning
+	 * If these fields are non-zero, but dma_mga_dma_bootstrap::agp_mode
+	 * is zero, it means that PCI memory (most likely through the use of
+	 * an IOMMU) is being used for "AGP" textures.
+	 */
+	/*@{*/
+	drm_handle_t texture_handle;  /**< Handle used to map AGP textures. */
+	uint32_t     texture_size;    /**< Size of the AGP texture region. */
+	/*@}*/
+
+
+	/**
+	 * Requested size of the primary DMA region.
+	 * 
+	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
+	 * filled in with the actual AGP mode.  If AGP was not available
+	 */
+	uint32_t primary_size;
+
+
+	/**
+	 * Requested number of secondary DMA buffers.
+	 * 
+	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
+	 * filled in with the actual number of secondary DMA buffers
+	 * allocated.  Particularly when PCI DMA is used, this may be
+	 * (subtantially) less than the number requested.
+	 */
+	uint32_t secondary_bin_count;
+	
+	
+	/**
+	 * Requested size of each secondary DMA buffer.
+	 * 
+	 * While the kernel \b is free to reduce
+	 * dma_mga_dma_bootstrap::secondary_bin_count, it is \b not allowed
+	 * to reduce dma_mga_dma_bootstrap::secondary_bin_size.
+	 */
+	uint32_t secondary_bin_size;
+
+
+	/**
+	 * Bit-wise mask of AGPSTAT2_* values.  Currently only \c AGPSTAT2_1X,
+	 * \c AGPSTAT2_2X, and \c AGPSTAT2_4X are supported.  If this value is
+	 * zero, it means that PCI DMA should be used, even if AGP is
+	 * possible.
+	 * 
+	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
+	 * filled in with the actual AGP mode.  If AGP was not available
+	 * (i.e., PCI DMA was used), this value will be zero.
+	 */
+	uint32_t agp_mode;
+
+
+	/**
+	 * Desired AGP GART size, measured in megabytes.
+	 */
+	uint8_t agp_size;
+} drm_mga_dma_bootstrap_t;
+
 typedef struct drm_mga_clear {
 	unsigned int flags;
 	unsigned int clear_color;
@@ -328,6 +409,14 @@ typedef struct _drm_mga_blit {
  */
 #define MGA_PARAM_IRQ_NR            1
 
+/* 3.2: Query the actual card type.  The DDX only distinguishes between
+ * G200 chips and non-G200 chips, which it calls G400.  It turns out that
+ * there are some very sublte differences between the G4x0 chips and the G550
+ * chips.  Using this parameter query, a client-side driver can detect the
+ * difference between a G4x0 and a G550.
+ */
+#define MGA_PARAM_CARD_TYPE         2
+
 typedef struct drm_mga_getparam {
 	int param;
 	void __user *value;
diff --git a/shared-core/mga_drv.h b/shared-core/mga_drv.h
index 180c1235..7a1799b9 100644
--- a/shared-core/mga_drv.h
+++ b/shared-core/mga_drv.h
@@ -38,11 +38,11 @@
 
 #define DRIVER_NAME		"mga"
 #define DRIVER_DESC		"Matrox G200/G400"
-#define DRIVER_DATE		"20050606"
+#define DRIVER_DATE		"20050607"
 
 #define DRIVER_MAJOR		3
-#define DRIVER_MINOR		1
-#define DRIVER_PATCHLEVEL	3
+#define DRIVER_MINOR		2
+#define DRIVER_PATCHLEVEL	0
 
 typedef struct drm_mga_primary_buffer {
 	u8 *start;
@@ -87,9 +87,55 @@ typedef struct drm_mga_private {
 	int chipset;
 	int usec_timeout;
 
+	/**
+	 * If set, the new DMA initialization sequence was used.  This is
+	 * primarilly used to select how the driver should uninitialized its
+	 * internal DMA structures.
+	 */
+	int used_new_dma_init;
+
+	/**
+	 * If AGP memory is used for DMA buffers, this will be the value
+	 * \c MGA_PAGPXFER.  Otherwise, it will be zero (for a PCI transfer).
+	 */
+	u32 dma_access;
+
+	/**
+	 * If AGP memory is used for DMA buffers, this will be the value
+	 * \c MGA_WAGP_ENABLE.  Otherwise, it will be zero (for a PCI
+	 * transfer).
+	 */
+	u32 wagp_enable;
+
+	
+	/**
+	 * \name Handles for PCI consistent memory.
+	 * 
+	 * \sa drm_mga_private_t::primary, drm_mga_private_t::warp
+	 */
+	/*@{*/
+	drm_dma_handle_t * warp_dmah;    /**< Handle for WARP ucode region. */
+	drm_dma_handle_t * primary_dmah; /**< Handle for primary DMA region. */
+	/*@}*/
+
+
+	/**
+	 * \name MMIO region parameters.
+	 * 
+	 * \sa drm_mga_private_t::mmio
+	 */
+	/*@{*/
+	u32 mmio_base;             /**< Bus address of base of MMIO. */
+	u32 mmio_size;             /**< Size of the MMIO region. */
+	/*@}*/
+
 	u32 clear_cmd;
 	u32 maccess;
 
+	wait_queue_head_t fence_queue;
+	atomic_t last_fence_retired;
+	u32 next_fence_to_post;
+
 	unsigned int fb_cpp;
 	unsigned int front_offset;
 	unsigned int front_pitch;
@@ -109,10 +155,14 @@ typedef struct drm_mga_private {
 	drm_local_map_t *warp;
 	drm_local_map_t *primary;
 	drm_local_map_t *agp_textures;
+	
+	DRM_AGP_MEM *agp_mem;
+	unsigned int agp_pages;
 } drm_mga_private_t;
 
 				/* mga_dma.c */
 extern int mga_driver_preinit(drm_device_t * dev, unsigned long flags);
+extern int mga_dma_bootstrap(DRM_IOCTL_ARGS);
 extern int mga_dma_init(DRM_IOCTL_ARGS);
 extern int mga_dma_flush(DRM_IOCTL_ARGS);
 extern int mga_dma_reset(DRM_IOCTL_ARGS);
@@ -134,6 +184,8 @@ extern unsigned int mga_warp_microcode_size(const drm_mga_private_t * dev_priv);
 extern int mga_warp_install_microcode(drm_mga_private_t * dev_priv);
 extern int mga_warp_init(drm_mga_private_t * dev_priv);
 
+				/* mga_irq.c */
+extern int mga_driver_fence_wait(drm_device_t * dev, unsigned int *sequence);
 extern int mga_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence);
 extern irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS);
 extern void mga_driver_irq_preinstall(drm_device_t * dev);
@@ -520,6 +572,12 @@ do {									\
  */
 #define MGA_EXEC 			0x0100
 
+/* AGP PLL encoding (for G200 only).
+ */
+#define MGA_AGP_PLL 			0x1e4c
+#	define MGA_AGP2XPLL_DISABLE		(0 << 0)
+#	define MGA_AGP2XPLL_ENABLE		(1 << 0)
+
 /* Warp registers
  */
 #define MGA_WR0				0x2d00
diff --git a/shared-core/mga_irq.c b/shared-core/mga_irq.c
index 99a23fe1..16dfd44b 100644
--- a/shared-core/mga_irq.c
+++ b/shared-core/mga_irq.c
@@ -40,6 +40,7 @@ irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
 	drm_device_t *dev = (drm_device_t *) arg;
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 	int status;
+	int handled = 0;
 
 	status = MGA_READ(MGA_STATUS);
 
@@ -49,6 +50,30 @@ irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
 		atomic_inc(&dev->vbl_received);
 		DRM_WAKEUP(&dev->vbl_queue);
 		drm_vbl_send_signals(dev);
+		handled = 1;
+	}
+
+	/* SOFTRAP interrupt */
+	if (status & MGA_SOFTRAPEN) {
+		const u32 prim_start = MGA_READ(MGA_PRIMADDRESS);
+		const u32 prim_end   = MGA_READ(MGA_PRIMEND);
+
+
+		MGA_WRITE(MGA_ICLEAR, MGA_SOFTRAPICLR);
+
+		/* In addition to clearing the interrupt-pending bit, we
+		 * have to write to MGA_PRIMEND to re-start the DMA operation.
+		 */
+		if ( (prim_start & ~0x03) != (prim_end & ~0x03) ) {
+			MGA_WRITE(MGA_PRIMEND, prim_end);
+		}
+
+		atomic_inc(&dev_priv->last_fence_retired);
+		DRM_WAKEUP(&dev_priv->fence_queue);
+		handled = 1;
+	}
+
+	if ( handled ) {
 		return IRQ_HANDLED;
 	}
 	return IRQ_NONE;
@@ -72,6 +97,25 @@ int mga_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence)
 	return ret;
 }
 
+int mga_driver_fence_wait(drm_device_t * dev, unsigned int *sequence)
+{
+	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
+	unsigned int cur_fence;
+	int ret = 0;
+
+	/* Assume that the user has missed the current sequence number
+	 * by about a day rather than she wants to wait for years
+	 * using fences.
+	 */
+	DRM_WAIT_ON(ret, dev_priv->fence_queue, 3 * DRM_HZ,
+		    (((cur_fence = atomic_read(&dev_priv->last_fence_retired))
+		      - *sequence) <= (1 << 23)));
+
+	*sequence = cur_fence;
+
+	return ret;
+}
+
 void mga_driver_irq_preinstall(drm_device_t * dev)
 {
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
@@ -86,8 +130,10 @@ void mga_driver_irq_postinstall(drm_device_t * dev)
 {
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 
-	/* Turn on VBL interrupt */
-	MGA_WRITE(MGA_IEN, MGA_VLINEIEN);
+	DRM_INIT_WAITQUEUE( &dev_priv->fence_queue );
+
+	/* Turn on vertical blank interrupt and soft trap interrupt. */
+	MGA_WRITE(MGA_IEN, MGA_VLINEIEN | MGA_SOFTRAPEN);
 }
 
 void mga_driver_irq_uninstall(drm_device_t * dev)
diff --git a/shared-core/mga_state.c b/shared-core/mga_state.c
index e7921bd2..273854cd 100644
--- a/shared-core/mga_state.c
+++ b/shared-core/mga_state.c
@@ -264,7 +264,7 @@ static __inline__ void mga_g200_emit_pipe(drm_mga_private_t * dev_priv)
 		  MGA_DMAPAD, 0xffffffff,
 		  MGA_DMAPAD, 0xffffffff,
 		  MGA_WIADDR, (dev_priv->warp_pipe_phys[pipe] |
-			       MGA_WMODE_START | MGA_WAGP_ENABLE));
+			       MGA_WMODE_START | dev_priv->wagp_enable));
 
 	ADVANCE_DMA();
 }
@@ -345,7 +345,7 @@ static __inline__ void mga_g400_emit_pipe(drm_mga_private_t * dev_priv)
 		  MGA_DMAPAD, 0xffffffff,
 		  MGA_DMAPAD, 0xffffffff,
 		  MGA_WIADDR2, (dev_priv->warp_pipe_phys[pipe] |
-				MGA_WMODE_START | MGA_WAGP_ENABLE));
+				MGA_WMODE_START | dev_priv->wagp_enable));
 
 	ADVANCE_DMA();
 }
@@ -455,7 +455,7 @@ static int mga_verify_state(drm_mga_private_t * dev_priv)
 	if (dirty & MGA_UPLOAD_TEX0)
 		ret |= mga_verify_tex(dev_priv, 0);
 
-	if (dev_priv->chipset == MGA_CARD_TYPE_G400) {
+	if (dev_priv->chipset >= MGA_CARD_TYPE_G400) {
 		if (dirty & MGA_UPLOAD_TEX1)
 			ret |= mga_verify_tex(dev_priv, 1);
 
@@ -679,7 +679,7 @@ static void mga_dma_dispatch_vertex(drm_device_t * dev, drm_buf_t * buf)
 				  MGA_SECADDRESS, (address |
 						   MGA_DMA_VERTEX),
 				  MGA_SECEND, ((address + length) |
-					       MGA_PAGPXFER));
+					       dev_priv->dma_access));
 
 			ADVANCE_DMA();
 		} while (++i < sarea_priv->nbox);
@@ -725,7 +725,7 @@ static void mga_dma_dispatch_indices(drm_device_t * dev, drm_buf_t * buf,
 				  MGA_DMAPAD, 0x00000000,
 				  MGA_SETUPADDRESS, address + start,
 				  MGA_SETUPEND, ((address + end) |
-						 MGA_PAGPXFER));
+						 dev_priv->dma_access));
 
 			ADVANCE_DMA();
 		} while (++i < sarea_priv->nbox);
@@ -752,7 +752,7 @@ static void mga_dma_dispatch_iload(drm_device_t * dev, drm_buf_t * buf,
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_buf_priv_t *buf_priv = buf->dev_private;
 	drm_mga_context_regs_t *ctx = &dev_priv->sarea_priv->context_state;
-	u32 srcorg = buf->bus_address | MGA_SRCACC_AGP | MGA_SRCMAP_SYSMEM;
+	u32 srcorg = buf->bus_address | dev_priv->dma_access | MGA_SRCMAP_SYSMEM;
 	u32 y2;
 	DMA_LOCALS;
 	DRM_DEBUG("buf=%d used=%d\n", buf->idx, buf->used);
@@ -1087,6 +1087,9 @@ static int mga_getparam(DRM_IOCTL_ARGS)
 	case MGA_PARAM_IRQ_NR:
 		value = dev->irq;
 		break;
+	case MGA_PARAM_CARD_TYPE:
+		value = dev_priv->chipset;
+		break;
 	default:
 		return DRM_ERR(EINVAL);
 	}
@@ -1099,6 +1102,67 @@ static int mga_getparam(DRM_IOCTL_ARGS)
 	return 0;
 }
 
+static int mga_set_fence(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_mga_private_t *dev_priv = dev->dev_private;
+	u32 temp;
+	DMA_LOCALS;
+
+	if (!dev_priv) {
+		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
+		return DRM_ERR(EINVAL);
+	}
+
+	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
+
+	/* I would normal do this assignment in the declaration of temp,
+	 * but dev_priv may be NULL.
+	 */
+
+	temp = dev_priv->next_fence_to_post;
+	dev_priv->next_fence_to_post++;
+
+	BEGIN_DMA(1);
+	DMA_BLOCK(MGA_DMAPAD, 0x00000000,
+		  MGA_DMAPAD, 0x00000000,
+		  MGA_DMAPAD, 0x00000000,
+		  MGA_SOFTRAP, 0x00000000);
+	ADVANCE_DMA();
+
+	if (DRM_COPY_TO_USER( (u32 __user *) data, & temp, sizeof(u32))) {
+		DRM_ERROR("copy_to_user\n");
+		return DRM_ERR(EFAULT);
+	}
+
+	return 0;
+}
+
+static int mga_wait_fence(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_mga_private_t *dev_priv = dev->dev_private;
+	u32 fence;
+
+	if (!dev_priv) {
+		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
+		return DRM_ERR(EINVAL);
+	}
+
+	DRM_COPY_FROM_USER_IOCTL(fence, (u32 __user *) data, sizeof(u32));
+
+	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
+
+	mga_driver_fence_wait(dev, & fence);
+
+	if (DRM_COPY_TO_USER( (u32 __user *) data, & fence, sizeof(u32))) {
+		DRM_ERROR("copy_to_user\n");
+		return DRM_ERR(EFAULT);
+	}
+
+	return 0;
+}
+
 drm_ioctl_desc_t mga_ioctls[] = {
 	[DRM_IOCTL_NR(DRM_MGA_INIT)] = {mga_dma_init, 1, 1},
 	[DRM_IOCTL_NR(DRM_MGA_FLUSH)] = {mga_dma_flush, 1, 0},
@@ -1110,6 +1174,10 @@ drm_ioctl_desc_t mga_ioctls[] = {
 	[DRM_IOCTL_NR(DRM_MGA_ILOAD)] = {mga_dma_iload, 1, 0},
 	[DRM_IOCTL_NR(DRM_MGA_BLIT)] = {mga_dma_blit, 1, 0},
 	[DRM_IOCTL_NR(DRM_MGA_GETPARAM)] = {mga_getparam, 1, 0},
+	[DRM_IOCTL_NR(DRM_MGA_SET_FENCE)] = {mga_set_fence, 1, 0},
+	[DRM_IOCTL_NR(DRM_MGA_WAIT_FENCE)] = {mga_wait_fence, 1, 0},
+	[DRM_IOCTL_NR(DRM_MGA_DMA_BOOTSTRAP)] = {mga_dma_bootstrap, 1, 1},
+
 };
 
 int mga_max_ioctl = DRM_ARRAY_SIZE(mga_ioctls);
diff --git a/shared-core/mga_warp.c b/shared-core/mga_warp.c
index b8fe48d1..304d9ff6 100644
--- a/shared-core/mga_warp.c
+++ b/shared-core/mga_warp.c
@@ -80,6 +80,7 @@ unsigned int mga_warp_microcode_size(const drm_mga_private_t * dev_priv)
 {
 	switch (dev_priv->chipset) {
 	case MGA_CARD_TYPE_G400:
+	case MGA_CARD_TYPE_G550:
 		return PAGE_ALIGN(mga_warp_g400_microcode_size);
 	case MGA_CARD_TYPE_G200:
 		return PAGE_ALIGN(mga_warp_g200_microcode_size);
@@ -148,6 +149,7 @@ int mga_warp_install_microcode(drm_mga_private_t * dev_priv)
 
 	switch (dev_priv->chipset) {
 	case MGA_CARD_TYPE_G400:
+	case MGA_CARD_TYPE_G550:
 		return mga_warp_install_g400_microcode(dev_priv);
 	case MGA_CARD_TYPE_G200:
 		return mga_warp_install_g200_microcode(dev_priv);
@@ -166,6 +168,7 @@ int mga_warp_init(drm_mga_private_t * dev_priv)
 	 */
 	switch (dev_priv->chipset) {
 	case MGA_CARD_TYPE_G400:
+	case MGA_CARD_TYPE_G550:
 		MGA_WRITE(MGA_WIADDR2, MGA_WMODE_SUSPEND);
 		MGA_WRITE(MGA_WGETMSB, 0x00000E00);
 		MGA_WRITE(MGA_WVRTXSZ, 0x00001807);