radeon: overhaul ring interactions

emit in 16-dword blocks, emit irqs at same time as everything else

6 files changed, 77 insertions, 45 deletions

diff --git a/linux-core/radeon_buffer.c b/linux-core/radeon_buffer.c
index e88378a4..571a0b9c 100644
--- a/linux-core/radeon_buffer.c
+++ b/linux-core/radeon_buffer.c
@@ -170,10 +170,12 @@ void radeon_emit_copy_blit(struct drm_device * dev,
 		ADVANCE_RING();
 	}
 
-	BEGIN_RING(4);
+	BEGIN_RING(6);
 	OUT_RING(CP_PACKET0(RADEON_RB2D_DSTCACHE_CTLSTAT, 0));
 	OUT_RING(RADEON_RB2D_DC_FLUSH_ALL);
 	RADEON_WAIT_UNTIL_2D_IDLE();
+	OUT_RING(CP_PACKET2());
+	OUT_RING(CP_PACKET2());
 	ADVANCE_RING();
 
 	COMMIT_RING();
@@ -265,10 +267,14 @@ void radeon_emit_solid_fill(struct drm_device * dev,
 		ADVANCE_RING();
 	}
 
-	BEGIN_RING(4);
+	BEGIN_RING(8);
 	OUT_RING(CP_PACKET0(RADEON_RB2D_DSTCACHE_CTLSTAT, 0));
 	OUT_RING(RADEON_RB2D_DC_FLUSH_ALL);
 	RADEON_WAIT_UNTIL_2D_IDLE();
+	OUT_RING(CP_PACKET2());
+	OUT_RING(CP_PACKET2());
+	OUT_RING(CP_PACKET2());
+	OUT_RING(CP_PACKET2());
 	ADVANCE_RING();
 
 	COMMIT_RING();
diff --git a/linux-core/radeon_fence.c b/linux-core/radeon_fence.c
index 591ad53b..b662da21 100644
--- a/linux-core/radeon_fence.c
+++ b/linux-core/radeon_fence.c
@@ -39,7 +39,6 @@ int radeon_fence_emit_sequence(struct drm_device *dev, uint32_t class,
 			       uint32_t *native_type)
 {
 	struct drm_radeon_private *dev_priv = (struct drm_radeon_private *) dev->dev_private;
-	RING_LOCALS;
 
 	if (!dev_priv)
 		return -EINVAL;
diff --git a/linux-core/radeon_gem.c b/linux-core/radeon_gem.c
index f5d6b94a..a785041e 100644
--- a/linux-core/radeon_gem.c
+++ b/linux-core/radeon_gem.c
@@ -1086,7 +1086,7 @@ int radeon_gem_object_unpin(struct drm_gem_object *obj)
 
 #define RADEON_NUM_IB (RADEON_IB_MEMORY / RADEON_IB_SIZE)
 
-int radeon_gem_ib_get(struct drm_radeon_cs_parser *parser, uint32_t *card_offset)
+int radeon_gem_ib_get(struct drm_radeon_cs_parser *parser)
 {
 	int i, index = -1;
 	int ret;
@@ -1132,8 +1132,8 @@ int radeon_gem_ib_get(struct drm_radeon_cs_parser *parser, uint32_t *card_offset
 		return -EINVAL;
 	}
 		
-	*card_offset = dev_priv->gart_vm_start + dev_priv->ib_objs[index]->bo->offset;
 	parser->ib = dev_priv->ib_objs[index]->kmap.virtual;
+	parser->card_offset = dev_priv->gart_vm_start + dev_priv->ib_objs[index]->bo->offset;
 	dev_priv->ib_alloc_bitmap |= (1 << i);
 	return 0;
 }
@@ -1150,6 +1150,7 @@ static void radeon_gem_ib_free(struct drm_radeon_cs_parser *parser)
 		if (dev_priv->ib_objs[i]->kmap.virtual == parser->ib) {
 			/* emit a fence object */
 			ret = drm_fence_buffer_objects(dev, NULL, 0, NULL, &fence);
+			dev_priv->irq_emitted = 0;
 			if (ret) {
 				drm_putback_buffer_objects(dev);
 			}
diff --git a/shared-core/radeon_cs.c b/shared-core/radeon_cs.c
index 31cd53db..4a231962 100644
--- a/shared-core/radeon_cs.c
+++ b/shared-core/radeon_cs.c
@@ -38,10 +38,8 @@ int radeon_cs2_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
 	struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
 	uint64_t *chunk_array;
 	uint64_t *chunk_array_ptr;
-	uint32_t card_offset;
 	long size;
 	int r, i;
-	RING_LOCALS;
 
 	/* set command stream id to 0 which is fake id */
 	cs_id = 0;
@@ -144,7 +142,7 @@ int radeon_cs2_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
 	}
 
 	/* get ib */
-	r = dev_priv->cs.ib_get(&parser, &card_offset);
+	r = dev_priv->cs.ib_get(&parser);
 	if (r) {
 		DRM_ERROR("ib_get failed\n");
 		goto out;
@@ -156,16 +154,8 @@ int radeon_cs2_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
 		goto out;
 	}
 
-	BEGIN_RING(4);
-	OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
-	OUT_RING(card_offset);
-	OUT_RING(parser.chunks[parser.ib_index].length_dw);
-	OUT_RING(CP_PACKET2());
-	ADVANCE_RING();
-
 	/* emit cs id sequence */
-	dev_priv->cs.id_emit(dev, &cs_id);
-	COMMIT_RING();
+	dev_priv->cs.id_emit(&parser, &cs_id);
 
 	cs->cs_id = cs_id;
 		
@@ -194,7 +184,6 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
 	long size;
 	int r;
 	struct drm_radeon_kernel_chunk chunk_fake[1];
-	RING_LOCALS;
 
 	/* set command stream id to 0 which is fake id */
 	cs_id = 0;
@@ -237,7 +226,7 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
 	parser.reloc_index = -1;
 
 	/* get ib */
-	r = dev_priv->cs.ib_get(&parser, &card_offset);
+	r = dev_priv->cs.ib_get(&parser);
 	if (r) {
 		goto out;
 	}
@@ -248,15 +237,8 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
 		goto out;
 	}
 
-	BEGIN_RING(4);
-	OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
-	OUT_RING(card_offset);
-	OUT_RING(cs->dwords);
-	OUT_RING(CP_PACKET2());
-	ADVANCE_RING();
-
 	/* emit cs id sequence */
-	dev_priv->cs.id_emit(dev, &cs_id);
+	dev_priv->cs.id_emit(&parser, &cs_id);
 	COMMIT_RING();
 
 	cs->cs_id = cs_id;
@@ -518,37 +500,75 @@ uint32_t radeon_cs_id_get(struct drm_radeon_private *radeon)
 	return (radeon->cs.id_scnt | radeon->cs.id_wcnt);
 }
 
-void r100_cs_id_emit(struct drm_device *dev, uint32_t *id)
+void r100_cs_id_emit(struct drm_radeon_cs_parser *parser, uint32_t *id)
 {
-	drm_radeon_private_t *dev_priv = dev->dev_private;
+	drm_radeon_private_t *dev_priv = parser->dev->dev_private;
 	RING_LOCALS;
 
+	dev_priv->irq_emitted = radeon_update_breadcrumb(parser->dev);
 	/* ISYNC_CNTL should have CPSCRACTH bit set */
 	*id = radeon_cs_id_get(dev_priv);
 	/* emit id in SCRATCH4 (not used yet in old drm) */
-	BEGIN_RING(2);
+	BEGIN_RING(10);
+	OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
+	OUT_RING(parser->card_offset);
+	OUT_RING(parser->chunks[parser->ib_index].length_dw);
+	OUT_RING(CP_PACKET2());
 	OUT_RING(CP_PACKET0(RADEON_SCRATCH_REG4, 0));
 	OUT_RING(*id);
+	OUT_RING_REG(RADEON_LAST_SWI_REG, dev_priv->irq_emitted);
+	OUT_RING_REG(RADEON_GEN_INT_STATUS, RADEON_SW_INT_FIRE);
 	ADVANCE_RING();	
+	COMMIT_RING();
+
 }
 
-void r300_cs_id_emit(struct drm_device *dev, uint32_t *id)
+void r300_cs_id_emit(struct drm_radeon_cs_parser *parser, uint32_t *id)
 {
-	drm_radeon_private_t *dev_priv = dev->dev_private;
+	drm_radeon_private_t *dev_priv = parser->dev->dev_private;
+	int i;
 	RING_LOCALS;
 
+	dev_priv->irq_emitted = radeon_update_breadcrumb(parser->dev);
+
 	/* ISYNC_CNTL should not have CPSCRACTH bit set */
 	*id = radeon_cs_id_get(dev_priv);
+
 	/* emit id in SCRATCH6 */
-	BEGIN_RING(8);
-	OUT_RING(CP_PACKET0(R300_CP_RESYNC_ADDR, 0));
+	BEGIN_RING(16);
+	OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
+	OUT_RING(parser->card_offset);
+	OUT_RING(parser->chunks[parser->ib_index].length_dw);
+	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
+	OUT_RING(0);
+	for (i = 0; i < 11; i++) /* emit fillers like fglrx */
+		OUT_RING(CP_PACKET2());
+	ADVANCE_RING();
+	COMMIT_RING();
+
+	BEGIN_RING(16);
+	OUT_RING_REG(R300_RB3D_DSTCACHE_CTLSTAT, R300_RB3D_DC_FLUSH);
+	OUT_RING(CP_PACKET0(R300_CP_RESYNC_ADDR, 1));
 	OUT_RING(6);
-	OUT_RING(CP_PACKET0(R300_CP_RESYNC_DATA, 0));
 	OUT_RING(*id);
-	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
-	OUT_RING(R300_RB3D_DC_FINISH);
+	OUT_RING_REG(R300_RB3D_DSTCACHE_CTLSTAT, R300_RB3D_DC_FINISH|R300_RB3D_DC_FLUSH);
+	/* emit inline breadcrumb for TTM fencing */
+#if 1
 	RADEON_WAIT_UNTIL_3D_IDLE();
+	OUT_RING_REG(RADEON_LAST_SWI_REG, dev_priv->irq_emitted);
+#else
+	OUT_RING(CP_PACKET0(R300_CP_RESYNC_ADDR, 1));
+	OUT_RING(3); /* breadcrumb register */
+	OUT_RING(dev_priv->irq_emitted);
+	OUT_RING(CP_PACKET2());
+#endif
+	OUT_RING_REG(RADEON_GEN_INT_STATUS, RADEON_SW_INT_FIRE);
+	OUT_RING(CP_PACKET2());
+	OUT_RING(CP_PACKET2());
+	OUT_RING(CP_PACKET2());
 	ADVANCE_RING();	
+	COMMIT_RING();
+
 }
 
 uint32_t r100_cs_id_last_get(struct drm_device *dev)
diff --git a/shared-core/radeon_drv.h b/shared-core/radeon_drv.h
index 8f77c850..83478a27 100644
--- a/shared-core/radeon_drv.h
+++ b/shared-core/radeon_drv.h
@@ -308,6 +308,7 @@ struct drm_radeon_cs_parser {
 	struct drm_radeon_kernel_chunk *chunks;
 	int ib_index;
 	int reloc_index;
+	uint32_t card_offset;
 	void *ib;
 };
 
@@ -319,12 +320,12 @@ struct drm_radeon_cs_priv {
 	uint32_t id_last_scnt;
 
 	int (*parse)(struct drm_radeon_cs_parser *parser);
-	void (*id_emit)(struct drm_device *dev, uint32_t *id);
+	void (*id_emit)(struct drm_radeon_cs_parser *parser, uint32_t *id);
 	uint32_t (*id_last_get)(struct drm_device *dev);
 	/* this ib handling callback are for hidding memory manager drm
 	 * from memory manager less drm, free have to emit ib discard
 	 * sequence into the ring */
-	int (*ib_get)(struct drm_radeon_cs_parser *parser, uint32_t *card_offset);
+	int (*ib_get)(struct drm_radeon_cs_parser *parser);
 	uint32_t (*ib_get_ptr)(struct drm_device *dev, void *ib);
 	void (*ib_free)(struct drm_radeon_cs_parser *parser);
 	/* do a relocation either MM or non-MM */
@@ -459,6 +460,7 @@ typedef struct drm_radeon_private {
 	struct drm_radeon_cs_priv cs;
 
 	struct radeon_pm_regs pmregs;
+	int irq_emitted;
 	atomic_t irq_received;
 } drm_radeon_private_t;
 
diff --git a/shared-core/radeon_irq.c b/shared-core/radeon_irq.c
index 311901c0..46a27943 100644
--- a/shared-core/radeon_irq.c
+++ b/shared-core/radeon_irq.c
@@ -223,15 +223,19 @@ int radeon_emit_irq(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	unsigned int ret;
+	int i;
 	RING_LOCALS;
 
-	ret = radeon_update_breadcrumb(dev);
+	if (!dev_priv->irq_emitted) {
+		ret = radeon_update_breadcrumb(dev);
 
-	BEGIN_RING(4);
-	OUT_RING_REG(RADEON_LAST_SWI_REG, ret);
-	OUT_RING_REG(RADEON_GEN_INT_STATUS, RADEON_SW_INT_FIRE);
-	ADVANCE_RING();
-	COMMIT_RING();
+		BEGIN_RING(4);
+		OUT_RING_REG(RADEON_LAST_SWI_REG, ret);
+		OUT_RING_REG(RADEON_GEN_INT_STATUS, RADEON_SW_INT_FIRE);
+		ADVANCE_RING();
+		COMMIT_RING();
+	} else
+		ret = dev_priv->irq_emitted;
 
 	return ret;
 }