drm/radeon/kms: Fix up vertical blank interrupt support.

Fixes 3D apps timing out in the WAIT_VBLANK ioctl.

AVIVO bits compile-tested only.

Signed-off-by: Michel Dänzer <daenzer@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

9 files changed, 217 insertions, 77 deletions

diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index f1ba8ff..e1a6e82 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -254,6 +254,72 @@ void r100_mc_fini(struct radeon_device *rdev)
 
 
 /*
+ * Interrupts
+ */
+int r100_irq_set(struct radeon_device *rdev)
+{
+	uint32_t tmp = 0;
+
+	if (rdev->irq.sw_int) {
+		tmp |= RADEON_SW_INT_ENABLE;
+	}
+	if (rdev->irq.crtc_vblank_int[0]) {
+		tmp |= RADEON_CRTC_VBLANK_MASK;
+	}
+	if (rdev->irq.crtc_vblank_int[1]) {
+		tmp |= RADEON_CRTC2_VBLANK_MASK;
+	}
+	WREG32(RADEON_GEN_INT_CNTL, tmp);
+	return 0;
+}
+
+static inline uint32_t r100_irq_ack(struct radeon_device *rdev)
+{
+	uint32_t irqs = RREG32(RADEON_GEN_INT_STATUS);
+	uint32_t irq_mask = RADEON_SW_INT_TEST | RADEON_CRTC_VBLANK_STAT |
+		RADEON_CRTC2_VBLANK_STAT;
+
+	if (irqs) {
+		WREG32(RADEON_GEN_INT_STATUS, irqs);
+	}
+	return irqs & irq_mask;
+}
+
+int r100_irq_process(struct radeon_device *rdev)
+{
+	uint32_t status;
+
+	status = r100_irq_ack(rdev);
+	if (!status) {
+		return IRQ_NONE;
+	}
+	while (status) {
+		/* SW interrupt */
+		if (status & RADEON_SW_INT_TEST) {
+			radeon_fence_process(rdev);
+		}
+		/* Vertical blank interrupts */
+		if (status & RADEON_CRTC_VBLANK_STAT) {
+			drm_handle_vblank(rdev->ddev, 0);
+		}
+		if (status & RADEON_CRTC2_VBLANK_STAT) {
+			drm_handle_vblank(rdev->ddev, 1);
+		}
+		status = r100_irq_ack(rdev);
+	}
+	return IRQ_HANDLED;
+}
+
+u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc)
+{
+	if (crtc == 0)
+		return RREG32(RADEON_CRTC_CRNT_FRAME);
+	else
+		return RREG32(RADEON_CRTC2_CRNT_FRAME);
+}
+
+
+/*
  * Fence emission
  */
 void r100_fence_ring_emit(struct radeon_device *rdev,
diff --git a/drivers/gpu/drm/radeon/r500_reg.h b/drivers/gpu/drm/radeon/r500_reg.h
index 036691b..e1d5e03 100644
--- a/drivers/gpu/drm/radeon/r500_reg.h
+++ b/drivers/gpu/drm/radeon/r500_reg.h
@@ -350,6 +350,7 @@
 #define AVIVO_D1CRTC_BLANK_CONTROL                              0x6084
 #define AVIVO_D1CRTC_INTERLACE_CONTROL                          0x6088
 #define AVIVO_D1CRTC_INTERLACE_STATUS                           0x608c
+#define AVIVO_D1CRTC_FRAME_COUNT                                0x60a4
 #define AVIVO_D1CRTC_STEREO_CONTROL                             0x60c4
 
 /* master controls */
@@ -438,14 +439,15 @@
 #       define AVIVO_DC_LB_DISP1_END_ADR_SHIFT  4
 #       define AVIVO_DC_LB_DISP1_END_ADR_MASK   0x7ff
 
-#define R500_DxMODE_INT_MASK 0x6540
-#define R500_D1MODE_INT_MASK (1<<0)
-#define R500_D2MODE_INT_MASK (1<<8)
-
 #define AVIVO_D1MODE_DATA_FORMAT                0x6528
 #       define AVIVO_D1MODE_INTERLEAVE_EN       (1 << 0)
 #define AVIVO_D1MODE_DESKTOP_HEIGHT             0x652C
+#define AVIVO_D1MODE_VBLANK_STATUS              0x6534
+#       define AVIVO_VBLANK_ACK                 (1 << 4)
 #define AVIVO_D1MODE_VLINE_START_END            0x6538
+#define AVIVO_DxMODE_INT_MASK                   0x6540
+#       define AVIVO_D1MODE_INT_MASK            (1 << 0)
+#       define AVIVO_D2MODE_INT_MASK            (1 << 8)
 #define AVIVO_D1MODE_VIEWPORT_START             0x6580
 #define AVIVO_D1MODE_VIEWPORT_SIZE              0x6584
 #define AVIVO_D1MODE_EXT_OVERSCAN_LEFT_RIGHT    0x6588
@@ -475,6 +477,7 @@
 #define AVIVO_D2CRTC_BLANK_CONTROL                              0x6884
 #define AVIVO_D2CRTC_INTERLACE_CONTROL                          0x6888
 #define AVIVO_D2CRTC_INTERLACE_STATUS                           0x688c
+#define AVIVO_D2CRTC_FRAME_COUNT                                0x68a4
 #define AVIVO_D2CRTC_STEREO_CONTROL                             0x68c4
 
 #define AVIVO_D2GRPH_ENABLE                                     0x6900
@@ -497,6 +500,7 @@
 #define AVIVO_D2CUR_SIZE                        0x6c10
 #define AVIVO_D2CUR_POSITION                    0x6c14
 
+#define AVIVO_D2MODE_VBLANK_STATUS              0x6d34
 #define AVIVO_D2MODE_VLINE_START_END            0x6d38
 #define AVIVO_D2MODE_VIEWPORT_START             0x6d80
 #define AVIVO_D2MODE_VIEWPORT_SIZE              0x6d84
@@ -748,4 +752,8 @@
 #	define AVIVO_I2C_EN							(1 << 0)
 #	define AVIVO_I2C_RESET						(1 << 8)
 
+#define AVIVO_DISP_INTERRUPT_STATUS                             0x7edc
+#       define AVIVO_D1_VBLANK_INTERRUPT                        (1 << 4)
+#       define AVIVO_D2_VBLANK_INTERRUPT                        (1 << 5)
+
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index b1d945b..3461127 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -574,6 +574,7 @@ struct radeon_asic {
 	void (*ring_start)(struct radeon_device *rdev);
 	int (*irq_set)(struct radeon_device *rdev);
 	int (*irq_process)(struct radeon_device *rdev);
+	u32 (*get_vblank_counter)(struct radeon_device *rdev, int crtc);
 	void (*fence_ring_emit)(struct radeon_device *rdev, struct radeon_fence *fence);
 	int (*cs_parse)(struct radeon_cs_parser *p);
 	int (*copy_blit)(struct radeon_device *rdev,
@@ -862,6 +863,7 @@ static inline void radeon_ring_write(struct radeon_device *rdev, uint32_t v)
 #define radeon_ring_start(rdev) (rdev)->asic->ring_start((rdev))
 #define radeon_irq_set(rdev) (rdev)->asic->irq_set((rdev))
 #define radeon_irq_process(rdev) (rdev)->asic->irq_process((rdev))
+#define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->get_vblank_counter((rdev), (crtc))
 #define radeon_fence_ring_emit(rdev, fence) (rdev)->asic->fence_ring_emit((rdev), (fence))
 #define radeon_copy_blit(rdev, s, d, np, f) (rdev)->asic->copy_blit((rdev), (s), (d), (np), (f))
 #define radeon_copy_dma(rdev, s, d, np, f) (rdev)->asic->copy_dma((rdev), (s), (d), (np), (f))
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index c0ae2d9..7ca6c13 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -49,6 +49,7 @@ void r100_vram_info(struct radeon_device *rdev);
 int r100_gpu_reset(struct radeon_device *rdev);
 int r100_mc_init(struct radeon_device *rdev);
 void r100_mc_fini(struct radeon_device *rdev);
+u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc);
 int r100_wb_init(struct radeon_device *rdev);
 void r100_wb_fini(struct radeon_device *rdev);
 int r100_gart_enable(struct radeon_device *rdev);
@@ -96,6 +97,7 @@ static struct radeon_asic r100_asic = {
 	.ring_start = &r100_ring_start,
 	.irq_set = &r100_irq_set,
 	.irq_process = &r100_irq_process,
+	.get_vblank_counter = &r100_get_vblank_counter,
 	.fence_ring_emit = &r100_fence_ring_emit,
 	.cs_parse = &r100_cs_parse,
 	.copy_blit = &r100_copy_blit,
@@ -156,6 +158,7 @@ static struct radeon_asic r300_asic = {
 	.ring_start = &r300_ring_start,
 	.irq_set = &r100_irq_set,
 	.irq_process = &r100_irq_process,
+	.get_vblank_counter = &r100_get_vblank_counter,
 	.fence_ring_emit = &r300_fence_ring_emit,
 	.cs_parse = &r300_cs_parse,
 	.copy_blit = &r100_copy_blit,
@@ -196,6 +199,7 @@ static struct radeon_asic r420_asic = {
 	.ring_start = &r300_ring_start,
 	.irq_set = &r100_irq_set,
 	.irq_process = &r100_irq_process,
+	.get_vblank_counter = &r100_get_vblank_counter,
 	.fence_ring_emit = &r300_fence_ring_emit,
 	.cs_parse = &r300_cs_parse,
 	.copy_blit = &r100_copy_blit,
@@ -243,6 +247,7 @@ static struct radeon_asic rs400_asic = {
 	.ring_start = &r300_ring_start,
 	.irq_set = &r100_irq_set,
 	.irq_process = &r100_irq_process,
+	.get_vblank_counter = &r100_get_vblank_counter,
 	.fence_ring_emit = &r300_fence_ring_emit,
 	.cs_parse = &r300_cs_parse,
 	.copy_blit = &r100_copy_blit,
@@ -266,6 +271,8 @@ void rs600_vram_info(struct radeon_device *rdev);
 int rs600_mc_init(struct radeon_device *rdev);
 void rs600_mc_fini(struct radeon_device *rdev);
 int rs600_irq_set(struct radeon_device *rdev);
+int rs600_irq_process(struct radeon_device *rdev);
+u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc);
 int rs600_gart_enable(struct radeon_device *rdev);
 void rs600_gart_disable(struct radeon_device *rdev);
 void rs600_gart_tlb_flush(struct radeon_device *rdev);
@@ -291,7 +298,8 @@ static struct radeon_asic rs600_asic = {
 	.cp_disable = &r100_cp_disable,
 	.ring_start = &r300_ring_start,
 	.irq_set = &rs600_irq_set,
-	.irq_process = &r100_irq_process,
+	.irq_process = &rs600_irq_process,
+	.get_vblank_counter = &rs600_get_vblank_counter,
 	.fence_ring_emit = &r300_fence_ring_emit,
 	.cs_parse = &r300_cs_parse,
 	.copy_blit = &r100_copy_blit,
@@ -334,7 +342,8 @@ static struct radeon_asic rs690_asic = {
 	.cp_disable = &r100_cp_disable,
 	.ring_start = &r300_ring_start,
 	.irq_set = &rs600_irq_set,
-	.irq_process = &r100_irq_process,
+	.irq_process = &rs600_irq_process,
+	.get_vblank_counter = &rs600_get_vblank_counter,
 	.fence_ring_emit = &r300_fence_ring_emit,
 	.cs_parse = &r300_cs_parse,
 	.copy_blit = &r100_copy_blit,
@@ -382,8 +391,9 @@ static struct radeon_asic rv515_asic = {
 	.cp_fini = &r100_cp_fini,
 	.cp_disable = &r100_cp_disable,
 	.ring_start = &rv515_ring_start,
-	.irq_set = &r100_irq_set,
-	.irq_process = &r100_irq_process,
+	.irq_set = &rs600_irq_set,
+	.irq_process = &rs600_irq_process,
+	.get_vblank_counter = &rs600_get_vblank_counter,
 	.fence_ring_emit = &r300_fence_ring_emit,
 	.cs_parse = &r300_cs_parse,
 	.copy_blit = &r100_copy_blit,
@@ -424,8 +434,9 @@ static struct radeon_asic r520_asic = {
 	.cp_fini = &r100_cp_fini,
 	.cp_disable = &r100_cp_disable,
 	.ring_start = &rv515_ring_start,
-	.irq_set = &r100_irq_set,
-	.irq_process = &r100_irq_process,
+	.irq_set = &rs600_irq_set,
+	.irq_process = &rs600_irq_process,
+	.get_vblank_counter = &rs600_get_vblank_counter,
 	.fence_ring_emit = &r300_fence_ring_emit,
 	.cs_parse = &r300_cs_parse,
 	.copy_blit = &r100_copy_blit,
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 491d569..9805e4b 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -32,60 +32,6 @@
 #include "radeon.h"
 #include "atom.h"
 
-static inline uint32_t r100_irq_ack(struct radeon_device *rdev)
-{
-	uint32_t irqs = RREG32(RADEON_GEN_INT_STATUS);
-	uint32_t irq_mask = RADEON_SW_INT_TEST;
-
-	if (irqs) {
-		WREG32(RADEON_GEN_INT_STATUS, irqs);
-	}
-	return irqs & irq_mask;
-}
-
-int r100_irq_set(struct radeon_device *rdev)
-{
-	uint32_t tmp = 0;
-
-	if (rdev->irq.sw_int) {
-		tmp |= RADEON_SW_INT_ENABLE;
-	}
-	/* Todo go through CRTC and enable vblank int or not */
-	WREG32(RADEON_GEN_INT_CNTL, tmp);
-	return 0;
-}
-
-int r100_irq_process(struct radeon_device *rdev)
-{
-	uint32_t status;
-
-	status = r100_irq_ack(rdev);
-	if (!status) {
-		return IRQ_NONE;
-	}
-	while (status) {
-		/* SW interrupt */
-		if (status & RADEON_SW_INT_TEST) {
-			radeon_fence_process(rdev);
-		}
-		status = r100_irq_ack(rdev);
-	}
-	return IRQ_HANDLED;
-}
-
-int rs600_irq_set(struct radeon_device *rdev)
-{
-	uint32_t tmp = 0;
-
-	if (rdev->irq.sw_int) {
-		tmp |= RADEON_SW_INT_ENABLE;
-	}
-	WREG32(RADEON_GEN_INT_CNTL, tmp);
-	/* Todo go through CRTC and enable vblank int or not */
-	WREG32(R500_DxMODE_INT_MASK, 0);
-	return 0;
-}
-
 irqreturn_t radeon_driver_irq_handler_kms(DRM_IRQ_ARGS)
 {
 	struct drm_device *dev = (struct drm_device *) arg;
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 3357110..d2764bf 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -141,19 +141,42 @@ void radeon_driver_preclose_kms(struct drm_device *dev,
  */
 u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc)
 {
-	/* FIXME: implement */
-	return 0;
+	struct radeon_device *rdev = dev->dev_private;
+
+	if (crtc < 0 || crtc > 1) {
+		DRM_ERROR("Invalid crtc %d\n", crtc);
+		return -EINVAL;
+	}
+
+	return radeon_get_vblank_counter(rdev, crtc);
 }
 
 int radeon_enable_vblank_kms(struct drm_device *dev, int crtc)
 {
-	/* FIXME: implement */
-	return 0;
+	struct radeon_device *rdev = dev->dev_private;
+
+	if (crtc < 0 || crtc > 1) {
+		DRM_ERROR("Invalid crtc %d\n", crtc);
+		return -EINVAL;
+	}
+
+	rdev->irq.crtc_vblank_int[crtc] = true;
+
+	return radeon_irq_set(rdev);
 }
 
 void radeon_disable_vblank_kms(struct drm_device *dev, int crtc)
 {
-	/* FIXME: implement */
+	struct radeon_device *rdev = dev->dev_private;
+
+	if (crtc < 0 || crtc > 1) {
+		DRM_ERROR("Invalid crtc %d\n", crtc);
+		return;
+	}
+
+	rdev->irq.crtc_vblank_int[crtc] = false;
+
+	radeon_irq_set(rdev);
 }
 
 
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 7d06dc9..0da72f1 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -310,10 +310,13 @@ void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
 									 RADEON_CRTC_DISP_REQ_EN_B));
 			WREG32_P(RADEON_CRTC_EXT_CNTL, 0, ~mask);
 		}
+		drm_vblank_post_modeset(dev, radeon_crtc->crtc_id);
+		radeon_crtc_load_lut(crtc);
 		break;
 	case DRM_MODE_DPMS_STANDBY:
 	case DRM_MODE_DPMS_SUSPEND:
 	case DRM_MODE_DPMS_OFF:
+		drm_vblank_pre_modeset(dev, radeon_crtc->crtc_id);
 		if (radeon_crtc->crtc_id)
 			WREG32_P(RADEON_CRTC2_GEN_CNTL, mask, ~mask);
 		else {
@@ -323,10 +326,6 @@ void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
 		}
 		break;
 	}
-
-	if (mode != DRM_MODE_DPMS_OFF) {
-		radeon_crtc_load_lut(crtc);
-	}
 }
 
 /* properly set crtc bpp when using atombios */
diff --git a/drivers/gpu/drm/radeon/radeon_reg.h b/drivers/gpu/drm/radeon/radeon_reg.h
index e1b6185..5a098f3 100644
--- a/drivers/gpu/drm/radeon/radeon_reg.h
+++ b/drivers/gpu/drm/radeon/radeon_reg.h
@@ -982,12 +982,15 @@
 #       define RS400_TMDS2_PLLRST           (1 << 1)
 
 #define RADEON_GEN_INT_CNTL                 0x0040
+#	define RADEON_CRTC_VBLANK_MASK		(1 << 0)
+#	define RADEON_CRTC2_VBLANK_MASK		(1 << 9)
 #	define RADEON_SW_INT_ENABLE		(1 << 25)
 #define RADEON_GEN_INT_STATUS               0x0044
-#       define RADEON_VSYNC_INT_AK          (1 <<  2)
-#       define RADEON_VSYNC_INT             (1 <<  2)
-#       define RADEON_VSYNC2_INT_AK         (1 <<  6)
-#       define RADEON_VSYNC2_INT            (1 <<  6)
+#	define AVIVO_DISPLAY_INT_STATUS		(1 << 0)
+#	define RADEON_CRTC_VBLANK_STAT		(1 << 0)
+#	define RADEON_CRTC_VBLANK_STAT_ACK	(1 << 0)
+#	define RADEON_CRTC2_VBLANK_STAT		(1 << 9)
+#	define RADEON_CRTC2_VBLANK_STAT_ACK	(1 << 9)
 #	define RADEON_SW_INT_FIRE		(1 << 26)
 #	define RADEON_SW_INT_TEST		(1 << 25)
 #	define RADEON_SW_INT_TEST_ACK		(1 << 25)
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index bbea6dee..7e8ce98 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -240,6 +240,88 @@ void rs600_mc_fini(struct radeon_device *rdev)
 
 
 /*
+ * Interrupts
+ */
+int rs600_irq_set(struct radeon_device *rdev)
+{
+	uint32_t tmp = 0;
+	uint32_t mode_int = 0;
+
+	if (rdev->irq.sw_int) {
+		tmp |= RADEON_SW_INT_ENABLE;
+	}
+	if (rdev->irq.crtc_vblank_int[0]) {
+		tmp |= AVIVO_DISPLAY_INT_STATUS;
+		mode_int |= AVIVO_D1MODE_INT_MASK;
+	}
+	if (rdev->irq.crtc_vblank_int[1]) {
+		tmp |= AVIVO_DISPLAY_INT_STATUS;
+		mode_int |= AVIVO_D2MODE_INT_MASK;
+	}
+	WREG32(RADEON_GEN_INT_CNTL, tmp);
+	WREG32(AVIVO_DxMODE_INT_MASK, mode_int);
+	return 0;
+}
+
+static inline uint32_t rs600_irq_ack(struct radeon_device *rdev, u32 *r500_disp_int)
+{
+	uint32_t irqs = RREG32(RADEON_GEN_INT_STATUS);
+	uint32_t irq_mask = RADEON_SW_INT_TEST;
+
+	if (irqs & AVIVO_DISPLAY_INT_STATUS) {
+		*r500_disp_int = RREG32(AVIVO_DISP_INTERRUPT_STATUS);
+		if (*r500_disp_int & AVIVO_D1_VBLANK_INTERRUPT) {
+			WREG32(AVIVO_D1MODE_VBLANK_STATUS, AVIVO_VBLANK_ACK);
+		}
+		if (*r500_disp_int & AVIVO_D2_VBLANK_INTERRUPT) {
+			WREG32(AVIVO_D2MODE_VBLANK_STATUS, AVIVO_VBLANK_ACK);
+		}
+	} else {
+		*r500_disp_int = 0;
+	}
+
+	if (irqs) {
+		WREG32(RADEON_GEN_INT_STATUS, irqs);
+	}
+	return irqs & irq_mask;
+}
+
+int rs600_irq_process(struct radeon_device *rdev)
+{
+	uint32_t status;
+	uint32_t r500_disp_int;
+
+	status = rs600_irq_ack(rdev, &r500_disp_int);
+	if (!status && !r500_disp_int) {
+		return IRQ_NONE;
+	}
+	while (status || r500_disp_int) {
+		/* SW interrupt */
+		if (status & RADEON_SW_INT_TEST) {
+			radeon_fence_process(rdev);
+		}
+		/* Vertical blank interrupts */
+		if (r500_disp_int & AVIVO_D1_VBLANK_INTERRUPT) {
+			drm_handle_vblank(rdev->ddev, 0);
+		}
+		if (r500_disp_int & AVIVO_D2_VBLANK_INTERRUPT) {
+			drm_handle_vblank(rdev->ddev, 1);
+		}
+		status = rs600_irq_ack(rdev, &r500_disp_int);
+	}
+	return IRQ_HANDLED;
+}
+
+u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc)
+{
+	if (crtc == 0)
+		return RREG32(AVIVO_D1CRTC_FRAME_COUNT);
+	else
+		return RREG32(AVIVO_D2CRTC_FRAME_COUNT);
+}
+
+
+/*
  * Global GPU functions
  */
 void rs600_disable_vga(struct radeon_device *rdev)