44 static inline bool fbc_supported(struct drm_i915_private *dev_priv)  in fbc_supported()  argument
46 	return dev_priv->fbc.enable_fbc != NULL;  in fbc_supported()
62 static void i8xx_fbc_disable(struct drm_i915_private *dev_priv)  in i8xx_fbc_disable()  argument
66 	dev_priv->fbc.enabled = false;  in i8xx_fbc_disable()
87 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in i8xx_fbc_enable()  local
94 	dev_priv->fbc.enabled = true;  in i8xx_fbc_enable()
97 	cfb_pitch = dev_priv->fbc.uncompressed_size / FBC_LL_SIZE;  in i8xx_fbc_enable()
102 	if (IS_GEN2(dev_priv))  in i8xx_fbc_enable()
111 	if (IS_GEN4(dev_priv)) {  in i8xx_fbc_enable()
125 	if (IS_I945GM(dev_priv))  in i8xx_fbc_enable()
135 static bool i8xx_fbc_enabled(struct drm_i915_private *dev_priv)  in i8xx_fbc_enabled()  argument
142 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in g4x_fbc_enable()  local
147 	dev_priv->fbc.enabled = true;  in g4x_fbc_enable()
164 static void g4x_fbc_disable(struct drm_i915_private *dev_priv)  in g4x_fbc_disable()  argument
168 	dev_priv->fbc.enabled = false;  in g4x_fbc_disable()
180 static bool g4x_fbc_enabled(struct drm_i915_private *dev_priv)  in g4x_fbc_enabled()  argument
185 static void intel_fbc_nuke(struct drm_i915_private *dev_priv)  in intel_fbc_nuke()  argument
193 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in ilk_fbc_enable()  local
197 	int threshold = dev_priv->fbc.threshold;  in ilk_fbc_enable()
200 	dev_priv->fbc.enabled = true;  in ilk_fbc_enable()
219 	if (IS_GEN5(dev_priv))  in ilk_fbc_enable()
228 	if (IS_GEN6(dev_priv)) {  in ilk_fbc_enable()
234 	intel_fbc_nuke(dev_priv);  in ilk_fbc_enable()
239 static void ilk_fbc_disable(struct drm_i915_private *dev_priv)  in ilk_fbc_disable()  argument
243 	dev_priv->fbc.enabled = false;  in ilk_fbc_disable()
255 static bool ilk_fbc_enabled(struct drm_i915_private *dev_priv)  in ilk_fbc_enabled()  argument
262 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in gen7_fbc_enable()  local
266 	int threshold = dev_priv->fbc.threshold;  in gen7_fbc_enable()
268 	dev_priv->fbc.enabled = true;  in gen7_fbc_enable()
271 	if (IS_IVYBRIDGE(dev_priv))  in gen7_fbc_enable()
292 	if (dev_priv->fbc.false_color)  in gen7_fbc_enable()
295 	if (IS_IVYBRIDGE(dev_priv)) {  in gen7_fbc_enable()
300 	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {  in gen7_fbc_enable()
313 	intel_fbc_nuke(dev_priv);  in gen7_fbc_enable()
326 bool intel_fbc_enabled(struct drm_i915_private *dev_priv)  in intel_fbc_enabled()  argument
328 	return dev_priv->fbc.enabled;  in intel_fbc_enabled()
334 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in intel_fbc_enable()  local
336 	dev_priv->fbc.enable_fbc(crtc);  in intel_fbc_enable()
338 	dev_priv->fbc.crtc = crtc;  in intel_fbc_enable()
339 	dev_priv->fbc.fb_id = fb->base.id;  in intel_fbc_enable()
340 	dev_priv->fbc.y = crtc->base.y;  in intel_fbc_enable()
348 	struct drm_i915_private *dev_priv = work->crtc->base.dev->dev_private;  in intel_fbc_work_fn()  local
351 	mutex_lock(&dev_priv->fbc.lock);  in intel_fbc_work_fn()
352 	if (work == dev_priv->fbc.fbc_work) {  in intel_fbc_work_fn()
359 		dev_priv->fbc.fbc_work = NULL;  in intel_fbc_work_fn()
361 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_work_fn()
366 static void intel_fbc_cancel_work(struct drm_i915_private *dev_priv)  in intel_fbc_cancel_work()  argument
368 	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));  in intel_fbc_cancel_work()
370 	if (dev_priv->fbc.fbc_work == NULL)  in intel_fbc_cancel_work()
379 	if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))  in intel_fbc_cancel_work()
381 		kfree(dev_priv->fbc.fbc_work);  in intel_fbc_cancel_work()
388 	dev_priv->fbc.fbc_work = NULL;  in intel_fbc_cancel_work()
394 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in intel_fbc_schedule_enable()  local
396 	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));  in intel_fbc_schedule_enable()
398 	intel_fbc_cancel_work(dev_priv);  in intel_fbc_schedule_enable()
411 	dev_priv->fbc.fbc_work = work;  in intel_fbc_schedule_enable()
429 static void __intel_fbc_disable(struct drm_i915_private *dev_priv)  in __intel_fbc_disable()  argument
431 	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));  in __intel_fbc_disable()
433 	intel_fbc_cancel_work(dev_priv);  in __intel_fbc_disable()
435 	dev_priv->fbc.disable_fbc(dev_priv);  in __intel_fbc_disable()
436 	dev_priv->fbc.crtc = NULL;  in __intel_fbc_disable()
445 void intel_fbc_disable(struct drm_i915_private *dev_priv)  in intel_fbc_disable()  argument
447 	if (!fbc_supported(dev_priv))  in intel_fbc_disable()
450 	mutex_lock(&dev_priv->fbc.lock);  in intel_fbc_disable()
451 	__intel_fbc_disable(dev_priv);  in intel_fbc_disable()
452 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_disable()
463 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in intel_fbc_disable_crtc()  local
465 	if (!fbc_supported(dev_priv))  in intel_fbc_disable_crtc()
468 	mutex_lock(&dev_priv->fbc.lock);  in intel_fbc_disable_crtc()
469 	if (dev_priv->fbc.crtc == crtc)  in intel_fbc_disable_crtc()
470 		__intel_fbc_disable(dev_priv);  in intel_fbc_disable_crtc()
471 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_disable_crtc()
515 static void set_no_fbc_reason(struct drm_i915_private *dev_priv,  in set_no_fbc_reason()  argument
518 	if (dev_priv->fbc.no_fbc_reason == reason)  in set_no_fbc_reason()
521 	dev_priv->fbc.no_fbc_reason = reason;  in set_no_fbc_reason()
525 static struct drm_crtc *intel_fbc_find_crtc(struct drm_i915_private *dev_priv)  in intel_fbc_find_crtc()  argument
531 	if (IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8)  in intel_fbc_find_crtc()
534 	for_each_pipe(dev_priv, pipe) {  in intel_fbc_find_crtc()
535 		tmp_crtc = dev_priv->pipe_to_crtc_mapping[pipe];  in intel_fbc_find_crtc()
551 static bool multiple_pipes_ok(struct drm_i915_private *dev_priv)  in multiple_pipes_ok()  argument
557 	if (INTEL_INFO(dev_priv)->gen > 4)  in multiple_pipes_ok()
560 	for_each_pipe(dev_priv, pipe) {  in multiple_pipes_ok()
561 		crtc = dev_priv->pipe_to_crtc_mapping[pipe];  in multiple_pipes_ok()
571 static int find_compression_threshold(struct drm_i915_private *dev_priv,  in find_compression_threshold()  argument
584 	if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv))  in find_compression_threshold()
585 		end = dev_priv->gtt.stolen_size - 8 * 1024 * 1024;  in find_compression_threshold()
587 		end = dev_priv->gtt.stolen_usable_size;  in find_compression_threshold()
597 	ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size <<= 1,  in find_compression_threshold()
608 	ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size >>= 1,  in find_compression_threshold()
610 	if (ret && INTEL_INFO(dev_priv)->gen <= 4) {  in find_compression_threshold()
620 static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv, int size,  in intel_fbc_alloc_cfb()  argument
626 	ret = find_compression_threshold(dev_priv, &dev_priv->fbc.compressed_fb,  in intel_fbc_alloc_cfb()
635 	dev_priv->fbc.threshold = ret;  in intel_fbc_alloc_cfb()
637 	if (INTEL_INFO(dev_priv)->gen >= 5)  in intel_fbc_alloc_cfb()
638 		I915_WRITE(ILK_DPFC_CB_BASE, dev_priv->fbc.compressed_fb.start);  in intel_fbc_alloc_cfb()
639 	else if (IS_GM45(dev_priv)) {  in intel_fbc_alloc_cfb()
640 		I915_WRITE(DPFC_CB_BASE, dev_priv->fbc.compressed_fb.start);  in intel_fbc_alloc_cfb()
646 		ret = i915_gem_stolen_insert_node(dev_priv, compressed_llb,  in intel_fbc_alloc_cfb()
651 		dev_priv->fbc.compressed_llb = compressed_llb;  in intel_fbc_alloc_cfb()
654 			   dev_priv->mm.stolen_base + dev_priv->fbc.compressed_fb.start);  in intel_fbc_alloc_cfb()
656 			   dev_priv->mm.stolen_base + compressed_llb->start);  in intel_fbc_alloc_cfb()
659 	dev_priv->fbc.uncompressed_size = size;  in intel_fbc_alloc_cfb()
662 		      dev_priv->fbc.compressed_fb.size,  in intel_fbc_alloc_cfb()
663 		      dev_priv->fbc.threshold);  in intel_fbc_alloc_cfb()
669 	i915_gem_stolen_remove_node(dev_priv, &dev_priv->fbc.compressed_fb);  in intel_fbc_alloc_cfb()
675 static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)  in __intel_fbc_cleanup_cfb()  argument
677 	if (dev_priv->fbc.uncompressed_size == 0)  in __intel_fbc_cleanup_cfb()
680 	i915_gem_stolen_remove_node(dev_priv, &dev_priv->fbc.compressed_fb);  in __intel_fbc_cleanup_cfb()
682 	if (dev_priv->fbc.compressed_llb) {  in __intel_fbc_cleanup_cfb()
683 		i915_gem_stolen_remove_node(dev_priv,  in __intel_fbc_cleanup_cfb()
684 					    dev_priv->fbc.compressed_llb);  in __intel_fbc_cleanup_cfb()
685 		kfree(dev_priv->fbc.compressed_llb);  in __intel_fbc_cleanup_cfb()
688 	dev_priv->fbc.uncompressed_size = 0;  in __intel_fbc_cleanup_cfb()
691 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)  in intel_fbc_cleanup_cfb()  argument
693 	if (!fbc_supported(dev_priv))  in intel_fbc_cleanup_cfb()
696 	mutex_lock(&dev_priv->fbc.lock);  in intel_fbc_cleanup_cfb()
697 	__intel_fbc_cleanup_cfb(dev_priv);  in intel_fbc_cleanup_cfb()
698 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_cleanup_cfb()
729 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in intel_fbc_calculate_cfb_size()  local
734 	if (INTEL_INFO(dev_priv)->gen >= 7)  in intel_fbc_calculate_cfb_size()
742 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in intel_fbc_setup_cfb()  local
749 	if (size <= dev_priv->fbc.uncompressed_size)  in intel_fbc_setup_cfb()
753 	__intel_fbc_cleanup_cfb(dev_priv);  in intel_fbc_setup_cfb()
755 	return intel_fbc_alloc_cfb(dev_priv, size, cpp);  in intel_fbc_setup_cfb()
758 static bool stride_is_valid(struct drm_i915_private *dev_priv,  in stride_is_valid()  argument
767 	if (IS_GEN2(dev_priv) || IS_GEN3(dev_priv))  in stride_is_valid()
770 	if (IS_GEN4(dev_priv) && !IS_G4X(dev_priv) && stride < 2048)  in stride_is_valid()
782 	struct drm_i915_private *dev_priv = dev->dev_private;  in pixel_format_is_valid()  local
794 		if (IS_G4X(dev_priv))  in pixel_format_is_valid()
810 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;  in intel_fbc_hw_tracking_covers_screen()  local
813 	if (INTEL_INFO(dev_priv)->gen >= 8 || IS_HASWELL(dev_priv)) {  in intel_fbc_hw_tracking_covers_screen()
816 	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {  in intel_fbc_hw_tracking_covers_screen()
850 static void __intel_fbc_update(struct drm_i915_private *dev_priv)  in __intel_fbc_update()  argument
858 	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));  in __intel_fbc_update()
861 	if (intel_vgpu_active(dev_priv->dev))  in __intel_fbc_update()
865 		set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT);  in __intel_fbc_update()
870 		set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM);  in __intel_fbc_update()
883 	crtc = intel_fbc_find_crtc(dev_priv);  in __intel_fbc_update()
885 		set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT);  in __intel_fbc_update()
889 	if (!multiple_pipes_ok(dev_priv)) {  in __intel_fbc_update()
890 		set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES);  in __intel_fbc_update()
901 		set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE);  in __intel_fbc_update()
906 		set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE);  in __intel_fbc_update()
910 	if ((INTEL_INFO(dev_priv)->gen < 4 || HAS_DDI(dev_priv)) &&  in __intel_fbc_update()
912 		set_no_fbc_reason(dev_priv, FBC_BAD_PLANE);  in __intel_fbc_update()
921 		set_no_fbc_reason(dev_priv, FBC_NOT_TILED);  in __intel_fbc_update()
924 	if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_G4X(dev_priv) &&  in __intel_fbc_update()
926 		set_no_fbc_reason(dev_priv, FBC_ROTATION);  in __intel_fbc_update()
930 	if (!stride_is_valid(dev_priv, fb->pitches[0])) {  in __intel_fbc_update()
931 		set_no_fbc_reason(dev_priv, FBC_BAD_STRIDE);  in __intel_fbc_update()
936 		set_no_fbc_reason(dev_priv, FBC_PIXEL_FORMAT);  in __intel_fbc_update()
942 		set_no_fbc_reason(dev_priv, FBC_IN_DBG_MASTER);  in __intel_fbc_update()
947 	if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) &&  in __intel_fbc_update()
949 	    dev_priv->cdclk_freq * 95 / 100) {  in __intel_fbc_update()
950 		set_no_fbc_reason(dev_priv, FBC_PIXEL_RATE);  in __intel_fbc_update()
955 		set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL);  in __intel_fbc_update()
964 	if (dev_priv->fbc.crtc == intel_crtc &&  in __intel_fbc_update()
965 	    dev_priv->fbc.fb_id == fb->base.id &&  in __intel_fbc_update()
966 	    dev_priv->fbc.y == crtc->y)  in __intel_fbc_update()
969 	if (intel_fbc_enabled(dev_priv)) {  in __intel_fbc_update()
994 		__intel_fbc_disable(dev_priv);  in __intel_fbc_update()
998 	dev_priv->fbc.no_fbc_reason = FBC_OK;  in __intel_fbc_update()
1003 	if (intel_fbc_enabled(dev_priv)) {  in __intel_fbc_update()
1005 		__intel_fbc_disable(dev_priv);  in __intel_fbc_update()
1007 	__intel_fbc_cleanup_cfb(dev_priv);  in __intel_fbc_update()
1016 void intel_fbc_update(struct drm_i915_private *dev_priv)  in intel_fbc_update()  argument
1018 	if (!fbc_supported(dev_priv))  in intel_fbc_update()
1021 	mutex_lock(&dev_priv->fbc.lock);  in intel_fbc_update()
1022 	__intel_fbc_update(dev_priv);  in intel_fbc_update()
1023 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_update()
1026 void intel_fbc_invalidate(struct drm_i915_private *dev_priv,  in intel_fbc_invalidate()  argument
1032 	if (!fbc_supported(dev_priv))  in intel_fbc_invalidate()
1038 	mutex_lock(&dev_priv->fbc.lock);  in intel_fbc_invalidate()
1040 	if (dev_priv->fbc.enabled)  in intel_fbc_invalidate()
1041 		fbc_bits = INTEL_FRONTBUFFER_PRIMARY(dev_priv->fbc.crtc->pipe);  in intel_fbc_invalidate()
1042 	else if (dev_priv->fbc.fbc_work)  in intel_fbc_invalidate()
1044 					dev_priv->fbc.fbc_work->crtc->pipe);  in intel_fbc_invalidate()
1046 		fbc_bits = dev_priv->fbc.possible_framebuffer_bits;  in intel_fbc_invalidate()
1048 	dev_priv->fbc.busy_bits |= (fbc_bits & frontbuffer_bits);  in intel_fbc_invalidate()
1050 	if (dev_priv->fbc.busy_bits)  in intel_fbc_invalidate()
1051 		__intel_fbc_disable(dev_priv);  in intel_fbc_invalidate()
1053 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_invalidate()
1056 void intel_fbc_flush(struct drm_i915_private *dev_priv,  in intel_fbc_flush()  argument
1059 	if (!fbc_supported(dev_priv))  in intel_fbc_flush()
1065 	mutex_lock(&dev_priv->fbc.lock);  in intel_fbc_flush()
1067 	dev_priv->fbc.busy_bits &= ~frontbuffer_bits;  in intel_fbc_flush()
1069 	if (!dev_priv->fbc.busy_bits) {  in intel_fbc_flush()
1070 		__intel_fbc_disable(dev_priv);  in intel_fbc_flush()
1071 		__intel_fbc_update(dev_priv);  in intel_fbc_flush()
1074 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_flush()
1083 void intel_fbc_init(struct drm_i915_private *dev_priv)  in intel_fbc_init()  argument
1087 	mutex_init(&dev_priv->fbc.lock);  in intel_fbc_init()
1089 	if (!HAS_FBC(dev_priv)) {  in intel_fbc_init()
1090 		dev_priv->fbc.enabled = false;  in intel_fbc_init()
1091 		dev_priv->fbc.no_fbc_reason = FBC_UNSUPPORTED;  in intel_fbc_init()
1095 	for_each_pipe(dev_priv, pipe) {  in intel_fbc_init()
1096 		dev_priv->fbc.possible_framebuffer_bits |=  in intel_fbc_init()
1099 		if (IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8)  in intel_fbc_init()
1103 	if (INTEL_INFO(dev_priv)->gen >= 7) {  in intel_fbc_init()
1104 		dev_priv->fbc.fbc_enabled = ilk_fbc_enabled;  in intel_fbc_init()
1105 		dev_priv->fbc.enable_fbc = gen7_fbc_enable;  in intel_fbc_init()
1106 		dev_priv->fbc.disable_fbc = ilk_fbc_disable;  in intel_fbc_init()
1107 	} else if (INTEL_INFO(dev_priv)->gen >= 5) {  in intel_fbc_init()
1108 		dev_priv->fbc.fbc_enabled = ilk_fbc_enabled;  in intel_fbc_init()
1109 		dev_priv->fbc.enable_fbc = ilk_fbc_enable;  in intel_fbc_init()
1110 		dev_priv->fbc.disable_fbc = ilk_fbc_disable;  in intel_fbc_init()
1111 	} else if (IS_GM45(dev_priv)) {  in intel_fbc_init()
1112 		dev_priv->fbc.fbc_enabled = g4x_fbc_enabled;  in intel_fbc_init()
1113 		dev_priv->fbc.enable_fbc = g4x_fbc_enable;  in intel_fbc_init()
1114 		dev_priv->fbc.disable_fbc = g4x_fbc_disable;  in intel_fbc_init()
1116 		dev_priv->fbc.fbc_enabled = i8xx_fbc_enabled;  in intel_fbc_init()
1117 		dev_priv->fbc.enable_fbc = i8xx_fbc_enable;  in intel_fbc_init()
1118 		dev_priv->fbc.disable_fbc = i8xx_fbc_disable;  in intel_fbc_init()
1124 	dev_priv->fbc.enabled = dev_priv->fbc.fbc_enabled(dev_priv);  in intel_fbc_init()