This source file includes following definitions.
- gen9_init_clock_gating
- bxt_init_clock_gating
- glk_init_clock_gating
- i915_pineview_get_mem_freq
- i915_ironlake_get_mem_freq
- intel_get_cxsr_latency
- chv_set_memory_dvfs
- chv_set_memory_pm5
- _intel_set_memory_cxsr
- intel_set_memory_cxsr
- vlv_get_fifo_size
- i9xx_get_fifo_size
- i830_get_fifo_size
- i845_get_fifo_size
- intel_wm_method1
- intel_wm_method2
- intel_calculate_wm
- is_disabling
- is_enabling
- intel_wm_num_levels
- intel_wm_plane_visible
- single_enabled_crtc
- pineview_update_wm
- g4x_tlb_miss_wa
- g4x_write_wm_values
- vlv_write_wm_values
- g4x_setup_wm_latency
- g4x_plane_fifo_size
- g4x_fbc_fifo_size
- g4x_compute_wm
- g4x_raw_plane_wm_set
- g4x_raw_fbc_wm_set
- g4x_raw_plane_wm_compute
- g4x_raw_plane_wm_is_valid
- g4x_raw_crtc_wm_is_valid
- g4x_invalidate_wms
- g4x_compute_pipe_wm
- g4x_compute_intermediate_wm
- g4x_merge_wm
- g4x_program_watermarks
- g4x_initial_watermarks
- g4x_optimize_watermarks
- vlv_wm_method2
- vlv_setup_wm_latency
- vlv_compute_wm_level
- vlv_need_sprite0_fifo_workaround
- vlv_compute_fifo
- vlv_invalidate_wms
- vlv_invert_wm_value
- vlv_raw_plane_wm_set
- vlv_raw_plane_wm_compute
- vlv_raw_plane_wm_is_valid
- vlv_raw_crtc_wm_is_valid
- vlv_compute_pipe_wm
- vlv_atomic_update_fifo
- vlv_compute_intermediate_wm
- vlv_merge_wm
- vlv_program_watermarks
- vlv_initial_watermarks
- vlv_optimize_watermarks
- i965_update_wm
- i9xx_update_wm
- i845_update_wm
- ilk_wm_method1
- ilk_wm_method2
- ilk_wm_fbc
- ilk_compute_pri_wm
- ilk_compute_spr_wm
- ilk_compute_cur_wm
- ilk_compute_fbc_wm
- ilk_display_fifo_size
- ilk_plane_wm_reg_max
- ilk_cursor_wm_reg_max
- ilk_fbc_wm_reg_max
- ilk_plane_wm_max
- ilk_cursor_wm_max
- ilk_compute_wm_maximums
- ilk_compute_wm_reg_maximums
- ilk_validate_wm_level
- ilk_compute_wm_level
- hsw_compute_linetime_wm
- intel_read_wm_latency
- intel_fixup_spr_wm_latency
- intel_fixup_cur_wm_latency
- ilk_wm_max_level
- intel_print_wm_latency
- ilk_increase_wm_latency
- snb_wm_latency_quirk
- snb_wm_lp3_irq_quirk
- ilk_setup_wm_latency
- skl_setup_wm_latency
- ilk_validate_pipe_wm
- ilk_compute_pipe_wm
- ilk_compute_intermediate_wm
- ilk_merge_wm_level
- ilk_wm_merge
- ilk_wm_lp_to_level
- ilk_wm_lp_latency
- ilk_compute_wm_results
- ilk_find_best_result
- ilk_compute_wm_dirty
- _ilk_disable_lp_wm
- ilk_write_wm_values
- ilk_disable_lp_wm
- intel_enabled_dbuf_slices_num
- skl_needs_memory_bw_wa
- intel_has_sagv
- intel_enable_sagv
- intel_disable_sagv
- intel_can_enable_sagv
- intel_get_ddb_size
- skl_ddb_get_pipe_allocation_limits
- skl_cursor_allocation
- skl_ddb_entry_init_from_hw
- skl_ddb_get_hw_plane_state
- skl_pipe_ddb_get_hw_state
- skl_ddb_get_hw_state
- skl_plane_downscale_amount
- skl_pipe_downscale_amount
- skl_check_pipe_max_pixel_rate
- skl_plane_relative_data_rate
- skl_get_total_relative_data_rate
- icl_get_total_relative_data_rate
- skl_allocate_pipe_ddb
- skl_wm_method1
- skl_wm_method2
- intel_get_linetime_us
- skl_adjusted_plane_pixel_rate
- skl_compute_wm_params
- skl_compute_plane_wm_params
- skl_wm_has_lines
- skl_compute_plane_wm
- skl_compute_wm_levels
- skl_compute_linetime_wm
- skl_compute_transition_wm
- skl_build_plane_wm_single
- skl_build_plane_wm_uv
- skl_build_plane_wm
- icl_build_plane_wm
- skl_build_pipe_wm
- skl_ddb_entry_write
- skl_write_wm_level
- skl_write_plane_wm
- skl_write_cursor_wm
- skl_wm_level_equals
- skl_plane_wm_equals
- skl_pipe_wm_equals
- skl_ddb_entries_overlap
- skl_ddb_allocation_overlaps
- pipes_modified
- skl_ddb_add_affected_planes
- skl_compute_ddb
- enast
- skl_print_wm_changes
- skl_ddb_add_affected_pipes
- skl_wm_add_affected_planes
- skl_compute_wm
- skl_atomic_update_crtc_wm
- skl_initial_wm
- ilk_compute_wm_config
- ilk_program_watermarks
- ilk_initial_watermarks
- ilk_optimize_watermarks
- skl_wm_level_from_reg_val
- skl_pipe_wm_get_hw_state
- skl_wm_get_hw_state
- ilk_pipe_wm_get_hw_state
- g4x_read_wm_values
- vlv_read_wm_values
- g4x_wm_get_hw_state
- g4x_wm_sanitize
- vlv_wm_get_hw_state
- vlv_wm_sanitize
- ilk_init_lp_watermarks
- ilk_wm_get_hw_state
- intel_update_watermarks
- intel_enable_ipc
- intel_can_enable_ipc
- intel_init_ipc
- ironlake_set_drps
- ironlake_enable_drps
- ironlake_disable_drps
- intel_rps_limits
- rps_set_power
- gen6_set_rps_thresholds
- intel_rps_mark_interactive
- gen6_rps_pm_mask
- gen6_set_rps
- valleyview_set_rps
- vlv_set_rps_idle
- gen6_rps_busy
- gen6_rps_idle
- gen6_rps_boost
- intel_set_rps
- gen9_disable_rc6
- gen9_disable_rps
- gen6_disable_rc6
- gen6_disable_rps
- cherryview_disable_rc6
- cherryview_disable_rps
- valleyview_disable_rc6
- valleyview_disable_rps
- bxt_check_bios_rc6_setup
- sanitize_rc6
- gen6_init_rps_frequencies
- reset_rps
- gen9_enable_rps
- gen11_enable_rc6
- gen9_enable_rc6
- gen8_enable_rc6
- gen8_enable_rps
- gen6_enable_rc6
- gen6_enable_rps
- gen6_update_ring_freq
- cherryview_rps_max_freq
- cherryview_rps_rpe_freq
- cherryview_rps_guar_freq
- cherryview_rps_min_freq
- valleyview_rps_guar_freq
- valleyview_rps_max_freq
- valleyview_rps_rpe_freq
- valleyview_rps_min_freq
- valleyview_check_pctx
- cherryview_check_pctx
- cherryview_setup_pctx
- valleyview_setup_pctx
- valleyview_cleanup_pctx
- vlv_init_gpll_ref_freq
- valleyview_init_gt_powersave
- cherryview_init_gt_powersave
- valleyview_cleanup_gt_powersave
- cherryview_enable_rc6
- cherryview_enable_rps
- valleyview_enable_rc6
- valleyview_enable_rps
- intel_pxfreq
- __i915_chipset_val
- i915_chipset_val
- i915_mch_val
- _pxvid_to_vd
- pvid_to_extvid
- __i915_update_gfx_val
- i915_update_gfx_val
- __i915_gfx_val
- i915_gfx_val
- mchdev_get
- i915_read_mch_val
- i915_gpu_raise
- i915_gpu_lower
- i915_gpu_busy
- i915_gpu_turbo_disable
- ips_ping_for_i915_load
- intel_gpu_ips_init
- intel_gpu_ips_teardown
- intel_init_emon
- i915_rc6_ctx_corrupted
- i915_rc6_ctx_wa_init
- i915_rc6_ctx_wa_cleanup
- i915_rc6_ctx_wa_suspend
- i915_rc6_ctx_wa_resume
- i915_rc6_ctx_wa_check
- intel_init_gt_powersave
- intel_cleanup_gt_powersave
- intel_sanitize_gt_powersave
- intel_disable_llc_pstate
- __intel_disable_rc6
- intel_disable_rc6
- intel_disable_rps
- intel_disable_gt_powersave
- intel_enable_llc_pstate
- intel_enable_rc6
- intel_enable_rps
- intel_enable_gt_powersave
- ibx_init_clock_gating
- g4x_disable_trickle_feed
- ilk_init_clock_gating
- cpt_init_clock_gating
- gen6_check_mch_setup
- gen6_init_clock_gating
- gen7_setup_fixed_func_scheduler
- lpt_init_clock_gating
- lpt_suspend_hw
- gen8_set_l3sqc_credits
- icl_init_clock_gating
- cnp_init_clock_gating
- cnl_init_clock_gating
- cfl_init_clock_gating
- kbl_init_clock_gating
- skl_init_clock_gating
- bdw_init_clock_gating
- hsw_init_clock_gating
- ivb_init_clock_gating
- vlv_init_clock_gating
- chv_init_clock_gating
- g4x_init_clock_gating
- i965gm_init_clock_gating
- i965g_init_clock_gating
- gen3_init_clock_gating
- i85x_init_clock_gating
- i830_init_clock_gating
- intel_init_clock_gating
- intel_suspend_hw
- nop_init_clock_gating
- intel_init_clock_gating_hooks
- intel_init_pm
- byt_gpu_freq
- byt_freq_opcode
- chv_gpu_freq
- chv_freq_opcode
- intel_gpu_freq
- intel_freq_opcode
- intel_pm_setup
- vlv_residency_raw
- intel_rc6_residency_ns
- intel_rc6_residency_us
- intel_get_cagf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28 #include <linux/cpufreq.h>
29 #include <linux/module.h>
30 #include <linux/pm_runtime.h>
31
32 #include <drm/drm_atomic_helper.h>
33 #include <drm/drm_fourcc.h>
34 #include <drm/drm_plane_helper.h>
35
36 #include "display/intel_atomic.h"
37 #include "display/intel_display_types.h"
38 #include "display/intel_fbc.h"
39 #include "display/intel_sprite.h"
40
41 #include "i915_drv.h"
42 #include "i915_irq.h"
43 #include "i915_trace.h"
44 #include "intel_pm.h"
45 #include "intel_sideband.h"
46 #include "../../../platform/x86/intel_ips.h"
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
69 {
70 if (HAS_LLC(dev_priv)) {
71
72
73
74
75
76
77
78 I915_WRITE(CHICKEN_PAR1_1,
79 I915_READ(CHICKEN_PAR1_1) |
80 SKL_DE_COMPRESSED_HASH_MODE);
81 }
82
83
84 I915_WRITE(CHICKEN_PAR1_1,
85 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
86
87
88 I915_WRITE(GEN8_CHICKEN_DCPR_1,
89 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
90
91
92
93 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
94 DISP_FBC_WM_DIS |
95 DISP_FBC_MEMORY_WAKE);
96
97
98 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
99 ILK_DPFC_DISABLE_DUMMY0);
100
101 if (IS_SKYLAKE(dev_priv)) {
102
103 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
104 & ~GEN7_DOP_CLOCK_GATE_ENABLE);
105 }
106 }
107
108 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
109 {
110 gen9_init_clock_gating(dev_priv);
111
112
113 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
114 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
115
116
117
118
119
120 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
121 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
122
123
124
125
126
127 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
128 PWM1_GATING_DIS | PWM2_GATING_DIS);
129
130
131
132
133
134
135
136 I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
137 }
138
139 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
140 {
141 gen9_init_clock_gating(dev_priv);
142
143
144
145
146
147
148 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
149 PWM1_GATING_DIS | PWM2_GATING_DIS);
150
151
152 if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
153 u32 val = I915_READ(CHICKEN_MISC_2);
154 val &= ~(GLK_CL0_PWR_DOWN |
155 GLK_CL1_PWR_DOWN |
156 GLK_CL2_PWR_DOWN);
157 I915_WRITE(CHICKEN_MISC_2, val);
158 }
159
160 }
161
162 static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
163 {
164 u32 tmp;
165
166 tmp = I915_READ(CLKCFG);
167
168 switch (tmp & CLKCFG_FSB_MASK) {
169 case CLKCFG_FSB_533:
170 dev_priv->fsb_freq = 533;
171 break;
172 case CLKCFG_FSB_800:
173 dev_priv->fsb_freq = 800;
174 break;
175 case CLKCFG_FSB_667:
176 dev_priv->fsb_freq = 667;
177 break;
178 case CLKCFG_FSB_400:
179 dev_priv->fsb_freq = 400;
180 break;
181 }
182
183 switch (tmp & CLKCFG_MEM_MASK) {
184 case CLKCFG_MEM_533:
185 dev_priv->mem_freq = 533;
186 break;
187 case CLKCFG_MEM_667:
188 dev_priv->mem_freq = 667;
189 break;
190 case CLKCFG_MEM_800:
191 dev_priv->mem_freq = 800;
192 break;
193 }
194
195
196 tmp = I915_READ(CSHRDDR3CTL);
197 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
198 }
199
200 static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
201 {
202 u16 ddrpll, csipll;
203
204 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
205 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
206
207 switch (ddrpll & 0xff) {
208 case 0xc:
209 dev_priv->mem_freq = 800;
210 break;
211 case 0x10:
212 dev_priv->mem_freq = 1066;
213 break;
214 case 0x14:
215 dev_priv->mem_freq = 1333;
216 break;
217 case 0x18:
218 dev_priv->mem_freq = 1600;
219 break;
220 default:
221 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
222 ddrpll & 0xff);
223 dev_priv->mem_freq = 0;
224 break;
225 }
226
227 dev_priv->ips.r_t = dev_priv->mem_freq;
228
229 switch (csipll & 0x3ff) {
230 case 0x00c:
231 dev_priv->fsb_freq = 3200;
232 break;
233 case 0x00e:
234 dev_priv->fsb_freq = 3733;
235 break;
236 case 0x010:
237 dev_priv->fsb_freq = 4266;
238 break;
239 case 0x012:
240 dev_priv->fsb_freq = 4800;
241 break;
242 case 0x014:
243 dev_priv->fsb_freq = 5333;
244 break;
245 case 0x016:
246 dev_priv->fsb_freq = 5866;
247 break;
248 case 0x018:
249 dev_priv->fsb_freq = 6400;
250 break;
251 default:
252 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
253 csipll & 0x3ff);
254 dev_priv->fsb_freq = 0;
255 break;
256 }
257
258 if (dev_priv->fsb_freq == 3200) {
259 dev_priv->ips.c_m = 0;
260 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
261 dev_priv->ips.c_m = 1;
262 } else {
263 dev_priv->ips.c_m = 2;
264 }
265 }
266
267 static const struct cxsr_latency cxsr_latency_table[] = {
268 {1, 0, 800, 400, 3382, 33382, 3983, 33983},
269 {1, 0, 800, 667, 3354, 33354, 3807, 33807},
270 {1, 0, 800, 800, 3347, 33347, 3763, 33763},
271 {1, 1, 800, 667, 6420, 36420, 6873, 36873},
272 {1, 1, 800, 800, 5902, 35902, 6318, 36318},
273
274 {1, 0, 667, 400, 3400, 33400, 4021, 34021},
275 {1, 0, 667, 667, 3372, 33372, 3845, 33845},
276 {1, 0, 667, 800, 3386, 33386, 3822, 33822},
277 {1, 1, 667, 667, 6438, 36438, 6911, 36911},
278 {1, 1, 667, 800, 5941, 35941, 6377, 36377},
279
280 {1, 0, 400, 400, 3472, 33472, 4173, 34173},
281 {1, 0, 400, 667, 3443, 33443, 3996, 33996},
282 {1, 0, 400, 800, 3430, 33430, 3946, 33946},
283 {1, 1, 400, 667, 6509, 36509, 7062, 37062},
284 {1, 1, 400, 800, 5985, 35985, 6501, 36501},
285
286 {0, 0, 800, 400, 3438, 33438, 4065, 34065},
287 {0, 0, 800, 667, 3410, 33410, 3889, 33889},
288 {0, 0, 800, 800, 3403, 33403, 3845, 33845},
289 {0, 1, 800, 667, 6476, 36476, 6955, 36955},
290 {0, 1, 800, 800, 5958, 35958, 6400, 36400},
291
292 {0, 0, 667, 400, 3456, 33456, 4103, 34106},
293 {0, 0, 667, 667, 3428, 33428, 3927, 33927},
294 {0, 0, 667, 800, 3443, 33443, 3905, 33905},
295 {0, 1, 667, 667, 6494, 36494, 6993, 36993},
296 {0, 1, 667, 800, 5998, 35998, 6460, 36460},
297
298 {0, 0, 400, 400, 3528, 33528, 4255, 34255},
299 {0, 0, 400, 667, 3500, 33500, 4079, 34079},
300 {0, 0, 400, 800, 3487, 33487, 4029, 34029},
301 {0, 1, 400, 667, 6566, 36566, 7145, 37145},
302 {0, 1, 400, 800, 6042, 36042, 6584, 36584},
303 };
304
305 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
306 bool is_ddr3,
307 int fsb,
308 int mem)
309 {
310 const struct cxsr_latency *latency;
311 int i;
312
313 if (fsb == 0 || mem == 0)
314 return NULL;
315
316 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
317 latency = &cxsr_latency_table[i];
318 if (is_desktop == latency->is_desktop &&
319 is_ddr3 == latency->is_ddr3 &&
320 fsb == latency->fsb_freq && mem == latency->mem_freq)
321 return latency;
322 }
323
324 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
325
326 return NULL;
327 }
328
329 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
330 {
331 u32 val;
332
333 vlv_punit_get(dev_priv);
334
335 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
336 if (enable)
337 val &= ~FORCE_DDR_HIGH_FREQ;
338 else
339 val |= FORCE_DDR_HIGH_FREQ;
340 val &= ~FORCE_DDR_LOW_FREQ;
341 val |= FORCE_DDR_FREQ_REQ_ACK;
342 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
343
344 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
345 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
346 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
347
348 vlv_punit_put(dev_priv);
349 }
350
351 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
352 {
353 u32 val;
354
355 vlv_punit_get(dev_priv);
356
357 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
358 if (enable)
359 val |= DSP_MAXFIFO_PM5_ENABLE;
360 else
361 val &= ~DSP_MAXFIFO_PM5_ENABLE;
362 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
363
364 vlv_punit_put(dev_priv);
365 }
366
367 #define FW_WM(value, plane) \
368 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
369
370 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
371 {
372 bool was_enabled;
373 u32 val;
374
375 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
376 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
377 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
378 POSTING_READ(FW_BLC_SELF_VLV);
379 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
380 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
381 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
382 POSTING_READ(FW_BLC_SELF);
383 } else if (IS_PINEVIEW(dev_priv)) {
384 val = I915_READ(DSPFW3);
385 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
386 if (enable)
387 val |= PINEVIEW_SELF_REFRESH_EN;
388 else
389 val &= ~PINEVIEW_SELF_REFRESH_EN;
390 I915_WRITE(DSPFW3, val);
391 POSTING_READ(DSPFW3);
392 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
393 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
394 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
395 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
396 I915_WRITE(FW_BLC_SELF, val);
397 POSTING_READ(FW_BLC_SELF);
398 } else if (IS_I915GM(dev_priv)) {
399
400
401
402
403
404 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
405 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
406 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
407 I915_WRITE(INSTPM, val);
408 POSTING_READ(INSTPM);
409 } else {
410 return false;
411 }
412
413 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
414
415 DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
416 enableddisabled(enable),
417 enableddisabled(was_enabled));
418
419 return was_enabled;
420 }
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
460 {
461 bool ret;
462
463 mutex_lock(&dev_priv->wm.wm_mutex);
464 ret = _intel_set_memory_cxsr(dev_priv, enable);
465 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
466 dev_priv->wm.vlv.cxsr = enable;
467 else if (IS_G4X(dev_priv))
468 dev_priv->wm.g4x.cxsr = enable;
469 mutex_unlock(&dev_priv->wm.wm_mutex);
470
471 return ret;
472 }
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488 static const int pessimal_latency_ns = 5000;
489
490 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
491 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
492
493 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
494 {
495 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
496 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
497 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
498 enum pipe pipe = crtc->pipe;
499 int sprite0_start, sprite1_start;
500
501 switch (pipe) {
502 u32 dsparb, dsparb2, dsparb3;
503 case PIPE_A:
504 dsparb = I915_READ(DSPARB);
505 dsparb2 = I915_READ(DSPARB2);
506 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
507 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
508 break;
509 case PIPE_B:
510 dsparb = I915_READ(DSPARB);
511 dsparb2 = I915_READ(DSPARB2);
512 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
513 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
514 break;
515 case PIPE_C:
516 dsparb2 = I915_READ(DSPARB2);
517 dsparb3 = I915_READ(DSPARB3);
518 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
519 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
520 break;
521 default:
522 MISSING_CASE(pipe);
523 return;
524 }
525
526 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
527 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
528 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
529 fifo_state->plane[PLANE_CURSOR] = 63;
530 }
531
532 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
533 enum i9xx_plane_id i9xx_plane)
534 {
535 u32 dsparb = I915_READ(DSPARB);
536 int size;
537
538 size = dsparb & 0x7f;
539 if (i9xx_plane == PLANE_B)
540 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
541
542 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
543 dsparb, plane_name(i9xx_plane), size);
544
545 return size;
546 }
547
548 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
549 enum i9xx_plane_id i9xx_plane)
550 {
551 u32 dsparb = I915_READ(DSPARB);
552 int size;
553
554 size = dsparb & 0x1ff;
555 if (i9xx_plane == PLANE_B)
556 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
557 size >>= 1;
558
559 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
560 dsparb, plane_name(i9xx_plane), size);
561
562 return size;
563 }
564
565 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
566 enum i9xx_plane_id i9xx_plane)
567 {
568 u32 dsparb = I915_READ(DSPARB);
569 int size;
570
571 size = dsparb & 0x7f;
572 size >>= 2;
573
574 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
575 dsparb, plane_name(i9xx_plane), size);
576
577 return size;
578 }
579
580
581 static const struct intel_watermark_params pineview_display_wm = {
582 .fifo_size = PINEVIEW_DISPLAY_FIFO,
583 .max_wm = PINEVIEW_MAX_WM,
584 .default_wm = PINEVIEW_DFT_WM,
585 .guard_size = PINEVIEW_GUARD_WM,
586 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
587 };
588 static const struct intel_watermark_params pineview_display_hplloff_wm = {
589 .fifo_size = PINEVIEW_DISPLAY_FIFO,
590 .max_wm = PINEVIEW_MAX_WM,
591 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
592 .guard_size = PINEVIEW_GUARD_WM,
593 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
594 };
595 static const struct intel_watermark_params pineview_cursor_wm = {
596 .fifo_size = PINEVIEW_CURSOR_FIFO,
597 .max_wm = PINEVIEW_CURSOR_MAX_WM,
598 .default_wm = PINEVIEW_CURSOR_DFT_WM,
599 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
600 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
601 };
602 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
603 .fifo_size = PINEVIEW_CURSOR_FIFO,
604 .max_wm = PINEVIEW_CURSOR_MAX_WM,
605 .default_wm = PINEVIEW_CURSOR_DFT_WM,
606 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
607 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
608 };
609 static const struct intel_watermark_params i965_cursor_wm_info = {
610 .fifo_size = I965_CURSOR_FIFO,
611 .max_wm = I965_CURSOR_MAX_WM,
612 .default_wm = I965_CURSOR_DFT_WM,
613 .guard_size = 2,
614 .cacheline_size = I915_FIFO_LINE_SIZE,
615 };
616 static const struct intel_watermark_params i945_wm_info = {
617 .fifo_size = I945_FIFO_SIZE,
618 .max_wm = I915_MAX_WM,
619 .default_wm = 1,
620 .guard_size = 2,
621 .cacheline_size = I915_FIFO_LINE_SIZE,
622 };
623 static const struct intel_watermark_params i915_wm_info = {
624 .fifo_size = I915_FIFO_SIZE,
625 .max_wm = I915_MAX_WM,
626 .default_wm = 1,
627 .guard_size = 2,
628 .cacheline_size = I915_FIFO_LINE_SIZE,
629 };
630 static const struct intel_watermark_params i830_a_wm_info = {
631 .fifo_size = I855GM_FIFO_SIZE,
632 .max_wm = I915_MAX_WM,
633 .default_wm = 1,
634 .guard_size = 2,
635 .cacheline_size = I830_FIFO_LINE_SIZE,
636 };
637 static const struct intel_watermark_params i830_bc_wm_info = {
638 .fifo_size = I855GM_FIFO_SIZE,
639 .max_wm = I915_MAX_WM/2,
640 .default_wm = 1,
641 .guard_size = 2,
642 .cacheline_size = I830_FIFO_LINE_SIZE,
643 };
644 static const struct intel_watermark_params i845_wm_info = {
645 .fifo_size = I830_FIFO_SIZE,
646 .max_wm = I915_MAX_WM,
647 .default_wm = 1,
648 .guard_size = 2,
649 .cacheline_size = I830_FIFO_LINE_SIZE,
650 };
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685 static unsigned int intel_wm_method1(unsigned int pixel_rate,
686 unsigned int cpp,
687 unsigned int latency)
688 {
689 u64 ret;
690
691 ret = mul_u32_u32(pixel_rate, cpp * latency);
692 ret = DIV_ROUND_UP_ULL(ret, 10000);
693
694 return ret;
695 }
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727 static unsigned int intel_wm_method2(unsigned int pixel_rate,
728 unsigned int htotal,
729 unsigned int width,
730 unsigned int cpp,
731 unsigned int latency)
732 {
733 unsigned int ret;
734
735
736
737
738
739 if (WARN_ON_ONCE(htotal == 0))
740 htotal = 1;
741
742 ret = (latency * pixel_rate) / (htotal * 10000);
743 ret = (ret + 1) * width * cpp;
744
745 return ret;
746 }
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767 static unsigned int intel_calculate_wm(int pixel_rate,
768 const struct intel_watermark_params *wm,
769 int fifo_size, int cpp,
770 unsigned int latency_ns)
771 {
772 int entries, wm_size;
773
774
775
776
777
778
779
780 entries = intel_wm_method1(pixel_rate, cpp,
781 latency_ns / 100);
782 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
783 wm->guard_size;
784 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
785
786 wm_size = fifo_size - entries;
787 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
788
789
790 if (wm_size > wm->max_wm)
791 wm_size = wm->max_wm;
792 if (wm_size <= 0)
793 wm_size = wm->default_wm;
794
795
796
797
798
799
800
801
802 if (wm_size <= 8)
803 wm_size = 8;
804
805 return wm_size;
806 }
807
808 static bool is_disabling(int old, int new, int threshold)
809 {
810 return old >= threshold && new < threshold;
811 }
812
813 static bool is_enabling(int old, int new, int threshold)
814 {
815 return old < threshold && new >= threshold;
816 }
817
818 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
819 {
820 return dev_priv->wm.max_level + 1;
821 }
822
823 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
824 const struct intel_plane_state *plane_state)
825 {
826 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
827
828
829 if (!crtc_state->base.active)
830 return false;
831
832
833
834
835
836
837
838
839
840 if (plane->id == PLANE_CURSOR)
841 return plane_state->base.fb != NULL;
842 else
843 return plane_state->base.visible;
844 }
845
846 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
847 {
848 struct intel_crtc *crtc, *enabled = NULL;
849
850 for_each_intel_crtc(&dev_priv->drm, crtc) {
851 if (intel_crtc_active(crtc)) {
852 if (enabled)
853 return NULL;
854 enabled = crtc;
855 }
856 }
857
858 return enabled;
859 }
860
861 static void pineview_update_wm(struct intel_crtc *unused_crtc)
862 {
863 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
864 struct intel_crtc *crtc;
865 const struct cxsr_latency *latency;
866 u32 reg;
867 unsigned int wm;
868
869 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
870 dev_priv->is_ddr3,
871 dev_priv->fsb_freq,
872 dev_priv->mem_freq);
873 if (!latency) {
874 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
875 intel_set_memory_cxsr(dev_priv, false);
876 return;
877 }
878
879 crtc = single_enabled_crtc(dev_priv);
880 if (crtc) {
881 const struct drm_display_mode *adjusted_mode =
882 &crtc->config->base.adjusted_mode;
883 const struct drm_framebuffer *fb =
884 crtc->base.primary->state->fb;
885 int cpp = fb->format->cpp[0];
886 int clock = adjusted_mode->crtc_clock;
887
888
889 wm = intel_calculate_wm(clock, &pineview_display_wm,
890 pineview_display_wm.fifo_size,
891 cpp, latency->display_sr);
892 reg = I915_READ(DSPFW1);
893 reg &= ~DSPFW_SR_MASK;
894 reg |= FW_WM(wm, SR);
895 I915_WRITE(DSPFW1, reg);
896 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
897
898
899 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
900 pineview_display_wm.fifo_size,
901 4, latency->cursor_sr);
902 reg = I915_READ(DSPFW3);
903 reg &= ~DSPFW_CURSOR_SR_MASK;
904 reg |= FW_WM(wm, CURSOR_SR);
905 I915_WRITE(DSPFW3, reg);
906
907
908 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
909 pineview_display_hplloff_wm.fifo_size,
910 cpp, latency->display_hpll_disable);
911 reg = I915_READ(DSPFW3);
912 reg &= ~DSPFW_HPLL_SR_MASK;
913 reg |= FW_WM(wm, HPLL_SR);
914 I915_WRITE(DSPFW3, reg);
915
916
917 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
918 pineview_display_hplloff_wm.fifo_size,
919 4, latency->cursor_hpll_disable);
920 reg = I915_READ(DSPFW3);
921 reg &= ~DSPFW_HPLL_CURSOR_MASK;
922 reg |= FW_WM(wm, HPLL_CURSOR);
923 I915_WRITE(DSPFW3, reg);
924 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
925
926 intel_set_memory_cxsr(dev_priv, true);
927 } else {
928 intel_set_memory_cxsr(dev_priv, false);
929 }
930 }
931
932
933
934
935
936
937
938
939
940
941
942 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
943 {
944 int tlb_miss = fifo_size * 64 - width * cpp * 8;
945
946 return max(0, tlb_miss);
947 }
948
949 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
950 const struct g4x_wm_values *wm)
951 {
952 enum pipe pipe;
953
954 for_each_pipe(dev_priv, pipe)
955 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
956
957 I915_WRITE(DSPFW1,
958 FW_WM(wm->sr.plane, SR) |
959 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
960 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
961 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
962 I915_WRITE(DSPFW2,
963 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
964 FW_WM(wm->sr.fbc, FBC_SR) |
965 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
966 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
967 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
968 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
969 I915_WRITE(DSPFW3,
970 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
971 FW_WM(wm->sr.cursor, CURSOR_SR) |
972 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
973 FW_WM(wm->hpll.plane, HPLL_SR));
974
975 POSTING_READ(DSPFW1);
976 }
977
978 #define FW_WM_VLV(value, plane) \
979 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
980
981 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
982 const struct vlv_wm_values *wm)
983 {
984 enum pipe pipe;
985
986 for_each_pipe(dev_priv, pipe) {
987 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
988
989 I915_WRITE(VLV_DDL(pipe),
990 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
991 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
992 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
993 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
994 }
995
996
997
998
999
1000
1001 I915_WRITE(DSPHOWM, 0);
1002 I915_WRITE(DSPHOWM1, 0);
1003 I915_WRITE(DSPFW4, 0);
1004 I915_WRITE(DSPFW5, 0);
1005 I915_WRITE(DSPFW6, 0);
1006
1007 I915_WRITE(DSPFW1,
1008 FW_WM(wm->sr.plane, SR) |
1009 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1010 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1011 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1012 I915_WRITE(DSPFW2,
1013 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1014 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1015 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1016 I915_WRITE(DSPFW3,
1017 FW_WM(wm->sr.cursor, CURSOR_SR));
1018
1019 if (IS_CHERRYVIEW(dev_priv)) {
1020 I915_WRITE(DSPFW7_CHV,
1021 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1022 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1023 I915_WRITE(DSPFW8_CHV,
1024 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1025 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1026 I915_WRITE(DSPFW9_CHV,
1027 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1028 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1029 I915_WRITE(DSPHOWM,
1030 FW_WM(wm->sr.plane >> 9, SR_HI) |
1031 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1032 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1033 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1034 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1035 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1036 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1037 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1038 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1039 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1040 } else {
1041 I915_WRITE(DSPFW7,
1042 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1043 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1044 I915_WRITE(DSPHOWM,
1045 FW_WM(wm->sr.plane >> 9, SR_HI) |
1046 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1047 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1048 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1049 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1050 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1051 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1052 }
1053
1054 POSTING_READ(DSPFW1);
1055 }
1056
1057 #undef FW_WM_VLV
1058
1059 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1060 {
1061
1062 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1063 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1064 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1065
1066 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1067 }
1068
1069 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1070 {
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085 switch (plane_id) {
1086 case PLANE_CURSOR:
1087 return 63;
1088 case PLANE_PRIMARY:
1089 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1090 case PLANE_SPRITE0:
1091 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1092 default:
1093 MISSING_CASE(plane_id);
1094 return 0;
1095 }
1096 }
1097
1098 static int g4x_fbc_fifo_size(int level)
1099 {
1100 switch (level) {
1101 case G4X_WM_LEVEL_SR:
1102 return 7;
1103 case G4X_WM_LEVEL_HPLL:
1104 return 15;
1105 default:
1106 MISSING_CASE(level);
1107 return 0;
1108 }
1109 }
1110
1111 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1112 const struct intel_plane_state *plane_state,
1113 int level)
1114 {
1115 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1116 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1117 const struct drm_display_mode *adjusted_mode =
1118 &crtc_state->base.adjusted_mode;
1119 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1120 unsigned int clock, htotal, cpp, width, wm;
1121
1122 if (latency == 0)
1123 return USHRT_MAX;
1124
1125 if (!intel_wm_plane_visible(crtc_state, plane_state))
1126 return 0;
1127
1128 cpp = plane_state->base.fb->format->cpp[0];
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1142 level != G4X_WM_LEVEL_NORMAL)
1143 cpp = max(cpp, 4u);
1144
1145 clock = adjusted_mode->crtc_clock;
1146 htotal = adjusted_mode->crtc_htotal;
1147
1148 if (plane->id == PLANE_CURSOR)
1149 width = plane_state->base.crtc_w;
1150 else
1151 width = drm_rect_width(&plane_state->base.dst);
1152
1153 if (plane->id == PLANE_CURSOR) {
1154 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1155 } else if (plane->id == PLANE_PRIMARY &&
1156 level == G4X_WM_LEVEL_NORMAL) {
1157 wm = intel_wm_method1(clock, cpp, latency);
1158 } else {
1159 unsigned int small, large;
1160
1161 small = intel_wm_method1(clock, cpp, latency);
1162 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1163
1164 wm = min(small, large);
1165 }
1166
1167 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1168 width, cpp);
1169
1170 wm = DIV_ROUND_UP(wm, 64) + 2;
1171
1172 return min_t(unsigned int, wm, USHRT_MAX);
1173 }
1174
1175 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1176 int level, enum plane_id plane_id, u16 value)
1177 {
1178 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1179 bool dirty = false;
1180
1181 for (; level < intel_wm_num_levels(dev_priv); level++) {
1182 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1183
1184 dirty |= raw->plane[plane_id] != value;
1185 raw->plane[plane_id] = value;
1186 }
1187
1188 return dirty;
1189 }
1190
1191 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1192 int level, u16 value)
1193 {
1194 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1195 bool dirty = false;
1196
1197
1198 level = max(level, G4X_WM_LEVEL_SR);
1199
1200 for (; level < intel_wm_num_levels(dev_priv); level++) {
1201 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1202
1203 dirty |= raw->fbc != value;
1204 raw->fbc = value;
1205 }
1206
1207 return dirty;
1208 }
1209
1210 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1211 const struct intel_plane_state *plane_state,
1212 u32 pri_val);
1213
1214 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1215 const struct intel_plane_state *plane_state)
1216 {
1217 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1218 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1219 enum plane_id plane_id = plane->id;
1220 bool dirty = false;
1221 int level;
1222
1223 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1224 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1225 if (plane_id == PLANE_PRIMARY)
1226 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1227 goto out;
1228 }
1229
1230 for (level = 0; level < num_levels; level++) {
1231 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1232 int wm, max_wm;
1233
1234 wm = g4x_compute_wm(crtc_state, plane_state, level);
1235 max_wm = g4x_plane_fifo_size(plane_id, level);
1236
1237 if (wm > max_wm)
1238 break;
1239
1240 dirty |= raw->plane[plane_id] != wm;
1241 raw->plane[plane_id] = wm;
1242
1243 if (plane_id != PLANE_PRIMARY ||
1244 level == G4X_WM_LEVEL_NORMAL)
1245 continue;
1246
1247 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1248 raw->plane[plane_id]);
1249 max_wm = g4x_fbc_fifo_size(level);
1250
1251
1252
1253
1254
1255 if (wm > max_wm)
1256 wm = USHRT_MAX;
1257
1258 dirty |= raw->fbc != wm;
1259 raw->fbc = wm;
1260 }
1261
1262
1263 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1264
1265 if (plane_id == PLANE_PRIMARY)
1266 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1267
1268 out:
1269 if (dirty) {
1270 DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1271 plane->base.name,
1272 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1273 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1274 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1275
1276 if (plane_id == PLANE_PRIMARY)
1277 DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
1278 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1279 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1280 }
1281
1282 return dirty;
1283 }
1284
1285 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1286 enum plane_id plane_id, int level)
1287 {
1288 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1289
1290 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1291 }
1292
1293 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1294 int level)
1295 {
1296 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1297
1298 if (level > dev_priv->wm.max_level)
1299 return false;
1300
1301 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1302 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1303 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1304 }
1305
1306
1307 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1308 struct g4x_wm_state *wm_state, int level)
1309 {
1310 if (level <= G4X_WM_LEVEL_NORMAL) {
1311 enum plane_id plane_id;
1312
1313 for_each_plane_id_on_crtc(crtc, plane_id)
1314 wm_state->wm.plane[plane_id] = USHRT_MAX;
1315 }
1316
1317 if (level <= G4X_WM_LEVEL_SR) {
1318 wm_state->cxsr = false;
1319 wm_state->sr.cursor = USHRT_MAX;
1320 wm_state->sr.plane = USHRT_MAX;
1321 wm_state->sr.fbc = USHRT_MAX;
1322 }
1323
1324 if (level <= G4X_WM_LEVEL_HPLL) {
1325 wm_state->hpll_en = false;
1326 wm_state->hpll.cursor = USHRT_MAX;
1327 wm_state->hpll.plane = USHRT_MAX;
1328 wm_state->hpll.fbc = USHRT_MAX;
1329 }
1330 }
1331
1332 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1333 {
1334 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1335 struct intel_atomic_state *state =
1336 to_intel_atomic_state(crtc_state->base.state);
1337 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1338 int num_active_planes = hweight32(crtc_state->active_planes &
1339 ~BIT(PLANE_CURSOR));
1340 const struct g4x_pipe_wm *raw;
1341 const struct intel_plane_state *old_plane_state;
1342 const struct intel_plane_state *new_plane_state;
1343 struct intel_plane *plane;
1344 enum plane_id plane_id;
1345 int i, level;
1346 unsigned int dirty = 0;
1347
1348 for_each_oldnew_intel_plane_in_state(state, plane,
1349 old_plane_state,
1350 new_plane_state, i) {
1351 if (new_plane_state->base.crtc != &crtc->base &&
1352 old_plane_state->base.crtc != &crtc->base)
1353 continue;
1354
1355 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1356 dirty |= BIT(plane->id);
1357 }
1358
1359 if (!dirty)
1360 return 0;
1361
1362 level = G4X_WM_LEVEL_NORMAL;
1363 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1364 goto out;
1365
1366 raw = &crtc_state->wm.g4x.raw[level];
1367 for_each_plane_id_on_crtc(crtc, plane_id)
1368 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1369
1370 level = G4X_WM_LEVEL_SR;
1371
1372 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1373 goto out;
1374
1375 raw = &crtc_state->wm.g4x.raw[level];
1376 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1377 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1378 wm_state->sr.fbc = raw->fbc;
1379
1380 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1381
1382 level = G4X_WM_LEVEL_HPLL;
1383
1384 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1385 goto out;
1386
1387 raw = &crtc_state->wm.g4x.raw[level];
1388 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1389 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1390 wm_state->hpll.fbc = raw->fbc;
1391
1392 wm_state->hpll_en = wm_state->cxsr;
1393
1394 level++;
1395
1396 out:
1397 if (level == G4X_WM_LEVEL_NORMAL)
1398 return -EINVAL;
1399
1400
1401 g4x_invalidate_wms(crtc, wm_state, level);
1402
1403
1404
1405
1406
1407
1408
1409 wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1410
1411 if (level >= G4X_WM_LEVEL_SR &&
1412 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1413 wm_state->fbc_en = false;
1414 else if (level >= G4X_WM_LEVEL_HPLL &&
1415 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1416 wm_state->fbc_en = false;
1417
1418 return 0;
1419 }
1420
1421 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1422 {
1423 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1424 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1425 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1426 struct intel_atomic_state *intel_state =
1427 to_intel_atomic_state(new_crtc_state->base.state);
1428 const struct intel_crtc_state *old_crtc_state =
1429 intel_atomic_get_old_crtc_state(intel_state, crtc);
1430 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1431 enum plane_id plane_id;
1432
1433 if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
1434 *intermediate = *optimal;
1435
1436 intermediate->cxsr = false;
1437 intermediate->hpll_en = false;
1438 goto out;
1439 }
1440
1441 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1442 !new_crtc_state->disable_cxsr;
1443 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1444 !new_crtc_state->disable_cxsr;
1445 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1446
1447 for_each_plane_id_on_crtc(crtc, plane_id) {
1448 intermediate->wm.plane[plane_id] =
1449 max(optimal->wm.plane[plane_id],
1450 active->wm.plane[plane_id]);
1451
1452 WARN_ON(intermediate->wm.plane[plane_id] >
1453 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1454 }
1455
1456 intermediate->sr.plane = max(optimal->sr.plane,
1457 active->sr.plane);
1458 intermediate->sr.cursor = max(optimal->sr.cursor,
1459 active->sr.cursor);
1460 intermediate->sr.fbc = max(optimal->sr.fbc,
1461 active->sr.fbc);
1462
1463 intermediate->hpll.plane = max(optimal->hpll.plane,
1464 active->hpll.plane);
1465 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1466 active->hpll.cursor);
1467 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1468 active->hpll.fbc);
1469
1470 WARN_ON((intermediate->sr.plane >
1471 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1472 intermediate->sr.cursor >
1473 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1474 intermediate->cxsr);
1475 WARN_ON((intermediate->sr.plane >
1476 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1477 intermediate->sr.cursor >
1478 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1479 intermediate->hpll_en);
1480
1481 WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1482 intermediate->fbc_en && intermediate->cxsr);
1483 WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1484 intermediate->fbc_en && intermediate->hpll_en);
1485
1486 out:
1487
1488
1489
1490
1491 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1492 new_crtc_state->wm.need_postvbl_update = true;
1493
1494 return 0;
1495 }
1496
1497 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1498 struct g4x_wm_values *wm)
1499 {
1500 struct intel_crtc *crtc;
1501 int num_active_crtcs = 0;
1502
1503 wm->cxsr = true;
1504 wm->hpll_en = true;
1505 wm->fbc_en = true;
1506
1507 for_each_intel_crtc(&dev_priv->drm, crtc) {
1508 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1509
1510 if (!crtc->active)
1511 continue;
1512
1513 if (!wm_state->cxsr)
1514 wm->cxsr = false;
1515 if (!wm_state->hpll_en)
1516 wm->hpll_en = false;
1517 if (!wm_state->fbc_en)
1518 wm->fbc_en = false;
1519
1520 num_active_crtcs++;
1521 }
1522
1523 if (num_active_crtcs != 1) {
1524 wm->cxsr = false;
1525 wm->hpll_en = false;
1526 wm->fbc_en = false;
1527 }
1528
1529 for_each_intel_crtc(&dev_priv->drm, crtc) {
1530 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1531 enum pipe pipe = crtc->pipe;
1532
1533 wm->pipe[pipe] = wm_state->wm;
1534 if (crtc->active && wm->cxsr)
1535 wm->sr = wm_state->sr;
1536 if (crtc->active && wm->hpll_en)
1537 wm->hpll = wm_state->hpll;
1538 }
1539 }
1540
1541 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1542 {
1543 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1544 struct g4x_wm_values new_wm = {};
1545
1546 g4x_merge_wm(dev_priv, &new_wm);
1547
1548 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1549 return;
1550
1551 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1552 _intel_set_memory_cxsr(dev_priv, false);
1553
1554 g4x_write_wm_values(dev_priv, &new_wm);
1555
1556 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1557 _intel_set_memory_cxsr(dev_priv, true);
1558
1559 *old_wm = new_wm;
1560 }
1561
1562 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1563 struct intel_crtc_state *crtc_state)
1564 {
1565 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1566 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1567
1568 mutex_lock(&dev_priv->wm.wm_mutex);
1569 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1570 g4x_program_watermarks(dev_priv);
1571 mutex_unlock(&dev_priv->wm.wm_mutex);
1572 }
1573
1574 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1575 struct intel_crtc_state *crtc_state)
1576 {
1577 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1578 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1579
1580 if (!crtc_state->wm.need_postvbl_update)
1581 return;
1582
1583 mutex_lock(&dev_priv->wm.wm_mutex);
1584 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1585 g4x_program_watermarks(dev_priv);
1586 mutex_unlock(&dev_priv->wm.wm_mutex);
1587 }
1588
1589
1590 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1591 unsigned int htotal,
1592 unsigned int width,
1593 unsigned int cpp,
1594 unsigned int latency)
1595 {
1596 unsigned int ret;
1597
1598 ret = intel_wm_method2(pixel_rate, htotal,
1599 width, cpp, latency);
1600 ret = DIV_ROUND_UP(ret, 64);
1601
1602 return ret;
1603 }
1604
1605 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1606 {
1607
1608 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1609
1610 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1611
1612 if (IS_CHERRYVIEW(dev_priv)) {
1613 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1614 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1615
1616 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1617 }
1618 }
1619
1620 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1621 const struct intel_plane_state *plane_state,
1622 int level)
1623 {
1624 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1625 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1626 const struct drm_display_mode *adjusted_mode =
1627 &crtc_state->base.adjusted_mode;
1628 unsigned int clock, htotal, cpp, width, wm;
1629
1630 if (dev_priv->wm.pri_latency[level] == 0)
1631 return USHRT_MAX;
1632
1633 if (!intel_wm_plane_visible(crtc_state, plane_state))
1634 return 0;
1635
1636 cpp = plane_state->base.fb->format->cpp[0];
1637 clock = adjusted_mode->crtc_clock;
1638 htotal = adjusted_mode->crtc_htotal;
1639 width = crtc_state->pipe_src_w;
1640
1641 if (plane->id == PLANE_CURSOR) {
1642
1643
1644
1645
1646
1647
1648 wm = 63;
1649 } else {
1650 wm = vlv_wm_method2(clock, htotal, width, cpp,
1651 dev_priv->wm.pri_latency[level] * 10);
1652 }
1653
1654 return min_t(unsigned int, wm, USHRT_MAX);
1655 }
1656
1657 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1658 {
1659 return (active_planes & (BIT(PLANE_SPRITE0) |
1660 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1661 }
1662
1663 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1664 {
1665 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1666 const struct g4x_pipe_wm *raw =
1667 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1668 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1669 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1670 int num_active_planes = hweight32(active_planes);
1671 const int fifo_size = 511;
1672 int fifo_extra, fifo_left = fifo_size;
1673 int sprite0_fifo_extra = 0;
1674 unsigned int total_rate;
1675 enum plane_id plane_id;
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685 if (vlv_need_sprite0_fifo_workaround(active_planes))
1686 sprite0_fifo_extra = 1;
1687
1688 total_rate = raw->plane[PLANE_PRIMARY] +
1689 raw->plane[PLANE_SPRITE0] +
1690 raw->plane[PLANE_SPRITE1] +
1691 sprite0_fifo_extra;
1692
1693 if (total_rate > fifo_size)
1694 return -EINVAL;
1695
1696 if (total_rate == 0)
1697 total_rate = 1;
1698
1699 for_each_plane_id_on_crtc(crtc, plane_id) {
1700 unsigned int rate;
1701
1702 if ((active_planes & BIT(plane_id)) == 0) {
1703 fifo_state->plane[plane_id] = 0;
1704 continue;
1705 }
1706
1707 rate = raw->plane[plane_id];
1708 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1709 fifo_left -= fifo_state->plane[plane_id];
1710 }
1711
1712 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1713 fifo_left -= sprite0_fifo_extra;
1714
1715 fifo_state->plane[PLANE_CURSOR] = 63;
1716
1717 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1718
1719
1720 for_each_plane_id_on_crtc(crtc, plane_id) {
1721 int plane_extra;
1722
1723 if (fifo_left == 0)
1724 break;
1725
1726 if ((active_planes & BIT(plane_id)) == 0)
1727 continue;
1728
1729 plane_extra = min(fifo_extra, fifo_left);
1730 fifo_state->plane[plane_id] += plane_extra;
1731 fifo_left -= plane_extra;
1732 }
1733
1734 WARN_ON(active_planes != 0 && fifo_left != 0);
1735
1736
1737 if (active_planes == 0) {
1738 WARN_ON(fifo_left != fifo_size);
1739 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1740 }
1741
1742 return 0;
1743 }
1744
1745
1746 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1747 struct vlv_wm_state *wm_state, int level)
1748 {
1749 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1750
1751 for (; level < intel_wm_num_levels(dev_priv); level++) {
1752 enum plane_id plane_id;
1753
1754 for_each_plane_id_on_crtc(crtc, plane_id)
1755 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1756
1757 wm_state->sr[level].cursor = USHRT_MAX;
1758 wm_state->sr[level].plane = USHRT_MAX;
1759 }
1760 }
1761
1762 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1763 {
1764 if (wm > fifo_size)
1765 return USHRT_MAX;
1766 else
1767 return fifo_size - wm;
1768 }
1769
1770
1771
1772
1773
1774 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1775 int level, enum plane_id plane_id, u16 value)
1776 {
1777 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1778 int num_levels = intel_wm_num_levels(dev_priv);
1779 bool dirty = false;
1780
1781 for (; level < num_levels; level++) {
1782 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1783
1784 dirty |= raw->plane[plane_id] != value;
1785 raw->plane[plane_id] = value;
1786 }
1787
1788 return dirty;
1789 }
1790
1791 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1792 const struct intel_plane_state *plane_state)
1793 {
1794 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1795 enum plane_id plane_id = plane->id;
1796 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1797 int level;
1798 bool dirty = false;
1799
1800 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1801 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1802 goto out;
1803 }
1804
1805 for (level = 0; level < num_levels; level++) {
1806 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1807 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1808 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1809
1810 if (wm > max_wm)
1811 break;
1812
1813 dirty |= raw->plane[plane_id] != wm;
1814 raw->plane[plane_id] = wm;
1815 }
1816
1817
1818 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1819
1820 out:
1821 if (dirty)
1822 DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1823 plane->base.name,
1824 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1825 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1826 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1827
1828 return dirty;
1829 }
1830
1831 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1832 enum plane_id plane_id, int level)
1833 {
1834 const struct g4x_pipe_wm *raw =
1835 &crtc_state->wm.vlv.raw[level];
1836 const struct vlv_fifo_state *fifo_state =
1837 &crtc_state->wm.vlv.fifo_state;
1838
1839 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1840 }
1841
1842 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1843 {
1844 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1845 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1846 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1847 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1848 }
1849
1850 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1851 {
1852 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1853 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1854 struct intel_atomic_state *state =
1855 to_intel_atomic_state(crtc_state->base.state);
1856 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1857 const struct vlv_fifo_state *fifo_state =
1858 &crtc_state->wm.vlv.fifo_state;
1859 int num_active_planes = hweight32(crtc_state->active_planes &
1860 ~BIT(PLANE_CURSOR));
1861 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1862 const struct intel_plane_state *old_plane_state;
1863 const struct intel_plane_state *new_plane_state;
1864 struct intel_plane *plane;
1865 enum plane_id plane_id;
1866 int level, ret, i;
1867 unsigned int dirty = 0;
1868
1869 for_each_oldnew_intel_plane_in_state(state, plane,
1870 old_plane_state,
1871 new_plane_state, i) {
1872 if (new_plane_state->base.crtc != &crtc->base &&
1873 old_plane_state->base.crtc != &crtc->base)
1874 continue;
1875
1876 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1877 dirty |= BIT(plane->id);
1878 }
1879
1880
1881
1882
1883
1884
1885
1886 if (needs_modeset)
1887 crtc_state->fifo_changed = true;
1888
1889 if (!dirty)
1890 return 0;
1891
1892
1893 if (dirty & ~BIT(PLANE_CURSOR)) {
1894 const struct intel_crtc_state *old_crtc_state =
1895 intel_atomic_get_old_crtc_state(state, crtc);
1896 const struct vlv_fifo_state *old_fifo_state =
1897 &old_crtc_state->wm.vlv.fifo_state;
1898
1899 ret = vlv_compute_fifo(crtc_state);
1900 if (ret)
1901 return ret;
1902
1903 if (needs_modeset ||
1904 memcmp(old_fifo_state, fifo_state,
1905 sizeof(*fifo_state)) != 0)
1906 crtc_state->fifo_changed = true;
1907 }
1908
1909
1910 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1911
1912
1913
1914
1915
1916 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1917
1918 for (level = 0; level < wm_state->num_levels; level++) {
1919 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1920 const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1921
1922 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1923 break;
1924
1925 for_each_plane_id_on_crtc(crtc, plane_id) {
1926 wm_state->wm[level].plane[plane_id] =
1927 vlv_invert_wm_value(raw->plane[plane_id],
1928 fifo_state->plane[plane_id]);
1929 }
1930
1931 wm_state->sr[level].plane =
1932 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1933 raw->plane[PLANE_SPRITE0],
1934 raw->plane[PLANE_SPRITE1]),
1935 sr_fifo_size);
1936
1937 wm_state->sr[level].cursor =
1938 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1939 63);
1940 }
1941
1942 if (level == 0)
1943 return -EINVAL;
1944
1945
1946 wm_state->num_levels = level;
1947
1948
1949 vlv_invalidate_wms(crtc, wm_state, level);
1950
1951 return 0;
1952 }
1953
1954 #define VLV_FIFO(plane, value) \
1955 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1956
1957 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1958 struct intel_crtc_state *crtc_state)
1959 {
1960 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1961 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1962 struct intel_uncore *uncore = &dev_priv->uncore;
1963 const struct vlv_fifo_state *fifo_state =
1964 &crtc_state->wm.vlv.fifo_state;
1965 int sprite0_start, sprite1_start, fifo_size;
1966
1967 if (!crtc_state->fifo_changed)
1968 return;
1969
1970 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1971 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1972 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1973
1974 WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1975 WARN_ON(fifo_size != 511);
1976
1977 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988 spin_lock(&uncore->lock);
1989
1990 switch (crtc->pipe) {
1991 u32 dsparb, dsparb2, dsparb3;
1992 case PIPE_A:
1993 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1994 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1995
1996 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1997 VLV_FIFO(SPRITEB, 0xff));
1998 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1999 VLV_FIFO(SPRITEB, sprite1_start));
2000
2001 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2002 VLV_FIFO(SPRITEB_HI, 0x1));
2003 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2004 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2005
2006 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2007 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2008 break;
2009 case PIPE_B:
2010 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2011 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2012
2013 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2014 VLV_FIFO(SPRITED, 0xff));
2015 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2016 VLV_FIFO(SPRITED, sprite1_start));
2017
2018 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2019 VLV_FIFO(SPRITED_HI, 0xff));
2020 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2021 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2022
2023 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2024 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2025 break;
2026 case PIPE_C:
2027 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2028 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2029
2030 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2031 VLV_FIFO(SPRITEF, 0xff));
2032 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2033 VLV_FIFO(SPRITEF, sprite1_start));
2034
2035 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2036 VLV_FIFO(SPRITEF_HI, 0xff));
2037 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2038 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2039
2040 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2041 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2042 break;
2043 default:
2044 break;
2045 }
2046
2047 intel_uncore_posting_read_fw(uncore, DSPARB);
2048
2049 spin_unlock(&uncore->lock);
2050 }
2051
2052 #undef VLV_FIFO
2053
2054 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2055 {
2056 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
2057 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2058 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2059 struct intel_atomic_state *intel_state =
2060 to_intel_atomic_state(new_crtc_state->base.state);
2061 const struct intel_crtc_state *old_crtc_state =
2062 intel_atomic_get_old_crtc_state(intel_state, crtc);
2063 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2064 int level;
2065
2066 if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
2067 *intermediate = *optimal;
2068
2069 intermediate->cxsr = false;
2070 goto out;
2071 }
2072
2073 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2074 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2075 !new_crtc_state->disable_cxsr;
2076
2077 for (level = 0; level < intermediate->num_levels; level++) {
2078 enum plane_id plane_id;
2079
2080 for_each_plane_id_on_crtc(crtc, plane_id) {
2081 intermediate->wm[level].plane[plane_id] =
2082 min(optimal->wm[level].plane[plane_id],
2083 active->wm[level].plane[plane_id]);
2084 }
2085
2086 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2087 active->sr[level].plane);
2088 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2089 active->sr[level].cursor);
2090 }
2091
2092 vlv_invalidate_wms(crtc, intermediate, level);
2093
2094 out:
2095
2096
2097
2098
2099 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2100 new_crtc_state->wm.need_postvbl_update = true;
2101
2102 return 0;
2103 }
2104
2105 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2106 struct vlv_wm_values *wm)
2107 {
2108 struct intel_crtc *crtc;
2109 int num_active_crtcs = 0;
2110
2111 wm->level = dev_priv->wm.max_level;
2112 wm->cxsr = true;
2113
2114 for_each_intel_crtc(&dev_priv->drm, crtc) {
2115 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2116
2117 if (!crtc->active)
2118 continue;
2119
2120 if (!wm_state->cxsr)
2121 wm->cxsr = false;
2122
2123 num_active_crtcs++;
2124 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2125 }
2126
2127 if (num_active_crtcs != 1)
2128 wm->cxsr = false;
2129
2130 if (num_active_crtcs > 1)
2131 wm->level = VLV_WM_LEVEL_PM2;
2132
2133 for_each_intel_crtc(&dev_priv->drm, crtc) {
2134 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2135 enum pipe pipe = crtc->pipe;
2136
2137 wm->pipe[pipe] = wm_state->wm[wm->level];
2138 if (crtc->active && wm->cxsr)
2139 wm->sr = wm_state->sr[wm->level];
2140
2141 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2142 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2143 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2144 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2145 }
2146 }
2147
2148 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2149 {
2150 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2151 struct vlv_wm_values new_wm = {};
2152
2153 vlv_merge_wm(dev_priv, &new_wm);
2154
2155 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2156 return;
2157
2158 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2159 chv_set_memory_dvfs(dev_priv, false);
2160
2161 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2162 chv_set_memory_pm5(dev_priv, false);
2163
2164 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2165 _intel_set_memory_cxsr(dev_priv, false);
2166
2167 vlv_write_wm_values(dev_priv, &new_wm);
2168
2169 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2170 _intel_set_memory_cxsr(dev_priv, true);
2171
2172 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2173 chv_set_memory_pm5(dev_priv, true);
2174
2175 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2176 chv_set_memory_dvfs(dev_priv, true);
2177
2178 *old_wm = new_wm;
2179 }
2180
2181 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2182 struct intel_crtc_state *crtc_state)
2183 {
2184 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2185 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2186
2187 mutex_lock(&dev_priv->wm.wm_mutex);
2188 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2189 vlv_program_watermarks(dev_priv);
2190 mutex_unlock(&dev_priv->wm.wm_mutex);
2191 }
2192
2193 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2194 struct intel_crtc_state *crtc_state)
2195 {
2196 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2197 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2198
2199 if (!crtc_state->wm.need_postvbl_update)
2200 return;
2201
2202 mutex_lock(&dev_priv->wm.wm_mutex);
2203 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2204 vlv_program_watermarks(dev_priv);
2205 mutex_unlock(&dev_priv->wm.wm_mutex);
2206 }
2207
2208 static void i965_update_wm(struct intel_crtc *unused_crtc)
2209 {
2210 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2211 struct intel_crtc *crtc;
2212 int srwm = 1;
2213 int cursor_sr = 16;
2214 bool cxsr_enabled;
2215
2216
2217 crtc = single_enabled_crtc(dev_priv);
2218 if (crtc) {
2219
2220 static const int sr_latency_ns = 12000;
2221 const struct drm_display_mode *adjusted_mode =
2222 &crtc->config->base.adjusted_mode;
2223 const struct drm_framebuffer *fb =
2224 crtc->base.primary->state->fb;
2225 int clock = adjusted_mode->crtc_clock;
2226 int htotal = adjusted_mode->crtc_htotal;
2227 int hdisplay = crtc->config->pipe_src_w;
2228 int cpp = fb->format->cpp[0];
2229 int entries;
2230
2231 entries = intel_wm_method2(clock, htotal,
2232 hdisplay, cpp, sr_latency_ns / 100);
2233 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2234 srwm = I965_FIFO_SIZE - entries;
2235 if (srwm < 0)
2236 srwm = 1;
2237 srwm &= 0x1ff;
2238 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
2239 entries, srwm);
2240
2241 entries = intel_wm_method2(clock, htotal,
2242 crtc->base.cursor->state->crtc_w, 4,
2243 sr_latency_ns / 100);
2244 entries = DIV_ROUND_UP(entries,
2245 i965_cursor_wm_info.cacheline_size) +
2246 i965_cursor_wm_info.guard_size;
2247
2248 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2249 if (cursor_sr > i965_cursor_wm_info.max_wm)
2250 cursor_sr = i965_cursor_wm_info.max_wm;
2251
2252 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
2253 "cursor %d\n", srwm, cursor_sr);
2254
2255 cxsr_enabled = true;
2256 } else {
2257 cxsr_enabled = false;
2258
2259 intel_set_memory_cxsr(dev_priv, false);
2260 }
2261
2262 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2263 srwm);
2264
2265
2266 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2267 FW_WM(8, CURSORB) |
2268 FW_WM(8, PLANEB) |
2269 FW_WM(8, PLANEA));
2270 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2271 FW_WM(8, PLANEC_OLD));
2272
2273 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2274
2275 if (cxsr_enabled)
2276 intel_set_memory_cxsr(dev_priv, true);
2277 }
2278
2279 #undef FW_WM
2280
2281 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2282 {
2283 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2284 const struct intel_watermark_params *wm_info;
2285 u32 fwater_lo;
2286 u32 fwater_hi;
2287 int cwm, srwm = 1;
2288 int fifo_size;
2289 int planea_wm, planeb_wm;
2290 struct intel_crtc *crtc, *enabled = NULL;
2291
2292 if (IS_I945GM(dev_priv))
2293 wm_info = &i945_wm_info;
2294 else if (!IS_GEN(dev_priv, 2))
2295 wm_info = &i915_wm_info;
2296 else
2297 wm_info = &i830_a_wm_info;
2298
2299 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2300 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2301 if (intel_crtc_active(crtc)) {
2302 const struct drm_display_mode *adjusted_mode =
2303 &crtc->config->base.adjusted_mode;
2304 const struct drm_framebuffer *fb =
2305 crtc->base.primary->state->fb;
2306 int cpp;
2307
2308 if (IS_GEN(dev_priv, 2))
2309 cpp = 4;
2310 else
2311 cpp = fb->format->cpp[0];
2312
2313 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2314 wm_info, fifo_size, cpp,
2315 pessimal_latency_ns);
2316 enabled = crtc;
2317 } else {
2318 planea_wm = fifo_size - wm_info->guard_size;
2319 if (planea_wm > (long)wm_info->max_wm)
2320 planea_wm = wm_info->max_wm;
2321 }
2322
2323 if (IS_GEN(dev_priv, 2))
2324 wm_info = &i830_bc_wm_info;
2325
2326 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2327 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2328 if (intel_crtc_active(crtc)) {
2329 const struct drm_display_mode *adjusted_mode =
2330 &crtc->config->base.adjusted_mode;
2331 const struct drm_framebuffer *fb =
2332 crtc->base.primary->state->fb;
2333 int cpp;
2334
2335 if (IS_GEN(dev_priv, 2))
2336 cpp = 4;
2337 else
2338 cpp = fb->format->cpp[0];
2339
2340 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2341 wm_info, fifo_size, cpp,
2342 pessimal_latency_ns);
2343 if (enabled == NULL)
2344 enabled = crtc;
2345 else
2346 enabled = NULL;
2347 } else {
2348 planeb_wm = fifo_size - wm_info->guard_size;
2349 if (planeb_wm > (long)wm_info->max_wm)
2350 planeb_wm = wm_info->max_wm;
2351 }
2352
2353 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2354
2355 if (IS_I915GM(dev_priv) && enabled) {
2356 struct drm_i915_gem_object *obj;
2357
2358 obj = intel_fb_obj(enabled->base.primary->state->fb);
2359
2360
2361 if (!i915_gem_object_is_tiled(obj))
2362 enabled = NULL;
2363 }
2364
2365
2366
2367
2368 cwm = 2;
2369
2370
2371 intel_set_memory_cxsr(dev_priv, false);
2372
2373
2374 if (HAS_FW_BLC(dev_priv) && enabled) {
2375
2376 static const int sr_latency_ns = 6000;
2377 const struct drm_display_mode *adjusted_mode =
2378 &enabled->config->base.adjusted_mode;
2379 const struct drm_framebuffer *fb =
2380 enabled->base.primary->state->fb;
2381 int clock = adjusted_mode->crtc_clock;
2382 int htotal = adjusted_mode->crtc_htotal;
2383 int hdisplay = enabled->config->pipe_src_w;
2384 int cpp;
2385 int entries;
2386
2387 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2388 cpp = 4;
2389 else
2390 cpp = fb->format->cpp[0];
2391
2392 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2393 sr_latency_ns / 100);
2394 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2395 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
2396 srwm = wm_info->fifo_size - entries;
2397 if (srwm < 0)
2398 srwm = 1;
2399
2400 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2401 I915_WRITE(FW_BLC_SELF,
2402 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2403 else
2404 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2405 }
2406
2407 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2408 planea_wm, planeb_wm, cwm, srwm);
2409
2410 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2411 fwater_hi = (cwm & 0x1f);
2412
2413
2414 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2415 fwater_hi = fwater_hi | (1 << 8);
2416
2417 I915_WRITE(FW_BLC, fwater_lo);
2418 I915_WRITE(FW_BLC2, fwater_hi);
2419
2420 if (enabled)
2421 intel_set_memory_cxsr(dev_priv, true);
2422 }
2423
2424 static void i845_update_wm(struct intel_crtc *unused_crtc)
2425 {
2426 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2427 struct intel_crtc *crtc;
2428 const struct drm_display_mode *adjusted_mode;
2429 u32 fwater_lo;
2430 int planea_wm;
2431
2432 crtc = single_enabled_crtc(dev_priv);
2433 if (crtc == NULL)
2434 return;
2435
2436 adjusted_mode = &crtc->config->base.adjusted_mode;
2437 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2438 &i845_wm_info,
2439 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2440 4, pessimal_latency_ns);
2441 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2442 fwater_lo |= (3<<8) | planea_wm;
2443
2444 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
2445
2446 I915_WRITE(FW_BLC, fwater_lo);
2447 }
2448
2449
2450 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2451 unsigned int cpp,
2452 unsigned int latency)
2453 {
2454 unsigned int ret;
2455
2456 ret = intel_wm_method1(pixel_rate, cpp, latency);
2457 ret = DIV_ROUND_UP(ret, 64) + 2;
2458
2459 return ret;
2460 }
2461
2462
2463 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2464 unsigned int htotal,
2465 unsigned int width,
2466 unsigned int cpp,
2467 unsigned int latency)
2468 {
2469 unsigned int ret;
2470
2471 ret = intel_wm_method2(pixel_rate, htotal,
2472 width, cpp, latency);
2473 ret = DIV_ROUND_UP(ret, 64) + 2;
2474
2475 return ret;
2476 }
2477
2478 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2479 {
2480
2481
2482
2483
2484
2485
2486 if (WARN_ON(!cpp))
2487 return 0;
2488 if (WARN_ON(!horiz_pixels))
2489 return 0;
2490
2491 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2492 }
2493
2494 struct ilk_wm_maximums {
2495 u16 pri;
2496 u16 spr;
2497 u16 cur;
2498 u16 fbc;
2499 };
2500
2501
2502
2503
2504
2505 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2506 const struct intel_plane_state *plane_state,
2507 u32 mem_value, bool is_lp)
2508 {
2509 u32 method1, method2;
2510 int cpp;
2511
2512 if (mem_value == 0)
2513 return U32_MAX;
2514
2515 if (!intel_wm_plane_visible(crtc_state, plane_state))
2516 return 0;
2517
2518 cpp = plane_state->base.fb->format->cpp[0];
2519
2520 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2521
2522 if (!is_lp)
2523 return method1;
2524
2525 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2526 crtc_state->base.adjusted_mode.crtc_htotal,
2527 drm_rect_width(&plane_state->base.dst),
2528 cpp, mem_value);
2529
2530 return min(method1, method2);
2531 }
2532
2533
2534
2535
2536
2537 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2538 const struct intel_plane_state *plane_state,
2539 u32 mem_value)
2540 {
2541 u32 method1, method2;
2542 int cpp;
2543
2544 if (mem_value == 0)
2545 return U32_MAX;
2546
2547 if (!intel_wm_plane_visible(crtc_state, plane_state))
2548 return 0;
2549
2550 cpp = plane_state->base.fb->format->cpp[0];
2551
2552 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2553 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2554 crtc_state->base.adjusted_mode.crtc_htotal,
2555 drm_rect_width(&plane_state->base.dst),
2556 cpp, mem_value);
2557 return min(method1, method2);
2558 }
2559
2560
2561
2562
2563
2564 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2565 const struct intel_plane_state *plane_state,
2566 u32 mem_value)
2567 {
2568 int cpp;
2569
2570 if (mem_value == 0)
2571 return U32_MAX;
2572
2573 if (!intel_wm_plane_visible(crtc_state, plane_state))
2574 return 0;
2575
2576 cpp = plane_state->base.fb->format->cpp[0];
2577
2578 return ilk_wm_method2(crtc_state->pixel_rate,
2579 crtc_state->base.adjusted_mode.crtc_htotal,
2580 plane_state->base.crtc_w, cpp, mem_value);
2581 }
2582
2583
2584 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2585 const struct intel_plane_state *plane_state,
2586 u32 pri_val)
2587 {
2588 int cpp;
2589
2590 if (!intel_wm_plane_visible(crtc_state, plane_state))
2591 return 0;
2592
2593 cpp = plane_state->base.fb->format->cpp[0];
2594
2595 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->base.dst), cpp);
2596 }
2597
2598 static unsigned int
2599 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2600 {
2601 if (INTEL_GEN(dev_priv) >= 8)
2602 return 3072;
2603 else if (INTEL_GEN(dev_priv) >= 7)
2604 return 768;
2605 else
2606 return 512;
2607 }
2608
2609 static unsigned int
2610 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2611 int level, bool is_sprite)
2612 {
2613 if (INTEL_GEN(dev_priv) >= 8)
2614
2615 return level == 0 ? 255 : 2047;
2616 else if (INTEL_GEN(dev_priv) >= 7)
2617
2618 return level == 0 ? 127 : 1023;
2619 else if (!is_sprite)
2620
2621 return level == 0 ? 127 : 511;
2622 else
2623
2624 return level == 0 ? 63 : 255;
2625 }
2626
2627 static unsigned int
2628 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2629 {
2630 if (INTEL_GEN(dev_priv) >= 7)
2631 return level == 0 ? 63 : 255;
2632 else
2633 return level == 0 ? 31 : 63;
2634 }
2635
2636 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2637 {
2638 if (INTEL_GEN(dev_priv) >= 8)
2639 return 31;
2640 else
2641 return 15;
2642 }
2643
2644
2645 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2646 int level,
2647 const struct intel_wm_config *config,
2648 enum intel_ddb_partitioning ddb_partitioning,
2649 bool is_sprite)
2650 {
2651 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2652
2653
2654 if (is_sprite && !config->sprites_enabled)
2655 return 0;
2656
2657
2658 if (level == 0 || config->num_pipes_active > 1) {
2659 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
2660
2661
2662
2663
2664
2665
2666 if (INTEL_GEN(dev_priv) <= 6)
2667 fifo_size /= 2;
2668 }
2669
2670 if (config->sprites_enabled) {
2671
2672 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2673 if (is_sprite)
2674 fifo_size *= 5;
2675 fifo_size /= 6;
2676 } else {
2677 fifo_size /= 2;
2678 }
2679 }
2680
2681
2682 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2683 }
2684
2685
2686 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2687 int level,
2688 const struct intel_wm_config *config)
2689 {
2690
2691 if (level > 0 && config->num_pipes_active > 1)
2692 return 64;
2693
2694
2695 return ilk_cursor_wm_reg_max(dev_priv, level);
2696 }
2697
2698 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2699 int level,
2700 const struct intel_wm_config *config,
2701 enum intel_ddb_partitioning ddb_partitioning,
2702 struct ilk_wm_maximums *max)
2703 {
2704 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2705 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2706 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2707 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2708 }
2709
2710 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2711 int level,
2712 struct ilk_wm_maximums *max)
2713 {
2714 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2715 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2716 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2717 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2718 }
2719
2720 static bool ilk_validate_wm_level(int level,
2721 const struct ilk_wm_maximums *max,
2722 struct intel_wm_level *result)
2723 {
2724 bool ret;
2725
2726
2727 if (!result->enable)
2728 return false;
2729
2730 result->enable = result->pri_val <= max->pri &&
2731 result->spr_val <= max->spr &&
2732 result->cur_val <= max->cur;
2733
2734 ret = result->enable;
2735
2736
2737
2738
2739
2740
2741 if (level == 0 && !result->enable) {
2742 if (result->pri_val > max->pri)
2743 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2744 level, result->pri_val, max->pri);
2745 if (result->spr_val > max->spr)
2746 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2747 level, result->spr_val, max->spr);
2748 if (result->cur_val > max->cur)
2749 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2750 level, result->cur_val, max->cur);
2751
2752 result->pri_val = min_t(u32, result->pri_val, max->pri);
2753 result->spr_val = min_t(u32, result->spr_val, max->spr);
2754 result->cur_val = min_t(u32, result->cur_val, max->cur);
2755 result->enable = true;
2756 }
2757
2758 return ret;
2759 }
2760
2761 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2762 const struct intel_crtc *intel_crtc,
2763 int level,
2764 struct intel_crtc_state *crtc_state,
2765 const struct intel_plane_state *pristate,
2766 const struct intel_plane_state *sprstate,
2767 const struct intel_plane_state *curstate,
2768 struct intel_wm_level *result)
2769 {
2770 u16 pri_latency = dev_priv->wm.pri_latency[level];
2771 u16 spr_latency = dev_priv->wm.spr_latency[level];
2772 u16 cur_latency = dev_priv->wm.cur_latency[level];
2773
2774
2775 if (level > 0) {
2776 pri_latency *= 5;
2777 spr_latency *= 5;
2778 cur_latency *= 5;
2779 }
2780
2781 if (pristate) {
2782 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2783 pri_latency, level);
2784 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2785 }
2786
2787 if (sprstate)
2788 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2789
2790 if (curstate)
2791 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2792
2793 result->enable = true;
2794 }
2795
2796 static u32
2797 hsw_compute_linetime_wm(const struct intel_crtc_state *crtc_state)
2798 {
2799 const struct intel_atomic_state *intel_state =
2800 to_intel_atomic_state(crtc_state->base.state);
2801 const struct drm_display_mode *adjusted_mode =
2802 &crtc_state->base.adjusted_mode;
2803 u32 linetime, ips_linetime;
2804
2805 if (!crtc_state->base.active)
2806 return 0;
2807 if (WARN_ON(adjusted_mode->crtc_clock == 0))
2808 return 0;
2809 if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2810 return 0;
2811
2812
2813
2814
2815 linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2816 adjusted_mode->crtc_clock);
2817 ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2818 intel_state->cdclk.logical.cdclk);
2819
2820 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2821 PIPE_WM_LINETIME_TIME(linetime);
2822 }
2823
2824 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2825 u16 wm[8])
2826 {
2827 struct intel_uncore *uncore = &dev_priv->uncore;
2828
2829 if (INTEL_GEN(dev_priv) >= 9) {
2830 u32 val;
2831 int ret, i;
2832 int level, max_level = ilk_wm_max_level(dev_priv);
2833
2834
2835 val = 0;
2836 ret = sandybridge_pcode_read(dev_priv,
2837 GEN9_PCODE_READ_MEM_LATENCY,
2838 &val, NULL);
2839
2840 if (ret) {
2841 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2842 return;
2843 }
2844
2845 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2846 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2847 GEN9_MEM_LATENCY_LEVEL_MASK;
2848 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2849 GEN9_MEM_LATENCY_LEVEL_MASK;
2850 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2851 GEN9_MEM_LATENCY_LEVEL_MASK;
2852
2853
2854 val = 1;
2855 ret = sandybridge_pcode_read(dev_priv,
2856 GEN9_PCODE_READ_MEM_LATENCY,
2857 &val, NULL);
2858 if (ret) {
2859 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2860 return;
2861 }
2862
2863 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2864 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2865 GEN9_MEM_LATENCY_LEVEL_MASK;
2866 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2867 GEN9_MEM_LATENCY_LEVEL_MASK;
2868 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2869 GEN9_MEM_LATENCY_LEVEL_MASK;
2870
2871
2872
2873
2874
2875
2876 for (level = 1; level <= max_level; level++) {
2877 if (wm[level] == 0) {
2878 for (i = level + 1; i <= max_level; i++)
2879 wm[i] = 0;
2880 break;
2881 }
2882 }
2883
2884
2885
2886
2887
2888
2889
2890
2891 if (wm[0] == 0) {
2892 wm[0] += 2;
2893 for (level = 1; level <= max_level; level++) {
2894 if (wm[level] == 0)
2895 break;
2896 wm[level] += 2;
2897 }
2898 }
2899
2900
2901
2902
2903
2904
2905
2906 if (dev_priv->dram_info.is_16gb_dimm)
2907 wm[0] += 1;
2908
2909 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2910 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2911
2912 wm[0] = (sskpd >> 56) & 0xFF;
2913 if (wm[0] == 0)
2914 wm[0] = sskpd & 0xF;
2915 wm[1] = (sskpd >> 4) & 0xFF;
2916 wm[2] = (sskpd >> 12) & 0xFF;
2917 wm[3] = (sskpd >> 20) & 0x1FF;
2918 wm[4] = (sskpd >> 32) & 0x1FF;
2919 } else if (INTEL_GEN(dev_priv) >= 6) {
2920 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2921
2922 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2923 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2924 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2925 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2926 } else if (INTEL_GEN(dev_priv) >= 5) {
2927 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2928
2929
2930 wm[0] = 7;
2931 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2932 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2933 } else {
2934 MISSING_CASE(INTEL_DEVID(dev_priv));
2935 }
2936 }
2937
2938 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2939 u16 wm[5])
2940 {
2941
2942 if (IS_GEN(dev_priv, 5))
2943 wm[0] = 13;
2944 }
2945
2946 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2947 u16 wm[5])
2948 {
2949
2950 if (IS_GEN(dev_priv, 5))
2951 wm[0] = 13;
2952 }
2953
2954 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2955 {
2956
2957 if (INTEL_GEN(dev_priv) >= 9)
2958 return 7;
2959 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2960 return 4;
2961 else if (INTEL_GEN(dev_priv) >= 6)
2962 return 3;
2963 else
2964 return 2;
2965 }
2966
2967 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2968 const char *name,
2969 const u16 wm[8])
2970 {
2971 int level, max_level = ilk_wm_max_level(dev_priv);
2972
2973 for (level = 0; level <= max_level; level++) {
2974 unsigned int latency = wm[level];
2975
2976 if (latency == 0) {
2977 DRM_DEBUG_KMS("%s WM%d latency not provided\n",
2978 name, level);
2979 continue;
2980 }
2981
2982
2983
2984
2985
2986 if (INTEL_GEN(dev_priv) >= 9)
2987 latency *= 10;
2988 else if (level > 0)
2989 latency *= 5;
2990
2991 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2992 name, level, wm[level],
2993 latency / 10, latency % 10);
2994 }
2995 }
2996
2997 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2998 u16 wm[5], u16 min)
2999 {
3000 int level, max_level = ilk_wm_max_level(dev_priv);
3001
3002 if (wm[0] >= min)
3003 return false;
3004
3005 wm[0] = max(wm[0], min);
3006 for (level = 1; level <= max_level; level++)
3007 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3008
3009 return true;
3010 }
3011
3012 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3013 {
3014 bool changed;
3015
3016
3017
3018
3019
3020 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3021 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3022 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3023
3024 if (!changed)
3025 return;
3026
3027 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
3028 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3029 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3030 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3031 }
3032
3033 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3034 {
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046 if (dev_priv->wm.pri_latency[3] == 0 &&
3047 dev_priv->wm.spr_latency[3] == 0 &&
3048 dev_priv->wm.cur_latency[3] == 0)
3049 return;
3050
3051 dev_priv->wm.pri_latency[3] = 0;
3052 dev_priv->wm.spr_latency[3] = 0;
3053 dev_priv->wm.cur_latency[3] = 0;
3054
3055 DRM_DEBUG_KMS("LP3 watermarks disabled due to potential for lost interrupts\n");
3056 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3057 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3058 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3059 }
3060
3061 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3062 {
3063 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3064
3065 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3066 sizeof(dev_priv->wm.pri_latency));
3067 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3068 sizeof(dev_priv->wm.pri_latency));
3069
3070 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3071 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3072
3073 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3074 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3075 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3076
3077 if (IS_GEN(dev_priv, 6)) {
3078 snb_wm_latency_quirk(dev_priv);
3079 snb_wm_lp3_irq_quirk(dev_priv);
3080 }
3081 }
3082
3083 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3084 {
3085 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3086 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3087 }
3088
3089 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3090 struct intel_pipe_wm *pipe_wm)
3091 {
3092
3093 const struct intel_wm_config config = {
3094 .num_pipes_active = 1,
3095 .sprites_enabled = pipe_wm->sprites_enabled,
3096 .sprites_scaled = pipe_wm->sprites_scaled,
3097 };
3098 struct ilk_wm_maximums max;
3099
3100
3101 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3102
3103
3104 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3105 DRM_DEBUG_KMS("LP0 watermark invalid\n");
3106 return false;
3107 }
3108
3109 return true;
3110 }
3111
3112
3113 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3114 {
3115 struct drm_atomic_state *state = crtc_state->base.state;
3116 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3117 struct intel_pipe_wm *pipe_wm;
3118 struct drm_device *dev = state->dev;
3119 const struct drm_i915_private *dev_priv = to_i915(dev);
3120 struct drm_plane *plane;
3121 const struct drm_plane_state *plane_state;
3122 const struct intel_plane_state *pristate = NULL;
3123 const struct intel_plane_state *sprstate = NULL;
3124 const struct intel_plane_state *curstate = NULL;
3125 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3126 struct ilk_wm_maximums max;
3127
3128 pipe_wm = &crtc_state->wm.ilk.optimal;
3129
3130 drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, &crtc_state->base) {
3131 const struct intel_plane_state *ps = to_intel_plane_state(plane_state);
3132
3133 if (plane->type == DRM_PLANE_TYPE_PRIMARY)
3134 pristate = ps;
3135 else if (plane->type == DRM_PLANE_TYPE_OVERLAY)
3136 sprstate = ps;
3137 else if (plane->type == DRM_PLANE_TYPE_CURSOR)
3138 curstate = ps;
3139 }
3140
3141 pipe_wm->pipe_enabled = crtc_state->base.active;
3142 if (sprstate) {
3143 pipe_wm->sprites_enabled = sprstate->base.visible;
3144 pipe_wm->sprites_scaled = sprstate->base.visible &&
3145 (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
3146 drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
3147 }
3148
3149 usable_level = max_level;
3150
3151
3152 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3153 usable_level = 1;
3154
3155
3156 if (pipe_wm->sprites_scaled)
3157 usable_level = 0;
3158
3159 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3160 ilk_compute_wm_level(dev_priv, intel_crtc, 0, crtc_state,
3161 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3162
3163 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3164 pipe_wm->linetime = hsw_compute_linetime_wm(crtc_state);
3165
3166 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3167 return -EINVAL;
3168
3169 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3170
3171 for (level = 1; level <= usable_level; level++) {
3172 struct intel_wm_level *wm = &pipe_wm->wm[level];
3173
3174 ilk_compute_wm_level(dev_priv, intel_crtc, level, crtc_state,
3175 pristate, sprstate, curstate, wm);
3176
3177
3178
3179
3180
3181
3182 if (!ilk_validate_wm_level(level, &max, wm)) {
3183 memset(wm, 0, sizeof(*wm));
3184 break;
3185 }
3186 }
3187
3188 return 0;
3189 }
3190
3191
3192
3193
3194
3195
3196 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3197 {
3198 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->base.crtc);
3199 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3200 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3201 struct intel_atomic_state *intel_state =
3202 to_intel_atomic_state(newstate->base.state);
3203 const struct intel_crtc_state *oldstate =
3204 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3205 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3206 int level, max_level = ilk_wm_max_level(dev_priv);
3207
3208
3209
3210
3211
3212
3213 *a = newstate->wm.ilk.optimal;
3214 if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base) ||
3215 intel_state->skip_intermediate_wm)
3216 return 0;
3217
3218 a->pipe_enabled |= b->pipe_enabled;
3219 a->sprites_enabled |= b->sprites_enabled;
3220 a->sprites_scaled |= b->sprites_scaled;
3221
3222 for (level = 0; level <= max_level; level++) {
3223 struct intel_wm_level *a_wm = &a->wm[level];
3224 const struct intel_wm_level *b_wm = &b->wm[level];
3225
3226 a_wm->enable &= b_wm->enable;
3227 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3228 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3229 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3230 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3231 }
3232
3233
3234
3235
3236
3237
3238
3239 if (!ilk_validate_pipe_wm(dev_priv, a))
3240 return -EINVAL;
3241
3242
3243
3244
3245
3246 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3247 newstate->wm.need_postvbl_update = true;
3248
3249 return 0;
3250 }
3251
3252
3253
3254
3255 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3256 int level,
3257 struct intel_wm_level *ret_wm)
3258 {
3259 const struct intel_crtc *intel_crtc;
3260
3261 ret_wm->enable = true;
3262
3263 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3264 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3265 const struct intel_wm_level *wm = &active->wm[level];
3266
3267 if (!active->pipe_enabled)
3268 continue;
3269
3270
3271
3272
3273
3274
3275 if (!wm->enable)
3276 ret_wm->enable = false;
3277
3278 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3279 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3280 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3281 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3282 }
3283 }
3284
3285
3286
3287
3288 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3289 const struct intel_wm_config *config,
3290 const struct ilk_wm_maximums *max,
3291 struct intel_pipe_wm *merged)
3292 {
3293 int level, max_level = ilk_wm_max_level(dev_priv);
3294 int last_enabled_level = max_level;
3295
3296
3297 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3298 config->num_pipes_active > 1)
3299 last_enabled_level = 0;
3300
3301
3302 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3303
3304
3305 for (level = 1; level <= max_level; level++) {
3306 struct intel_wm_level *wm = &merged->wm[level];
3307
3308 ilk_merge_wm_level(dev_priv, level, wm);
3309
3310 if (level > last_enabled_level)
3311 wm->enable = false;
3312 else if (!ilk_validate_wm_level(level, max, wm))
3313
3314 last_enabled_level = level - 1;
3315
3316
3317
3318
3319
3320 if (wm->fbc_val > max->fbc) {
3321 if (wm->enable)
3322 merged->fbc_wm_enabled = false;
3323 wm->fbc_val = 0;
3324 }
3325 }
3326
3327
3328
3329
3330
3331
3332
3333 if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3334 intel_fbc_is_active(dev_priv)) {
3335 for (level = 2; level <= max_level; level++) {
3336 struct intel_wm_level *wm = &merged->wm[level];
3337
3338 wm->enable = false;
3339 }
3340 }
3341 }
3342
3343 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3344 {
3345
3346 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3347 }
3348
3349
3350 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3351 int level)
3352 {
3353 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3354 return 2 * level;
3355 else
3356 return dev_priv->wm.pri_latency[level];
3357 }
3358
3359 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3360 const struct intel_pipe_wm *merged,
3361 enum intel_ddb_partitioning partitioning,
3362 struct ilk_wm_values *results)
3363 {
3364 struct intel_crtc *intel_crtc;
3365 int level, wm_lp;
3366
3367 results->enable_fbc_wm = merged->fbc_wm_enabled;
3368 results->partitioning = partitioning;
3369
3370
3371 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3372 const struct intel_wm_level *r;
3373
3374 level = ilk_wm_lp_to_level(wm_lp, merged);
3375
3376 r = &merged->wm[level];
3377
3378
3379
3380
3381
3382 results->wm_lp[wm_lp - 1] =
3383 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3384 (r->pri_val << WM1_LP_SR_SHIFT) |
3385 r->cur_val;
3386
3387 if (r->enable)
3388 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3389
3390 if (INTEL_GEN(dev_priv) >= 8)
3391 results->wm_lp[wm_lp - 1] |=
3392 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3393 else
3394 results->wm_lp[wm_lp - 1] |=
3395 r->fbc_val << WM1_LP_FBC_SHIFT;
3396
3397
3398
3399
3400
3401 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3402 WARN_ON(wm_lp != 1);
3403 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3404 } else
3405 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3406 }
3407
3408
3409 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3410 enum pipe pipe = intel_crtc->pipe;
3411 const struct intel_wm_level *r =
3412 &intel_crtc->wm.active.ilk.wm[0];
3413
3414 if (WARN_ON(!r->enable))
3415 continue;
3416
3417 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
3418
3419 results->wm_pipe[pipe] =
3420 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3421 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3422 r->cur_val;
3423 }
3424 }
3425
3426
3427
3428 static struct intel_pipe_wm *
3429 ilk_find_best_result(struct drm_i915_private *dev_priv,
3430 struct intel_pipe_wm *r1,
3431 struct intel_pipe_wm *r2)
3432 {
3433 int level, max_level = ilk_wm_max_level(dev_priv);
3434 int level1 = 0, level2 = 0;
3435
3436 for (level = 1; level <= max_level; level++) {
3437 if (r1->wm[level].enable)
3438 level1 = level;
3439 if (r2->wm[level].enable)
3440 level2 = level;
3441 }
3442
3443 if (level1 == level2) {
3444 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3445 return r2;
3446 else
3447 return r1;
3448 } else if (level1 > level2) {
3449 return r1;
3450 } else {
3451 return r2;
3452 }
3453 }
3454
3455
3456 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3457 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
3458 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3459 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3460 #define WM_DIRTY_FBC (1 << 24)
3461 #define WM_DIRTY_DDB (1 << 25)
3462
3463 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3464 const struct ilk_wm_values *old,
3465 const struct ilk_wm_values *new)
3466 {
3467 unsigned int dirty = 0;
3468 enum pipe pipe;
3469 int wm_lp;
3470
3471 for_each_pipe(dev_priv, pipe) {
3472 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
3473 dirty |= WM_DIRTY_LINETIME(pipe);
3474
3475 dirty |= WM_DIRTY_LP_ALL;
3476 }
3477
3478 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3479 dirty |= WM_DIRTY_PIPE(pipe);
3480
3481 dirty |= WM_DIRTY_LP_ALL;
3482 }
3483 }
3484
3485 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3486 dirty |= WM_DIRTY_FBC;
3487
3488 dirty |= WM_DIRTY_LP_ALL;
3489 }
3490
3491 if (old->partitioning != new->partitioning) {
3492 dirty |= WM_DIRTY_DDB;
3493
3494 dirty |= WM_DIRTY_LP_ALL;
3495 }
3496
3497
3498 if (dirty & WM_DIRTY_LP_ALL)
3499 return dirty;
3500
3501
3502 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3503 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3504 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3505 break;
3506 }
3507
3508
3509 for (; wm_lp <= 3; wm_lp++)
3510 dirty |= WM_DIRTY_LP(wm_lp);
3511
3512 return dirty;
3513 }
3514
3515 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3516 unsigned int dirty)
3517 {
3518 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3519 bool changed = false;
3520
3521 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3522 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3523 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3524 changed = true;
3525 }
3526 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3527 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3528 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3529 changed = true;
3530 }
3531 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3532 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3533 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3534 changed = true;
3535 }
3536
3537
3538
3539
3540
3541
3542 return changed;
3543 }
3544
3545
3546
3547
3548
3549 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3550 struct ilk_wm_values *results)
3551 {
3552 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3553 unsigned int dirty;
3554 u32 val;
3555
3556 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3557 if (!dirty)
3558 return;
3559
3560 _ilk_disable_lp_wm(dev_priv, dirty);
3561
3562 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3563 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3564 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3565 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3566 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3567 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3568
3569 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3570 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3571 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3572 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3573 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3574 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3575
3576 if (dirty & WM_DIRTY_DDB) {
3577 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3578 val = I915_READ(WM_MISC);
3579 if (results->partitioning == INTEL_DDB_PART_1_2)
3580 val &= ~WM_MISC_DATA_PARTITION_5_6;
3581 else
3582 val |= WM_MISC_DATA_PARTITION_5_6;
3583 I915_WRITE(WM_MISC, val);
3584 } else {
3585 val = I915_READ(DISP_ARB_CTL2);
3586 if (results->partitioning == INTEL_DDB_PART_1_2)
3587 val &= ~DISP_DATA_PARTITION_5_6;
3588 else
3589 val |= DISP_DATA_PARTITION_5_6;
3590 I915_WRITE(DISP_ARB_CTL2, val);
3591 }
3592 }
3593
3594 if (dirty & WM_DIRTY_FBC) {
3595 val = I915_READ(DISP_ARB_CTL);
3596 if (results->enable_fbc_wm)
3597 val &= ~DISP_FBC_WM_DIS;
3598 else
3599 val |= DISP_FBC_WM_DIS;
3600 I915_WRITE(DISP_ARB_CTL, val);
3601 }
3602
3603 if (dirty & WM_DIRTY_LP(1) &&
3604 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3605 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3606
3607 if (INTEL_GEN(dev_priv) >= 7) {
3608 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3609 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3610 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3611 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3612 }
3613
3614 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3615 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3616 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3617 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3618 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3619 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3620
3621 dev_priv->wm.hw = *results;
3622 }
3623
3624 bool ilk_disable_lp_wm(struct drm_device *dev)
3625 {
3626 struct drm_i915_private *dev_priv = to_i915(dev);
3627
3628 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3629 }
3630
3631 static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
3632 {
3633 u8 enabled_slices;
3634
3635
3636 enabled_slices = 1;
3637
3638
3639 if (INTEL_GEN(dev_priv) < 11)
3640 return enabled_slices;
3641
3642
3643
3644
3645
3646
3647 if (0 && I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
3648 enabled_slices++;
3649
3650 return enabled_slices;
3651 }
3652
3653
3654
3655
3656
3657 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3658 {
3659 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3660 }
3661
3662 static bool
3663 intel_has_sagv(struct drm_i915_private *dev_priv)
3664 {
3665 return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3666 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3667 }
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680 int
3681 intel_enable_sagv(struct drm_i915_private *dev_priv)
3682 {
3683 int ret;
3684
3685 if (!intel_has_sagv(dev_priv))
3686 return 0;
3687
3688 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3689 return 0;
3690
3691 DRM_DEBUG_KMS("Enabling SAGV\n");
3692 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3693 GEN9_SAGV_ENABLE);
3694
3695
3696
3697
3698
3699
3700
3701 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3702 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3703 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3704 return 0;
3705 } else if (ret < 0) {
3706 DRM_ERROR("Failed to enable SAGV\n");
3707 return ret;
3708 }
3709
3710 dev_priv->sagv_status = I915_SAGV_ENABLED;
3711 return 0;
3712 }
3713
3714 int
3715 intel_disable_sagv(struct drm_i915_private *dev_priv)
3716 {
3717 int ret;
3718
3719 if (!intel_has_sagv(dev_priv))
3720 return 0;
3721
3722 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3723 return 0;
3724
3725 DRM_DEBUG_KMS("Disabling SAGV\n");
3726
3727 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3728 GEN9_SAGV_DISABLE,
3729 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3730 1);
3731
3732
3733
3734
3735 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3736 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3737 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3738 return 0;
3739 } else if (ret < 0) {
3740 DRM_ERROR("Failed to disable SAGV (%d)\n", ret);
3741 return ret;
3742 }
3743
3744 dev_priv->sagv_status = I915_SAGV_DISABLED;
3745 return 0;
3746 }
3747
3748 bool intel_can_enable_sagv(struct intel_atomic_state *state)
3749 {
3750 struct drm_device *dev = state->base.dev;
3751 struct drm_i915_private *dev_priv = to_i915(dev);
3752 struct intel_crtc *crtc;
3753 struct intel_plane *plane;
3754 struct intel_crtc_state *crtc_state;
3755 enum pipe pipe;
3756 int level, latency;
3757 int sagv_block_time_us;
3758
3759 if (!intel_has_sagv(dev_priv))
3760 return false;
3761
3762 if (IS_GEN(dev_priv, 9))
3763 sagv_block_time_us = 30;
3764 else if (IS_GEN(dev_priv, 10))
3765 sagv_block_time_us = 20;
3766 else
3767 sagv_block_time_us = 10;
3768
3769
3770
3771
3772 if (hweight32(state->active_crtcs) == 0)
3773 return true;
3774
3775
3776
3777
3778
3779 if (hweight32(state->active_crtcs) > 1)
3780 return false;
3781
3782
3783 pipe = ffs(state->active_crtcs) - 1;
3784 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3785 crtc_state = to_intel_crtc_state(crtc->base.state);
3786
3787 if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3788 return false;
3789
3790 for_each_intel_plane_on_crtc(dev, crtc, plane) {
3791 struct skl_plane_wm *wm =
3792 &crtc_state->wm.skl.optimal.planes[plane->id];
3793
3794
3795 if (!wm->wm[0].plane_en)
3796 continue;
3797
3798
3799 for (level = ilk_wm_max_level(dev_priv);
3800 !wm->wm[level].plane_en; --level)
3801 { }
3802
3803 latency = dev_priv->wm.skl_latency[level];
3804
3805 if (skl_needs_memory_bw_wa(dev_priv) &&
3806 plane->base.state->fb->modifier ==
3807 I915_FORMAT_MOD_X_TILED)
3808 latency += 15;
3809
3810
3811
3812
3813
3814
3815 if (latency < sagv_block_time_us)
3816 return false;
3817 }
3818
3819 return true;
3820 }
3821
3822 static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv,
3823 const struct intel_crtc_state *crtc_state,
3824 const u64 total_data_rate,
3825 const int num_active,
3826 struct skl_ddb_allocation *ddb)
3827 {
3828 const struct drm_display_mode *adjusted_mode;
3829 u64 total_data_bw;
3830 u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3831
3832 WARN_ON(ddb_size == 0);
3833
3834 if (INTEL_GEN(dev_priv) < 11)
3835 return ddb_size - 4;
3836
3837 adjusted_mode = &crtc_state->base.adjusted_mode;
3838 total_data_bw = total_data_rate * drm_mode_vrefresh(adjusted_mode);
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848 if (0 && (num_active > 1 || total_data_bw >= GBps(12))) {
3849 ddb->enabled_slices = 2;
3850 } else {
3851 ddb->enabled_slices = 1;
3852 ddb_size /= 2;
3853 }
3854
3855 return ddb_size;
3856 }
3857
3858 static void
3859 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
3860 const struct intel_crtc_state *crtc_state,
3861 const u64 total_data_rate,
3862 struct skl_ddb_allocation *ddb,
3863 struct skl_ddb_entry *alloc,
3864 int *num_active )
3865 {
3866 struct drm_atomic_state *state = crtc_state->base.state;
3867 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3868 struct drm_crtc *for_crtc = crtc_state->base.crtc;
3869 const struct intel_crtc *crtc;
3870 u32 pipe_width = 0, total_width = 0, width_before_pipe = 0;
3871 enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
3872 u16 ddb_size;
3873 u32 i;
3874
3875 if (WARN_ON(!state) || !crtc_state->base.active) {
3876 alloc->start = 0;
3877 alloc->end = 0;
3878 *num_active = hweight32(dev_priv->active_crtcs);
3879 return;
3880 }
3881
3882 if (intel_state->active_pipe_changes)
3883 *num_active = hweight32(intel_state->active_crtcs);
3884 else
3885 *num_active = hweight32(dev_priv->active_crtcs);
3886
3887 ddb_size = intel_get_ddb_size(dev_priv, crtc_state, total_data_rate,
3888 *num_active, ddb);
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898 if (!intel_state->active_pipe_changes && !intel_state->modeset) {
3899
3900
3901
3902
3903 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3904 return;
3905 }
3906
3907
3908
3909
3910
3911
3912 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3913 const struct drm_display_mode *adjusted_mode =
3914 &crtc_state->base.adjusted_mode;
3915 enum pipe pipe = crtc->pipe;
3916 int hdisplay, vdisplay;
3917
3918 if (!crtc_state->base.enable)
3919 continue;
3920
3921 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
3922 total_width += hdisplay;
3923
3924 if (pipe < for_pipe)
3925 width_before_pipe += hdisplay;
3926 else if (pipe == for_pipe)
3927 pipe_width = hdisplay;
3928 }
3929
3930 alloc->start = ddb_size * width_before_pipe / total_width;
3931 alloc->end = ddb_size * (width_before_pipe + pipe_width) / total_width;
3932 }
3933
3934 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
3935 int width, const struct drm_format_info *format,
3936 u64 modifier, unsigned int rotation,
3937 u32 plane_pixel_rate, struct skl_wm_params *wp,
3938 int color_plane);
3939 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
3940 int level,
3941 const struct skl_wm_params *wp,
3942 const struct skl_wm_level *result_prev,
3943 struct skl_wm_level *result );
3944
3945 static unsigned int
3946 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
3947 int num_active)
3948 {
3949 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
3950 int level, max_level = ilk_wm_max_level(dev_priv);
3951 struct skl_wm_level wm = {};
3952 int ret, min_ddb_alloc = 0;
3953 struct skl_wm_params wp;
3954
3955 ret = skl_compute_wm_params(crtc_state, 256,
3956 drm_format_info(DRM_FORMAT_ARGB8888),
3957 DRM_FORMAT_MOD_LINEAR,
3958 DRM_MODE_ROTATE_0,
3959 crtc_state->pixel_rate, &wp, 0);
3960 WARN_ON(ret);
3961
3962 for (level = 0; level <= max_level; level++) {
3963 skl_compute_plane_wm(crtc_state, level, &wp, &wm, &wm);
3964 if (wm.min_ddb_alloc == U16_MAX)
3965 break;
3966
3967 min_ddb_alloc = wm.min_ddb_alloc;
3968 }
3969
3970 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
3971 }
3972
3973 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
3974 struct skl_ddb_entry *entry, u32 reg)
3975 {
3976
3977 entry->start = reg & DDB_ENTRY_MASK;
3978 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
3979
3980 if (entry->end)
3981 entry->end += 1;
3982 }
3983
3984 static void
3985 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
3986 const enum pipe pipe,
3987 const enum plane_id plane_id,
3988 struct skl_ddb_entry *ddb_y,
3989 struct skl_ddb_entry *ddb_uv)
3990 {
3991 u32 val, val2;
3992 u32 fourcc = 0;
3993
3994
3995 if (plane_id == PLANE_CURSOR) {
3996 val = I915_READ(CUR_BUF_CFG(pipe));
3997 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
3998 return;
3999 }
4000
4001 val = I915_READ(PLANE_CTL(pipe, plane_id));
4002
4003
4004 if (val & PLANE_CTL_ENABLE)
4005 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4006 val & PLANE_CTL_ORDER_RGBX,
4007 val & PLANE_CTL_ALPHA_MASK);
4008
4009 if (INTEL_GEN(dev_priv) >= 11) {
4010 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4011 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4012 } else {
4013 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4014 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4015
4016 if (is_planar_yuv_format(fourcc))
4017 swap(val, val2);
4018
4019 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4020 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4021 }
4022 }
4023
4024 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4025 struct skl_ddb_entry *ddb_y,
4026 struct skl_ddb_entry *ddb_uv)
4027 {
4028 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4029 enum intel_display_power_domain power_domain;
4030 enum pipe pipe = crtc->pipe;
4031 intel_wakeref_t wakeref;
4032 enum plane_id plane_id;
4033
4034 power_domain = POWER_DOMAIN_PIPE(pipe);
4035 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4036 if (!wakeref)
4037 return;
4038
4039 for_each_plane_id_on_crtc(crtc, plane_id)
4040 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4041 plane_id,
4042 &ddb_y[plane_id],
4043 &ddb_uv[plane_id]);
4044
4045 intel_display_power_put(dev_priv, power_domain, wakeref);
4046 }
4047
4048 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
4049 struct skl_ddb_allocation *ddb )
4050 {
4051 ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv);
4052 }
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070 static uint_fixed_16_16_t
4071 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4072 const struct intel_plane_state *plane_state)
4073 {
4074 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4075 u32 src_w, src_h, dst_w, dst_h;
4076 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4077 uint_fixed_16_16_t downscale_h, downscale_w;
4078
4079 if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
4080 return u32_to_fixed16(0);
4081
4082
4083 if (plane->id == PLANE_CURSOR) {
4084
4085
4086
4087
4088 src_w = plane_state->base.src_w >> 16;
4089 src_h = plane_state->base.src_h >> 16;
4090 dst_w = plane_state->base.crtc_w;
4091 dst_h = plane_state->base.crtc_h;
4092 } else {
4093
4094
4095
4096
4097
4098 src_w = drm_rect_width(&plane_state->base.src) >> 16;
4099 src_h = drm_rect_height(&plane_state->base.src) >> 16;
4100 dst_w = drm_rect_width(&plane_state->base.dst);
4101 dst_h = drm_rect_height(&plane_state->base.dst);
4102 }
4103
4104 fp_w_ratio = div_fixed16(src_w, dst_w);
4105 fp_h_ratio = div_fixed16(src_h, dst_h);
4106 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4107 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4108
4109 return mul_fixed16(downscale_w, downscale_h);
4110 }
4111
4112 static uint_fixed_16_16_t
4113 skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
4114 {
4115 uint_fixed_16_16_t pipe_downscale = u32_to_fixed16(1);
4116
4117 if (!crtc_state->base.enable)
4118 return pipe_downscale;
4119
4120 if (crtc_state->pch_pfit.enabled) {
4121 u32 src_w, src_h, dst_w, dst_h;
4122 u32 pfit_size = crtc_state->pch_pfit.size;
4123 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4124 uint_fixed_16_16_t downscale_h, downscale_w;
4125
4126 src_w = crtc_state->pipe_src_w;
4127 src_h = crtc_state->pipe_src_h;
4128 dst_w = pfit_size >> 16;
4129 dst_h = pfit_size & 0xffff;
4130
4131 if (!dst_w || !dst_h)
4132 return pipe_downscale;
4133
4134 fp_w_ratio = div_fixed16(src_w, dst_w);
4135 fp_h_ratio = div_fixed16(src_h, dst_h);
4136 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4137 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4138
4139 pipe_downscale = mul_fixed16(downscale_w, downscale_h);
4140 }
4141
4142 return pipe_downscale;
4143 }
4144
4145 int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
4146 struct intel_crtc_state *crtc_state)
4147 {
4148 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4149 struct drm_atomic_state *state = crtc_state->base.state;
4150 struct drm_plane *plane;
4151 const struct drm_plane_state *drm_plane_state;
4152 int crtc_clock, dotclk;
4153 u32 pipe_max_pixel_rate;
4154 uint_fixed_16_16_t pipe_downscale;
4155 uint_fixed_16_16_t max_downscale = u32_to_fixed16(1);
4156
4157 if (!crtc_state->base.enable)
4158 return 0;
4159
4160 drm_atomic_crtc_state_for_each_plane_state(plane, drm_plane_state, &crtc_state->base) {
4161 uint_fixed_16_16_t plane_downscale;
4162 uint_fixed_16_16_t fp_9_div_8 = div_fixed16(9, 8);
4163 int bpp;
4164 const struct intel_plane_state *plane_state =
4165 to_intel_plane_state(drm_plane_state);
4166
4167 if (!intel_wm_plane_visible(crtc_state, plane_state))
4168 continue;
4169
4170 if (WARN_ON(!plane_state->base.fb))
4171 return -EINVAL;
4172
4173 plane_downscale = skl_plane_downscale_amount(crtc_state, plane_state);
4174 bpp = plane_state->base.fb->format->cpp[0] * 8;
4175 if (bpp == 64)
4176 plane_downscale = mul_fixed16(plane_downscale,
4177 fp_9_div_8);
4178
4179 max_downscale = max_fixed16(plane_downscale, max_downscale);
4180 }
4181 pipe_downscale = skl_pipe_downscale_amount(crtc_state);
4182
4183 pipe_downscale = mul_fixed16(pipe_downscale, max_downscale);
4184
4185 crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
4186 dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
4187
4188 if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
4189 dotclk *= 2;
4190
4191 pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
4192
4193 if (pipe_max_pixel_rate < crtc_clock) {
4194 DRM_DEBUG_KMS("Max supported pixel clock with scaling exceeded\n");
4195 return -EINVAL;
4196 }
4197
4198 return 0;
4199 }
4200
4201 static u64
4202 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4203 const struct intel_plane_state *plane_state,
4204 const int plane)
4205 {
4206 struct intel_plane *intel_plane = to_intel_plane(plane_state->base.plane);
4207 u32 data_rate;
4208 u32 width = 0, height = 0;
4209 struct drm_framebuffer *fb;
4210 u32 format;
4211 uint_fixed_16_16_t down_scale_amount;
4212 u64 rate;
4213
4214 if (!plane_state->base.visible)
4215 return 0;
4216
4217 fb = plane_state->base.fb;
4218 format = fb->format->format;
4219
4220 if (intel_plane->id == PLANE_CURSOR)
4221 return 0;
4222 if (plane == 1 && !is_planar_yuv_format(format))
4223 return 0;
4224
4225
4226
4227
4228
4229
4230 width = drm_rect_width(&plane_state->base.src) >> 16;
4231 height = drm_rect_height(&plane_state->base.src) >> 16;
4232
4233
4234 if (plane == 1 && is_planar_yuv_format(format)) {
4235 width /= 2;
4236 height /= 2;
4237 }
4238
4239 data_rate = width * height;
4240
4241 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4242
4243 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4244
4245 rate *= fb->format->cpp[plane];
4246 return rate;
4247 }
4248
4249 static u64
4250 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4251 u64 *plane_data_rate,
4252 u64 *uv_plane_data_rate)
4253 {
4254 struct drm_atomic_state *state = crtc_state->base.state;
4255 struct drm_plane *plane;
4256 const struct drm_plane_state *drm_plane_state;
4257 u64 total_data_rate = 0;
4258
4259 if (WARN_ON(!state))
4260 return 0;
4261
4262
4263 drm_atomic_crtc_state_for_each_plane_state(plane, drm_plane_state, &crtc_state->base) {
4264 enum plane_id plane_id = to_intel_plane(plane)->id;
4265 const struct intel_plane_state *plane_state =
4266 to_intel_plane_state(drm_plane_state);
4267 u64 rate;
4268
4269
4270 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4271 plane_data_rate[plane_id] = rate;
4272 total_data_rate += rate;
4273
4274
4275 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4276 uv_plane_data_rate[plane_id] = rate;
4277 total_data_rate += rate;
4278 }
4279
4280 return total_data_rate;
4281 }
4282
4283 static u64
4284 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4285 u64 *plane_data_rate)
4286 {
4287 struct drm_plane *plane;
4288 const struct drm_plane_state *drm_plane_state;
4289 u64 total_data_rate = 0;
4290
4291 if (WARN_ON(!crtc_state->base.state))
4292 return 0;
4293
4294
4295 drm_atomic_crtc_state_for_each_plane_state(plane, drm_plane_state, &crtc_state->base) {
4296 const struct intel_plane_state *plane_state =
4297 to_intel_plane_state(drm_plane_state);
4298 enum plane_id plane_id = to_intel_plane(plane)->id;
4299 u64 rate;
4300
4301 if (!plane_state->linked_plane) {
4302 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4303 plane_data_rate[plane_id] = rate;
4304 total_data_rate += rate;
4305 } else {
4306 enum plane_id y_plane_id;
4307
4308
4309
4310
4311
4312
4313
4314
4315 if (plane_state->slave)
4316 continue;
4317
4318
4319 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4320 y_plane_id = plane_state->linked_plane->id;
4321 plane_data_rate[y_plane_id] = rate;
4322 total_data_rate += rate;
4323
4324 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4325 plane_data_rate[plane_id] = rate;
4326 total_data_rate += rate;
4327 }
4328 }
4329
4330 return total_data_rate;
4331 }
4332
4333 static int
4334 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state,
4335 struct skl_ddb_allocation *ddb )
4336 {
4337 struct drm_atomic_state *state = crtc_state->base.state;
4338 struct drm_crtc *crtc = crtc_state->base.crtc;
4339 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
4340 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4341 struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4342 u16 alloc_size, start = 0;
4343 u16 total[I915_MAX_PLANES] = {};
4344 u16 uv_total[I915_MAX_PLANES] = {};
4345 u64 total_data_rate;
4346 enum plane_id plane_id;
4347 int num_active;
4348 u64 plane_data_rate[I915_MAX_PLANES] = {};
4349 u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4350 u32 blocks;
4351 int level;
4352
4353
4354 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4355 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4356
4357 if (WARN_ON(!state))
4358 return 0;
4359
4360 if (!crtc_state->base.active) {
4361 alloc->start = alloc->end = 0;
4362 return 0;
4363 }
4364
4365 if (INTEL_GEN(dev_priv) >= 11)
4366 total_data_rate =
4367 icl_get_total_relative_data_rate(crtc_state,
4368 plane_data_rate);
4369 else
4370 total_data_rate =
4371 skl_get_total_relative_data_rate(crtc_state,
4372 plane_data_rate,
4373 uv_plane_data_rate);
4374
4375
4376 skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state, total_data_rate,
4377 ddb, alloc, &num_active);
4378 alloc_size = skl_ddb_entry_size(alloc);
4379 if (alloc_size == 0)
4380 return 0;
4381
4382
4383 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4384 alloc_size -= total[PLANE_CURSOR];
4385 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4386 alloc->end - total[PLANE_CURSOR];
4387 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4388
4389 if (total_data_rate == 0)
4390 return 0;
4391
4392
4393
4394
4395
4396 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4397 blocks = 0;
4398 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4399 const struct skl_plane_wm *wm =
4400 &crtc_state->wm.skl.optimal.planes[plane_id];
4401
4402 if (plane_id == PLANE_CURSOR) {
4403 if (WARN_ON(wm->wm[level].min_ddb_alloc >
4404 total[PLANE_CURSOR])) {
4405 blocks = U32_MAX;
4406 break;
4407 }
4408 continue;
4409 }
4410
4411 blocks += wm->wm[level].min_ddb_alloc;
4412 blocks += wm->uv_wm[level].min_ddb_alloc;
4413 }
4414
4415 if (blocks <= alloc_size) {
4416 alloc_size -= blocks;
4417 break;
4418 }
4419 }
4420
4421 if (level < 0) {
4422 DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
4423 DRM_DEBUG_KMS("minimum required %d/%d\n", blocks,
4424 alloc_size);
4425 return -EINVAL;
4426 }
4427
4428
4429
4430
4431
4432
4433 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4434 const struct skl_plane_wm *wm =
4435 &crtc_state->wm.skl.optimal.planes[plane_id];
4436 u64 rate;
4437 u16 extra;
4438
4439 if (plane_id == PLANE_CURSOR)
4440 continue;
4441
4442
4443
4444
4445
4446 if (total_data_rate == 0)
4447 break;
4448
4449 rate = plane_data_rate[plane_id];
4450 extra = min_t(u16, alloc_size,
4451 DIV64_U64_ROUND_UP(alloc_size * rate,
4452 total_data_rate));
4453 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4454 alloc_size -= extra;
4455 total_data_rate -= rate;
4456
4457 if (total_data_rate == 0)
4458 break;
4459
4460 rate = uv_plane_data_rate[plane_id];
4461 extra = min_t(u16, alloc_size,
4462 DIV64_U64_ROUND_UP(alloc_size * rate,
4463 total_data_rate));
4464 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4465 alloc_size -= extra;
4466 total_data_rate -= rate;
4467 }
4468 WARN_ON(alloc_size != 0 || total_data_rate != 0);
4469
4470
4471 start = alloc->start;
4472 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4473 struct skl_ddb_entry *plane_alloc =
4474 &crtc_state->wm.skl.plane_ddb_y[plane_id];
4475 struct skl_ddb_entry *uv_plane_alloc =
4476 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4477
4478 if (plane_id == PLANE_CURSOR)
4479 continue;
4480
4481
4482 WARN_ON(INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4483
4484
4485 if (total[plane_id]) {
4486 plane_alloc->start = start;
4487 start += total[plane_id];
4488 plane_alloc->end = start;
4489 }
4490
4491 if (uv_total[plane_id]) {
4492 uv_plane_alloc->start = start;
4493 start += uv_total[plane_id];
4494 uv_plane_alloc->end = start;
4495 }
4496 }
4497
4498
4499
4500
4501
4502
4503
4504 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4505 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4506 struct skl_plane_wm *wm =
4507 &crtc_state->wm.skl.optimal.planes[plane_id];
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521 if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
4522 wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
4523 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
4524
4525
4526
4527
4528
4529 if (IS_GEN(dev_priv, 11) &&
4530 level == 1 && wm->wm[0].plane_en) {
4531 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
4532 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
4533 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
4534 }
4535 }
4536 }
4537
4538
4539
4540
4541
4542 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4543 struct skl_plane_wm *wm =
4544 &crtc_state->wm.skl.optimal.planes[plane_id];
4545
4546 if (wm->trans_wm.plane_res_b >= total[plane_id])
4547 memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
4548 }
4549
4550 return 0;
4551 }
4552
4553
4554
4555
4556
4557
4558
4559 static uint_fixed_16_16_t
4560 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
4561 u8 cpp, u32 latency, u32 dbuf_block_size)
4562 {
4563 u32 wm_intermediate_val;
4564 uint_fixed_16_16_t ret;
4565
4566 if (latency == 0)
4567 return FP_16_16_MAX;
4568
4569 wm_intermediate_val = latency * pixel_rate * cpp;
4570 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4571
4572 if (INTEL_GEN(dev_priv) >= 10)
4573 ret = add_fixed16_u32(ret, 1);
4574
4575 return ret;
4576 }
4577
4578 static uint_fixed_16_16_t
4579 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
4580 uint_fixed_16_16_t plane_blocks_per_line)
4581 {
4582 u32 wm_intermediate_val;
4583 uint_fixed_16_16_t ret;
4584
4585 if (latency == 0)
4586 return FP_16_16_MAX;
4587
4588 wm_intermediate_val = latency * pixel_rate;
4589 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4590 pipe_htotal * 1000);
4591 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4592 return ret;
4593 }
4594
4595 static uint_fixed_16_16_t
4596 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
4597 {
4598 u32 pixel_rate;
4599 u32 crtc_htotal;
4600 uint_fixed_16_16_t linetime_us;
4601
4602 if (!crtc_state->base.active)
4603 return u32_to_fixed16(0);
4604
4605 pixel_rate = crtc_state->pixel_rate;
4606
4607 if (WARN_ON(pixel_rate == 0))
4608 return u32_to_fixed16(0);
4609
4610 crtc_htotal = crtc_state->base.adjusted_mode.crtc_htotal;
4611 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4612
4613 return linetime_us;
4614 }
4615
4616 static u32
4617 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
4618 const struct intel_plane_state *plane_state)
4619 {
4620 u64 adjusted_pixel_rate;
4621 uint_fixed_16_16_t downscale_amount;
4622
4623
4624 if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
4625 return 0;
4626
4627
4628
4629
4630
4631 adjusted_pixel_rate = crtc_state->pixel_rate;
4632 downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4633
4634 return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4635 downscale_amount);
4636 }
4637
4638 static int
4639 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4640 int width, const struct drm_format_info *format,
4641 u64 modifier, unsigned int rotation,
4642 u32 plane_pixel_rate, struct skl_wm_params *wp,
4643 int color_plane)
4644 {
4645 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4646 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4647 u32 interm_pbpl;
4648
4649
4650 if (color_plane == 1 && !is_planar_yuv_format(format->format)) {
4651 DRM_DEBUG_KMS("Non planar format have single plane\n");
4652 return -EINVAL;
4653 }
4654
4655 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
4656 modifier == I915_FORMAT_MOD_Yf_TILED ||
4657 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4658 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4659 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
4660 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4661 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4662 wp->is_planar = is_planar_yuv_format(format->format);
4663
4664 wp->width = width;
4665 if (color_plane == 1 && wp->is_planar)
4666 wp->width /= 2;
4667
4668 wp->cpp = format->cpp[color_plane];
4669 wp->plane_pixel_rate = plane_pixel_rate;
4670
4671 if (INTEL_GEN(dev_priv) >= 11 &&
4672 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
4673 wp->dbuf_block_size = 256;
4674 else
4675 wp->dbuf_block_size = 512;
4676
4677 if (drm_rotation_90_or_270(rotation)) {
4678 switch (wp->cpp) {
4679 case 1:
4680 wp->y_min_scanlines = 16;
4681 break;
4682 case 2:
4683 wp->y_min_scanlines = 8;
4684 break;
4685 case 4:
4686 wp->y_min_scanlines = 4;
4687 break;
4688 default:
4689 MISSING_CASE(wp->cpp);
4690 return -EINVAL;
4691 }
4692 } else {
4693 wp->y_min_scanlines = 4;
4694 }
4695
4696 if (skl_needs_memory_bw_wa(dev_priv))
4697 wp->y_min_scanlines *= 2;
4698
4699 wp->plane_bytes_per_line = wp->width * wp->cpp;
4700 if (wp->y_tiled) {
4701 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4702 wp->y_min_scanlines,
4703 wp->dbuf_block_size);
4704
4705 if (INTEL_GEN(dev_priv) >= 10)
4706 interm_pbpl++;
4707
4708 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4709 wp->y_min_scanlines);
4710 } else if (wp->x_tiled && IS_GEN(dev_priv, 9)) {
4711 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4712 wp->dbuf_block_size);
4713 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4714 } else {
4715 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4716 wp->dbuf_block_size) + 1;
4717 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4718 }
4719
4720 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4721 wp->plane_blocks_per_line);
4722
4723 wp->linetime_us = fixed16_to_u32_round_up(
4724 intel_get_linetime_us(crtc_state));
4725
4726 return 0;
4727 }
4728
4729 static int
4730 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
4731 const struct intel_plane_state *plane_state,
4732 struct skl_wm_params *wp, int color_plane)
4733 {
4734 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4735 const struct drm_framebuffer *fb = plane_state->base.fb;
4736 int width;
4737
4738 if (plane->id == PLANE_CURSOR) {
4739 width = plane_state->base.crtc_w;
4740 } else {
4741
4742
4743
4744
4745
4746 width = drm_rect_width(&plane_state->base.src) >> 16;
4747 }
4748
4749 return skl_compute_wm_params(crtc_state, width,
4750 fb->format, fb->modifier,
4751 plane_state->base.rotation,
4752 skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
4753 wp, color_plane);
4754 }
4755
4756 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
4757 {
4758 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4759 return true;
4760
4761
4762 return level > 0;
4763 }
4764
4765 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4766 int level,
4767 const struct skl_wm_params *wp,
4768 const struct skl_wm_level *result_prev,
4769 struct skl_wm_level *result )
4770 {
4771 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4772 u32 latency = dev_priv->wm.skl_latency[level];
4773 uint_fixed_16_16_t method1, method2;
4774 uint_fixed_16_16_t selected_result;
4775 u32 res_blocks, res_lines, min_ddb_alloc = 0;
4776
4777 if (latency == 0) {
4778
4779 result->min_ddb_alloc = U16_MAX;
4780 return;
4781 }
4782
4783
4784
4785
4786
4787 if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) &&
4788 dev_priv->ipc_enabled)
4789 latency += 4;
4790
4791 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
4792 latency += 15;
4793
4794 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
4795 wp->cpp, latency, wp->dbuf_block_size);
4796 method2 = skl_wm_method2(wp->plane_pixel_rate,
4797 crtc_state->base.adjusted_mode.crtc_htotal,
4798 latency,
4799 wp->plane_blocks_per_line);
4800
4801 if (wp->y_tiled) {
4802 selected_result = max_fixed16(method2, wp->y_tile_minimum);
4803 } else {
4804 if ((wp->cpp * crtc_state->base.adjusted_mode.crtc_htotal /
4805 wp->dbuf_block_size < 1) &&
4806 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
4807 selected_result = method2;
4808 } else if (latency >= wp->linetime_us) {
4809 if (IS_GEN(dev_priv, 9) &&
4810 !IS_GEMINILAKE(dev_priv))
4811 selected_result = min_fixed16(method1, method2);
4812 else
4813 selected_result = method2;
4814 } else {
4815 selected_result = method1;
4816 }
4817 }
4818
4819 res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
4820 res_lines = div_round_up_fixed16(selected_result,
4821 wp->plane_blocks_per_line);
4822
4823 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
4824
4825 if (level == 0 && wp->rc_surface)
4826 res_blocks +=
4827 fixed16_to_u32_round_up(wp->y_tile_minimum);
4828
4829
4830 if (level >= 1 && level <= 7) {
4831 if (wp->y_tiled) {
4832 res_blocks +=
4833 fixed16_to_u32_round_up(wp->y_tile_minimum);
4834 res_lines += wp->y_min_scanlines;
4835 } else {
4836 res_blocks++;
4837 }
4838
4839
4840
4841
4842
4843
4844
4845 if (result_prev->plane_res_b > res_blocks)
4846 res_blocks = result_prev->plane_res_b;
4847 }
4848 }
4849
4850 if (INTEL_GEN(dev_priv) >= 11) {
4851 if (wp->y_tiled) {
4852 int extra_lines;
4853
4854 if (res_lines % wp->y_min_scanlines == 0)
4855 extra_lines = wp->y_min_scanlines;
4856 else
4857 extra_lines = wp->y_min_scanlines * 2 -
4858 res_lines % wp->y_min_scanlines;
4859
4860 min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
4861 wp->plane_blocks_per_line);
4862 } else {
4863 min_ddb_alloc = res_blocks +
4864 DIV_ROUND_UP(res_blocks, 10);
4865 }
4866 }
4867
4868 if (!skl_wm_has_lines(dev_priv, level))
4869 res_lines = 0;
4870
4871 if (res_lines > 31) {
4872
4873 result->min_ddb_alloc = U16_MAX;
4874 return;
4875 }
4876
4877
4878
4879
4880
4881
4882
4883 result->plane_res_b = res_blocks;
4884 result->plane_res_l = res_lines;
4885
4886 result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
4887 result->plane_en = true;
4888 }
4889
4890 static void
4891 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
4892 const struct skl_wm_params *wm_params,
4893 struct skl_wm_level *levels)
4894 {
4895 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4896 int level, max_level = ilk_wm_max_level(dev_priv);
4897 struct skl_wm_level *result_prev = &levels[0];
4898
4899 for (level = 0; level <= max_level; level++) {
4900 struct skl_wm_level *result = &levels[level];
4901
4902 skl_compute_plane_wm(crtc_state, level, wm_params,
4903 result_prev, result);
4904
4905 result_prev = result;
4906 }
4907 }
4908
4909 static u32
4910 skl_compute_linetime_wm(const struct intel_crtc_state *crtc_state)
4911 {
4912 struct drm_atomic_state *state = crtc_state->base.state;
4913 struct drm_i915_private *dev_priv = to_i915(state->dev);
4914 uint_fixed_16_16_t linetime_us;
4915 u32 linetime_wm;
4916
4917 linetime_us = intel_get_linetime_us(crtc_state);
4918 linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
4919
4920
4921 if (IS_GEN9_LP(dev_priv) && dev_priv->ipc_enabled)
4922 linetime_wm /= 2;
4923
4924 return linetime_wm;
4925 }
4926
4927 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
4928 const struct skl_wm_params *wp,
4929 struct skl_plane_wm *wm)
4930 {
4931 struct drm_device *dev = crtc_state->base.crtc->dev;
4932 const struct drm_i915_private *dev_priv = to_i915(dev);
4933 u16 trans_min, trans_y_tile_min;
4934 const u16 trans_amount = 10;
4935 u16 wm0_sel_res_b, trans_offset_b, res_blocks;
4936
4937
4938 if (INTEL_GEN(dev_priv) <= 9)
4939 return;
4940
4941
4942 if (!dev_priv->ipc_enabled)
4943 return;
4944
4945 trans_min = 14;
4946 if (INTEL_GEN(dev_priv) >= 11)
4947 trans_min = 4;
4948
4949 trans_offset_b = trans_min + trans_amount;
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961 wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
4962
4963 if (wp->y_tiled) {
4964 trans_y_tile_min =
4965 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
4966 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
4967 trans_offset_b;
4968 } else {
4969 res_blocks = wm0_sel_res_b + trans_offset_b;
4970
4971
4972 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
4973 res_blocks += 1;
4974
4975 }
4976
4977
4978
4979
4980
4981
4982 wm->trans_wm.plane_res_b = res_blocks + 1;
4983 wm->trans_wm.plane_en = true;
4984 }
4985
4986 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
4987 const struct intel_plane_state *plane_state,
4988 enum plane_id plane_id, int color_plane)
4989 {
4990 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
4991 struct skl_wm_params wm_params;
4992 int ret;
4993
4994 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
4995 &wm_params, color_plane);
4996 if (ret)
4997 return ret;
4998
4999 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5000 skl_compute_transition_wm(crtc_state, &wm_params, wm);
5001
5002 return 0;
5003 }
5004
5005 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5006 const struct intel_plane_state *plane_state,
5007 enum plane_id plane_id)
5008 {
5009 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5010 struct skl_wm_params wm_params;
5011 int ret;
5012
5013 wm->is_planar = true;
5014
5015
5016 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5017 &wm_params, 1);
5018 if (ret)
5019 return ret;
5020
5021 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5022
5023 return 0;
5024 }
5025
5026 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5027 const struct intel_plane_state *plane_state)
5028 {
5029 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
5030 const struct drm_framebuffer *fb = plane_state->base.fb;
5031 enum plane_id plane_id = plane->id;
5032 int ret;
5033
5034 if (!intel_wm_plane_visible(crtc_state, plane_state))
5035 return 0;
5036
5037 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5038 plane_id, 0);
5039 if (ret)
5040 return ret;
5041
5042 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5043 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5044 plane_id);
5045 if (ret)
5046 return ret;
5047 }
5048
5049 return 0;
5050 }
5051
5052 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5053 const struct intel_plane_state *plane_state)
5054 {
5055 enum plane_id plane_id = to_intel_plane(plane_state->base.plane)->id;
5056 int ret;
5057
5058
5059 if (plane_state->slave)
5060 return 0;
5061
5062 if (plane_state->linked_plane) {
5063 const struct drm_framebuffer *fb = plane_state->base.fb;
5064 enum plane_id y_plane_id = plane_state->linked_plane->id;
5065
5066 WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state));
5067 WARN_ON(!fb->format->is_yuv ||
5068 fb->format->num_planes == 1);
5069
5070 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5071 y_plane_id, 0);
5072 if (ret)
5073 return ret;
5074
5075 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5076 plane_id, 1);
5077 if (ret)
5078 return ret;
5079 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5080 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5081 plane_id, 0);
5082 if (ret)
5083 return ret;
5084 }
5085
5086 return 0;
5087 }
5088
5089 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5090 {
5091 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
5092 struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5093 struct drm_plane *plane;
5094 const struct drm_plane_state *drm_plane_state;
5095 int ret;
5096
5097
5098
5099
5100
5101 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5102
5103 drm_atomic_crtc_state_for_each_plane_state(plane, drm_plane_state,
5104 &crtc_state->base) {
5105 const struct intel_plane_state *plane_state =
5106 to_intel_plane_state(drm_plane_state);
5107
5108 if (INTEL_GEN(dev_priv) >= 11)
5109 ret = icl_build_plane_wm(crtc_state, plane_state);
5110 else
5111 ret = skl_build_plane_wm(crtc_state, plane_state);
5112 if (ret)
5113 return ret;
5114 }
5115
5116 pipe_wm->linetime = skl_compute_linetime_wm(crtc_state);
5117
5118 return 0;
5119 }
5120
5121 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5122 i915_reg_t reg,
5123 const struct skl_ddb_entry *entry)
5124 {
5125 if (entry->end)
5126 I915_WRITE_FW(reg, (entry->end - 1) << 16 | entry->start);
5127 else
5128 I915_WRITE_FW(reg, 0);
5129 }
5130
5131 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5132 i915_reg_t reg,
5133 const struct skl_wm_level *level)
5134 {
5135 u32 val = 0;
5136
5137 if (level->plane_en)
5138 val |= PLANE_WM_EN;
5139 if (level->ignore_lines)
5140 val |= PLANE_WM_IGNORE_LINES;
5141 val |= level->plane_res_b;
5142 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5143
5144 I915_WRITE_FW(reg, val);
5145 }
5146
5147 void skl_write_plane_wm(struct intel_plane *plane,
5148 const struct intel_crtc_state *crtc_state)
5149 {
5150 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5151 int level, max_level = ilk_wm_max_level(dev_priv);
5152 enum plane_id plane_id = plane->id;
5153 enum pipe pipe = plane->pipe;
5154 const struct skl_plane_wm *wm =
5155 &crtc_state->wm.skl.optimal.planes[plane_id];
5156 const struct skl_ddb_entry *ddb_y =
5157 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5158 const struct skl_ddb_entry *ddb_uv =
5159 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5160
5161 for (level = 0; level <= max_level; level++) {
5162 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5163 &wm->wm[level]);
5164 }
5165 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5166 &wm->trans_wm);
5167
5168 if (INTEL_GEN(dev_priv) >= 11) {
5169 skl_ddb_entry_write(dev_priv,
5170 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5171 return;
5172 }
5173
5174 if (wm->is_planar)
5175 swap(ddb_y, ddb_uv);
5176
5177 skl_ddb_entry_write(dev_priv,
5178 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5179 skl_ddb_entry_write(dev_priv,
5180 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5181 }
5182
5183 void skl_write_cursor_wm(struct intel_plane *plane,
5184 const struct intel_crtc_state *crtc_state)
5185 {
5186 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5187 int level, max_level = ilk_wm_max_level(dev_priv);
5188 enum plane_id plane_id = plane->id;
5189 enum pipe pipe = plane->pipe;
5190 const struct skl_plane_wm *wm =
5191 &crtc_state->wm.skl.optimal.planes[plane_id];
5192 const struct skl_ddb_entry *ddb =
5193 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5194
5195 for (level = 0; level <= max_level; level++) {
5196 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5197 &wm->wm[level]);
5198 }
5199 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5200
5201 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5202 }
5203
5204 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5205 const struct skl_wm_level *l2)
5206 {
5207 return l1->plane_en == l2->plane_en &&
5208 l1->ignore_lines == l2->ignore_lines &&
5209 l1->plane_res_l == l2->plane_res_l &&
5210 l1->plane_res_b == l2->plane_res_b;
5211 }
5212
5213 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5214 const struct skl_plane_wm *wm1,
5215 const struct skl_plane_wm *wm2)
5216 {
5217 int level, max_level = ilk_wm_max_level(dev_priv);
5218
5219 for (level = 0; level <= max_level; level++) {
5220 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]) ||
5221 !skl_wm_level_equals(&wm1->uv_wm[level], &wm2->uv_wm[level]))
5222 return false;
5223 }
5224
5225 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5226 }
5227
5228 static bool skl_pipe_wm_equals(struct intel_crtc *crtc,
5229 const struct skl_pipe_wm *wm1,
5230 const struct skl_pipe_wm *wm2)
5231 {
5232 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5233 enum plane_id plane_id;
5234
5235 for_each_plane_id_on_crtc(crtc, plane_id) {
5236 if (!skl_plane_wm_equals(dev_priv,
5237 &wm1->planes[plane_id],
5238 &wm2->planes[plane_id]))
5239 return false;
5240 }
5241
5242 return wm1->linetime == wm2->linetime;
5243 }
5244
5245 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5246 const struct skl_ddb_entry *b)
5247 {
5248 return a->start < b->end && b->start < a->end;
5249 }
5250
5251 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5252 const struct skl_ddb_entry *entries,
5253 int num_entries, int ignore_idx)
5254 {
5255 int i;
5256
5257 for (i = 0; i < num_entries; i++) {
5258 if (i != ignore_idx &&
5259 skl_ddb_entries_overlap(ddb, &entries[i]))
5260 return true;
5261 }
5262
5263 return false;
5264 }
5265
5266 static u32
5267 pipes_modified(struct intel_atomic_state *state)
5268 {
5269 struct intel_crtc *crtc;
5270 struct intel_crtc_state *crtc_state;
5271 u32 i, ret = 0;
5272
5273 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i)
5274 ret |= drm_crtc_mask(&crtc->base);
5275
5276 return ret;
5277 }
5278
5279 static int
5280 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5281 struct intel_crtc_state *new_crtc_state)
5282 {
5283 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->base.state);
5284 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5285 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5286 struct intel_plane *plane;
5287
5288 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5289 struct intel_plane_state *plane_state;
5290 enum plane_id plane_id = plane->id;
5291
5292 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5293 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5294 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5295 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5296 continue;
5297
5298 plane_state = intel_atomic_get_plane_state(state, plane);
5299 if (IS_ERR(plane_state))
5300 return PTR_ERR(plane_state);
5301
5302 new_crtc_state->update_planes |= BIT(plane_id);
5303 }
5304
5305 return 0;
5306 }
5307
5308 static int
5309 skl_compute_ddb(struct intel_atomic_state *state)
5310 {
5311 const struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5312 struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
5313 struct intel_crtc_state *old_crtc_state;
5314 struct intel_crtc_state *new_crtc_state;
5315 struct intel_crtc *crtc;
5316 int ret, i;
5317
5318 memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
5319
5320 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5321 new_crtc_state, i) {
5322 ret = skl_allocate_pipe_ddb(new_crtc_state, ddb);
5323 if (ret)
5324 return ret;
5325
5326 ret = skl_ddb_add_affected_planes(old_crtc_state,
5327 new_crtc_state);
5328 if (ret)
5329 return ret;
5330 }
5331
5332 return 0;
5333 }
5334
5335 static char enast(bool enable)
5336 {
5337 return enable ? '*' : ' ';
5338 }
5339
5340 static void
5341 skl_print_wm_changes(struct intel_atomic_state *state)
5342 {
5343 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5344 const struct intel_crtc_state *old_crtc_state;
5345 const struct intel_crtc_state *new_crtc_state;
5346 struct intel_plane *plane;
5347 struct intel_crtc *crtc;
5348 int i;
5349
5350 if ((drm_debug & DRM_UT_KMS) == 0)
5351 return;
5352
5353 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5354 new_crtc_state, i) {
5355 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5356
5357 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5358 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5359
5360 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5361 enum plane_id plane_id = plane->id;
5362 const struct skl_ddb_entry *old, *new;
5363
5364 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5365 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5366
5367 if (skl_ddb_entry_equal(old, new))
5368 continue;
5369
5370 DRM_DEBUG_KMS("[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5371 plane->base.base.id, plane->base.name,
5372 old->start, old->end, new->start, new->end,
5373 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5374 }
5375
5376 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5377 enum plane_id plane_id = plane->id;
5378 const struct skl_plane_wm *old_wm, *new_wm;
5379
5380 old_wm = &old_pipe_wm->planes[plane_id];
5381 new_wm = &new_pipe_wm->planes[plane_id];
5382
5383 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5384 continue;
5385
5386 DRM_DEBUG_KMS("[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm"
5387 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm\n",
5388 plane->base.base.id, plane->base.name,
5389 enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5390 enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5391 enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5392 enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5393 enast(old_wm->trans_wm.plane_en),
5394 enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5395 enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5396 enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5397 enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5398 enast(new_wm->trans_wm.plane_en));
5399
5400 DRM_DEBUG_KMS("[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5401 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5402 plane->base.base.id, plane->base.name,
5403 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5404 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5405 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5406 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5407 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5408 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5409 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5410 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5411 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5412
5413 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5414 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5415 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5416 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5417 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5418 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5419 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5420 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5421 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l);
5422
5423 DRM_DEBUG_KMS("[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5424 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5425 plane->base.base.id, plane->base.name,
5426 old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5427 old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5428 old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5429 old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5430 old_wm->trans_wm.plane_res_b,
5431 new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5432 new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5433 new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5434 new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5435 new_wm->trans_wm.plane_res_b);
5436
5437 DRM_DEBUG_KMS("[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5438 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5439 plane->base.base.id, plane->base.name,
5440 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5441 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5442 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5443 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5444 old_wm->trans_wm.min_ddb_alloc,
5445 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5446 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5447 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5448 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5449 new_wm->trans_wm.min_ddb_alloc);
5450 }
5451 }
5452 }
5453
5454 static int
5455 skl_ddb_add_affected_pipes(struct intel_atomic_state *state, bool *changed)
5456 {
5457 struct drm_device *dev = state->base.dev;
5458 const struct drm_i915_private *dev_priv = to_i915(dev);
5459 struct intel_crtc *crtc;
5460 struct intel_crtc_state *crtc_state;
5461 u32 realloc_pipes = pipes_modified(state);
5462 int ret, i;
5463
5464
5465
5466
5467
5468 if (dev_priv->wm.distrust_bios_wm)
5469 (*changed) = true;
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i)
5480 (*changed) = true;
5481
5482 if (!*changed)
5483 return 0;
5484
5485
5486
5487
5488
5489
5490
5491 if (dev_priv->wm.distrust_bios_wm) {
5492 ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
5493 state->base.acquire_ctx);
5494 if (ret)
5495 return ret;
5496
5497 state->active_pipe_changes = ~0;
5498
5499
5500
5501
5502
5503
5504
5505 if (!state->modeset)
5506 state->active_crtcs = dev_priv->active_crtcs;
5507 }
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522 if (state->active_pipe_changes || state->modeset) {
5523 realloc_pipes = ~0;
5524 state->wm_results.dirty_pipes = ~0;
5525 }
5526
5527
5528
5529
5530
5531 for_each_intel_crtc_mask(dev, crtc, realloc_pipes) {
5532 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5533 if (IS_ERR(crtc_state))
5534 return PTR_ERR(crtc_state);
5535 }
5536
5537 return 0;
5538 }
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
5563 struct intel_crtc *crtc)
5564 {
5565 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5566 const struct intel_crtc_state *old_crtc_state =
5567 intel_atomic_get_old_crtc_state(state, crtc);
5568 struct intel_crtc_state *new_crtc_state =
5569 intel_atomic_get_new_crtc_state(state, crtc);
5570 struct intel_plane *plane;
5571
5572 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5573 struct intel_plane_state *plane_state;
5574 enum plane_id plane_id = plane->id;
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->base) &&
5585 skl_plane_wm_equals(dev_priv,
5586 &old_crtc_state->wm.skl.optimal.planes[plane_id],
5587 &new_crtc_state->wm.skl.optimal.planes[plane_id]))
5588 continue;
5589
5590 plane_state = intel_atomic_get_plane_state(state, plane);
5591 if (IS_ERR(plane_state))
5592 return PTR_ERR(plane_state);
5593
5594 new_crtc_state->update_planes |= BIT(plane_id);
5595 }
5596
5597 return 0;
5598 }
5599
5600 static int
5601 skl_compute_wm(struct intel_atomic_state *state)
5602 {
5603 struct intel_crtc *crtc;
5604 struct intel_crtc_state *new_crtc_state;
5605 struct intel_crtc_state *old_crtc_state;
5606 struct skl_ddb_values *results = &state->wm_results;
5607 bool changed = false;
5608 int ret, i;
5609
5610
5611 results->dirty_pipes = 0;
5612
5613 ret = skl_ddb_add_affected_pipes(state, &changed);
5614 if (ret || !changed)
5615 return ret;
5616
5617
5618
5619
5620
5621
5622
5623 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5624 new_crtc_state, i) {
5625 ret = skl_build_pipe_wm(new_crtc_state);
5626 if (ret)
5627 return ret;
5628
5629 ret = skl_wm_add_affected_planes(state, crtc);
5630 if (ret)
5631 return ret;
5632
5633 if (!skl_pipe_wm_equals(crtc,
5634 &old_crtc_state->wm.skl.optimal,
5635 &new_crtc_state->wm.skl.optimal))
5636 results->dirty_pipes |= drm_crtc_mask(&crtc->base);
5637 }
5638
5639 ret = skl_compute_ddb(state);
5640 if (ret)
5641 return ret;
5642
5643 skl_print_wm_changes(state);
5644
5645 return 0;
5646 }
5647
5648 static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
5649 struct intel_crtc_state *crtc_state)
5650 {
5651 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5652 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5653 struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5654 enum pipe pipe = crtc->pipe;
5655
5656 if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
5657 return;
5658
5659 I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
5660 }
5661
5662 static void skl_initial_wm(struct intel_atomic_state *state,
5663 struct intel_crtc_state *crtc_state)
5664 {
5665 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
5666 struct drm_device *dev = intel_crtc->base.dev;
5667 struct drm_i915_private *dev_priv = to_i915(dev);
5668 struct skl_ddb_values *results = &state->wm_results;
5669
5670 if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
5671 return;
5672
5673 mutex_lock(&dev_priv->wm.wm_mutex);
5674
5675 if (crtc_state->base.active_changed)
5676 skl_atomic_update_crtc_wm(state, crtc_state);
5677
5678 mutex_unlock(&dev_priv->wm.wm_mutex);
5679 }
5680
5681 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
5682 struct intel_wm_config *config)
5683 {
5684 struct intel_crtc *crtc;
5685
5686
5687 for_each_intel_crtc(&dev_priv->drm, crtc) {
5688 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5689
5690 if (!wm->pipe_enabled)
5691 continue;
5692
5693 config->sprites_enabled |= wm->sprites_enabled;
5694 config->sprites_scaled |= wm->sprites_scaled;
5695 config->num_pipes_active++;
5696 }
5697 }
5698
5699 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5700 {
5701 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5702 struct ilk_wm_maximums max;
5703 struct intel_wm_config config = {};
5704 struct ilk_wm_values results = {};
5705 enum intel_ddb_partitioning partitioning;
5706
5707 ilk_compute_wm_config(dev_priv, &config);
5708
5709 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
5710 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
5711
5712
5713 if (INTEL_GEN(dev_priv) >= 7 &&
5714 config.num_pipes_active == 1 && config.sprites_enabled) {
5715 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
5716 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
5717
5718 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
5719 } else {
5720 best_lp_wm = &lp_wm_1_2;
5721 }
5722
5723 partitioning = (best_lp_wm == &lp_wm_1_2) ?
5724 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5725
5726 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
5727
5728 ilk_write_wm_values(dev_priv, &results);
5729 }
5730
5731 static void ilk_initial_watermarks(struct intel_atomic_state *state,
5732 struct intel_crtc_state *crtc_state)
5733 {
5734 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
5735 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5736
5737 mutex_lock(&dev_priv->wm.wm_mutex);
5738 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
5739 ilk_program_watermarks(dev_priv);
5740 mutex_unlock(&dev_priv->wm.wm_mutex);
5741 }
5742
5743 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5744 struct intel_crtc_state *crtc_state)
5745 {
5746 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
5747 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5748
5749 if (!crtc_state->wm.need_postvbl_update)
5750 return;
5751
5752 mutex_lock(&dev_priv->wm.wm_mutex);
5753 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
5754 ilk_program_watermarks(dev_priv);
5755 mutex_unlock(&dev_priv->wm.wm_mutex);
5756 }
5757
5758 static inline void skl_wm_level_from_reg_val(u32 val,
5759 struct skl_wm_level *level)
5760 {
5761 level->plane_en = val & PLANE_WM_EN;
5762 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
5763 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5764 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5765 PLANE_WM_LINES_MASK;
5766 }
5767
5768 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
5769 struct skl_pipe_wm *out)
5770 {
5771 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5772 enum pipe pipe = crtc->pipe;
5773 int level, max_level;
5774 enum plane_id plane_id;
5775 u32 val;
5776
5777 max_level = ilk_wm_max_level(dev_priv);
5778
5779 for_each_plane_id_on_crtc(crtc, plane_id) {
5780 struct skl_plane_wm *wm = &out->planes[plane_id];
5781
5782 for (level = 0; level <= max_level; level++) {
5783 if (plane_id != PLANE_CURSOR)
5784 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5785 else
5786 val = I915_READ(CUR_WM(pipe, level));
5787
5788 skl_wm_level_from_reg_val(val, &wm->wm[level]);
5789 }
5790
5791 if (plane_id != PLANE_CURSOR)
5792 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5793 else
5794 val = I915_READ(CUR_WM_TRANS(pipe));
5795
5796 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5797 }
5798
5799 if (!crtc->active)
5800 return;
5801
5802 out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
5803 }
5804
5805 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
5806 {
5807 struct skl_ddb_values *hw = &dev_priv->wm.skl_hw;
5808 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
5809 struct intel_crtc *crtc;
5810 struct intel_crtc_state *crtc_state;
5811
5812 skl_ddb_get_hw_state(dev_priv, ddb);
5813 for_each_intel_crtc(&dev_priv->drm, crtc) {
5814 crtc_state = to_intel_crtc_state(crtc->base.state);
5815
5816 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
5817
5818 if (crtc->active)
5819 hw->dirty_pipes |= drm_crtc_mask(&crtc->base);
5820 }
5821
5822 if (dev_priv->active_crtcs) {
5823
5824 dev_priv->wm.distrust_bios_wm = true;
5825 }
5826 }
5827
5828 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
5829 {
5830 struct drm_device *dev = crtc->base.dev;
5831 struct drm_i915_private *dev_priv = to_i915(dev);
5832 struct ilk_wm_values *hw = &dev_priv->wm.hw;
5833 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
5834 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
5835 enum pipe pipe = crtc->pipe;
5836 static const i915_reg_t wm0_pipe_reg[] = {
5837 [PIPE_A] = WM0_PIPEA_ILK,
5838 [PIPE_B] = WM0_PIPEB_ILK,
5839 [PIPE_C] = WM0_PIPEC_IVB,
5840 };
5841
5842 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5843 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5844 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5845
5846 memset(active, 0, sizeof(*active));
5847
5848 active->pipe_enabled = crtc->active;
5849
5850 if (active->pipe_enabled) {
5851 u32 tmp = hw->wm_pipe[pipe];
5852
5853
5854
5855
5856
5857
5858
5859 active->wm[0].enable = true;
5860 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5861 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5862 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5863 active->linetime = hw->wm_linetime[pipe];
5864 } else {
5865 int level, max_level = ilk_wm_max_level(dev_priv);
5866
5867
5868
5869
5870
5871
5872 for (level = 0; level <= max_level; level++)
5873 active->wm[level].enable = true;
5874 }
5875
5876 crtc->wm.active.ilk = *active;
5877 }
5878
5879 #define _FW_WM(value, plane) \
5880 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5881 #define _FW_WM_VLV(value, plane) \
5882 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5883
5884 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5885 struct g4x_wm_values *wm)
5886 {
5887 u32 tmp;
5888
5889 tmp = I915_READ(DSPFW1);
5890 wm->sr.plane = _FW_WM(tmp, SR);
5891 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5892 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5893 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5894
5895 tmp = I915_READ(DSPFW2);
5896 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5897 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5898 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5899 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5900 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5901 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5902
5903 tmp = I915_READ(DSPFW3);
5904 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
5905 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5906 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
5907 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
5908 }
5909
5910 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
5911 struct vlv_wm_values *wm)
5912 {
5913 enum pipe pipe;
5914 u32 tmp;
5915
5916 for_each_pipe(dev_priv, pipe) {
5917 tmp = I915_READ(VLV_DDL(pipe));
5918
5919 wm->ddl[pipe].plane[PLANE_PRIMARY] =
5920 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5921 wm->ddl[pipe].plane[PLANE_CURSOR] =
5922 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5923 wm->ddl[pipe].plane[PLANE_SPRITE0] =
5924 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5925 wm->ddl[pipe].plane[PLANE_SPRITE1] =
5926 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5927 }
5928
5929 tmp = I915_READ(DSPFW1);
5930 wm->sr.plane = _FW_WM(tmp, SR);
5931 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5932 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
5933 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
5934
5935 tmp = I915_READ(DSPFW2);
5936 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
5937 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5938 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
5939
5940 tmp = I915_READ(DSPFW3);
5941 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5942
5943 if (IS_CHERRYVIEW(dev_priv)) {
5944 tmp = I915_READ(DSPFW7_CHV);
5945 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5946 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5947
5948 tmp = I915_READ(DSPFW8_CHV);
5949 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
5950 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
5951
5952 tmp = I915_READ(DSPFW9_CHV);
5953 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
5954 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
5955
5956 tmp = I915_READ(DSPHOWM);
5957 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5958 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
5959 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
5960 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
5961 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5962 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5963 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5964 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5965 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5966 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5967 } else {
5968 tmp = I915_READ(DSPFW7);
5969 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5970 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5971
5972 tmp = I915_READ(DSPHOWM);
5973 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5974 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5975 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5976 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5977 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5978 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5979 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5980 }
5981 }
5982
5983 #undef _FW_WM
5984 #undef _FW_WM_VLV
5985
5986 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
5987 {
5988 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
5989 struct intel_crtc *crtc;
5990
5991 g4x_read_wm_values(dev_priv, wm);
5992
5993 wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
5994
5995 for_each_intel_crtc(&dev_priv->drm, crtc) {
5996 struct intel_crtc_state *crtc_state =
5997 to_intel_crtc_state(crtc->base.state);
5998 struct g4x_wm_state *active = &crtc->wm.active.g4x;
5999 struct g4x_pipe_wm *raw;
6000 enum pipe pipe = crtc->pipe;
6001 enum plane_id plane_id;
6002 int level, max_level;
6003
6004 active->cxsr = wm->cxsr;
6005 active->hpll_en = wm->hpll_en;
6006 active->fbc_en = wm->fbc_en;
6007
6008 active->sr = wm->sr;
6009 active->hpll = wm->hpll;
6010
6011 for_each_plane_id_on_crtc(crtc, plane_id) {
6012 active->wm.plane[plane_id] =
6013 wm->pipe[pipe].plane[plane_id];
6014 }
6015
6016 if (wm->cxsr && wm->hpll_en)
6017 max_level = G4X_WM_LEVEL_HPLL;
6018 else if (wm->cxsr)
6019 max_level = G4X_WM_LEVEL_SR;
6020 else
6021 max_level = G4X_WM_LEVEL_NORMAL;
6022
6023 level = G4X_WM_LEVEL_NORMAL;
6024 raw = &crtc_state->wm.g4x.raw[level];
6025 for_each_plane_id_on_crtc(crtc, plane_id)
6026 raw->plane[plane_id] = active->wm.plane[plane_id];
6027
6028 if (++level > max_level)
6029 goto out;
6030
6031 raw = &crtc_state->wm.g4x.raw[level];
6032 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6033 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6034 raw->plane[PLANE_SPRITE0] = 0;
6035 raw->fbc = active->sr.fbc;
6036
6037 if (++level > max_level)
6038 goto out;
6039
6040 raw = &crtc_state->wm.g4x.raw[level];
6041 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6042 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6043 raw->plane[PLANE_SPRITE0] = 0;
6044 raw->fbc = active->hpll.fbc;
6045
6046 out:
6047 for_each_plane_id_on_crtc(crtc, plane_id)
6048 g4x_raw_plane_wm_set(crtc_state, level,
6049 plane_id, USHRT_MAX);
6050 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6051
6052 crtc_state->wm.g4x.optimal = *active;
6053 crtc_state->wm.g4x.intermediate = *active;
6054
6055 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6056 pipe_name(pipe),
6057 wm->pipe[pipe].plane[PLANE_PRIMARY],
6058 wm->pipe[pipe].plane[PLANE_CURSOR],
6059 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6060 }
6061
6062 DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6063 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6064 DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6065 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6066 DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
6067 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6068 }
6069
6070 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6071 {
6072 struct intel_plane *plane;
6073 struct intel_crtc *crtc;
6074
6075 mutex_lock(&dev_priv->wm.wm_mutex);
6076
6077 for_each_intel_plane(&dev_priv->drm, plane) {
6078 struct intel_crtc *crtc =
6079 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6080 struct intel_crtc_state *crtc_state =
6081 to_intel_crtc_state(crtc->base.state);
6082 struct intel_plane_state *plane_state =
6083 to_intel_plane_state(plane->base.state);
6084 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6085 enum plane_id plane_id = plane->id;
6086 int level;
6087
6088 if (plane_state->base.visible)
6089 continue;
6090
6091 for (level = 0; level < 3; level++) {
6092 struct g4x_pipe_wm *raw =
6093 &crtc_state->wm.g4x.raw[level];
6094
6095 raw->plane[plane_id] = 0;
6096 wm_state->wm.plane[plane_id] = 0;
6097 }
6098
6099 if (plane_id == PLANE_PRIMARY) {
6100 for (level = 0; level < 3; level++) {
6101 struct g4x_pipe_wm *raw =
6102 &crtc_state->wm.g4x.raw[level];
6103 raw->fbc = 0;
6104 }
6105
6106 wm_state->sr.fbc = 0;
6107 wm_state->hpll.fbc = 0;
6108 wm_state->fbc_en = false;
6109 }
6110 }
6111
6112 for_each_intel_crtc(&dev_priv->drm, crtc) {
6113 struct intel_crtc_state *crtc_state =
6114 to_intel_crtc_state(crtc->base.state);
6115
6116 crtc_state->wm.g4x.intermediate =
6117 crtc_state->wm.g4x.optimal;
6118 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6119 }
6120
6121 g4x_program_watermarks(dev_priv);
6122
6123 mutex_unlock(&dev_priv->wm.wm_mutex);
6124 }
6125
6126 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6127 {
6128 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6129 struct intel_crtc *crtc;
6130 u32 val;
6131
6132 vlv_read_wm_values(dev_priv, wm);
6133
6134 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6135 wm->level = VLV_WM_LEVEL_PM2;
6136
6137 if (IS_CHERRYVIEW(dev_priv)) {
6138 vlv_punit_get(dev_priv);
6139
6140 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6141 if (val & DSP_MAXFIFO_PM5_ENABLE)
6142 wm->level = VLV_WM_LEVEL_PM5;
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6154 val |= FORCE_DDR_FREQ_REQ_ACK;
6155 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6156
6157 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6158 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6159 DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
6160 "assuming DDR DVFS is disabled\n");
6161 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6162 } else {
6163 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6164 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6165 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6166 }
6167
6168 vlv_punit_put(dev_priv);
6169 }
6170
6171 for_each_intel_crtc(&dev_priv->drm, crtc) {
6172 struct intel_crtc_state *crtc_state =
6173 to_intel_crtc_state(crtc->base.state);
6174 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6175 const struct vlv_fifo_state *fifo_state =
6176 &crtc_state->wm.vlv.fifo_state;
6177 enum pipe pipe = crtc->pipe;
6178 enum plane_id plane_id;
6179 int level;
6180
6181 vlv_get_fifo_size(crtc_state);
6182
6183 active->num_levels = wm->level + 1;
6184 active->cxsr = wm->cxsr;
6185
6186 for (level = 0; level < active->num_levels; level++) {
6187 struct g4x_pipe_wm *raw =
6188 &crtc_state->wm.vlv.raw[level];
6189
6190 active->sr[level].plane = wm->sr.plane;
6191 active->sr[level].cursor = wm->sr.cursor;
6192
6193 for_each_plane_id_on_crtc(crtc, plane_id) {
6194 active->wm[level].plane[plane_id] =
6195 wm->pipe[pipe].plane[plane_id];
6196
6197 raw->plane[plane_id] =
6198 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6199 fifo_state->plane[plane_id]);
6200 }
6201 }
6202
6203 for_each_plane_id_on_crtc(crtc, plane_id)
6204 vlv_raw_plane_wm_set(crtc_state, level,
6205 plane_id, USHRT_MAX);
6206 vlv_invalidate_wms(crtc, active, level);
6207
6208 crtc_state->wm.vlv.optimal = *active;
6209 crtc_state->wm.vlv.intermediate = *active;
6210
6211 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6212 pipe_name(pipe),
6213 wm->pipe[pipe].plane[PLANE_PRIMARY],
6214 wm->pipe[pipe].plane[PLANE_CURSOR],
6215 wm->pipe[pipe].plane[PLANE_SPRITE0],
6216 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6217 }
6218
6219 DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6220 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6221 }
6222
6223 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6224 {
6225 struct intel_plane *plane;
6226 struct intel_crtc *crtc;
6227
6228 mutex_lock(&dev_priv->wm.wm_mutex);
6229
6230 for_each_intel_plane(&dev_priv->drm, plane) {
6231 struct intel_crtc *crtc =
6232 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6233 struct intel_crtc_state *crtc_state =
6234 to_intel_crtc_state(crtc->base.state);
6235 struct intel_plane_state *plane_state =
6236 to_intel_plane_state(plane->base.state);
6237 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6238 const struct vlv_fifo_state *fifo_state =
6239 &crtc_state->wm.vlv.fifo_state;
6240 enum plane_id plane_id = plane->id;
6241 int level;
6242
6243 if (plane_state->base.visible)
6244 continue;
6245
6246 for (level = 0; level < wm_state->num_levels; level++) {
6247 struct g4x_pipe_wm *raw =
6248 &crtc_state->wm.vlv.raw[level];
6249
6250 raw->plane[plane_id] = 0;
6251
6252 wm_state->wm[level].plane[plane_id] =
6253 vlv_invert_wm_value(raw->plane[plane_id],
6254 fifo_state->plane[plane_id]);
6255 }
6256 }
6257
6258 for_each_intel_crtc(&dev_priv->drm, crtc) {
6259 struct intel_crtc_state *crtc_state =
6260 to_intel_crtc_state(crtc->base.state);
6261
6262 crtc_state->wm.vlv.intermediate =
6263 crtc_state->wm.vlv.optimal;
6264 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6265 }
6266
6267 vlv_program_watermarks(dev_priv);
6268
6269 mutex_unlock(&dev_priv->wm.wm_mutex);
6270 }
6271
6272
6273
6274
6275
6276 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6277 {
6278 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6279 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6280 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6281
6282
6283
6284
6285
6286 }
6287
6288 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6289 {
6290 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6291 struct intel_crtc *crtc;
6292
6293 ilk_init_lp_watermarks(dev_priv);
6294
6295 for_each_intel_crtc(&dev_priv->drm, crtc)
6296 ilk_pipe_wm_get_hw_state(crtc);
6297
6298 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6299 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6300 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6301
6302 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6303 if (INTEL_GEN(dev_priv) >= 7) {
6304 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6305 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6306 }
6307
6308 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6309 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6310 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6311 else if (IS_IVYBRIDGE(dev_priv))
6312 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6313 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6314
6315 hw->enable_fbc_wm =
6316 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6317 }
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352 void intel_update_watermarks(struct intel_crtc *crtc)
6353 {
6354 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6355
6356 if (dev_priv->display.update_wm)
6357 dev_priv->display.update_wm(crtc);
6358 }
6359
6360 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6361 {
6362 u32 val;
6363
6364 if (!HAS_IPC(dev_priv))
6365 return;
6366
6367 val = I915_READ(DISP_ARB_CTL2);
6368
6369 if (dev_priv->ipc_enabled)
6370 val |= DISP_IPC_ENABLE;
6371 else
6372 val &= ~DISP_IPC_ENABLE;
6373
6374 I915_WRITE(DISP_ARB_CTL2, val);
6375 }
6376
6377 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6378 {
6379
6380 if (IS_SKYLAKE(dev_priv))
6381 return false;
6382
6383
6384 if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
6385 return dev_priv->dram_info.symmetric_memory;
6386
6387 return true;
6388 }
6389
6390 void intel_init_ipc(struct drm_i915_private *dev_priv)
6391 {
6392 if (!HAS_IPC(dev_priv))
6393 return;
6394
6395 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6396
6397 intel_enable_ipc(dev_priv);
6398 }
6399
6400
6401
6402
6403 DEFINE_SPINLOCK(mchdev_lock);
6404
6405 bool ironlake_set_drps(struct drm_i915_private *i915, u8 val)
6406 {
6407 struct intel_uncore *uncore = &i915->uncore;
6408 u16 rgvswctl;
6409
6410 lockdep_assert_held(&mchdev_lock);
6411
6412 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
6413 if (rgvswctl & MEMCTL_CMD_STS) {
6414 DRM_DEBUG("gpu busy, RCS change rejected\n");
6415 return false;
6416 }
6417
6418 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
6419 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
6420 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
6421 intel_uncore_posting_read16(uncore, MEMSWCTL);
6422
6423 rgvswctl |= MEMCTL_CMD_STS;
6424 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
6425
6426 return true;
6427 }
6428
6429 static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
6430 {
6431 struct intel_uncore *uncore = &dev_priv->uncore;
6432 u32 rgvmodectl;
6433 u8 fmax, fmin, fstart, vstart;
6434
6435 spin_lock_irq(&mchdev_lock);
6436
6437 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
6438
6439
6440 intel_uncore_write16(uncore, PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
6441 intel_uncore_write16(uncore, TSC1, I915_READ(TSC1) | TSE);
6442
6443
6444 intel_uncore_write(uncore, RCUPEI, 100000);
6445 intel_uncore_write(uncore, RCDNEI, 100000);
6446
6447
6448 intel_uncore_write(uncore, RCBMAXAVG, 90000);
6449 intel_uncore_write(uncore, RCBMINAVG, 80000);
6450
6451 intel_uncore_write(uncore, MEMIHYST, 1);
6452
6453
6454 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
6455 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
6456 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
6457 MEMMODE_FSTART_SHIFT;
6458
6459 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
6460 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
6461
6462 dev_priv->ips.fmax = fmax;
6463 dev_priv->ips.fstart = fstart;
6464
6465 dev_priv->ips.max_delay = fstart;
6466 dev_priv->ips.min_delay = fmin;
6467 dev_priv->ips.cur_delay = fstart;
6468
6469 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
6470 fmax, fmin, fstart);
6471
6472 intel_uncore_write(uncore,
6473 MEMINTREN,
6474 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
6475
6476
6477
6478
6479
6480 intel_uncore_write(uncore, VIDSTART, vstart);
6481 intel_uncore_posting_read(uncore, VIDSTART);
6482
6483 rgvmodectl |= MEMMODE_SWMODE_EN;
6484 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
6485
6486 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
6487 MEMCTL_CMD_STS) == 0, 10))
6488 DRM_ERROR("stuck trying to change perf mode\n");
6489 mdelay(1);
6490
6491 ironlake_set_drps(dev_priv, fstart);
6492
6493 dev_priv->ips.last_count1 =
6494 intel_uncore_read(uncore, DMIEC) +
6495 intel_uncore_read(uncore, DDREC) +
6496 intel_uncore_read(uncore, CSIEC);
6497 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6498 dev_priv->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
6499 dev_priv->ips.last_time2 = ktime_get_raw_ns();
6500
6501 spin_unlock_irq(&mchdev_lock);
6502 }
6503
6504 static void ironlake_disable_drps(struct drm_i915_private *i915)
6505 {
6506 struct intel_uncore *uncore = &i915->uncore;
6507 u16 rgvswctl;
6508
6509 spin_lock_irq(&mchdev_lock);
6510
6511 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
6512
6513
6514 intel_uncore_write(uncore,
6515 MEMINTREN,
6516 intel_uncore_read(uncore, MEMINTREN) &
6517 ~MEMINT_EVAL_CHG_EN);
6518 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
6519 intel_uncore_write(uncore,
6520 DEIER,
6521 intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
6522 intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
6523 intel_uncore_write(uncore,
6524 DEIMR,
6525 intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
6526
6527
6528 ironlake_set_drps(i915, i915->ips.fstart);
6529 mdelay(1);
6530 rgvswctl |= MEMCTL_CMD_STS;
6531 intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
6532 mdelay(1);
6533
6534 spin_unlock_irq(&mchdev_lock);
6535 }
6536
6537
6538
6539
6540
6541
6542 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
6543 {
6544 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6545 u32 limits;
6546
6547
6548
6549
6550
6551
6552
6553 if (INTEL_GEN(dev_priv) >= 9) {
6554 limits = (rps->max_freq_softlimit) << 23;
6555 if (val <= rps->min_freq_softlimit)
6556 limits |= (rps->min_freq_softlimit) << 14;
6557 } else {
6558 limits = rps->max_freq_softlimit << 24;
6559 if (val <= rps->min_freq_softlimit)
6560 limits |= rps->min_freq_softlimit << 16;
6561 }
6562
6563 return limits;
6564 }
6565
6566 static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
6567 {
6568 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6569 u32 threshold_up = 0, threshold_down = 0;
6570 u32 ei_up = 0, ei_down = 0;
6571
6572 lockdep_assert_held(&rps->power.mutex);
6573
6574 if (new_power == rps->power.mode)
6575 return;
6576
6577
6578 switch (new_power) {
6579 case LOW_POWER:
6580
6581 ei_up = 16000;
6582 threshold_up = 95;
6583
6584
6585 ei_down = 32000;
6586 threshold_down = 85;
6587 break;
6588
6589 case BETWEEN:
6590
6591 ei_up = 13000;
6592 threshold_up = 90;
6593
6594
6595 ei_down = 32000;
6596 threshold_down = 75;
6597 break;
6598
6599 case HIGH_POWER:
6600
6601 ei_up = 10000;
6602 threshold_up = 85;
6603
6604
6605 ei_down = 32000;
6606 threshold_down = 60;
6607 break;
6608 }
6609
6610
6611
6612
6613 if (IS_VALLEYVIEW(dev_priv))
6614 goto skip_hw_write;
6615
6616 I915_WRITE(GEN6_RP_UP_EI,
6617 GT_INTERVAL_FROM_US(dev_priv, ei_up));
6618 I915_WRITE(GEN6_RP_UP_THRESHOLD,
6619 GT_INTERVAL_FROM_US(dev_priv,
6620 ei_up * threshold_up / 100));
6621
6622 I915_WRITE(GEN6_RP_DOWN_EI,
6623 GT_INTERVAL_FROM_US(dev_priv, ei_down));
6624 I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
6625 GT_INTERVAL_FROM_US(dev_priv,
6626 ei_down * threshold_down / 100));
6627
6628 I915_WRITE(GEN6_RP_CONTROL,
6629 (INTEL_GEN(dev_priv) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
6630 GEN6_RP_MEDIA_HW_NORMAL_MODE |
6631 GEN6_RP_MEDIA_IS_GFX |
6632 GEN6_RP_ENABLE |
6633 GEN6_RP_UP_BUSY_AVG |
6634 GEN6_RP_DOWN_IDLE_AVG);
6635
6636 skip_hw_write:
6637 rps->power.mode = new_power;
6638 rps->power.up_threshold = threshold_up;
6639 rps->power.down_threshold = threshold_down;
6640 }
6641
6642 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
6643 {
6644 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6645 int new_power;
6646
6647 new_power = rps->power.mode;
6648 switch (rps->power.mode) {
6649 case LOW_POWER:
6650 if (val > rps->efficient_freq + 1 &&
6651 val > rps->cur_freq)
6652 new_power = BETWEEN;
6653 break;
6654
6655 case BETWEEN:
6656 if (val <= rps->efficient_freq &&
6657 val < rps->cur_freq)
6658 new_power = LOW_POWER;
6659 else if (val >= rps->rp0_freq &&
6660 val > rps->cur_freq)
6661 new_power = HIGH_POWER;
6662 break;
6663
6664 case HIGH_POWER:
6665 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
6666 val < rps->cur_freq)
6667 new_power = BETWEEN;
6668 break;
6669 }
6670
6671 if (val <= rps->min_freq_softlimit)
6672 new_power = LOW_POWER;
6673 if (val >= rps->max_freq_softlimit)
6674 new_power = HIGH_POWER;
6675
6676 mutex_lock(&rps->power.mutex);
6677 if (rps->power.interactive)
6678 new_power = HIGH_POWER;
6679 rps_set_power(dev_priv, new_power);
6680 mutex_unlock(&rps->power.mutex);
6681 }
6682
6683 void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
6684 {
6685 struct intel_rps *rps = &i915->gt_pm.rps;
6686
6687 if (INTEL_GEN(i915) < 6)
6688 return;
6689
6690 mutex_lock(&rps->power.mutex);
6691 if (interactive) {
6692 if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
6693 rps_set_power(i915, HIGH_POWER);
6694 } else {
6695 GEM_BUG_ON(!rps->power.interactive);
6696 rps->power.interactive--;
6697 }
6698 mutex_unlock(&rps->power.mutex);
6699 }
6700
6701 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
6702 {
6703 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6704 u32 mask = 0;
6705
6706
6707 if (val > rps->min_freq_softlimit)
6708 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
6709 if (val < rps->max_freq_softlimit)
6710 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
6711
6712 mask &= dev_priv->pm_rps_events;
6713
6714 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
6715 }
6716
6717
6718
6719
6720 static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
6721 {
6722 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6723
6724
6725
6726
6727 if (val != rps->cur_freq) {
6728 gen6_set_rps_thresholds(dev_priv, val);
6729
6730 if (INTEL_GEN(dev_priv) >= 9)
6731 I915_WRITE(GEN6_RPNSWREQ,
6732 GEN9_FREQUENCY(val));
6733 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6734 I915_WRITE(GEN6_RPNSWREQ,
6735 HSW_FREQUENCY(val));
6736 else
6737 I915_WRITE(GEN6_RPNSWREQ,
6738 GEN6_FREQUENCY(val) |
6739 GEN6_OFFSET(0) |
6740 GEN6_AGGRESSIVE_TURBO);
6741 }
6742
6743
6744
6745
6746 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
6747 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6748
6749 rps->cur_freq = val;
6750 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6751
6752 return 0;
6753 }
6754
6755 static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
6756 {
6757 int err;
6758
6759 if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
6760 "Odd GPU freq value\n"))
6761 val &= ~1;
6762
6763 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6764
6765 if (val != dev_priv->gt_pm.rps.cur_freq) {
6766 vlv_punit_get(dev_priv);
6767 err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
6768 vlv_punit_put(dev_priv);
6769 if (err)
6770 return err;
6771
6772 gen6_set_rps_thresholds(dev_priv, val);
6773 }
6774
6775 dev_priv->gt_pm.rps.cur_freq = val;
6776 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6777
6778 return 0;
6779 }
6780
6781
6782
6783
6784
6785
6786
6787
6788 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
6789 {
6790 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6791 u32 val = rps->idle_freq;
6792 int err;
6793
6794 if (rps->cur_freq <= val)
6795 return;
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_MEDIA);
6810 err = valleyview_set_rps(dev_priv, val);
6811 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_MEDIA);
6812
6813 if (err)
6814 DRM_ERROR("Failed to set RPS for idle\n");
6815 }
6816
6817 void gen6_rps_busy(struct drm_i915_private *dev_priv)
6818 {
6819 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6820
6821 mutex_lock(&rps->lock);
6822 if (rps->enabled) {
6823 u8 freq;
6824
6825 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
6826 gen6_rps_reset_ei(dev_priv);
6827 I915_WRITE(GEN6_PMINTRMSK,
6828 gen6_rps_pm_mask(dev_priv, rps->cur_freq));
6829
6830 gen6_enable_rps_interrupts(dev_priv);
6831
6832
6833
6834
6835 freq = max(rps->cur_freq,
6836 rps->efficient_freq);
6837
6838 if (intel_set_rps(dev_priv,
6839 clamp(freq,
6840 rps->min_freq_softlimit,
6841 rps->max_freq_softlimit)))
6842 DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
6843 }
6844 mutex_unlock(&rps->lock);
6845 }
6846
6847 void gen6_rps_idle(struct drm_i915_private *dev_priv)
6848 {
6849 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6850
6851
6852
6853
6854
6855
6856 gen6_disable_rps_interrupts(dev_priv);
6857
6858 mutex_lock(&rps->lock);
6859 if (rps->enabled) {
6860 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6861 vlv_set_rps_idle(dev_priv);
6862 else
6863 gen6_set_rps(dev_priv, rps->idle_freq);
6864 rps->last_adj = 0;
6865 I915_WRITE(GEN6_PMINTRMSK,
6866 gen6_sanitize_rps_pm_mask(dev_priv, ~0));
6867 }
6868 mutex_unlock(&rps->lock);
6869 }
6870
6871 void gen6_rps_boost(struct i915_request *rq)
6872 {
6873 struct intel_rps *rps = &rq->i915->gt_pm.rps;
6874 unsigned long flags;
6875 bool boost;
6876
6877
6878
6879
6880 if (!rps->enabled)
6881 return;
6882
6883 if (i915_request_signaled(rq))
6884 return;
6885
6886
6887 boost = false;
6888 spin_lock_irqsave(&rq->lock, flags);
6889 if (!i915_request_has_waitboost(rq) &&
6890 !dma_fence_is_signaled_locked(&rq->fence)) {
6891 boost = !atomic_fetch_inc(&rps->num_waiters);
6892 rq->flags |= I915_REQUEST_WAITBOOST;
6893 }
6894 spin_unlock_irqrestore(&rq->lock, flags);
6895 if (!boost)
6896 return;
6897
6898 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
6899 schedule_work(&rps->work);
6900
6901 atomic_inc(&rps->boosts);
6902 }
6903
6904 int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
6905 {
6906 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6907 int err;
6908
6909 lockdep_assert_held(&rps->lock);
6910 GEM_BUG_ON(val > rps->max_freq);
6911 GEM_BUG_ON(val < rps->min_freq);
6912
6913 if (!rps->enabled) {
6914 rps->cur_freq = val;
6915 return 0;
6916 }
6917
6918 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6919 err = valleyview_set_rps(dev_priv, val);
6920 else
6921 err = gen6_set_rps(dev_priv, val);
6922
6923 return err;
6924 }
6925
6926 static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
6927 {
6928 I915_WRITE(GEN6_RC_CONTROL, 0);
6929 I915_WRITE(GEN9_PG_ENABLE, 0);
6930 }
6931
6932 static void gen9_disable_rps(struct drm_i915_private *dev_priv)
6933 {
6934 I915_WRITE(GEN6_RP_CONTROL, 0);
6935 }
6936
6937 static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
6938 {
6939 I915_WRITE(GEN6_RC_CONTROL, 0);
6940 }
6941
6942 static void gen6_disable_rps(struct drm_i915_private *dev_priv)
6943 {
6944 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
6945 I915_WRITE(GEN6_RP_CONTROL, 0);
6946 }
6947
6948 static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
6949 {
6950 I915_WRITE(GEN6_RC_CONTROL, 0);
6951 }
6952
6953 static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
6954 {
6955 I915_WRITE(GEN6_RP_CONTROL, 0);
6956 }
6957
6958 static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
6959 {
6960
6961
6962 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
6963
6964 I915_WRITE(GEN6_RC_CONTROL, 0);
6965
6966 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
6967 }
6968
6969 static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
6970 {
6971 I915_WRITE(GEN6_RP_CONTROL, 0);
6972 }
6973
6974 static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
6975 {
6976 bool enable_rc6 = true;
6977 unsigned long rc6_ctx_base;
6978 u32 rc_ctl;
6979 int rc_sw_target;
6980
6981 rc_ctl = I915_READ(GEN6_RC_CONTROL);
6982 rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
6983 RC_SW_TARGET_STATE_SHIFT;
6984 DRM_DEBUG_DRIVER("BIOS enabled RC states: "
6985 "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
6986 onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
6987 onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
6988 rc_sw_target);
6989
6990 if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
6991 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
6992 enable_rc6 = false;
6993 }
6994
6995
6996
6997
6998
6999 rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
7000 if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
7001 (rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
7002 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
7003 enable_rc6 = false;
7004 }
7005
7006 if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
7007 ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
7008 ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
7009 ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
7010 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
7011 enable_rc6 = false;
7012 }
7013
7014 if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
7015 !I915_READ(GEN8_PUSHBUS_ENABLE) ||
7016 !I915_READ(GEN8_PUSHBUS_SHIFT)) {
7017 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
7018 enable_rc6 = false;
7019 }
7020
7021 if (!I915_READ(GEN6_GFXPAUSE)) {
7022 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
7023 enable_rc6 = false;
7024 }
7025
7026 if (!I915_READ(GEN8_MISC_CTRL0)) {
7027 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
7028 enable_rc6 = false;
7029 }
7030
7031 return enable_rc6;
7032 }
7033
7034 static bool sanitize_rc6(struct drm_i915_private *i915)
7035 {
7036 struct intel_device_info *info = mkwrite_device_info(i915);
7037
7038
7039 if (intel_vgpu_active(i915)) {
7040 info->has_rc6 = 0;
7041 info->has_rps = false;
7042 }
7043
7044 if (info->has_rc6 &&
7045 IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
7046 DRM_INFO("RC6 disabled by BIOS\n");
7047 info->has_rc6 = 0;
7048 }
7049
7050
7051
7052
7053
7054
7055
7056 if (!info->has_rc6 && info->has_rc6p)
7057 info->has_rc6p = 0;
7058
7059 return info->has_rc6;
7060 }
7061
7062 static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
7063 {
7064 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7065
7066
7067
7068
7069 if (IS_GEN9_LP(dev_priv)) {
7070 u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
7071 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
7072 rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
7073 rps->min_freq = (rp_state_cap >> 0) & 0xff;
7074 } else {
7075 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
7076 rps->rp0_freq = (rp_state_cap >> 0) & 0xff;
7077 rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
7078 rps->min_freq = (rp_state_cap >> 16) & 0xff;
7079 }
7080
7081 rps->max_freq = rps->rp0_freq;
7082
7083 rps->efficient_freq = rps->rp1_freq;
7084 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
7085 IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7086 u32 ddcc_status = 0;
7087
7088 if (sandybridge_pcode_read(dev_priv,
7089 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
7090 &ddcc_status, NULL) == 0)
7091 rps->efficient_freq =
7092 clamp_t(u8,
7093 ((ddcc_status >> 8) & 0xff),
7094 rps->min_freq,
7095 rps->max_freq);
7096 }
7097
7098 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7099
7100
7101
7102 rps->rp0_freq *= GEN9_FREQ_SCALER;
7103 rps->rp1_freq *= GEN9_FREQ_SCALER;
7104 rps->min_freq *= GEN9_FREQ_SCALER;
7105 rps->max_freq *= GEN9_FREQ_SCALER;
7106 rps->efficient_freq *= GEN9_FREQ_SCALER;
7107 }
7108 }
7109
7110 static void reset_rps(struct drm_i915_private *dev_priv,
7111 int (*set)(struct drm_i915_private *, u8))
7112 {
7113 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7114 u8 freq = rps->cur_freq;
7115
7116
7117 rps->power.mode = -1;
7118 rps->cur_freq = -1;
7119
7120 if (set(dev_priv, freq))
7121 DRM_ERROR("Failed to reset RPS to initial values\n");
7122 }
7123
7124
7125 static void gen9_enable_rps(struct drm_i915_private *dev_priv)
7126 {
7127 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7128
7129
7130 if (IS_GEN(dev_priv, 9))
7131 I915_WRITE(GEN6_RC_VIDEO_FREQ,
7132 GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
7133
7134
7135 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
7136 GT_INTERVAL_FROM_US(dev_priv, 1000000));
7137
7138 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
7139
7140
7141
7142
7143 reset_rps(dev_priv, gen6_set_rps);
7144
7145 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7146 }
7147
7148 static void gen11_enable_rc6(struct drm_i915_private *dev_priv)
7149 {
7150 struct intel_engine_cs *engine;
7151 enum intel_engine_id id;
7152
7153
7154 I915_WRITE(GEN6_RC_STATE, 0);
7155
7156
7157
7158
7159
7160 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7161
7162
7163 I915_WRITE(GEN6_RC_CONTROL, 0);
7164
7165
7166 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
7167 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
7168
7169 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7170 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7171 for_each_engine(engine, dev_priv, id)
7172 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7173
7174 if (HAS_GT_UC(dev_priv))
7175 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
7176
7177 I915_WRITE(GEN6_RC_SLEEP, 0);
7178
7179 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
7203 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
7204
7205
7206 I915_WRITE(GEN6_RC_CONTROL,
7207 GEN6_RC_CTL_HW_ENABLE |
7208 GEN6_RC_CTL_RC6_ENABLE |
7209 GEN6_RC_CTL_EI_MODE(1));
7210
7211
7212 I915_WRITE(GEN9_PG_ENABLE,
7213 GEN9_RENDER_PG_ENABLE |
7214 GEN9_MEDIA_PG_ENABLE |
7215 GEN11_MEDIA_SAMPLER_PG_ENABLE);
7216
7217 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7218 }
7219
7220 static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
7221 {
7222 struct intel_engine_cs *engine;
7223 enum intel_engine_id id;
7224 u32 rc6_mode;
7225
7226
7227 I915_WRITE(GEN6_RC_STATE, 0);
7228
7229
7230
7231 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7232
7233
7234 I915_WRITE(GEN6_RC_CONTROL, 0);
7235
7236
7237 if (INTEL_GEN(dev_priv) >= 10) {
7238 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
7239 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
7240 } else if (IS_SKYLAKE(dev_priv)) {
7241
7242
7243
7244
7245 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
7246 } else {
7247 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
7248 }
7249
7250 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7251 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7252 for_each_engine(engine, dev_priv, id)
7253 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7254
7255 if (HAS_GT_UC(dev_priv))
7256 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
7257
7258 I915_WRITE(GEN6_RC_SLEEP, 0);
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
7282 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
7283
7284
7285 I915_WRITE(GEN6_RC6_THRESHOLD, 37500);
7286
7287
7288 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_C0))
7289 rc6_mode = GEN7_RC_CTL_TO_MODE;
7290 else
7291 rc6_mode = GEN6_RC_CTL_EI_MODE(1);
7292
7293 I915_WRITE(GEN6_RC_CONTROL,
7294 GEN6_RC_CTL_HW_ENABLE |
7295 GEN6_RC_CTL_RC6_ENABLE |
7296 rc6_mode);
7297
7298
7299
7300
7301
7302 if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
7303 I915_WRITE(GEN9_PG_ENABLE, 0);
7304 else
7305 I915_WRITE(GEN9_PG_ENABLE,
7306 GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
7307
7308 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7309 }
7310
7311 static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
7312 {
7313 struct intel_engine_cs *engine;
7314 enum intel_engine_id id;
7315
7316
7317 I915_WRITE(GEN6_RC_STATE, 0);
7318
7319
7320
7321 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7322
7323
7324 I915_WRITE(GEN6_RC_CONTROL, 0);
7325
7326
7327 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7328 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7329 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7330 for_each_engine(engine, dev_priv, id)
7331 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7332 I915_WRITE(GEN6_RC_SLEEP, 0);
7333 I915_WRITE(GEN6_RC6_THRESHOLD, 625);
7334
7335
7336
7337 I915_WRITE(GEN6_RC_CONTROL,
7338 GEN6_RC_CTL_HW_ENABLE |
7339 GEN7_RC_CTL_TO_MODE |
7340 GEN6_RC_CTL_RC6_ENABLE);
7341
7342 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7343 }
7344
7345 static void gen8_enable_rps(struct drm_i915_private *dev_priv)
7346 {
7347 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7348
7349 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7350
7351
7352 I915_WRITE(GEN6_RPNSWREQ,
7353 HSW_FREQUENCY(rps->rp1_freq));
7354 I915_WRITE(GEN6_RC_VIDEO_FREQ,
7355 HSW_FREQUENCY(rps->rp1_freq));
7356
7357 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128);
7358
7359
7360 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
7361 rps->max_freq_softlimit << 24 |
7362 rps->min_freq_softlimit << 16);
7363
7364 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128);
7365 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128);
7366 I915_WRITE(GEN6_RP_UP_EI, 66000);
7367 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7368
7369 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7370
7371
7372 I915_WRITE(GEN6_RP_CONTROL,
7373 GEN6_RP_MEDIA_TURBO |
7374 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7375 GEN6_RP_MEDIA_IS_GFX |
7376 GEN6_RP_ENABLE |
7377 GEN6_RP_UP_BUSY_AVG |
7378 GEN6_RP_DOWN_IDLE_AVG);
7379
7380 reset_rps(dev_priv, gen6_set_rps);
7381
7382 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7383 }
7384
7385 static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
7386 {
7387 struct intel_engine_cs *engine;
7388 enum intel_engine_id id;
7389 u32 rc6vids, rc6_mask;
7390 u32 gtfifodbg;
7391 int ret;
7392
7393 I915_WRITE(GEN6_RC_STATE, 0);
7394
7395
7396 gtfifodbg = I915_READ(GTFIFODBG);
7397 if (gtfifodbg) {
7398 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
7399 I915_WRITE(GTFIFODBG, gtfifodbg);
7400 }
7401
7402 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7403
7404
7405 I915_WRITE(GEN6_RC_CONTROL, 0);
7406
7407 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
7408 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
7409 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
7410 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7411 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7412
7413 for_each_engine(engine, dev_priv, id)
7414 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7415
7416 I915_WRITE(GEN6_RC_SLEEP, 0);
7417 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
7418 if (IS_IVYBRIDGE(dev_priv))
7419 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
7420 else
7421 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7422 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
7423 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000);
7424
7425
7426 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
7427 if (HAS_RC6p(dev_priv))
7428 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
7429 if (HAS_RC6pp(dev_priv))
7430 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
7431 I915_WRITE(GEN6_RC_CONTROL,
7432 rc6_mask |
7433 GEN6_RC_CTL_EI_MODE(1) |
7434 GEN6_RC_CTL_HW_ENABLE);
7435
7436 rc6vids = 0;
7437 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS,
7438 &rc6vids, NULL);
7439 if (IS_GEN(dev_priv, 6) && ret) {
7440 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
7441 } else if (IS_GEN(dev_priv, 6) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
7442 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
7443 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
7444 rc6vids &= 0xffff00;
7445 rc6vids |= GEN6_ENCODE_RC6_VID(450);
7446 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
7447 if (ret)
7448 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
7449 }
7450
7451 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7452 }
7453
7454 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
7455 {
7456
7457
7458
7459
7460
7461
7462 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7463
7464
7465 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
7466 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7467
7468 reset_rps(dev_priv, gen6_set_rps);
7469
7470 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7471 }
7472
7473 static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
7474 {
7475 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7476 const int min_freq = 15;
7477 const int scaling_factor = 180;
7478 unsigned int gpu_freq;
7479 unsigned int max_ia_freq, min_ring_freq;
7480 unsigned int max_gpu_freq, min_gpu_freq;
7481 struct cpufreq_policy *policy;
7482
7483 lockdep_assert_held(&rps->lock);
7484
7485 if (rps->max_freq <= rps->min_freq)
7486 return;
7487
7488 policy = cpufreq_cpu_get(0);
7489 if (policy) {
7490 max_ia_freq = policy->cpuinfo.max_freq;
7491 cpufreq_cpu_put(policy);
7492 } else {
7493
7494
7495
7496
7497 max_ia_freq = tsc_khz;
7498 }
7499
7500
7501 max_ia_freq /= 1000;
7502
7503 min_ring_freq = I915_READ(DCLK) & 0xf;
7504
7505 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
7506
7507 min_gpu_freq = rps->min_freq;
7508 max_gpu_freq = rps->max_freq;
7509 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7510
7511 min_gpu_freq /= GEN9_FREQ_SCALER;
7512 max_gpu_freq /= GEN9_FREQ_SCALER;
7513 }
7514
7515
7516
7517
7518
7519
7520 for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
7521 const int diff = max_gpu_freq - gpu_freq;
7522 unsigned int ia_freq = 0, ring_freq = 0;
7523
7524 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7525
7526
7527
7528
7529 ring_freq = gpu_freq;
7530 } else if (INTEL_GEN(dev_priv) >= 8) {
7531
7532 ring_freq = max(min_ring_freq, gpu_freq);
7533 } else if (IS_HASWELL(dev_priv)) {
7534 ring_freq = mult_frac(gpu_freq, 5, 4);
7535 ring_freq = max(min_ring_freq, ring_freq);
7536
7537 } else {
7538
7539
7540
7541
7542
7543
7544
7545 if (gpu_freq < min_freq)
7546 ia_freq = 800;
7547 else
7548 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
7549 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
7550 }
7551
7552 sandybridge_pcode_write(dev_priv,
7553 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
7554 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
7555 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
7556 gpu_freq);
7557 }
7558 }
7559
7560 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
7561 {
7562 u32 val, rp0;
7563
7564 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7565
7566 switch (RUNTIME_INFO(dev_priv)->sseu.eu_total) {
7567 case 8:
7568
7569 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
7570 break;
7571 case 12:
7572
7573 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
7574 break;
7575 case 16:
7576
7577 default:
7578
7579 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
7580 break;
7581 }
7582
7583 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
7584
7585 return rp0;
7586 }
7587
7588 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7589 {
7590 u32 val, rpe;
7591
7592 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
7593 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
7594
7595 return rpe;
7596 }
7597
7598 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
7599 {
7600 u32 val, rp1;
7601
7602 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7603 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
7604
7605 return rp1;
7606 }
7607
7608 static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
7609 {
7610 u32 val, rpn;
7611
7612 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
7613 rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
7614 FB_GFX_FREQ_FUSE_MASK);
7615
7616 return rpn;
7617 }
7618
7619 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
7620 {
7621 u32 val, rp1;
7622
7623 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7624
7625 rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
7626
7627 return rp1;
7628 }
7629
7630 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
7631 {
7632 u32 val, rp0;
7633
7634 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7635
7636 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
7637
7638 rp0 = min_t(u32, rp0, 0xea);
7639
7640 return rp0;
7641 }
7642
7643 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7644 {
7645 u32 val, rpe;
7646
7647 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
7648 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
7649 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
7650 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
7651
7652 return rpe;
7653 }
7654
7655 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
7656 {
7657 u32 val;
7658
7659 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
7660
7661
7662
7663
7664
7665
7666
7667 return max_t(u32, val, 0xc0);
7668 }
7669
7670
7671 static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
7672 {
7673 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7674
7675 WARN_ON(pctx_addr != dev_priv->dsm.start +
7676 dev_priv->vlv_pctx->stolen->start);
7677 }
7678
7679
7680
7681 static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
7682 {
7683 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7684
7685 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
7686 }
7687
7688 static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
7689 {
7690 resource_size_t pctx_paddr, paddr;
7691 resource_size_t pctx_size = 32*1024;
7692 u32 pcbr;
7693
7694 pcbr = I915_READ(VLV_PCBR);
7695 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
7696 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7697 paddr = dev_priv->dsm.end + 1 - pctx_size;
7698 GEM_BUG_ON(paddr > U32_MAX);
7699
7700 pctx_paddr = (paddr & (~4095));
7701 I915_WRITE(VLV_PCBR, pctx_paddr);
7702 }
7703
7704 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7705 }
7706
7707 static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
7708 {
7709 struct drm_i915_gem_object *pctx;
7710 resource_size_t pctx_paddr;
7711 resource_size_t pctx_size = 24*1024;
7712 u32 pcbr;
7713
7714 pcbr = I915_READ(VLV_PCBR);
7715 if (pcbr) {
7716
7717 resource_size_t pcbr_offset;
7718
7719 pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
7720 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
7721 pcbr_offset,
7722 I915_GTT_OFFSET_NONE,
7723 pctx_size);
7724 goto out;
7725 }
7726
7727 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737 pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
7738 if (!pctx) {
7739 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
7740 goto out;
7741 }
7742
7743 GEM_BUG_ON(range_overflows_t(u64,
7744 dev_priv->dsm.start,
7745 pctx->stolen->start,
7746 U32_MAX));
7747 pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
7748 I915_WRITE(VLV_PCBR, pctx_paddr);
7749
7750 out:
7751 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7752 dev_priv->vlv_pctx = pctx;
7753 }
7754
7755 static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
7756 {
7757 struct drm_i915_gem_object *pctx;
7758
7759 pctx = fetch_and_zero(&dev_priv->vlv_pctx);
7760 if (pctx)
7761 i915_gem_object_put(pctx);
7762 }
7763
7764 static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
7765 {
7766 dev_priv->gt_pm.rps.gpll_ref_freq =
7767 vlv_get_cck_clock(dev_priv, "GPLL ref",
7768 CCK_GPLL_CLOCK_CONTROL,
7769 dev_priv->czclk_freq);
7770
7771 DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
7772 dev_priv->gt_pm.rps.gpll_ref_freq);
7773 }
7774
7775 static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
7776 {
7777 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7778 u32 val;
7779
7780 valleyview_setup_pctx(dev_priv);
7781
7782 vlv_iosf_sb_get(dev_priv,
7783 BIT(VLV_IOSF_SB_PUNIT) |
7784 BIT(VLV_IOSF_SB_NC) |
7785 BIT(VLV_IOSF_SB_CCK));
7786
7787 vlv_init_gpll_ref_freq(dev_priv);
7788
7789 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7790 switch ((val >> 6) & 3) {
7791 case 0:
7792 case 1:
7793 dev_priv->mem_freq = 800;
7794 break;
7795 case 2:
7796 dev_priv->mem_freq = 1066;
7797 break;
7798 case 3:
7799 dev_priv->mem_freq = 1333;
7800 break;
7801 }
7802 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7803
7804 rps->max_freq = valleyview_rps_max_freq(dev_priv);
7805 rps->rp0_freq = rps->max_freq;
7806 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7807 intel_gpu_freq(dev_priv, rps->max_freq),
7808 rps->max_freq);
7809
7810 rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
7811 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7812 intel_gpu_freq(dev_priv, rps->efficient_freq),
7813 rps->efficient_freq);
7814
7815 rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
7816 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
7817 intel_gpu_freq(dev_priv, rps->rp1_freq),
7818 rps->rp1_freq);
7819
7820 rps->min_freq = valleyview_rps_min_freq(dev_priv);
7821 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7822 intel_gpu_freq(dev_priv, rps->min_freq),
7823 rps->min_freq);
7824
7825 vlv_iosf_sb_put(dev_priv,
7826 BIT(VLV_IOSF_SB_PUNIT) |
7827 BIT(VLV_IOSF_SB_NC) |
7828 BIT(VLV_IOSF_SB_CCK));
7829 }
7830
7831 static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
7832 {
7833 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7834 u32 val;
7835
7836 cherryview_setup_pctx(dev_priv);
7837
7838 vlv_iosf_sb_get(dev_priv,
7839 BIT(VLV_IOSF_SB_PUNIT) |
7840 BIT(VLV_IOSF_SB_NC) |
7841 BIT(VLV_IOSF_SB_CCK));
7842
7843 vlv_init_gpll_ref_freq(dev_priv);
7844
7845 val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
7846
7847 switch ((val >> 2) & 0x7) {
7848 case 3:
7849 dev_priv->mem_freq = 2000;
7850 break;
7851 default:
7852 dev_priv->mem_freq = 1600;
7853 break;
7854 }
7855 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7856
7857 rps->max_freq = cherryview_rps_max_freq(dev_priv);
7858 rps->rp0_freq = rps->max_freq;
7859 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7860 intel_gpu_freq(dev_priv, rps->max_freq),
7861 rps->max_freq);
7862
7863 rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
7864 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7865 intel_gpu_freq(dev_priv, rps->efficient_freq),
7866 rps->efficient_freq);
7867
7868 rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
7869 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
7870 intel_gpu_freq(dev_priv, rps->rp1_freq),
7871 rps->rp1_freq);
7872
7873 rps->min_freq = cherryview_rps_min_freq(dev_priv);
7874 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7875 intel_gpu_freq(dev_priv, rps->min_freq),
7876 rps->min_freq);
7877
7878 vlv_iosf_sb_put(dev_priv,
7879 BIT(VLV_IOSF_SB_PUNIT) |
7880 BIT(VLV_IOSF_SB_NC) |
7881 BIT(VLV_IOSF_SB_CCK));
7882
7883 WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
7884 rps->min_freq) & 1,
7885 "Odd GPU freq values\n");
7886 }
7887
7888 static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7889 {
7890 valleyview_cleanup_pctx(dev_priv);
7891 }
7892
7893 static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
7894 {
7895 struct intel_engine_cs *engine;
7896 enum intel_engine_id id;
7897 u32 gtfifodbg, rc6_mode, pcbr;
7898
7899 gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
7900 GT_FIFO_FREE_ENTRIES_CHV);
7901 if (gtfifodbg) {
7902 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7903 gtfifodbg);
7904 I915_WRITE(GTFIFODBG, gtfifodbg);
7905 }
7906
7907 cherryview_check_pctx(dev_priv);
7908
7909
7910
7911 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7912
7913
7914 I915_WRITE(GEN6_RC_CONTROL, 0);
7915
7916
7917 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7918 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7919 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7920
7921 for_each_engine(engine, dev_priv, id)
7922 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7923 I915_WRITE(GEN6_RC_SLEEP, 0);
7924
7925
7926 I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
7927
7928
7929 I915_WRITE(VLV_COUNTER_CONTROL,
7930 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7931 VLV_MEDIA_RC6_COUNT_EN |
7932 VLV_RENDER_RC6_COUNT_EN));
7933
7934
7935 pcbr = I915_READ(VLV_PCBR);
7936
7937
7938 rc6_mode = 0;
7939 if (pcbr >> VLV_PCBR_ADDR_SHIFT)
7940 rc6_mode = GEN7_RC_CTL_TO_MODE;
7941 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
7942
7943 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7944 }
7945
7946 static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
7947 {
7948 u32 val;
7949
7950 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7951
7952
7953 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7954 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7955 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7956 I915_WRITE(GEN6_RP_UP_EI, 66000);
7957 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7958
7959 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7960
7961
7962 I915_WRITE(GEN6_RP_CONTROL,
7963 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7964 GEN6_RP_MEDIA_IS_GFX |
7965 GEN6_RP_ENABLE |
7966 GEN6_RP_UP_BUSY_AVG |
7967 GEN6_RP_DOWN_IDLE_AVG);
7968
7969
7970 vlv_punit_get(dev_priv);
7971
7972 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
7973 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7974
7975 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7976
7977 vlv_punit_put(dev_priv);
7978
7979
7980 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7981
7982 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7983 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7984
7985 reset_rps(dev_priv, valleyview_set_rps);
7986
7987 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7988 }
7989
7990 static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
7991 {
7992 struct intel_engine_cs *engine;
7993 enum intel_engine_id id;
7994 u32 gtfifodbg;
7995
7996 valleyview_check_pctx(dev_priv);
7997
7998 gtfifodbg = I915_READ(GTFIFODBG);
7999 if (gtfifodbg) {
8000 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
8001 gtfifodbg);
8002 I915_WRITE(GTFIFODBG, gtfifodbg);
8003 }
8004
8005 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
8006
8007
8008 I915_WRITE(GEN6_RC_CONTROL, 0);
8009
8010 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
8011 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
8012 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
8013
8014 for_each_engine(engine, dev_priv, id)
8015 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
8016
8017 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
8018
8019
8020 I915_WRITE(VLV_COUNTER_CONTROL,
8021 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
8022 VLV_MEDIA_RC0_COUNT_EN |
8023 VLV_RENDER_RC0_COUNT_EN |
8024 VLV_MEDIA_RC6_COUNT_EN |
8025 VLV_RENDER_RC6_COUNT_EN));
8026
8027 I915_WRITE(GEN6_RC_CONTROL,
8028 GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
8029
8030 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
8031 }
8032
8033 static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
8034 {
8035 u32 val;
8036
8037 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
8038
8039 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
8040 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
8041 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
8042 I915_WRITE(GEN6_RP_UP_EI, 66000);
8043 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
8044
8045 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
8046
8047 I915_WRITE(GEN6_RP_CONTROL,
8048 GEN6_RP_MEDIA_TURBO |
8049 GEN6_RP_MEDIA_HW_NORMAL_MODE |
8050 GEN6_RP_MEDIA_IS_GFX |
8051 GEN6_RP_ENABLE |
8052 GEN6_RP_UP_BUSY_AVG |
8053 GEN6_RP_DOWN_IDLE_CONT);
8054
8055 vlv_punit_get(dev_priv);
8056
8057
8058 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
8059 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
8060
8061 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
8062
8063 vlv_punit_put(dev_priv);
8064
8065
8066 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
8067
8068 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
8069 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
8070
8071 reset_rps(dev_priv, valleyview_set_rps);
8072
8073 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
8074 }
8075
8076 static unsigned long intel_pxfreq(u32 vidfreq)
8077 {
8078 unsigned long freq;
8079 int div = (vidfreq & 0x3f0000) >> 16;
8080 int post = (vidfreq & 0x3000) >> 12;
8081 int pre = (vidfreq & 0x7);
8082
8083 if (!pre)
8084 return 0;
8085
8086 freq = ((div * 133333) / ((1<<post) * pre));
8087
8088 return freq;
8089 }
8090
8091 static const struct cparams {
8092 u16 i;
8093 u16 t;
8094 u16 m;
8095 u16 c;
8096 } cparams[] = {
8097 { 1, 1333, 301, 28664 },
8098 { 1, 1066, 294, 24460 },
8099 { 1, 800, 294, 25192 },
8100 { 0, 1333, 276, 27605 },
8101 { 0, 1066, 276, 27605 },
8102 { 0, 800, 231, 23784 },
8103 };
8104
8105 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
8106 {
8107 u64 total_count, diff, ret;
8108 u32 count1, count2, count3, m = 0, c = 0;
8109 unsigned long now = jiffies_to_msecs(jiffies), diff1;
8110 int i;
8111
8112 lockdep_assert_held(&mchdev_lock);
8113
8114 diff1 = now - dev_priv->ips.last_time1;
8115
8116
8117
8118
8119
8120
8121 if (diff1 <= 10)
8122 return dev_priv->ips.chipset_power;
8123
8124 count1 = I915_READ(DMIEC);
8125 count2 = I915_READ(DDREC);
8126 count3 = I915_READ(CSIEC);
8127
8128 total_count = count1 + count2 + count3;
8129
8130
8131 if (total_count < dev_priv->ips.last_count1) {
8132 diff = ~0UL - dev_priv->ips.last_count1;
8133 diff += total_count;
8134 } else {
8135 diff = total_count - dev_priv->ips.last_count1;
8136 }
8137
8138 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
8139 if (cparams[i].i == dev_priv->ips.c_m &&
8140 cparams[i].t == dev_priv->ips.r_t) {
8141 m = cparams[i].m;
8142 c = cparams[i].c;
8143 break;
8144 }
8145 }
8146
8147 diff = div_u64(diff, diff1);
8148 ret = ((m * diff) + c);
8149 ret = div_u64(ret, 10);
8150
8151 dev_priv->ips.last_count1 = total_count;
8152 dev_priv->ips.last_time1 = now;
8153
8154 dev_priv->ips.chipset_power = ret;
8155
8156 return ret;
8157 }
8158
8159 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
8160 {
8161 intel_wakeref_t wakeref;
8162 unsigned long val = 0;
8163
8164 if (!IS_GEN(dev_priv, 5))
8165 return 0;
8166
8167 with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
8168 spin_lock_irq(&mchdev_lock);
8169 val = __i915_chipset_val(dev_priv);
8170 spin_unlock_irq(&mchdev_lock);
8171 }
8172
8173 return val;
8174 }
8175
8176 unsigned long i915_mch_val(struct drm_i915_private *i915)
8177 {
8178 unsigned long m, x, b;
8179 u32 tsfs;
8180
8181 tsfs = intel_uncore_read(&i915->uncore, TSFS);
8182
8183 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
8184 x = intel_uncore_read8(&i915->uncore, TR1);
8185
8186 b = tsfs & TSFS_INTR_MASK;
8187
8188 return ((m * x) / 127) - b;
8189 }
8190
8191 static int _pxvid_to_vd(u8 pxvid)
8192 {
8193 if (pxvid == 0)
8194 return 0;
8195
8196 if (pxvid >= 8 && pxvid < 31)
8197 pxvid = 31;
8198
8199 return (pxvid + 2) * 125;
8200 }
8201
8202 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
8203 {
8204 const int vd = _pxvid_to_vd(pxvid);
8205 const int vm = vd - 1125;
8206
8207 if (INTEL_INFO(dev_priv)->is_mobile)
8208 return vm > 0 ? vm : 0;
8209
8210 return vd;
8211 }
8212
8213 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
8214 {
8215 u64 now, diff, diffms;
8216 u32 count;
8217
8218 lockdep_assert_held(&mchdev_lock);
8219
8220 now = ktime_get_raw_ns();
8221 diffms = now - dev_priv->ips.last_time2;
8222 do_div(diffms, NSEC_PER_MSEC);
8223
8224
8225 if (!diffms)
8226 return;
8227
8228 count = I915_READ(GFXEC);
8229
8230 if (count < dev_priv->ips.last_count2) {
8231 diff = ~0UL - dev_priv->ips.last_count2;
8232 diff += count;
8233 } else {
8234 diff = count - dev_priv->ips.last_count2;
8235 }
8236
8237 dev_priv->ips.last_count2 = count;
8238 dev_priv->ips.last_time2 = now;
8239
8240
8241 diff = diff * 1181;
8242 diff = div_u64(diff, diffms * 10);
8243 dev_priv->ips.gfx_power = diff;
8244 }
8245
8246 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
8247 {
8248 intel_wakeref_t wakeref;
8249
8250 if (!IS_GEN(dev_priv, 5))
8251 return;
8252
8253 with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
8254 spin_lock_irq(&mchdev_lock);
8255 __i915_update_gfx_val(dev_priv);
8256 spin_unlock_irq(&mchdev_lock);
8257 }
8258 }
8259
8260 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
8261 {
8262 unsigned long t, corr, state1, corr2, state2;
8263 u32 pxvid, ext_v;
8264
8265 lockdep_assert_held(&mchdev_lock);
8266
8267 pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
8268 pxvid = (pxvid >> 24) & 0x7f;
8269 ext_v = pvid_to_extvid(dev_priv, pxvid);
8270
8271 state1 = ext_v;
8272
8273 t = i915_mch_val(dev_priv);
8274
8275
8276
8277
8278 if (t > 80)
8279 corr = ((t * 2349) + 135940);
8280 else if (t >= 50)
8281 corr = ((t * 964) + 29317);
8282 else
8283 corr = ((t * 301) + 1004);
8284
8285 corr = corr * ((150142 * state1) / 10000 - 78642);
8286 corr /= 100000;
8287 corr2 = (corr * dev_priv->ips.corr);
8288
8289 state2 = (corr2 * state1) / 10000;
8290 state2 /= 100;
8291
8292 __i915_update_gfx_val(dev_priv);
8293
8294 return dev_priv->ips.gfx_power + state2;
8295 }
8296
8297 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
8298 {
8299 intel_wakeref_t wakeref;
8300 unsigned long val = 0;
8301
8302 if (!IS_GEN(dev_priv, 5))
8303 return 0;
8304
8305 with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
8306 spin_lock_irq(&mchdev_lock);
8307 val = __i915_gfx_val(dev_priv);
8308 spin_unlock_irq(&mchdev_lock);
8309 }
8310
8311 return val;
8312 }
8313
8314 static struct drm_i915_private __rcu *i915_mch_dev;
8315
8316 static struct drm_i915_private *mchdev_get(void)
8317 {
8318 struct drm_i915_private *i915;
8319
8320 rcu_read_lock();
8321 i915 = rcu_dereference(i915_mch_dev);
8322 if (!kref_get_unless_zero(&i915->drm.ref))
8323 i915 = NULL;
8324 rcu_read_unlock();
8325
8326 return i915;
8327 }
8328
8329
8330
8331
8332
8333
8334
8335 unsigned long i915_read_mch_val(void)
8336 {
8337 struct drm_i915_private *i915;
8338 unsigned long chipset_val = 0;
8339 unsigned long graphics_val = 0;
8340 intel_wakeref_t wakeref;
8341
8342 i915 = mchdev_get();
8343 if (!i915)
8344 return 0;
8345
8346 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
8347 spin_lock_irq(&mchdev_lock);
8348 chipset_val = __i915_chipset_val(i915);
8349 graphics_val = __i915_gfx_val(i915);
8350 spin_unlock_irq(&mchdev_lock);
8351 }
8352
8353 drm_dev_put(&i915->drm);
8354 return chipset_val + graphics_val;
8355 }
8356 EXPORT_SYMBOL_GPL(i915_read_mch_val);
8357
8358
8359
8360
8361
8362
8363 bool i915_gpu_raise(void)
8364 {
8365 struct drm_i915_private *i915;
8366
8367 i915 = mchdev_get();
8368 if (!i915)
8369 return false;
8370
8371 spin_lock_irq(&mchdev_lock);
8372 if (i915->ips.max_delay > i915->ips.fmax)
8373 i915->ips.max_delay--;
8374 spin_unlock_irq(&mchdev_lock);
8375
8376 drm_dev_put(&i915->drm);
8377 return true;
8378 }
8379 EXPORT_SYMBOL_GPL(i915_gpu_raise);
8380
8381
8382
8383
8384
8385
8386
8387 bool i915_gpu_lower(void)
8388 {
8389 struct drm_i915_private *i915;
8390
8391 i915 = mchdev_get();
8392 if (!i915)
8393 return false;
8394
8395 spin_lock_irq(&mchdev_lock);
8396 if (i915->ips.max_delay < i915->ips.min_delay)
8397 i915->ips.max_delay++;
8398 spin_unlock_irq(&mchdev_lock);
8399
8400 drm_dev_put(&i915->drm);
8401 return true;
8402 }
8403 EXPORT_SYMBOL_GPL(i915_gpu_lower);
8404
8405
8406
8407
8408
8409
8410 bool i915_gpu_busy(void)
8411 {
8412 struct drm_i915_private *i915;
8413 bool ret;
8414
8415 i915 = mchdev_get();
8416 if (!i915)
8417 return false;
8418
8419 ret = i915->gt.awake;
8420
8421 drm_dev_put(&i915->drm);
8422 return ret;
8423 }
8424 EXPORT_SYMBOL_GPL(i915_gpu_busy);
8425
8426
8427
8428
8429
8430
8431
8432 bool i915_gpu_turbo_disable(void)
8433 {
8434 struct drm_i915_private *i915;
8435 bool ret;
8436
8437 i915 = mchdev_get();
8438 if (!i915)
8439 return false;
8440
8441 spin_lock_irq(&mchdev_lock);
8442 i915->ips.max_delay = i915->ips.fstart;
8443 ret = ironlake_set_drps(i915, i915->ips.fstart);
8444 spin_unlock_irq(&mchdev_lock);
8445
8446 drm_dev_put(&i915->drm);
8447 return ret;
8448 }
8449 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459 static void
8460 ips_ping_for_i915_load(void)
8461 {
8462 void (*link)(void);
8463
8464 link = symbol_get(ips_link_to_i915_driver);
8465 if (link) {
8466 link();
8467 symbol_put(ips_link_to_i915_driver);
8468 }
8469 }
8470
8471 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
8472 {
8473
8474
8475 rcu_assign_pointer(i915_mch_dev, dev_priv);
8476
8477 ips_ping_for_i915_load();
8478 }
8479
8480 void intel_gpu_ips_teardown(void)
8481 {
8482 rcu_assign_pointer(i915_mch_dev, NULL);
8483 }
8484
8485 static void intel_init_emon(struct drm_i915_private *dev_priv)
8486 {
8487 u32 lcfuse;
8488 u8 pxw[16];
8489 int i;
8490
8491
8492 I915_WRITE(ECR, 0);
8493 POSTING_READ(ECR);
8494
8495
8496 I915_WRITE(SDEW, 0x15040d00);
8497 I915_WRITE(CSIEW0, 0x007f0000);
8498 I915_WRITE(CSIEW1, 0x1e220004);
8499 I915_WRITE(CSIEW2, 0x04000004);
8500
8501 for (i = 0; i < 5; i++)
8502 I915_WRITE(PEW(i), 0);
8503 for (i = 0; i < 3; i++)
8504 I915_WRITE(DEW(i), 0);
8505
8506
8507 for (i = 0; i < 16; i++) {
8508 u32 pxvidfreq = I915_READ(PXVFREQ(i));
8509 unsigned long freq = intel_pxfreq(pxvidfreq);
8510 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
8511 PXVFREQ_PX_SHIFT;
8512 unsigned long val;
8513
8514 val = vid * vid;
8515 val *= (freq / 1000);
8516 val *= 255;
8517 val /= (127*127*900);
8518 if (val > 0xff)
8519 DRM_ERROR("bad pxval: %ld\n", val);
8520 pxw[i] = val;
8521 }
8522
8523 pxw[14] = 0;
8524 pxw[15] = 0;
8525
8526 for (i = 0; i < 4; i++) {
8527 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
8528 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
8529 I915_WRITE(PXW(i), val);
8530 }
8531
8532
8533 I915_WRITE(OGW0, 0);
8534 I915_WRITE(OGW1, 0);
8535 I915_WRITE(EG0, 0x00007f00);
8536 I915_WRITE(EG1, 0x0000000e);
8537 I915_WRITE(EG2, 0x000e0000);
8538 I915_WRITE(EG3, 0x68000300);
8539 I915_WRITE(EG4, 0x42000000);
8540 I915_WRITE(EG5, 0x00140031);
8541 I915_WRITE(EG6, 0);
8542 I915_WRITE(EG7, 0);
8543
8544 for (i = 0; i < 8; i++)
8545 I915_WRITE(PXWL(i), 0);
8546
8547
8548 I915_WRITE(ECR, 0x80000019);
8549
8550 lcfuse = I915_READ(LCFUSE02);
8551
8552 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
8553 }
8554
8555 static bool i915_rc6_ctx_corrupted(struct drm_i915_private *dev_priv)
8556 {
8557 return !I915_READ(GEN8_RC6_CTX_INFO);
8558 }
8559
8560 static void i915_rc6_ctx_wa_init(struct drm_i915_private *i915)
8561 {
8562 if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
8563 return;
8564
8565 if (i915_rc6_ctx_corrupted(i915)) {
8566 DRM_INFO("RC6 context corrupted, disabling runtime power management\n");
8567 i915->gt_pm.rc6.ctx_corrupted = true;
8568 i915->gt_pm.rc6.ctx_corrupted_wakeref =
8569 intel_runtime_pm_get(&i915->runtime_pm);
8570 }
8571 }
8572
8573 static void i915_rc6_ctx_wa_cleanup(struct drm_i915_private *i915)
8574 {
8575 if (i915->gt_pm.rc6.ctx_corrupted) {
8576 intel_runtime_pm_put(&i915->runtime_pm,
8577 i915->gt_pm.rc6.ctx_corrupted_wakeref);
8578 i915->gt_pm.rc6.ctx_corrupted = false;
8579 }
8580 }
8581
8582
8583
8584
8585
8586
8587
8588 void i915_rc6_ctx_wa_suspend(struct drm_i915_private *i915)
8589 {
8590 if (i915->gt_pm.rc6.ctx_corrupted)
8591 intel_runtime_pm_put(&i915->runtime_pm,
8592 i915->gt_pm.rc6.ctx_corrupted_wakeref);
8593 }
8594
8595
8596
8597
8598
8599
8600
8601 void i915_rc6_ctx_wa_resume(struct drm_i915_private *i915)
8602 {
8603 if (!i915->gt_pm.rc6.ctx_corrupted)
8604 return;
8605
8606 if (i915_rc6_ctx_corrupted(i915)) {
8607 i915->gt_pm.rc6.ctx_corrupted_wakeref =
8608 intel_runtime_pm_get(&i915->runtime_pm);
8609 return;
8610 }
8611
8612 DRM_INFO("RC6 context restored, re-enabling runtime power management\n");
8613 i915->gt_pm.rc6.ctx_corrupted = false;
8614 }
8615
8616 static void intel_disable_rc6(struct drm_i915_private *dev_priv);
8617
8618
8619
8620
8621
8622
8623
8624
8625
8626
8627
8628 bool i915_rc6_ctx_wa_check(struct drm_i915_private *i915)
8629 {
8630 if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
8631 return false;
8632
8633 if (i915->gt_pm.rc6.ctx_corrupted)
8634 return false;
8635
8636 if (!i915_rc6_ctx_corrupted(i915))
8637 return false;
8638
8639 DRM_NOTE("RC6 context corruption, disabling runtime power management\n");
8640
8641 intel_disable_rc6(i915);
8642 i915->gt_pm.rc6.ctx_corrupted = true;
8643 i915->gt_pm.rc6.ctx_corrupted_wakeref =
8644 intel_runtime_pm_get_noresume(&i915->runtime_pm);
8645
8646 return true;
8647 }
8648
8649 void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
8650 {
8651 struct intel_rps *rps = &dev_priv->gt_pm.rps;
8652
8653
8654
8655
8656
8657 if (!sanitize_rc6(dev_priv)) {
8658 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
8659 pm_runtime_get(&dev_priv->drm.pdev->dev);
8660 }
8661
8662 i915_rc6_ctx_wa_init(dev_priv);
8663
8664
8665 if (IS_CHERRYVIEW(dev_priv))
8666 cherryview_init_gt_powersave(dev_priv);
8667 else if (IS_VALLEYVIEW(dev_priv))
8668 valleyview_init_gt_powersave(dev_priv);
8669 else if (INTEL_GEN(dev_priv) >= 6)
8670 gen6_init_rps_frequencies(dev_priv);
8671
8672
8673 rps->max_freq_softlimit = rps->max_freq;
8674 rps->min_freq_softlimit = rps->min_freq;
8675
8676
8677 if (IS_GEN(dev_priv, 6) ||
8678 IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
8679 u32 params = 0;
8680
8681 sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS,
8682 ¶ms, NULL);
8683 if (params & BIT(31)) {
8684 DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
8685 (rps->max_freq & 0xff) * 50,
8686 (params & 0xff) * 50);
8687 rps->max_freq = params & 0xff;
8688 }
8689 }
8690
8691
8692 rps->boost_freq = rps->max_freq;
8693 rps->idle_freq = rps->min_freq;
8694 rps->cur_freq = rps->idle_freq;
8695 }
8696
8697 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
8698 {
8699 if (IS_VALLEYVIEW(dev_priv))
8700 valleyview_cleanup_gt_powersave(dev_priv);
8701
8702 i915_rc6_ctx_wa_cleanup(dev_priv);
8703
8704 if (!HAS_RC6(dev_priv))
8705 pm_runtime_put(&dev_priv->drm.pdev->dev);
8706 }
8707
8708 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
8709 {
8710 dev_priv->gt_pm.rps.enabled = true;
8711 dev_priv->gt_pm.rc6.enabled = true;
8712 intel_disable_gt_powersave(dev_priv);
8713
8714 if (INTEL_GEN(dev_priv) >= 11)
8715 gen11_reset_rps_interrupts(dev_priv);
8716 else if (INTEL_GEN(dev_priv) >= 6)
8717 gen6_reset_rps_interrupts(dev_priv);
8718 }
8719
8720 static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
8721 {
8722 lockdep_assert_held(&i915->gt_pm.rps.lock);
8723
8724 if (!i915->gt_pm.llc_pstate.enabled)
8725 return;
8726
8727
8728
8729 i915->gt_pm.llc_pstate.enabled = false;
8730 }
8731
8732 static void __intel_disable_rc6(struct drm_i915_private *dev_priv)
8733 {
8734 lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
8735
8736 if (!dev_priv->gt_pm.rc6.enabled)
8737 return;
8738
8739 if (INTEL_GEN(dev_priv) >= 9)
8740 gen9_disable_rc6(dev_priv);
8741 else if (IS_CHERRYVIEW(dev_priv))
8742 cherryview_disable_rc6(dev_priv);
8743 else if (IS_VALLEYVIEW(dev_priv))
8744 valleyview_disable_rc6(dev_priv);
8745 else if (INTEL_GEN(dev_priv) >= 6)
8746 gen6_disable_rc6(dev_priv);
8747
8748 dev_priv->gt_pm.rc6.enabled = false;
8749 }
8750
8751 static void intel_disable_rc6(struct drm_i915_private *dev_priv)
8752 {
8753 struct intel_rps *rps = &dev_priv->gt_pm.rps;
8754
8755 mutex_lock(&rps->lock);
8756 __intel_disable_rc6(dev_priv);
8757 mutex_unlock(&rps->lock);
8758 }
8759
8760 static void intel_disable_rps(struct drm_i915_private *dev_priv)
8761 {
8762 lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
8763
8764 if (!dev_priv->gt_pm.rps.enabled)
8765 return;
8766
8767 if (INTEL_GEN(dev_priv) >= 9)
8768 gen9_disable_rps(dev_priv);
8769 else if (IS_CHERRYVIEW(dev_priv))
8770 cherryview_disable_rps(dev_priv);
8771 else if (IS_VALLEYVIEW(dev_priv))
8772 valleyview_disable_rps(dev_priv);
8773 else if (INTEL_GEN(dev_priv) >= 6)
8774 gen6_disable_rps(dev_priv);
8775 else if (IS_IRONLAKE_M(dev_priv))
8776 ironlake_disable_drps(dev_priv);
8777
8778 dev_priv->gt_pm.rps.enabled = false;
8779 }
8780
8781 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
8782 {
8783 mutex_lock(&dev_priv->gt_pm.rps.lock);
8784
8785 __intel_disable_rc6(dev_priv);
8786 intel_disable_rps(dev_priv);
8787 if (HAS_LLC(dev_priv))
8788 intel_disable_llc_pstate(dev_priv);
8789
8790 mutex_unlock(&dev_priv->gt_pm.rps.lock);
8791 }
8792
8793 static inline void intel_enable_llc_pstate(struct drm_i915_private *i915)
8794 {
8795 lockdep_assert_held(&i915->gt_pm.rps.lock);
8796
8797 if (i915->gt_pm.llc_pstate.enabled)
8798 return;
8799
8800 gen6_update_ring_freq(i915);
8801
8802 i915->gt_pm.llc_pstate.enabled = true;
8803 }
8804
8805 static void intel_enable_rc6(struct drm_i915_private *dev_priv)
8806 {
8807 lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
8808
8809 if (dev_priv->gt_pm.rc6.enabled)
8810 return;
8811
8812 if (dev_priv->gt_pm.rc6.ctx_corrupted)
8813 return;
8814
8815 if (IS_CHERRYVIEW(dev_priv))
8816 cherryview_enable_rc6(dev_priv);
8817 else if (IS_VALLEYVIEW(dev_priv))
8818 valleyview_enable_rc6(dev_priv);
8819 else if (INTEL_GEN(dev_priv) >= 11)
8820 gen11_enable_rc6(dev_priv);
8821 else if (INTEL_GEN(dev_priv) >= 9)
8822 gen9_enable_rc6(dev_priv);
8823 else if (IS_BROADWELL(dev_priv))
8824 gen8_enable_rc6(dev_priv);
8825 else if (INTEL_GEN(dev_priv) >= 6)
8826 gen6_enable_rc6(dev_priv);
8827
8828 dev_priv->gt_pm.rc6.enabled = true;
8829 }
8830
8831 static void intel_enable_rps(struct drm_i915_private *dev_priv)
8832 {
8833 struct intel_rps *rps = &dev_priv->gt_pm.rps;
8834
8835 lockdep_assert_held(&rps->lock);
8836
8837 if (rps->enabled)
8838 return;
8839
8840 if (IS_CHERRYVIEW(dev_priv)) {
8841 cherryview_enable_rps(dev_priv);
8842 } else if (IS_VALLEYVIEW(dev_priv)) {
8843 valleyview_enable_rps(dev_priv);
8844 } else if (INTEL_GEN(dev_priv) >= 9) {
8845 gen9_enable_rps(dev_priv);
8846 } else if (IS_BROADWELL(dev_priv)) {
8847 gen8_enable_rps(dev_priv);
8848 } else if (INTEL_GEN(dev_priv) >= 6) {
8849 gen6_enable_rps(dev_priv);
8850 } else if (IS_IRONLAKE_M(dev_priv)) {
8851 ironlake_enable_drps(dev_priv);
8852 intel_init_emon(dev_priv);
8853 }
8854
8855 WARN_ON(rps->max_freq < rps->min_freq);
8856 WARN_ON(rps->idle_freq > rps->max_freq);
8857
8858 WARN_ON(rps->efficient_freq < rps->min_freq);
8859 WARN_ON(rps->efficient_freq > rps->max_freq);
8860
8861 rps->enabled = true;
8862 }
8863
8864 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
8865 {
8866
8867 if (intel_vgpu_active(dev_priv))
8868 return;
8869
8870 mutex_lock(&dev_priv->gt_pm.rps.lock);
8871
8872 if (HAS_RC6(dev_priv))
8873 intel_enable_rc6(dev_priv);
8874 if (HAS_RPS(dev_priv))
8875 intel_enable_rps(dev_priv);
8876 if (HAS_LLC(dev_priv))
8877 intel_enable_llc_pstate(dev_priv);
8878
8879 mutex_unlock(&dev_priv->gt_pm.rps.lock);
8880 }
8881
8882 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
8883 {
8884
8885
8886
8887
8888
8889 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8890 }
8891
8892 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
8893 {
8894 enum pipe pipe;
8895
8896 for_each_pipe(dev_priv, pipe) {
8897 I915_WRITE(DSPCNTR(pipe),
8898 I915_READ(DSPCNTR(pipe)) |
8899 DISPPLANE_TRICKLE_FEED_DISABLE);
8900
8901 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
8902 POSTING_READ(DSPSURF(pipe));
8903 }
8904 }
8905
8906 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
8907 {
8908 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8909
8910
8911
8912
8913
8914 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
8915 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
8916 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
8917
8918 I915_WRITE(PCH_3DCGDIS0,
8919 MARIUNIT_CLOCK_GATE_DISABLE |
8920 SVSMUNIT_CLOCK_GATE_DISABLE);
8921 I915_WRITE(PCH_3DCGDIS1,
8922 VFMUNIT_CLOCK_GATE_DISABLE);
8923
8924
8925
8926
8927
8928
8929
8930
8931 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8932 (I915_READ(ILK_DISPLAY_CHICKEN2) |
8933 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8934 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
8935 I915_WRITE(DISP_ARB_CTL,
8936 (I915_READ(DISP_ARB_CTL) |
8937 DISP_FBC_WM_DIS));
8938
8939
8940
8941
8942
8943
8944
8945
8946 if (IS_IRONLAKE_M(dev_priv)) {
8947
8948 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8949 I915_READ(ILK_DISPLAY_CHICKEN1) |
8950 ILK_FBCQ_DIS);
8951 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8952 I915_READ(ILK_DISPLAY_CHICKEN2) |
8953 ILK_DPARB_GATE);
8954 }
8955
8956 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8957
8958 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8959 I915_READ(ILK_DISPLAY_CHICKEN2) |
8960 ILK_ELPIN_409_SELECT);
8961 I915_WRITE(_3D_CHICKEN2,
8962 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8963 _3D_CHICKEN2_WM_READ_PIPELINED);
8964
8965
8966 I915_WRITE(CACHE_MODE_0,
8967 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
8968
8969
8970 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8971
8972 g4x_disable_trickle_feed(dev_priv);
8973
8974 ibx_init_clock_gating(dev_priv);
8975 }
8976
8977 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
8978 {
8979 int pipe;
8980 u32 val;
8981
8982
8983
8984
8985
8986
8987 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
8988 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
8989 PCH_CPUNIT_CLOCK_GATE_DISABLE);
8990 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
8991 DPLS_EDP_PPS_FIX_DIS);
8992
8993
8994
8995 for_each_pipe(dev_priv, pipe) {
8996 val = I915_READ(TRANS_CHICKEN2(pipe));
8997 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
8998 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8999 if (dev_priv->vbt.fdi_rx_polarity_inverted)
9000 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
9001 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
9002 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
9003 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
9004 I915_WRITE(TRANS_CHICKEN2(pipe), val);
9005 }
9006
9007 for_each_pipe(dev_priv, pipe) {
9008 I915_WRITE(TRANS_CHICKEN1(pipe),
9009 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
9010 }
9011 }
9012
9013 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
9014 {
9015 u32 tmp;
9016
9017 tmp = I915_READ(MCH_SSKPD);
9018 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
9019 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
9020 tmp);
9021 }
9022
9023 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
9024 {
9025 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
9026
9027 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
9028
9029 I915_WRITE(ILK_DISPLAY_CHICKEN2,
9030 I915_READ(ILK_DISPLAY_CHICKEN2) |
9031 ILK_ELPIN_409_SELECT);
9032
9033
9034 I915_WRITE(_3D_CHICKEN,
9035 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
9036
9037
9038 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9039
9040
9041
9042
9043
9044
9045
9046
9047
9048 I915_WRITE(GEN6_GT_MODE,
9049 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9050
9051 I915_WRITE(CACHE_MODE_0,
9052 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
9053
9054 I915_WRITE(GEN6_UCGCTL1,
9055 I915_READ(GEN6_UCGCTL1) |
9056 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
9057 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
9058
9059
9060
9061
9062
9063
9064
9065
9066
9067
9068
9069
9070
9071
9072 I915_WRITE(GEN6_UCGCTL2,
9073 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
9074 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
9075
9076
9077 I915_WRITE(_3D_CHICKEN3,
9078 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
9079
9080
9081
9082
9083
9084
9085 I915_WRITE(_3D_CHICKEN3,
9086 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
9087
9088
9089
9090
9091
9092
9093
9094
9095
9096
9097
9098
9099 I915_WRITE(ILK_DISPLAY_CHICKEN1,
9100 I915_READ(ILK_DISPLAY_CHICKEN1) |
9101 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
9102 I915_WRITE(ILK_DISPLAY_CHICKEN2,
9103 I915_READ(ILK_DISPLAY_CHICKEN2) |
9104 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
9105 I915_WRITE(ILK_DSPCLK_GATE_D,
9106 I915_READ(ILK_DSPCLK_GATE_D) |
9107 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
9108 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
9109
9110 g4x_disable_trickle_feed(dev_priv);
9111
9112 cpt_init_clock_gating(dev_priv);
9113
9114 gen6_check_mch_setup(dev_priv);
9115 }
9116
9117 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
9118 {
9119 u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
9120
9121
9122
9123
9124
9125
9126
9127 reg &= ~GEN7_FF_SCHED_MASK;
9128 reg |= GEN7_FF_TS_SCHED_HW;
9129 reg |= GEN7_FF_VS_SCHED_HW;
9130 reg |= GEN7_FF_DS_SCHED_HW;
9131
9132 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
9133 }
9134
9135 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
9136 {
9137
9138
9139
9140
9141 if (HAS_PCH_LPT_LP(dev_priv))
9142 I915_WRITE(SOUTH_DSPCLK_GATE_D,
9143 I915_READ(SOUTH_DSPCLK_GATE_D) |
9144 PCH_LP_PARTITION_LEVEL_DISABLE);
9145
9146
9147 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
9148 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
9149 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
9150 }
9151
9152 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
9153 {
9154 if (HAS_PCH_LPT_LP(dev_priv)) {
9155 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9156
9157 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9158 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9159 }
9160 }
9161
9162 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
9163 int general_prio_credits,
9164 int high_prio_credits)
9165 {
9166 u32 misccpctl;
9167 u32 val;
9168
9169
9170 misccpctl = I915_READ(GEN7_MISCCPCTL);
9171 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
9172
9173 val = I915_READ(GEN8_L3SQCREG1);
9174 val &= ~L3_PRIO_CREDITS_MASK;
9175 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
9176 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
9177 I915_WRITE(GEN8_L3SQCREG1, val);
9178
9179
9180
9181
9182
9183 POSTING_READ(GEN8_L3SQCREG1);
9184 udelay(1);
9185 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
9186 }
9187
9188 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
9189 {
9190
9191 I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
9192 I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
9193
9194
9195 I915_WRITE(GEN9_CSFE_CHICKEN1_RCS,
9196 _MASKED_BIT_ENABLE(GEN11_ENABLE_32_PLANE_MODE));
9197
9198
9199
9200
9201
9202 intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE,
9203 0, VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
9204
9205
9206 intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2,
9207 0, PSDUNIT_CLKGATE_DIS);
9208 }
9209
9210 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
9211 {
9212 if (!HAS_PCH_CNP(dev_priv))
9213 return;
9214
9215
9216 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
9217 CNP_PWM_CGE_GATING_DISABLE);
9218 }
9219
9220 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
9221 {
9222 u32 val;
9223 cnp_init_clock_gating(dev_priv);
9224
9225
9226 I915_WRITE(_3D_CHICKEN3,
9227 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
9228
9229
9230 I915_WRITE(GEN8_CHICKEN_DCPR_1,
9231 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
9232
9233
9234 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
9235 DISP_FBC_MEMORY_WAKE);
9236
9237 val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
9238
9239 val |= RCCUNIT_CLKGATE_DIS;
9240
9241 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
9242 val |= SARBUNIT_CLKGATE_DIS;
9243 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
9244
9245
9246 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
9247 val |= GWUNIT_CLKGATE_DIS;
9248 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
9249
9250
9251
9252 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
9253 val |= VFUNIT_CLKGATE_DIS;
9254 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
9255 }
9256
9257 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
9258 {
9259 cnp_init_clock_gating(dev_priv);
9260 gen9_init_clock_gating(dev_priv);
9261
9262
9263 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
9264 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
9265 }
9266
9267 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
9268 {
9269 gen9_init_clock_gating(dev_priv);
9270
9271
9272 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
9273 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9274 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9275
9276
9277 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
9278 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
9279 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
9280
9281
9282 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
9283 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
9284 }
9285
9286 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
9287 {
9288 gen9_init_clock_gating(dev_priv);
9289
9290
9291 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
9292 FBC_LLC_FULLY_OPEN);
9293
9294
9295 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
9296 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
9297 }
9298
9299 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
9300 {
9301 enum pipe pipe;
9302
9303
9304 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
9305
9306
9307 I915_WRITE(CHICKEN_PAR1_1,
9308 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
9309
9310
9311 for_each_pipe(dev_priv, pipe) {
9312 I915_WRITE(CHICKEN_PIPESL_1(pipe),
9313 I915_READ(CHICKEN_PIPESL_1(pipe)) |
9314 BDW_DPRS_MASK_VBLANK_SRD);
9315 }
9316
9317
9318
9319 I915_WRITE(GEN7_FF_THREAD_MODE,
9320 I915_READ(GEN7_FF_THREAD_MODE) &
9321 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
9322
9323 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
9324 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
9325
9326
9327 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9328 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9329
9330
9331 gen8_set_l3sqc_credits(dev_priv, 30, 2);
9332
9333
9334 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
9335 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
9336
9337 lpt_init_clock_gating(dev_priv);
9338
9339
9340
9341
9342
9343
9344 I915_WRITE(GEN6_UCGCTL1,
9345 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
9346 }
9347
9348 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
9349 {
9350
9351 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
9352 I915_WRITE(HSW_ROW_CHICKEN3,
9353 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
9354
9355
9356 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9357 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9358 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9359
9360
9361 I915_WRITE(GEN7_FF_THREAD_MODE,
9362 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
9363
9364
9365 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9366
9367
9368 I915_WRITE(CACHE_MODE_0_GEN7,
9369 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
9370
9371
9372 I915_WRITE(CACHE_MODE_1,
9373 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9374
9375
9376
9377
9378
9379
9380
9381
9382
9383 I915_WRITE(GEN7_GT_MODE,
9384 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9385
9386
9387 I915_WRITE(HALF_SLICE_CHICKEN3,
9388 _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
9389
9390
9391 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
9392
9393 lpt_init_clock_gating(dev_priv);
9394 }
9395
9396 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
9397 {
9398 u32 snpcr;
9399
9400 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
9401
9402
9403 I915_WRITE(_3D_CHICKEN3,
9404 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
9405
9406
9407 I915_WRITE(IVB_CHICKEN3,
9408 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9409 CHICKEN3_DGMG_DONE_FIX_DISABLE);
9410
9411
9412 if (IS_IVB_GT1(dev_priv))
9413 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9414 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9415
9416
9417 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9418
9419
9420 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
9421 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
9422
9423
9424 I915_WRITE(GEN7_L3CNTLREG1,
9425 GEN7_WA_FOR_GEN7_L3_CONTROL);
9426 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
9427 GEN7_WA_L3_CHICKEN_MODE);
9428 if (IS_IVB_GT1(dev_priv))
9429 I915_WRITE(GEN7_ROW_CHICKEN2,
9430 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9431 else {
9432
9433 I915_WRITE(GEN7_ROW_CHICKEN2,
9434 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9435 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
9436 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9437 }
9438
9439
9440 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9441 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9442
9443
9444
9445
9446
9447 I915_WRITE(GEN6_UCGCTL2,
9448 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9449
9450
9451 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9452 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9453 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9454
9455 g4x_disable_trickle_feed(dev_priv);
9456
9457 gen7_setup_fixed_func_scheduler(dev_priv);
9458
9459 if (0) {
9460
9461 I915_WRITE(CACHE_MODE_0_GEN7,
9462 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
9463 }
9464
9465
9466 I915_WRITE(CACHE_MODE_1,
9467 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9468
9469
9470
9471
9472
9473
9474
9475
9476
9477 I915_WRITE(GEN7_GT_MODE,
9478 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9479
9480 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
9481 snpcr &= ~GEN6_MBC_SNPCR_MASK;
9482 snpcr |= GEN6_MBC_SNPCR_MED;
9483 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
9484
9485 if (!HAS_PCH_NOP(dev_priv))
9486 cpt_init_clock_gating(dev_priv);
9487
9488 gen6_check_mch_setup(dev_priv);
9489 }
9490
9491 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
9492 {
9493
9494 I915_WRITE(_3D_CHICKEN3,
9495 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
9496
9497
9498 I915_WRITE(IVB_CHICKEN3,
9499 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9500 CHICKEN3_DGMG_DONE_FIX_DISABLE);
9501
9502
9503
9504 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9505 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
9506 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9507
9508
9509 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9510
9511
9512 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9513 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9514
9515
9516 I915_WRITE(GEN7_ROW_CHICKEN2,
9517 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9518
9519
9520 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9521 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9522 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9523
9524 gen7_setup_fixed_func_scheduler(dev_priv);
9525
9526
9527
9528
9529
9530 I915_WRITE(GEN6_UCGCTL2,
9531 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9532
9533
9534
9535
9536 I915_WRITE(GEN7_UCGCTL4,
9537 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
9538
9539
9540
9541
9542
9543 I915_WRITE(CACHE_MODE_1,
9544 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9545
9546
9547
9548
9549
9550
9551
9552
9553
9554 I915_WRITE(GEN7_GT_MODE,
9555 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9556
9557
9558
9559
9560
9561 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
9562
9563
9564
9565
9566
9567
9568 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
9569 }
9570
9571 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
9572 {
9573
9574
9575 I915_WRITE(GEN7_FF_THREAD_MODE,
9576 I915_READ(GEN7_FF_THREAD_MODE) &
9577 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
9578
9579
9580 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
9581 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
9582
9583
9584 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
9585 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
9586
9587
9588 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9589 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9590
9591
9592
9593
9594
9595
9596 gen8_set_l3sqc_credits(dev_priv, 38, 2);
9597 }
9598
9599 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
9600 {
9601 u32 dspclk_gate;
9602
9603 I915_WRITE(RENCLK_GATE_D1, 0);
9604 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
9605 GS_UNIT_CLOCK_GATE_DISABLE |
9606 CL_UNIT_CLOCK_GATE_DISABLE);
9607 I915_WRITE(RAMCLK_GATE_D, 0);
9608 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
9609 OVRUNIT_CLOCK_GATE_DISABLE |
9610 OVCUNIT_CLOCK_GATE_DISABLE;
9611 if (IS_GM45(dev_priv))
9612 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
9613 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
9614
9615
9616 I915_WRITE(CACHE_MODE_0,
9617 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
9618
9619
9620 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9621
9622 g4x_disable_trickle_feed(dev_priv);
9623 }
9624
9625 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
9626 {
9627 struct intel_uncore *uncore = &dev_priv->uncore;
9628
9629 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
9630 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
9631 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
9632 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
9633 intel_uncore_write16(uncore, DEUC, 0);
9634 intel_uncore_write(uncore,
9635 MI_ARB_STATE,
9636 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9637
9638
9639 intel_uncore_write(uncore,
9640 CACHE_MODE_0,
9641 _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9642 }
9643
9644 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
9645 {
9646 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
9647 I965_RCC_CLOCK_GATE_DISABLE |
9648 I965_RCPB_CLOCK_GATE_DISABLE |
9649 I965_ISC_CLOCK_GATE_DISABLE |
9650 I965_FBC_CLOCK_GATE_DISABLE);
9651 I915_WRITE(RENCLK_GATE_D2, 0);
9652 I915_WRITE(MI_ARB_STATE,
9653 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9654
9655
9656 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9657 }
9658
9659 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
9660 {
9661 u32 dstate = I915_READ(D_STATE);
9662
9663 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
9664 DSTATE_DOT_CLOCK_GATING;
9665 I915_WRITE(D_STATE, dstate);
9666
9667 if (IS_PINEVIEW(dev_priv))
9668 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
9669
9670
9671 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
9672
9673
9674 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
9675
9676
9677 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
9678
9679 I915_WRITE(MI_ARB_STATE,
9680 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9681 }
9682
9683 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
9684 {
9685 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
9686
9687
9688 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
9689 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
9690
9691 I915_WRITE(MEM_MODE,
9692 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
9693 }
9694
9695 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
9696 {
9697 I915_WRITE(MEM_MODE,
9698 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
9699 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
9700 }
9701
9702 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
9703 {
9704 dev_priv->display.init_clock_gating(dev_priv);
9705 }
9706
9707 void intel_suspend_hw(struct drm_i915_private *dev_priv)
9708 {
9709 if (HAS_PCH_LPT(dev_priv))
9710 lpt_suspend_hw(dev_priv);
9711 }
9712
9713 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
9714 {
9715 DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
9716 }
9717
9718
9719
9720
9721
9722
9723
9724
9725
9726
9727 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
9728 {
9729 if (IS_GEN(dev_priv, 12))
9730 dev_priv->display.init_clock_gating = nop_init_clock_gating;
9731 else if (IS_GEN(dev_priv, 11))
9732 dev_priv->display.init_clock_gating = icl_init_clock_gating;
9733 else if (IS_CANNONLAKE(dev_priv))
9734 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
9735 else if (IS_COFFEELAKE(dev_priv))
9736 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
9737 else if (IS_SKYLAKE(dev_priv))
9738 dev_priv->display.init_clock_gating = skl_init_clock_gating;
9739 else if (IS_KABYLAKE(dev_priv))
9740 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
9741 else if (IS_BROXTON(dev_priv))
9742 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
9743 else if (IS_GEMINILAKE(dev_priv))
9744 dev_priv->display.init_clock_gating = glk_init_clock_gating;
9745 else if (IS_BROADWELL(dev_priv))
9746 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
9747 else if (IS_CHERRYVIEW(dev_priv))
9748 dev_priv->display.init_clock_gating = chv_init_clock_gating;
9749 else if (IS_HASWELL(dev_priv))
9750 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
9751 else if (IS_IVYBRIDGE(dev_priv))
9752 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
9753 else if (IS_VALLEYVIEW(dev_priv))
9754 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
9755 else if (IS_GEN(dev_priv, 6))
9756 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
9757 else if (IS_GEN(dev_priv, 5))
9758 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
9759 else if (IS_G4X(dev_priv))
9760 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
9761 else if (IS_I965GM(dev_priv))
9762 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
9763 else if (IS_I965G(dev_priv))
9764 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
9765 else if (IS_GEN(dev_priv, 3))
9766 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
9767 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
9768 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
9769 else if (IS_GEN(dev_priv, 2))
9770 dev_priv->display.init_clock_gating = i830_init_clock_gating;
9771 else {
9772 MISSING_CASE(INTEL_DEVID(dev_priv));
9773 dev_priv->display.init_clock_gating = nop_init_clock_gating;
9774 }
9775 }
9776
9777
9778 void intel_init_pm(struct drm_i915_private *dev_priv)
9779 {
9780
9781 if (IS_PINEVIEW(dev_priv))
9782 i915_pineview_get_mem_freq(dev_priv);
9783 else if (IS_GEN(dev_priv, 5))
9784 i915_ironlake_get_mem_freq(dev_priv);
9785
9786
9787 if (INTEL_GEN(dev_priv) >= 9) {
9788 skl_setup_wm_latency(dev_priv);
9789 dev_priv->display.initial_watermarks = skl_initial_wm;
9790 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
9791 dev_priv->display.compute_global_watermarks = skl_compute_wm;
9792 } else if (HAS_PCH_SPLIT(dev_priv)) {
9793 ilk_setup_wm_latency(dev_priv);
9794
9795 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
9796 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
9797 (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
9798 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
9799 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
9800 dev_priv->display.compute_intermediate_wm =
9801 ilk_compute_intermediate_wm;
9802 dev_priv->display.initial_watermarks =
9803 ilk_initial_watermarks;
9804 dev_priv->display.optimize_watermarks =
9805 ilk_optimize_watermarks;
9806 } else {
9807 DRM_DEBUG_KMS("Failed to read display plane latency. "
9808 "Disable CxSR\n");
9809 }
9810 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9811 vlv_setup_wm_latency(dev_priv);
9812 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
9813 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
9814 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
9815 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
9816 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
9817 } else if (IS_G4X(dev_priv)) {
9818 g4x_setup_wm_latency(dev_priv);
9819 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
9820 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
9821 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
9822 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
9823 } else if (IS_PINEVIEW(dev_priv)) {
9824 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
9825 dev_priv->is_ddr3,
9826 dev_priv->fsb_freq,
9827 dev_priv->mem_freq)) {
9828 DRM_INFO("failed to find known CxSR latency "
9829 "(found ddr%s fsb freq %d, mem freq %d), "
9830 "disabling CxSR\n",
9831 (dev_priv->is_ddr3 == 1) ? "3" : "2",
9832 dev_priv->fsb_freq, dev_priv->mem_freq);
9833
9834 intel_set_memory_cxsr(dev_priv, false);
9835 dev_priv->display.update_wm = NULL;
9836 } else
9837 dev_priv->display.update_wm = pineview_update_wm;
9838 } else if (IS_GEN(dev_priv, 4)) {
9839 dev_priv->display.update_wm = i965_update_wm;
9840 } else if (IS_GEN(dev_priv, 3)) {
9841 dev_priv->display.update_wm = i9xx_update_wm;
9842 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
9843 } else if (IS_GEN(dev_priv, 2)) {
9844 if (INTEL_INFO(dev_priv)->num_pipes == 1) {
9845 dev_priv->display.update_wm = i845_update_wm;
9846 dev_priv->display.get_fifo_size = i845_get_fifo_size;
9847 } else {
9848 dev_priv->display.update_wm = i9xx_update_wm;
9849 dev_priv->display.get_fifo_size = i830_get_fifo_size;
9850 }
9851 } else {
9852 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
9853 }
9854 }
9855
9856 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
9857 {
9858 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9859
9860
9861
9862
9863
9864 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
9865 }
9866
9867 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
9868 {
9869 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9870
9871 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
9872 }
9873
9874 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
9875 {
9876 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9877
9878
9879
9880
9881
9882 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
9883 }
9884
9885 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
9886 {
9887 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9888
9889
9890 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
9891 }
9892
9893 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
9894 {
9895 if (INTEL_GEN(dev_priv) >= 9)
9896 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
9897 GEN9_FREQ_SCALER);
9898 else if (IS_CHERRYVIEW(dev_priv))
9899 return chv_gpu_freq(dev_priv, val);
9900 else if (IS_VALLEYVIEW(dev_priv))
9901 return byt_gpu_freq(dev_priv, val);
9902 else
9903 return val * GT_FREQUENCY_MULTIPLIER;
9904 }
9905
9906 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
9907 {
9908 if (INTEL_GEN(dev_priv) >= 9)
9909 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
9910 GT_FREQUENCY_MULTIPLIER);
9911 else if (IS_CHERRYVIEW(dev_priv))
9912 return chv_freq_opcode(dev_priv, val);
9913 else if (IS_VALLEYVIEW(dev_priv))
9914 return byt_freq_opcode(dev_priv, val);
9915 else
9916 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
9917 }
9918
9919 void intel_pm_setup(struct drm_i915_private *dev_priv)
9920 {
9921 mutex_init(&dev_priv->gt_pm.rps.lock);
9922 mutex_init(&dev_priv->gt_pm.rps.power.mutex);
9923
9924 atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
9925
9926 dev_priv->runtime_pm.suspended = false;
9927 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
9928 }
9929
9930 static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
9931 const i915_reg_t reg)
9932 {
9933 u32 lower, upper, tmp;
9934 int loop = 2;
9935
9936
9937
9938
9939
9940 lockdep_assert_held(&dev_priv->uncore.lock);
9941
9942
9943
9944
9945
9946
9947
9948
9949
9950
9951
9952 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9953 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9954 upper = I915_READ_FW(reg);
9955 do {
9956 tmp = upper;
9957
9958 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9959 _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
9960 lower = I915_READ_FW(reg);
9961
9962 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9963 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9964 upper = I915_READ_FW(reg);
9965 } while (upper != tmp && --loop);
9966
9967
9968
9969
9970
9971
9972
9973 return lower | (u64)upper << 8;
9974 }
9975
9976 u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
9977 const i915_reg_t reg)
9978 {
9979 struct intel_uncore *uncore = &dev_priv->uncore;
9980 u64 time_hw, prev_hw, overflow_hw;
9981 unsigned int fw_domains;
9982 unsigned long flags;
9983 unsigned int i;
9984 u32 mul, div;
9985
9986 if (!HAS_RC6(dev_priv))
9987 return 0;
9988
9989
9990
9991
9992
9993
9994
9995
9996 i = (i915_mmio_reg_offset(reg) -
9997 i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32);
9998 if (WARN_ON_ONCE(i >= ARRAY_SIZE(dev_priv->gt_pm.rc6.cur_residency)))
9999 return 0;
10000
10001 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
10002
10003 spin_lock_irqsave(&uncore->lock, flags);
10004 intel_uncore_forcewake_get__locked(uncore, fw_domains);
10005
10006
10007 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
10008 mul = 1000000;
10009 div = dev_priv->czclk_freq;
10010 overflow_hw = BIT_ULL(40);
10011 time_hw = vlv_residency_raw(dev_priv, reg);
10012 } else {
10013
10014 if (IS_GEN9_LP(dev_priv)) {
10015 mul = 10000;
10016 div = 12;
10017 } else {
10018 mul = 1280;
10019 div = 1;
10020 }
10021
10022 overflow_hw = BIT_ULL(32);
10023 time_hw = intel_uncore_read_fw(uncore, reg);
10024 }
10025
10026
10027
10028
10029
10030
10031
10032 prev_hw = dev_priv->gt_pm.rc6.prev_hw_residency[i];
10033 dev_priv->gt_pm.rc6.prev_hw_residency[i] = time_hw;
10034
10035
10036 if (time_hw >= prev_hw)
10037 time_hw -= prev_hw;
10038 else
10039 time_hw += overflow_hw - prev_hw;
10040
10041
10042 time_hw += dev_priv->gt_pm.rc6.cur_residency[i];
10043 dev_priv->gt_pm.rc6.cur_residency[i] = time_hw;
10044
10045 intel_uncore_forcewake_put__locked(uncore, fw_domains);
10046 spin_unlock_irqrestore(&uncore->lock, flags);
10047
10048 return mul_u64_u32_div(time_hw, mul, div);
10049 }
10050
10051 u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
10052 i915_reg_t reg)
10053 {
10054 return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(dev_priv, reg), 1000);
10055 }
10056
10057 u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
10058 {
10059 u32 cagf;
10060
10061 if (INTEL_GEN(dev_priv) >= 9)
10062 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
10063 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
10064 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
10065 else
10066 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
10067
10068 return cagf;
10069 }