Lines Matching refs:dev_priv
57 struct drm_i915_private *dev_priv = dev->dev_private; in bxt_init_clock_gating() local
73 struct drm_i915_private *dev_priv = dev->dev_private; in i915_pineview_get_mem_freq() local
80 dev_priv->fsb_freq = 533; /* 133*4 */ in i915_pineview_get_mem_freq()
83 dev_priv->fsb_freq = 800; /* 200*4 */ in i915_pineview_get_mem_freq()
86 dev_priv->fsb_freq = 667; /* 167*4 */ in i915_pineview_get_mem_freq()
89 dev_priv->fsb_freq = 400; /* 100*4 */ in i915_pineview_get_mem_freq()
95 dev_priv->mem_freq = 533; in i915_pineview_get_mem_freq()
98 dev_priv->mem_freq = 667; in i915_pineview_get_mem_freq()
101 dev_priv->mem_freq = 800; in i915_pineview_get_mem_freq()
107 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; in i915_pineview_get_mem_freq()
112 struct drm_i915_private *dev_priv = dev->dev_private; in i915_ironlake_get_mem_freq() local
120 dev_priv->mem_freq = 800; in i915_ironlake_get_mem_freq()
123 dev_priv->mem_freq = 1066; in i915_ironlake_get_mem_freq()
126 dev_priv->mem_freq = 1333; in i915_ironlake_get_mem_freq()
129 dev_priv->mem_freq = 1600; in i915_ironlake_get_mem_freq()
134 dev_priv->mem_freq = 0; in i915_ironlake_get_mem_freq()
138 dev_priv->ips.r_t = dev_priv->mem_freq; in i915_ironlake_get_mem_freq()
142 dev_priv->fsb_freq = 3200; in i915_ironlake_get_mem_freq()
145 dev_priv->fsb_freq = 3733; in i915_ironlake_get_mem_freq()
148 dev_priv->fsb_freq = 4266; in i915_ironlake_get_mem_freq()
151 dev_priv->fsb_freq = 4800; in i915_ironlake_get_mem_freq()
154 dev_priv->fsb_freq = 5333; in i915_ironlake_get_mem_freq()
157 dev_priv->fsb_freq = 5866; in i915_ironlake_get_mem_freq()
160 dev_priv->fsb_freq = 6400; in i915_ironlake_get_mem_freq()
165 dev_priv->fsb_freq = 0; in i915_ironlake_get_mem_freq()
169 if (dev_priv->fsb_freq == 3200) { in i915_ironlake_get_mem_freq()
170 dev_priv->ips.c_m = 0; in i915_ironlake_get_mem_freq()
171 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) { in i915_ironlake_get_mem_freq()
172 dev_priv->ips.c_m = 1; in i915_ironlake_get_mem_freq()
174 dev_priv->ips.c_m = 2; in i915_ironlake_get_mem_freq()
240 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable) in chv_set_memory_dvfs() argument
244 mutex_lock(&dev_priv->rps.hw_lock); in chv_set_memory_dvfs()
246 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); in chv_set_memory_dvfs()
253 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); in chv_set_memory_dvfs()
255 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & in chv_set_memory_dvfs()
259 mutex_unlock(&dev_priv->rps.hw_lock); in chv_set_memory_dvfs()
262 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) in chv_set_memory_pm5() argument
266 mutex_lock(&dev_priv->rps.hw_lock); in chv_set_memory_pm5()
268 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); in chv_set_memory_pm5()
273 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val); in chv_set_memory_pm5()
275 mutex_unlock(&dev_priv->rps.hw_lock); in chv_set_memory_pm5()
281 void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) in intel_set_memory_cxsr() argument
283 struct drm_device *dev = dev_priv->dev; in intel_set_memory_cxsr()
289 dev_priv->wm.vlv.cxsr = enable; in intel_set_memory_cxsr()
339 struct drm_i915_private *dev_priv = dev->dev_private; in vlv_get_fifo_size() local
390 struct drm_i915_private *dev_priv = dev->dev_private; in i9xx_get_fifo_size() local
406 struct drm_i915_private *dev_priv = dev->dev_private; in i830_get_fifo_size() local
423 struct drm_i915_private *dev_priv = dev->dev_private; in i845_get_fifo_size() local
616 struct drm_i915_private *dev_priv = dev->dev_private; in pineview_update_wm() local
622 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3, in pineview_update_wm()
623 dev_priv->fsb_freq, dev_priv->mem_freq); in pineview_update_wm()
626 intel_set_memory_cxsr(dev_priv, false); in pineview_update_wm()
674 intel_set_memory_cxsr(dev_priv, true); in pineview_update_wm()
676 intel_set_memory_cxsr(dev_priv, false); in pineview_update_wm()
822 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); in vlv_write_wm_values() local
843 if (IS_CHERRYVIEW(dev_priv)) { in vlv_write_wm_values()
913 struct drm_i915_private *dev_priv = dev->dev_private; in vlv_setup_wm_latency() local
916 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; in vlv_setup_wm_latency()
918 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2; in vlv_setup_wm_latency()
920 if (IS_CHERRYVIEW(dev_priv)) { in vlv_setup_wm_latency()
921 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12; in vlv_setup_wm_latency()
922 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33; in vlv_setup_wm_latency()
924 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS; in vlv_setup_wm_latency()
933 struct drm_i915_private *dev_priv = to_i915(plane->base.dev); in vlv_compute_wm_level() local
936 if (dev_priv->wm.pri_latency[level] == 0) in vlv_compute_wm_level()
959 dev_priv->wm.pri_latency[level] * 10); in vlv_compute_wm_level()
1171 struct drm_i915_private *dev_priv = to_i915(dev); in vlv_pipe_set_fifo_size() local
1304 struct drm_i915_private *dev_priv = dev->dev_private; in vlv_update_wm() local
1312 if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) { in vlv_update_wm()
1319 dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS) in vlv_update_wm()
1320 chv_set_memory_dvfs(dev_priv, false); in vlv_update_wm()
1323 dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5) in vlv_update_wm()
1324 chv_set_memory_pm5(dev_priv, false); in vlv_update_wm()
1326 if (!wm.cxsr && dev_priv->wm.vlv.cxsr) in vlv_update_wm()
1327 intel_set_memory_cxsr(dev_priv, false); in vlv_update_wm()
1340 if (wm.cxsr && !dev_priv->wm.vlv.cxsr) in vlv_update_wm()
1341 intel_set_memory_cxsr(dev_priv, true); in vlv_update_wm()
1344 dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5) in vlv_update_wm()
1345 chv_set_memory_pm5(dev_priv, true); in vlv_update_wm()
1348 dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS) in vlv_update_wm()
1349 chv_set_memory_dvfs(dev_priv, true); in vlv_update_wm()
1351 dev_priv->wm.vlv = wm; in vlv_update_wm()
1360 struct drm_i915_private *dev_priv = dev->dev_private; in g4x_update_wm() local
1387 intel_set_memory_cxsr(dev_priv, false); in g4x_update_wm()
1411 intel_set_memory_cxsr(dev_priv, true); in g4x_update_wm()
1417 struct drm_i915_private *dev_priv = dev->dev_private; in i965_update_wm() local
1466 intel_set_memory_cxsr(dev_priv, false); in i965_update_wm()
1483 intel_set_memory_cxsr(dev_priv, true); in i965_update_wm()
1491 struct drm_i915_private *dev_priv = dev->dev_private; in i9xx_update_wm() local
1507 fifo_size = dev_priv->display.get_fifo_size(dev, 0); in i9xx_update_wm()
1529 fifo_size = dev_priv->display.get_fifo_size(dev, 1); in i9xx_update_wm()
1569 intel_set_memory_cxsr(dev_priv, false); in i9xx_update_wm()
1615 intel_set_memory_cxsr(dev_priv, true); in i9xx_update_wm()
1621 struct drm_i915_private *dev_priv = dev->dev_private; in i845_update_wm() local
1634 dev_priv->display.get_fifo_size(dev, 0), in i845_update_wm()
1978 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, in ilk_compute_wm_level() argument
1985 uint16_t pri_latency = dev_priv->wm.pri_latency[level]; in ilk_compute_wm_level()
1986 uint16_t spr_latency = dev_priv->wm.spr_latency[level]; in ilk_compute_wm_level()
1987 uint16_t cur_latency = dev_priv->wm.cur_latency[level]; in ilk_compute_wm_level()
1996 for_each_intel_plane_on_crtc(dev_priv->dev, intel_crtc, intel_plane) { in ilk_compute_wm_level()
2025 struct drm_i915_private *dev_priv = dev->dev_private; in hsw_compute_linetime_wm() local
2039 dev_priv->cdclk_freq); in hsw_compute_linetime_wm()
2047 struct drm_i915_private *dev_priv = dev->dev_private; in intel_read_wm_latency() local
2056 mutex_lock(&dev_priv->rps.hw_lock); in intel_read_wm_latency()
2057 ret = sandybridge_pcode_read(dev_priv, in intel_read_wm_latency()
2060 mutex_unlock(&dev_priv->rps.hw_lock); in intel_read_wm_latency()
2077 mutex_lock(&dev_priv->rps.hw_lock); in intel_read_wm_latency()
2078 ret = sandybridge_pcode_read(dev_priv, in intel_read_wm_latency()
2081 mutex_unlock(&dev_priv->rps.hw_lock); in intel_read_wm_latency()
2210 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, in ilk_increase_wm_latency() argument
2213 int level, max_level = ilk_wm_max_level(dev_priv->dev); in ilk_increase_wm_latency()
2227 struct drm_i915_private *dev_priv = dev->dev_private; in snb_wm_latency_quirk() local
2234 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) | in snb_wm_latency_quirk()
2235 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) | in snb_wm_latency_quirk()
2236 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12); in snb_wm_latency_quirk()
2242 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); in snb_wm_latency_quirk()
2243 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); in snb_wm_latency_quirk()
2244 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); in snb_wm_latency_quirk()
2249 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_setup_wm_latency() local
2251 intel_read_wm_latency(dev, dev_priv->wm.pri_latency); in ilk_setup_wm_latency()
2253 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency, in ilk_setup_wm_latency()
2254 sizeof(dev_priv->wm.pri_latency)); in ilk_setup_wm_latency()
2255 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency, in ilk_setup_wm_latency()
2256 sizeof(dev_priv->wm.pri_latency)); in ilk_setup_wm_latency()
2258 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency); in ilk_setup_wm_latency()
2259 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency); in ilk_setup_wm_latency()
2261 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); in ilk_setup_wm_latency()
2262 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); in ilk_setup_wm_latency()
2263 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); in ilk_setup_wm_latency()
2271 struct drm_i915_private *dev_priv = dev->dev_private; in skl_setup_wm_latency() local
2273 intel_read_wm_latency(dev, dev_priv->wm.skl_latency); in skl_setup_wm_latency()
2274 intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency); in skl_setup_wm_latency()
2301 const struct drm_i915_private *dev_priv = dev->dev_private; in intel_compute_pipe_wm() local
2336 ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate, &pipe_wm->wm[0]); in intel_compute_pipe_wm()
2353 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate, &wm); in intel_compute_pipe_wm()
2410 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_wm_merge() local
2452 intel_fbc_enabled(dev_priv)) { in ilk_wm_merge()
2470 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_wm_lp_latency() local
2475 return dev_priv->wm.pri_latency[level]; in ilk_wm_lp_latency()
2581 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, in ilk_compute_wm_dirty() argument
2589 for_each_pipe(dev_priv, pipe) { in ilk_compute_wm_dirty()
2633 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, in _ilk_disable_lp_wm() argument
2636 struct ilk_wm_values *previous = &dev_priv->wm.hw; in _ilk_disable_lp_wm()
2667 static void ilk_write_wm_values(struct drm_i915_private *dev_priv, in ilk_write_wm_values() argument
2670 struct drm_device *dev = dev_priv->dev; in ilk_write_wm_values()
2671 struct ilk_wm_values *previous = &dev_priv->wm.hw; in ilk_write_wm_values()
2675 dirty = ilk_compute_wm_dirty(dev_priv, previous, results); in ilk_write_wm_values()
2679 _ilk_disable_lp_wm(dev_priv, dirty); in ilk_write_wm_values()
2740 dev_priv->wm.hw = *results; in ilk_write_wm_values()
2745 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_disable_lp_wm() local
2747 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); in ilk_disable_lp_wm()
2814 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, in skl_ddb_get_hw_state() argument
2823 for_each_pipe(dev_priv, pipe) { in skl_ddb_get_hw_state()
2824 if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PIPE(pipe))) in skl_ddb_get_hw_state()
2827 for_each_plane(dev_priv, pipe, plane) { in skl_ddb_get_hw_state()
2890 struct drm_i915_private *dev_priv = dev->dev_private; in skl_allocate_pipe_ddb() local
2917 for_each_plane(dev_priv, pipe, plane) { in skl_allocate_pipe_ddb()
3042 struct drm_i915_private *dev_priv = dev->dev_private; in skl_ddb_allocation_changed() local
3043 const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb; in skl_ddb_allocation_changed()
3134 static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, in skl_compute_plane_wm() argument
3142 uint32_t latency = dev_priv->wm.skl_latency[level]; in skl_compute_plane_wm()
3213 static void skl_compute_wm_level(const struct drm_i915_private *dev_priv, in skl_compute_wm_level() argument
3227 result->plane_en[i] = skl_compute_plane_wm(dev_priv, in skl_compute_wm_level()
3236 result->plane_en[PLANE_CURSOR] = skl_compute_plane_wm(dev_priv, p, in skl_compute_wm_level()
3277 const struct drm_i915_private *dev_priv = dev->dev_private; in skl_compute_pipe_wm() local
3282 skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe, in skl_compute_pipe_wm()
3349 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg, in skl_ddb_entry_write() argument
3358 static void skl_write_wm_values(struct drm_i915_private *dev_priv, in skl_write_wm_values() argument
3361 struct drm_device *dev = dev_priv->dev; in skl_write_wm_values()
3387 skl_ddb_entry_write(dev_priv, in skl_write_wm_values()
3390 skl_ddb_entry_write(dev_priv, in skl_write_wm_values()
3395 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), in skl_write_wm_values()
3425 skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass) in skl_wm_flush_pipe() argument
3431 for_each_plane(dev_priv, pipe, plane) { in skl_wm_flush_pipe()
3453 static void skl_flush_wm_values(struct drm_i915_private *dev_priv, in skl_flush_wm_values() argument
3456 struct drm_device *dev = dev_priv->dev; in skl_flush_wm_values()
3463 cur_ddb = &dev_priv->wm.skl_hw.ddb; in skl_flush_wm_values()
3481 skl_wm_flush_pipe(dev_priv, pipe, 1); in skl_flush_wm_values()
3506 skl_wm_flush_pipe(dev_priv, pipe, 2); in skl_flush_wm_values()
3531 skl_wm_flush_pipe(dev_priv, pipe, 3); in skl_flush_wm_values()
3628 struct drm_i915_private *dev_priv = dev->dev_private; in skl_update_wm() local
3630 struct skl_wm_values *results = &dev_priv->wm.skl_results; in skl_update_wm()
3650 skl_write_wm_values(dev_priv, results); in skl_update_wm()
3651 skl_flush_wm_values(dev_priv, results); in skl_update_wm()
3654 dev_priv->wm.skl_hw = *results; in skl_update_wm()
3695 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_update_wm() local
3733 ilk_write_wm_values(dev_priv, &results); in ilk_update_wm()
3805 struct drm_i915_private *dev_priv = dev->dev_private; in skl_pipe_wm_get_hw_state() local
3806 struct skl_wm_values *hw = &dev_priv->wm.skl_hw; in skl_pipe_wm_get_hw_state()
3856 struct drm_i915_private *dev_priv = dev->dev_private; in skl_wm_get_hw_state() local
3857 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb; in skl_wm_get_hw_state()
3860 skl_ddb_get_hw_state(dev_priv, ddb); in skl_wm_get_hw_state()
3868 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_pipe_wm_get_hw_state() local
3869 struct ilk_wm_values *hw = &dev_priv->wm.hw; in ilk_pipe_wm_get_hw_state()
3919 static void vlv_read_wm_values(struct drm_i915_private *dev_priv, in vlv_read_wm_values() argument
3925 for_each_pipe(dev_priv, pipe) { in vlv_read_wm_values()
3952 if (IS_CHERRYVIEW(dev_priv)) { in vlv_read_wm_values()
3997 struct drm_i915_private *dev_priv = to_i915(dev); in vlv_wm_get_hw_state() local
3998 struct vlv_wm_values *wm = &dev_priv->wm.vlv; in vlv_wm_get_hw_state()
4003 vlv_read_wm_values(dev_priv, wm); in vlv_wm_get_hw_state()
4024 if (IS_CHERRYVIEW(dev_priv)) { in vlv_wm_get_hw_state()
4025 mutex_lock(&dev_priv->rps.hw_lock); in vlv_wm_get_hw_state()
4027 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); in vlv_wm_get_hw_state()
4040 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); in vlv_wm_get_hw_state()
4042 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); in vlv_wm_get_hw_state()
4044 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & in vlv_wm_get_hw_state()
4048 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5; in vlv_wm_get_hw_state()
4050 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); in vlv_wm_get_hw_state()
4055 mutex_unlock(&dev_priv->rps.hw_lock); in vlv_wm_get_hw_state()
4058 for_each_pipe(dev_priv, pipe) in vlv_wm_get_hw_state()
4069 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_wm_get_hw_state() local
4070 struct ilk_wm_values *hw = &dev_priv->wm.hw; in ilk_wm_get_hw_state()
4131 struct drm_i915_private *dev_priv = crtc->dev->dev_private; in intel_update_watermarks() local
4133 if (dev_priv->display.update_wm) in intel_update_watermarks()
4134 dev_priv->display.update_wm(crtc); in intel_update_watermarks()
4144 struct drm_i915_private *dev_priv = plane->dev->dev_private; in intel_update_sprite_watermarks() local
4146 if (dev_priv->display.update_sprite_wm) in intel_update_sprite_watermarks()
4147 dev_priv->display.update_sprite_wm(plane, crtc, in intel_update_sprite_watermarks()
4163 struct drm_i915_private *dev_priv = dev->dev_private; in ironlake_set_drps() local
4187 struct drm_i915_private *dev_priv = dev->dev_private; in ironlake_enable_drps() local
4216 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */ in ironlake_enable_drps()
4217 dev_priv->ips.fstart = fstart; in ironlake_enable_drps()
4219 dev_priv->ips.max_delay = fstart; in ironlake_enable_drps()
4220 dev_priv->ips.min_delay = fmin; in ironlake_enable_drps()
4221 dev_priv->ips.cur_delay = fstart; in ironlake_enable_drps()
4244 dev_priv->ips.last_count1 = I915_READ(DMIEC) + in ironlake_enable_drps()
4246 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies); in ironlake_enable_drps()
4247 dev_priv->ips.last_count2 = I915_READ(GFXEC); in ironlake_enable_drps()
4248 dev_priv->ips.last_time2 = ktime_get_raw_ns(); in ironlake_enable_drps()
4255 struct drm_i915_private *dev_priv = dev->dev_private; in ironlake_disable_drps() local
4270 ironlake_set_drps(dev, dev_priv->ips.fstart); in ironlake_disable_drps()
4284 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val) in intel_rps_limits() argument
4294 if (IS_GEN9(dev_priv->dev)) { in intel_rps_limits()
4295 limits = (dev_priv->rps.max_freq_softlimit) << 23; in intel_rps_limits()
4296 if (val <= dev_priv->rps.min_freq_softlimit) in intel_rps_limits()
4297 limits |= (dev_priv->rps.min_freq_softlimit) << 14; in intel_rps_limits()
4299 limits = dev_priv->rps.max_freq_softlimit << 24; in intel_rps_limits()
4300 if (val <= dev_priv->rps.min_freq_softlimit) in intel_rps_limits()
4301 limits |= dev_priv->rps.min_freq_softlimit << 16; in intel_rps_limits()
4307 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) in gen6_set_rps_thresholds() argument
4313 new_power = dev_priv->rps.power; in gen6_set_rps_thresholds()
4314 switch (dev_priv->rps.power) { in gen6_set_rps_thresholds()
4316 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq) in gen6_set_rps_thresholds()
4321 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq) in gen6_set_rps_thresholds()
4323 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq) in gen6_set_rps_thresholds()
4328 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq) in gen6_set_rps_thresholds()
4333 if (val <= dev_priv->rps.min_freq_softlimit) in gen6_set_rps_thresholds()
4335 if (val >= dev_priv->rps.max_freq_softlimit) in gen6_set_rps_thresholds()
4337 if (new_power == dev_priv->rps.power) in gen6_set_rps_thresholds()
4374 GT_INTERVAL_FROM_US(dev_priv, ei_up)); in gen6_set_rps_thresholds()
4376 GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100))); in gen6_set_rps_thresholds()
4379 GT_INTERVAL_FROM_US(dev_priv, ei_down)); in gen6_set_rps_thresholds()
4381 GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100))); in gen6_set_rps_thresholds()
4391 dev_priv->rps.power = new_power; in gen6_set_rps_thresholds()
4392 dev_priv->rps.up_threshold = threshold_up; in gen6_set_rps_thresholds()
4393 dev_priv->rps.down_threshold = threshold_down; in gen6_set_rps_thresholds()
4394 dev_priv->rps.last_adj = 0; in gen6_set_rps_thresholds()
4397 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) in gen6_rps_pm_mask() argument
4401 if (val > dev_priv->rps.min_freq_softlimit) in gen6_rps_pm_mask()
4403 if (val < dev_priv->rps.max_freq_softlimit) in gen6_rps_pm_mask()
4406 mask &= dev_priv->pm_rps_events; in gen6_rps_pm_mask()
4408 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask); in gen6_rps_pm_mask()
4416 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_set_rps() local
4422 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in gen6_set_rps()
4423 WARN_ON(val > dev_priv->rps.max_freq); in gen6_set_rps()
4424 WARN_ON(val < dev_priv->rps.min_freq); in gen6_set_rps()
4429 if (val != dev_priv->rps.cur_freq) { in gen6_set_rps()
4430 gen6_set_rps_thresholds(dev_priv, val); in gen6_set_rps()
4448 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val)); in gen6_set_rps()
4449 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); in gen6_set_rps()
4453 dev_priv->rps.cur_freq = val; in gen6_set_rps()
4454 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); in gen6_set_rps()
4459 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_set_rps() local
4461 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in valleyview_set_rps()
4462 WARN_ON(val > dev_priv->rps.max_freq); in valleyview_set_rps()
4463 WARN_ON(val < dev_priv->rps.min_freq); in valleyview_set_rps()
4469 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); in valleyview_set_rps()
4471 if (val != dev_priv->rps.cur_freq) { in valleyview_set_rps()
4472 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); in valleyview_set_rps()
4473 if (!IS_CHERRYVIEW(dev_priv)) in valleyview_set_rps()
4474 gen6_set_rps_thresholds(dev_priv, val); in valleyview_set_rps()
4477 dev_priv->rps.cur_freq = val; in valleyview_set_rps()
4478 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); in valleyview_set_rps()
4488 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) in vlv_set_rps_idle() argument
4490 u32 val = dev_priv->rps.idle_freq; in vlv_set_rps_idle()
4492 if (dev_priv->rps.cur_freq <= val) in vlv_set_rps_idle()
4497 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA); in vlv_set_rps_idle()
4498 valleyview_set_rps(dev_priv->dev, val); in vlv_set_rps_idle()
4499 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA); in vlv_set_rps_idle()
4502 void gen6_rps_busy(struct drm_i915_private *dev_priv) in gen6_rps_busy() argument
4504 mutex_lock(&dev_priv->rps.hw_lock); in gen6_rps_busy()
4505 if (dev_priv->rps.enabled) { in gen6_rps_busy()
4506 if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) in gen6_rps_busy()
4507 gen6_rps_reset_ei(dev_priv); in gen6_rps_busy()
4509 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq)); in gen6_rps_busy()
4511 mutex_unlock(&dev_priv->rps.hw_lock); in gen6_rps_busy()
4514 void gen6_rps_idle(struct drm_i915_private *dev_priv) in gen6_rps_idle() argument
4516 struct drm_device *dev = dev_priv->dev; in gen6_rps_idle()
4518 mutex_lock(&dev_priv->rps.hw_lock); in gen6_rps_idle()
4519 if (dev_priv->rps.enabled) { in gen6_rps_idle()
4521 vlv_set_rps_idle(dev_priv); in gen6_rps_idle()
4523 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); in gen6_rps_idle()
4524 dev_priv->rps.last_adj = 0; in gen6_rps_idle()
4527 mutex_unlock(&dev_priv->rps.hw_lock); in gen6_rps_idle()
4529 spin_lock(&dev_priv->rps.client_lock); in gen6_rps_idle()
4530 while (!list_empty(&dev_priv->rps.clients)) in gen6_rps_idle()
4531 list_del_init(dev_priv->rps.clients.next); in gen6_rps_idle()
4532 spin_unlock(&dev_priv->rps.client_lock); in gen6_rps_idle()
4535 void gen6_rps_boost(struct drm_i915_private *dev_priv, in gen6_rps_boost() argument
4542 if (!(dev_priv->mm.busy && in gen6_rps_boost()
4543 dev_priv->rps.enabled && in gen6_rps_boost()
4544 dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit)) in gen6_rps_boost()
4553 spin_lock(&dev_priv->rps.client_lock); in gen6_rps_boost()
4555 spin_lock_irq(&dev_priv->irq_lock); in gen6_rps_boost()
4556 if (dev_priv->rps.interrupts_enabled) { in gen6_rps_boost()
4557 dev_priv->rps.client_boost = true; in gen6_rps_boost()
4558 queue_work(dev_priv->wq, &dev_priv->rps.work); in gen6_rps_boost()
4560 spin_unlock_irq(&dev_priv->irq_lock); in gen6_rps_boost()
4563 list_add(&rps->link, &dev_priv->rps.clients); in gen6_rps_boost()
4566 dev_priv->rps.boosts++; in gen6_rps_boost()
4568 spin_unlock(&dev_priv->rps.client_lock); in gen6_rps_boost()
4581 struct drm_i915_private *dev_priv = dev->dev_private; in gen9_disable_rps() local
4589 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_disable_rps() local
4597 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_disable_rps() local
4604 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_disable_rps() local
4608 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in valleyview_disable_rps()
4612 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in valleyview_disable_rps()
4670 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_init_rps_frequencies() local
4676 dev_priv->rps.cur_freq = 0; in gen6_init_rps_frequencies()
4680 dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff; in gen6_init_rps_frequencies()
4681 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; in gen6_init_rps_frequencies()
4682 dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff; in gen6_init_rps_frequencies()
4685 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff; in gen6_init_rps_frequencies()
4686 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; in gen6_init_rps_frequencies()
4687 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff; in gen6_init_rps_frequencies()
4691 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; in gen6_init_rps_frequencies()
4693 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; in gen6_init_rps_frequencies()
4695 ret = sandybridge_pcode_read(dev_priv, in gen6_init_rps_frequencies()
4699 dev_priv->rps.efficient_freq = in gen6_init_rps_frequencies()
4702 dev_priv->rps.min_freq, in gen6_init_rps_frequencies()
4703 dev_priv->rps.max_freq); in gen6_init_rps_frequencies()
4709 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER; in gen6_init_rps_frequencies()
4710 dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER; in gen6_init_rps_frequencies()
4711 dev_priv->rps.min_freq *= GEN9_FREQ_SCALER; in gen6_init_rps_frequencies()
4712 dev_priv->rps.max_freq *= GEN9_FREQ_SCALER; in gen6_init_rps_frequencies()
4713 dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER; in gen6_init_rps_frequencies()
4716 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; in gen6_init_rps_frequencies()
4719 if (dev_priv->rps.max_freq_softlimit == 0) in gen6_init_rps_frequencies()
4720 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; in gen6_init_rps_frequencies()
4722 if (dev_priv->rps.min_freq_softlimit == 0) { in gen6_init_rps_frequencies()
4724 dev_priv->rps.min_freq_softlimit = in gen6_init_rps_frequencies()
4725 max_t(int, dev_priv->rps.efficient_freq, in gen6_init_rps_frequencies()
4726 intel_freq_opcode(dev_priv, 450)); in gen6_init_rps_frequencies()
4728 dev_priv->rps.min_freq_softlimit = in gen6_init_rps_frequencies()
4729 dev_priv->rps.min_freq; in gen6_init_rps_frequencies()
4736 struct drm_i915_private *dev_priv = dev->dev_private; in gen9_enable_rps() local
4738 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in gen9_enable_rps()
4744 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen9_enable_rps()
4750 GEN9_FREQUENCY(dev_priv->rps.rp1_freq)); in gen9_enable_rps()
4754 GT_INTERVAL_FROM_US(dev_priv, 1000000)); in gen9_enable_rps()
4761 dev_priv->rps.power = HIGH_POWER; /* force a reset */ in gen9_enable_rps()
4762 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); in gen9_enable_rps()
4764 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen9_enable_rps()
4769 struct drm_i915_private *dev_priv = dev->dev_private; in gen9_enable_rc6() local
4779 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in gen9_enable_rc6()
4793 for_each_ring(ring, dev_priv, unused) in gen9_enable_rc6()
4835 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen9_enable_rc6()
4841 struct drm_i915_private *dev_priv = dev->dev_private; in gen8_enable_rps() local
4851 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in gen8_enable_rps()
4863 for_each_ring(ring, dev_priv, unused) in gen8_enable_rps()
4886 HSW_FREQUENCY(dev_priv->rps.rp1_freq)); in gen8_enable_rps()
4888 HSW_FREQUENCY(dev_priv->rps.rp1_freq)); in gen8_enable_rps()
4894 dev_priv->rps.max_freq_softlimit << 24 | in gen8_enable_rps()
4895 dev_priv->rps.min_freq_softlimit << 16); in gen8_enable_rps()
4915 dev_priv->rps.power = HIGH_POWER; /* force a reset */ in gen8_enable_rps()
4916 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); in gen8_enable_rps()
4918 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen8_enable_rps()
4923 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_enable_rps() local
4930 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in gen6_enable_rps()
4946 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in gen6_enable_rps()
4960 for_each_ring(ring, dev_priv, i) in gen6_enable_rps()
4973 rc6_mode = intel_enable_rc6(dev_priv->dev); in gen6_enable_rps()
4997 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0); in gen6_enable_rps()
5001 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox); in gen6_enable_rps()
5004 (dev_priv->rps.max_freq_softlimit & 0xff) * 50, in gen6_enable_rps()
5006 dev_priv->rps.max_freq = pcu_mbox & 0xff; in gen6_enable_rps()
5009 dev_priv->rps.power = HIGH_POWER; /* force a reset */ in gen6_enable_rps()
5010 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); in gen6_enable_rps()
5013 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids); in gen6_enable_rps()
5021 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); in gen6_enable_rps()
5026 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen6_enable_rps()
5031 struct drm_i915_private *dev_priv = dev->dev_private; in __gen6_update_ring_freq() local
5039 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in __gen6_update_ring_freq()
5062 min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER; in __gen6_update_ring_freq()
5063 max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER; in __gen6_update_ring_freq()
5065 min_gpu_freq = dev_priv->rps.min_freq; in __gen6_update_ring_freq()
5066 max_gpu_freq = dev_priv->rps.max_freq; in __gen6_update_ring_freq()
5106 sandybridge_pcode_write(dev_priv, in __gen6_update_ring_freq()
5116 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_update_ring_freq() local
5121 mutex_lock(&dev_priv->rps.hw_lock); in gen6_update_ring_freq()
5123 mutex_unlock(&dev_priv->rps.hw_lock); in gen6_update_ring_freq()
5126 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) in cherryview_rps_max_freq() argument
5128 struct drm_device *dev = dev_priv->dev; in cherryview_rps_max_freq()
5131 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); in cherryview_rps_max_freq()
5155 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv) in cherryview_rps_rpe_freq() argument
5159 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG); in cherryview_rps_rpe_freq()
5165 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv) in cherryview_rps_guar_freq() argument
5169 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); in cherryview_rps_guar_freq()
5175 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv) in valleyview_rps_guar_freq() argument
5179 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); in valleyview_rps_guar_freq()
5186 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv) in valleyview_rps_max_freq() argument
5190 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); in valleyview_rps_max_freq()
5199 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv) in valleyview_rps_rpe_freq() argument
5203 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO); in valleyview_rps_rpe_freq()
5205 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI); in valleyview_rps_rpe_freq()
5211 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv) in valleyview_rps_min_freq() argument
5213 return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff; in valleyview_rps_min_freq()
5217 static void valleyview_check_pctx(struct drm_i915_private *dev_priv) in valleyview_check_pctx() argument
5221 WARN_ON(pctx_addr != dev_priv->mm.stolen_base + in valleyview_check_pctx()
5222 dev_priv->vlv_pctx->stolen->start); in valleyview_check_pctx()
5227 static void cherryview_check_pctx(struct drm_i915_private *dev_priv) in cherryview_check_pctx() argument
5236 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_setup_pctx() local
5238 struct i915_gtt *gtt = &dev_priv->gtt; in cherryview_setup_pctx()
5247 paddr = (dev_priv->mm.stolen_base + in cherryview_setup_pctx()
5259 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_setup_pctx() local
5272 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base; in valleyview_setup_pctx()
5273 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev, in valleyview_setup_pctx()
5296 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start; in valleyview_setup_pctx()
5301 dev_priv->vlv_pctx = pctx; in valleyview_setup_pctx()
5306 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_cleanup_pctx() local
5308 if (WARN_ON(!dev_priv->vlv_pctx)) in valleyview_cleanup_pctx()
5311 drm_gem_object_unreference(&dev_priv->vlv_pctx->base); in valleyview_cleanup_pctx()
5312 dev_priv->vlv_pctx = NULL; in valleyview_cleanup_pctx()
5317 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_init_gt_powersave() local
5322 mutex_lock(&dev_priv->rps.hw_lock); in valleyview_init_gt_powersave()
5324 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); in valleyview_init_gt_powersave()
5328 dev_priv->mem_freq = 800; in valleyview_init_gt_powersave()
5331 dev_priv->mem_freq = 1066; in valleyview_init_gt_powersave()
5334 dev_priv->mem_freq = 1333; in valleyview_init_gt_powersave()
5337 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); in valleyview_init_gt_powersave()
5339 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv); in valleyview_init_gt_powersave()
5340 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; in valleyview_init_gt_powersave()
5342 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq), in valleyview_init_gt_powersave()
5343 dev_priv->rps.max_freq); in valleyview_init_gt_powersave()
5345 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv); in valleyview_init_gt_powersave()
5347 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), in valleyview_init_gt_powersave()
5348 dev_priv->rps.efficient_freq); in valleyview_init_gt_powersave()
5350 dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv); in valleyview_init_gt_powersave()
5352 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq), in valleyview_init_gt_powersave()
5353 dev_priv->rps.rp1_freq); in valleyview_init_gt_powersave()
5355 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv); in valleyview_init_gt_powersave()
5357 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq), in valleyview_init_gt_powersave()
5358 dev_priv->rps.min_freq); in valleyview_init_gt_powersave()
5360 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; in valleyview_init_gt_powersave()
5363 if (dev_priv->rps.max_freq_softlimit == 0) in valleyview_init_gt_powersave()
5364 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; in valleyview_init_gt_powersave()
5366 if (dev_priv->rps.min_freq_softlimit == 0) in valleyview_init_gt_powersave()
5367 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; in valleyview_init_gt_powersave()
5369 mutex_unlock(&dev_priv->rps.hw_lock); in valleyview_init_gt_powersave()
5374 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_init_gt_powersave() local
5379 mutex_lock(&dev_priv->rps.hw_lock); in cherryview_init_gt_powersave()
5381 mutex_lock(&dev_priv->sb_lock); in cherryview_init_gt_powersave()
5382 val = vlv_cck_read(dev_priv, CCK_FUSE_REG); in cherryview_init_gt_powersave()
5383 mutex_unlock(&dev_priv->sb_lock); in cherryview_init_gt_powersave()
5387 dev_priv->mem_freq = 2000; in cherryview_init_gt_powersave()
5390 dev_priv->mem_freq = 1600; in cherryview_init_gt_powersave()
5393 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); in cherryview_init_gt_powersave()
5395 dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv); in cherryview_init_gt_powersave()
5396 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; in cherryview_init_gt_powersave()
5398 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq), in cherryview_init_gt_powersave()
5399 dev_priv->rps.max_freq); in cherryview_init_gt_powersave()
5401 dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv); in cherryview_init_gt_powersave()
5403 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), in cherryview_init_gt_powersave()
5404 dev_priv->rps.efficient_freq); in cherryview_init_gt_powersave()
5406 dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv); in cherryview_init_gt_powersave()
5408 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq), in cherryview_init_gt_powersave()
5409 dev_priv->rps.rp1_freq); in cherryview_init_gt_powersave()
5412 dev_priv->rps.min_freq = dev_priv->rps.efficient_freq; in cherryview_init_gt_powersave()
5414 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq), in cherryview_init_gt_powersave()
5415 dev_priv->rps.min_freq); in cherryview_init_gt_powersave()
5417 WARN_ONCE((dev_priv->rps.max_freq | in cherryview_init_gt_powersave()
5418 dev_priv->rps.efficient_freq | in cherryview_init_gt_powersave()
5419 dev_priv->rps.rp1_freq | in cherryview_init_gt_powersave()
5420 dev_priv->rps.min_freq) & 1, in cherryview_init_gt_powersave()
5423 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; in cherryview_init_gt_powersave()
5426 if (dev_priv->rps.max_freq_softlimit == 0) in cherryview_init_gt_powersave()
5427 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; in cherryview_init_gt_powersave()
5429 if (dev_priv->rps.min_freq_softlimit == 0) in cherryview_init_gt_powersave()
5430 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; in cherryview_init_gt_powersave()
5432 mutex_unlock(&dev_priv->rps.hw_lock); in cherryview_init_gt_powersave()
5442 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_enable_rps() local
5447 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in cherryview_enable_rps()
5456 cherryview_check_pctx(dev_priv); in cherryview_enable_rps()
5460 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in cherryview_enable_rps()
5470 for_each_ring(ring, dev_priv, i) in cherryview_enable_rps()
5514 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val); in cherryview_enable_rps()
5516 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); in cherryview_enable_rps()
5524 dev_priv->rps.cur_freq = (val >> 8) & 0xff; in cherryview_enable_rps()
5526 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq), in cherryview_enable_rps()
5527 dev_priv->rps.cur_freq); in cherryview_enable_rps()
5530 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), in cherryview_enable_rps()
5531 dev_priv->rps.efficient_freq); in cherryview_enable_rps()
5533 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); in cherryview_enable_rps()
5535 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in cherryview_enable_rps()
5540 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_enable_rps() local
5545 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in valleyview_enable_rps()
5547 valleyview_check_pctx(dev_priv); in valleyview_enable_rps()
5556 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in valleyview_enable_rps()
5581 for_each_ring(ring, dev_priv, i) in valleyview_enable_rps()
5604 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val); in valleyview_enable_rps()
5606 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); in valleyview_enable_rps()
5614 dev_priv->rps.cur_freq = (val >> 8) & 0xff; in valleyview_enable_rps()
5616 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq), in valleyview_enable_rps()
5617 dev_priv->rps.cur_freq); in valleyview_enable_rps()
5620 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), in valleyview_enable_rps()
5621 dev_priv->rps.efficient_freq); in valleyview_enable_rps()
5623 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); in valleyview_enable_rps()
5625 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in valleyview_enable_rps()
5657 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) in __i915_chipset_val() argument
5666 diff1 = now - dev_priv->ips.last_time1; in __i915_chipset_val()
5674 return dev_priv->ips.chipset_power; in __i915_chipset_val()
5683 if (total_count < dev_priv->ips.last_count1) { in __i915_chipset_val()
5684 diff = ~0UL - dev_priv->ips.last_count1; in __i915_chipset_val()
5687 diff = total_count - dev_priv->ips.last_count1; in __i915_chipset_val()
5691 if (cparams[i].i == dev_priv->ips.c_m && in __i915_chipset_val()
5692 cparams[i].t == dev_priv->ips.r_t) { in __i915_chipset_val()
5703 dev_priv->ips.last_count1 = total_count; in __i915_chipset_val()
5704 dev_priv->ips.last_time1 = now; in __i915_chipset_val()
5706 dev_priv->ips.chipset_power = ret; in __i915_chipset_val()
5711 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) in i915_chipset_val() argument
5713 struct drm_device *dev = dev_priv->dev; in i915_chipset_val()
5721 val = __i915_chipset_val(dev_priv); in i915_chipset_val()
5728 unsigned long i915_mch_val(struct drm_i915_private *dev_priv) in i915_mch_val() argument
5754 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) in pvid_to_extvid() argument
5756 struct drm_device *dev = dev_priv->dev; in pvid_to_extvid()
5766 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) in __i915_update_gfx_val() argument
5774 diffms = now - dev_priv->ips.last_time2; in __i915_update_gfx_val()
5783 if (count < dev_priv->ips.last_count2) { in __i915_update_gfx_val()
5784 diff = ~0UL - dev_priv->ips.last_count2; in __i915_update_gfx_val()
5787 diff = count - dev_priv->ips.last_count2; in __i915_update_gfx_val()
5790 dev_priv->ips.last_count2 = count; in __i915_update_gfx_val()
5791 dev_priv->ips.last_time2 = now; in __i915_update_gfx_val()
5796 dev_priv->ips.gfx_power = diff; in __i915_update_gfx_val()
5799 void i915_update_gfx_val(struct drm_i915_private *dev_priv) in i915_update_gfx_val() argument
5801 struct drm_device *dev = dev_priv->dev; in i915_update_gfx_val()
5808 __i915_update_gfx_val(dev_priv); in i915_update_gfx_val()
5813 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) in __i915_gfx_val() argument
5820 pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq)); in __i915_gfx_val()
5822 ext_v = pvid_to_extvid(dev_priv, pxvid); in __i915_gfx_val()
5826 t = i915_mch_val(dev_priv); in __i915_gfx_val()
5840 corr2 = (corr * dev_priv->ips.corr); in __i915_gfx_val()
5845 __i915_update_gfx_val(dev_priv); in __i915_gfx_val()
5847 return dev_priv->ips.gfx_power + state2; in __i915_gfx_val()
5850 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) in i915_gfx_val() argument
5852 struct drm_device *dev = dev_priv->dev; in i915_gfx_val()
5860 val = __i915_gfx_val(dev_priv); in i915_gfx_val()
5875 struct drm_i915_private *dev_priv; in i915_read_mch_val() local
5881 dev_priv = i915_mch_dev; in i915_read_mch_val()
5883 chipset_val = __i915_chipset_val(dev_priv); in i915_read_mch_val()
5884 graphics_val = __i915_gfx_val(dev_priv); in i915_read_mch_val()
5902 struct drm_i915_private *dev_priv; in i915_gpu_raise() local
5910 dev_priv = i915_mch_dev; in i915_gpu_raise()
5912 if (dev_priv->ips.max_delay > dev_priv->ips.fmax) in i915_gpu_raise()
5913 dev_priv->ips.max_delay--; in i915_gpu_raise()
5930 struct drm_i915_private *dev_priv; in i915_gpu_lower() local
5938 dev_priv = i915_mch_dev; in i915_gpu_lower()
5940 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay) in i915_gpu_lower()
5941 dev_priv->ips.max_delay++; in i915_gpu_lower()
5957 struct drm_i915_private *dev_priv; in i915_gpu_busy() local
5965 dev_priv = i915_mch_dev; in i915_gpu_busy()
5967 for_each_ring(ring, dev_priv, i) in i915_gpu_busy()
5985 struct drm_i915_private *dev_priv; in i915_gpu_turbo_disable() local
5993 dev_priv = i915_mch_dev; in i915_gpu_turbo_disable()
5995 dev_priv->ips.max_delay = dev_priv->ips.fstart; in i915_gpu_turbo_disable()
5997 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart)) in i915_gpu_turbo_disable()
6027 void intel_gpu_ips_init(struct drm_i915_private *dev_priv) in intel_gpu_ips_init() argument
6032 i915_mch_dev = dev_priv; in intel_gpu_ips_init()
6047 struct drm_i915_private *dev_priv = dev->dev_private; in intel_init_emon() local
6113 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); in intel_init_emon()
6136 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_suspend_rps() local
6138 flush_delayed_work(&dev_priv->rps.delayed_resume_work); in gen6_suspend_rps()
6153 struct drm_i915_private *dev_priv = dev->dev_private; in intel_suspend_gt_powersave() local
6161 gen6_rps_idle(dev_priv); in intel_suspend_gt_powersave()
6166 struct drm_i915_private *dev_priv = dev->dev_private; in intel_disable_gt_powersave() local
6173 mutex_lock(&dev_priv->rps.hw_lock); in intel_disable_gt_powersave()
6183 dev_priv->rps.enabled = false; in intel_disable_gt_powersave()
6184 mutex_unlock(&dev_priv->rps.hw_lock); in intel_disable_gt_powersave()
6190 struct drm_i915_private *dev_priv = in intel_gen6_powersave_work() local
6193 struct drm_device *dev = dev_priv->dev; in intel_gen6_powersave_work()
6195 mutex_lock(&dev_priv->rps.hw_lock); in intel_gen6_powersave_work()
6216 WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq); in intel_gen6_powersave_work()
6217 WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq); in intel_gen6_powersave_work()
6219 WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq); in intel_gen6_powersave_work()
6220 WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq); in intel_gen6_powersave_work()
6222 dev_priv->rps.enabled = true; in intel_gen6_powersave_work()
6226 mutex_unlock(&dev_priv->rps.hw_lock); in intel_gen6_powersave_work()
6228 intel_runtime_pm_put(dev_priv); in intel_gen6_powersave_work()
6233 struct drm_i915_private *dev_priv = dev->dev_private; in intel_enable_gt_powersave() local
6257 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work, in intel_enable_gt_powersave()
6259 intel_runtime_pm_get_noresume(dev_priv); in intel_enable_gt_powersave()
6265 struct drm_i915_private *dev_priv = dev->dev_private; in intel_reset_gt_powersave() local
6271 dev_priv->rps.enabled = false; in intel_reset_gt_powersave()
6276 struct drm_i915_private *dev_priv = dev->dev_private; in ibx_init_clock_gating() local
6288 struct drm_i915_private *dev_priv = dev->dev_private; in g4x_disable_trickle_feed() local
6291 for_each_pipe(dev_priv, pipe) { in g4x_disable_trickle_feed()
6303 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_init_lp_watermarks() local
6317 struct drm_i915_private *dev_priv = dev->dev_private; in ironlake_init_clock_gating() local
6391 struct drm_i915_private *dev_priv = dev->dev_private; in cpt_init_clock_gating() local
6408 for_each_pipe(dev_priv, pipe) { in cpt_init_clock_gating()
6412 if (dev_priv->vbt.fdi_rx_polarity_inverted) in cpt_init_clock_gating()
6420 for_each_pipe(dev_priv, pipe) { in cpt_init_clock_gating()
6428 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_check_mch_setup() local
6439 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_init_clock_gating() local
6534 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) in gen7_setup_fixed_func_scheduler() argument
6554 struct drm_i915_private *dev_priv = dev->dev_private; in lpt_init_clock_gating() local
6573 struct drm_i915_private *dev_priv = dev->dev_private; in lpt_suspend_hw() local
6585 struct drm_i915_private *dev_priv = dev->dev_private; in broadwell_init_clock_gating() local
6599 for_each_pipe(dev_priv, pipe) { in broadwell_init_clock_gating()
6645 struct drm_i915_private *dev_priv = dev->dev_private; in haswell_init_clock_gating() local
6701 struct drm_i915_private *dev_priv = dev->dev_private; in ivybridge_init_clock_gating() local
6763 gen7_setup_fixed_func_scheduler(dev_priv); in ivybridge_init_clock_gating()
6797 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv) in vlv_init_display_clock_gating() argument
6810 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_init_clock_gating() local
6812 vlv_init_display_clock_gating(dev_priv); in valleyview_init_clock_gating()
6845 gen7_setup_fixed_func_scheduler(dev_priv); in valleyview_init_clock_gating()
6894 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_init_clock_gating() local
6896 vlv_init_display_clock_gating(dev_priv); in cherryview_init_clock_gating()
6925 struct drm_i915_private *dev_priv = dev->dev_private; in g4x_init_clock_gating() local
6952 struct drm_i915_private *dev_priv = dev->dev_private; in crestline_init_clock_gating() local
6968 struct drm_i915_private *dev_priv = dev->dev_private; in broadwater_init_clock_gating() local
6985 struct drm_i915_private *dev_priv = dev->dev_private; in gen3_init_clock_gating() local
7010 struct drm_i915_private *dev_priv = dev->dev_private; in i85x_init_clock_gating() local
7024 struct drm_i915_private *dev_priv = dev->dev_private; in i830_init_clock_gating() local
7035 struct drm_i915_private *dev_priv = dev->dev_private; in intel_init_clock_gating() local
7037 if (dev_priv->display.init_clock_gating) in intel_init_clock_gating()
7038 dev_priv->display.init_clock_gating(dev); in intel_init_clock_gating()
7050 struct drm_i915_private *dev_priv = dev->dev_private; in intel_init_pm() local
7052 intel_fbc_init(dev_priv); in intel_init_pm()
7065 dev_priv->display.init_clock_gating = in intel_init_pm()
7067 dev_priv->display.update_wm = skl_update_wm; in intel_init_pm()
7068 dev_priv->display.update_sprite_wm = skl_update_sprite_wm; in intel_init_pm()
7072 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] && in intel_init_pm()
7073 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || in intel_init_pm()
7074 (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] && in intel_init_pm()
7075 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { in intel_init_pm()
7076 dev_priv->display.update_wm = ilk_update_wm; in intel_init_pm()
7077 dev_priv->display.update_sprite_wm = ilk_update_sprite_wm; in intel_init_pm()
7084 dev_priv->display.init_clock_gating = ironlake_init_clock_gating; in intel_init_pm()
7086 dev_priv->display.init_clock_gating = gen6_init_clock_gating; in intel_init_pm()
7088 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating; in intel_init_pm()
7090 dev_priv->display.init_clock_gating = haswell_init_clock_gating; in intel_init_pm()
7092 dev_priv->display.init_clock_gating = broadwell_init_clock_gating; in intel_init_pm()
7096 dev_priv->display.update_wm = vlv_update_wm; in intel_init_pm()
7097 dev_priv->display.init_clock_gating = in intel_init_pm()
7102 dev_priv->display.update_wm = vlv_update_wm; in intel_init_pm()
7103 dev_priv->display.init_clock_gating = in intel_init_pm()
7107 dev_priv->is_ddr3, in intel_init_pm()
7108 dev_priv->fsb_freq, in intel_init_pm()
7109 dev_priv->mem_freq)) { in intel_init_pm()
7113 (dev_priv->is_ddr3 == 1) ? "3" : "2", in intel_init_pm()
7114 dev_priv->fsb_freq, dev_priv->mem_freq); in intel_init_pm()
7116 intel_set_memory_cxsr(dev_priv, false); in intel_init_pm()
7117 dev_priv->display.update_wm = NULL; in intel_init_pm()
7119 dev_priv->display.update_wm = pineview_update_wm; in intel_init_pm()
7120 dev_priv->display.init_clock_gating = gen3_init_clock_gating; in intel_init_pm()
7122 dev_priv->display.update_wm = g4x_update_wm; in intel_init_pm()
7123 dev_priv->display.init_clock_gating = g4x_init_clock_gating; in intel_init_pm()
7125 dev_priv->display.update_wm = i965_update_wm; in intel_init_pm()
7127 dev_priv->display.init_clock_gating = crestline_init_clock_gating; in intel_init_pm()
7129 dev_priv->display.init_clock_gating = broadwater_init_clock_gating; in intel_init_pm()
7131 dev_priv->display.update_wm = i9xx_update_wm; in intel_init_pm()
7132 dev_priv->display.get_fifo_size = i9xx_get_fifo_size; in intel_init_pm()
7133 dev_priv->display.init_clock_gating = gen3_init_clock_gating; in intel_init_pm()
7136 dev_priv->display.update_wm = i845_update_wm; in intel_init_pm()
7137 dev_priv->display.get_fifo_size = i845_get_fifo_size; in intel_init_pm()
7139 dev_priv->display.update_wm = i9xx_update_wm; in intel_init_pm()
7140 dev_priv->display.get_fifo_size = i830_get_fifo_size; in intel_init_pm()
7144 dev_priv->display.init_clock_gating = i85x_init_clock_gating; in intel_init_pm()
7146 dev_priv->display.init_clock_gating = i830_init_clock_gating; in intel_init_pm()
7152 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val) in sandybridge_pcode_read() argument
7154 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in sandybridge_pcode_read()
7177 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val) in sandybridge_pcode_write() argument
7179 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in sandybridge_pcode_write()
7217 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) in byt_gpu_freq() argument
7219 int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); in byt_gpu_freq()
7228 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) in byt_freq_opcode() argument
7230 int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); in byt_freq_opcode()
7239 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) in chv_gpu_freq() argument
7241 int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); in chv_gpu_freq()
7250 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) in chv_freq_opcode() argument
7252 int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); in chv_freq_opcode()
7262 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) in intel_gpu_freq() argument
7264 if (IS_GEN9(dev_priv->dev)) in intel_gpu_freq()
7267 else if (IS_CHERRYVIEW(dev_priv->dev)) in intel_gpu_freq()
7268 return chv_gpu_freq(dev_priv, val); in intel_gpu_freq()
7269 else if (IS_VALLEYVIEW(dev_priv->dev)) in intel_gpu_freq()
7270 return byt_gpu_freq(dev_priv, val); in intel_gpu_freq()
7275 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val) in intel_freq_opcode() argument
7277 if (IS_GEN9(dev_priv->dev)) in intel_freq_opcode()
7280 else if (IS_CHERRYVIEW(dev_priv->dev)) in intel_freq_opcode()
7281 return chv_freq_opcode(dev_priv, val); in intel_freq_opcode()
7282 else if (IS_VALLEYVIEW(dev_priv->dev)) in intel_freq_opcode()
7283 return byt_freq_opcode(dev_priv, val); in intel_freq_opcode()
7330 struct drm_i915_private *dev_priv = dev->dev_private; in intel_pm_setup() local
7332 mutex_init(&dev_priv->rps.hw_lock); in intel_pm_setup()
7333 spin_lock_init(&dev_priv->rps.client_lock); in intel_pm_setup()
7335 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work, in intel_pm_setup()
7337 INIT_LIST_HEAD(&dev_priv->rps.clients); in intel_pm_setup()
7338 INIT_LIST_HEAD(&dev_priv->rps.semaphores.link); in intel_pm_setup()
7339 INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link); in intel_pm_setup()
7341 dev_priv->pm.suspended = false; in intel_pm_setup()