Lines Matching refs:dev_priv
57 struct drm_i915_private *dev_priv = dev->dev_private; in gen9_init_clock_gating() local
66 struct drm_i915_private *dev_priv = dev->dev_private; in skl_init_clock_gating() local
99 struct drm_i915_private *dev_priv = dev->dev_private; in i915_pineview_get_mem_freq() local
106 dev_priv->fsb_freq = 533; /* 133*4 */ in i915_pineview_get_mem_freq()
109 dev_priv->fsb_freq = 800; /* 200*4 */ in i915_pineview_get_mem_freq()
112 dev_priv->fsb_freq = 667; /* 167*4 */ in i915_pineview_get_mem_freq()
115 dev_priv->fsb_freq = 400; /* 100*4 */ in i915_pineview_get_mem_freq()
121 dev_priv->mem_freq = 533; in i915_pineview_get_mem_freq()
124 dev_priv->mem_freq = 667; in i915_pineview_get_mem_freq()
127 dev_priv->mem_freq = 800; in i915_pineview_get_mem_freq()
133 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; in i915_pineview_get_mem_freq()
138 struct drm_i915_private *dev_priv = dev->dev_private; in i915_ironlake_get_mem_freq() local
146 dev_priv->mem_freq = 800; in i915_ironlake_get_mem_freq()
149 dev_priv->mem_freq = 1066; in i915_ironlake_get_mem_freq()
152 dev_priv->mem_freq = 1333; in i915_ironlake_get_mem_freq()
155 dev_priv->mem_freq = 1600; in i915_ironlake_get_mem_freq()
160 dev_priv->mem_freq = 0; in i915_ironlake_get_mem_freq()
164 dev_priv->ips.r_t = dev_priv->mem_freq; in i915_ironlake_get_mem_freq()
168 dev_priv->fsb_freq = 3200; in i915_ironlake_get_mem_freq()
171 dev_priv->fsb_freq = 3733; in i915_ironlake_get_mem_freq()
174 dev_priv->fsb_freq = 4266; in i915_ironlake_get_mem_freq()
177 dev_priv->fsb_freq = 4800; in i915_ironlake_get_mem_freq()
180 dev_priv->fsb_freq = 5333; in i915_ironlake_get_mem_freq()
183 dev_priv->fsb_freq = 5866; in i915_ironlake_get_mem_freq()
186 dev_priv->fsb_freq = 6400; in i915_ironlake_get_mem_freq()
191 dev_priv->fsb_freq = 0; in i915_ironlake_get_mem_freq()
195 if (dev_priv->fsb_freq == 3200) { in i915_ironlake_get_mem_freq()
196 dev_priv->ips.c_m = 0; in i915_ironlake_get_mem_freq()
197 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) { in i915_ironlake_get_mem_freq()
198 dev_priv->ips.c_m = 1; in i915_ironlake_get_mem_freq()
200 dev_priv->ips.c_m = 2; in i915_ironlake_get_mem_freq()
266 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable) in chv_set_memory_dvfs() argument
270 mutex_lock(&dev_priv->rps.hw_lock); in chv_set_memory_dvfs()
272 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); in chv_set_memory_dvfs()
279 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); in chv_set_memory_dvfs()
281 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & in chv_set_memory_dvfs()
285 mutex_unlock(&dev_priv->rps.hw_lock); in chv_set_memory_dvfs()
288 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) in chv_set_memory_pm5() argument
292 mutex_lock(&dev_priv->rps.hw_lock); in chv_set_memory_pm5()
294 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); in chv_set_memory_pm5()
299 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val); in chv_set_memory_pm5()
301 mutex_unlock(&dev_priv->rps.hw_lock); in chv_set_memory_pm5()
307 void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) in intel_set_memory_cxsr() argument
309 struct drm_device *dev = dev_priv->dev; in intel_set_memory_cxsr()
315 chv_set_memory_pm5(dev_priv, enable); in intel_set_memory_cxsr()
361 struct drm_i915_private *dev_priv = dev->dev_private; in vlv_get_fifo_size() local
412 struct drm_i915_private *dev_priv = dev->dev_private; in i9xx_get_fifo_size() local
428 struct drm_i915_private *dev_priv = dev->dev_private; in i830_get_fifo_size() local
445 struct drm_i915_private *dev_priv = dev->dev_private; in i845_get_fifo_size() local
638 struct drm_i915_private *dev_priv = dev->dev_private; in pineview_update_wm() local
644 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3, in pineview_update_wm()
645 dev_priv->fsb_freq, dev_priv->mem_freq); in pineview_update_wm()
648 intel_set_memory_cxsr(dev_priv, false); in pineview_update_wm()
699 intel_set_memory_cxsr(dev_priv, true); in pineview_update_wm()
701 intel_set_memory_cxsr(dev_priv, false); in pineview_update_wm()
847 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); in vlv_write_wm_values() local
868 if (IS_CHERRYVIEW(dev_priv)) { in vlv_write_wm_values()
905 dev_priv->wm.vlv = *wm; in vlv_write_wm_values()
984 struct drm_i915_private *dev_priv = to_i915(dev); in vlv_compute_sr_wm() local
1000 fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1; in vlv_compute_sr_wm()
1028 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_update_wm() local
1032 struct vlv_wm_values wm = dev_priv->wm.vlv; in valleyview_update_wm()
1046 if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0) in valleyview_update_wm()
1062 if (IS_CHERRYVIEW(dev_priv)) in valleyview_update_wm()
1063 chv_set_memory_dvfs(dev_priv, false); in valleyview_update_wm()
1066 intel_set_memory_cxsr(dev_priv, false); in valleyview_update_wm()
1071 intel_set_memory_cxsr(dev_priv, true); in valleyview_update_wm()
1082 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_update_sprite_wm() local
1087 struct vlv_wm_values wm = dev_priv->wm.vlv; in valleyview_update_sprite_wm()
1104 if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0) in valleyview_update_sprite_wm()
1114 intel_set_memory_cxsr(dev_priv, false); in valleyview_update_sprite_wm()
1119 intel_set_memory_cxsr(dev_priv, true); in valleyview_update_sprite_wm()
1128 struct drm_i915_private *dev_priv = dev->dev_private; in g4x_update_wm() local
1155 intel_set_memory_cxsr(dev_priv, false); in g4x_update_wm()
1179 intel_set_memory_cxsr(dev_priv, true); in g4x_update_wm()
1185 struct drm_i915_private *dev_priv = dev->dev_private; in i965_update_wm() local
1235 intel_set_memory_cxsr(dev_priv, false); in i965_update_wm()
1252 intel_set_memory_cxsr(dev_priv, true); in i965_update_wm()
1260 struct drm_i915_private *dev_priv = dev->dev_private; in i9xx_update_wm() local
1276 fifo_size = dev_priv->display.get_fifo_size(dev, 0); in i9xx_update_wm()
1298 fifo_size = dev_priv->display.get_fifo_size(dev, 1); in i9xx_update_wm()
1338 intel_set_memory_cxsr(dev_priv, false); in i9xx_update_wm()
1385 intel_set_memory_cxsr(dev_priv, true); in i9xx_update_wm()
1391 struct drm_i915_private *dev_priv = dev->dev_private; in i845_update_wm() local
1404 dev_priv->display.get_fifo_size(dev, 0), in i845_update_wm()
1755 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, in ilk_compute_wm_level() argument
1760 uint16_t pri_latency = dev_priv->wm.pri_latency[level]; in ilk_compute_wm_level()
1761 uint16_t spr_latency = dev_priv->wm.spr_latency[level]; in ilk_compute_wm_level()
1762 uint16_t cur_latency = dev_priv->wm.cur_latency[level]; in ilk_compute_wm_level()
1781 struct drm_i915_private *dev_priv = dev->dev_private; in hsw_compute_linetime_wm() local
1795 intel_ddi_get_cdclk_freq(dev_priv)); in hsw_compute_linetime_wm()
1803 struct drm_i915_private *dev_priv = dev->dev_private; in intel_read_wm_latency() local
1812 mutex_lock(&dev_priv->rps.hw_lock); in intel_read_wm_latency()
1813 ret = sandybridge_pcode_read(dev_priv, in intel_read_wm_latency()
1816 mutex_unlock(&dev_priv->rps.hw_lock); in intel_read_wm_latency()
1833 mutex_lock(&dev_priv->rps.hw_lock); in intel_read_wm_latency()
1834 ret = sandybridge_pcode_read(dev_priv, in intel_read_wm_latency()
1837 mutex_unlock(&dev_priv->rps.hw_lock); in intel_read_wm_latency()
1966 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, in ilk_increase_wm_latency() argument
1969 int level, max_level = ilk_wm_max_level(dev_priv->dev); in ilk_increase_wm_latency()
1983 struct drm_i915_private *dev_priv = dev->dev_private; in snb_wm_latency_quirk() local
1990 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) | in snb_wm_latency_quirk()
1991 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) | in snb_wm_latency_quirk()
1992 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12); in snb_wm_latency_quirk()
1998 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); in snb_wm_latency_quirk()
1999 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); in snb_wm_latency_quirk()
2000 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); in snb_wm_latency_quirk()
2005 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_setup_wm_latency() local
2007 intel_read_wm_latency(dev, dev_priv->wm.pri_latency); in ilk_setup_wm_latency()
2009 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency, in ilk_setup_wm_latency()
2010 sizeof(dev_priv->wm.pri_latency)); in ilk_setup_wm_latency()
2011 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency, in ilk_setup_wm_latency()
2012 sizeof(dev_priv->wm.pri_latency)); in ilk_setup_wm_latency()
2014 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency); in ilk_setup_wm_latency()
2015 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency); in ilk_setup_wm_latency()
2017 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); in ilk_setup_wm_latency()
2018 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); in ilk_setup_wm_latency()
2019 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); in ilk_setup_wm_latency()
2027 struct drm_i915_private *dev_priv = dev->dev_private; in skl_setup_wm_latency() local
2029 intel_read_wm_latency(dev, dev_priv->wm.skl_latency); in skl_setup_wm_latency()
2030 intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency); in skl_setup_wm_latency()
2099 const struct drm_i915_private *dev_priv = dev->dev_private; in intel_compute_pipe_wm() local
2121 ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]); in intel_compute_pipe_wm()
2138 ilk_compute_wm_level(dev_priv, level, params, &wm); in intel_compute_pipe_wm()
2253 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_wm_lp_latency() local
2258 return dev_priv->wm.pri_latency[level]; in ilk_wm_lp_latency()
2364 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, in ilk_compute_wm_dirty() argument
2372 for_each_pipe(dev_priv, pipe) { in ilk_compute_wm_dirty()
2416 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, in _ilk_disable_lp_wm() argument
2419 struct ilk_wm_values *previous = &dev_priv->wm.hw; in _ilk_disable_lp_wm()
2450 static void ilk_write_wm_values(struct drm_i915_private *dev_priv, in ilk_write_wm_values() argument
2453 struct drm_device *dev = dev_priv->dev; in ilk_write_wm_values()
2454 struct ilk_wm_values *previous = &dev_priv->wm.hw; in ilk_write_wm_values()
2458 dirty = ilk_compute_wm_dirty(dev_priv, previous, results); in ilk_write_wm_values()
2462 _ilk_disable_lp_wm(dev_priv, dirty); in ilk_write_wm_values()
2523 dev_priv->wm.hw = *results; in ilk_write_wm_values()
2528 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_disable_lp_wm() local
2530 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); in ilk_disable_lp_wm()
2593 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, in skl_ddb_get_hw_state() argument
2600 for_each_pipe(dev_priv, pipe) { in skl_ddb_get_hw_state()
2601 for_each_plane(dev_priv, pipe, plane) { in skl_ddb_get_hw_state()
2650 struct drm_i915_private *dev_priv = dev->dev_private; in skl_allocate_pipe_ddb() local
2675 for_each_plane(dev_priv, pipe, plane) { in skl_allocate_pipe_ddb()
2781 struct drm_i915_private *dev_priv = dev->dev_private; in skl_ddb_allocation_changed() local
2782 const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb; in skl_ddb_allocation_changed()
2866 static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, in skl_compute_plane_wm() argument
2874 uint32_t latency = dev_priv->wm.skl_latency[level]; in skl_compute_plane_wm()
2942 static void skl_compute_wm_level(const struct drm_i915_private *dev_priv, in skl_compute_wm_level() argument
2956 result->plane_en[i] = skl_compute_plane_wm(dev_priv, in skl_compute_wm_level()
2965 result->cursor_en = skl_compute_plane_wm(dev_priv, p, &p->cursor, in skl_compute_wm_level()
3003 const struct drm_i915_private *dev_priv = dev->dev_private; in skl_compute_pipe_wm() local
3008 skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe, in skl_compute_pipe_wm()
3075 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg, in skl_ddb_entry_write() argument
3084 static void skl_write_wm_values(struct drm_i915_private *dev_priv, in skl_write_wm_values() argument
3087 struct drm_device *dev = dev_priv->dev; in skl_write_wm_values()
3112 skl_ddb_entry_write(dev_priv, in skl_write_wm_values()
3116 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), in skl_write_wm_values()
3146 skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass) in skl_wm_flush_pipe() argument
3152 for_each_plane(dev_priv, pipe, plane) { in skl_wm_flush_pipe()
3174 static void skl_flush_wm_values(struct drm_i915_private *dev_priv, in skl_flush_wm_values() argument
3177 struct drm_device *dev = dev_priv->dev; in skl_flush_wm_values()
3184 cur_ddb = &dev_priv->wm.skl_hw.ddb; in skl_flush_wm_values()
3202 skl_wm_flush_pipe(dev_priv, pipe, 1); in skl_flush_wm_values()
3227 skl_wm_flush_pipe(dev_priv, pipe, 2); in skl_flush_wm_values()
3252 skl_wm_flush_pipe(dev_priv, pipe, 3); in skl_flush_wm_values()
3328 struct drm_i915_private *dev_priv = dev->dev_private; in skl_update_wm() local
3330 struct skl_wm_values *results = &dev_priv->wm.skl_results; in skl_update_wm()
3346 skl_write_wm_values(dev_priv, results); in skl_update_wm()
3347 skl_flush_wm_values(dev_priv, results); in skl_update_wm()
3350 dev_priv->wm.skl_hw = *results; in skl_update_wm()
3382 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_update_wm() local
3421 ilk_write_wm_values(dev_priv, &results); in ilk_update_wm()
3499 struct drm_i915_private *dev_priv = dev->dev_private; in skl_pipe_wm_get_hw_state() local
3500 struct skl_wm_values *hw = &dev_priv->wm.skl_hw; in skl_pipe_wm_get_hw_state()
3550 struct drm_i915_private *dev_priv = dev->dev_private; in skl_wm_get_hw_state() local
3551 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb; in skl_wm_get_hw_state()
3554 skl_ddb_get_hw_state(dev_priv, ddb); in skl_wm_get_hw_state()
3562 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_pipe_wm_get_hw_state() local
3563 struct ilk_wm_values *hw = &dev_priv->wm.hw; in ilk_pipe_wm_get_hw_state()
3610 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_wm_get_hw_state() local
3611 struct ilk_wm_values *hw = &dev_priv->wm.hw; in ilk_wm_get_hw_state()
3672 struct drm_i915_private *dev_priv = crtc->dev->dev_private; in intel_update_watermarks() local
3674 if (dev_priv->display.update_wm) in intel_update_watermarks()
3675 dev_priv->display.update_wm(crtc); in intel_update_watermarks()
3685 struct drm_i915_private *dev_priv = plane->dev->dev_private; in intel_update_sprite_watermarks() local
3687 if (dev_priv->display.update_sprite_wm) in intel_update_sprite_watermarks()
3688 dev_priv->display.update_sprite_wm(plane, crtc, in intel_update_sprite_watermarks()
3704 struct drm_i915_private *dev_priv = dev->dev_private; in ironlake_set_drps() local
3728 struct drm_i915_private *dev_priv = dev->dev_private; in ironlake_enable_drps() local
3757 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */ in ironlake_enable_drps()
3758 dev_priv->ips.fstart = fstart; in ironlake_enable_drps()
3760 dev_priv->ips.max_delay = fstart; in ironlake_enable_drps()
3761 dev_priv->ips.min_delay = fmin; in ironlake_enable_drps()
3762 dev_priv->ips.cur_delay = fstart; in ironlake_enable_drps()
3785 dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) + in ironlake_enable_drps()
3787 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies); in ironlake_enable_drps()
3788 dev_priv->ips.last_count2 = I915_READ(0x112f4); in ironlake_enable_drps()
3789 dev_priv->ips.last_time2 = ktime_get_raw_ns(); in ironlake_enable_drps()
3796 struct drm_i915_private *dev_priv = dev->dev_private; in ironlake_disable_drps() local
3811 ironlake_set_drps(dev, dev_priv->ips.fstart); in ironlake_disable_drps()
3825 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val) in intel_rps_limits() argument
3835 if (IS_GEN9(dev_priv->dev)) { in intel_rps_limits()
3836 limits = (dev_priv->rps.max_freq_softlimit) << 23; in intel_rps_limits()
3837 if (val <= dev_priv->rps.min_freq_softlimit) in intel_rps_limits()
3838 limits |= (dev_priv->rps.min_freq_softlimit) << 14; in intel_rps_limits()
3840 limits = dev_priv->rps.max_freq_softlimit << 24; in intel_rps_limits()
3841 if (val <= dev_priv->rps.min_freq_softlimit) in intel_rps_limits()
3842 limits |= dev_priv->rps.min_freq_softlimit << 16; in intel_rps_limits()
3848 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) in gen6_set_rps_thresholds() argument
3854 new_power = dev_priv->rps.power; in gen6_set_rps_thresholds()
3855 switch (dev_priv->rps.power) { in gen6_set_rps_thresholds()
3857 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq) in gen6_set_rps_thresholds()
3862 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq) in gen6_set_rps_thresholds()
3864 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq) in gen6_set_rps_thresholds()
3869 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq) in gen6_set_rps_thresholds()
3874 if (val <= dev_priv->rps.min_freq_softlimit) in gen6_set_rps_thresholds()
3876 if (val >= dev_priv->rps.max_freq_softlimit) in gen6_set_rps_thresholds()
3878 if (new_power == dev_priv->rps.power) in gen6_set_rps_thresholds()
3915 GT_INTERVAL_FROM_US(dev_priv, ei_up)); in gen6_set_rps_thresholds()
3917 GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100))); in gen6_set_rps_thresholds()
3920 GT_INTERVAL_FROM_US(dev_priv, ei_down)); in gen6_set_rps_thresholds()
3922 GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100))); in gen6_set_rps_thresholds()
3932 dev_priv->rps.power = new_power; in gen6_set_rps_thresholds()
3933 dev_priv->rps.last_adj = 0; in gen6_set_rps_thresholds()
3936 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) in gen6_rps_pm_mask() argument
3940 if (val > dev_priv->rps.min_freq_softlimit) in gen6_rps_pm_mask()
3942 if (val < dev_priv->rps.max_freq_softlimit) in gen6_rps_pm_mask()
3945 mask &= dev_priv->pm_rps_events; in gen6_rps_pm_mask()
3947 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask); in gen6_rps_pm_mask()
3955 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_set_rps() local
3957 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in gen6_set_rps()
3958 WARN_ON(val > dev_priv->rps.max_freq); in gen6_set_rps()
3959 WARN_ON(val < dev_priv->rps.min_freq); in gen6_set_rps()
3964 if (val != dev_priv->rps.cur_freq) { in gen6_set_rps()
3965 gen6_set_rps_thresholds(dev_priv, val); in gen6_set_rps()
3983 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val)); in gen6_set_rps()
3984 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); in gen6_set_rps()
3988 dev_priv->rps.cur_freq = val; in gen6_set_rps()
3994 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_set_rps() local
3996 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in valleyview_set_rps()
3997 WARN_ON(val > dev_priv->rps.max_freq); in valleyview_set_rps()
3998 WARN_ON(val < dev_priv->rps.min_freq); in valleyview_set_rps()
4004 if (val != dev_priv->rps.cur_freq) in valleyview_set_rps()
4005 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); in valleyview_set_rps()
4007 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); in valleyview_set_rps()
4009 dev_priv->rps.cur_freq = val; in valleyview_set_rps()
4010 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); in valleyview_set_rps()
4022 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) in vlv_set_rps_idle() argument
4024 struct drm_device *dev = dev_priv->dev; in vlv_set_rps_idle()
4025 u32 val = dev_priv->rps.idle_freq; in vlv_set_rps_idle()
4029 valleyview_set_rps(dev_priv->dev, val); in vlv_set_rps_idle()
4037 if (dev_priv->rps.cur_freq <= val) in vlv_set_rps_idle()
4042 gen6_sanitize_rps_pm_mask(dev_priv, ~0)); in vlv_set_rps_idle()
4044 vlv_force_gfx_clock(dev_priv, true); in vlv_set_rps_idle()
4046 dev_priv->rps.cur_freq = val; in vlv_set_rps_idle()
4048 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); in vlv_set_rps_idle()
4050 if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) in vlv_set_rps_idle()
4054 vlv_force_gfx_clock(dev_priv, false); in vlv_set_rps_idle()
4056 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); in vlv_set_rps_idle()
4059 void gen6_rps_busy(struct drm_i915_private *dev_priv) in gen6_rps_busy() argument
4061 mutex_lock(&dev_priv->rps.hw_lock); in gen6_rps_busy()
4062 if (dev_priv->rps.enabled) { in gen6_rps_busy()
4063 if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) in gen6_rps_busy()
4064 gen6_rps_reset_ei(dev_priv); in gen6_rps_busy()
4066 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq)); in gen6_rps_busy()
4068 mutex_unlock(&dev_priv->rps.hw_lock); in gen6_rps_busy()
4071 void gen6_rps_idle(struct drm_i915_private *dev_priv) in gen6_rps_idle() argument
4073 struct drm_device *dev = dev_priv->dev; in gen6_rps_idle()
4075 mutex_lock(&dev_priv->rps.hw_lock); in gen6_rps_idle()
4076 if (dev_priv->rps.enabled) { in gen6_rps_idle()
4078 vlv_set_rps_idle(dev_priv); in gen6_rps_idle()
4080 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); in gen6_rps_idle()
4081 dev_priv->rps.last_adj = 0; in gen6_rps_idle()
4084 mutex_unlock(&dev_priv->rps.hw_lock); in gen6_rps_idle()
4087 void gen6_rps_boost(struct drm_i915_private *dev_priv) in gen6_rps_boost() argument
4091 mutex_lock(&dev_priv->rps.hw_lock); in gen6_rps_boost()
4092 val = dev_priv->rps.max_freq_softlimit; in gen6_rps_boost()
4093 if (dev_priv->rps.enabled && in gen6_rps_boost()
4094 dev_priv->mm.busy && in gen6_rps_boost()
4095 dev_priv->rps.cur_freq < val) { in gen6_rps_boost()
4096 intel_set_rps(dev_priv->dev, val); in gen6_rps_boost()
4097 dev_priv->rps.last_adj = 0; in gen6_rps_boost()
4099 mutex_unlock(&dev_priv->rps.hw_lock); in gen6_rps_boost()
4112 struct drm_i915_private *dev_priv = dev->dev_private; in gen9_disable_rps() local
4120 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_disable_rps() local
4128 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_disable_rps() local
4135 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_disable_rps() local
4139 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in valleyview_disable_rps()
4143 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in valleyview_disable_rps()
4209 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_init_rps_frequencies() local
4216 dev_priv->rps.cur_freq = 0; in gen6_init_rps_frequencies()
4218 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff; in gen6_init_rps_frequencies()
4219 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; in gen6_init_rps_frequencies()
4220 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff; in gen6_init_rps_frequencies()
4224 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER; in gen6_init_rps_frequencies()
4225 dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER; in gen6_init_rps_frequencies()
4226 dev_priv->rps.min_freq *= GEN9_FREQ_SCALER; in gen6_init_rps_frequencies()
4229 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; in gen6_init_rps_frequencies()
4231 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; in gen6_init_rps_frequencies()
4233 ret = sandybridge_pcode_read(dev_priv, in gen6_init_rps_frequencies()
4237 dev_priv->rps.efficient_freq = in gen6_init_rps_frequencies()
4240 dev_priv->rps.min_freq, in gen6_init_rps_frequencies()
4241 dev_priv->rps.max_freq); in gen6_init_rps_frequencies()
4244 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; in gen6_init_rps_frequencies()
4247 if (dev_priv->rps.max_freq_softlimit == 0) in gen6_init_rps_frequencies()
4248 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; in gen6_init_rps_frequencies()
4250 if (dev_priv->rps.min_freq_softlimit == 0) { in gen6_init_rps_frequencies()
4252 dev_priv->rps.min_freq_softlimit = in gen6_init_rps_frequencies()
4254 max(dev_priv->rps.efficient_freq, (u8) 9); in gen6_init_rps_frequencies()
4256 dev_priv->rps.min_freq_softlimit = in gen6_init_rps_frequencies()
4257 dev_priv->rps.min_freq; in gen6_init_rps_frequencies()
4264 struct drm_i915_private *dev_priv = dev->dev_private; in gen9_enable_rps() local
4266 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in gen9_enable_rps()
4272 GEN9_FREQUENCY(dev_priv->rps.rp1_freq)); in gen9_enable_rps()
4276 GT_INTERVAL_FROM_US(dev_priv, 1000000)); in gen9_enable_rps()
4283 dev_priv->rps.power = HIGH_POWER; /* force a reset */ in gen9_enable_rps()
4284 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); in gen9_enable_rps()
4286 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen9_enable_rps()
4291 struct drm_i915_private *dev_priv = dev->dev_private; in gen9_enable_rc6() local
4301 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in gen9_enable_rc6()
4310 for_each_ring(ring, dev_priv, unused) in gen9_enable_rc6()
4331 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen9_enable_rc6()
4337 struct drm_i915_private *dev_priv = dev->dev_private; in gen8_enable_rps() local
4347 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in gen8_enable_rps()
4359 for_each_ring(ring, dev_priv, unused) in gen8_enable_rps()
4382 HSW_FREQUENCY(dev_priv->rps.rp1_freq)); in gen8_enable_rps()
4384 HSW_FREQUENCY(dev_priv->rps.rp1_freq)); in gen8_enable_rps()
4390 dev_priv->rps.max_freq_softlimit << 24 | in gen8_enable_rps()
4391 dev_priv->rps.min_freq_softlimit << 16); in gen8_enable_rps()
4411 dev_priv->rps.power = HIGH_POWER; /* force a reset */ in gen8_enable_rps()
4412 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); in gen8_enable_rps()
4414 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen8_enable_rps()
4419 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_enable_rps() local
4426 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in gen6_enable_rps()
4442 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in gen6_enable_rps()
4456 for_each_ring(ring, dev_priv, i) in gen6_enable_rps()
4469 rc6_mode = intel_enable_rc6(dev_priv->dev); in gen6_enable_rps()
4493 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0); in gen6_enable_rps()
4497 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox); in gen6_enable_rps()
4500 (dev_priv->rps.max_freq_softlimit & 0xff) * 50, in gen6_enable_rps()
4502 dev_priv->rps.max_freq = pcu_mbox & 0xff; in gen6_enable_rps()
4505 dev_priv->rps.power = HIGH_POWER; /* force a reset */ in gen6_enable_rps()
4506 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); in gen6_enable_rps()
4509 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids); in gen6_enable_rps()
4517 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); in gen6_enable_rps()
4522 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in gen6_enable_rps()
4527 struct drm_i915_private *dev_priv = dev->dev_private; in __gen6_update_ring_freq() local
4534 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in __gen6_update_ring_freq()
4560 for (gpu_freq = dev_priv->rps.max_freq; gpu_freq >= dev_priv->rps.min_freq; in __gen6_update_ring_freq()
4562 int diff = dev_priv->rps.max_freq - gpu_freq; in __gen6_update_ring_freq()
4587 sandybridge_pcode_write(dev_priv, in __gen6_update_ring_freq()
4597 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_update_ring_freq() local
4602 mutex_lock(&dev_priv->rps.hw_lock); in gen6_update_ring_freq()
4604 mutex_unlock(&dev_priv->rps.hw_lock); in gen6_update_ring_freq()
4607 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) in cherryview_rps_max_freq() argument
4609 struct drm_device *dev = dev_priv->dev; in cherryview_rps_max_freq()
4613 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); in cherryview_rps_max_freq()
4634 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG); in cherryview_rps_max_freq()
4641 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv) in cherryview_rps_rpe_freq() argument
4645 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG); in cherryview_rps_rpe_freq()
4651 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv) in cherryview_rps_guar_freq() argument
4653 struct drm_device *dev = dev_priv->dev; in cherryview_rps_guar_freq()
4657 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); in cherryview_rps_guar_freq()
4661 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); in cherryview_rps_guar_freq()
4668 static int cherryview_rps_min_freq(struct drm_i915_private *dev_priv) in cherryview_rps_min_freq() argument
4670 struct drm_device *dev = dev_priv->dev; in cherryview_rps_min_freq()
4674 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE); in cherryview_rps_min_freq()
4678 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG); in cherryview_rps_min_freq()
4686 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv) in valleyview_rps_guar_freq() argument
4690 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); in valleyview_rps_guar_freq()
4697 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv) in valleyview_rps_max_freq() argument
4701 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); in valleyview_rps_max_freq()
4710 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv) in valleyview_rps_rpe_freq() argument
4714 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO); in valleyview_rps_rpe_freq()
4716 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI); in valleyview_rps_rpe_freq()
4722 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv) in valleyview_rps_min_freq() argument
4724 return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff; in valleyview_rps_min_freq()
4728 static void valleyview_check_pctx(struct drm_i915_private *dev_priv) in valleyview_check_pctx() argument
4732 WARN_ON(pctx_addr != dev_priv->mm.stolen_base + in valleyview_check_pctx()
4733 dev_priv->vlv_pctx->stolen->start); in valleyview_check_pctx()
4738 static void cherryview_check_pctx(struct drm_i915_private *dev_priv) in cherryview_check_pctx() argument
4747 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_setup_pctx() local
4749 struct i915_gtt *gtt = &dev_priv->gtt; in cherryview_setup_pctx()
4758 paddr = (dev_priv->mm.stolen_base + in cherryview_setup_pctx()
4770 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_setup_pctx() local
4783 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base; in valleyview_setup_pctx()
4784 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev, in valleyview_setup_pctx()
4807 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start; in valleyview_setup_pctx()
4812 dev_priv->vlv_pctx = pctx; in valleyview_setup_pctx()
4817 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_cleanup_pctx() local
4819 if (WARN_ON(!dev_priv->vlv_pctx)) in valleyview_cleanup_pctx()
4822 drm_gem_object_unreference(&dev_priv->vlv_pctx->base); in valleyview_cleanup_pctx()
4823 dev_priv->vlv_pctx = NULL; in valleyview_cleanup_pctx()
4828 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_init_gt_powersave() local
4833 mutex_lock(&dev_priv->rps.hw_lock); in valleyview_init_gt_powersave()
4835 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); in valleyview_init_gt_powersave()
4839 dev_priv->mem_freq = 800; in valleyview_init_gt_powersave()
4842 dev_priv->mem_freq = 1066; in valleyview_init_gt_powersave()
4845 dev_priv->mem_freq = 1333; in valleyview_init_gt_powersave()
4848 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); in valleyview_init_gt_powersave()
4850 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv); in valleyview_init_gt_powersave()
4851 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; in valleyview_init_gt_powersave()
4853 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq), in valleyview_init_gt_powersave()
4854 dev_priv->rps.max_freq); in valleyview_init_gt_powersave()
4856 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv); in valleyview_init_gt_powersave()
4858 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), in valleyview_init_gt_powersave()
4859 dev_priv->rps.efficient_freq); in valleyview_init_gt_powersave()
4861 dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv); in valleyview_init_gt_powersave()
4863 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq), in valleyview_init_gt_powersave()
4864 dev_priv->rps.rp1_freq); in valleyview_init_gt_powersave()
4866 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv); in valleyview_init_gt_powersave()
4868 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq), in valleyview_init_gt_powersave()
4869 dev_priv->rps.min_freq); in valleyview_init_gt_powersave()
4871 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; in valleyview_init_gt_powersave()
4874 if (dev_priv->rps.max_freq_softlimit == 0) in valleyview_init_gt_powersave()
4875 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; in valleyview_init_gt_powersave()
4877 if (dev_priv->rps.min_freq_softlimit == 0) in valleyview_init_gt_powersave()
4878 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; in valleyview_init_gt_powersave()
4880 mutex_unlock(&dev_priv->rps.hw_lock); in valleyview_init_gt_powersave()
4885 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_init_gt_powersave() local
4890 mutex_lock(&dev_priv->rps.hw_lock); in cherryview_init_gt_powersave()
4892 mutex_lock(&dev_priv->dpio_lock); in cherryview_init_gt_powersave()
4893 val = vlv_cck_read(dev_priv, CCK_FUSE_REG); in cherryview_init_gt_powersave()
4894 mutex_unlock(&dev_priv->dpio_lock); in cherryview_init_gt_powersave()
4899 dev_priv->rps.cz_freq = 200; in cherryview_init_gt_powersave()
4900 dev_priv->mem_freq = 1600; in cherryview_init_gt_powersave()
4903 dev_priv->rps.cz_freq = 267; in cherryview_init_gt_powersave()
4904 dev_priv->mem_freq = 1600; in cherryview_init_gt_powersave()
4907 dev_priv->rps.cz_freq = 333; in cherryview_init_gt_powersave()
4908 dev_priv->mem_freq = 2000; in cherryview_init_gt_powersave()
4911 dev_priv->rps.cz_freq = 320; in cherryview_init_gt_powersave()
4912 dev_priv->mem_freq = 1600; in cherryview_init_gt_powersave()
4915 dev_priv->rps.cz_freq = 400; in cherryview_init_gt_powersave()
4916 dev_priv->mem_freq = 1600; in cherryview_init_gt_powersave()
4919 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); in cherryview_init_gt_powersave()
4921 dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv); in cherryview_init_gt_powersave()
4922 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; in cherryview_init_gt_powersave()
4924 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq), in cherryview_init_gt_powersave()
4925 dev_priv->rps.max_freq); in cherryview_init_gt_powersave()
4927 dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv); in cherryview_init_gt_powersave()
4929 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), in cherryview_init_gt_powersave()
4930 dev_priv->rps.efficient_freq); in cherryview_init_gt_powersave()
4932 dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv); in cherryview_init_gt_powersave()
4934 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq), in cherryview_init_gt_powersave()
4935 dev_priv->rps.rp1_freq); in cherryview_init_gt_powersave()
4937 dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv); in cherryview_init_gt_powersave()
4939 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq), in cherryview_init_gt_powersave()
4940 dev_priv->rps.min_freq); in cherryview_init_gt_powersave()
4942 WARN_ONCE((dev_priv->rps.max_freq | in cherryview_init_gt_powersave()
4943 dev_priv->rps.efficient_freq | in cherryview_init_gt_powersave()
4944 dev_priv->rps.rp1_freq | in cherryview_init_gt_powersave()
4945 dev_priv->rps.min_freq) & 1, in cherryview_init_gt_powersave()
4948 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; in cherryview_init_gt_powersave()
4951 if (dev_priv->rps.max_freq_softlimit == 0) in cherryview_init_gt_powersave()
4952 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; in cherryview_init_gt_powersave()
4954 if (dev_priv->rps.min_freq_softlimit == 0) in cherryview_init_gt_powersave()
4955 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; in cherryview_init_gt_powersave()
4957 mutex_unlock(&dev_priv->rps.hw_lock); in cherryview_init_gt_powersave()
4967 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_enable_rps() local
4972 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in cherryview_enable_rps()
4981 cherryview_check_pctx(dev_priv); in cherryview_enable_rps()
4985 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in cherryview_enable_rps()
4995 for_each_ring(ring, dev_priv, i) in cherryview_enable_rps()
5035 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); in cherryview_enable_rps()
5043 dev_priv->rps.cur_freq = (val >> 8) & 0xff; in cherryview_enable_rps()
5045 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq), in cherryview_enable_rps()
5046 dev_priv->rps.cur_freq); in cherryview_enable_rps()
5049 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), in cherryview_enable_rps()
5050 dev_priv->rps.efficient_freq); in cherryview_enable_rps()
5052 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); in cherryview_enable_rps()
5054 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in cherryview_enable_rps()
5059 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_enable_rps() local
5064 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in valleyview_enable_rps()
5066 valleyview_check_pctx(dev_priv); in valleyview_enable_rps()
5075 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in valleyview_enable_rps()
5100 for_each_ring(ring, dev_priv, i) in valleyview_enable_rps()
5119 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); in valleyview_enable_rps()
5127 dev_priv->rps.cur_freq = (val >> 8) & 0xff; in valleyview_enable_rps()
5129 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq), in valleyview_enable_rps()
5130 dev_priv->rps.cur_freq); in valleyview_enable_rps()
5133 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), in valleyview_enable_rps()
5134 dev_priv->rps.efficient_freq); in valleyview_enable_rps()
5136 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); in valleyview_enable_rps()
5138 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in valleyview_enable_rps()
5170 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) in __i915_chipset_val() argument
5179 diff1 = now - dev_priv->ips.last_time1; in __i915_chipset_val()
5187 return dev_priv->ips.chipset_power; in __i915_chipset_val()
5196 if (total_count < dev_priv->ips.last_count1) { in __i915_chipset_val()
5197 diff = ~0UL - dev_priv->ips.last_count1; in __i915_chipset_val()
5200 diff = total_count - dev_priv->ips.last_count1; in __i915_chipset_val()
5204 if (cparams[i].i == dev_priv->ips.c_m && in __i915_chipset_val()
5205 cparams[i].t == dev_priv->ips.r_t) { in __i915_chipset_val()
5216 dev_priv->ips.last_count1 = total_count; in __i915_chipset_val()
5217 dev_priv->ips.last_time1 = now; in __i915_chipset_val()
5219 dev_priv->ips.chipset_power = ret; in __i915_chipset_val()
5224 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) in i915_chipset_val() argument
5226 struct drm_device *dev = dev_priv->dev; in i915_chipset_val()
5234 val = __i915_chipset_val(dev_priv); in i915_chipset_val()
5241 unsigned long i915_mch_val(struct drm_i915_private *dev_priv) in i915_mch_val() argument
5267 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) in pvid_to_extvid() argument
5269 struct drm_device *dev = dev_priv->dev; in pvid_to_extvid()
5279 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) in __i915_update_gfx_val() argument
5287 diffms = now - dev_priv->ips.last_time2; in __i915_update_gfx_val()
5296 if (count < dev_priv->ips.last_count2) { in __i915_update_gfx_val()
5297 diff = ~0UL - dev_priv->ips.last_count2; in __i915_update_gfx_val()
5300 diff = count - dev_priv->ips.last_count2; in __i915_update_gfx_val()
5303 dev_priv->ips.last_count2 = count; in __i915_update_gfx_val()
5304 dev_priv->ips.last_time2 = now; in __i915_update_gfx_val()
5309 dev_priv->ips.gfx_power = diff; in __i915_update_gfx_val()
5312 void i915_update_gfx_val(struct drm_i915_private *dev_priv) in i915_update_gfx_val() argument
5314 struct drm_device *dev = dev_priv->dev; in i915_update_gfx_val()
5321 __i915_update_gfx_val(dev_priv); in i915_update_gfx_val()
5326 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) in __i915_gfx_val() argument
5333 pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4)); in __i915_gfx_val()
5335 ext_v = pvid_to_extvid(dev_priv, pxvid); in __i915_gfx_val()
5339 t = i915_mch_val(dev_priv); in __i915_gfx_val()
5353 corr2 = (corr * dev_priv->ips.corr); in __i915_gfx_val()
5358 __i915_update_gfx_val(dev_priv); in __i915_gfx_val()
5360 return dev_priv->ips.gfx_power + state2; in __i915_gfx_val()
5363 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) in i915_gfx_val() argument
5365 struct drm_device *dev = dev_priv->dev; in i915_gfx_val()
5373 val = __i915_gfx_val(dev_priv); in i915_gfx_val()
5388 struct drm_i915_private *dev_priv; in i915_read_mch_val() local
5394 dev_priv = i915_mch_dev; in i915_read_mch_val()
5396 chipset_val = __i915_chipset_val(dev_priv); in i915_read_mch_val()
5397 graphics_val = __i915_gfx_val(dev_priv); in i915_read_mch_val()
5415 struct drm_i915_private *dev_priv; in i915_gpu_raise() local
5423 dev_priv = i915_mch_dev; in i915_gpu_raise()
5425 if (dev_priv->ips.max_delay > dev_priv->ips.fmax) in i915_gpu_raise()
5426 dev_priv->ips.max_delay--; in i915_gpu_raise()
5443 struct drm_i915_private *dev_priv; in i915_gpu_lower() local
5451 dev_priv = i915_mch_dev; in i915_gpu_lower()
5453 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay) in i915_gpu_lower()
5454 dev_priv->ips.max_delay++; in i915_gpu_lower()
5470 struct drm_i915_private *dev_priv; in i915_gpu_busy() local
5478 dev_priv = i915_mch_dev; in i915_gpu_busy()
5480 for_each_ring(ring, dev_priv, i) in i915_gpu_busy()
5498 struct drm_i915_private *dev_priv; in i915_gpu_turbo_disable() local
5506 dev_priv = i915_mch_dev; in i915_gpu_turbo_disable()
5508 dev_priv->ips.max_delay = dev_priv->ips.fstart; in i915_gpu_turbo_disable()
5510 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart)) in i915_gpu_turbo_disable()
5540 void intel_gpu_ips_init(struct drm_i915_private *dev_priv) in intel_gpu_ips_init() argument
5545 i915_mch_dev = dev_priv; in intel_gpu_ips_init()
5560 struct drm_i915_private *dev_priv = dev->dev_private; in intel_init_emon() local
5626 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); in intel_init_emon()
5649 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_suspend_rps() local
5651 flush_delayed_work(&dev_priv->rps.delayed_resume_work); in gen6_suspend_rps()
5666 struct drm_i915_private *dev_priv = dev->dev_private; in intel_suspend_gt_powersave() local
5674 gen6_rps_idle(dev_priv); in intel_suspend_gt_powersave()
5679 struct drm_i915_private *dev_priv = dev->dev_private; in intel_disable_gt_powersave() local
5686 mutex_lock(&dev_priv->rps.hw_lock); in intel_disable_gt_powersave()
5696 dev_priv->rps.enabled = false; in intel_disable_gt_powersave()
5697 mutex_unlock(&dev_priv->rps.hw_lock); in intel_disable_gt_powersave()
5703 struct drm_i915_private *dev_priv = in intel_gen6_powersave_work() local
5706 struct drm_device *dev = dev_priv->dev; in intel_gen6_powersave_work()
5708 mutex_lock(&dev_priv->rps.hw_lock); in intel_gen6_powersave_work()
5728 WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq); in intel_gen6_powersave_work()
5729 WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq); in intel_gen6_powersave_work()
5731 WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq); in intel_gen6_powersave_work()
5732 WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq); in intel_gen6_powersave_work()
5734 dev_priv->rps.enabled = true; in intel_gen6_powersave_work()
5738 mutex_unlock(&dev_priv->rps.hw_lock); in intel_gen6_powersave_work()
5740 intel_runtime_pm_put(dev_priv); in intel_gen6_powersave_work()
5745 struct drm_i915_private *dev_priv = dev->dev_private; in intel_enable_gt_powersave() local
5769 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work, in intel_enable_gt_powersave()
5771 intel_runtime_pm_get_noresume(dev_priv); in intel_enable_gt_powersave()
5777 struct drm_i915_private *dev_priv = dev->dev_private; in intel_reset_gt_powersave() local
5783 dev_priv->rps.enabled = false; in intel_reset_gt_powersave()
5788 struct drm_i915_private *dev_priv = dev->dev_private; in ibx_init_clock_gating() local
5800 struct drm_i915_private *dev_priv = dev->dev_private; in g4x_disable_trickle_feed() local
5803 for_each_pipe(dev_priv, pipe) { in g4x_disable_trickle_feed()
5807 intel_flush_primary_plane(dev_priv, pipe); in g4x_disable_trickle_feed()
5813 struct drm_i915_private *dev_priv = dev->dev_private; in ilk_init_lp_watermarks() local
5827 struct drm_i915_private *dev_priv = dev->dev_private; in ironlake_init_clock_gating() local
5901 struct drm_i915_private *dev_priv = dev->dev_private; in cpt_init_clock_gating() local
5918 for_each_pipe(dev_priv, pipe) { in cpt_init_clock_gating()
5922 if (dev_priv->vbt.fdi_rx_polarity_inverted) in cpt_init_clock_gating()
5930 for_each_pipe(dev_priv, pipe) { in cpt_init_clock_gating()
5938 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_check_mch_setup() local
5949 struct drm_i915_private *dev_priv = dev->dev_private; in gen6_init_clock_gating() local
6044 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) in gen7_setup_fixed_func_scheduler() argument
6064 struct drm_i915_private *dev_priv = dev->dev_private; in lpt_init_clock_gating() local
6070 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) in lpt_init_clock_gating()
6083 struct drm_i915_private *dev_priv = dev->dev_private; in lpt_suspend_hw() local
6085 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { in lpt_suspend_hw()
6095 struct drm_i915_private *dev_priv = dev->dev_private; in broadwell_init_clock_gating() local
6110 for_each_pipe(dev_priv, pipe) { in broadwell_init_clock_gating()
6134 struct drm_i915_private *dev_priv = dev->dev_private; in haswell_init_clock_gating() local
6190 struct drm_i915_private *dev_priv = dev->dev_private; in ivybridge_init_clock_gating() local
6252 gen7_setup_fixed_func_scheduler(dev_priv); in ivybridge_init_clock_gating()
6286 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv) in vlv_init_display_clock_gating() argument
6299 struct drm_i915_private *dev_priv = dev->dev_private; in valleyview_init_clock_gating() local
6301 vlv_init_display_clock_gating(dev_priv); in valleyview_init_clock_gating()
6334 gen7_setup_fixed_func_scheduler(dev_priv); in valleyview_init_clock_gating()
6383 struct drm_i915_private *dev_priv = dev->dev_private; in cherryview_init_clock_gating() local
6385 vlv_init_display_clock_gating(dev_priv); in cherryview_init_clock_gating()
6408 struct drm_i915_private *dev_priv = dev->dev_private; in g4x_init_clock_gating() local
6435 struct drm_i915_private *dev_priv = dev->dev_private; in crestline_init_clock_gating() local
6451 struct drm_i915_private *dev_priv = dev->dev_private; in broadwater_init_clock_gating() local
6468 struct drm_i915_private *dev_priv = dev->dev_private; in gen3_init_clock_gating() local
6493 struct drm_i915_private *dev_priv = dev->dev_private; in i85x_init_clock_gating() local
6507 struct drm_i915_private *dev_priv = dev->dev_private; in i830_init_clock_gating() local
6518 struct drm_i915_private *dev_priv = dev->dev_private; in intel_init_clock_gating() local
6520 if (dev_priv->display.init_clock_gating) in intel_init_clock_gating()
6521 dev_priv->display.init_clock_gating(dev); in intel_init_clock_gating()
6533 struct drm_i915_private *dev_priv = dev->dev_private; in intel_init_pm() local
6535 intel_fbc_init(dev_priv); in intel_init_pm()
6547 dev_priv->display.init_clock_gating = skl_init_clock_gating; in intel_init_pm()
6548 dev_priv->display.update_wm = skl_update_wm; in intel_init_pm()
6549 dev_priv->display.update_sprite_wm = skl_update_sprite_wm; in intel_init_pm()
6553 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] && in intel_init_pm()
6554 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || in intel_init_pm()
6555 (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] && in intel_init_pm()
6556 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { in intel_init_pm()
6557 dev_priv->display.update_wm = ilk_update_wm; in intel_init_pm()
6558 dev_priv->display.update_sprite_wm = ilk_update_sprite_wm; in intel_init_pm()
6565 dev_priv->display.init_clock_gating = ironlake_init_clock_gating; in intel_init_pm()
6567 dev_priv->display.init_clock_gating = gen6_init_clock_gating; in intel_init_pm()
6569 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating; in intel_init_pm()
6571 dev_priv->display.init_clock_gating = haswell_init_clock_gating; in intel_init_pm()
6573 dev_priv->display.init_clock_gating = broadwell_init_clock_gating; in intel_init_pm()
6575 dev_priv->display.update_wm = valleyview_update_wm; in intel_init_pm()
6576 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; in intel_init_pm()
6577 dev_priv->display.init_clock_gating = in intel_init_pm()
6580 dev_priv->display.update_wm = valleyview_update_wm; in intel_init_pm()
6581 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; in intel_init_pm()
6582 dev_priv->display.init_clock_gating = in intel_init_pm()
6586 dev_priv->is_ddr3, in intel_init_pm()
6587 dev_priv->fsb_freq, in intel_init_pm()
6588 dev_priv->mem_freq)) { in intel_init_pm()
6592 (dev_priv->is_ddr3 == 1) ? "3" : "2", in intel_init_pm()
6593 dev_priv->fsb_freq, dev_priv->mem_freq); in intel_init_pm()
6595 intel_set_memory_cxsr(dev_priv, false); in intel_init_pm()
6596 dev_priv->display.update_wm = NULL; in intel_init_pm()
6598 dev_priv->display.update_wm = pineview_update_wm; in intel_init_pm()
6599 dev_priv->display.init_clock_gating = gen3_init_clock_gating; in intel_init_pm()
6601 dev_priv->display.update_wm = g4x_update_wm; in intel_init_pm()
6602 dev_priv->display.init_clock_gating = g4x_init_clock_gating; in intel_init_pm()
6604 dev_priv->display.update_wm = i965_update_wm; in intel_init_pm()
6606 dev_priv->display.init_clock_gating = crestline_init_clock_gating; in intel_init_pm()
6608 dev_priv->display.init_clock_gating = broadwater_init_clock_gating; in intel_init_pm()
6610 dev_priv->display.update_wm = i9xx_update_wm; in intel_init_pm()
6611 dev_priv->display.get_fifo_size = i9xx_get_fifo_size; in intel_init_pm()
6612 dev_priv->display.init_clock_gating = gen3_init_clock_gating; in intel_init_pm()
6615 dev_priv->display.update_wm = i845_update_wm; in intel_init_pm()
6616 dev_priv->display.get_fifo_size = i845_get_fifo_size; in intel_init_pm()
6618 dev_priv->display.update_wm = i9xx_update_wm; in intel_init_pm()
6619 dev_priv->display.get_fifo_size = i830_get_fifo_size; in intel_init_pm()
6623 dev_priv->display.init_clock_gating = i85x_init_clock_gating; in intel_init_pm()
6625 dev_priv->display.init_clock_gating = i830_init_clock_gating; in intel_init_pm()
6631 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val) in sandybridge_pcode_read() argument
6633 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in sandybridge_pcode_read()
6656 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val) in sandybridge_pcode_write() argument
6658 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); in sandybridge_pcode_write()
6696 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) in byt_gpu_freq() argument
6698 int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4); in byt_gpu_freq()
6707 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) in byt_freq_opcode() argument
6709 int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4); in byt_freq_opcode()
6718 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) in chv_gpu_freq() argument
6720 int div, czclk_freq = dev_priv->rps.cz_freq; in chv_gpu_freq()
6729 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) in chv_freq_opcode() argument
6731 int mul, czclk_freq = dev_priv->rps.cz_freq; in chv_freq_opcode()
6741 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) in intel_gpu_freq() argument
6743 if (IS_GEN9(dev_priv->dev)) in intel_gpu_freq()
6745 else if (IS_CHERRYVIEW(dev_priv->dev)) in intel_gpu_freq()
6746 return chv_gpu_freq(dev_priv, val); in intel_gpu_freq()
6747 else if (IS_VALLEYVIEW(dev_priv->dev)) in intel_gpu_freq()
6748 return byt_gpu_freq(dev_priv, val); in intel_gpu_freq()
6753 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val) in intel_freq_opcode() argument
6755 if (IS_GEN9(dev_priv->dev)) in intel_freq_opcode()
6757 else if (IS_CHERRYVIEW(dev_priv->dev)) in intel_freq_opcode()
6758 return chv_freq_opcode(dev_priv, val); in intel_freq_opcode()
6759 else if (IS_VALLEYVIEW(dev_priv->dev)) in intel_freq_opcode()
6760 return byt_freq_opcode(dev_priv, val); in intel_freq_opcode()
6767 struct drm_i915_private *dev_priv = dev->dev_private; in intel_pm_setup() local
6769 mutex_init(&dev_priv->rps.hw_lock); in intel_pm_setup()
6771 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work, in intel_pm_setup()
6774 dev_priv->pm.suspended = false; in intel_pm_setup()