1/* 2 * Copyright © 2012 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eugeni Dodonov <eugeni.dodonov@intel.com> 25 * 26 */ 27 28#include <linux/cpufreq.h> 29#include "i915_drv.h" 30#include "intel_drv.h" 31#include "../../../platform/x86/intel_ips.h" 32#include <linux/module.h> 33 34/** 35 * RC6 is a special power stage which allows the GPU to enter an very 36 * low-voltage mode when idle, using down to 0V while at this stage. This 37 * stage is entered automatically when the GPU is idle when RC6 support is 38 * enabled, and as soon as new workload arises GPU wakes up automatically as well. 39 * 40 * There are different RC6 modes available in Intel GPU, which differentiate 41 * among each other with the latency required to enter and leave RC6 and 42 * voltage consumed by the GPU in different states. 43 * 44 * The combination of the following flags define which states GPU is allowed 45 * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and 46 * RC6pp is deepest RC6. Their support by hardware varies according to the 47 * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one 48 * which brings the most power savings; deeper states save more power, but 49 * require higher latency to switch to and wake up. 50 */ 51#define INTEL_RC6_ENABLE (1<<0) 52#define INTEL_RC6p_ENABLE (1<<1) 53#define INTEL_RC6pp_ENABLE (1<<2) 54 55static void gen9_init_clock_gating(struct drm_device *dev) 56{ 57 struct drm_i915_private *dev_priv = dev->dev_private; 58 59 /* WaEnableLbsSlaRetryTimerDecrement:skl */ 60 I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) | 61 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE); 62} 63 64static void skl_init_clock_gating(struct drm_device *dev) 65{ 66 struct drm_i915_private *dev_priv = dev->dev_private; 67 68 gen9_init_clock_gating(dev); 69 70 if (INTEL_REVID(dev) == SKL_REVID_A0) { 71 /* 72 * WaDisableSDEUnitClockGating:skl 73 * WaSetGAPSunitClckGateDisable:skl 74 */ 75 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 76 GEN8_GAPSUNIT_CLOCK_GATE_DISABLE | 77 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 78 } 79 80 if (INTEL_REVID(dev) <= SKL_REVID_D0) { 81 /* WaDisableHDCInvalidation:skl */ 82 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | 83 BDW_DISABLE_HDC_INVALIDATION); 84 85 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */ 86 I915_WRITE(FF_SLICE_CS_CHICKEN2, 87 I915_READ(FF_SLICE_CS_CHICKEN2) | 88 GEN9_TSG_BARRIER_ACK_DISABLE); 89 } 90 91 if (INTEL_REVID(dev) <= SKL_REVID_E0) 92 /* WaDisableLSQCROPERFforOCL:skl */ 93 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | 94 GEN8_LQSC_RO_PERF_DIS); 95} 96 97static void i915_pineview_get_mem_freq(struct drm_device *dev) 98{ 99 struct drm_i915_private *dev_priv = dev->dev_private; 100 u32 tmp; 101 102 tmp = I915_READ(CLKCFG); 103 104 switch (tmp & CLKCFG_FSB_MASK) { 105 case CLKCFG_FSB_533: 106 dev_priv->fsb_freq = 533; /* 133*4 */ 107 break; 108 case CLKCFG_FSB_800: 109 dev_priv->fsb_freq = 800; /* 200*4 */ 110 break; 111 case CLKCFG_FSB_667: 112 dev_priv->fsb_freq = 667; /* 167*4 */ 113 break; 114 case CLKCFG_FSB_400: 115 dev_priv->fsb_freq = 400; /* 100*4 */ 116 break; 117 } 118 119 switch (tmp & CLKCFG_MEM_MASK) { 120 case CLKCFG_MEM_533: 121 dev_priv->mem_freq = 533; 122 break; 123 case CLKCFG_MEM_667: 124 dev_priv->mem_freq = 667; 125 break; 126 case CLKCFG_MEM_800: 127 dev_priv->mem_freq = 800; 128 break; 129 } 130 131 /* detect pineview DDR3 setting */ 132 tmp = I915_READ(CSHRDDR3CTL); 133 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; 134} 135 136static void i915_ironlake_get_mem_freq(struct drm_device *dev) 137{ 138 struct drm_i915_private *dev_priv = dev->dev_private; 139 u16 ddrpll, csipll; 140 141 ddrpll = I915_READ16(DDRMPLL1); 142 csipll = I915_READ16(CSIPLL0); 143 144 switch (ddrpll & 0xff) { 145 case 0xc: 146 dev_priv->mem_freq = 800; 147 break; 148 case 0x10: 149 dev_priv->mem_freq = 1066; 150 break; 151 case 0x14: 152 dev_priv->mem_freq = 1333; 153 break; 154 case 0x18: 155 dev_priv->mem_freq = 1600; 156 break; 157 default: 158 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n", 159 ddrpll & 0xff); 160 dev_priv->mem_freq = 0; 161 break; 162 } 163 164 dev_priv->ips.r_t = dev_priv->mem_freq; 165 166 switch (csipll & 0x3ff) { 167 case 0x00c: 168 dev_priv->fsb_freq = 3200; 169 break; 170 case 0x00e: 171 dev_priv->fsb_freq = 3733; 172 break; 173 case 0x010: 174 dev_priv->fsb_freq = 4266; 175 break; 176 case 0x012: 177 dev_priv->fsb_freq = 4800; 178 break; 179 case 0x014: 180 dev_priv->fsb_freq = 5333; 181 break; 182 case 0x016: 183 dev_priv->fsb_freq = 5866; 184 break; 185 case 0x018: 186 dev_priv->fsb_freq = 6400; 187 break; 188 default: 189 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n", 190 csipll & 0x3ff); 191 dev_priv->fsb_freq = 0; 192 break; 193 } 194 195 if (dev_priv->fsb_freq == 3200) { 196 dev_priv->ips.c_m = 0; 197 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) { 198 dev_priv->ips.c_m = 1; 199 } else { 200 dev_priv->ips.c_m = 2; 201 } 202} 203 204static const struct cxsr_latency cxsr_latency_table[] = { 205 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */ 206 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */ 207 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */ 208 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */ 209 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */ 210 211 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */ 212 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */ 213 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */ 214 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */ 215 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */ 216 217 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */ 218 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */ 219 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */ 220 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */ 221 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */ 222 223 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */ 224 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */ 225 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */ 226 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */ 227 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */ 228 229 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */ 230 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */ 231 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */ 232 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */ 233 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */ 234 235 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */ 236 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */ 237 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */ 238 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */ 239 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */ 240}; 241 242static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop, 243 int is_ddr3, 244 int fsb, 245 int mem) 246{ 247 const struct cxsr_latency *latency; 248 int i; 249 250 if (fsb == 0 || mem == 0) 251 return NULL; 252 253 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) { 254 latency = &cxsr_latency_table[i]; 255 if (is_desktop == latency->is_desktop && 256 is_ddr3 == latency->is_ddr3 && 257 fsb == latency->fsb_freq && mem == latency->mem_freq) 258 return latency; 259 } 260 261 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n"); 262 263 return NULL; 264} 265 266static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable) 267{ 268 u32 val; 269 270 mutex_lock(&dev_priv->rps.hw_lock); 271 272 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); 273 if (enable) 274 val &= ~FORCE_DDR_HIGH_FREQ; 275 else 276 val |= FORCE_DDR_HIGH_FREQ; 277 val &= ~FORCE_DDR_LOW_FREQ; 278 val |= FORCE_DDR_FREQ_REQ_ACK; 279 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); 280 281 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & 282 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) 283 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n"); 284 285 mutex_unlock(&dev_priv->rps.hw_lock); 286} 287 288static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) 289{ 290 u32 val; 291 292 mutex_lock(&dev_priv->rps.hw_lock); 293 294 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); 295 if (enable) 296 val |= DSP_MAXFIFO_PM5_ENABLE; 297 else 298 val &= ~DSP_MAXFIFO_PM5_ENABLE; 299 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val); 300 301 mutex_unlock(&dev_priv->rps.hw_lock); 302} 303 304#define FW_WM(value, plane) \ 305 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK) 306 307void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) 308{ 309 struct drm_device *dev = dev_priv->dev; 310 u32 val; 311 312 if (IS_VALLEYVIEW(dev)) { 313 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0); 314 if (IS_CHERRYVIEW(dev)) 315 chv_set_memory_pm5(dev_priv, enable); 316 } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) { 317 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); 318 } else if (IS_PINEVIEW(dev)) { 319 val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN; 320 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0; 321 I915_WRITE(DSPFW3, val); 322 } else if (IS_I945G(dev) || IS_I945GM(dev)) { 323 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) : 324 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN); 325 I915_WRITE(FW_BLC_SELF, val); 326 } else if (IS_I915GM(dev)) { 327 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) : 328 _MASKED_BIT_DISABLE(INSTPM_SELF_EN); 329 I915_WRITE(INSTPM, val); 330 } else { 331 return; 332 } 333 334 DRM_DEBUG_KMS("memory self-refresh is %s\n", 335 enable ? "enabled" : "disabled"); 336} 337 338 339/* 340 * Latency for FIFO fetches is dependent on several factors: 341 * - memory configuration (speed, channels) 342 * - chipset 343 * - current MCH state 344 * It can be fairly high in some situations, so here we assume a fairly 345 * pessimal value. It's a tradeoff between extra memory fetches (if we 346 * set this value too high, the FIFO will fetch frequently to stay full) 347 * and power consumption (set it too low to save power and we might see 348 * FIFO underruns and display "flicker"). 349 * 350 * A value of 5us seems to be a good balance; safe for very low end 351 * platforms but not overly aggressive on lower latency configs. 352 */ 353static const int pessimal_latency_ns = 5000; 354 355#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \ 356 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8)) 357 358static int vlv_get_fifo_size(struct drm_device *dev, 359 enum pipe pipe, int plane) 360{ 361 struct drm_i915_private *dev_priv = dev->dev_private; 362 int sprite0_start, sprite1_start, size; 363 364 switch (pipe) { 365 uint32_t dsparb, dsparb2, dsparb3; 366 case PIPE_A: 367 dsparb = I915_READ(DSPARB); 368 dsparb2 = I915_READ(DSPARB2); 369 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0); 370 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4); 371 break; 372 case PIPE_B: 373 dsparb = I915_READ(DSPARB); 374 dsparb2 = I915_READ(DSPARB2); 375 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8); 376 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12); 377 break; 378 case PIPE_C: 379 dsparb2 = I915_READ(DSPARB2); 380 dsparb3 = I915_READ(DSPARB3); 381 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16); 382 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20); 383 break; 384 default: 385 return 0; 386 } 387 388 switch (plane) { 389 case 0: 390 size = sprite0_start; 391 break; 392 case 1: 393 size = sprite1_start - sprite0_start; 394 break; 395 case 2: 396 size = 512 - 1 - sprite1_start; 397 break; 398 default: 399 return 0; 400 } 401 402 DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n", 403 pipe_name(pipe), plane == 0 ? "primary" : "sprite", 404 plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1), 405 size); 406 407 return size; 408} 409 410static int i9xx_get_fifo_size(struct drm_device *dev, int plane) 411{ 412 struct drm_i915_private *dev_priv = dev->dev_private; 413 uint32_t dsparb = I915_READ(DSPARB); 414 int size; 415 416 size = dsparb & 0x7f; 417 if (plane) 418 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size; 419 420 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb, 421 plane ? "B" : "A", size); 422 423 return size; 424} 425 426static int i830_get_fifo_size(struct drm_device *dev, int plane) 427{ 428 struct drm_i915_private *dev_priv = dev->dev_private; 429 uint32_t dsparb = I915_READ(DSPARB); 430 int size; 431 432 size = dsparb & 0x1ff; 433 if (plane) 434 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size; 435 size >>= 1; /* Convert to cachelines */ 436 437 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb, 438 plane ? "B" : "A", size); 439 440 return size; 441} 442 443static int i845_get_fifo_size(struct drm_device *dev, int plane) 444{ 445 struct drm_i915_private *dev_priv = dev->dev_private; 446 uint32_t dsparb = I915_READ(DSPARB); 447 int size; 448 449 size = dsparb & 0x7f; 450 size >>= 2; /* Convert to cachelines */ 451 452 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb, 453 plane ? "B" : "A", 454 size); 455 456 return size; 457} 458 459/* Pineview has different values for various configs */ 460static const struct intel_watermark_params pineview_display_wm = { 461 .fifo_size = PINEVIEW_DISPLAY_FIFO, 462 .max_wm = PINEVIEW_MAX_WM, 463 .default_wm = PINEVIEW_DFT_WM, 464 .guard_size = PINEVIEW_GUARD_WM, 465 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 466}; 467static const struct intel_watermark_params pineview_display_hplloff_wm = { 468 .fifo_size = PINEVIEW_DISPLAY_FIFO, 469 .max_wm = PINEVIEW_MAX_WM, 470 .default_wm = PINEVIEW_DFT_HPLLOFF_WM, 471 .guard_size = PINEVIEW_GUARD_WM, 472 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 473}; 474static const struct intel_watermark_params pineview_cursor_wm = { 475 .fifo_size = PINEVIEW_CURSOR_FIFO, 476 .max_wm = PINEVIEW_CURSOR_MAX_WM, 477 .default_wm = PINEVIEW_CURSOR_DFT_WM, 478 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 479 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 480}; 481static const struct intel_watermark_params pineview_cursor_hplloff_wm = { 482 .fifo_size = PINEVIEW_CURSOR_FIFO, 483 .max_wm = PINEVIEW_CURSOR_MAX_WM, 484 .default_wm = PINEVIEW_CURSOR_DFT_WM, 485 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 486 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 487}; 488static const struct intel_watermark_params g4x_wm_info = { 489 .fifo_size = G4X_FIFO_SIZE, 490 .max_wm = G4X_MAX_WM, 491 .default_wm = G4X_MAX_WM, 492 .guard_size = 2, 493 .cacheline_size = G4X_FIFO_LINE_SIZE, 494}; 495static const struct intel_watermark_params g4x_cursor_wm_info = { 496 .fifo_size = I965_CURSOR_FIFO, 497 .max_wm = I965_CURSOR_MAX_WM, 498 .default_wm = I965_CURSOR_DFT_WM, 499 .guard_size = 2, 500 .cacheline_size = G4X_FIFO_LINE_SIZE, 501}; 502static const struct intel_watermark_params valleyview_wm_info = { 503 .fifo_size = VALLEYVIEW_FIFO_SIZE, 504 .max_wm = VALLEYVIEW_MAX_WM, 505 .default_wm = VALLEYVIEW_MAX_WM, 506 .guard_size = 2, 507 .cacheline_size = G4X_FIFO_LINE_SIZE, 508}; 509static const struct intel_watermark_params valleyview_cursor_wm_info = { 510 .fifo_size = I965_CURSOR_FIFO, 511 .max_wm = VALLEYVIEW_CURSOR_MAX_WM, 512 .default_wm = I965_CURSOR_DFT_WM, 513 .guard_size = 2, 514 .cacheline_size = G4X_FIFO_LINE_SIZE, 515}; 516static const struct intel_watermark_params i965_cursor_wm_info = { 517 .fifo_size = I965_CURSOR_FIFO, 518 .max_wm = I965_CURSOR_MAX_WM, 519 .default_wm = I965_CURSOR_DFT_WM, 520 .guard_size = 2, 521 .cacheline_size = I915_FIFO_LINE_SIZE, 522}; 523static const struct intel_watermark_params i945_wm_info = { 524 .fifo_size = I945_FIFO_SIZE, 525 .max_wm = I915_MAX_WM, 526 .default_wm = 1, 527 .guard_size = 2, 528 .cacheline_size = I915_FIFO_LINE_SIZE, 529}; 530static const struct intel_watermark_params i915_wm_info = { 531 .fifo_size = I915_FIFO_SIZE, 532 .max_wm = I915_MAX_WM, 533 .default_wm = 1, 534 .guard_size = 2, 535 .cacheline_size = I915_FIFO_LINE_SIZE, 536}; 537static const struct intel_watermark_params i830_a_wm_info = { 538 .fifo_size = I855GM_FIFO_SIZE, 539 .max_wm = I915_MAX_WM, 540 .default_wm = 1, 541 .guard_size = 2, 542 .cacheline_size = I830_FIFO_LINE_SIZE, 543}; 544static const struct intel_watermark_params i830_bc_wm_info = { 545 .fifo_size = I855GM_FIFO_SIZE, 546 .max_wm = I915_MAX_WM/2, 547 .default_wm = 1, 548 .guard_size = 2, 549 .cacheline_size = I830_FIFO_LINE_SIZE, 550}; 551static const struct intel_watermark_params i845_wm_info = { 552 .fifo_size = I830_FIFO_SIZE, 553 .max_wm = I915_MAX_WM, 554 .default_wm = 1, 555 .guard_size = 2, 556 .cacheline_size = I830_FIFO_LINE_SIZE, 557}; 558 559/** 560 * intel_calculate_wm - calculate watermark level 561 * @clock_in_khz: pixel clock 562 * @wm: chip FIFO params 563 * @pixel_size: display pixel size 564 * @latency_ns: memory latency for the platform 565 * 566 * Calculate the watermark level (the level at which the display plane will 567 * start fetching from memory again). Each chip has a different display 568 * FIFO size and allocation, so the caller needs to figure that out and pass 569 * in the correct intel_watermark_params structure. 570 * 571 * As the pixel clock runs, the FIFO will be drained at a rate that depends 572 * on the pixel size. When it reaches the watermark level, it'll start 573 * fetching FIFO line sized based chunks from memory until the FIFO fills 574 * past the watermark point. If the FIFO drains completely, a FIFO underrun 575 * will occur, and a display engine hang could result. 576 */ 577static unsigned long intel_calculate_wm(unsigned long clock_in_khz, 578 const struct intel_watermark_params *wm, 579 int fifo_size, 580 int pixel_size, 581 unsigned long latency_ns) 582{ 583 long entries_required, wm_size; 584 585 /* 586 * Note: we need to make sure we don't overflow for various clock & 587 * latency values. 588 * clocks go from a few thousand to several hundred thousand. 589 * latency is usually a few thousand 590 */ 591 entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) / 592 1000; 593 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size); 594 595 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required); 596 597 wm_size = fifo_size - (entries_required + wm->guard_size); 598 599 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size); 600 601 /* Don't promote wm_size to unsigned... */ 602 if (wm_size > (long)wm->max_wm) 603 wm_size = wm->max_wm; 604 if (wm_size <= 0) 605 wm_size = wm->default_wm; 606 607 /* 608 * Bspec seems to indicate that the value shouldn't be lower than 609 * 'burst size + 1'. Certainly 830 is quite unhappy with low values. 610 * Lets go for 8 which is the burst size since certain platforms 611 * already use a hardcoded 8 (which is what the spec says should be 612 * done). 613 */ 614 if (wm_size <= 8) 615 wm_size = 8; 616 617 return wm_size; 618} 619 620static struct drm_crtc *single_enabled_crtc(struct drm_device *dev) 621{ 622 struct drm_crtc *crtc, *enabled = NULL; 623 624 for_each_crtc(dev, crtc) { 625 if (intel_crtc_active(crtc)) { 626 if (enabled) 627 return NULL; 628 enabled = crtc; 629 } 630 } 631 632 return enabled; 633} 634 635static void pineview_update_wm(struct drm_crtc *unused_crtc) 636{ 637 struct drm_device *dev = unused_crtc->dev; 638 struct drm_i915_private *dev_priv = dev->dev_private; 639 struct drm_crtc *crtc; 640 const struct cxsr_latency *latency; 641 u32 reg; 642 unsigned long wm; 643 644 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3, 645 dev_priv->fsb_freq, dev_priv->mem_freq); 646 if (!latency) { 647 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n"); 648 intel_set_memory_cxsr(dev_priv, false); 649 return; 650 } 651 652 crtc = single_enabled_crtc(dev); 653 if (crtc) { 654 const struct drm_display_mode *adjusted_mode; 655 int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8; 656 int clock; 657 658 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; 659 clock = adjusted_mode->crtc_clock; 660 661 /* Display SR */ 662 wm = intel_calculate_wm(clock, &pineview_display_wm, 663 pineview_display_wm.fifo_size, 664 pixel_size, latency->display_sr); 665 reg = I915_READ(DSPFW1); 666 reg &= ~DSPFW_SR_MASK; 667 reg |= FW_WM(wm, SR); 668 I915_WRITE(DSPFW1, reg); 669 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg); 670 671 /* cursor SR */ 672 wm = intel_calculate_wm(clock, &pineview_cursor_wm, 673 pineview_display_wm.fifo_size, 674 pixel_size, latency->cursor_sr); 675 reg = I915_READ(DSPFW3); 676 reg &= ~DSPFW_CURSOR_SR_MASK; 677 reg |= FW_WM(wm, CURSOR_SR); 678 I915_WRITE(DSPFW3, reg); 679 680 /* Display HPLL off SR */ 681 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm, 682 pineview_display_hplloff_wm.fifo_size, 683 pixel_size, latency->display_hpll_disable); 684 reg = I915_READ(DSPFW3); 685 reg &= ~DSPFW_HPLL_SR_MASK; 686 reg |= FW_WM(wm, HPLL_SR); 687 I915_WRITE(DSPFW3, reg); 688 689 /* cursor HPLL off SR */ 690 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm, 691 pineview_display_hplloff_wm.fifo_size, 692 pixel_size, latency->cursor_hpll_disable); 693 reg = I915_READ(DSPFW3); 694 reg &= ~DSPFW_HPLL_CURSOR_MASK; 695 reg |= FW_WM(wm, HPLL_CURSOR); 696 I915_WRITE(DSPFW3, reg); 697 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg); 698 699 intel_set_memory_cxsr(dev_priv, true); 700 } else { 701 intel_set_memory_cxsr(dev_priv, false); 702 } 703} 704 705static bool g4x_compute_wm0(struct drm_device *dev, 706 int plane, 707 const struct intel_watermark_params *display, 708 int display_latency_ns, 709 const struct intel_watermark_params *cursor, 710 int cursor_latency_ns, 711 int *plane_wm, 712 int *cursor_wm) 713{ 714 struct drm_crtc *crtc; 715 const struct drm_display_mode *adjusted_mode; 716 int htotal, hdisplay, clock, pixel_size; 717 int line_time_us, line_count; 718 int entries, tlb_miss; 719 720 crtc = intel_get_crtc_for_plane(dev, plane); 721 if (!intel_crtc_active(crtc)) { 722 *cursor_wm = cursor->guard_size; 723 *plane_wm = display->guard_size; 724 return false; 725 } 726 727 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; 728 clock = adjusted_mode->crtc_clock; 729 htotal = adjusted_mode->crtc_htotal; 730 hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; 731 pixel_size = crtc->primary->state->fb->bits_per_pixel / 8; 732 733 /* Use the small buffer method to calculate plane watermark */ 734 entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000; 735 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8; 736 if (tlb_miss > 0) 737 entries += tlb_miss; 738 entries = DIV_ROUND_UP(entries, display->cacheline_size); 739 *plane_wm = entries + display->guard_size; 740 if (*plane_wm > (int)display->max_wm) 741 *plane_wm = display->max_wm; 742 743 /* Use the large buffer method to calculate cursor watermark */ 744 line_time_us = max(htotal * 1000 / clock, 1); 745 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000; 746 entries = line_count * crtc->cursor->state->crtc_w * pixel_size; 747 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8; 748 if (tlb_miss > 0) 749 entries += tlb_miss; 750 entries = DIV_ROUND_UP(entries, cursor->cacheline_size); 751 *cursor_wm = entries + cursor->guard_size; 752 if (*cursor_wm > (int)cursor->max_wm) 753 *cursor_wm = (int)cursor->max_wm; 754 755 return true; 756} 757 758/* 759 * Check the wm result. 760 * 761 * If any calculated watermark values is larger than the maximum value that 762 * can be programmed into the associated watermark register, that watermark 763 * must be disabled. 764 */ 765static bool g4x_check_srwm(struct drm_device *dev, 766 int display_wm, int cursor_wm, 767 const struct intel_watermark_params *display, 768 const struct intel_watermark_params *cursor) 769{ 770 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n", 771 display_wm, cursor_wm); 772 773 if (display_wm > display->max_wm) { 774 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n", 775 display_wm, display->max_wm); 776 return false; 777 } 778 779 if (cursor_wm > cursor->max_wm) { 780 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n", 781 cursor_wm, cursor->max_wm); 782 return false; 783 } 784 785 if (!(display_wm || cursor_wm)) { 786 DRM_DEBUG_KMS("SR latency is 0, disabling\n"); 787 return false; 788 } 789 790 return true; 791} 792 793static bool g4x_compute_srwm(struct drm_device *dev, 794 int plane, 795 int latency_ns, 796 const struct intel_watermark_params *display, 797 const struct intel_watermark_params *cursor, 798 int *display_wm, int *cursor_wm) 799{ 800 struct drm_crtc *crtc; 801 const struct drm_display_mode *adjusted_mode; 802 int hdisplay, htotal, pixel_size, clock; 803 unsigned long line_time_us; 804 int line_count, line_size; 805 int small, large; 806 int entries; 807 808 if (!latency_ns) { 809 *display_wm = *cursor_wm = 0; 810 return false; 811 } 812 813 crtc = intel_get_crtc_for_plane(dev, plane); 814 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; 815 clock = adjusted_mode->crtc_clock; 816 htotal = adjusted_mode->crtc_htotal; 817 hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; 818 pixel_size = crtc->primary->state->fb->bits_per_pixel / 8; 819 820 line_time_us = max(htotal * 1000 / clock, 1); 821 line_count = (latency_ns / line_time_us + 1000) / 1000; 822 line_size = hdisplay * pixel_size; 823 824 /* Use the minimum of the small and large buffer method for primary */ 825 small = ((clock * pixel_size / 1000) * latency_ns) / 1000; 826 large = line_count * line_size; 827 828 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size); 829 *display_wm = entries + display->guard_size; 830 831 /* calculate the self-refresh watermark for display cursor */ 832 entries = line_count * pixel_size * crtc->cursor->state->crtc_w; 833 entries = DIV_ROUND_UP(entries, cursor->cacheline_size); 834 *cursor_wm = entries + cursor->guard_size; 835 836 return g4x_check_srwm(dev, 837 *display_wm, *cursor_wm, 838 display, cursor); 839} 840 841#define FW_WM_VLV(value, plane) \ 842 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV) 843 844static void vlv_write_wm_values(struct intel_crtc *crtc, 845 const struct vlv_wm_values *wm) 846{ 847 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 848 enum pipe pipe = crtc->pipe; 849 850 I915_WRITE(VLV_DDL(pipe), 851 (wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) | 852 (wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) | 853 (wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) | 854 (wm->ddl[pipe].primary << DDL_PLANE_SHIFT)); 855 856 I915_WRITE(DSPFW1, 857 FW_WM(wm->sr.plane, SR) | 858 FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) | 859 FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) | 860 FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA)); 861 I915_WRITE(DSPFW2, 862 FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) | 863 FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) | 864 FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA)); 865 I915_WRITE(DSPFW3, 866 FW_WM(wm->sr.cursor, CURSOR_SR)); 867 868 if (IS_CHERRYVIEW(dev_priv)) { 869 I915_WRITE(DSPFW7_CHV, 870 FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) | 871 FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC)); 872 I915_WRITE(DSPFW8_CHV, 873 FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) | 874 FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE)); 875 I915_WRITE(DSPFW9_CHV, 876 FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) | 877 FW_WM(wm->pipe[PIPE_C].cursor, CURSORC)); 878 I915_WRITE(DSPHOWM, 879 FW_WM(wm->sr.plane >> 9, SR_HI) | 880 FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) | 881 FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) | 882 FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) | 883 FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) | 884 FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) | 885 FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) | 886 FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) | 887 FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) | 888 FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI)); 889 } else { 890 I915_WRITE(DSPFW7, 891 FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) | 892 FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC)); 893 I915_WRITE(DSPHOWM, 894 FW_WM(wm->sr.plane >> 9, SR_HI) | 895 FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) | 896 FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) | 897 FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) | 898 FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) | 899 FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) | 900 FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI)); 901 } 902 903 POSTING_READ(DSPFW1); 904 905 dev_priv->wm.vlv = *wm; 906} 907 908#undef FW_WM_VLV 909 910static uint8_t vlv_compute_drain_latency(struct drm_crtc *crtc, 911 struct drm_plane *plane) 912{ 913 struct drm_device *dev = crtc->dev; 914 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 915 int entries, prec_mult, drain_latency, pixel_size; 916 int clock = intel_crtc->config->base.adjusted_mode.crtc_clock; 917 const int high_precision = IS_CHERRYVIEW(dev) ? 16 : 64; 918 919 /* 920 * FIXME the plane might have an fb 921 * but be invisible (eg. due to clipping) 922 */ 923 if (!intel_crtc->active || !plane->state->fb) 924 return 0; 925 926 if (WARN(clock == 0, "Pixel clock is zero!\n")) 927 return 0; 928 929 pixel_size = drm_format_plane_cpp(plane->state->fb->pixel_format, 0); 930 931 if (WARN(pixel_size == 0, "Pixel size is zero!\n")) 932 return 0; 933 934 entries = DIV_ROUND_UP(clock, 1000) * pixel_size; 935 936 prec_mult = high_precision; 937 drain_latency = 64 * prec_mult * 4 / entries; 938 939 if (drain_latency > DRAIN_LATENCY_MASK) { 940 prec_mult /= 2; 941 drain_latency = 64 * prec_mult * 4 / entries; 942 } 943 944 if (drain_latency > DRAIN_LATENCY_MASK) 945 drain_latency = DRAIN_LATENCY_MASK; 946 947 return drain_latency | (prec_mult == high_precision ? 948 DDL_PRECISION_HIGH : DDL_PRECISION_LOW); 949} 950 951static int vlv_compute_wm(struct intel_crtc *crtc, 952 struct intel_plane *plane, 953 int fifo_size) 954{ 955 int clock, entries, pixel_size; 956 957 /* 958 * FIXME the plane might have an fb 959 * but be invisible (eg. due to clipping) 960 */ 961 if (!crtc->active || !plane->base.state->fb) 962 return 0; 963 964 pixel_size = drm_format_plane_cpp(plane->base.state->fb->pixel_format, 0); 965 clock = crtc->config->base.adjusted_mode.crtc_clock; 966 967 entries = DIV_ROUND_UP(clock, 1000) * pixel_size; 968 969 /* 970 * Set up the watermark such that we don't start issuing memory 971 * requests until we are within PND's max deadline value (256us). 972 * Idea being to be idle as long as possible while still taking 973 * advatange of PND's deadline scheduling. The limit of 8 974 * cachelines (used when the FIFO will anyway drain in less time 975 * than 256us) should match what we would be done if trickle 976 * feed were enabled. 977 */ 978 return fifo_size - clamp(DIV_ROUND_UP(256 * entries, 64), 0, fifo_size - 8); 979} 980 981static bool vlv_compute_sr_wm(struct drm_device *dev, 982 struct vlv_wm_values *wm) 983{ 984 struct drm_i915_private *dev_priv = to_i915(dev); 985 struct drm_crtc *crtc; 986 enum pipe pipe = INVALID_PIPE; 987 int num_planes = 0; 988 int fifo_size = 0; 989 struct intel_plane *plane; 990 991 wm->sr.cursor = wm->sr.plane = 0; 992 993 crtc = single_enabled_crtc(dev); 994 /* maxfifo not supported on pipe C */ 995 if (crtc && to_intel_crtc(crtc)->pipe != PIPE_C) { 996 pipe = to_intel_crtc(crtc)->pipe; 997 num_planes = !!wm->pipe[pipe].primary + 998 !!wm->pipe[pipe].sprite[0] + 999 !!wm->pipe[pipe].sprite[1]; 1000 fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1; 1001 } 1002 1003 if (fifo_size == 0 || num_planes > 1) 1004 return false; 1005 1006 wm->sr.cursor = vlv_compute_wm(to_intel_crtc(crtc), 1007 to_intel_plane(crtc->cursor), 0x3f); 1008 1009 list_for_each_entry(plane, &dev->mode_config.plane_list, base.head) { 1010 if (plane->base.type == DRM_PLANE_TYPE_CURSOR) 1011 continue; 1012 1013 if (plane->pipe != pipe) 1014 continue; 1015 1016 wm->sr.plane = vlv_compute_wm(to_intel_crtc(crtc), 1017 plane, fifo_size); 1018 if (wm->sr.plane != 0) 1019 break; 1020 } 1021 1022 return true; 1023} 1024 1025static void valleyview_update_wm(struct drm_crtc *crtc) 1026{ 1027 struct drm_device *dev = crtc->dev; 1028 struct drm_i915_private *dev_priv = dev->dev_private; 1029 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1030 enum pipe pipe = intel_crtc->pipe; 1031 bool cxsr_enabled; 1032 struct vlv_wm_values wm = dev_priv->wm.vlv; 1033 1034 wm.ddl[pipe].primary = vlv_compute_drain_latency(crtc, crtc->primary); 1035 wm.pipe[pipe].primary = vlv_compute_wm(intel_crtc, 1036 to_intel_plane(crtc->primary), 1037 vlv_get_fifo_size(dev, pipe, 0)); 1038 1039 wm.ddl[pipe].cursor = vlv_compute_drain_latency(crtc, crtc->cursor); 1040 wm.pipe[pipe].cursor = vlv_compute_wm(intel_crtc, 1041 to_intel_plane(crtc->cursor), 1042 0x3f); 1043 1044 cxsr_enabled = vlv_compute_sr_wm(dev, &wm); 1045 1046 if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0) 1047 return; 1048 1049 DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, " 1050 "SR: plane=%d, cursor=%d\n", pipe_name(pipe), 1051 wm.pipe[pipe].primary, wm.pipe[pipe].cursor, 1052 wm.sr.plane, wm.sr.cursor); 1053 1054 /* 1055 * FIXME DDR DVFS introduces massive memory latencies which 1056 * are not known to system agent so any deadline specified 1057 * by the display may not be respected. To support DDR DVFS 1058 * the watermark code needs to be rewritten to essentially 1059 * bypass deadline mechanism and rely solely on the 1060 * watermarks. For now disable DDR DVFS. 1061 */ 1062 if (IS_CHERRYVIEW(dev_priv)) 1063 chv_set_memory_dvfs(dev_priv, false); 1064 1065 if (!cxsr_enabled) 1066 intel_set_memory_cxsr(dev_priv, false); 1067 1068 vlv_write_wm_values(intel_crtc, &wm); 1069 1070 if (cxsr_enabled) 1071 intel_set_memory_cxsr(dev_priv, true); 1072} 1073 1074static void valleyview_update_sprite_wm(struct drm_plane *plane, 1075 struct drm_crtc *crtc, 1076 uint32_t sprite_width, 1077 uint32_t sprite_height, 1078 int pixel_size, 1079 bool enabled, bool scaled) 1080{ 1081 struct drm_device *dev = crtc->dev; 1082 struct drm_i915_private *dev_priv = dev->dev_private; 1083 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1084 enum pipe pipe = intel_crtc->pipe; 1085 int sprite = to_intel_plane(plane)->plane; 1086 bool cxsr_enabled; 1087 struct vlv_wm_values wm = dev_priv->wm.vlv; 1088 1089 if (enabled) { 1090 wm.ddl[pipe].sprite[sprite] = 1091 vlv_compute_drain_latency(crtc, plane); 1092 1093 wm.pipe[pipe].sprite[sprite] = 1094 vlv_compute_wm(intel_crtc, 1095 to_intel_plane(plane), 1096 vlv_get_fifo_size(dev, pipe, sprite+1)); 1097 } else { 1098 wm.ddl[pipe].sprite[sprite] = 0; 1099 wm.pipe[pipe].sprite[sprite] = 0; 1100 } 1101 1102 cxsr_enabled = vlv_compute_sr_wm(dev, &wm); 1103 1104 if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0) 1105 return; 1106 1107 DRM_DEBUG_KMS("Setting FIFO watermarks - %c: sprite %c=%d, " 1108 "SR: plane=%d, cursor=%d\n", pipe_name(pipe), 1109 sprite_name(pipe, sprite), 1110 wm.pipe[pipe].sprite[sprite], 1111 wm.sr.plane, wm.sr.cursor); 1112 1113 if (!cxsr_enabled) 1114 intel_set_memory_cxsr(dev_priv, false); 1115 1116 vlv_write_wm_values(intel_crtc, &wm); 1117 1118 if (cxsr_enabled) 1119 intel_set_memory_cxsr(dev_priv, true); 1120} 1121 1122#define single_plane_enabled(mask) is_power_of_2(mask) 1123 1124static void g4x_update_wm(struct drm_crtc *crtc) 1125{ 1126 struct drm_device *dev = crtc->dev; 1127 static const int sr_latency_ns = 12000; 1128 struct drm_i915_private *dev_priv = dev->dev_private; 1129 int planea_wm, planeb_wm, cursora_wm, cursorb_wm; 1130 int plane_sr, cursor_sr; 1131 unsigned int enabled = 0; 1132 bool cxsr_enabled; 1133 1134 if (g4x_compute_wm0(dev, PIPE_A, 1135 &g4x_wm_info, pessimal_latency_ns, 1136 &g4x_cursor_wm_info, pessimal_latency_ns, 1137 &planea_wm, &cursora_wm)) 1138 enabled |= 1 << PIPE_A; 1139 1140 if (g4x_compute_wm0(dev, PIPE_B, 1141 &g4x_wm_info, pessimal_latency_ns, 1142 &g4x_cursor_wm_info, pessimal_latency_ns, 1143 &planeb_wm, &cursorb_wm)) 1144 enabled |= 1 << PIPE_B; 1145 1146 if (single_plane_enabled(enabled) && 1147 g4x_compute_srwm(dev, ffs(enabled) - 1, 1148 sr_latency_ns, 1149 &g4x_wm_info, 1150 &g4x_cursor_wm_info, 1151 &plane_sr, &cursor_sr)) { 1152 cxsr_enabled = true; 1153 } else { 1154 cxsr_enabled = false; 1155 intel_set_memory_cxsr(dev_priv, false); 1156 plane_sr = cursor_sr = 0; 1157 } 1158 1159 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " 1160 "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", 1161 planea_wm, cursora_wm, 1162 planeb_wm, cursorb_wm, 1163 plane_sr, cursor_sr); 1164 1165 I915_WRITE(DSPFW1, 1166 FW_WM(plane_sr, SR) | 1167 FW_WM(cursorb_wm, CURSORB) | 1168 FW_WM(planeb_wm, PLANEB) | 1169 FW_WM(planea_wm, PLANEA)); 1170 I915_WRITE(DSPFW2, 1171 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | 1172 FW_WM(cursora_wm, CURSORA)); 1173 /* HPLL off in SR has some issues on G4x... disable it */ 1174 I915_WRITE(DSPFW3, 1175 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) | 1176 FW_WM(cursor_sr, CURSOR_SR)); 1177 1178 if (cxsr_enabled) 1179 intel_set_memory_cxsr(dev_priv, true); 1180} 1181 1182static void i965_update_wm(struct drm_crtc *unused_crtc) 1183{ 1184 struct drm_device *dev = unused_crtc->dev; 1185 struct drm_i915_private *dev_priv = dev->dev_private; 1186 struct drm_crtc *crtc; 1187 int srwm = 1; 1188 int cursor_sr = 16; 1189 bool cxsr_enabled; 1190 1191 /* Calc sr entries for one plane configs */ 1192 crtc = single_enabled_crtc(dev); 1193 if (crtc) { 1194 /* self-refresh has much higher latency */ 1195 static const int sr_latency_ns = 12000; 1196 const struct drm_display_mode *adjusted_mode = 1197 &to_intel_crtc(crtc)->config->base.adjusted_mode; 1198 int clock = adjusted_mode->crtc_clock; 1199 int htotal = adjusted_mode->crtc_htotal; 1200 int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; 1201 int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8; 1202 unsigned long line_time_us; 1203 int entries; 1204 1205 line_time_us = max(htotal * 1000 / clock, 1); 1206 1207 /* Use ns/us then divide to preserve precision */ 1208 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * 1209 pixel_size * hdisplay; 1210 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE); 1211 srwm = I965_FIFO_SIZE - entries; 1212 if (srwm < 0) 1213 srwm = 1; 1214 srwm &= 0x1ff; 1215 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n", 1216 entries, srwm); 1217 1218 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * 1219 pixel_size * crtc->cursor->state->crtc_w; 1220 entries = DIV_ROUND_UP(entries, 1221 i965_cursor_wm_info.cacheline_size); 1222 cursor_sr = i965_cursor_wm_info.fifo_size - 1223 (entries + i965_cursor_wm_info.guard_size); 1224 1225 if (cursor_sr > i965_cursor_wm_info.max_wm) 1226 cursor_sr = i965_cursor_wm_info.max_wm; 1227 1228 DRM_DEBUG_KMS("self-refresh watermark: display plane %d " 1229 "cursor %d\n", srwm, cursor_sr); 1230 1231 cxsr_enabled = true; 1232 } else { 1233 cxsr_enabled = false; 1234 /* Turn off self refresh if both pipes are enabled */ 1235 intel_set_memory_cxsr(dev_priv, false); 1236 } 1237 1238 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n", 1239 srwm); 1240 1241 /* 965 has limitations... */ 1242 I915_WRITE(DSPFW1, FW_WM(srwm, SR) | 1243 FW_WM(8, CURSORB) | 1244 FW_WM(8, PLANEB) | 1245 FW_WM(8, PLANEA)); 1246 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) | 1247 FW_WM(8, PLANEC_OLD)); 1248 /* update cursor SR watermark */ 1249 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR)); 1250 1251 if (cxsr_enabled) 1252 intel_set_memory_cxsr(dev_priv, true); 1253} 1254 1255#undef FW_WM 1256 1257static void i9xx_update_wm(struct drm_crtc *unused_crtc) 1258{ 1259 struct drm_device *dev = unused_crtc->dev; 1260 struct drm_i915_private *dev_priv = dev->dev_private; 1261 const struct intel_watermark_params *wm_info; 1262 uint32_t fwater_lo; 1263 uint32_t fwater_hi; 1264 int cwm, srwm = 1; 1265 int fifo_size; 1266 int planea_wm, planeb_wm; 1267 struct drm_crtc *crtc, *enabled = NULL; 1268 1269 if (IS_I945GM(dev)) 1270 wm_info = &i945_wm_info; 1271 else if (!IS_GEN2(dev)) 1272 wm_info = &i915_wm_info; 1273 else 1274 wm_info = &i830_a_wm_info; 1275 1276 fifo_size = dev_priv->display.get_fifo_size(dev, 0); 1277 crtc = intel_get_crtc_for_plane(dev, 0); 1278 if (intel_crtc_active(crtc)) { 1279 const struct drm_display_mode *adjusted_mode; 1280 int cpp = crtc->primary->state->fb->bits_per_pixel / 8; 1281 if (IS_GEN2(dev)) 1282 cpp = 4; 1283 1284 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; 1285 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 1286 wm_info, fifo_size, cpp, 1287 pessimal_latency_ns); 1288 enabled = crtc; 1289 } else { 1290 planea_wm = fifo_size - wm_info->guard_size; 1291 if (planea_wm > (long)wm_info->max_wm) 1292 planea_wm = wm_info->max_wm; 1293 } 1294 1295 if (IS_GEN2(dev)) 1296 wm_info = &i830_bc_wm_info; 1297 1298 fifo_size = dev_priv->display.get_fifo_size(dev, 1); 1299 crtc = intel_get_crtc_for_plane(dev, 1); 1300 if (intel_crtc_active(crtc)) { 1301 const struct drm_display_mode *adjusted_mode; 1302 int cpp = crtc->primary->state->fb->bits_per_pixel / 8; 1303 if (IS_GEN2(dev)) 1304 cpp = 4; 1305 1306 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; 1307 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 1308 wm_info, fifo_size, cpp, 1309 pessimal_latency_ns); 1310 if (enabled == NULL) 1311 enabled = crtc; 1312 else 1313 enabled = NULL; 1314 } else { 1315 planeb_wm = fifo_size - wm_info->guard_size; 1316 if (planeb_wm > (long)wm_info->max_wm) 1317 planeb_wm = wm_info->max_wm; 1318 } 1319 1320 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); 1321 1322 if (IS_I915GM(dev) && enabled) { 1323 struct drm_i915_gem_object *obj; 1324 1325 obj = intel_fb_obj(enabled->primary->state->fb); 1326 1327 /* self-refresh seems busted with untiled */ 1328 if (obj->tiling_mode == I915_TILING_NONE) 1329 enabled = NULL; 1330 } 1331 1332 /* 1333 * Overlay gets an aggressive default since video jitter is bad. 1334 */ 1335 cwm = 2; 1336 1337 /* Play safe and disable self-refresh before adjusting watermarks. */ 1338 intel_set_memory_cxsr(dev_priv, false); 1339 1340 /* Calc sr entries for one plane configs */ 1341 if (HAS_FW_BLC(dev) && enabled) { 1342 /* self-refresh has much higher latency */ 1343 static const int sr_latency_ns = 6000; 1344 const struct drm_display_mode *adjusted_mode = 1345 &to_intel_crtc(enabled)->config->base.adjusted_mode; 1346 int clock = adjusted_mode->crtc_clock; 1347 int htotal = adjusted_mode->crtc_htotal; 1348 int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w; 1349 int pixel_size = enabled->primary->state->fb->bits_per_pixel / 8; 1350 unsigned long line_time_us; 1351 int entries; 1352 1353 line_time_us = max(htotal * 1000 / clock, 1); 1354 1355 /* Use ns/us then divide to preserve precision */ 1356 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * 1357 pixel_size * hdisplay; 1358 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size); 1359 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries); 1360 srwm = wm_info->fifo_size - entries; 1361 if (srwm < 0) 1362 srwm = 1; 1363 1364 if (IS_I945G(dev) || IS_I945GM(dev)) 1365 I915_WRITE(FW_BLC_SELF, 1366 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff)); 1367 else if (IS_I915GM(dev)) 1368 I915_WRITE(FW_BLC_SELF, srwm & 0x3f); 1369 } 1370 1371 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n", 1372 planea_wm, planeb_wm, cwm, srwm); 1373 1374 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f); 1375 fwater_hi = (cwm & 0x1f); 1376 1377 /* Set request length to 8 cachelines per fetch */ 1378 fwater_lo = fwater_lo | (1 << 24) | (1 << 8); 1379 fwater_hi = fwater_hi | (1 << 8); 1380 1381 I915_WRITE(FW_BLC, fwater_lo); 1382 I915_WRITE(FW_BLC2, fwater_hi); 1383 1384 if (enabled) 1385 intel_set_memory_cxsr(dev_priv, true); 1386} 1387 1388static void i845_update_wm(struct drm_crtc *unused_crtc) 1389{ 1390 struct drm_device *dev = unused_crtc->dev; 1391 struct drm_i915_private *dev_priv = dev->dev_private; 1392 struct drm_crtc *crtc; 1393 const struct drm_display_mode *adjusted_mode; 1394 uint32_t fwater_lo; 1395 int planea_wm; 1396 1397 crtc = single_enabled_crtc(dev); 1398 if (crtc == NULL) 1399 return; 1400 1401 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; 1402 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 1403 &i845_wm_info, 1404 dev_priv->display.get_fifo_size(dev, 0), 1405 4, pessimal_latency_ns); 1406 fwater_lo = I915_READ(FW_BLC) & ~0xfff; 1407 fwater_lo |= (3<<8) | planea_wm; 1408 1409 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm); 1410 1411 I915_WRITE(FW_BLC, fwater_lo); 1412} 1413 1414static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev, 1415 struct drm_crtc *crtc) 1416{ 1417 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1418 uint32_t pixel_rate; 1419 1420 pixel_rate = intel_crtc->config->base.adjusted_mode.crtc_clock; 1421 1422 /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to 1423 * adjust the pixel_rate here. */ 1424 1425 if (intel_crtc->config->pch_pfit.enabled) { 1426 uint64_t pipe_w, pipe_h, pfit_w, pfit_h; 1427 uint32_t pfit_size = intel_crtc->config->pch_pfit.size; 1428 1429 pipe_w = intel_crtc->config->pipe_src_w; 1430 pipe_h = intel_crtc->config->pipe_src_h; 1431 pfit_w = (pfit_size >> 16) & 0xFFFF; 1432 pfit_h = pfit_size & 0xFFFF; 1433 if (pipe_w < pfit_w) 1434 pipe_w = pfit_w; 1435 if (pipe_h < pfit_h) 1436 pipe_h = pfit_h; 1437 1438 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h, 1439 pfit_w * pfit_h); 1440 } 1441 1442 return pixel_rate; 1443} 1444 1445/* latency must be in 0.1us units. */ 1446static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel, 1447 uint32_t latency) 1448{ 1449 uint64_t ret; 1450 1451 if (WARN(latency == 0, "Latency value missing\n")) 1452 return UINT_MAX; 1453 1454 ret = (uint64_t) pixel_rate * bytes_per_pixel * latency; 1455 ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2; 1456 1457 return ret; 1458} 1459 1460/* latency must be in 0.1us units. */ 1461static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal, 1462 uint32_t horiz_pixels, uint8_t bytes_per_pixel, 1463 uint32_t latency) 1464{ 1465 uint32_t ret; 1466 1467 if (WARN(latency == 0, "Latency value missing\n")) 1468 return UINT_MAX; 1469 1470 ret = (latency * pixel_rate) / (pipe_htotal * 10000); 1471 ret = (ret + 1) * horiz_pixels * bytes_per_pixel; 1472 ret = DIV_ROUND_UP(ret, 64) + 2; 1473 return ret; 1474} 1475 1476static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels, 1477 uint8_t bytes_per_pixel) 1478{ 1479 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2; 1480} 1481 1482struct skl_pipe_wm_parameters { 1483 bool active; 1484 uint32_t pipe_htotal; 1485 uint32_t pixel_rate; /* in KHz */ 1486 struct intel_plane_wm_parameters plane[I915_MAX_PLANES]; 1487 struct intel_plane_wm_parameters cursor; 1488}; 1489 1490struct ilk_pipe_wm_parameters { 1491 bool active; 1492 uint32_t pipe_htotal; 1493 uint32_t pixel_rate; 1494 struct intel_plane_wm_parameters pri; 1495 struct intel_plane_wm_parameters spr; 1496 struct intel_plane_wm_parameters cur; 1497}; 1498 1499struct ilk_wm_maximums { 1500 uint16_t pri; 1501 uint16_t spr; 1502 uint16_t cur; 1503 uint16_t fbc; 1504}; 1505 1506/* used in computing the new watermarks state */ 1507struct intel_wm_config { 1508 unsigned int num_pipes_active; 1509 bool sprites_enabled; 1510 bool sprites_scaled; 1511}; 1512 1513/* 1514 * For both WM_PIPE and WM_LP. 1515 * mem_value must be in 0.1us units. 1516 */ 1517static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params, 1518 uint32_t mem_value, 1519 bool is_lp) 1520{ 1521 uint32_t method1, method2; 1522 1523 if (!params->active || !params->pri.enabled) 1524 return 0; 1525 1526 method1 = ilk_wm_method1(params->pixel_rate, 1527 params->pri.bytes_per_pixel, 1528 mem_value); 1529 1530 if (!is_lp) 1531 return method1; 1532 1533 method2 = ilk_wm_method2(params->pixel_rate, 1534 params->pipe_htotal, 1535 params->pri.horiz_pixels, 1536 params->pri.bytes_per_pixel, 1537 mem_value); 1538 1539 return min(method1, method2); 1540} 1541 1542/* 1543 * For both WM_PIPE and WM_LP. 1544 * mem_value must be in 0.1us units. 1545 */ 1546static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params, 1547 uint32_t mem_value) 1548{ 1549 uint32_t method1, method2; 1550 1551 if (!params->active || !params->spr.enabled) 1552 return 0; 1553 1554 method1 = ilk_wm_method1(params->pixel_rate, 1555 params->spr.bytes_per_pixel, 1556 mem_value); 1557 method2 = ilk_wm_method2(params->pixel_rate, 1558 params->pipe_htotal, 1559 params->spr.horiz_pixels, 1560 params->spr.bytes_per_pixel, 1561 mem_value); 1562 return min(method1, method2); 1563} 1564 1565/* 1566 * For both WM_PIPE and WM_LP. 1567 * mem_value must be in 0.1us units. 1568 */ 1569static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params, 1570 uint32_t mem_value) 1571{ 1572 if (!params->active || !params->cur.enabled) 1573 return 0; 1574 1575 return ilk_wm_method2(params->pixel_rate, 1576 params->pipe_htotal, 1577 params->cur.horiz_pixels, 1578 params->cur.bytes_per_pixel, 1579 mem_value); 1580} 1581 1582/* Only for WM_LP. */ 1583static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params, 1584 uint32_t pri_val) 1585{ 1586 if (!params->active || !params->pri.enabled) 1587 return 0; 1588 1589 return ilk_wm_fbc(pri_val, 1590 params->pri.horiz_pixels, 1591 params->pri.bytes_per_pixel); 1592} 1593 1594static unsigned int ilk_display_fifo_size(const struct drm_device *dev) 1595{ 1596 if (INTEL_INFO(dev)->gen >= 8) 1597 return 3072; 1598 else if (INTEL_INFO(dev)->gen >= 7) 1599 return 768; 1600 else 1601 return 512; 1602} 1603 1604static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev, 1605 int level, bool is_sprite) 1606{ 1607 if (INTEL_INFO(dev)->gen >= 8) 1608 /* BDW primary/sprite plane watermarks */ 1609 return level == 0 ? 255 : 2047; 1610 else if (INTEL_INFO(dev)->gen >= 7) 1611 /* IVB/HSW primary/sprite plane watermarks */ 1612 return level == 0 ? 127 : 1023; 1613 else if (!is_sprite) 1614 /* ILK/SNB primary plane watermarks */ 1615 return level == 0 ? 127 : 511; 1616 else 1617 /* ILK/SNB sprite plane watermarks */ 1618 return level == 0 ? 63 : 255; 1619} 1620 1621static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev, 1622 int level) 1623{ 1624 if (INTEL_INFO(dev)->gen >= 7) 1625 return level == 0 ? 63 : 255; 1626 else 1627 return level == 0 ? 31 : 63; 1628} 1629 1630static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev) 1631{ 1632 if (INTEL_INFO(dev)->gen >= 8) 1633 return 31; 1634 else 1635 return 15; 1636} 1637 1638/* Calculate the maximum primary/sprite plane watermark */ 1639static unsigned int ilk_plane_wm_max(const struct drm_device *dev, 1640 int level, 1641 const struct intel_wm_config *config, 1642 enum intel_ddb_partitioning ddb_partitioning, 1643 bool is_sprite) 1644{ 1645 unsigned int fifo_size = ilk_display_fifo_size(dev); 1646 1647 /* if sprites aren't enabled, sprites get nothing */ 1648 if (is_sprite && !config->sprites_enabled) 1649 return 0; 1650 1651 /* HSW allows LP1+ watermarks even with multiple pipes */ 1652 if (level == 0 || config->num_pipes_active > 1) { 1653 fifo_size /= INTEL_INFO(dev)->num_pipes; 1654 1655 /* 1656 * For some reason the non self refresh 1657 * FIFO size is only half of the self 1658 * refresh FIFO size on ILK/SNB. 1659 */ 1660 if (INTEL_INFO(dev)->gen <= 6) 1661 fifo_size /= 2; 1662 } 1663 1664 if (config->sprites_enabled) { 1665 /* level 0 is always calculated with 1:1 split */ 1666 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) { 1667 if (is_sprite) 1668 fifo_size *= 5; 1669 fifo_size /= 6; 1670 } else { 1671 fifo_size /= 2; 1672 } 1673 } 1674 1675 /* clamp to max that the registers can hold */ 1676 return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite)); 1677} 1678 1679/* Calculate the maximum cursor plane watermark */ 1680static unsigned int ilk_cursor_wm_max(const struct drm_device *dev, 1681 int level, 1682 const struct intel_wm_config *config) 1683{ 1684 /* HSW LP1+ watermarks w/ multiple pipes */ 1685 if (level > 0 && config->num_pipes_active > 1) 1686 return 64; 1687 1688 /* otherwise just report max that registers can hold */ 1689 return ilk_cursor_wm_reg_max(dev, level); 1690} 1691 1692static void ilk_compute_wm_maximums(const struct drm_device *dev, 1693 int level, 1694 const struct intel_wm_config *config, 1695 enum intel_ddb_partitioning ddb_partitioning, 1696 struct ilk_wm_maximums *max) 1697{ 1698 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false); 1699 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true); 1700 max->cur = ilk_cursor_wm_max(dev, level, config); 1701 max->fbc = ilk_fbc_wm_reg_max(dev); 1702} 1703 1704static void ilk_compute_wm_reg_maximums(struct drm_device *dev, 1705 int level, 1706 struct ilk_wm_maximums *max) 1707{ 1708 max->pri = ilk_plane_wm_reg_max(dev, level, false); 1709 max->spr = ilk_plane_wm_reg_max(dev, level, true); 1710 max->cur = ilk_cursor_wm_reg_max(dev, level); 1711 max->fbc = ilk_fbc_wm_reg_max(dev); 1712} 1713 1714static bool ilk_validate_wm_level(int level, 1715 const struct ilk_wm_maximums *max, 1716 struct intel_wm_level *result) 1717{ 1718 bool ret; 1719 1720 /* already determined to be invalid? */ 1721 if (!result->enable) 1722 return false; 1723 1724 result->enable = result->pri_val <= max->pri && 1725 result->spr_val <= max->spr && 1726 result->cur_val <= max->cur; 1727 1728 ret = result->enable; 1729 1730 /* 1731 * HACK until we can pre-compute everything, 1732 * and thus fail gracefully if LP0 watermarks 1733 * are exceeded... 1734 */ 1735 if (level == 0 && !result->enable) { 1736 if (result->pri_val > max->pri) 1737 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n", 1738 level, result->pri_val, max->pri); 1739 if (result->spr_val > max->spr) 1740 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n", 1741 level, result->spr_val, max->spr); 1742 if (result->cur_val > max->cur) 1743 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n", 1744 level, result->cur_val, max->cur); 1745 1746 result->pri_val = min_t(uint32_t, result->pri_val, max->pri); 1747 result->spr_val = min_t(uint32_t, result->spr_val, max->spr); 1748 result->cur_val = min_t(uint32_t, result->cur_val, max->cur); 1749 result->enable = true; 1750 } 1751 1752 return ret; 1753} 1754 1755static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, 1756 int level, 1757 const struct ilk_pipe_wm_parameters *p, 1758 struct intel_wm_level *result) 1759{ 1760 uint16_t pri_latency = dev_priv->wm.pri_latency[level]; 1761 uint16_t spr_latency = dev_priv->wm.spr_latency[level]; 1762 uint16_t cur_latency = dev_priv->wm.cur_latency[level]; 1763 1764 /* WM1+ latency values stored in 0.5us units */ 1765 if (level > 0) { 1766 pri_latency *= 5; 1767 spr_latency *= 5; 1768 cur_latency *= 5; 1769 } 1770 1771 result->pri_val = ilk_compute_pri_wm(p, pri_latency, level); 1772 result->spr_val = ilk_compute_spr_wm(p, spr_latency); 1773 result->cur_val = ilk_compute_cur_wm(p, cur_latency); 1774 result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val); 1775 result->enable = true; 1776} 1777 1778static uint32_t 1779hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc) 1780{ 1781 struct drm_i915_private *dev_priv = dev->dev_private; 1782 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1783 struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode; 1784 u32 linetime, ips_linetime; 1785 1786 if (!intel_crtc->active) 1787 return 0; 1788 1789 /* The WM are computed with base on how long it takes to fill a single 1790 * row at the given clock rate, multiplied by 8. 1791 * */ 1792 linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8, 1793 mode->crtc_clock); 1794 ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8, 1795 intel_ddi_get_cdclk_freq(dev_priv)); 1796 1797 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) | 1798 PIPE_WM_LINETIME_TIME(linetime); 1799} 1800 1801static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) 1802{ 1803 struct drm_i915_private *dev_priv = dev->dev_private; 1804 1805 if (IS_GEN9(dev)) { 1806 uint32_t val; 1807 int ret, i; 1808 int level, max_level = ilk_wm_max_level(dev); 1809 1810 /* read the first set of memory latencies[0:3] */ 1811 val = 0; /* data0 to be programmed to 0 for first set */ 1812 mutex_lock(&dev_priv->rps.hw_lock); 1813 ret = sandybridge_pcode_read(dev_priv, 1814 GEN9_PCODE_READ_MEM_LATENCY, 1815 &val); 1816 mutex_unlock(&dev_priv->rps.hw_lock); 1817 1818 if (ret) { 1819 DRM_ERROR("SKL Mailbox read error = %d\n", ret); 1820 return; 1821 } 1822 1823 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK; 1824 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & 1825 GEN9_MEM_LATENCY_LEVEL_MASK; 1826 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & 1827 GEN9_MEM_LATENCY_LEVEL_MASK; 1828 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & 1829 GEN9_MEM_LATENCY_LEVEL_MASK; 1830 1831 /* read the second set of memory latencies[4:7] */ 1832 val = 1; /* data0 to be programmed to 1 for second set */ 1833 mutex_lock(&dev_priv->rps.hw_lock); 1834 ret = sandybridge_pcode_read(dev_priv, 1835 GEN9_PCODE_READ_MEM_LATENCY, 1836 &val); 1837 mutex_unlock(&dev_priv->rps.hw_lock); 1838 if (ret) { 1839 DRM_ERROR("SKL Mailbox read error = %d\n", ret); 1840 return; 1841 } 1842 1843 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK; 1844 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & 1845 GEN9_MEM_LATENCY_LEVEL_MASK; 1846 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & 1847 GEN9_MEM_LATENCY_LEVEL_MASK; 1848 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & 1849 GEN9_MEM_LATENCY_LEVEL_MASK; 1850 1851 /* 1852 * WaWmMemoryReadLatency:skl 1853 * 1854 * punit doesn't take into account the read latency so we need 1855 * to add 2us to the various latency levels we retrieve from 1856 * the punit. 1857 * - W0 is a bit special in that it's the only level that 1858 * can't be disabled if we want to have display working, so 1859 * we always add 2us there. 1860 * - For levels >=1, punit returns 0us latency when they are 1861 * disabled, so we respect that and don't add 2us then 1862 * 1863 * Additionally, if a level n (n > 1) has a 0us latency, all 1864 * levels m (m >= n) need to be disabled. We make sure to 1865 * sanitize the values out of the punit to satisfy this 1866 * requirement. 1867 */ 1868 wm[0] += 2; 1869 for (level = 1; level <= max_level; level++) 1870 if (wm[level] != 0) 1871 wm[level] += 2; 1872 else { 1873 for (i = level + 1; i <= max_level; i++) 1874 wm[i] = 0; 1875 1876 break; 1877 } 1878 } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { 1879 uint64_t sskpd = I915_READ64(MCH_SSKPD); 1880 1881 wm[0] = (sskpd >> 56) & 0xFF; 1882 if (wm[0] == 0) 1883 wm[0] = sskpd & 0xF; 1884 wm[1] = (sskpd >> 4) & 0xFF; 1885 wm[2] = (sskpd >> 12) & 0xFF; 1886 wm[3] = (sskpd >> 20) & 0x1FF; 1887 wm[4] = (sskpd >> 32) & 0x1FF; 1888 } else if (INTEL_INFO(dev)->gen >= 6) { 1889 uint32_t sskpd = I915_READ(MCH_SSKPD); 1890 1891 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK; 1892 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK; 1893 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK; 1894 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK; 1895 } else if (INTEL_INFO(dev)->gen >= 5) { 1896 uint32_t mltr = I915_READ(MLTR_ILK); 1897 1898 /* ILK primary LP0 latency is 700 ns */ 1899 wm[0] = 7; 1900 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK; 1901 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK; 1902 } 1903} 1904 1905static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5]) 1906{ 1907 /* ILK sprite LP0 latency is 1300 ns */ 1908 if (INTEL_INFO(dev)->gen == 5) 1909 wm[0] = 13; 1910} 1911 1912static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5]) 1913{ 1914 /* ILK cursor LP0 latency is 1300 ns */ 1915 if (INTEL_INFO(dev)->gen == 5) 1916 wm[0] = 13; 1917 1918 /* WaDoubleCursorLP3Latency:ivb */ 1919 if (IS_IVYBRIDGE(dev)) 1920 wm[3] *= 2; 1921} 1922 1923int ilk_wm_max_level(const struct drm_device *dev) 1924{ 1925 /* how many WM levels are we expecting */ 1926 if (IS_GEN9(dev)) 1927 return 7; 1928 else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 1929 return 4; 1930 else if (INTEL_INFO(dev)->gen >= 6) 1931 return 3; 1932 else 1933 return 2; 1934} 1935 1936static void intel_print_wm_latency(struct drm_device *dev, 1937 const char *name, 1938 const uint16_t wm[8]) 1939{ 1940 int level, max_level = ilk_wm_max_level(dev); 1941 1942 for (level = 0; level <= max_level; level++) { 1943 unsigned int latency = wm[level]; 1944 1945 if (latency == 0) { 1946 DRM_ERROR("%s WM%d latency not provided\n", 1947 name, level); 1948 continue; 1949 } 1950 1951 /* 1952 * - latencies are in us on gen9. 1953 * - before then, WM1+ latency values are in 0.5us units 1954 */ 1955 if (IS_GEN9(dev)) 1956 latency *= 10; 1957 else if (level > 0) 1958 latency *= 5; 1959 1960 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n", 1961 name, level, wm[level], 1962 latency / 10, latency % 10); 1963 } 1964} 1965 1966static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, 1967 uint16_t wm[5], uint16_t min) 1968{ 1969 int level, max_level = ilk_wm_max_level(dev_priv->dev); 1970 1971 if (wm[0] >= min) 1972 return false; 1973 1974 wm[0] = max(wm[0], min); 1975 for (level = 1; level <= max_level; level++) 1976 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5)); 1977 1978 return true; 1979} 1980 1981static void snb_wm_latency_quirk(struct drm_device *dev) 1982{ 1983 struct drm_i915_private *dev_priv = dev->dev_private; 1984 bool changed; 1985 1986 /* 1987 * The BIOS provided WM memory latency values are often 1988 * inadequate for high resolution displays. Adjust them. 1989 */ 1990 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) | 1991 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) | 1992 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12); 1993 1994 if (!changed) 1995 return; 1996 1997 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n"); 1998 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); 1999 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); 2000 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); 2001} 2002 2003static void ilk_setup_wm_latency(struct drm_device *dev) 2004{ 2005 struct drm_i915_private *dev_priv = dev->dev_private; 2006 2007 intel_read_wm_latency(dev, dev_priv->wm.pri_latency); 2008 2009 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency, 2010 sizeof(dev_priv->wm.pri_latency)); 2011 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency, 2012 sizeof(dev_priv->wm.pri_latency)); 2013 2014 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency); 2015 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency); 2016 2017 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); 2018 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); 2019 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); 2020 2021 if (IS_GEN6(dev)) 2022 snb_wm_latency_quirk(dev); 2023} 2024 2025static void skl_setup_wm_latency(struct drm_device *dev) 2026{ 2027 struct drm_i915_private *dev_priv = dev->dev_private; 2028 2029 intel_read_wm_latency(dev, dev_priv->wm.skl_latency); 2030 intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency); 2031} 2032 2033static void ilk_compute_wm_parameters(struct drm_crtc *crtc, 2034 struct ilk_pipe_wm_parameters *p) 2035{ 2036 struct drm_device *dev = crtc->dev; 2037 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 2038 enum pipe pipe = intel_crtc->pipe; 2039 struct drm_plane *plane; 2040 2041 if (!intel_crtc->active) 2042 return; 2043 2044 p->active = true; 2045 p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal; 2046 p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc); 2047 2048 if (crtc->primary->state->fb) 2049 p->pri.bytes_per_pixel = 2050 crtc->primary->state->fb->bits_per_pixel / 8; 2051 else 2052 p->pri.bytes_per_pixel = 4; 2053 2054 p->cur.bytes_per_pixel = 4; 2055 /* 2056 * TODO: for now, assume primary and cursor planes are always enabled. 2057 * Setting them to false makes the screen flicker. 2058 */ 2059 p->pri.enabled = true; 2060 p->cur.enabled = true; 2061 2062 p->pri.horiz_pixels = intel_crtc->config->pipe_src_w; 2063 p->cur.horiz_pixels = intel_crtc->base.cursor->state->crtc_w; 2064 2065 drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) { 2066 struct intel_plane *intel_plane = to_intel_plane(plane); 2067 2068 if (intel_plane->pipe == pipe) { 2069 p->spr = intel_plane->wm; 2070 break; 2071 } 2072 } 2073} 2074 2075static void ilk_compute_wm_config(struct drm_device *dev, 2076 struct intel_wm_config *config) 2077{ 2078 struct intel_crtc *intel_crtc; 2079 2080 /* Compute the currently _active_ config */ 2081 for_each_intel_crtc(dev, intel_crtc) { 2082 const struct intel_pipe_wm *wm = &intel_crtc->wm.active; 2083 2084 if (!wm->pipe_enabled) 2085 continue; 2086 2087 config->sprites_enabled |= wm->sprites_enabled; 2088 config->sprites_scaled |= wm->sprites_scaled; 2089 config->num_pipes_active++; 2090 } 2091} 2092 2093/* Compute new watermarks for the pipe */ 2094static bool intel_compute_pipe_wm(struct drm_crtc *crtc, 2095 const struct ilk_pipe_wm_parameters *params, 2096 struct intel_pipe_wm *pipe_wm) 2097{ 2098 struct drm_device *dev = crtc->dev; 2099 const struct drm_i915_private *dev_priv = dev->dev_private; 2100 int level, max_level = ilk_wm_max_level(dev); 2101 /* LP0 watermark maximums depend on this pipe alone */ 2102 struct intel_wm_config config = { 2103 .num_pipes_active = 1, 2104 .sprites_enabled = params->spr.enabled, 2105 .sprites_scaled = params->spr.scaled, 2106 }; 2107 struct ilk_wm_maximums max; 2108 2109 pipe_wm->pipe_enabled = params->active; 2110 pipe_wm->sprites_enabled = params->spr.enabled; 2111 pipe_wm->sprites_scaled = params->spr.scaled; 2112 2113 /* ILK/SNB: LP2+ watermarks only w/o sprites */ 2114 if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled) 2115 max_level = 1; 2116 2117 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ 2118 if (params->spr.scaled) 2119 max_level = 0; 2120 2121 ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]); 2122 2123 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 2124 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc); 2125 2126 /* LP0 watermarks always use 1/2 DDB partitioning */ 2127 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); 2128 2129 /* At least LP0 must be valid */ 2130 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) 2131 return false; 2132 2133 ilk_compute_wm_reg_maximums(dev, 1, &max); 2134 2135 for (level = 1; level <= max_level; level++) { 2136 struct intel_wm_level wm = {}; 2137 2138 ilk_compute_wm_level(dev_priv, level, params, &wm); 2139 2140 /* 2141 * Disable any watermark level that exceeds the 2142 * register maximums since such watermarks are 2143 * always invalid. 2144 */ 2145 if (!ilk_validate_wm_level(level, &max, &wm)) 2146 break; 2147 2148 pipe_wm->wm[level] = wm; 2149 } 2150 2151 return true; 2152} 2153 2154/* 2155 * Merge the watermarks from all active pipes for a specific level. 2156 */ 2157static void ilk_merge_wm_level(struct drm_device *dev, 2158 int level, 2159 struct intel_wm_level *ret_wm) 2160{ 2161 const struct intel_crtc *intel_crtc; 2162 2163 ret_wm->enable = true; 2164 2165 for_each_intel_crtc(dev, intel_crtc) { 2166 const struct intel_pipe_wm *active = &intel_crtc->wm.active; 2167 const struct intel_wm_level *wm = &active->wm[level]; 2168 2169 if (!active->pipe_enabled) 2170 continue; 2171 2172 /* 2173 * The watermark values may have been used in the past, 2174 * so we must maintain them in the registers for some 2175 * time even if the level is now disabled. 2176 */ 2177 if (!wm->enable) 2178 ret_wm->enable = false; 2179 2180 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val); 2181 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val); 2182 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val); 2183 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val); 2184 } 2185} 2186 2187/* 2188 * Merge all low power watermarks for all active pipes. 2189 */ 2190static void ilk_wm_merge(struct drm_device *dev, 2191 const struct intel_wm_config *config, 2192 const struct ilk_wm_maximums *max, 2193 struct intel_pipe_wm *merged) 2194{ 2195 int level, max_level = ilk_wm_max_level(dev); 2196 int last_enabled_level = max_level; 2197 2198 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */ 2199 if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) && 2200 config->num_pipes_active > 1) 2201 return; 2202 2203 /* ILK: FBC WM must be disabled always */ 2204 merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6; 2205 2206 /* merge each WM1+ level */ 2207 for (level = 1; level <= max_level; level++) { 2208 struct intel_wm_level *wm = &merged->wm[level]; 2209 2210 ilk_merge_wm_level(dev, level, wm); 2211 2212 if (level > last_enabled_level) 2213 wm->enable = false; 2214 else if (!ilk_validate_wm_level(level, max, wm)) 2215 /* make sure all following levels get disabled */ 2216 last_enabled_level = level - 1; 2217 2218 /* 2219 * The spec says it is preferred to disable 2220 * FBC WMs instead of disabling a WM level. 2221 */ 2222 if (wm->fbc_val > max->fbc) { 2223 if (wm->enable) 2224 merged->fbc_wm_enabled = false; 2225 wm->fbc_val = 0; 2226 } 2227 } 2228 2229 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */ 2230 /* 2231 * FIXME this is racy. FBC might get enabled later. 2232 * What we should check here is whether FBC can be 2233 * enabled sometime later. 2234 */ 2235 if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) { 2236 for (level = 2; level <= max_level; level++) { 2237 struct intel_wm_level *wm = &merged->wm[level]; 2238 2239 wm->enable = false; 2240 } 2241 } 2242} 2243 2244static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) 2245{ 2246 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */ 2247 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable); 2248} 2249 2250/* The value we need to program into the WM_LPx latency field */ 2251static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level) 2252{ 2253 struct drm_i915_private *dev_priv = dev->dev_private; 2254 2255 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 2256 return 2 * level; 2257 else 2258 return dev_priv->wm.pri_latency[level]; 2259} 2260 2261static void ilk_compute_wm_results(struct drm_device *dev, 2262 const struct intel_pipe_wm *merged, 2263 enum intel_ddb_partitioning partitioning, 2264 struct ilk_wm_values *results) 2265{ 2266 struct intel_crtc *intel_crtc; 2267 int level, wm_lp; 2268 2269 results->enable_fbc_wm = merged->fbc_wm_enabled; 2270 results->partitioning = partitioning; 2271 2272 /* LP1+ register values */ 2273 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 2274 const struct intel_wm_level *r; 2275 2276 level = ilk_wm_lp_to_level(wm_lp, merged); 2277 2278 r = &merged->wm[level]; 2279 2280 /* 2281 * Maintain the watermark values even if the level is 2282 * disabled. Doing otherwise could cause underruns. 2283 */ 2284 results->wm_lp[wm_lp - 1] = 2285 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) | 2286 (r->pri_val << WM1_LP_SR_SHIFT) | 2287 r->cur_val; 2288 2289 if (r->enable) 2290 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN; 2291 2292 if (INTEL_INFO(dev)->gen >= 8) 2293 results->wm_lp[wm_lp - 1] |= 2294 r->fbc_val << WM1_LP_FBC_SHIFT_BDW; 2295 else 2296 results->wm_lp[wm_lp - 1] |= 2297 r->fbc_val << WM1_LP_FBC_SHIFT; 2298 2299 /* 2300 * Always set WM1S_LP_EN when spr_val != 0, even if the 2301 * level is disabled. Doing otherwise could cause underruns. 2302 */ 2303 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) { 2304 WARN_ON(wm_lp != 1); 2305 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val; 2306 } else 2307 results->wm_lp_spr[wm_lp - 1] = r->spr_val; 2308 } 2309 2310 /* LP0 register values */ 2311 for_each_intel_crtc(dev, intel_crtc) { 2312 enum pipe pipe = intel_crtc->pipe; 2313 const struct intel_wm_level *r = 2314 &intel_crtc->wm.active.wm[0]; 2315 2316 if (WARN_ON(!r->enable)) 2317 continue; 2318 2319 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime; 2320 2321 results->wm_pipe[pipe] = 2322 (r->pri_val << WM0_PIPE_PLANE_SHIFT) | 2323 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) | 2324 r->cur_val; 2325 } 2326} 2327 2328/* Find the result with the highest level enabled. Check for enable_fbc_wm in 2329 * case both are at the same level. Prefer r1 in case they're the same. */ 2330static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev, 2331 struct intel_pipe_wm *r1, 2332 struct intel_pipe_wm *r2) 2333{ 2334 int level, max_level = ilk_wm_max_level(dev); 2335 int level1 = 0, level2 = 0; 2336 2337 for (level = 1; level <= max_level; level++) { 2338 if (r1->wm[level].enable) 2339 level1 = level; 2340 if (r2->wm[level].enable) 2341 level2 = level; 2342 } 2343 2344 if (level1 == level2) { 2345 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled) 2346 return r2; 2347 else 2348 return r1; 2349 } else if (level1 > level2) { 2350 return r1; 2351 } else { 2352 return r2; 2353 } 2354} 2355 2356/* dirty bits used to track which watermarks need changes */ 2357#define WM_DIRTY_PIPE(pipe) (1 << (pipe)) 2358#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe))) 2359#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp))) 2360#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3)) 2361#define WM_DIRTY_FBC (1 << 24) 2362#define WM_DIRTY_DDB (1 << 25) 2363 2364static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, 2365 const struct ilk_wm_values *old, 2366 const struct ilk_wm_values *new) 2367{ 2368 unsigned int dirty = 0; 2369 enum pipe pipe; 2370 int wm_lp; 2371 2372 for_each_pipe(dev_priv, pipe) { 2373 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) { 2374 dirty |= WM_DIRTY_LINETIME(pipe); 2375 /* Must disable LP1+ watermarks too */ 2376 dirty |= WM_DIRTY_LP_ALL; 2377 } 2378 2379 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) { 2380 dirty |= WM_DIRTY_PIPE(pipe); 2381 /* Must disable LP1+ watermarks too */ 2382 dirty |= WM_DIRTY_LP_ALL; 2383 } 2384 } 2385 2386 if (old->enable_fbc_wm != new->enable_fbc_wm) { 2387 dirty |= WM_DIRTY_FBC; 2388 /* Must disable LP1+ watermarks too */ 2389 dirty |= WM_DIRTY_LP_ALL; 2390 } 2391 2392 if (old->partitioning != new->partitioning) { 2393 dirty |= WM_DIRTY_DDB; 2394 /* Must disable LP1+ watermarks too */ 2395 dirty |= WM_DIRTY_LP_ALL; 2396 } 2397 2398 /* LP1+ watermarks already deemed dirty, no need to continue */ 2399 if (dirty & WM_DIRTY_LP_ALL) 2400 return dirty; 2401 2402 /* Find the lowest numbered LP1+ watermark in need of an update... */ 2403 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 2404 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] || 2405 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1]) 2406 break; 2407 } 2408 2409 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */ 2410 for (; wm_lp <= 3; wm_lp++) 2411 dirty |= WM_DIRTY_LP(wm_lp); 2412 2413 return dirty; 2414} 2415 2416static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, 2417 unsigned int dirty) 2418{ 2419 struct ilk_wm_values *previous = &dev_priv->wm.hw; 2420 bool changed = false; 2421 2422 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) { 2423 previous->wm_lp[2] &= ~WM1_LP_SR_EN; 2424 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]); 2425 changed = true; 2426 } 2427 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) { 2428 previous->wm_lp[1] &= ~WM1_LP_SR_EN; 2429 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]); 2430 changed = true; 2431 } 2432 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) { 2433 previous->wm_lp[0] &= ~WM1_LP_SR_EN; 2434 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]); 2435 changed = true; 2436 } 2437 2438 /* 2439 * Don't touch WM1S_LP_EN here. 2440 * Doing so could cause underruns. 2441 */ 2442 2443 return changed; 2444} 2445 2446/* 2447 * The spec says we shouldn't write when we don't need, because every write 2448 * causes WMs to be re-evaluated, expending some power. 2449 */ 2450static void ilk_write_wm_values(struct drm_i915_private *dev_priv, 2451 struct ilk_wm_values *results) 2452{ 2453 struct drm_device *dev = dev_priv->dev; 2454 struct ilk_wm_values *previous = &dev_priv->wm.hw; 2455 unsigned int dirty; 2456 uint32_t val; 2457 2458 dirty = ilk_compute_wm_dirty(dev_priv, previous, results); 2459 if (!dirty) 2460 return; 2461 2462 _ilk_disable_lp_wm(dev_priv, dirty); 2463 2464 if (dirty & WM_DIRTY_PIPE(PIPE_A)) 2465 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]); 2466 if (dirty & WM_DIRTY_PIPE(PIPE_B)) 2467 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]); 2468 if (dirty & WM_DIRTY_PIPE(PIPE_C)) 2469 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]); 2470 2471 if (dirty & WM_DIRTY_LINETIME(PIPE_A)) 2472 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]); 2473 if (dirty & WM_DIRTY_LINETIME(PIPE_B)) 2474 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]); 2475 if (dirty & WM_DIRTY_LINETIME(PIPE_C)) 2476 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]); 2477 2478 if (dirty & WM_DIRTY_DDB) { 2479 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { 2480 val = I915_READ(WM_MISC); 2481 if (results->partitioning == INTEL_DDB_PART_1_2) 2482 val &= ~WM_MISC_DATA_PARTITION_5_6; 2483 else 2484 val |= WM_MISC_DATA_PARTITION_5_6; 2485 I915_WRITE(WM_MISC, val); 2486 } else { 2487 val = I915_READ(DISP_ARB_CTL2); 2488 if (results->partitioning == INTEL_DDB_PART_1_2) 2489 val &= ~DISP_DATA_PARTITION_5_6; 2490 else 2491 val |= DISP_DATA_PARTITION_5_6; 2492 I915_WRITE(DISP_ARB_CTL2, val); 2493 } 2494 } 2495 2496 if (dirty & WM_DIRTY_FBC) { 2497 val = I915_READ(DISP_ARB_CTL); 2498 if (results->enable_fbc_wm) 2499 val &= ~DISP_FBC_WM_DIS; 2500 else 2501 val |= DISP_FBC_WM_DIS; 2502 I915_WRITE(DISP_ARB_CTL, val); 2503 } 2504 2505 if (dirty & WM_DIRTY_LP(1) && 2506 previous->wm_lp_spr[0] != results->wm_lp_spr[0]) 2507 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]); 2508 2509 if (INTEL_INFO(dev)->gen >= 7) { 2510 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) 2511 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]); 2512 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) 2513 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]); 2514 } 2515 2516 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0]) 2517 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]); 2518 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1]) 2519 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]); 2520 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2]) 2521 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]); 2522 2523 dev_priv->wm.hw = *results; 2524} 2525 2526static bool ilk_disable_lp_wm(struct drm_device *dev) 2527{ 2528 struct drm_i915_private *dev_priv = dev->dev_private; 2529 2530 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); 2531} 2532 2533/* 2534 * On gen9, we need to allocate Display Data Buffer (DDB) portions to the 2535 * different active planes. 2536 */ 2537 2538#define SKL_DDB_SIZE 896 /* in blocks */ 2539 2540static void 2541skl_ddb_get_pipe_allocation_limits(struct drm_device *dev, 2542 struct drm_crtc *for_crtc, 2543 const struct intel_wm_config *config, 2544 const struct skl_pipe_wm_parameters *params, 2545 struct skl_ddb_entry *alloc /* out */) 2546{ 2547 struct drm_crtc *crtc; 2548 unsigned int pipe_size, ddb_size; 2549 int nth_active_pipe; 2550 2551 if (!params->active) { 2552 alloc->start = 0; 2553 alloc->end = 0; 2554 return; 2555 } 2556 2557 ddb_size = SKL_DDB_SIZE; 2558 2559 ddb_size -= 4; /* 4 blocks for bypass path allocation */ 2560 2561 nth_active_pipe = 0; 2562 for_each_crtc(dev, crtc) { 2563 if (!to_intel_crtc(crtc)->active) 2564 continue; 2565 2566 if (crtc == for_crtc) 2567 break; 2568 2569 nth_active_pipe++; 2570 } 2571 2572 pipe_size = ddb_size / config->num_pipes_active; 2573 alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active; 2574 alloc->end = alloc->start + pipe_size; 2575} 2576 2577static unsigned int skl_cursor_allocation(const struct intel_wm_config *config) 2578{ 2579 if (config->num_pipes_active == 1) 2580 return 32; 2581 2582 return 8; 2583} 2584 2585static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg) 2586{ 2587 entry->start = reg & 0x3ff; 2588 entry->end = (reg >> 16) & 0x3ff; 2589 if (entry->end) 2590 entry->end += 1; 2591} 2592 2593void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, 2594 struct skl_ddb_allocation *ddb /* out */) 2595{ 2596 enum pipe pipe; 2597 int plane; 2598 u32 val; 2599 2600 for_each_pipe(dev_priv, pipe) { 2601 for_each_plane(dev_priv, pipe, plane) { 2602 val = I915_READ(PLANE_BUF_CFG(pipe, plane)); 2603 skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane], 2604 val); 2605 } 2606 2607 val = I915_READ(CUR_BUF_CFG(pipe)); 2608 skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val); 2609 } 2610} 2611 2612static unsigned int 2613skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p) 2614{ 2615 return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel; 2616} 2617 2618/* 2619 * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching 2620 * a 8192x4096@32bpp framebuffer: 2621 * 3 * 4096 * 8192 * 4 < 2^32 2622 */ 2623static unsigned int 2624skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc, 2625 const struct skl_pipe_wm_parameters *params) 2626{ 2627 unsigned int total_data_rate = 0; 2628 int plane; 2629 2630 for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) { 2631 const struct intel_plane_wm_parameters *p; 2632 2633 p = ¶ms->plane[plane]; 2634 if (!p->enabled) 2635 continue; 2636 2637 total_data_rate += skl_plane_relative_data_rate(p); 2638 } 2639 2640 return total_data_rate; 2641} 2642 2643static void 2644skl_allocate_pipe_ddb(struct drm_crtc *crtc, 2645 const struct intel_wm_config *config, 2646 const struct skl_pipe_wm_parameters *params, 2647 struct skl_ddb_allocation *ddb /* out */) 2648{ 2649 struct drm_device *dev = crtc->dev; 2650 struct drm_i915_private *dev_priv = dev->dev_private; 2651 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 2652 enum pipe pipe = intel_crtc->pipe; 2653 struct skl_ddb_entry *alloc = &ddb->pipe[pipe]; 2654 uint16_t alloc_size, start, cursor_blocks; 2655 uint16_t minimum[I915_MAX_PLANES]; 2656 unsigned int total_data_rate; 2657 int plane; 2658 2659 skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc); 2660 alloc_size = skl_ddb_entry_size(alloc); 2661 if (alloc_size == 0) { 2662 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); 2663 memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe])); 2664 return; 2665 } 2666 2667 cursor_blocks = skl_cursor_allocation(config); 2668 ddb->cursor[pipe].start = alloc->end - cursor_blocks; 2669 ddb->cursor[pipe].end = alloc->end; 2670 2671 alloc_size -= cursor_blocks; 2672 alloc->end -= cursor_blocks; 2673 2674 /* 1. Allocate the mininum required blocks for each active plane */ 2675 for_each_plane(dev_priv, pipe, plane) { 2676 const struct intel_plane_wm_parameters *p; 2677 2678 p = ¶ms->plane[plane]; 2679 if (!p->enabled) 2680 continue; 2681 2682 minimum[plane] = 8; 2683 alloc_size -= minimum[plane]; 2684 } 2685 2686 /* 2687 * 2. Distribute the remaining space in proportion to the amount of 2688 * data each plane needs to fetch from memory. 2689 * 2690 * FIXME: we may not allocate every single block here. 2691 */ 2692 total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params); 2693 2694 start = alloc->start; 2695 for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) { 2696 const struct intel_plane_wm_parameters *p; 2697 unsigned int data_rate; 2698 uint16_t plane_blocks; 2699 2700 p = ¶ms->plane[plane]; 2701 if (!p->enabled) 2702 continue; 2703 2704 data_rate = skl_plane_relative_data_rate(p); 2705 2706 /* 2707 * promote the expression to 64 bits to avoid overflowing, the 2708 * result is < available as data_rate / total_data_rate < 1 2709 */ 2710 plane_blocks = minimum[plane]; 2711 plane_blocks += div_u64((uint64_t)alloc_size * data_rate, 2712 total_data_rate); 2713 2714 ddb->plane[pipe][plane].start = start; 2715 ddb->plane[pipe][plane].end = start + plane_blocks; 2716 2717 start += plane_blocks; 2718 } 2719 2720} 2721 2722static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config) 2723{ 2724 /* TODO: Take into account the scalers once we support them */ 2725 return config->base.adjusted_mode.crtc_clock; 2726} 2727 2728/* 2729 * The max latency should be 257 (max the punit can code is 255 and we add 2us 2730 * for the read latency) and bytes_per_pixel should always be <= 8, so that 2731 * should allow pixel_rate up to ~2 GHz which seems sufficient since max 2732 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that. 2733*/ 2734static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel, 2735 uint32_t latency) 2736{ 2737 uint32_t wm_intermediate_val, ret; 2738 2739 if (latency == 0) 2740 return UINT_MAX; 2741 2742 wm_intermediate_val = latency * pixel_rate * bytes_per_pixel / 512; 2743 ret = DIV_ROUND_UP(wm_intermediate_val, 1000); 2744 2745 return ret; 2746} 2747 2748static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal, 2749 uint32_t horiz_pixels, uint8_t bytes_per_pixel, 2750 uint64_t tiling, uint32_t latency) 2751{ 2752 uint32_t ret; 2753 uint32_t plane_bytes_per_line, plane_blocks_per_line; 2754 uint32_t wm_intermediate_val; 2755 2756 if (latency == 0) 2757 return UINT_MAX; 2758 2759 plane_bytes_per_line = horiz_pixels * bytes_per_pixel; 2760 2761 if (tiling == I915_FORMAT_MOD_Y_TILED || 2762 tiling == I915_FORMAT_MOD_Yf_TILED) { 2763 plane_bytes_per_line *= 4; 2764 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512); 2765 plane_blocks_per_line /= 4; 2766 } else { 2767 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512); 2768 } 2769 2770 wm_intermediate_val = latency * pixel_rate; 2771 ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) * 2772 plane_blocks_per_line; 2773 2774 return ret; 2775} 2776 2777static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb, 2778 const struct intel_crtc *intel_crtc) 2779{ 2780 struct drm_device *dev = intel_crtc->base.dev; 2781 struct drm_i915_private *dev_priv = dev->dev_private; 2782 const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb; 2783 enum pipe pipe = intel_crtc->pipe; 2784 2785 if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe], 2786 sizeof(new_ddb->plane[pipe]))) 2787 return true; 2788 2789 if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe], 2790 sizeof(new_ddb->cursor[pipe]))) 2791 return true; 2792 2793 return false; 2794} 2795 2796static void skl_compute_wm_global_parameters(struct drm_device *dev, 2797 struct intel_wm_config *config) 2798{ 2799 struct drm_crtc *crtc; 2800 struct drm_plane *plane; 2801 2802 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) 2803 config->num_pipes_active += to_intel_crtc(crtc)->active; 2804 2805 /* FIXME: I don't think we need those two global parameters on SKL */ 2806 list_for_each_entry(plane, &dev->mode_config.plane_list, head) { 2807 struct intel_plane *intel_plane = to_intel_plane(plane); 2808 2809 config->sprites_enabled |= intel_plane->wm.enabled; 2810 config->sprites_scaled |= intel_plane->wm.scaled; 2811 } 2812} 2813 2814static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc, 2815 struct skl_pipe_wm_parameters *p) 2816{ 2817 struct drm_device *dev = crtc->dev; 2818 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 2819 enum pipe pipe = intel_crtc->pipe; 2820 struct drm_plane *plane; 2821 struct drm_framebuffer *fb; 2822 int i = 1; /* Index for sprite planes start */ 2823 2824 p->active = intel_crtc->active; 2825 if (p->active) { 2826 p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal; 2827 p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config); 2828 2829 fb = crtc->primary->state->fb; 2830 if (fb) { 2831 p->plane[0].enabled = true; 2832 p->plane[0].bytes_per_pixel = fb->bits_per_pixel / 8; 2833 p->plane[0].tiling = fb->modifier[0]; 2834 } else { 2835 p->plane[0].enabled = false; 2836 p->plane[0].bytes_per_pixel = 0; 2837 p->plane[0].tiling = DRM_FORMAT_MOD_NONE; 2838 } 2839 p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w; 2840 p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h; 2841 p->plane[0].rotation = crtc->primary->state->rotation; 2842 2843 fb = crtc->cursor->state->fb; 2844 if (fb) { 2845 p->cursor.enabled = true; 2846 p->cursor.bytes_per_pixel = fb->bits_per_pixel / 8; 2847 p->cursor.horiz_pixels = crtc->cursor->state->crtc_w; 2848 p->cursor.vert_pixels = crtc->cursor->state->crtc_h; 2849 } else { 2850 p->cursor.enabled = false; 2851 p->cursor.bytes_per_pixel = 0; 2852 p->cursor.horiz_pixels = 64; 2853 p->cursor.vert_pixels = 64; 2854 } 2855 } 2856 2857 list_for_each_entry(plane, &dev->mode_config.plane_list, head) { 2858 struct intel_plane *intel_plane = to_intel_plane(plane); 2859 2860 if (intel_plane->pipe == pipe && 2861 plane->type == DRM_PLANE_TYPE_OVERLAY) 2862 p->plane[i++] = intel_plane->wm; 2863 } 2864} 2865 2866static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, 2867 struct skl_pipe_wm_parameters *p, 2868 struct intel_plane_wm_parameters *p_params, 2869 uint16_t ddb_allocation, 2870 int level, 2871 uint16_t *out_blocks, /* out */ 2872 uint8_t *out_lines /* out */) 2873{ 2874 uint32_t latency = dev_priv->wm.skl_latency[level]; 2875 uint32_t method1, method2; 2876 uint32_t plane_bytes_per_line, plane_blocks_per_line; 2877 uint32_t res_blocks, res_lines; 2878 uint32_t selected_result; 2879 2880 if (latency == 0 || !p->active || !p_params->enabled) 2881 return false; 2882 2883 method1 = skl_wm_method1(p->pixel_rate, 2884 p_params->bytes_per_pixel, 2885 latency); 2886 method2 = skl_wm_method2(p->pixel_rate, 2887 p->pipe_htotal, 2888 p_params->horiz_pixels, 2889 p_params->bytes_per_pixel, 2890 p_params->tiling, 2891 latency); 2892 2893 plane_bytes_per_line = p_params->horiz_pixels * 2894 p_params->bytes_per_pixel; 2895 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512); 2896 2897 if (p_params->tiling == I915_FORMAT_MOD_Y_TILED || 2898 p_params->tiling == I915_FORMAT_MOD_Yf_TILED) { 2899 uint32_t min_scanlines = 4; 2900 uint32_t y_tile_minimum; 2901 if (intel_rotation_90_or_270(p_params->rotation)) { 2902 switch (p_params->bytes_per_pixel) { 2903 case 1: 2904 min_scanlines = 16; 2905 break; 2906 case 2: 2907 min_scanlines = 8; 2908 break; 2909 case 8: 2910 WARN(1, "Unsupported pixel depth for rotation"); 2911 } 2912 } 2913 y_tile_minimum = plane_blocks_per_line * min_scanlines; 2914 selected_result = max(method2, y_tile_minimum); 2915 } else { 2916 if ((ddb_allocation / plane_blocks_per_line) >= 1) 2917 selected_result = min(method1, method2); 2918 else 2919 selected_result = method1; 2920 } 2921 2922 res_blocks = selected_result + 1; 2923 res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line); 2924 2925 if (level >= 1 && level <= 7) { 2926 if (p_params->tiling == I915_FORMAT_MOD_Y_TILED || 2927 p_params->tiling == I915_FORMAT_MOD_Yf_TILED) 2928 res_lines += 4; 2929 else 2930 res_blocks++; 2931 } 2932 2933 if (res_blocks >= ddb_allocation || res_lines > 31) 2934 return false; 2935 2936 *out_blocks = res_blocks; 2937 *out_lines = res_lines; 2938 2939 return true; 2940} 2941 2942static void skl_compute_wm_level(const struct drm_i915_private *dev_priv, 2943 struct skl_ddb_allocation *ddb, 2944 struct skl_pipe_wm_parameters *p, 2945 enum pipe pipe, 2946 int level, 2947 int num_planes, 2948 struct skl_wm_level *result) 2949{ 2950 uint16_t ddb_blocks; 2951 int i; 2952 2953 for (i = 0; i < num_planes; i++) { 2954 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]); 2955 2956 result->plane_en[i] = skl_compute_plane_wm(dev_priv, 2957 p, &p->plane[i], 2958 ddb_blocks, 2959 level, 2960 &result->plane_res_b[i], 2961 &result->plane_res_l[i]); 2962 } 2963 2964 ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]); 2965 result->cursor_en = skl_compute_plane_wm(dev_priv, p, &p->cursor, 2966 ddb_blocks, level, 2967 &result->cursor_res_b, 2968 &result->cursor_res_l); 2969} 2970 2971static uint32_t 2972skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p) 2973{ 2974 if (!to_intel_crtc(crtc)->active) 2975 return 0; 2976 2977 return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate); 2978 2979} 2980 2981static void skl_compute_transition_wm(struct drm_crtc *crtc, 2982 struct skl_pipe_wm_parameters *params, 2983 struct skl_wm_level *trans_wm /* out */) 2984{ 2985 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 2986 int i; 2987 2988 if (!params->active) 2989 return; 2990 2991 /* Until we know more, just disable transition WMs */ 2992 for (i = 0; i < intel_num_planes(intel_crtc); i++) 2993 trans_wm->plane_en[i] = false; 2994 trans_wm->cursor_en = false; 2995} 2996 2997static void skl_compute_pipe_wm(struct drm_crtc *crtc, 2998 struct skl_ddb_allocation *ddb, 2999 struct skl_pipe_wm_parameters *params, 3000 struct skl_pipe_wm *pipe_wm) 3001{ 3002 struct drm_device *dev = crtc->dev; 3003 const struct drm_i915_private *dev_priv = dev->dev_private; 3004 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3005 int level, max_level = ilk_wm_max_level(dev); 3006 3007 for (level = 0; level <= max_level; level++) { 3008 skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe, 3009 level, intel_num_planes(intel_crtc), 3010 &pipe_wm->wm[level]); 3011 } 3012 pipe_wm->linetime = skl_compute_linetime_wm(crtc, params); 3013 3014 skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm); 3015} 3016 3017static void skl_compute_wm_results(struct drm_device *dev, 3018 struct skl_pipe_wm_parameters *p, 3019 struct skl_pipe_wm *p_wm, 3020 struct skl_wm_values *r, 3021 struct intel_crtc *intel_crtc) 3022{ 3023 int level, max_level = ilk_wm_max_level(dev); 3024 enum pipe pipe = intel_crtc->pipe; 3025 uint32_t temp; 3026 int i; 3027 3028 for (level = 0; level <= max_level; level++) { 3029 for (i = 0; i < intel_num_planes(intel_crtc); i++) { 3030 temp = 0; 3031 3032 temp |= p_wm->wm[level].plane_res_l[i] << 3033 PLANE_WM_LINES_SHIFT; 3034 temp |= p_wm->wm[level].plane_res_b[i]; 3035 if (p_wm->wm[level].plane_en[i]) 3036 temp |= PLANE_WM_EN; 3037 3038 r->plane[pipe][i][level] = temp; 3039 } 3040 3041 temp = 0; 3042 3043 temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT; 3044 temp |= p_wm->wm[level].cursor_res_b; 3045 3046 if (p_wm->wm[level].cursor_en) 3047 temp |= PLANE_WM_EN; 3048 3049 r->cursor[pipe][level] = temp; 3050 3051 } 3052 3053 /* transition WMs */ 3054 for (i = 0; i < intel_num_planes(intel_crtc); i++) { 3055 temp = 0; 3056 temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT; 3057 temp |= p_wm->trans_wm.plane_res_b[i]; 3058 if (p_wm->trans_wm.plane_en[i]) 3059 temp |= PLANE_WM_EN; 3060 3061 r->plane_trans[pipe][i] = temp; 3062 } 3063 3064 temp = 0; 3065 temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT; 3066 temp |= p_wm->trans_wm.cursor_res_b; 3067 if (p_wm->trans_wm.cursor_en) 3068 temp |= PLANE_WM_EN; 3069 3070 r->cursor_trans[pipe] = temp; 3071 3072 r->wm_linetime[pipe] = p_wm->linetime; 3073} 3074 3075static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg, 3076 const struct skl_ddb_entry *entry) 3077{ 3078 if (entry->end) 3079 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start); 3080 else 3081 I915_WRITE(reg, 0); 3082} 3083 3084static void skl_write_wm_values(struct drm_i915_private *dev_priv, 3085 const struct skl_wm_values *new) 3086{ 3087 struct drm_device *dev = dev_priv->dev; 3088 struct intel_crtc *crtc; 3089 3090 list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) { 3091 int i, level, max_level = ilk_wm_max_level(dev); 3092 enum pipe pipe = crtc->pipe; 3093 3094 if (!new->dirty[pipe]) 3095 continue; 3096 3097 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]); 3098 3099 for (level = 0; level <= max_level; level++) { 3100 for (i = 0; i < intel_num_planes(crtc); i++) 3101 I915_WRITE(PLANE_WM(pipe, i, level), 3102 new->plane[pipe][i][level]); 3103 I915_WRITE(CUR_WM(pipe, level), 3104 new->cursor[pipe][level]); 3105 } 3106 for (i = 0; i < intel_num_planes(crtc); i++) 3107 I915_WRITE(PLANE_WM_TRANS(pipe, i), 3108 new->plane_trans[pipe][i]); 3109 I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]); 3110 3111 for (i = 0; i < intel_num_planes(crtc); i++) 3112 skl_ddb_entry_write(dev_priv, 3113 PLANE_BUF_CFG(pipe, i), 3114 &new->ddb.plane[pipe][i]); 3115 3116 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), 3117 &new->ddb.cursor[pipe]); 3118 } 3119} 3120 3121/* 3122 * When setting up a new DDB allocation arrangement, we need to correctly 3123 * sequence the times at which the new allocations for the pipes are taken into 3124 * account or we'll have pipes fetching from space previously allocated to 3125 * another pipe. 3126 * 3127 * Roughly the sequence looks like: 3128 * 1. re-allocate the pipe(s) with the allocation being reduced and not 3129 * overlapping with a previous light-up pipe (another way to put it is: 3130 * pipes with their new allocation strickly included into their old ones). 3131 * 2. re-allocate the other pipes that get their allocation reduced 3132 * 3. allocate the pipes having their allocation increased 3133 * 3134 * Steps 1. and 2. are here to take care of the following case: 3135 * - Initially DDB looks like this: 3136 * | B | C | 3137 * - enable pipe A. 3138 * - pipe B has a reduced DDB allocation that overlaps with the old pipe C 3139 * allocation 3140 * | A | B | C | 3141 * 3142 * We need to sequence the re-allocation: C, B, A (and not B, C, A). 3143 */ 3144 3145static void 3146skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass) 3147{ 3148 int plane; 3149 3150 DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass); 3151 3152 for_each_plane(dev_priv, pipe, plane) { 3153 I915_WRITE(PLANE_SURF(pipe, plane), 3154 I915_READ(PLANE_SURF(pipe, plane))); 3155 } 3156 I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe))); 3157} 3158 3159static bool 3160skl_ddb_allocation_included(const struct skl_ddb_allocation *old, 3161 const struct skl_ddb_allocation *new, 3162 enum pipe pipe) 3163{ 3164 uint16_t old_size, new_size; 3165 3166 old_size = skl_ddb_entry_size(&old->pipe[pipe]); 3167 new_size = skl_ddb_entry_size(&new->pipe[pipe]); 3168 3169 return old_size != new_size && 3170 new->pipe[pipe].start >= old->pipe[pipe].start && 3171 new->pipe[pipe].end <= old->pipe[pipe].end; 3172} 3173 3174static void skl_flush_wm_values(struct drm_i915_private *dev_priv, 3175 struct skl_wm_values *new_values) 3176{ 3177 struct drm_device *dev = dev_priv->dev; 3178 struct skl_ddb_allocation *cur_ddb, *new_ddb; 3179 bool reallocated[I915_MAX_PIPES] = {false, false, false}; 3180 struct intel_crtc *crtc; 3181 enum pipe pipe; 3182 3183 new_ddb = &new_values->ddb; 3184 cur_ddb = &dev_priv->wm.skl_hw.ddb; 3185 3186 /* 3187 * First pass: flush the pipes with the new allocation contained into 3188 * the old space. 3189 * 3190 * We'll wait for the vblank on those pipes to ensure we can safely 3191 * re-allocate the freed space without this pipe fetching from it. 3192 */ 3193 for_each_intel_crtc(dev, crtc) { 3194 if (!crtc->active) 3195 continue; 3196 3197 pipe = crtc->pipe; 3198 3199 if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe)) 3200 continue; 3201 3202 skl_wm_flush_pipe(dev_priv, pipe, 1); 3203 intel_wait_for_vblank(dev, pipe); 3204 3205 reallocated[pipe] = true; 3206 } 3207 3208 3209 /* 3210 * Second pass: flush the pipes that are having their allocation 3211 * reduced, but overlapping with a previous allocation. 3212 * 3213 * Here as well we need to wait for the vblank to make sure the freed 3214 * space is not used anymore. 3215 */ 3216 for_each_intel_crtc(dev, crtc) { 3217 if (!crtc->active) 3218 continue; 3219 3220 pipe = crtc->pipe; 3221 3222 if (reallocated[pipe]) 3223 continue; 3224 3225 if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) < 3226 skl_ddb_entry_size(&cur_ddb->pipe[pipe])) { 3227 skl_wm_flush_pipe(dev_priv, pipe, 2); 3228 intel_wait_for_vblank(dev, pipe); 3229 reallocated[pipe] = true; 3230 } 3231 } 3232 3233 /* 3234 * Third pass: flush the pipes that got more space allocated. 3235 * 3236 * We don't need to actively wait for the update here, next vblank 3237 * will just get more DDB space with the correct WM values. 3238 */ 3239 for_each_intel_crtc(dev, crtc) { 3240 if (!crtc->active) 3241 continue; 3242 3243 pipe = crtc->pipe; 3244 3245 /* 3246 * At this point, only the pipes more space than before are 3247 * left to re-allocate. 3248 */ 3249 if (reallocated[pipe]) 3250 continue; 3251 3252 skl_wm_flush_pipe(dev_priv, pipe, 3); 3253 } 3254} 3255 3256static bool skl_update_pipe_wm(struct drm_crtc *crtc, 3257 struct skl_pipe_wm_parameters *params, 3258 struct intel_wm_config *config, 3259 struct skl_ddb_allocation *ddb, /* out */ 3260 struct skl_pipe_wm *pipe_wm /* out */) 3261{ 3262 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3263 3264 skl_compute_wm_pipe_parameters(crtc, params); 3265 skl_allocate_pipe_ddb(crtc, config, params, ddb); 3266 skl_compute_pipe_wm(crtc, ddb, params, pipe_wm); 3267 3268 if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm))) 3269 return false; 3270 3271 intel_crtc->wm.skl_active = *pipe_wm; 3272 return true; 3273} 3274 3275static void skl_update_other_pipe_wm(struct drm_device *dev, 3276 struct drm_crtc *crtc, 3277 struct intel_wm_config *config, 3278 struct skl_wm_values *r) 3279{ 3280 struct intel_crtc *intel_crtc; 3281 struct intel_crtc *this_crtc = to_intel_crtc(crtc); 3282 3283 /* 3284 * If the WM update hasn't changed the allocation for this_crtc (the 3285 * crtc we are currently computing the new WM values for), other 3286 * enabled crtcs will keep the same allocation and we don't need to 3287 * recompute anything for them. 3288 */ 3289 if (!skl_ddb_allocation_changed(&r->ddb, this_crtc)) 3290 return; 3291 3292 /* 3293 * Otherwise, because of this_crtc being freshly enabled/disabled, the 3294 * other active pipes need new DDB allocation and WM values. 3295 */ 3296 list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, 3297 base.head) { 3298 struct skl_pipe_wm_parameters params = {}; 3299 struct skl_pipe_wm pipe_wm = {}; 3300 bool wm_changed; 3301 3302 if (this_crtc->pipe == intel_crtc->pipe) 3303 continue; 3304 3305 if (!intel_crtc->active) 3306 continue; 3307 3308 wm_changed = skl_update_pipe_wm(&intel_crtc->base, 3309 ¶ms, config, 3310 &r->ddb, &pipe_wm); 3311 3312 /* 3313 * If we end up re-computing the other pipe WM values, it's 3314 * because it was really needed, so we expect the WM values to 3315 * be different. 3316 */ 3317 WARN_ON(!wm_changed); 3318 3319 skl_compute_wm_results(dev, ¶ms, &pipe_wm, r, intel_crtc); 3320 r->dirty[intel_crtc->pipe] = true; 3321 } 3322} 3323 3324static void skl_update_wm(struct drm_crtc *crtc) 3325{ 3326 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3327 struct drm_device *dev = crtc->dev; 3328 struct drm_i915_private *dev_priv = dev->dev_private; 3329 struct skl_pipe_wm_parameters params = {}; 3330 struct skl_wm_values *results = &dev_priv->wm.skl_results; 3331 struct skl_pipe_wm pipe_wm = {}; 3332 struct intel_wm_config config = {}; 3333 3334 memset(results, 0, sizeof(*results)); 3335 3336 skl_compute_wm_global_parameters(dev, &config); 3337 3338 if (!skl_update_pipe_wm(crtc, ¶ms, &config, 3339 &results->ddb, &pipe_wm)) 3340 return; 3341 3342 skl_compute_wm_results(dev, ¶ms, &pipe_wm, results, intel_crtc); 3343 results->dirty[intel_crtc->pipe] = true; 3344 3345 skl_update_other_pipe_wm(dev, crtc, &config, results); 3346 skl_write_wm_values(dev_priv, results); 3347 skl_flush_wm_values(dev_priv, results); 3348 3349 /* store the new configuration */ 3350 dev_priv->wm.skl_hw = *results; 3351} 3352 3353static void 3354skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc, 3355 uint32_t sprite_width, uint32_t sprite_height, 3356 int pixel_size, bool enabled, bool scaled) 3357{ 3358 struct intel_plane *intel_plane = to_intel_plane(plane); 3359 struct drm_framebuffer *fb = plane->state->fb; 3360 3361 intel_plane->wm.enabled = enabled; 3362 intel_plane->wm.scaled = scaled; 3363 intel_plane->wm.horiz_pixels = sprite_width; 3364 intel_plane->wm.vert_pixels = sprite_height; 3365 intel_plane->wm.bytes_per_pixel = pixel_size; 3366 intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE; 3367 /* 3368 * Framebuffer can be NULL on plane disable, but it does not 3369 * matter for watermarks if we assume no tiling in that case. 3370 */ 3371 if (fb) 3372 intel_plane->wm.tiling = fb->modifier[0]; 3373 intel_plane->wm.rotation = plane->state->rotation; 3374 3375 skl_update_wm(crtc); 3376} 3377 3378static void ilk_update_wm(struct drm_crtc *crtc) 3379{ 3380 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3381 struct drm_device *dev = crtc->dev; 3382 struct drm_i915_private *dev_priv = dev->dev_private; 3383 struct ilk_wm_maximums max; 3384 struct ilk_pipe_wm_parameters params = {}; 3385 struct ilk_wm_values results = {}; 3386 enum intel_ddb_partitioning partitioning; 3387 struct intel_pipe_wm pipe_wm = {}; 3388 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; 3389 struct intel_wm_config config = {}; 3390 3391 ilk_compute_wm_parameters(crtc, ¶ms); 3392 3393 intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm); 3394 3395 if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm))) 3396 return; 3397 3398 intel_crtc->wm.active = pipe_wm; 3399 3400 ilk_compute_wm_config(dev, &config); 3401 3402 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max); 3403 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2); 3404 3405 /* 5/6 split only in single pipe config on IVB+ */ 3406 if (INTEL_INFO(dev)->gen >= 7 && 3407 config.num_pipes_active == 1 && config.sprites_enabled) { 3408 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max); 3409 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6); 3410 3411 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6); 3412 } else { 3413 best_lp_wm = &lp_wm_1_2; 3414 } 3415 3416 partitioning = (best_lp_wm == &lp_wm_1_2) ? 3417 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; 3418 3419 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results); 3420 3421 ilk_write_wm_values(dev_priv, &results); 3422} 3423 3424static void 3425ilk_update_sprite_wm(struct drm_plane *plane, 3426 struct drm_crtc *crtc, 3427 uint32_t sprite_width, uint32_t sprite_height, 3428 int pixel_size, bool enabled, bool scaled) 3429{ 3430 struct drm_device *dev = plane->dev; 3431 struct intel_plane *intel_plane = to_intel_plane(plane); 3432 3433 intel_plane->wm.enabled = enabled; 3434 intel_plane->wm.scaled = scaled; 3435 intel_plane->wm.horiz_pixels = sprite_width; 3436 intel_plane->wm.vert_pixels = sprite_width; 3437 intel_plane->wm.bytes_per_pixel = pixel_size; 3438 3439 /* 3440 * IVB workaround: must disable low power watermarks for at least 3441 * one frame before enabling scaling. LP watermarks can be re-enabled 3442 * when scaling is disabled. 3443 * 3444 * WaCxSRDisabledForSpriteScaling:ivb 3445 */ 3446 if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev)) 3447 intel_wait_for_vblank(dev, intel_plane->pipe); 3448 3449 ilk_update_wm(crtc); 3450} 3451 3452static void skl_pipe_wm_active_state(uint32_t val, 3453 struct skl_pipe_wm *active, 3454 bool is_transwm, 3455 bool is_cursor, 3456 int i, 3457 int level) 3458{ 3459 bool is_enabled = (val & PLANE_WM_EN) != 0; 3460 3461 if (!is_transwm) { 3462 if (!is_cursor) { 3463 active->wm[level].plane_en[i] = is_enabled; 3464 active->wm[level].plane_res_b[i] = 3465 val & PLANE_WM_BLOCKS_MASK; 3466 active->wm[level].plane_res_l[i] = 3467 (val >> PLANE_WM_LINES_SHIFT) & 3468 PLANE_WM_LINES_MASK; 3469 } else { 3470 active->wm[level].cursor_en = is_enabled; 3471 active->wm[level].cursor_res_b = 3472 val & PLANE_WM_BLOCKS_MASK; 3473 active->wm[level].cursor_res_l = 3474 (val >> PLANE_WM_LINES_SHIFT) & 3475 PLANE_WM_LINES_MASK; 3476 } 3477 } else { 3478 if (!is_cursor) { 3479 active->trans_wm.plane_en[i] = is_enabled; 3480 active->trans_wm.plane_res_b[i] = 3481 val & PLANE_WM_BLOCKS_MASK; 3482 active->trans_wm.plane_res_l[i] = 3483 (val >> PLANE_WM_LINES_SHIFT) & 3484 PLANE_WM_LINES_MASK; 3485 } else { 3486 active->trans_wm.cursor_en = is_enabled; 3487 active->trans_wm.cursor_res_b = 3488 val & PLANE_WM_BLOCKS_MASK; 3489 active->trans_wm.cursor_res_l = 3490 (val >> PLANE_WM_LINES_SHIFT) & 3491 PLANE_WM_LINES_MASK; 3492 } 3493 } 3494} 3495 3496static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) 3497{ 3498 struct drm_device *dev = crtc->dev; 3499 struct drm_i915_private *dev_priv = dev->dev_private; 3500 struct skl_wm_values *hw = &dev_priv->wm.skl_hw; 3501 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3502 struct skl_pipe_wm *active = &intel_crtc->wm.skl_active; 3503 enum pipe pipe = intel_crtc->pipe; 3504 int level, i, max_level; 3505 uint32_t temp; 3506 3507 max_level = ilk_wm_max_level(dev); 3508 3509 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe)); 3510 3511 for (level = 0; level <= max_level; level++) { 3512 for (i = 0; i < intel_num_planes(intel_crtc); i++) 3513 hw->plane[pipe][i][level] = 3514 I915_READ(PLANE_WM(pipe, i, level)); 3515 hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level)); 3516 } 3517 3518 for (i = 0; i < intel_num_planes(intel_crtc); i++) 3519 hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i)); 3520 hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe)); 3521 3522 if (!intel_crtc->active) 3523 return; 3524 3525 hw->dirty[pipe] = true; 3526 3527 active->linetime = hw->wm_linetime[pipe]; 3528 3529 for (level = 0; level <= max_level; level++) { 3530 for (i = 0; i < intel_num_planes(intel_crtc); i++) { 3531 temp = hw->plane[pipe][i][level]; 3532 skl_pipe_wm_active_state(temp, active, false, 3533 false, i, level); 3534 } 3535 temp = hw->cursor[pipe][level]; 3536 skl_pipe_wm_active_state(temp, active, false, true, i, level); 3537 } 3538 3539 for (i = 0; i < intel_num_planes(intel_crtc); i++) { 3540 temp = hw->plane_trans[pipe][i]; 3541 skl_pipe_wm_active_state(temp, active, true, false, i, 0); 3542 } 3543 3544 temp = hw->cursor_trans[pipe]; 3545 skl_pipe_wm_active_state(temp, active, true, true, i, 0); 3546} 3547 3548void skl_wm_get_hw_state(struct drm_device *dev) 3549{ 3550 struct drm_i915_private *dev_priv = dev->dev_private; 3551 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb; 3552 struct drm_crtc *crtc; 3553 3554 skl_ddb_get_hw_state(dev_priv, ddb); 3555 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) 3556 skl_pipe_wm_get_hw_state(crtc); 3557} 3558 3559static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) 3560{ 3561 struct drm_device *dev = crtc->dev; 3562 struct drm_i915_private *dev_priv = dev->dev_private; 3563 struct ilk_wm_values *hw = &dev_priv->wm.hw; 3564 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3565 struct intel_pipe_wm *active = &intel_crtc->wm.active; 3566 enum pipe pipe = intel_crtc->pipe; 3567 static const unsigned int wm0_pipe_reg[] = { 3568 [PIPE_A] = WM0_PIPEA_ILK, 3569 [PIPE_B] = WM0_PIPEB_ILK, 3570 [PIPE_C] = WM0_PIPEC_IVB, 3571 }; 3572 3573 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]); 3574 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 3575 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe)); 3576 3577 memset(active, 0, sizeof(*active)); 3578 3579 active->pipe_enabled = intel_crtc->active; 3580 3581 if (active->pipe_enabled) { 3582 u32 tmp = hw->wm_pipe[pipe]; 3583 3584 /* 3585 * For active pipes LP0 watermark is marked as 3586 * enabled, and LP1+ watermaks as disabled since 3587 * we can't really reverse compute them in case 3588 * multiple pipes are active. 3589 */ 3590 active->wm[0].enable = true; 3591 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT; 3592 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT; 3593 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK; 3594 active->linetime = hw->wm_linetime[pipe]; 3595 } else { 3596 int level, max_level = ilk_wm_max_level(dev); 3597 3598 /* 3599 * For inactive pipes, all watermark levels 3600 * should be marked as enabled but zeroed, 3601 * which is what we'd compute them to. 3602 */ 3603 for (level = 0; level <= max_level; level++) 3604 active->wm[level].enable = true; 3605 } 3606} 3607 3608void ilk_wm_get_hw_state(struct drm_device *dev) 3609{ 3610 struct drm_i915_private *dev_priv = dev->dev_private; 3611 struct ilk_wm_values *hw = &dev_priv->wm.hw; 3612 struct drm_crtc *crtc; 3613 3614 for_each_crtc(dev, crtc) 3615 ilk_pipe_wm_get_hw_state(crtc); 3616 3617 hw->wm_lp[0] = I915_READ(WM1_LP_ILK); 3618 hw->wm_lp[1] = I915_READ(WM2_LP_ILK); 3619 hw->wm_lp[2] = I915_READ(WM3_LP_ILK); 3620 3621 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK); 3622 if (INTEL_INFO(dev)->gen >= 7) { 3623 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); 3624 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); 3625 } 3626 3627 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 3628 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ? 3629 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 3630 else if (IS_IVYBRIDGE(dev)) 3631 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ? 3632 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 3633 3634 hw->enable_fbc_wm = 3635 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS); 3636} 3637 3638/** 3639 * intel_update_watermarks - update FIFO watermark values based on current modes 3640 * 3641 * Calculate watermark values for the various WM regs based on current mode 3642 * and plane configuration. 3643 * 3644 * There are several cases to deal with here: 3645 * - normal (i.e. non-self-refresh) 3646 * - self-refresh (SR) mode 3647 * - lines are large relative to FIFO size (buffer can hold up to 2) 3648 * - lines are small relative to FIFO size (buffer can hold more than 2 3649 * lines), so need to account for TLB latency 3650 * 3651 * The normal calculation is: 3652 * watermark = dotclock * bytes per pixel * latency 3653 * where latency is platform & configuration dependent (we assume pessimal 3654 * values here). 3655 * 3656 * The SR calculation is: 3657 * watermark = (trunc(latency/line time)+1) * surface width * 3658 * bytes per pixel 3659 * where 3660 * line time = htotal / dotclock 3661 * surface width = hdisplay for normal plane and 64 for cursor 3662 * and latency is assumed to be high, as above. 3663 * 3664 * The final value programmed to the register should always be rounded up, 3665 * and include an extra 2 entries to account for clock crossings. 3666 * 3667 * We don't use the sprite, so we can ignore that. And on Crestline we have 3668 * to set the non-SR watermarks to 8. 3669 */ 3670void intel_update_watermarks(struct drm_crtc *crtc) 3671{ 3672 struct drm_i915_private *dev_priv = crtc->dev->dev_private; 3673 3674 if (dev_priv->display.update_wm) 3675 dev_priv->display.update_wm(crtc); 3676} 3677 3678void intel_update_sprite_watermarks(struct drm_plane *plane, 3679 struct drm_crtc *crtc, 3680 uint32_t sprite_width, 3681 uint32_t sprite_height, 3682 int pixel_size, 3683 bool enabled, bool scaled) 3684{ 3685 struct drm_i915_private *dev_priv = plane->dev->dev_private; 3686 3687 if (dev_priv->display.update_sprite_wm) 3688 dev_priv->display.update_sprite_wm(plane, crtc, 3689 sprite_width, sprite_height, 3690 pixel_size, enabled, scaled); 3691} 3692 3693/** 3694 * Lock protecting IPS related data structures 3695 */ 3696DEFINE_SPINLOCK(mchdev_lock); 3697 3698/* Global for IPS driver to get at the current i915 device. Protected by 3699 * mchdev_lock. */ 3700static struct drm_i915_private *i915_mch_dev; 3701 3702bool ironlake_set_drps(struct drm_device *dev, u8 val) 3703{ 3704 struct drm_i915_private *dev_priv = dev->dev_private; 3705 u16 rgvswctl; 3706 3707 assert_spin_locked(&mchdev_lock); 3708 3709 rgvswctl = I915_READ16(MEMSWCTL); 3710 if (rgvswctl & MEMCTL_CMD_STS) { 3711 DRM_DEBUG("gpu busy, RCS change rejected\n"); 3712 return false; /* still busy with another command */ 3713 } 3714 3715 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 3716 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM; 3717 I915_WRITE16(MEMSWCTL, rgvswctl); 3718 POSTING_READ16(MEMSWCTL); 3719 3720 rgvswctl |= MEMCTL_CMD_STS; 3721 I915_WRITE16(MEMSWCTL, rgvswctl); 3722 3723 return true; 3724} 3725 3726static void ironlake_enable_drps(struct drm_device *dev) 3727{ 3728 struct drm_i915_private *dev_priv = dev->dev_private; 3729 u32 rgvmodectl = I915_READ(MEMMODECTL); 3730 u8 fmax, fmin, fstart, vstart; 3731 3732 spin_lock_irq(&mchdev_lock); 3733 3734 /* Enable temp reporting */ 3735 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN); 3736 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE); 3737 3738 /* 100ms RC evaluation intervals */ 3739 I915_WRITE(RCUPEI, 100000); 3740 I915_WRITE(RCDNEI, 100000); 3741 3742 /* Set max/min thresholds to 90ms and 80ms respectively */ 3743 I915_WRITE(RCBMAXAVG, 90000); 3744 I915_WRITE(RCBMINAVG, 80000); 3745 3746 I915_WRITE(MEMIHYST, 1); 3747 3748 /* Set up min, max, and cur for interrupt handling */ 3749 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 3750 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 3751 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 3752 MEMMODE_FSTART_SHIFT; 3753 3754 vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >> 3755 PXVFREQ_PX_SHIFT; 3756 3757 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */ 3758 dev_priv->ips.fstart = fstart; 3759 3760 dev_priv->ips.max_delay = fstart; 3761 dev_priv->ips.min_delay = fmin; 3762 dev_priv->ips.cur_delay = fstart; 3763 3764 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", 3765 fmax, fmin, fstart); 3766 3767 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 3768 3769 /* 3770 * Interrupts will be enabled in ironlake_irq_postinstall 3771 */ 3772 3773 I915_WRITE(VIDSTART, vstart); 3774 POSTING_READ(VIDSTART); 3775 3776 rgvmodectl |= MEMMODE_SWMODE_EN; 3777 I915_WRITE(MEMMODECTL, rgvmodectl); 3778 3779 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10)) 3780 DRM_ERROR("stuck trying to change perf mode\n"); 3781 mdelay(1); 3782 3783 ironlake_set_drps(dev, fstart); 3784 3785 dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) + 3786 I915_READ(0x112e0); 3787 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies); 3788 dev_priv->ips.last_count2 = I915_READ(0x112f4); 3789 dev_priv->ips.last_time2 = ktime_get_raw_ns(); 3790 3791 spin_unlock_irq(&mchdev_lock); 3792} 3793 3794static void ironlake_disable_drps(struct drm_device *dev) 3795{ 3796 struct drm_i915_private *dev_priv = dev->dev_private; 3797 u16 rgvswctl; 3798 3799 spin_lock_irq(&mchdev_lock); 3800 3801 rgvswctl = I915_READ16(MEMSWCTL); 3802 3803 /* Ack interrupts, disable EFC interrupt */ 3804 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN); 3805 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG); 3806 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT); 3807 I915_WRITE(DEIIR, DE_PCU_EVENT); 3808 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT); 3809 3810 /* Go back to the starting frequency */ 3811 ironlake_set_drps(dev, dev_priv->ips.fstart); 3812 mdelay(1); 3813 rgvswctl |= MEMCTL_CMD_STS; 3814 I915_WRITE(MEMSWCTL, rgvswctl); 3815 mdelay(1); 3816 3817 spin_unlock_irq(&mchdev_lock); 3818} 3819 3820/* There's a funny hw issue where the hw returns all 0 when reading from 3821 * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value 3822 * ourselves, instead of doing a rmw cycle (which might result in us clearing 3823 * all limits and the gpu stuck at whatever frequency it is at atm). 3824 */ 3825static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val) 3826{ 3827 u32 limits; 3828 3829 /* Only set the down limit when we've reached the lowest level to avoid 3830 * getting more interrupts, otherwise leave this clear. This prevents a 3831 * race in the hw when coming out of rc6: There's a tiny window where 3832 * the hw runs at the minimal clock before selecting the desired 3833 * frequency, if the down threshold expires in that window we will not 3834 * receive a down interrupt. */ 3835 if (IS_GEN9(dev_priv->dev)) { 3836 limits = (dev_priv->rps.max_freq_softlimit) << 23; 3837 if (val <= dev_priv->rps.min_freq_softlimit) 3838 limits |= (dev_priv->rps.min_freq_softlimit) << 14; 3839 } else { 3840 limits = dev_priv->rps.max_freq_softlimit << 24; 3841 if (val <= dev_priv->rps.min_freq_softlimit) 3842 limits |= dev_priv->rps.min_freq_softlimit << 16; 3843 } 3844 3845 return limits; 3846} 3847 3848static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) 3849{ 3850 int new_power; 3851 u32 threshold_up = 0, threshold_down = 0; /* in % */ 3852 u32 ei_up = 0, ei_down = 0; 3853 3854 new_power = dev_priv->rps.power; 3855 switch (dev_priv->rps.power) { 3856 case LOW_POWER: 3857 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq) 3858 new_power = BETWEEN; 3859 break; 3860 3861 case BETWEEN: 3862 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq) 3863 new_power = LOW_POWER; 3864 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq) 3865 new_power = HIGH_POWER; 3866 break; 3867 3868 case HIGH_POWER: 3869 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq) 3870 new_power = BETWEEN; 3871 break; 3872 } 3873 /* Max/min bins are special */ 3874 if (val <= dev_priv->rps.min_freq_softlimit) 3875 new_power = LOW_POWER; 3876 if (val >= dev_priv->rps.max_freq_softlimit) 3877 new_power = HIGH_POWER; 3878 if (new_power == dev_priv->rps.power) 3879 return; 3880 3881 /* Note the units here are not exactly 1us, but 1280ns. */ 3882 switch (new_power) { 3883 case LOW_POWER: 3884 /* Upclock if more than 95% busy over 16ms */ 3885 ei_up = 16000; 3886 threshold_up = 95; 3887 3888 /* Downclock if less than 85% busy over 32ms */ 3889 ei_down = 32000; 3890 threshold_down = 85; 3891 break; 3892 3893 case BETWEEN: 3894 /* Upclock if more than 90% busy over 13ms */ 3895 ei_up = 13000; 3896 threshold_up = 90; 3897 3898 /* Downclock if less than 75% busy over 32ms */ 3899 ei_down = 32000; 3900 threshold_down = 75; 3901 break; 3902 3903 case HIGH_POWER: 3904 /* Upclock if more than 85% busy over 10ms */ 3905 ei_up = 10000; 3906 threshold_up = 85; 3907 3908 /* Downclock if less than 60% busy over 32ms */ 3909 ei_down = 32000; 3910 threshold_down = 60; 3911 break; 3912 } 3913 3914 I915_WRITE(GEN6_RP_UP_EI, 3915 GT_INTERVAL_FROM_US(dev_priv, ei_up)); 3916 I915_WRITE(GEN6_RP_UP_THRESHOLD, 3917 GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100))); 3918 3919 I915_WRITE(GEN6_RP_DOWN_EI, 3920 GT_INTERVAL_FROM_US(dev_priv, ei_down)); 3921 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 3922 GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100))); 3923 3924 I915_WRITE(GEN6_RP_CONTROL, 3925 GEN6_RP_MEDIA_TURBO | 3926 GEN6_RP_MEDIA_HW_NORMAL_MODE | 3927 GEN6_RP_MEDIA_IS_GFX | 3928 GEN6_RP_ENABLE | 3929 GEN6_RP_UP_BUSY_AVG | 3930 GEN6_RP_DOWN_IDLE_AVG); 3931 3932 dev_priv->rps.power = new_power; 3933 dev_priv->rps.last_adj = 0; 3934} 3935 3936static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) 3937{ 3938 u32 mask = 0; 3939 3940 if (val > dev_priv->rps.min_freq_softlimit) 3941 mask |= GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT; 3942 if (val < dev_priv->rps.max_freq_softlimit) 3943 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 3944 3945 mask &= dev_priv->pm_rps_events; 3946 3947 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask); 3948} 3949 3950/* gen6_set_rps is called to update the frequency request, but should also be 3951 * called when the range (min_delay and max_delay) is modified so that we can 3952 * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ 3953static void gen6_set_rps(struct drm_device *dev, u8 val) 3954{ 3955 struct drm_i915_private *dev_priv = dev->dev_private; 3956 3957 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 3958 WARN_ON(val > dev_priv->rps.max_freq); 3959 WARN_ON(val < dev_priv->rps.min_freq); 3960 3961 /* min/max delay may still have been modified so be sure to 3962 * write the limits value. 3963 */ 3964 if (val != dev_priv->rps.cur_freq) { 3965 gen6_set_rps_thresholds(dev_priv, val); 3966 3967 if (IS_GEN9(dev)) 3968 I915_WRITE(GEN6_RPNSWREQ, 3969 GEN9_FREQUENCY(val)); 3970 else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 3971 I915_WRITE(GEN6_RPNSWREQ, 3972 HSW_FREQUENCY(val)); 3973 else 3974 I915_WRITE(GEN6_RPNSWREQ, 3975 GEN6_FREQUENCY(val) | 3976 GEN6_OFFSET(0) | 3977 GEN6_AGGRESSIVE_TURBO); 3978 } 3979 3980 /* Make sure we continue to get interrupts 3981 * until we hit the minimum or maximum frequencies. 3982 */ 3983 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val)); 3984 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); 3985 3986 POSTING_READ(GEN6_RPNSWREQ); 3987 3988 dev_priv->rps.cur_freq = val; 3989 trace_intel_gpu_freq_change(val * 50); 3990} 3991 3992static void valleyview_set_rps(struct drm_device *dev, u8 val) 3993{ 3994 struct drm_i915_private *dev_priv = dev->dev_private; 3995 3996 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 3997 WARN_ON(val > dev_priv->rps.max_freq); 3998 WARN_ON(val < dev_priv->rps.min_freq); 3999 4000 if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1), 4001 "Odd GPU freq value\n")) 4002 val &= ~1; 4003 4004 if (val != dev_priv->rps.cur_freq) 4005 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); 4006 4007 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); 4008 4009 dev_priv->rps.cur_freq = val; 4010 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); 4011} 4012 4013/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down 4014 * 4015 * * If Gfx is Idle, then 4016 * 1. Mask Turbo interrupts 4017 * 2. Bring up Gfx clock 4018 * 3. Change the freq to Rpn and wait till P-Unit updates freq 4019 * 4. Clear the Force GFX CLK ON bit so that Gfx can down 4020 * 5. Unmask Turbo interrupts 4021*/ 4022static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) 4023{ 4024 struct drm_device *dev = dev_priv->dev; 4025 u32 val = dev_priv->rps.idle_freq; 4026 4027 /* CHV and latest VLV don't need to force the gfx clock */ 4028 if (IS_CHERRYVIEW(dev) || dev->pdev->revision >= 0xd) { 4029 valleyview_set_rps(dev_priv->dev, val); 4030 return; 4031 } 4032 4033 /* 4034 * When we are idle. Drop to min voltage state. 4035 */ 4036 4037 if (dev_priv->rps.cur_freq <= val) 4038 return; 4039 4040 /* Mask turbo interrupt so that they will not come in between */ 4041 I915_WRITE(GEN6_PMINTRMSK, 4042 gen6_sanitize_rps_pm_mask(dev_priv, ~0)); 4043 4044 vlv_force_gfx_clock(dev_priv, true); 4045 4046 dev_priv->rps.cur_freq = val; 4047 4048 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); 4049 4050 if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) 4051 & GENFREQSTATUS) == 0, 100)) 4052 DRM_ERROR("timed out waiting for Punit\n"); 4053 4054 vlv_force_gfx_clock(dev_priv, false); 4055 4056 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); 4057} 4058 4059void gen6_rps_busy(struct drm_i915_private *dev_priv) 4060{ 4061 mutex_lock(&dev_priv->rps.hw_lock); 4062 if (dev_priv->rps.enabled) { 4063 if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) 4064 gen6_rps_reset_ei(dev_priv); 4065 I915_WRITE(GEN6_PMINTRMSK, 4066 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq)); 4067 } 4068 mutex_unlock(&dev_priv->rps.hw_lock); 4069} 4070 4071void gen6_rps_idle(struct drm_i915_private *dev_priv) 4072{ 4073 struct drm_device *dev = dev_priv->dev; 4074 4075 mutex_lock(&dev_priv->rps.hw_lock); 4076 if (dev_priv->rps.enabled) { 4077 if (IS_VALLEYVIEW(dev)) 4078 vlv_set_rps_idle(dev_priv); 4079 else 4080 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 4081 dev_priv->rps.last_adj = 0; 4082 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); 4083 } 4084 mutex_unlock(&dev_priv->rps.hw_lock); 4085} 4086 4087void gen6_rps_boost(struct drm_i915_private *dev_priv) 4088{ 4089 u32 val; 4090 4091 mutex_lock(&dev_priv->rps.hw_lock); 4092 val = dev_priv->rps.max_freq_softlimit; 4093 if (dev_priv->rps.enabled && 4094 dev_priv->mm.busy && 4095 dev_priv->rps.cur_freq < val) { 4096 intel_set_rps(dev_priv->dev, val); 4097 dev_priv->rps.last_adj = 0; 4098 } 4099 mutex_unlock(&dev_priv->rps.hw_lock); 4100} 4101 4102void intel_set_rps(struct drm_device *dev, u8 val) 4103{ 4104 if (IS_VALLEYVIEW(dev)) 4105 valleyview_set_rps(dev, val); 4106 else 4107 gen6_set_rps(dev, val); 4108} 4109 4110static void gen9_disable_rps(struct drm_device *dev) 4111{ 4112 struct drm_i915_private *dev_priv = dev->dev_private; 4113 4114 I915_WRITE(GEN6_RC_CONTROL, 0); 4115 I915_WRITE(GEN9_PG_ENABLE, 0); 4116} 4117 4118static void gen6_disable_rps(struct drm_device *dev) 4119{ 4120 struct drm_i915_private *dev_priv = dev->dev_private; 4121 4122 I915_WRITE(GEN6_RC_CONTROL, 0); 4123 I915_WRITE(GEN6_RPNSWREQ, 1 << 31); 4124} 4125 4126static void cherryview_disable_rps(struct drm_device *dev) 4127{ 4128 struct drm_i915_private *dev_priv = dev->dev_private; 4129 4130 I915_WRITE(GEN6_RC_CONTROL, 0); 4131} 4132 4133static void valleyview_disable_rps(struct drm_device *dev) 4134{ 4135 struct drm_i915_private *dev_priv = dev->dev_private; 4136 4137 /* we're doing forcewake before Disabling RC6, 4138 * This what the BIOS expects when going into suspend */ 4139 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 4140 4141 I915_WRITE(GEN6_RC_CONTROL, 0); 4142 4143 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 4144} 4145 4146static void intel_print_rc6_info(struct drm_device *dev, u32 mode) 4147{ 4148 if (IS_VALLEYVIEW(dev)) { 4149 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1))) 4150 mode = GEN6_RC_CTL_RC6_ENABLE; 4151 else 4152 mode = 0; 4153 } 4154 if (HAS_RC6p(dev)) 4155 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n", 4156 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off", 4157 (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off", 4158 (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off"); 4159 4160 else 4161 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n", 4162 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off"); 4163} 4164 4165static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) 4166{ 4167 /* No RC6 before Ironlake */ 4168 if (INTEL_INFO(dev)->gen < 5) 4169 return 0; 4170 4171 /* RC6 is only on Ironlake mobile not on desktop */ 4172 if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev)) 4173 return 0; 4174 4175 /* Respect the kernel parameter if it is set */ 4176 if (enable_rc6 >= 0) { 4177 int mask; 4178 4179 if (HAS_RC6p(dev)) 4180 mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE | 4181 INTEL_RC6pp_ENABLE; 4182 else 4183 mask = INTEL_RC6_ENABLE; 4184 4185 if ((enable_rc6 & mask) != enable_rc6) 4186 DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n", 4187 enable_rc6 & mask, enable_rc6, mask); 4188 4189 return enable_rc6 & mask; 4190 } 4191 4192 /* Disable RC6 on Ironlake */ 4193 if (INTEL_INFO(dev)->gen == 5) 4194 return 0; 4195 4196 if (IS_IVYBRIDGE(dev)) 4197 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); 4198 4199 return INTEL_RC6_ENABLE; 4200} 4201 4202int intel_enable_rc6(const struct drm_device *dev) 4203{ 4204 return i915.enable_rc6; 4205} 4206 4207static void gen6_init_rps_frequencies(struct drm_device *dev) 4208{ 4209 struct drm_i915_private *dev_priv = dev->dev_private; 4210 uint32_t rp_state_cap; 4211 u32 ddcc_status = 0; 4212 int ret; 4213 4214 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); 4215 /* All of these values are in units of 50MHz */ 4216 dev_priv->rps.cur_freq = 0; 4217 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 4218 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff; 4219 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; 4220 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff; 4221 if (IS_SKYLAKE(dev)) { 4222 /* Store the frequency values in 16.66 MHZ units, which is 4223 the natural hardware unit for SKL */ 4224 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER; 4225 dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER; 4226 dev_priv->rps.min_freq *= GEN9_FREQ_SCALER; 4227 } 4228 /* hw_max = RP0 until we check for overclocking */ 4229 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; 4230 4231 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; 4232 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { 4233 ret = sandybridge_pcode_read(dev_priv, 4234 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 4235 &ddcc_status); 4236 if (0 == ret) 4237 dev_priv->rps.efficient_freq = 4238 clamp_t(u8, 4239 ((ddcc_status >> 8) & 0xff), 4240 dev_priv->rps.min_freq, 4241 dev_priv->rps.max_freq); 4242 } 4243 4244 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; 4245 4246 /* Preserve min/max settings in case of re-init */ 4247 if (dev_priv->rps.max_freq_softlimit == 0) 4248 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; 4249 4250 if (dev_priv->rps.min_freq_softlimit == 0) { 4251 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 4252 dev_priv->rps.min_freq_softlimit = 4253 /* max(RPe, 450 MHz) */ 4254 max(dev_priv->rps.efficient_freq, (u8) 9); 4255 else 4256 dev_priv->rps.min_freq_softlimit = 4257 dev_priv->rps.min_freq; 4258 } 4259} 4260 4261/* See the Gen9_GT_PM_Programming_Guide doc for the below */ 4262static void gen9_enable_rps(struct drm_device *dev) 4263{ 4264 struct drm_i915_private *dev_priv = dev->dev_private; 4265 4266 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 4267 4268 gen6_init_rps_frequencies(dev); 4269 4270 /* Program defaults and thresholds for RPS*/ 4271 I915_WRITE(GEN6_RC_VIDEO_FREQ, 4272 GEN9_FREQUENCY(dev_priv->rps.rp1_freq)); 4273 4274 /* 1 second timeout*/ 4275 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 4276 GT_INTERVAL_FROM_US(dev_priv, 1000000)); 4277 4278 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa); 4279 4280 /* Leaning on the below call to gen6_set_rps to program/setup the 4281 * Up/Down EI & threshold registers, as well as the RP_CONTROL, 4282 * RP_INTERRUPT_LIMITS & RPNSWREQ registers */ 4283 dev_priv->rps.power = HIGH_POWER; /* force a reset */ 4284 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); 4285 4286 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 4287} 4288 4289static void gen9_enable_rc6(struct drm_device *dev) 4290{ 4291 struct drm_i915_private *dev_priv = dev->dev_private; 4292 struct intel_engine_cs *ring; 4293 uint32_t rc6_mask = 0; 4294 int unused; 4295 4296 /* 1a: Software RC state - RC0 */ 4297 I915_WRITE(GEN6_RC_STATE, 0); 4298 4299 /* 1b: Get forcewake during program sequence. Although the driver 4300 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/ 4301 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 4302 4303 /* 2a: Disable RC states. */ 4304 I915_WRITE(GEN6_RC_CONTROL, 0); 4305 4306 /* 2b: Program RC6 thresholds.*/ 4307 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); 4308 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 4309 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 4310 for_each_ring(ring, dev_priv, unused) 4311 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 4312 I915_WRITE(GEN6_RC_SLEEP, 0); 4313 I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */ 4314 4315 /* 2c: Program Coarse Power Gating Policies. */ 4316 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25); 4317 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25); 4318 4319 /* 3a: Enable RC6 */ 4320 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) 4321 rc6_mask = GEN6_RC_CTL_RC6_ENABLE; 4322 DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? 4323 "on" : "off"); 4324 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | 4325 GEN6_RC_CTL_EI_MODE(1) | 4326 rc6_mask); 4327 4328 /* 3b: Enable Coarse Power Gating only when RC6 is enabled */ 4329 I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? 3 : 0); 4330 4331 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 4332 4333} 4334 4335static void gen8_enable_rps(struct drm_device *dev) 4336{ 4337 struct drm_i915_private *dev_priv = dev->dev_private; 4338 struct intel_engine_cs *ring; 4339 uint32_t rc6_mask = 0; 4340 int unused; 4341 4342 /* 1a: Software RC state - RC0 */ 4343 I915_WRITE(GEN6_RC_STATE, 0); 4344 4345 /* 1c & 1d: Get forcewake during program sequence. Although the driver 4346 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/ 4347 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 4348 4349 /* 2a: Disable RC states. */ 4350 I915_WRITE(GEN6_RC_CONTROL, 0); 4351 4352 /* Initialize rps frequencies */ 4353 gen6_init_rps_frequencies(dev); 4354 4355 /* 2b: Program RC6 thresholds.*/ 4356 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); 4357 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 4358 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 4359 for_each_ring(ring, dev_priv, unused) 4360 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 4361 I915_WRITE(GEN6_RC_SLEEP, 0); 4362 if (IS_BROADWELL(dev)) 4363 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ 4364 else 4365 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ 4366 4367 /* 3: Enable RC6 */ 4368 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) 4369 rc6_mask = GEN6_RC_CTL_RC6_ENABLE; 4370 intel_print_rc6_info(dev, rc6_mask); 4371 if (IS_BROADWELL(dev)) 4372 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | 4373 GEN7_RC_CTL_TO_MODE | 4374 rc6_mask); 4375 else 4376 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | 4377 GEN6_RC_CTL_EI_MODE(1) | 4378 rc6_mask); 4379 4380 /* 4 Program defaults and thresholds for RPS*/ 4381 I915_WRITE(GEN6_RPNSWREQ, 4382 HSW_FREQUENCY(dev_priv->rps.rp1_freq)); 4383 I915_WRITE(GEN6_RC_VIDEO_FREQ, 4384 HSW_FREQUENCY(dev_priv->rps.rp1_freq)); 4385 /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ 4386 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */ 4387 4388 /* Docs recommend 900MHz, and 300 MHz respectively */ 4389 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, 4390 dev_priv->rps.max_freq_softlimit << 24 | 4391 dev_priv->rps.min_freq_softlimit << 16); 4392 4393 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */ 4394 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/ 4395 I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */ 4396 I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */ 4397 4398 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); 4399 4400 /* 5: Enable RPS */ 4401 I915_WRITE(GEN6_RP_CONTROL, 4402 GEN6_RP_MEDIA_TURBO | 4403 GEN6_RP_MEDIA_HW_NORMAL_MODE | 4404 GEN6_RP_MEDIA_IS_GFX | 4405 GEN6_RP_ENABLE | 4406 GEN6_RP_UP_BUSY_AVG | 4407 GEN6_RP_DOWN_IDLE_AVG); 4408 4409 /* 6: Ring frequency + overclocking (our driver does this later */ 4410 4411 dev_priv->rps.power = HIGH_POWER; /* force a reset */ 4412 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 4413 4414 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 4415} 4416 4417static void gen6_enable_rps(struct drm_device *dev) 4418{ 4419 struct drm_i915_private *dev_priv = dev->dev_private; 4420 struct intel_engine_cs *ring; 4421 u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; 4422 u32 gtfifodbg; 4423 int rc6_mode; 4424 int i, ret; 4425 4426 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 4427 4428 /* Here begins a magic sequence of register writes to enable 4429 * auto-downclocking. 4430 * 4431 * Perhaps there might be some value in exposing these to 4432 * userspace... 4433 */ 4434 I915_WRITE(GEN6_RC_STATE, 0); 4435 4436 /* Clear the DBG now so we don't confuse earlier errors */ 4437 if ((gtfifodbg = I915_READ(GTFIFODBG))) { 4438 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg); 4439 I915_WRITE(GTFIFODBG, gtfifodbg); 4440 } 4441 4442 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 4443 4444 /* Initialize rps frequencies */ 4445 gen6_init_rps_frequencies(dev); 4446 4447 /* disable the counters and set deterministic thresholds */ 4448 I915_WRITE(GEN6_RC_CONTROL, 0); 4449 4450 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16); 4451 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30); 4452 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30); 4453 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); 4454 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); 4455 4456 for_each_ring(ring, dev_priv, i) 4457 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 4458 4459 I915_WRITE(GEN6_RC_SLEEP, 0); 4460 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000); 4461 if (IS_IVYBRIDGE(dev)) 4462 I915_WRITE(GEN6_RC6_THRESHOLD, 125000); 4463 else 4464 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); 4465 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000); 4466 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */ 4467 4468 /* Check if we are enabling RC6 */ 4469 rc6_mode = intel_enable_rc6(dev_priv->dev); 4470 if (rc6_mode & INTEL_RC6_ENABLE) 4471 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE; 4472 4473 /* We don't use those on Haswell */ 4474 if (!IS_HASWELL(dev)) { 4475 if (rc6_mode & INTEL_RC6p_ENABLE) 4476 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE; 4477 4478 if (rc6_mode & INTEL_RC6pp_ENABLE) 4479 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE; 4480 } 4481 4482 intel_print_rc6_info(dev, rc6_mask); 4483 4484 I915_WRITE(GEN6_RC_CONTROL, 4485 rc6_mask | 4486 GEN6_RC_CTL_EI_MODE(1) | 4487 GEN6_RC_CTL_HW_ENABLE); 4488 4489 /* Power down if completely idle for over 50ms */ 4490 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000); 4491 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); 4492 4493 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0); 4494 if (ret) 4495 DRM_DEBUG_DRIVER("Failed to set the min frequency\n"); 4496 4497 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox); 4498 if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */ 4499 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n", 4500 (dev_priv->rps.max_freq_softlimit & 0xff) * 50, 4501 (pcu_mbox & 0xff) * 50); 4502 dev_priv->rps.max_freq = pcu_mbox & 0xff; 4503 } 4504 4505 dev_priv->rps.power = HIGH_POWER; /* force a reset */ 4506 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 4507 4508 rc6vids = 0; 4509 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids); 4510 if (IS_GEN6(dev) && ret) { 4511 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n"); 4512 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { 4513 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", 4514 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); 4515 rc6vids &= 0xffff00; 4516 rc6vids |= GEN6_ENCODE_RC6_VID(450); 4517 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); 4518 if (ret) 4519 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n"); 4520 } 4521 4522 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 4523} 4524 4525static void __gen6_update_ring_freq(struct drm_device *dev) 4526{ 4527 struct drm_i915_private *dev_priv = dev->dev_private; 4528 int min_freq = 15; 4529 unsigned int gpu_freq; 4530 unsigned int max_ia_freq, min_ring_freq; 4531 int scaling_factor = 180; 4532 struct cpufreq_policy *policy; 4533 4534 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 4535 4536 policy = cpufreq_cpu_get(0); 4537 if (policy) { 4538 max_ia_freq = policy->cpuinfo.max_freq; 4539 cpufreq_cpu_put(policy); 4540 } else { 4541 /* 4542 * Default to measured freq if none found, PCU will ensure we 4543 * don't go over 4544 */ 4545 max_ia_freq = tsc_khz; 4546 } 4547 4548 /* Convert from kHz to MHz */ 4549 max_ia_freq /= 1000; 4550 4551 min_ring_freq = I915_READ(DCLK) & 0xf; 4552 /* convert DDR frequency from units of 266.6MHz to bandwidth */ 4553 min_ring_freq = mult_frac(min_ring_freq, 8, 3); 4554 4555 /* 4556 * For each potential GPU frequency, load a ring frequency we'd like 4557 * to use for memory access. We do this by specifying the IA frequency 4558 * the PCU should use as a reference to determine the ring frequency. 4559 */ 4560 for (gpu_freq = dev_priv->rps.max_freq; gpu_freq >= dev_priv->rps.min_freq; 4561 gpu_freq--) { 4562 int diff = dev_priv->rps.max_freq - gpu_freq; 4563 unsigned int ia_freq = 0, ring_freq = 0; 4564 4565 if (INTEL_INFO(dev)->gen >= 8) { 4566 /* max(2 * GT, DDR). NB: GT is 50MHz units */ 4567 ring_freq = max(min_ring_freq, gpu_freq); 4568 } else if (IS_HASWELL(dev)) { 4569 ring_freq = mult_frac(gpu_freq, 5, 4); 4570 ring_freq = max(min_ring_freq, ring_freq); 4571 /* leave ia_freq as the default, chosen by cpufreq */ 4572 } else { 4573 /* On older processors, there is no separate ring 4574 * clock domain, so in order to boost the bandwidth 4575 * of the ring, we need to upclock the CPU (ia_freq). 4576 * 4577 * For GPU frequencies less than 750MHz, 4578 * just use the lowest ring freq. 4579 */ 4580 if (gpu_freq < min_freq) 4581 ia_freq = 800; 4582 else 4583 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2); 4584 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100); 4585 } 4586 4587 sandybridge_pcode_write(dev_priv, 4588 GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 4589 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT | 4590 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT | 4591 gpu_freq); 4592 } 4593} 4594 4595void gen6_update_ring_freq(struct drm_device *dev) 4596{ 4597 struct drm_i915_private *dev_priv = dev->dev_private; 4598 4599 if (INTEL_INFO(dev)->gen < 6 || IS_VALLEYVIEW(dev)) 4600 return; 4601 4602 mutex_lock(&dev_priv->rps.hw_lock); 4603 __gen6_update_ring_freq(dev); 4604 mutex_unlock(&dev_priv->rps.hw_lock); 4605} 4606 4607static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) 4608{ 4609 struct drm_device *dev = dev_priv->dev; 4610 u32 val, rp0; 4611 4612 if (dev->pdev->revision >= 0x20) { 4613 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); 4614 4615 switch (INTEL_INFO(dev)->eu_total) { 4616 case 8: 4617 /* (2 * 4) config */ 4618 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT); 4619 break; 4620 case 12: 4621 /* (2 * 6) config */ 4622 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT); 4623 break; 4624 case 16: 4625 /* (2 * 8) config */ 4626 default: 4627 /* Setting (2 * 8) Min RP0 for any other combination */ 4628 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT); 4629 break; 4630 } 4631 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK); 4632 } else { 4633 /* For pre-production hardware */ 4634 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG); 4635 rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & 4636 PUNIT_GPU_STATUS_MAX_FREQ_MASK; 4637 } 4638 return rp0; 4639} 4640 4641static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv) 4642{ 4643 u32 val, rpe; 4644 4645 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG); 4646 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 4647 4648 return rpe; 4649} 4650 4651static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv) 4652{ 4653 struct drm_device *dev = dev_priv->dev; 4654 u32 val, rp1; 4655 4656 if (dev->pdev->revision >= 0x20) { 4657 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); 4658 rp1 = (val & FB_GFX_FREQ_FUSE_MASK); 4659 } else { 4660 /* For pre-production hardware */ 4661 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); 4662 rp1 = ((val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & 4663 PUNIT_GPU_STATUS_MAX_FREQ_MASK); 4664 } 4665 return rp1; 4666} 4667 4668static int cherryview_rps_min_freq(struct drm_i915_private *dev_priv) 4669{ 4670 struct drm_device *dev = dev_priv->dev; 4671 u32 val, rpn; 4672 4673 if (dev->pdev->revision >= 0x20) { 4674 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE); 4675 rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) & 4676 FB_GFX_FREQ_FUSE_MASK); 4677 } else { /* For pre-production hardware */ 4678 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG); 4679 rpn = ((val >> PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT) & 4680 PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK); 4681 } 4682 4683 return rpn; 4684} 4685 4686static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv) 4687{ 4688 u32 val, rp1; 4689 4690 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); 4691 4692 rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 4693 4694 return rp1; 4695} 4696 4697static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv) 4698{ 4699 u32 val, rp0; 4700 4701 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); 4702 4703 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 4704 /* Clamp to max */ 4705 rp0 = min_t(u32, rp0, 0xea); 4706 4707 return rp0; 4708} 4709 4710static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv) 4711{ 4712 u32 val, rpe; 4713 4714 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 4715 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 4716 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 4717 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 4718 4719 return rpe; 4720} 4721 4722static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv) 4723{ 4724 return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff; 4725} 4726 4727/* Check that the pctx buffer wasn't move under us. */ 4728static void valleyview_check_pctx(struct drm_i915_private *dev_priv) 4729{ 4730 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095; 4731 4732 WARN_ON(pctx_addr != dev_priv->mm.stolen_base + 4733 dev_priv->vlv_pctx->stolen->start); 4734} 4735 4736 4737/* Check that the pcbr address is not empty. */ 4738static void cherryview_check_pctx(struct drm_i915_private *dev_priv) 4739{ 4740 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095; 4741 4742 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0); 4743} 4744 4745static void cherryview_setup_pctx(struct drm_device *dev) 4746{ 4747 struct drm_i915_private *dev_priv = dev->dev_private; 4748 unsigned long pctx_paddr, paddr; 4749 struct i915_gtt *gtt = &dev_priv->gtt; 4750 u32 pcbr; 4751 int pctx_size = 32*1024; 4752 4753 WARN_ON(!mutex_is_locked(&dev->struct_mutex)); 4754 4755 pcbr = I915_READ(VLV_PCBR); 4756 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) { 4757 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n"); 4758 paddr = (dev_priv->mm.stolen_base + 4759 (gtt->stolen_size - pctx_size)); 4760 4761 pctx_paddr = (paddr & (~4095)); 4762 I915_WRITE(VLV_PCBR, pctx_paddr); 4763 } 4764 4765 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR)); 4766} 4767 4768static void valleyview_setup_pctx(struct drm_device *dev) 4769{ 4770 struct drm_i915_private *dev_priv = dev->dev_private; 4771 struct drm_i915_gem_object *pctx; 4772 unsigned long pctx_paddr; 4773 u32 pcbr; 4774 int pctx_size = 24*1024; 4775 4776 WARN_ON(!mutex_is_locked(&dev->struct_mutex)); 4777 4778 pcbr = I915_READ(VLV_PCBR); 4779 if (pcbr) { 4780 /* BIOS set it up already, grab the pre-alloc'd space */ 4781 int pcbr_offset; 4782 4783 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base; 4784 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev, 4785 pcbr_offset, 4786 I915_GTT_OFFSET_NONE, 4787 pctx_size); 4788 goto out; 4789 } 4790 4791 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n"); 4792 4793 /* 4794 * From the Gunit register HAS: 4795 * The Gfx driver is expected to program this register and ensure 4796 * proper allocation within Gfx stolen memory. For example, this 4797 * register should be programmed such than the PCBR range does not 4798 * overlap with other ranges, such as the frame buffer, protected 4799 * memory, or any other relevant ranges. 4800 */ 4801 pctx = i915_gem_object_create_stolen(dev, pctx_size); 4802 if (!pctx) { 4803 DRM_DEBUG("not enough stolen space for PCTX, disabling\n"); 4804 return; 4805 } 4806 4807 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start; 4808 I915_WRITE(VLV_PCBR, pctx_paddr); 4809 4810out: 4811 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR)); 4812 dev_priv->vlv_pctx = pctx; 4813} 4814 4815static void valleyview_cleanup_pctx(struct drm_device *dev) 4816{ 4817 struct drm_i915_private *dev_priv = dev->dev_private; 4818 4819 if (WARN_ON(!dev_priv->vlv_pctx)) 4820 return; 4821 4822 drm_gem_object_unreference(&dev_priv->vlv_pctx->base); 4823 dev_priv->vlv_pctx = NULL; 4824} 4825 4826static void valleyview_init_gt_powersave(struct drm_device *dev) 4827{ 4828 struct drm_i915_private *dev_priv = dev->dev_private; 4829 u32 val; 4830 4831 valleyview_setup_pctx(dev); 4832 4833 mutex_lock(&dev_priv->rps.hw_lock); 4834 4835 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); 4836 switch ((val >> 6) & 3) { 4837 case 0: 4838 case 1: 4839 dev_priv->mem_freq = 800; 4840 break; 4841 case 2: 4842 dev_priv->mem_freq = 1066; 4843 break; 4844 case 3: 4845 dev_priv->mem_freq = 1333; 4846 break; 4847 } 4848 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); 4849 4850 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv); 4851 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; 4852 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", 4853 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq), 4854 dev_priv->rps.max_freq); 4855 4856 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv); 4857 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", 4858 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), 4859 dev_priv->rps.efficient_freq); 4860 4861 dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv); 4862 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 4863 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq), 4864 dev_priv->rps.rp1_freq); 4865 4866 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv); 4867 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", 4868 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq), 4869 dev_priv->rps.min_freq); 4870 4871 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; 4872 4873 /* Preserve min/max settings in case of re-init */ 4874 if (dev_priv->rps.max_freq_softlimit == 0) 4875 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; 4876 4877 if (dev_priv->rps.min_freq_softlimit == 0) 4878 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; 4879 4880 mutex_unlock(&dev_priv->rps.hw_lock); 4881} 4882 4883static void cherryview_init_gt_powersave(struct drm_device *dev) 4884{ 4885 struct drm_i915_private *dev_priv = dev->dev_private; 4886 u32 val; 4887 4888 cherryview_setup_pctx(dev); 4889 4890 mutex_lock(&dev_priv->rps.hw_lock); 4891 4892 mutex_lock(&dev_priv->dpio_lock); 4893 val = vlv_cck_read(dev_priv, CCK_FUSE_REG); 4894 mutex_unlock(&dev_priv->dpio_lock); 4895 4896 switch ((val >> 2) & 0x7) { 4897 case 0: 4898 case 1: 4899 dev_priv->rps.cz_freq = 200; 4900 dev_priv->mem_freq = 1600; 4901 break; 4902 case 2: 4903 dev_priv->rps.cz_freq = 267; 4904 dev_priv->mem_freq = 1600; 4905 break; 4906 case 3: 4907 dev_priv->rps.cz_freq = 333; 4908 dev_priv->mem_freq = 2000; 4909 break; 4910 case 4: 4911 dev_priv->rps.cz_freq = 320; 4912 dev_priv->mem_freq = 1600; 4913 break; 4914 case 5: 4915 dev_priv->rps.cz_freq = 400; 4916 dev_priv->mem_freq = 1600; 4917 break; 4918 } 4919 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); 4920 4921 dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv); 4922 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; 4923 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", 4924 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq), 4925 dev_priv->rps.max_freq); 4926 4927 dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv); 4928 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", 4929 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), 4930 dev_priv->rps.efficient_freq); 4931 4932 dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv); 4933 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n", 4934 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq), 4935 dev_priv->rps.rp1_freq); 4936 4937 dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv); 4938 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", 4939 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq), 4940 dev_priv->rps.min_freq); 4941 4942 WARN_ONCE((dev_priv->rps.max_freq | 4943 dev_priv->rps.efficient_freq | 4944 dev_priv->rps.rp1_freq | 4945 dev_priv->rps.min_freq) & 1, 4946 "Odd GPU freq values\n"); 4947 4948 dev_priv->rps.idle_freq = dev_priv->rps.min_freq; 4949 4950 /* Preserve min/max settings in case of re-init */ 4951 if (dev_priv->rps.max_freq_softlimit == 0) 4952 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; 4953 4954 if (dev_priv->rps.min_freq_softlimit == 0) 4955 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; 4956 4957 mutex_unlock(&dev_priv->rps.hw_lock); 4958} 4959 4960static void valleyview_cleanup_gt_powersave(struct drm_device *dev) 4961{ 4962 valleyview_cleanup_pctx(dev); 4963} 4964 4965static void cherryview_enable_rps(struct drm_device *dev) 4966{ 4967 struct drm_i915_private *dev_priv = dev->dev_private; 4968 struct intel_engine_cs *ring; 4969 u32 gtfifodbg, val, rc6_mode = 0, pcbr; 4970 int i; 4971 4972 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 4973 4974 gtfifodbg = I915_READ(GTFIFODBG); 4975 if (gtfifodbg) { 4976 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n", 4977 gtfifodbg); 4978 I915_WRITE(GTFIFODBG, gtfifodbg); 4979 } 4980 4981 cherryview_check_pctx(dev_priv); 4982 4983 /* 1a & 1b: Get forcewake during program sequence. Although the driver 4984 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/ 4985 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 4986 4987 /* Disable RC states. */ 4988 I915_WRITE(GEN6_RC_CONTROL, 0); 4989 4990 /* 2a: Program RC6 thresholds.*/ 4991 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); 4992 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 4993 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 4994 4995 for_each_ring(ring, dev_priv, i) 4996 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 4997 I915_WRITE(GEN6_RC_SLEEP, 0); 4998 4999 /* TO threshold set to 1750 us ( 0x557 * 1.28 us) */ 5000 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557); 5001 5002 /* allows RC6 residency counter to work */ 5003 I915_WRITE(VLV_COUNTER_CONTROL, 5004 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | 5005 VLV_MEDIA_RC6_COUNT_EN | 5006 VLV_RENDER_RC6_COUNT_EN)); 5007 5008 /* For now we assume BIOS is allocating and populating the PCBR */ 5009 pcbr = I915_READ(VLV_PCBR); 5010 5011 /* 3: Enable RC6 */ 5012 if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) && 5013 (pcbr >> VLV_PCBR_ADDR_SHIFT)) 5014 rc6_mode = GEN7_RC_CTL_TO_MODE; 5015 5016 I915_WRITE(GEN6_RC_CONTROL, rc6_mode); 5017 5018 /* 4 Program defaults and thresholds for RPS*/ 5019 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000); 5020 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); 5021 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); 5022 I915_WRITE(GEN6_RP_UP_EI, 66000); 5023 I915_WRITE(GEN6_RP_DOWN_EI, 350000); 5024 5025 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); 5026 5027 /* 5: Enable RPS */ 5028 I915_WRITE(GEN6_RP_CONTROL, 5029 GEN6_RP_MEDIA_HW_NORMAL_MODE | 5030 GEN6_RP_MEDIA_IS_GFX | 5031 GEN6_RP_ENABLE | 5032 GEN6_RP_UP_BUSY_AVG | 5033 GEN6_RP_DOWN_IDLE_AVG); 5034 5035 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); 5036 5037 /* RPS code assumes GPLL is used */ 5038 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); 5039 5040 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no"); 5041 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); 5042 5043 dev_priv->rps.cur_freq = (val >> 8) & 0xff; 5044 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n", 5045 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq), 5046 dev_priv->rps.cur_freq); 5047 5048 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", 5049 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), 5050 dev_priv->rps.efficient_freq); 5051 5052 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); 5053 5054 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5055} 5056 5057static void valleyview_enable_rps(struct drm_device *dev) 5058{ 5059 struct drm_i915_private *dev_priv = dev->dev_private; 5060 struct intel_engine_cs *ring; 5061 u32 gtfifodbg, val, rc6_mode = 0; 5062 int i; 5063 5064 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 5065 5066 valleyview_check_pctx(dev_priv); 5067 5068 if ((gtfifodbg = I915_READ(GTFIFODBG))) { 5069 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n", 5070 gtfifodbg); 5071 I915_WRITE(GTFIFODBG, gtfifodbg); 5072 } 5073 5074 /* If VLV, Forcewake all wells, else re-direct to regular path */ 5075 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5076 5077 /* Disable RC states. */ 5078 I915_WRITE(GEN6_RC_CONTROL, 0); 5079 5080 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000); 5081 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); 5082 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); 5083 I915_WRITE(GEN6_RP_UP_EI, 66000); 5084 I915_WRITE(GEN6_RP_DOWN_EI, 350000); 5085 5086 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); 5087 5088 I915_WRITE(GEN6_RP_CONTROL, 5089 GEN6_RP_MEDIA_TURBO | 5090 GEN6_RP_MEDIA_HW_NORMAL_MODE | 5091 GEN6_RP_MEDIA_IS_GFX | 5092 GEN6_RP_ENABLE | 5093 GEN6_RP_UP_BUSY_AVG | 5094 GEN6_RP_DOWN_IDLE_CONT); 5095 5096 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000); 5097 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); 5098 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); 5099 5100 for_each_ring(ring, dev_priv, i) 5101 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 5102 5103 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557); 5104 5105 /* allows RC6 residency counter to work */ 5106 I915_WRITE(VLV_COUNTER_CONTROL, 5107 _MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN | 5108 VLV_RENDER_RC0_COUNT_EN | 5109 VLV_MEDIA_RC6_COUNT_EN | 5110 VLV_RENDER_RC6_COUNT_EN)); 5111 5112 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) 5113 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL; 5114 5115 intel_print_rc6_info(dev, rc6_mode); 5116 5117 I915_WRITE(GEN6_RC_CONTROL, rc6_mode); 5118 5119 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); 5120 5121 /* RPS code assumes GPLL is used */ 5122 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); 5123 5124 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no"); 5125 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); 5126 5127 dev_priv->rps.cur_freq = (val >> 8) & 0xff; 5128 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n", 5129 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq), 5130 dev_priv->rps.cur_freq); 5131 5132 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", 5133 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), 5134 dev_priv->rps.efficient_freq); 5135 5136 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); 5137 5138 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5139} 5140 5141static unsigned long intel_pxfreq(u32 vidfreq) 5142{ 5143 unsigned long freq; 5144 int div = (vidfreq & 0x3f0000) >> 16; 5145 int post = (vidfreq & 0x3000) >> 12; 5146 int pre = (vidfreq & 0x7); 5147 5148 if (!pre) 5149 return 0; 5150 5151 freq = ((div * 133333) / ((1<<post) * pre)); 5152 5153 return freq; 5154} 5155 5156static const struct cparams { 5157 u16 i; 5158 u16 t; 5159 u16 m; 5160 u16 c; 5161} cparams[] = { 5162 { 1, 1333, 301, 28664 }, 5163 { 1, 1066, 294, 24460 }, 5164 { 1, 800, 294, 25192 }, 5165 { 0, 1333, 276, 27605 }, 5166 { 0, 1066, 276, 27605 }, 5167 { 0, 800, 231, 23784 }, 5168}; 5169 5170static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) 5171{ 5172 u64 total_count, diff, ret; 5173 u32 count1, count2, count3, m = 0, c = 0; 5174 unsigned long now = jiffies_to_msecs(jiffies), diff1; 5175 int i; 5176 5177 assert_spin_locked(&mchdev_lock); 5178 5179 diff1 = now - dev_priv->ips.last_time1; 5180 5181 /* Prevent division-by-zero if we are asking too fast. 5182 * Also, we don't get interesting results if we are polling 5183 * faster than once in 10ms, so just return the saved value 5184 * in such cases. 5185 */ 5186 if (diff1 <= 10) 5187 return dev_priv->ips.chipset_power; 5188 5189 count1 = I915_READ(DMIEC); 5190 count2 = I915_READ(DDREC); 5191 count3 = I915_READ(CSIEC); 5192 5193 total_count = count1 + count2 + count3; 5194 5195 /* FIXME: handle per-counter overflow */ 5196 if (total_count < dev_priv->ips.last_count1) { 5197 diff = ~0UL - dev_priv->ips.last_count1; 5198 diff += total_count; 5199 } else { 5200 diff = total_count - dev_priv->ips.last_count1; 5201 } 5202 5203 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 5204 if (cparams[i].i == dev_priv->ips.c_m && 5205 cparams[i].t == dev_priv->ips.r_t) { 5206 m = cparams[i].m; 5207 c = cparams[i].c; 5208 break; 5209 } 5210 } 5211 5212 diff = div_u64(diff, diff1); 5213 ret = ((m * diff) + c); 5214 ret = div_u64(ret, 10); 5215 5216 dev_priv->ips.last_count1 = total_count; 5217 dev_priv->ips.last_time1 = now; 5218 5219 dev_priv->ips.chipset_power = ret; 5220 5221 return ret; 5222} 5223 5224unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) 5225{ 5226 struct drm_device *dev = dev_priv->dev; 5227 unsigned long val; 5228 5229 if (INTEL_INFO(dev)->gen != 5) 5230 return 0; 5231 5232 spin_lock_irq(&mchdev_lock); 5233 5234 val = __i915_chipset_val(dev_priv); 5235 5236 spin_unlock_irq(&mchdev_lock); 5237 5238 return val; 5239} 5240 5241unsigned long i915_mch_val(struct drm_i915_private *dev_priv) 5242{ 5243 unsigned long m, x, b; 5244 u32 tsfs; 5245 5246 tsfs = I915_READ(TSFS); 5247 5248 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT); 5249 x = I915_READ8(TR1); 5250 5251 b = tsfs & TSFS_INTR_MASK; 5252 5253 return ((m * x) / 127) - b; 5254} 5255 5256static int _pxvid_to_vd(u8 pxvid) 5257{ 5258 if (pxvid == 0) 5259 return 0; 5260 5261 if (pxvid >= 8 && pxvid < 31) 5262 pxvid = 31; 5263 5264 return (pxvid + 2) * 125; 5265} 5266 5267static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) 5268{ 5269 struct drm_device *dev = dev_priv->dev; 5270 const int vd = _pxvid_to_vd(pxvid); 5271 const int vm = vd - 1125; 5272 5273 if (INTEL_INFO(dev)->is_mobile) 5274 return vm > 0 ? vm : 0; 5275 5276 return vd; 5277} 5278 5279static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) 5280{ 5281 u64 now, diff, diffms; 5282 u32 count; 5283 5284 assert_spin_locked(&mchdev_lock); 5285 5286 now = ktime_get_raw_ns(); 5287 diffms = now - dev_priv->ips.last_time2; 5288 do_div(diffms, NSEC_PER_MSEC); 5289 5290 /* Don't divide by 0 */ 5291 if (!diffms) 5292 return; 5293 5294 count = I915_READ(GFXEC); 5295 5296 if (count < dev_priv->ips.last_count2) { 5297 diff = ~0UL - dev_priv->ips.last_count2; 5298 diff += count; 5299 } else { 5300 diff = count - dev_priv->ips.last_count2; 5301 } 5302 5303 dev_priv->ips.last_count2 = count; 5304 dev_priv->ips.last_time2 = now; 5305 5306 /* More magic constants... */ 5307 diff = diff * 1181; 5308 diff = div_u64(diff, diffms * 10); 5309 dev_priv->ips.gfx_power = diff; 5310} 5311 5312void i915_update_gfx_val(struct drm_i915_private *dev_priv) 5313{ 5314 struct drm_device *dev = dev_priv->dev; 5315 5316 if (INTEL_INFO(dev)->gen != 5) 5317 return; 5318 5319 spin_lock_irq(&mchdev_lock); 5320 5321 __i915_update_gfx_val(dev_priv); 5322 5323 spin_unlock_irq(&mchdev_lock); 5324} 5325 5326static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) 5327{ 5328 unsigned long t, corr, state1, corr2, state2; 5329 u32 pxvid, ext_v; 5330 5331 assert_spin_locked(&mchdev_lock); 5332 5333 pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4)); 5334 pxvid = (pxvid >> 24) & 0x7f; 5335 ext_v = pvid_to_extvid(dev_priv, pxvid); 5336 5337 state1 = ext_v; 5338 5339 t = i915_mch_val(dev_priv); 5340 5341 /* Revel in the empirically derived constants */ 5342 5343 /* Correction factor in 1/100000 units */ 5344 if (t > 80) 5345 corr = ((t * 2349) + 135940); 5346 else if (t >= 50) 5347 corr = ((t * 964) + 29317); 5348 else /* < 50 */ 5349 corr = ((t * 301) + 1004); 5350 5351 corr = corr * ((150142 * state1) / 10000 - 78642); 5352 corr /= 100000; 5353 corr2 = (corr * dev_priv->ips.corr); 5354 5355 state2 = (corr2 * state1) / 10000; 5356 state2 /= 100; /* convert to mW */ 5357 5358 __i915_update_gfx_val(dev_priv); 5359 5360 return dev_priv->ips.gfx_power + state2; 5361} 5362 5363unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) 5364{ 5365 struct drm_device *dev = dev_priv->dev; 5366 unsigned long val; 5367 5368 if (INTEL_INFO(dev)->gen != 5) 5369 return 0; 5370 5371 spin_lock_irq(&mchdev_lock); 5372 5373 val = __i915_gfx_val(dev_priv); 5374 5375 spin_unlock_irq(&mchdev_lock); 5376 5377 return val; 5378} 5379 5380/** 5381 * i915_read_mch_val - return value for IPS use 5382 * 5383 * Calculate and return a value for the IPS driver to use when deciding whether 5384 * we have thermal and power headroom to increase CPU or GPU power budget. 5385 */ 5386unsigned long i915_read_mch_val(void) 5387{ 5388 struct drm_i915_private *dev_priv; 5389 unsigned long chipset_val, graphics_val, ret = 0; 5390 5391 spin_lock_irq(&mchdev_lock); 5392 if (!i915_mch_dev) 5393 goto out_unlock; 5394 dev_priv = i915_mch_dev; 5395 5396 chipset_val = __i915_chipset_val(dev_priv); 5397 graphics_val = __i915_gfx_val(dev_priv); 5398 5399 ret = chipset_val + graphics_val; 5400 5401out_unlock: 5402 spin_unlock_irq(&mchdev_lock); 5403 5404 return ret; 5405} 5406EXPORT_SYMBOL_GPL(i915_read_mch_val); 5407 5408/** 5409 * i915_gpu_raise - raise GPU frequency limit 5410 * 5411 * Raise the limit; IPS indicates we have thermal headroom. 5412 */ 5413bool i915_gpu_raise(void) 5414{ 5415 struct drm_i915_private *dev_priv; 5416 bool ret = true; 5417 5418 spin_lock_irq(&mchdev_lock); 5419 if (!i915_mch_dev) { 5420 ret = false; 5421 goto out_unlock; 5422 } 5423 dev_priv = i915_mch_dev; 5424 5425 if (dev_priv->ips.max_delay > dev_priv->ips.fmax) 5426 dev_priv->ips.max_delay--; 5427 5428out_unlock: 5429 spin_unlock_irq(&mchdev_lock); 5430 5431 return ret; 5432} 5433EXPORT_SYMBOL_GPL(i915_gpu_raise); 5434 5435/** 5436 * i915_gpu_lower - lower GPU frequency limit 5437 * 5438 * IPS indicates we're close to a thermal limit, so throttle back the GPU 5439 * frequency maximum. 5440 */ 5441bool i915_gpu_lower(void) 5442{ 5443 struct drm_i915_private *dev_priv; 5444 bool ret = true; 5445 5446 spin_lock_irq(&mchdev_lock); 5447 if (!i915_mch_dev) { 5448 ret = false; 5449 goto out_unlock; 5450 } 5451 dev_priv = i915_mch_dev; 5452 5453 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay) 5454 dev_priv->ips.max_delay++; 5455 5456out_unlock: 5457 spin_unlock_irq(&mchdev_lock); 5458 5459 return ret; 5460} 5461EXPORT_SYMBOL_GPL(i915_gpu_lower); 5462 5463/** 5464 * i915_gpu_busy - indicate GPU business to IPS 5465 * 5466 * Tell the IPS driver whether or not the GPU is busy. 5467 */ 5468bool i915_gpu_busy(void) 5469{ 5470 struct drm_i915_private *dev_priv; 5471 struct intel_engine_cs *ring; 5472 bool ret = false; 5473 int i; 5474 5475 spin_lock_irq(&mchdev_lock); 5476 if (!i915_mch_dev) 5477 goto out_unlock; 5478 dev_priv = i915_mch_dev; 5479 5480 for_each_ring(ring, dev_priv, i) 5481 ret |= !list_empty(&ring->request_list); 5482 5483out_unlock: 5484 spin_unlock_irq(&mchdev_lock); 5485 5486 return ret; 5487} 5488EXPORT_SYMBOL_GPL(i915_gpu_busy); 5489 5490/** 5491 * i915_gpu_turbo_disable - disable graphics turbo 5492 * 5493 * Disable graphics turbo by resetting the max frequency and setting the 5494 * current frequency to the default. 5495 */ 5496bool i915_gpu_turbo_disable(void) 5497{ 5498 struct drm_i915_private *dev_priv; 5499 bool ret = true; 5500 5501 spin_lock_irq(&mchdev_lock); 5502 if (!i915_mch_dev) { 5503 ret = false; 5504 goto out_unlock; 5505 } 5506 dev_priv = i915_mch_dev; 5507 5508 dev_priv->ips.max_delay = dev_priv->ips.fstart; 5509 5510 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart)) 5511 ret = false; 5512 5513out_unlock: 5514 spin_unlock_irq(&mchdev_lock); 5515 5516 return ret; 5517} 5518EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 5519 5520/** 5521 * Tells the intel_ips driver that the i915 driver is now loaded, if 5522 * IPS got loaded first. 5523 * 5524 * This awkward dance is so that neither module has to depend on the 5525 * other in order for IPS to do the appropriate communication of 5526 * GPU turbo limits to i915. 5527 */ 5528static void 5529ips_ping_for_i915_load(void) 5530{ 5531 void (*link)(void); 5532 5533 link = symbol_get(ips_link_to_i915_driver); 5534 if (link) { 5535 link(); 5536 symbol_put(ips_link_to_i915_driver); 5537 } 5538} 5539 5540void intel_gpu_ips_init(struct drm_i915_private *dev_priv) 5541{ 5542 /* We only register the i915 ips part with intel-ips once everything is 5543 * set up, to avoid intel-ips sneaking in and reading bogus values. */ 5544 spin_lock_irq(&mchdev_lock); 5545 i915_mch_dev = dev_priv; 5546 spin_unlock_irq(&mchdev_lock); 5547 5548 ips_ping_for_i915_load(); 5549} 5550 5551void intel_gpu_ips_teardown(void) 5552{ 5553 spin_lock_irq(&mchdev_lock); 5554 i915_mch_dev = NULL; 5555 spin_unlock_irq(&mchdev_lock); 5556} 5557 5558static void intel_init_emon(struct drm_device *dev) 5559{ 5560 struct drm_i915_private *dev_priv = dev->dev_private; 5561 u32 lcfuse; 5562 u8 pxw[16]; 5563 int i; 5564 5565 /* Disable to program */ 5566 I915_WRITE(ECR, 0); 5567 POSTING_READ(ECR); 5568 5569 /* Program energy weights for various events */ 5570 I915_WRITE(SDEW, 0x15040d00); 5571 I915_WRITE(CSIEW0, 0x007f0000); 5572 I915_WRITE(CSIEW1, 0x1e220004); 5573 I915_WRITE(CSIEW2, 0x04000004); 5574 5575 for (i = 0; i < 5; i++) 5576 I915_WRITE(PEW + (i * 4), 0); 5577 for (i = 0; i < 3; i++) 5578 I915_WRITE(DEW + (i * 4), 0); 5579 5580 /* Program P-state weights to account for frequency power adjustment */ 5581 for (i = 0; i < 16; i++) { 5582 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4)); 5583 unsigned long freq = intel_pxfreq(pxvidfreq); 5584 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >> 5585 PXVFREQ_PX_SHIFT; 5586 unsigned long val; 5587 5588 val = vid * vid; 5589 val *= (freq / 1000); 5590 val *= 255; 5591 val /= (127*127*900); 5592 if (val > 0xff) 5593 DRM_ERROR("bad pxval: %ld\n", val); 5594 pxw[i] = val; 5595 } 5596 /* Render standby states get 0 weight */ 5597 pxw[14] = 0; 5598 pxw[15] = 0; 5599 5600 for (i = 0; i < 4; i++) { 5601 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) | 5602 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]); 5603 I915_WRITE(PXW + (i * 4), val); 5604 } 5605 5606 /* Adjust magic regs to magic values (more experimental results) */ 5607 I915_WRITE(OGW0, 0); 5608 I915_WRITE(OGW1, 0); 5609 I915_WRITE(EG0, 0x00007f00); 5610 I915_WRITE(EG1, 0x0000000e); 5611 I915_WRITE(EG2, 0x000e0000); 5612 I915_WRITE(EG3, 0x68000300); 5613 I915_WRITE(EG4, 0x42000000); 5614 I915_WRITE(EG5, 0x00140031); 5615 I915_WRITE(EG6, 0); 5616 I915_WRITE(EG7, 0); 5617 5618 for (i = 0; i < 8; i++) 5619 I915_WRITE(PXWL + (i * 4), 0); 5620 5621 /* Enable PMON + select events */ 5622 I915_WRITE(ECR, 0x80000019); 5623 5624 lcfuse = I915_READ(LCFUSE02); 5625 5626 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); 5627} 5628 5629void intel_init_gt_powersave(struct drm_device *dev) 5630{ 5631 i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6); 5632 5633 if (IS_CHERRYVIEW(dev)) 5634 cherryview_init_gt_powersave(dev); 5635 else if (IS_VALLEYVIEW(dev)) 5636 valleyview_init_gt_powersave(dev); 5637} 5638 5639void intel_cleanup_gt_powersave(struct drm_device *dev) 5640{ 5641 if (IS_CHERRYVIEW(dev)) 5642 return; 5643 else if (IS_VALLEYVIEW(dev)) 5644 valleyview_cleanup_gt_powersave(dev); 5645} 5646 5647static void gen6_suspend_rps(struct drm_device *dev) 5648{ 5649 struct drm_i915_private *dev_priv = dev->dev_private; 5650 5651 flush_delayed_work(&dev_priv->rps.delayed_resume_work); 5652 5653 gen6_disable_rps_interrupts(dev); 5654} 5655 5656/** 5657 * intel_suspend_gt_powersave - suspend PM work and helper threads 5658 * @dev: drm device 5659 * 5660 * We don't want to disable RC6 or other features here, we just want 5661 * to make sure any work we've queued has finished and won't bother 5662 * us while we're suspended. 5663 */ 5664void intel_suspend_gt_powersave(struct drm_device *dev) 5665{ 5666 struct drm_i915_private *dev_priv = dev->dev_private; 5667 5668 if (INTEL_INFO(dev)->gen < 6) 5669 return; 5670 5671 gen6_suspend_rps(dev); 5672 5673 /* Force GPU to min freq during suspend */ 5674 gen6_rps_idle(dev_priv); 5675} 5676 5677void intel_disable_gt_powersave(struct drm_device *dev) 5678{ 5679 struct drm_i915_private *dev_priv = dev->dev_private; 5680 5681 if (IS_IRONLAKE_M(dev)) { 5682 ironlake_disable_drps(dev); 5683 } else if (INTEL_INFO(dev)->gen >= 6) { 5684 intel_suspend_gt_powersave(dev); 5685 5686 mutex_lock(&dev_priv->rps.hw_lock); 5687 if (INTEL_INFO(dev)->gen >= 9) 5688 gen9_disable_rps(dev); 5689 else if (IS_CHERRYVIEW(dev)) 5690 cherryview_disable_rps(dev); 5691 else if (IS_VALLEYVIEW(dev)) 5692 valleyview_disable_rps(dev); 5693 else 5694 gen6_disable_rps(dev); 5695 5696 dev_priv->rps.enabled = false; 5697 mutex_unlock(&dev_priv->rps.hw_lock); 5698 } 5699} 5700 5701static void intel_gen6_powersave_work(struct work_struct *work) 5702{ 5703 struct drm_i915_private *dev_priv = 5704 container_of(work, struct drm_i915_private, 5705 rps.delayed_resume_work.work); 5706 struct drm_device *dev = dev_priv->dev; 5707 5708 mutex_lock(&dev_priv->rps.hw_lock); 5709 5710 gen6_reset_rps_interrupts(dev); 5711 5712 if (IS_CHERRYVIEW(dev)) { 5713 cherryview_enable_rps(dev); 5714 } else if (IS_VALLEYVIEW(dev)) { 5715 valleyview_enable_rps(dev); 5716 } else if (INTEL_INFO(dev)->gen >= 9) { 5717 gen9_enable_rc6(dev); 5718 gen9_enable_rps(dev); 5719 __gen6_update_ring_freq(dev); 5720 } else if (IS_BROADWELL(dev)) { 5721 gen8_enable_rps(dev); 5722 __gen6_update_ring_freq(dev); 5723 } else { 5724 gen6_enable_rps(dev); 5725 __gen6_update_ring_freq(dev); 5726 } 5727 5728 WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq); 5729 WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq); 5730 5731 WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq); 5732 WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq); 5733 5734 dev_priv->rps.enabled = true; 5735 5736 gen6_enable_rps_interrupts(dev); 5737 5738 mutex_unlock(&dev_priv->rps.hw_lock); 5739 5740 intel_runtime_pm_put(dev_priv); 5741} 5742 5743void intel_enable_gt_powersave(struct drm_device *dev) 5744{ 5745 struct drm_i915_private *dev_priv = dev->dev_private; 5746 5747 /* Powersaving is controlled by the host when inside a VM */ 5748 if (intel_vgpu_active(dev)) 5749 return; 5750 5751 if (IS_IRONLAKE_M(dev)) { 5752 mutex_lock(&dev->struct_mutex); 5753 ironlake_enable_drps(dev); 5754 intel_init_emon(dev); 5755 mutex_unlock(&dev->struct_mutex); 5756 } else if (INTEL_INFO(dev)->gen >= 6) { 5757 /* 5758 * PCU communication is slow and this doesn't need to be 5759 * done at any specific time, so do this out of our fast path 5760 * to make resume and init faster. 5761 * 5762 * We depend on the HW RC6 power context save/restore 5763 * mechanism when entering D3 through runtime PM suspend. So 5764 * disable RPM until RPS/RC6 is properly setup. We can only 5765 * get here via the driver load/system resume/runtime resume 5766 * paths, so the _noresume version is enough (and in case of 5767 * runtime resume it's necessary). 5768 */ 5769 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work, 5770 round_jiffies_up_relative(HZ))) 5771 intel_runtime_pm_get_noresume(dev_priv); 5772 } 5773} 5774 5775void intel_reset_gt_powersave(struct drm_device *dev) 5776{ 5777 struct drm_i915_private *dev_priv = dev->dev_private; 5778 5779 if (INTEL_INFO(dev)->gen < 6) 5780 return; 5781 5782 gen6_suspend_rps(dev); 5783 dev_priv->rps.enabled = false; 5784} 5785 5786static void ibx_init_clock_gating(struct drm_device *dev) 5787{ 5788 struct drm_i915_private *dev_priv = dev->dev_private; 5789 5790 /* 5791 * On Ibex Peak and Cougar Point, we need to disable clock 5792 * gating for the panel power sequencer or it will fail to 5793 * start up when no ports are active. 5794 */ 5795 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); 5796} 5797 5798static void g4x_disable_trickle_feed(struct drm_device *dev) 5799{ 5800 struct drm_i915_private *dev_priv = dev->dev_private; 5801 int pipe; 5802 5803 for_each_pipe(dev_priv, pipe) { 5804 I915_WRITE(DSPCNTR(pipe), 5805 I915_READ(DSPCNTR(pipe)) | 5806 DISPPLANE_TRICKLE_FEED_DISABLE); 5807 intel_flush_primary_plane(dev_priv, pipe); 5808 } 5809} 5810 5811static void ilk_init_lp_watermarks(struct drm_device *dev) 5812{ 5813 struct drm_i915_private *dev_priv = dev->dev_private; 5814 5815 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); 5816 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); 5817 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN); 5818 5819 /* 5820 * Don't touch WM1S_LP_EN here. 5821 * Doing so could cause underruns. 5822 */ 5823} 5824 5825static void ironlake_init_clock_gating(struct drm_device *dev) 5826{ 5827 struct drm_i915_private *dev_priv = dev->dev_private; 5828 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; 5829 5830 /* 5831 * Required for FBC 5832 * WaFbcDisableDpfcClockGating:ilk 5833 */ 5834 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE | 5835 ILK_DPFCUNIT_CLOCK_GATE_DISABLE | 5836 ILK_DPFDUNIT_CLOCK_GATE_ENABLE; 5837 5838 I915_WRITE(PCH_3DCGDIS0, 5839 MARIUNIT_CLOCK_GATE_DISABLE | 5840 SVSMUNIT_CLOCK_GATE_DISABLE); 5841 I915_WRITE(PCH_3DCGDIS1, 5842 VFMUNIT_CLOCK_GATE_DISABLE); 5843 5844 /* 5845 * According to the spec the following bits should be set in 5846 * order to enable memory self-refresh 5847 * The bit 22/21 of 0x42004 5848 * The bit 5 of 0x42020 5849 * The bit 15 of 0x45000 5850 */ 5851 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5852 (I915_READ(ILK_DISPLAY_CHICKEN2) | 5853 ILK_DPARB_GATE | ILK_VSDPFD_FULL)); 5854 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE; 5855 I915_WRITE(DISP_ARB_CTL, 5856 (I915_READ(DISP_ARB_CTL) | 5857 DISP_FBC_WM_DIS)); 5858 5859 ilk_init_lp_watermarks(dev); 5860 5861 /* 5862 * Based on the document from hardware guys the following bits 5863 * should be set unconditionally in order to enable FBC. 5864 * The bit 22 of 0x42000 5865 * The bit 22 of 0x42004 5866 * The bit 7,8,9 of 0x42020. 5867 */ 5868 if (IS_IRONLAKE_M(dev)) { 5869 /* WaFbcAsynchFlipDisableFbcQueue:ilk */ 5870 I915_WRITE(ILK_DISPLAY_CHICKEN1, 5871 I915_READ(ILK_DISPLAY_CHICKEN1) | 5872 ILK_FBCQ_DIS); 5873 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5874 I915_READ(ILK_DISPLAY_CHICKEN2) | 5875 ILK_DPARB_GATE); 5876 } 5877 5878 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); 5879 5880 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5881 I915_READ(ILK_DISPLAY_CHICKEN2) | 5882 ILK_ELPIN_409_SELECT); 5883 I915_WRITE(_3D_CHICKEN2, 5884 _3D_CHICKEN2_WM_READ_PIPELINED << 16 | 5885 _3D_CHICKEN2_WM_READ_PIPELINED); 5886 5887 /* WaDisableRenderCachePipelinedFlush:ilk */ 5888 I915_WRITE(CACHE_MODE_0, 5889 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); 5890 5891 /* WaDisable_RenderCache_OperationalFlush:ilk */ 5892 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5893 5894 g4x_disable_trickle_feed(dev); 5895 5896 ibx_init_clock_gating(dev); 5897} 5898 5899static void cpt_init_clock_gating(struct drm_device *dev) 5900{ 5901 struct drm_i915_private *dev_priv = dev->dev_private; 5902 int pipe; 5903 uint32_t val; 5904 5905 /* 5906 * On Ibex Peak and Cougar Point, we need to disable clock 5907 * gating for the panel power sequencer or it will fail to 5908 * start up when no ports are active. 5909 */ 5910 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | 5911 PCH_DPLUNIT_CLOCK_GATE_DISABLE | 5912 PCH_CPUNIT_CLOCK_GATE_DISABLE); 5913 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) | 5914 DPLS_EDP_PPS_FIX_DIS); 5915 /* The below fixes the weird display corruption, a few pixels shifted 5916 * downward, on (only) LVDS of some HP laptops with IVY. 5917 */ 5918 for_each_pipe(dev_priv, pipe) { 5919 val = I915_READ(TRANS_CHICKEN2(pipe)); 5920 val |= TRANS_CHICKEN2_TIMING_OVERRIDE; 5921 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; 5922 if (dev_priv->vbt.fdi_rx_polarity_inverted) 5923 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; 5924 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK; 5925 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; 5926 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; 5927 I915_WRITE(TRANS_CHICKEN2(pipe), val); 5928 } 5929 /* WADP0ClockGatingDisable */ 5930 for_each_pipe(dev_priv, pipe) { 5931 I915_WRITE(TRANS_CHICKEN1(pipe), 5932 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); 5933 } 5934} 5935 5936static void gen6_check_mch_setup(struct drm_device *dev) 5937{ 5938 struct drm_i915_private *dev_priv = dev->dev_private; 5939 uint32_t tmp; 5940 5941 tmp = I915_READ(MCH_SSKPD); 5942 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) 5943 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", 5944 tmp); 5945} 5946 5947static void gen6_init_clock_gating(struct drm_device *dev) 5948{ 5949 struct drm_i915_private *dev_priv = dev->dev_private; 5950 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; 5951 5952 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); 5953 5954 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5955 I915_READ(ILK_DISPLAY_CHICKEN2) | 5956 ILK_ELPIN_409_SELECT); 5957 5958 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ 5959 I915_WRITE(_3D_CHICKEN, 5960 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB)); 5961 5962 /* WaDisable_RenderCache_OperationalFlush:snb */ 5963 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5964 5965 /* 5966 * BSpec recoomends 8x4 when MSAA is used, 5967 * however in practice 16x4 seems fastest. 5968 * 5969 * Note that PS/WM thread counts depend on the WIZ hashing 5970 * disable bit, which we don't touch here, but it's good 5971 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 5972 */ 5973 I915_WRITE(GEN6_GT_MODE, 5974 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); 5975 5976 ilk_init_lp_watermarks(dev); 5977 5978 I915_WRITE(CACHE_MODE_0, 5979 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); 5980 5981 I915_WRITE(GEN6_UCGCTL1, 5982 I915_READ(GEN6_UCGCTL1) | 5983 GEN6_BLBUNIT_CLOCK_GATE_DISABLE | 5984 GEN6_CSUNIT_CLOCK_GATE_DISABLE); 5985 5986 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock 5987 * gating disable must be set. Failure to set it results in 5988 * flickering pixels due to Z write ordering failures after 5989 * some amount of runtime in the Mesa "fire" demo, and Unigine 5990 * Sanctuary and Tropics, and apparently anything else with 5991 * alpha test or pixel discard. 5992 * 5993 * According to the spec, bit 11 (RCCUNIT) must also be set, 5994 * but we didn't debug actual testcases to find it out. 5995 * 5996 * WaDisableRCCUnitClockGating:snb 5997 * WaDisableRCPBUnitClockGating:snb 5998 */ 5999 I915_WRITE(GEN6_UCGCTL2, 6000 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | 6001 GEN6_RCCUNIT_CLOCK_GATE_DISABLE); 6002 6003 /* WaStripsFansDisableFastClipPerformanceFix:snb */ 6004 I915_WRITE(_3D_CHICKEN3, 6005 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); 6006 6007 /* 6008 * Bspec says: 6009 * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and 6010 * 3DSTATE_SF number of SF output attributes is more than 16." 6011 */ 6012 I915_WRITE(_3D_CHICKEN3, 6013 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); 6014 6015 /* 6016 * According to the spec the following bits should be 6017 * set in order to enable memory self-refresh and fbc: 6018 * The bit21 and bit22 of 0x42000 6019 * The bit21 and bit22 of 0x42004 6020 * The bit5 and bit7 of 0x42020 6021 * The bit14 of 0x70180 6022 * The bit14 of 0x71180 6023 * 6024 * WaFbcAsynchFlipDisableFbcQueue:snb 6025 */ 6026 I915_WRITE(ILK_DISPLAY_CHICKEN1, 6027 I915_READ(ILK_DISPLAY_CHICKEN1) | 6028 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); 6029 I915_WRITE(ILK_DISPLAY_CHICKEN2, 6030 I915_READ(ILK_DISPLAY_CHICKEN2) | 6031 ILK_DPARB_GATE | ILK_VSDPFD_FULL); 6032 I915_WRITE(ILK_DSPCLK_GATE_D, 6033 I915_READ(ILK_DSPCLK_GATE_D) | 6034 ILK_DPARBUNIT_CLOCK_GATE_ENABLE | 6035 ILK_DPFDUNIT_CLOCK_GATE_ENABLE); 6036 6037 g4x_disable_trickle_feed(dev); 6038 6039 cpt_init_clock_gating(dev); 6040 6041 gen6_check_mch_setup(dev); 6042} 6043 6044static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) 6045{ 6046 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE); 6047 6048 /* 6049 * WaVSThreadDispatchOverride:ivb,vlv 6050 * 6051 * This actually overrides the dispatch 6052 * mode for all thread types. 6053 */ 6054 reg &= ~GEN7_FF_SCHED_MASK; 6055 reg |= GEN7_FF_TS_SCHED_HW; 6056 reg |= GEN7_FF_VS_SCHED_HW; 6057 reg |= GEN7_FF_DS_SCHED_HW; 6058 6059 I915_WRITE(GEN7_FF_THREAD_MODE, reg); 6060} 6061 6062static void lpt_init_clock_gating(struct drm_device *dev) 6063{ 6064 struct drm_i915_private *dev_priv = dev->dev_private; 6065 6066 /* 6067 * TODO: this bit should only be enabled when really needed, then 6068 * disabled when not needed anymore in order to save power. 6069 */ 6070 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) 6071 I915_WRITE(SOUTH_DSPCLK_GATE_D, 6072 I915_READ(SOUTH_DSPCLK_GATE_D) | 6073 PCH_LP_PARTITION_LEVEL_DISABLE); 6074 6075 /* WADPOClockGatingDisable:hsw */ 6076 I915_WRITE(_TRANSA_CHICKEN1, 6077 I915_READ(_TRANSA_CHICKEN1) | 6078 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); 6079} 6080 6081static void lpt_suspend_hw(struct drm_device *dev) 6082{ 6083 struct drm_i915_private *dev_priv = dev->dev_private; 6084 6085 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { 6086 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D); 6087 6088 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; 6089 I915_WRITE(SOUTH_DSPCLK_GATE_D, val); 6090 } 6091} 6092 6093static void broadwell_init_clock_gating(struct drm_device *dev) 6094{ 6095 struct drm_i915_private *dev_priv = dev->dev_private; 6096 enum pipe pipe; 6097 6098 I915_WRITE(WM3_LP_ILK, 0); 6099 I915_WRITE(WM2_LP_ILK, 0); 6100 I915_WRITE(WM1_LP_ILK, 0); 6101 6102 /* WaSwitchSolVfFArbitrationPriority:bdw */ 6103 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); 6104 6105 /* WaPsrDPAMaskVBlankInSRD:bdw */ 6106 I915_WRITE(CHICKEN_PAR1_1, 6107 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); 6108 6109 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ 6110 for_each_pipe(dev_priv, pipe) { 6111 I915_WRITE(CHICKEN_PIPESL_1(pipe), 6112 I915_READ(CHICKEN_PIPESL_1(pipe)) | 6113 BDW_DPRS_MASK_VBLANK_SRD); 6114 } 6115 6116 /* WaVSRefCountFullforceMissDisable:bdw */ 6117 /* WaDSRefCountFullforceMissDisable:bdw */ 6118 I915_WRITE(GEN7_FF_THREAD_MODE, 6119 I915_READ(GEN7_FF_THREAD_MODE) & 6120 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); 6121 6122 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, 6123 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); 6124 6125 /* WaDisableSDEUnitClockGating:bdw */ 6126 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 6127 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 6128 6129 lpt_init_clock_gating(dev); 6130} 6131 6132static void haswell_init_clock_gating(struct drm_device *dev) 6133{ 6134 struct drm_i915_private *dev_priv = dev->dev_private; 6135 6136 ilk_init_lp_watermarks(dev); 6137 6138 /* L3 caching of data atomics doesn't work -- disable it. */ 6139 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); 6140 I915_WRITE(HSW_ROW_CHICKEN3, 6141 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE)); 6142 6143 /* This is required by WaCatErrorRejectionIssue:hsw */ 6144 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 6145 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 6146 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 6147 6148 /* WaVSRefCountFullforceMissDisable:hsw */ 6149 I915_WRITE(GEN7_FF_THREAD_MODE, 6150 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME); 6151 6152 /* WaDisable_RenderCache_OperationalFlush:hsw */ 6153 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6154 6155 /* enable HiZ Raw Stall Optimization */ 6156 I915_WRITE(CACHE_MODE_0_GEN7, 6157 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); 6158 6159 /* WaDisable4x2SubspanOptimization:hsw */ 6160 I915_WRITE(CACHE_MODE_1, 6161 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 6162 6163 /* 6164 * BSpec recommends 8x4 when MSAA is used, 6165 * however in practice 16x4 seems fastest. 6166 * 6167 * Note that PS/WM thread counts depend on the WIZ hashing 6168 * disable bit, which we don't touch here, but it's good 6169 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 6170 */ 6171 I915_WRITE(GEN7_GT_MODE, 6172 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); 6173 6174 /* WaSampleCChickenBitEnable:hsw */ 6175 I915_WRITE(HALF_SLICE_CHICKEN3, 6176 _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE)); 6177 6178 /* WaSwitchSolVfFArbitrationPriority:hsw */ 6179 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); 6180 6181 /* WaRsPkgCStateDisplayPMReq:hsw */ 6182 I915_WRITE(CHICKEN_PAR1_1, 6183 I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES); 6184 6185 lpt_init_clock_gating(dev); 6186} 6187 6188static void ivybridge_init_clock_gating(struct drm_device *dev) 6189{ 6190 struct drm_i915_private *dev_priv = dev->dev_private; 6191 uint32_t snpcr; 6192 6193 ilk_init_lp_watermarks(dev); 6194 6195 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); 6196 6197 /* WaDisableEarlyCull:ivb */ 6198 I915_WRITE(_3D_CHICKEN3, 6199 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); 6200 6201 /* WaDisableBackToBackFlipFix:ivb */ 6202 I915_WRITE(IVB_CHICKEN3, 6203 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | 6204 CHICKEN3_DGMG_DONE_FIX_DISABLE); 6205 6206 /* WaDisablePSDDualDispatchEnable:ivb */ 6207 if (IS_IVB_GT1(dev)) 6208 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, 6209 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); 6210 6211 /* WaDisable_RenderCache_OperationalFlush:ivb */ 6212 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6213 6214 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ 6215 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, 6216 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); 6217 6218 /* WaApplyL3ControlAndL3ChickenMode:ivb */ 6219 I915_WRITE(GEN7_L3CNTLREG1, 6220 GEN7_WA_FOR_GEN7_L3_CONTROL); 6221 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, 6222 GEN7_WA_L3_CHICKEN_MODE); 6223 if (IS_IVB_GT1(dev)) 6224 I915_WRITE(GEN7_ROW_CHICKEN2, 6225 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 6226 else { 6227 /* must write both registers */ 6228 I915_WRITE(GEN7_ROW_CHICKEN2, 6229 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 6230 I915_WRITE(GEN7_ROW_CHICKEN2_GT2, 6231 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 6232 } 6233 6234 /* WaForceL3Serialization:ivb */ 6235 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & 6236 ~L3SQ_URB_READ_CAM_MATCH_DISABLE); 6237 6238 /* 6239 * According to the spec, bit 13 (RCZUNIT) must be set on IVB. 6240 * This implements the WaDisableRCZUnitClockGating:ivb workaround. 6241 */ 6242 I915_WRITE(GEN6_UCGCTL2, 6243 GEN6_RCZUNIT_CLOCK_GATE_DISABLE); 6244 6245 /* This is required by WaCatErrorRejectionIssue:ivb */ 6246 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 6247 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 6248 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 6249 6250 g4x_disable_trickle_feed(dev); 6251 6252 gen7_setup_fixed_func_scheduler(dev_priv); 6253 6254 if (0) { /* causes HiZ corruption on ivb:gt1 */ 6255 /* enable HiZ Raw Stall Optimization */ 6256 I915_WRITE(CACHE_MODE_0_GEN7, 6257 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); 6258 } 6259 6260 /* WaDisable4x2SubspanOptimization:ivb */ 6261 I915_WRITE(CACHE_MODE_1, 6262 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 6263 6264 /* 6265 * BSpec recommends 8x4 when MSAA is used, 6266 * however in practice 16x4 seems fastest. 6267 * 6268 * Note that PS/WM thread counts depend on the WIZ hashing 6269 * disable bit, which we don't touch here, but it's good 6270 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 6271 */ 6272 I915_WRITE(GEN7_GT_MODE, 6273 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); 6274 6275 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); 6276 snpcr &= ~GEN6_MBC_SNPCR_MASK; 6277 snpcr |= GEN6_MBC_SNPCR_MED; 6278 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); 6279 6280 if (!HAS_PCH_NOP(dev)) 6281 cpt_init_clock_gating(dev); 6282 6283 gen6_check_mch_setup(dev); 6284} 6285 6286static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv) 6287{ 6288 I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); 6289 6290 /* 6291 * Disable trickle feed and enable pnd deadline calculation 6292 */ 6293 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); 6294 I915_WRITE(CBR1_VLV, 0); 6295} 6296 6297static void valleyview_init_clock_gating(struct drm_device *dev) 6298{ 6299 struct drm_i915_private *dev_priv = dev->dev_private; 6300 6301 vlv_init_display_clock_gating(dev_priv); 6302 6303 /* WaDisableEarlyCull:vlv */ 6304 I915_WRITE(_3D_CHICKEN3, 6305 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); 6306 6307 /* WaDisableBackToBackFlipFix:vlv */ 6308 I915_WRITE(IVB_CHICKEN3, 6309 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | 6310 CHICKEN3_DGMG_DONE_FIX_DISABLE); 6311 6312 /* WaPsdDispatchEnable:vlv */ 6313 /* WaDisablePSDDualDispatchEnable:vlv */ 6314 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, 6315 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP | 6316 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); 6317 6318 /* WaDisable_RenderCache_OperationalFlush:vlv */ 6319 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6320 6321 /* WaForceL3Serialization:vlv */ 6322 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & 6323 ~L3SQ_URB_READ_CAM_MATCH_DISABLE); 6324 6325 /* WaDisableDopClockGating:vlv */ 6326 I915_WRITE(GEN7_ROW_CHICKEN2, 6327 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 6328 6329 /* This is required by WaCatErrorRejectionIssue:vlv */ 6330 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 6331 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 6332 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 6333 6334 gen7_setup_fixed_func_scheduler(dev_priv); 6335 6336 /* 6337 * According to the spec, bit 13 (RCZUNIT) must be set on IVB. 6338 * This implements the WaDisableRCZUnitClockGating:vlv workaround. 6339 */ 6340 I915_WRITE(GEN6_UCGCTL2, 6341 GEN6_RCZUNIT_CLOCK_GATE_DISABLE); 6342 6343 /* WaDisableL3Bank2xClockGate:vlv 6344 * Disabling L3 clock gating- MMIO 940c[25] = 1 6345 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */ 6346 I915_WRITE(GEN7_UCGCTL4, 6347 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE); 6348 6349 /* 6350 * BSpec says this must be set, even though 6351 * WaDisable4x2SubspanOptimization isn't listed for VLV. 6352 */ 6353 I915_WRITE(CACHE_MODE_1, 6354 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 6355 6356 /* 6357 * BSpec recommends 8x4 when MSAA is used, 6358 * however in practice 16x4 seems fastest. 6359 * 6360 * Note that PS/WM thread counts depend on the WIZ hashing 6361 * disable bit, which we don't touch here, but it's good 6362 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 6363 */ 6364 I915_WRITE(GEN7_GT_MODE, 6365 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); 6366 6367 /* 6368 * WaIncreaseL3CreditsForVLVB0:vlv 6369 * This is the hardware default actually. 6370 */ 6371 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); 6372 6373 /* 6374 * WaDisableVLVClockGating_VBIIssue:vlv 6375 * Disable clock gating on th GCFG unit to prevent a delay 6376 * in the reporting of vblank events. 6377 */ 6378 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); 6379} 6380 6381static void cherryview_init_clock_gating(struct drm_device *dev) 6382{ 6383 struct drm_i915_private *dev_priv = dev->dev_private; 6384 6385 vlv_init_display_clock_gating(dev_priv); 6386 6387 /* WaVSRefCountFullforceMissDisable:chv */ 6388 /* WaDSRefCountFullforceMissDisable:chv */ 6389 I915_WRITE(GEN7_FF_THREAD_MODE, 6390 I915_READ(GEN7_FF_THREAD_MODE) & 6391 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); 6392 6393 /* WaDisableSemaphoreAndSyncFlipWait:chv */ 6394 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, 6395 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); 6396 6397 /* WaDisableCSUnitClockGating:chv */ 6398 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | 6399 GEN6_CSUNIT_CLOCK_GATE_DISABLE); 6400 6401 /* WaDisableSDEUnitClockGating:chv */ 6402 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 6403 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 6404} 6405 6406static void g4x_init_clock_gating(struct drm_device *dev) 6407{ 6408 struct drm_i915_private *dev_priv = dev->dev_private; 6409 uint32_t dspclk_gate; 6410 6411 I915_WRITE(RENCLK_GATE_D1, 0); 6412 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE | 6413 GS_UNIT_CLOCK_GATE_DISABLE | 6414 CL_UNIT_CLOCK_GATE_DISABLE); 6415 I915_WRITE(RAMCLK_GATE_D, 0); 6416 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE | 6417 OVRUNIT_CLOCK_GATE_DISABLE | 6418 OVCUNIT_CLOCK_GATE_DISABLE; 6419 if (IS_GM45(dev)) 6420 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE; 6421 I915_WRITE(DSPCLK_GATE_D, dspclk_gate); 6422 6423 /* WaDisableRenderCachePipelinedFlush */ 6424 I915_WRITE(CACHE_MODE_0, 6425 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); 6426 6427 /* WaDisable_RenderCache_OperationalFlush:g4x */ 6428 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6429 6430 g4x_disable_trickle_feed(dev); 6431} 6432 6433static void crestline_init_clock_gating(struct drm_device *dev) 6434{ 6435 struct drm_i915_private *dev_priv = dev->dev_private; 6436 6437 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE); 6438 I915_WRITE(RENCLK_GATE_D2, 0); 6439 I915_WRITE(DSPCLK_GATE_D, 0); 6440 I915_WRITE(RAMCLK_GATE_D, 0); 6441 I915_WRITE16(DEUC, 0); 6442 I915_WRITE(MI_ARB_STATE, 6443 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 6444 6445 /* WaDisable_RenderCache_OperationalFlush:gen4 */ 6446 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6447} 6448 6449static void broadwater_init_clock_gating(struct drm_device *dev) 6450{ 6451 struct drm_i915_private *dev_priv = dev->dev_private; 6452 6453 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | 6454 I965_RCC_CLOCK_GATE_DISABLE | 6455 I965_RCPB_CLOCK_GATE_DISABLE | 6456 I965_ISC_CLOCK_GATE_DISABLE | 6457 I965_FBC_CLOCK_GATE_DISABLE); 6458 I915_WRITE(RENCLK_GATE_D2, 0); 6459 I915_WRITE(MI_ARB_STATE, 6460 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 6461 6462 /* WaDisable_RenderCache_OperationalFlush:gen4 */ 6463 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6464} 6465 6466static void gen3_init_clock_gating(struct drm_device *dev) 6467{ 6468 struct drm_i915_private *dev_priv = dev->dev_private; 6469 u32 dstate = I915_READ(D_STATE); 6470 6471 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING | 6472 DSTATE_DOT_CLOCK_GATING; 6473 I915_WRITE(D_STATE, dstate); 6474 6475 if (IS_PINEVIEW(dev)) 6476 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); 6477 6478 /* IIR "flip pending" means done if this bit is set */ 6479 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); 6480 6481 /* interrupts should cause a wake up from C3 */ 6482 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN)); 6483 6484 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 6485 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 6486 6487 I915_WRITE(MI_ARB_STATE, 6488 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 6489} 6490 6491static void i85x_init_clock_gating(struct drm_device *dev) 6492{ 6493 struct drm_i915_private *dev_priv = dev->dev_private; 6494 6495 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); 6496 6497 /* interrupts should cause a wake up from C3 */ 6498 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) | 6499 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE)); 6500 6501 I915_WRITE(MEM_MODE, 6502 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE)); 6503} 6504 6505static void i830_init_clock_gating(struct drm_device *dev) 6506{ 6507 struct drm_i915_private *dev_priv = dev->dev_private; 6508 6509 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); 6510 6511 I915_WRITE(MEM_MODE, 6512 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) | 6513 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE)); 6514} 6515 6516void intel_init_clock_gating(struct drm_device *dev) 6517{ 6518 struct drm_i915_private *dev_priv = dev->dev_private; 6519 6520 if (dev_priv->display.init_clock_gating) 6521 dev_priv->display.init_clock_gating(dev); 6522} 6523 6524void intel_suspend_hw(struct drm_device *dev) 6525{ 6526 if (HAS_PCH_LPT(dev)) 6527 lpt_suspend_hw(dev); 6528} 6529 6530/* Set up chip specific power management-related functions */ 6531void intel_init_pm(struct drm_device *dev) 6532{ 6533 struct drm_i915_private *dev_priv = dev->dev_private; 6534 6535 intel_fbc_init(dev_priv); 6536 6537 /* For cxsr */ 6538 if (IS_PINEVIEW(dev)) 6539 i915_pineview_get_mem_freq(dev); 6540 else if (IS_GEN5(dev)) 6541 i915_ironlake_get_mem_freq(dev); 6542 6543 /* For FIFO watermark updates */ 6544 if (INTEL_INFO(dev)->gen >= 9) { 6545 skl_setup_wm_latency(dev); 6546 6547 dev_priv->display.init_clock_gating = skl_init_clock_gating; 6548 dev_priv->display.update_wm = skl_update_wm; 6549 dev_priv->display.update_sprite_wm = skl_update_sprite_wm; 6550 } else if (HAS_PCH_SPLIT(dev)) { 6551 ilk_setup_wm_latency(dev); 6552 6553 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] && 6554 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || 6555 (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] && 6556 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { 6557 dev_priv->display.update_wm = ilk_update_wm; 6558 dev_priv->display.update_sprite_wm = ilk_update_sprite_wm; 6559 } else { 6560 DRM_DEBUG_KMS("Failed to read display plane latency. " 6561 "Disable CxSR\n"); 6562 } 6563 6564 if (IS_GEN5(dev)) 6565 dev_priv->display.init_clock_gating = ironlake_init_clock_gating; 6566 else if (IS_GEN6(dev)) 6567 dev_priv->display.init_clock_gating = gen6_init_clock_gating; 6568 else if (IS_IVYBRIDGE(dev)) 6569 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating; 6570 else if (IS_HASWELL(dev)) 6571 dev_priv->display.init_clock_gating = haswell_init_clock_gating; 6572 else if (INTEL_INFO(dev)->gen == 8) 6573 dev_priv->display.init_clock_gating = broadwell_init_clock_gating; 6574 } else if (IS_CHERRYVIEW(dev)) { 6575 dev_priv->display.update_wm = valleyview_update_wm; 6576 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; 6577 dev_priv->display.init_clock_gating = 6578 cherryview_init_clock_gating; 6579 } else if (IS_VALLEYVIEW(dev)) { 6580 dev_priv->display.update_wm = valleyview_update_wm; 6581 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; 6582 dev_priv->display.init_clock_gating = 6583 valleyview_init_clock_gating; 6584 } else if (IS_PINEVIEW(dev)) { 6585 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev), 6586 dev_priv->is_ddr3, 6587 dev_priv->fsb_freq, 6588 dev_priv->mem_freq)) { 6589 DRM_INFO("failed to find known CxSR latency " 6590 "(found ddr%s fsb freq %d, mem freq %d), " 6591 "disabling CxSR\n", 6592 (dev_priv->is_ddr3 == 1) ? "3" : "2", 6593 dev_priv->fsb_freq, dev_priv->mem_freq); 6594 /* Disable CxSR and never update its watermark again */ 6595 intel_set_memory_cxsr(dev_priv, false); 6596 dev_priv->display.update_wm = NULL; 6597 } else 6598 dev_priv->display.update_wm = pineview_update_wm; 6599 dev_priv->display.init_clock_gating = gen3_init_clock_gating; 6600 } else if (IS_G4X(dev)) { 6601 dev_priv->display.update_wm = g4x_update_wm; 6602 dev_priv->display.init_clock_gating = g4x_init_clock_gating; 6603 } else if (IS_GEN4(dev)) { 6604 dev_priv->display.update_wm = i965_update_wm; 6605 if (IS_CRESTLINE(dev)) 6606 dev_priv->display.init_clock_gating = crestline_init_clock_gating; 6607 else if (IS_BROADWATER(dev)) 6608 dev_priv->display.init_clock_gating = broadwater_init_clock_gating; 6609 } else if (IS_GEN3(dev)) { 6610 dev_priv->display.update_wm = i9xx_update_wm; 6611 dev_priv->display.get_fifo_size = i9xx_get_fifo_size; 6612 dev_priv->display.init_clock_gating = gen3_init_clock_gating; 6613 } else if (IS_GEN2(dev)) { 6614 if (INTEL_INFO(dev)->num_pipes == 1) { 6615 dev_priv->display.update_wm = i845_update_wm; 6616 dev_priv->display.get_fifo_size = i845_get_fifo_size; 6617 } else { 6618 dev_priv->display.update_wm = i9xx_update_wm; 6619 dev_priv->display.get_fifo_size = i830_get_fifo_size; 6620 } 6621 6622 if (IS_I85X(dev) || IS_I865G(dev)) 6623 dev_priv->display.init_clock_gating = i85x_init_clock_gating; 6624 else 6625 dev_priv->display.init_clock_gating = i830_init_clock_gating; 6626 } else { 6627 DRM_ERROR("unexpected fall-through in intel_init_pm\n"); 6628 } 6629} 6630 6631int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val) 6632{ 6633 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 6634 6635 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { 6636 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n"); 6637 return -EAGAIN; 6638 } 6639 6640 I915_WRITE(GEN6_PCODE_DATA, *val); 6641 I915_WRITE(GEN6_PCODE_DATA1, 0); 6642 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); 6643 6644 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 6645 500)) { 6646 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox); 6647 return -ETIMEDOUT; 6648 } 6649 6650 *val = I915_READ(GEN6_PCODE_DATA); 6651 I915_WRITE(GEN6_PCODE_DATA, 0); 6652 6653 return 0; 6654} 6655 6656int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val) 6657{ 6658 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 6659 6660 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { 6661 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n"); 6662 return -EAGAIN; 6663 } 6664 6665 I915_WRITE(GEN6_PCODE_DATA, val); 6666 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); 6667 6668 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 6669 500)) { 6670 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox); 6671 return -ETIMEDOUT; 6672 } 6673 6674 I915_WRITE(GEN6_PCODE_DATA, 0); 6675 6676 return 0; 6677} 6678 6679static int vlv_gpu_freq_div(unsigned int czclk_freq) 6680{ 6681 switch (czclk_freq) { 6682 case 200: 6683 return 10; 6684 case 267: 6685 return 12; 6686 case 320: 6687 case 333: 6688 return 16; 6689 case 400: 6690 return 20; 6691 default: 6692 return -1; 6693 } 6694} 6695 6696static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) 6697{ 6698 int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4); 6699 6700 div = vlv_gpu_freq_div(czclk_freq); 6701 if (div < 0) 6702 return div; 6703 6704 return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div); 6705} 6706 6707static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) 6708{ 6709 int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4); 6710 6711 mul = vlv_gpu_freq_div(czclk_freq); 6712 if (mul < 0) 6713 return mul; 6714 6715 return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6; 6716} 6717 6718static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) 6719{ 6720 int div, czclk_freq = dev_priv->rps.cz_freq; 6721 6722 div = vlv_gpu_freq_div(czclk_freq) / 2; 6723 if (div < 0) 6724 return div; 6725 6726 return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2; 6727} 6728 6729static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) 6730{ 6731 int mul, czclk_freq = dev_priv->rps.cz_freq; 6732 6733 mul = vlv_gpu_freq_div(czclk_freq) / 2; 6734 if (mul < 0) 6735 return mul; 6736 6737 /* CHV needs even values */ 6738 return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2; 6739} 6740 6741int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) 6742{ 6743 if (IS_GEN9(dev_priv->dev)) 6744 return (val * GT_FREQUENCY_MULTIPLIER) / GEN9_FREQ_SCALER; 6745 else if (IS_CHERRYVIEW(dev_priv->dev)) 6746 return chv_gpu_freq(dev_priv, val); 6747 else if (IS_VALLEYVIEW(dev_priv->dev)) 6748 return byt_gpu_freq(dev_priv, val); 6749 else 6750 return val * GT_FREQUENCY_MULTIPLIER; 6751} 6752 6753int intel_freq_opcode(struct drm_i915_private *dev_priv, int val) 6754{ 6755 if (IS_GEN9(dev_priv->dev)) 6756 return (val * GEN9_FREQ_SCALER) / GT_FREQUENCY_MULTIPLIER; 6757 else if (IS_CHERRYVIEW(dev_priv->dev)) 6758 return chv_freq_opcode(dev_priv, val); 6759 else if (IS_VALLEYVIEW(dev_priv->dev)) 6760 return byt_freq_opcode(dev_priv, val); 6761 else 6762 return val / GT_FREQUENCY_MULTIPLIER; 6763} 6764 6765void intel_pm_setup(struct drm_device *dev) 6766{ 6767 struct drm_i915_private *dev_priv = dev->dev_private; 6768 6769 mutex_init(&dev_priv->rps.hw_lock); 6770 6771 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work, 6772 intel_gen6_powersave_work); 6773 6774 dev_priv->pm.suspended = false; 6775} 6776