1/* 2 * MTRR (Memory Type Range Register) cleanup 3 * 4 * Copyright (C) 2009 Yinghai Lu 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Library General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Library General Public License for more details. 15 * 16 * You should have received a copy of the GNU Library General Public 17 * License along with this library; if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20#include <linux/module.h> 21#include <linux/init.h> 22#include <linux/pci.h> 23#include <linux/smp.h> 24#include <linux/cpu.h> 25#include <linux/mutex.h> 26#include <linux/uaccess.h> 27#include <linux/kvm_para.h> 28#include <linux/range.h> 29 30#include <asm/processor.h> 31#include <asm/e820.h> 32#include <asm/mtrr.h> 33#include <asm/msr.h> 34 35#include "mtrr.h" 36 37struct var_mtrr_range_state { 38 unsigned long base_pfn; 39 unsigned long size_pfn; 40 mtrr_type type; 41}; 42 43struct var_mtrr_state { 44 unsigned long range_startk; 45 unsigned long range_sizek; 46 unsigned long chunk_sizek; 47 unsigned long gran_sizek; 48 unsigned int reg; 49}; 50 51/* Should be related to MTRR_VAR_RANGES nums */ 52#define RANGE_NUM 256 53 54static struct range __initdata range[RANGE_NUM]; 55static int __initdata nr_range; 56 57static struct var_mtrr_range_state __initdata range_state[RANGE_NUM]; 58 59static int __initdata debug_print; 60#define Dprintk(x...) do { if (debug_print) printk(KERN_DEBUG x); } while (0) 61 62#define BIOS_BUG_MSG KERN_WARNING \ 63 "WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n" 64 65static int __init 66x86_get_mtrr_mem_range(struct range *range, int nr_range, 67 unsigned long extra_remove_base, 68 unsigned long extra_remove_size) 69{ 70 unsigned long base, size; 71 mtrr_type type; 72 int i; 73 74 for (i = 0; i < num_var_ranges; i++) { 75 type = range_state[i].type; 76 if (type != MTRR_TYPE_WRBACK) 77 continue; 78 base = range_state[i].base_pfn; 79 size = range_state[i].size_pfn; 80 nr_range = add_range_with_merge(range, RANGE_NUM, nr_range, 81 base, base + size); 82 } 83 if (debug_print) { 84 printk(KERN_DEBUG "After WB checking\n"); 85 for (i = 0; i < nr_range; i++) 86 printk(KERN_DEBUG "MTRR MAP PFN: %016llx - %016llx\n", 87 range[i].start, range[i].end); 88 } 89 90 /* Take out UC ranges: */ 91 for (i = 0; i < num_var_ranges; i++) { 92 type = range_state[i].type; 93 if (type != MTRR_TYPE_UNCACHABLE && 94 type != MTRR_TYPE_WRPROT) 95 continue; 96 size = range_state[i].size_pfn; 97 if (!size) 98 continue; 99 base = range_state[i].base_pfn; 100 if (base < (1<<(20-PAGE_SHIFT)) && mtrr_state.have_fixed && 101 (mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) && 102 (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) { 103 /* Var MTRR contains UC entry below 1M? Skip it: */ 104 printk(BIOS_BUG_MSG, i); 105 if (base + size <= (1<<(20-PAGE_SHIFT))) 106 continue; 107 size -= (1<<(20-PAGE_SHIFT)) - base; 108 base = 1<<(20-PAGE_SHIFT); 109 } 110 subtract_range(range, RANGE_NUM, base, base + size); 111 } 112 if (extra_remove_size) 113 subtract_range(range, RANGE_NUM, extra_remove_base, 114 extra_remove_base + extra_remove_size); 115 116 if (debug_print) { 117 printk(KERN_DEBUG "After UC checking\n"); 118 for (i = 0; i < RANGE_NUM; i++) { 119 if (!range[i].end) 120 continue; 121 printk(KERN_DEBUG "MTRR MAP PFN: %016llx - %016llx\n", 122 range[i].start, range[i].end); 123 } 124 } 125 126 /* sort the ranges */ 127 nr_range = clean_sort_range(range, RANGE_NUM); 128 if (debug_print) { 129 printk(KERN_DEBUG "After sorting\n"); 130 for (i = 0; i < nr_range; i++) 131 printk(KERN_DEBUG "MTRR MAP PFN: %016llx - %016llx\n", 132 range[i].start, range[i].end); 133 } 134 135 return nr_range; 136} 137 138#ifdef CONFIG_MTRR_SANITIZER 139 140static unsigned long __init sum_ranges(struct range *range, int nr_range) 141{ 142 unsigned long sum = 0; 143 int i; 144 145 for (i = 0; i < nr_range; i++) 146 sum += range[i].end - range[i].start; 147 148 return sum; 149} 150 151static int enable_mtrr_cleanup __initdata = 152 CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT; 153 154static int __init disable_mtrr_cleanup_setup(char *str) 155{ 156 enable_mtrr_cleanup = 0; 157 return 0; 158} 159early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup); 160 161static int __init enable_mtrr_cleanup_setup(char *str) 162{ 163 enable_mtrr_cleanup = 1; 164 return 0; 165} 166early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup); 167 168static int __init mtrr_cleanup_debug_setup(char *str) 169{ 170 debug_print = 1; 171 return 0; 172} 173early_param("mtrr_cleanup_debug", mtrr_cleanup_debug_setup); 174 175static void __init 176set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek, 177 unsigned char type, unsigned int address_bits) 178{ 179 u32 base_lo, base_hi, mask_lo, mask_hi; 180 u64 base, mask; 181 182 if (!sizek) { 183 fill_mtrr_var_range(reg, 0, 0, 0, 0); 184 return; 185 } 186 187 mask = (1ULL << address_bits) - 1; 188 mask &= ~((((u64)sizek) << 10) - 1); 189 190 base = ((u64)basek) << 10; 191 192 base |= type; 193 mask |= 0x800; 194 195 base_lo = base & ((1ULL<<32) - 1); 196 base_hi = base >> 32; 197 198 mask_lo = mask & ((1ULL<<32) - 1); 199 mask_hi = mask >> 32; 200 201 fill_mtrr_var_range(reg, base_lo, base_hi, mask_lo, mask_hi); 202} 203 204static void __init 205save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek, 206 unsigned char type) 207{ 208 range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10); 209 range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10); 210 range_state[reg].type = type; 211} 212 213static void __init set_var_mtrr_all(unsigned int address_bits) 214{ 215 unsigned long basek, sizek; 216 unsigned char type; 217 unsigned int reg; 218 219 for (reg = 0; reg < num_var_ranges; reg++) { 220 basek = range_state[reg].base_pfn << (PAGE_SHIFT - 10); 221 sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10); 222 type = range_state[reg].type; 223 224 set_var_mtrr(reg, basek, sizek, type, address_bits); 225 } 226} 227 228static unsigned long to_size_factor(unsigned long sizek, char *factorp) 229{ 230 unsigned long base = sizek; 231 char factor; 232 233 if (base & ((1<<10) - 1)) { 234 /* Not MB-aligned: */ 235 factor = 'K'; 236 } else if (base & ((1<<20) - 1)) { 237 factor = 'M'; 238 base >>= 10; 239 } else { 240 factor = 'G'; 241 base >>= 20; 242 } 243 244 *factorp = factor; 245 246 return base; 247} 248 249static unsigned int __init 250range_to_mtrr(unsigned int reg, unsigned long range_startk, 251 unsigned long range_sizek, unsigned char type) 252{ 253 if (!range_sizek || (reg >= num_var_ranges)) 254 return reg; 255 256 while (range_sizek) { 257 unsigned long max_align, align; 258 unsigned long sizek; 259 260 /* Compute the maximum size with which we can make a range: */ 261 if (range_startk) 262 max_align = __ffs(range_startk); 263 else 264 max_align = BITS_PER_LONG - 1; 265 266 align = __fls(range_sizek); 267 if (align > max_align) 268 align = max_align; 269 270 sizek = 1UL << align; 271 if (debug_print) { 272 char start_factor = 'K', size_factor = 'K'; 273 unsigned long start_base, size_base; 274 275 start_base = to_size_factor(range_startk, &start_factor); 276 size_base = to_size_factor(sizek, &size_factor); 277 278 Dprintk("Setting variable MTRR %d, " 279 "base: %ld%cB, range: %ld%cB, type %s\n", 280 reg, start_base, start_factor, 281 size_base, size_factor, 282 (type == MTRR_TYPE_UNCACHABLE) ? "UC" : 283 ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other") 284 ); 285 } 286 save_var_mtrr(reg++, range_startk, sizek, type); 287 range_startk += sizek; 288 range_sizek -= sizek; 289 if (reg >= num_var_ranges) 290 break; 291 } 292 return reg; 293} 294 295static unsigned __init 296range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek, 297 unsigned long sizek) 298{ 299 unsigned long hole_basek, hole_sizek; 300 unsigned long second_basek, second_sizek; 301 unsigned long range0_basek, range0_sizek; 302 unsigned long range_basek, range_sizek; 303 unsigned long chunk_sizek; 304 unsigned long gran_sizek; 305 306 hole_basek = 0; 307 hole_sizek = 0; 308 second_basek = 0; 309 second_sizek = 0; 310 chunk_sizek = state->chunk_sizek; 311 gran_sizek = state->gran_sizek; 312 313 /* Align with gran size, prevent small block used up MTRRs: */ 314 range_basek = ALIGN(state->range_startk, gran_sizek); 315 if ((range_basek > basek) && basek) 316 return second_sizek; 317 318 state->range_sizek -= (range_basek - state->range_startk); 319 range_sizek = ALIGN(state->range_sizek, gran_sizek); 320 321 while (range_sizek > state->range_sizek) { 322 range_sizek -= gran_sizek; 323 if (!range_sizek) 324 return 0; 325 } 326 state->range_sizek = range_sizek; 327 328 /* Try to append some small hole: */ 329 range0_basek = state->range_startk; 330 range0_sizek = ALIGN(state->range_sizek, chunk_sizek); 331 332 /* No increase: */ 333 if (range0_sizek == state->range_sizek) { 334 Dprintk("rangeX: %016lx - %016lx\n", 335 range0_basek<<10, 336 (range0_basek + state->range_sizek)<<10); 337 state->reg = range_to_mtrr(state->reg, range0_basek, 338 state->range_sizek, MTRR_TYPE_WRBACK); 339 return 0; 340 } 341 342 /* Only cut back when it is not the last: */ 343 if (sizek) { 344 while (range0_basek + range0_sizek > (basek + sizek)) { 345 if (range0_sizek >= chunk_sizek) 346 range0_sizek -= chunk_sizek; 347 else 348 range0_sizek = 0; 349 350 if (!range0_sizek) 351 break; 352 } 353 } 354 355second_try: 356 range_basek = range0_basek + range0_sizek; 357 358 /* One hole in the middle: */ 359 if (range_basek > basek && range_basek <= (basek + sizek)) 360 second_sizek = range_basek - basek; 361 362 if (range0_sizek > state->range_sizek) { 363 364 /* One hole in middle or at the end: */ 365 hole_sizek = range0_sizek - state->range_sizek - second_sizek; 366 367 /* Hole size should be less than half of range0 size: */ 368 if (hole_sizek >= (range0_sizek >> 1) && 369 range0_sizek >= chunk_sizek) { 370 range0_sizek -= chunk_sizek; 371 second_sizek = 0; 372 hole_sizek = 0; 373 374 goto second_try; 375 } 376 } 377 378 if (range0_sizek) { 379 Dprintk("range0: %016lx - %016lx\n", 380 range0_basek<<10, 381 (range0_basek + range0_sizek)<<10); 382 state->reg = range_to_mtrr(state->reg, range0_basek, 383 range0_sizek, MTRR_TYPE_WRBACK); 384 } 385 386 if (range0_sizek < state->range_sizek) { 387 /* Need to handle left over range: */ 388 range_sizek = state->range_sizek - range0_sizek; 389 390 Dprintk("range: %016lx - %016lx\n", 391 range_basek<<10, 392 (range_basek + range_sizek)<<10); 393 394 state->reg = range_to_mtrr(state->reg, range_basek, 395 range_sizek, MTRR_TYPE_WRBACK); 396 } 397 398 if (hole_sizek) { 399 hole_basek = range_basek - hole_sizek - second_sizek; 400 Dprintk("hole: %016lx - %016lx\n", 401 hole_basek<<10, 402 (hole_basek + hole_sizek)<<10); 403 state->reg = range_to_mtrr(state->reg, hole_basek, 404 hole_sizek, MTRR_TYPE_UNCACHABLE); 405 } 406 407 return second_sizek; 408} 409 410static void __init 411set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn, 412 unsigned long size_pfn) 413{ 414 unsigned long basek, sizek; 415 unsigned long second_sizek = 0; 416 417 if (state->reg >= num_var_ranges) 418 return; 419 420 basek = base_pfn << (PAGE_SHIFT - 10); 421 sizek = size_pfn << (PAGE_SHIFT - 10); 422 423 /* See if I can merge with the last range: */ 424 if ((basek <= 1024) || 425 (state->range_startk + state->range_sizek == basek)) { 426 unsigned long endk = basek + sizek; 427 state->range_sizek = endk - state->range_startk; 428 return; 429 } 430 /* Write the range mtrrs: */ 431 if (state->range_sizek != 0) 432 second_sizek = range_to_mtrr_with_hole(state, basek, sizek); 433 434 /* Allocate an msr: */ 435 state->range_startk = basek + second_sizek; 436 state->range_sizek = sizek - second_sizek; 437} 438 439/* Mininum size of mtrr block that can take hole: */ 440static u64 mtrr_chunk_size __initdata = (256ULL<<20); 441 442static int __init parse_mtrr_chunk_size_opt(char *p) 443{ 444 if (!p) 445 return -EINVAL; 446 mtrr_chunk_size = memparse(p, &p); 447 return 0; 448} 449early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt); 450 451/* Granularity of mtrr of block: */ 452static u64 mtrr_gran_size __initdata; 453 454static int __init parse_mtrr_gran_size_opt(char *p) 455{ 456 if (!p) 457 return -EINVAL; 458 mtrr_gran_size = memparse(p, &p); 459 return 0; 460} 461early_param("mtrr_gran_size", parse_mtrr_gran_size_opt); 462 463static unsigned long nr_mtrr_spare_reg __initdata = 464 CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT; 465 466static int __init parse_mtrr_spare_reg(char *arg) 467{ 468 if (arg) 469 nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0); 470 return 0; 471} 472early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg); 473 474static int __init 475x86_setup_var_mtrrs(struct range *range, int nr_range, 476 u64 chunk_size, u64 gran_size) 477{ 478 struct var_mtrr_state var_state; 479 int num_reg; 480 int i; 481 482 var_state.range_startk = 0; 483 var_state.range_sizek = 0; 484 var_state.reg = 0; 485 var_state.chunk_sizek = chunk_size >> 10; 486 var_state.gran_sizek = gran_size >> 10; 487 488 memset(range_state, 0, sizeof(range_state)); 489 490 /* Write the range: */ 491 for (i = 0; i < nr_range; i++) { 492 set_var_mtrr_range(&var_state, range[i].start, 493 range[i].end - range[i].start); 494 } 495 496 /* Write the last range: */ 497 if (var_state.range_sizek != 0) 498 range_to_mtrr_with_hole(&var_state, 0, 0); 499 500 num_reg = var_state.reg; 501 /* Clear out the extra MTRR's: */ 502 while (var_state.reg < num_var_ranges) { 503 save_var_mtrr(var_state.reg, 0, 0, 0); 504 var_state.reg++; 505 } 506 507 return num_reg; 508} 509 510struct mtrr_cleanup_result { 511 unsigned long gran_sizek; 512 unsigned long chunk_sizek; 513 unsigned long lose_cover_sizek; 514 unsigned int num_reg; 515 int bad; 516}; 517 518/* 519 * gran_size: 64K, 128K, 256K, 512K, 1M, 2M, ..., 2G 520 * chunk size: gran_size, ..., 2G 521 * so we need (1+16)*8 522 */ 523#define NUM_RESULT 136 524#define PSHIFT (PAGE_SHIFT - 10) 525 526static struct mtrr_cleanup_result __initdata result[NUM_RESULT]; 527static unsigned long __initdata min_loss_pfn[RANGE_NUM]; 528 529static void __init print_out_mtrr_range_state(void) 530{ 531 char start_factor = 'K', size_factor = 'K'; 532 unsigned long start_base, size_base; 533 mtrr_type type; 534 int i; 535 536 for (i = 0; i < num_var_ranges; i++) { 537 538 size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10); 539 if (!size_base) 540 continue; 541 542 size_base = to_size_factor(size_base, &size_factor), 543 start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10); 544 start_base = to_size_factor(start_base, &start_factor), 545 type = range_state[i].type; 546 547 printk(KERN_DEBUG "reg %d, base: %ld%cB, range: %ld%cB, type %s\n", 548 i, start_base, start_factor, 549 size_base, size_factor, 550 (type == MTRR_TYPE_UNCACHABLE) ? "UC" : 551 ((type == MTRR_TYPE_WRPROT) ? "WP" : 552 ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other")) 553 ); 554 } 555} 556 557static int __init mtrr_need_cleanup(void) 558{ 559 int i; 560 mtrr_type type; 561 unsigned long size; 562 /* Extra one for all 0: */ 563 int num[MTRR_NUM_TYPES + 1]; 564 565 /* Check entries number: */ 566 memset(num, 0, sizeof(num)); 567 for (i = 0; i < num_var_ranges; i++) { 568 type = range_state[i].type; 569 size = range_state[i].size_pfn; 570 if (type >= MTRR_NUM_TYPES) 571 continue; 572 if (!size) 573 type = MTRR_NUM_TYPES; 574 num[type]++; 575 } 576 577 /* Check if we got UC entries: */ 578 if (!num[MTRR_TYPE_UNCACHABLE]) 579 return 0; 580 581 /* Check if we only had WB and UC */ 582 if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] != 583 num_var_ranges - num[MTRR_NUM_TYPES]) 584 return 0; 585 586 return 1; 587} 588 589static unsigned long __initdata range_sums; 590 591static void __init 592mtrr_calc_range_state(u64 chunk_size, u64 gran_size, 593 unsigned long x_remove_base, 594 unsigned long x_remove_size, int i) 595{ 596 static struct range range_new[RANGE_NUM]; 597 unsigned long range_sums_new; 598 static int nr_range_new; 599 int num_reg; 600 601 /* Convert ranges to var ranges state: */ 602 num_reg = x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size); 603 604 /* We got new setting in range_state, check it: */ 605 memset(range_new, 0, sizeof(range_new)); 606 nr_range_new = x86_get_mtrr_mem_range(range_new, 0, 607 x_remove_base, x_remove_size); 608 range_sums_new = sum_ranges(range_new, nr_range_new); 609 610 result[i].chunk_sizek = chunk_size >> 10; 611 result[i].gran_sizek = gran_size >> 10; 612 result[i].num_reg = num_reg; 613 614 if (range_sums < range_sums_new) { 615 result[i].lose_cover_sizek = (range_sums_new - range_sums) << PSHIFT; 616 result[i].bad = 1; 617 } else { 618 result[i].lose_cover_sizek = (range_sums - range_sums_new) << PSHIFT; 619 } 620 621 /* Double check it: */ 622 if (!result[i].bad && !result[i].lose_cover_sizek) { 623 if (nr_range_new != nr_range || memcmp(range, range_new, sizeof(range))) 624 result[i].bad = 1; 625 } 626 627 if (!result[i].bad && (range_sums - range_sums_new < min_loss_pfn[num_reg])) 628 min_loss_pfn[num_reg] = range_sums - range_sums_new; 629} 630 631static void __init mtrr_print_out_one_result(int i) 632{ 633 unsigned long gran_base, chunk_base, lose_base; 634 char gran_factor, chunk_factor, lose_factor; 635 636 gran_base = to_size_factor(result[i].gran_sizek, &gran_factor); 637 chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor); 638 lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor); 639 640 pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t", 641 result[i].bad ? "*BAD*" : " ", 642 gran_base, gran_factor, chunk_base, chunk_factor); 643 pr_cont("num_reg: %d \tlose cover RAM: %s%ld%c\n", 644 result[i].num_reg, result[i].bad ? "-" : "", 645 lose_base, lose_factor); 646} 647 648static int __init mtrr_search_optimal_index(void) 649{ 650 int num_reg_good; 651 int index_good; 652 int i; 653 654 if (nr_mtrr_spare_reg >= num_var_ranges) 655 nr_mtrr_spare_reg = num_var_ranges - 1; 656 657 num_reg_good = -1; 658 for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) { 659 if (!min_loss_pfn[i]) 660 num_reg_good = i; 661 } 662 663 index_good = -1; 664 if (num_reg_good != -1) { 665 for (i = 0; i < NUM_RESULT; i++) { 666 if (!result[i].bad && 667 result[i].num_reg == num_reg_good && 668 !result[i].lose_cover_sizek) { 669 index_good = i; 670 break; 671 } 672 } 673 } 674 675 return index_good; 676} 677 678int __init mtrr_cleanup(unsigned address_bits) 679{ 680 unsigned long x_remove_base, x_remove_size; 681 unsigned long base, size, def, dummy; 682 u64 chunk_size, gran_size; 683 mtrr_type type; 684 int index_good; 685 int i; 686 687 if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1) 688 return 0; 689 690 rdmsr(MSR_MTRRdefType, def, dummy); 691 def &= 0xff; 692 if (def != MTRR_TYPE_UNCACHABLE) 693 return 0; 694 695 /* Get it and store it aside: */ 696 memset(range_state, 0, sizeof(range_state)); 697 for (i = 0; i < num_var_ranges; i++) { 698 mtrr_if->get(i, &base, &size, &type); 699 range_state[i].base_pfn = base; 700 range_state[i].size_pfn = size; 701 range_state[i].type = type; 702 } 703 704 /* Check if we need handle it and can handle it: */ 705 if (!mtrr_need_cleanup()) 706 return 0; 707 708 /* Print original var MTRRs at first, for debugging: */ 709 printk(KERN_DEBUG "original variable MTRRs\n"); 710 print_out_mtrr_range_state(); 711 712 memset(range, 0, sizeof(range)); 713 x_remove_size = 0; 714 x_remove_base = 1 << (32 - PAGE_SHIFT); 715 if (mtrr_tom2) 716 x_remove_size = (mtrr_tom2 >> PAGE_SHIFT) - x_remove_base; 717 718 /* 719 * [0, 1M) should always be covered by var mtrr with WB 720 * and fixed mtrrs should take effect before var mtrr for it: 721 */ 722 nr_range = add_range_with_merge(range, RANGE_NUM, 0, 0, 723 1ULL<<(20 - PAGE_SHIFT)); 724 /* add from var mtrr at last */ 725 nr_range = x86_get_mtrr_mem_range(range, nr_range, 726 x_remove_base, x_remove_size); 727 728 range_sums = sum_ranges(range, nr_range); 729 printk(KERN_INFO "total RAM covered: %ldM\n", 730 range_sums >> (20 - PAGE_SHIFT)); 731 732 if (mtrr_chunk_size && mtrr_gran_size) { 733 i = 0; 734 mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size, 735 x_remove_base, x_remove_size, i); 736 737 mtrr_print_out_one_result(i); 738 739 if (!result[i].bad) { 740 set_var_mtrr_all(address_bits); 741 printk(KERN_DEBUG "New variable MTRRs\n"); 742 print_out_mtrr_range_state(); 743 return 1; 744 } 745 printk(KERN_INFO "invalid mtrr_gran_size or mtrr_chunk_size, " 746 "will find optimal one\n"); 747 } 748 749 i = 0; 750 memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn)); 751 memset(result, 0, sizeof(result)); 752 for (gran_size = (1ULL<<16); gran_size < (1ULL<<32); gran_size <<= 1) { 753 754 for (chunk_size = gran_size; chunk_size < (1ULL<<32); 755 chunk_size <<= 1) { 756 757 if (i >= NUM_RESULT) 758 continue; 759 760 mtrr_calc_range_state(chunk_size, gran_size, 761 x_remove_base, x_remove_size, i); 762 if (debug_print) { 763 mtrr_print_out_one_result(i); 764 printk(KERN_INFO "\n"); 765 } 766 767 i++; 768 } 769 } 770 771 /* Try to find the optimal index: */ 772 index_good = mtrr_search_optimal_index(); 773 774 if (index_good != -1) { 775 printk(KERN_INFO "Found optimal setting for mtrr clean up\n"); 776 i = index_good; 777 mtrr_print_out_one_result(i); 778 779 /* Convert ranges to var ranges state: */ 780 chunk_size = result[i].chunk_sizek; 781 chunk_size <<= 10; 782 gran_size = result[i].gran_sizek; 783 gran_size <<= 10; 784 x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size); 785 set_var_mtrr_all(address_bits); 786 printk(KERN_DEBUG "New variable MTRRs\n"); 787 print_out_mtrr_range_state(); 788 return 1; 789 } else { 790 /* print out all */ 791 for (i = 0; i < NUM_RESULT; i++) 792 mtrr_print_out_one_result(i); 793 } 794 795 printk(KERN_INFO "mtrr_cleanup: can not find optimal value\n"); 796 printk(KERN_INFO "please specify mtrr_gran_size/mtrr_chunk_size\n"); 797 798 return 0; 799} 800#else 801int __init mtrr_cleanup(unsigned address_bits) 802{ 803 return 0; 804} 805#endif 806 807static int disable_mtrr_trim; 808 809static int __init disable_mtrr_trim_setup(char *str) 810{ 811 disable_mtrr_trim = 1; 812 return 0; 813} 814early_param("disable_mtrr_trim", disable_mtrr_trim_setup); 815 816/* 817 * Newer AMD K8s and later CPUs have a special magic MSR way to force WB 818 * for memory >4GB. Check for that here. 819 * Note this won't check if the MTRRs < 4GB where the magic bit doesn't 820 * apply to are wrong, but so far we don't know of any such case in the wild. 821 */ 822#define Tom2Enabled (1U << 21) 823#define Tom2ForceMemTypeWB (1U << 22) 824 825int __init amd_special_default_mtrr(void) 826{ 827 u32 l, h; 828 829 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 830 return 0; 831 if (boot_cpu_data.x86 < 0xf) 832 return 0; 833 /* In case some hypervisor doesn't pass SYSCFG through: */ 834 if (rdmsr_safe(MSR_K8_SYSCFG, &l, &h) < 0) 835 return 0; 836 /* 837 * Memory between 4GB and top of mem is forced WB by this magic bit. 838 * Reserved before K8RevF, but should be zero there. 839 */ 840 if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) == 841 (Tom2Enabled | Tom2ForceMemTypeWB)) 842 return 1; 843 return 0; 844} 845 846static u64 __init 847real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn) 848{ 849 u64 trim_start, trim_size; 850 851 trim_start = start_pfn; 852 trim_start <<= PAGE_SHIFT; 853 854 trim_size = limit_pfn; 855 trim_size <<= PAGE_SHIFT; 856 trim_size -= trim_start; 857 858 return e820_update_range(trim_start, trim_size, E820_RAM, E820_RESERVED); 859} 860 861/** 862 * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs 863 * @end_pfn: ending page frame number 864 * 865 * Some buggy BIOSes don't setup the MTRRs properly for systems with certain 866 * memory configurations. This routine checks that the highest MTRR matches 867 * the end of memory, to make sure the MTRRs having a write back type cover 868 * all of the memory the kernel is intending to use. If not, it'll trim any 869 * memory off the end by adjusting end_pfn, removing it from the kernel's 870 * allocation pools, warning the user with an obnoxious message. 871 */ 872int __init mtrr_trim_uncached_memory(unsigned long end_pfn) 873{ 874 unsigned long i, base, size, highest_pfn = 0, def, dummy; 875 mtrr_type type; 876 u64 total_trim_size; 877 /* extra one for all 0 */ 878 int num[MTRR_NUM_TYPES + 1]; 879 880 /* 881 * Make sure we only trim uncachable memory on machines that 882 * support the Intel MTRR architecture: 883 */ 884 if (!is_cpu(INTEL) || disable_mtrr_trim) 885 return 0; 886 887 rdmsr(MSR_MTRRdefType, def, dummy); 888 def &= 0xff; 889 if (def != MTRR_TYPE_UNCACHABLE) 890 return 0; 891 892 /* Get it and store it aside: */ 893 memset(range_state, 0, sizeof(range_state)); 894 for (i = 0; i < num_var_ranges; i++) { 895 mtrr_if->get(i, &base, &size, &type); 896 range_state[i].base_pfn = base; 897 range_state[i].size_pfn = size; 898 range_state[i].type = type; 899 } 900 901 /* Find highest cached pfn: */ 902 for (i = 0; i < num_var_ranges; i++) { 903 type = range_state[i].type; 904 if (type != MTRR_TYPE_WRBACK) 905 continue; 906 base = range_state[i].base_pfn; 907 size = range_state[i].size_pfn; 908 if (highest_pfn < base + size) 909 highest_pfn = base + size; 910 } 911 912 /* kvm/qemu doesn't have mtrr set right, don't trim them all: */ 913 if (!highest_pfn) { 914 printk(KERN_INFO "CPU MTRRs all blank - virtualized system.\n"); 915 return 0; 916 } 917 918 /* Check entries number: */ 919 memset(num, 0, sizeof(num)); 920 for (i = 0; i < num_var_ranges; i++) { 921 type = range_state[i].type; 922 if (type >= MTRR_NUM_TYPES) 923 continue; 924 size = range_state[i].size_pfn; 925 if (!size) 926 type = MTRR_NUM_TYPES; 927 num[type]++; 928 } 929 930 /* No entry for WB? */ 931 if (!num[MTRR_TYPE_WRBACK]) 932 return 0; 933 934 /* Check if we only had WB and UC: */ 935 if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] != 936 num_var_ranges - num[MTRR_NUM_TYPES]) 937 return 0; 938 939 memset(range, 0, sizeof(range)); 940 nr_range = 0; 941 if (mtrr_tom2) { 942 range[nr_range].start = (1ULL<<(32 - PAGE_SHIFT)); 943 range[nr_range].end = mtrr_tom2 >> PAGE_SHIFT; 944 if (highest_pfn < range[nr_range].end) 945 highest_pfn = range[nr_range].end; 946 nr_range++; 947 } 948 nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0); 949 950 /* Check the head: */ 951 total_trim_size = 0; 952 if (range[0].start) 953 total_trim_size += real_trim_memory(0, range[0].start); 954 955 /* Check the holes: */ 956 for (i = 0; i < nr_range - 1; i++) { 957 if (range[i].end < range[i+1].start) 958 total_trim_size += real_trim_memory(range[i].end, 959 range[i+1].start); 960 } 961 962 /* Check the top: */ 963 i = nr_range - 1; 964 if (range[i].end < end_pfn) 965 total_trim_size += real_trim_memory(range[i].end, 966 end_pfn); 967 968 if (total_trim_size) { 969 pr_warning("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n", total_trim_size >> 20); 970 971 if (!changed_by_mtrr_cleanup) 972 WARN_ON(1); 973 974 pr_info("update e820 for mtrr\n"); 975 update_e820(); 976 977 return 1; 978 } 979 980 return 0; 981} 982