1/* 2 * bootmem - A boot-time physical memory allocator and configurator 3 * 4 * Copyright (C) 1999 Ingo Molnar 5 * 1999 Kanoj Sarcar, SGI 6 * 2008 Johannes Weiner 7 * 8 * Access to this subsystem has to be serialized externally (which is true 9 * for the boot process anyway). 10 */ 11#include <linux/init.h> 12#include <linux/pfn.h> 13#include <linux/slab.h> 14#include <linux/bootmem.h> 15#include <linux/export.h> 16#include <linux/kmemleak.h> 17#include <linux/range.h> 18#include <linux/memblock.h> 19#include <linux/bug.h> 20#include <linux/io.h> 21 22#include <asm/processor.h> 23 24#include "internal.h" 25 26#ifndef CONFIG_NEED_MULTIPLE_NODES 27struct pglist_data __refdata contig_page_data = { 28 .bdata = &bootmem_node_data[0] 29}; 30EXPORT_SYMBOL(contig_page_data); 31#endif 32 33unsigned long max_low_pfn; 34unsigned long min_low_pfn; 35unsigned long max_pfn; 36 37bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata; 38 39static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list); 40 41static int bootmem_debug; 42 43static int __init bootmem_debug_setup(char *buf) 44{ 45 bootmem_debug = 1; 46 return 0; 47} 48early_param("bootmem_debug", bootmem_debug_setup); 49 50#define bdebug(fmt, args...) ({ \ 51 if (unlikely(bootmem_debug)) \ 52 printk(KERN_INFO \ 53 "bootmem::%s " fmt, \ 54 __func__, ## args); \ 55}) 56 57static unsigned long __init bootmap_bytes(unsigned long pages) 58{ 59 unsigned long bytes = DIV_ROUND_UP(pages, 8); 60 61 return ALIGN(bytes, sizeof(long)); 62} 63 64/** 65 * bootmem_bootmap_pages - calculate bitmap size in pages 66 * @pages: number of pages the bitmap has to represent 67 */ 68unsigned long __init bootmem_bootmap_pages(unsigned long pages) 69{ 70 unsigned long bytes = bootmap_bytes(pages); 71 72 return PAGE_ALIGN(bytes) >> PAGE_SHIFT; 73} 74 75/* 76 * link bdata in order 77 */ 78static void __init link_bootmem(bootmem_data_t *bdata) 79{ 80 bootmem_data_t *ent; 81 82 list_for_each_entry(ent, &bdata_list, list) { 83 if (bdata->node_min_pfn < ent->node_min_pfn) { 84 list_add_tail(&bdata->list, &ent->list); 85 return; 86 } 87 } 88 89 list_add_tail(&bdata->list, &bdata_list); 90} 91 92/* 93 * Called once to set up the allocator itself. 94 */ 95static unsigned long __init init_bootmem_core(bootmem_data_t *bdata, 96 unsigned long mapstart, unsigned long start, unsigned long end) 97{ 98 unsigned long mapsize; 99 100 mminit_validate_memmodel_limits(&start, &end); 101 bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart)); 102 bdata->node_min_pfn = start; 103 bdata->node_low_pfn = end; 104 link_bootmem(bdata); 105 106 /* 107 * Initially all pages are reserved - setup_arch() has to 108 * register free RAM areas explicitly. 109 */ 110 mapsize = bootmap_bytes(end - start); 111 memset(bdata->node_bootmem_map, 0xff, mapsize); 112 113 bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx\n", 114 bdata - bootmem_node_data, start, mapstart, end, mapsize); 115 116 return mapsize; 117} 118 119/** 120 * init_bootmem_node - register a node as boot memory 121 * @pgdat: node to register 122 * @freepfn: pfn where the bitmap for this node is to be placed 123 * @startpfn: first pfn on the node 124 * @endpfn: first pfn after the node 125 * 126 * Returns the number of bytes needed to hold the bitmap for this node. 127 */ 128unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn, 129 unsigned long startpfn, unsigned long endpfn) 130{ 131 return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn); 132} 133 134/** 135 * init_bootmem - register boot memory 136 * @start: pfn where the bitmap is to be placed 137 * @pages: number of available physical pages 138 * 139 * Returns the number of bytes needed to hold the bitmap. 140 */ 141unsigned long __init init_bootmem(unsigned long start, unsigned long pages) 142{ 143 max_low_pfn = pages; 144 min_low_pfn = start; 145 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages); 146} 147 148/* 149 * free_bootmem_late - free bootmem pages directly to page allocator 150 * @addr: starting physical address of the range 151 * @size: size of the range in bytes 152 * 153 * This is only useful when the bootmem allocator has already been torn 154 * down, but we are still initializing the system. Pages are given directly 155 * to the page allocator, no bootmem metadata is updated because it is gone. 156 */ 157void __init free_bootmem_late(unsigned long physaddr, unsigned long size) 158{ 159 unsigned long cursor, end; 160 161 kmemleak_free_part(__va(physaddr), size); 162 163 cursor = PFN_UP(physaddr); 164 end = PFN_DOWN(physaddr + size); 165 166 for (; cursor < end; cursor++) { 167 __free_pages_bootmem(pfn_to_page(cursor), cursor, 0); 168 totalram_pages++; 169 } 170} 171 172static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) 173{ 174 struct page *page; 175 unsigned long *map, start, end, pages, cur, count = 0; 176 177 if (!bdata->node_bootmem_map) 178 return 0; 179 180 map = bdata->node_bootmem_map; 181 start = bdata->node_min_pfn; 182 end = bdata->node_low_pfn; 183 184 bdebug("nid=%td start=%lx end=%lx\n", 185 bdata - bootmem_node_data, start, end); 186 187 while (start < end) { 188 unsigned long idx, vec; 189 unsigned shift; 190 191 idx = start - bdata->node_min_pfn; 192 shift = idx & (BITS_PER_LONG - 1); 193 /* 194 * vec holds at most BITS_PER_LONG map bits, 195 * bit 0 corresponds to start. 196 */ 197 vec = ~map[idx / BITS_PER_LONG]; 198 199 if (shift) { 200 vec >>= shift; 201 if (end - start >= BITS_PER_LONG) 202 vec |= ~map[idx / BITS_PER_LONG + 1] << 203 (BITS_PER_LONG - shift); 204 } 205 /* 206 * If we have a properly aligned and fully unreserved 207 * BITS_PER_LONG block of pages in front of us, free 208 * it in one go. 209 */ 210 if (IS_ALIGNED(start, BITS_PER_LONG) && vec == ~0UL) { 211 int order = ilog2(BITS_PER_LONG); 212 213 __free_pages_bootmem(pfn_to_page(start), start, order); 214 count += BITS_PER_LONG; 215 start += BITS_PER_LONG; 216 } else { 217 cur = start; 218 219 start = ALIGN(start + 1, BITS_PER_LONG); 220 while (vec && cur != start) { 221 if (vec & 1) { 222 page = pfn_to_page(cur); 223 __free_pages_bootmem(page, cur, 0); 224 count++; 225 } 226 vec >>= 1; 227 ++cur; 228 } 229 } 230 } 231 232 cur = bdata->node_min_pfn; 233 page = virt_to_page(bdata->node_bootmem_map); 234 pages = bdata->node_low_pfn - bdata->node_min_pfn; 235 pages = bootmem_bootmap_pages(pages); 236 count += pages; 237 while (pages--) 238 __free_pages_bootmem(page++, cur++, 0); 239 bdata->node_bootmem_map = NULL; 240 241 bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count); 242 243 return count; 244} 245 246static int reset_managed_pages_done __initdata; 247 248void reset_node_managed_pages(pg_data_t *pgdat) 249{ 250 struct zone *z; 251 252 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 253 z->managed_pages = 0; 254} 255 256void __init reset_all_zones_managed_pages(void) 257{ 258 struct pglist_data *pgdat; 259 260 if (reset_managed_pages_done) 261 return; 262 263 for_each_online_pgdat(pgdat) 264 reset_node_managed_pages(pgdat); 265 266 reset_managed_pages_done = 1; 267} 268 269/** 270 * free_all_bootmem - release free pages to the buddy allocator 271 * 272 * Returns the number of pages actually released. 273 */ 274unsigned long __init free_all_bootmem(void) 275{ 276 unsigned long total_pages = 0; 277 bootmem_data_t *bdata; 278 279 reset_all_zones_managed_pages(); 280 281 list_for_each_entry(bdata, &bdata_list, list) 282 total_pages += free_all_bootmem_core(bdata); 283 284 totalram_pages += total_pages; 285 286 return total_pages; 287} 288 289static void __init __free(bootmem_data_t *bdata, 290 unsigned long sidx, unsigned long eidx) 291{ 292 unsigned long idx; 293 294 bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data, 295 sidx + bdata->node_min_pfn, 296 eidx + bdata->node_min_pfn); 297 298 if (WARN_ON(bdata->node_bootmem_map == NULL)) 299 return; 300 301 if (bdata->hint_idx > sidx) 302 bdata->hint_idx = sidx; 303 304 for (idx = sidx; idx < eidx; idx++) 305 if (!test_and_clear_bit(idx, bdata->node_bootmem_map)) 306 BUG(); 307} 308 309static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx, 310 unsigned long eidx, int flags) 311{ 312 unsigned long idx; 313 int exclusive = flags & BOOTMEM_EXCLUSIVE; 314 315 bdebug("nid=%td start=%lx end=%lx flags=%x\n", 316 bdata - bootmem_node_data, 317 sidx + bdata->node_min_pfn, 318 eidx + bdata->node_min_pfn, 319 flags); 320 321 if (WARN_ON(bdata->node_bootmem_map == NULL)) 322 return 0; 323 324 for (idx = sidx; idx < eidx; idx++) 325 if (test_and_set_bit(idx, bdata->node_bootmem_map)) { 326 if (exclusive) { 327 __free(bdata, sidx, idx); 328 return -EBUSY; 329 } 330 bdebug("silent double reserve of PFN %lx\n", 331 idx + bdata->node_min_pfn); 332 } 333 return 0; 334} 335 336static int __init mark_bootmem_node(bootmem_data_t *bdata, 337 unsigned long start, unsigned long end, 338 int reserve, int flags) 339{ 340 unsigned long sidx, eidx; 341 342 bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x\n", 343 bdata - bootmem_node_data, start, end, reserve, flags); 344 345 BUG_ON(start < bdata->node_min_pfn); 346 BUG_ON(end > bdata->node_low_pfn); 347 348 sidx = start - bdata->node_min_pfn; 349 eidx = end - bdata->node_min_pfn; 350 351 if (reserve) 352 return __reserve(bdata, sidx, eidx, flags); 353 else 354 __free(bdata, sidx, eidx); 355 return 0; 356} 357 358static int __init mark_bootmem(unsigned long start, unsigned long end, 359 int reserve, int flags) 360{ 361 unsigned long pos; 362 bootmem_data_t *bdata; 363 364 pos = start; 365 list_for_each_entry(bdata, &bdata_list, list) { 366 int err; 367 unsigned long max; 368 369 if (pos < bdata->node_min_pfn || 370 pos >= bdata->node_low_pfn) { 371 BUG_ON(pos != start); 372 continue; 373 } 374 375 max = min(bdata->node_low_pfn, end); 376 377 err = mark_bootmem_node(bdata, pos, max, reserve, flags); 378 if (reserve && err) { 379 mark_bootmem(start, pos, 0, 0); 380 return err; 381 } 382 383 if (max == end) 384 return 0; 385 pos = bdata->node_low_pfn; 386 } 387 BUG(); 388} 389 390/** 391 * free_bootmem_node - mark a page range as usable 392 * @pgdat: node the range resides on 393 * @physaddr: starting address of the range 394 * @size: size of the range in bytes 395 * 396 * Partial pages will be considered reserved and left as they are. 397 * 398 * The range must reside completely on the specified node. 399 */ 400void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 401 unsigned long size) 402{ 403 unsigned long start, end; 404 405 kmemleak_free_part(__va(physaddr), size); 406 407 start = PFN_UP(physaddr); 408 end = PFN_DOWN(physaddr + size); 409 410 mark_bootmem_node(pgdat->bdata, start, end, 0, 0); 411} 412 413/** 414 * free_bootmem - mark a page range as usable 415 * @addr: starting physical address of the range 416 * @size: size of the range in bytes 417 * 418 * Partial pages will be considered reserved and left as they are. 419 * 420 * The range must be contiguous but may span node boundaries. 421 */ 422void __init free_bootmem(unsigned long physaddr, unsigned long size) 423{ 424 unsigned long start, end; 425 426 kmemleak_free_part(__va(physaddr), size); 427 428 start = PFN_UP(physaddr); 429 end = PFN_DOWN(physaddr + size); 430 431 mark_bootmem(start, end, 0, 0); 432} 433 434/** 435 * reserve_bootmem_node - mark a page range as reserved 436 * @pgdat: node the range resides on 437 * @physaddr: starting address of the range 438 * @size: size of the range in bytes 439 * @flags: reservation flags (see linux/bootmem.h) 440 * 441 * Partial pages will be reserved. 442 * 443 * The range must reside completely on the specified node. 444 */ 445int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 446 unsigned long size, int flags) 447{ 448 unsigned long start, end; 449 450 start = PFN_DOWN(physaddr); 451 end = PFN_UP(physaddr + size); 452 453 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags); 454} 455 456/** 457 * reserve_bootmem - mark a page range as reserved 458 * @addr: starting address of the range 459 * @size: size of the range in bytes 460 * @flags: reservation flags (see linux/bootmem.h) 461 * 462 * Partial pages will be reserved. 463 * 464 * The range must be contiguous but may span node boundaries. 465 */ 466int __init reserve_bootmem(unsigned long addr, unsigned long size, 467 int flags) 468{ 469 unsigned long start, end; 470 471 start = PFN_DOWN(addr); 472 end = PFN_UP(addr + size); 473 474 return mark_bootmem(start, end, 1, flags); 475} 476 477static unsigned long __init align_idx(struct bootmem_data *bdata, 478 unsigned long idx, unsigned long step) 479{ 480 unsigned long base = bdata->node_min_pfn; 481 482 /* 483 * Align the index with respect to the node start so that the 484 * combination of both satisfies the requested alignment. 485 */ 486 487 return ALIGN(base + idx, step) - base; 488} 489 490static unsigned long __init align_off(struct bootmem_data *bdata, 491 unsigned long off, unsigned long align) 492{ 493 unsigned long base = PFN_PHYS(bdata->node_min_pfn); 494 495 /* Same as align_idx for byte offsets */ 496 497 return ALIGN(base + off, align) - base; 498} 499 500static void * __init alloc_bootmem_bdata(struct bootmem_data *bdata, 501 unsigned long size, unsigned long align, 502 unsigned long goal, unsigned long limit) 503{ 504 unsigned long fallback = 0; 505 unsigned long min, max, start, sidx, midx, step; 506 507 bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n", 508 bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT, 509 align, goal, limit); 510 511 BUG_ON(!size); 512 BUG_ON(align & (align - 1)); 513 BUG_ON(limit && goal + size > limit); 514 515 if (!bdata->node_bootmem_map) 516 return NULL; 517 518 min = bdata->node_min_pfn; 519 max = bdata->node_low_pfn; 520 521 goal >>= PAGE_SHIFT; 522 limit >>= PAGE_SHIFT; 523 524 if (limit && max > limit) 525 max = limit; 526 if (max <= min) 527 return NULL; 528 529 step = max(align >> PAGE_SHIFT, 1UL); 530 531 if (goal && min < goal && goal < max) 532 start = ALIGN(goal, step); 533 else 534 start = ALIGN(min, step); 535 536 sidx = start - bdata->node_min_pfn; 537 midx = max - bdata->node_min_pfn; 538 539 if (bdata->hint_idx > sidx) { 540 /* 541 * Handle the valid case of sidx being zero and still 542 * catch the fallback below. 543 */ 544 fallback = sidx + 1; 545 sidx = align_idx(bdata, bdata->hint_idx, step); 546 } 547 548 while (1) { 549 int merge; 550 void *region; 551 unsigned long eidx, i, start_off, end_off; 552find_block: 553 sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx); 554 sidx = align_idx(bdata, sidx, step); 555 eidx = sidx + PFN_UP(size); 556 557 if (sidx >= midx || eidx > midx) 558 break; 559 560 for (i = sidx; i < eidx; i++) 561 if (test_bit(i, bdata->node_bootmem_map)) { 562 sidx = align_idx(bdata, i, step); 563 if (sidx == i) 564 sidx += step; 565 goto find_block; 566 } 567 568 if (bdata->last_end_off & (PAGE_SIZE - 1) && 569 PFN_DOWN(bdata->last_end_off) + 1 == sidx) 570 start_off = align_off(bdata, bdata->last_end_off, align); 571 else 572 start_off = PFN_PHYS(sidx); 573 574 merge = PFN_DOWN(start_off) < sidx; 575 end_off = start_off + size; 576 577 bdata->last_end_off = end_off; 578 bdata->hint_idx = PFN_UP(end_off); 579 580 /* 581 * Reserve the area now: 582 */ 583 if (__reserve(bdata, PFN_DOWN(start_off) + merge, 584 PFN_UP(end_off), BOOTMEM_EXCLUSIVE)) 585 BUG(); 586 587 region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) + 588 start_off); 589 memset(region, 0, size); 590 /* 591 * The min_count is set to 0 so that bootmem allocated blocks 592 * are never reported as leaks. 593 */ 594 kmemleak_alloc(region, size, 0, 0); 595 return region; 596 } 597 598 if (fallback) { 599 sidx = align_idx(bdata, fallback - 1, step); 600 fallback = 0; 601 goto find_block; 602 } 603 604 return NULL; 605} 606 607static void * __init alloc_bootmem_core(unsigned long size, 608 unsigned long align, 609 unsigned long goal, 610 unsigned long limit) 611{ 612 bootmem_data_t *bdata; 613 void *region; 614 615 if (WARN_ON_ONCE(slab_is_available())) 616 return kzalloc(size, GFP_NOWAIT); 617 618 list_for_each_entry(bdata, &bdata_list, list) { 619 if (goal && bdata->node_low_pfn <= PFN_DOWN(goal)) 620 continue; 621 if (limit && bdata->node_min_pfn >= PFN_DOWN(limit)) 622 break; 623 624 region = alloc_bootmem_bdata(bdata, size, align, goal, limit); 625 if (region) 626 return region; 627 } 628 629 return NULL; 630} 631 632static void * __init ___alloc_bootmem_nopanic(unsigned long size, 633 unsigned long align, 634 unsigned long goal, 635 unsigned long limit) 636{ 637 void *ptr; 638 639restart: 640 ptr = alloc_bootmem_core(size, align, goal, limit); 641 if (ptr) 642 return ptr; 643 if (goal) { 644 goal = 0; 645 goto restart; 646 } 647 648 return NULL; 649} 650 651/** 652 * __alloc_bootmem_nopanic - allocate boot memory without panicking 653 * @size: size of the request in bytes 654 * @align: alignment of the region 655 * @goal: preferred starting address of the region 656 * 657 * The goal is dropped if it can not be satisfied and the allocation will 658 * fall back to memory below @goal. 659 * 660 * Allocation may happen on any node in the system. 661 * 662 * Returns NULL on failure. 663 */ 664void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, 665 unsigned long goal) 666{ 667 unsigned long limit = 0; 668 669 return ___alloc_bootmem_nopanic(size, align, goal, limit); 670} 671 672static void * __init ___alloc_bootmem(unsigned long size, unsigned long align, 673 unsigned long goal, unsigned long limit) 674{ 675 void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit); 676 677 if (mem) 678 return mem; 679 /* 680 * Whoops, we cannot satisfy the allocation request. 681 */ 682 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size); 683 panic("Out of memory"); 684 return NULL; 685} 686 687/** 688 * __alloc_bootmem - allocate boot memory 689 * @size: size of the request in bytes 690 * @align: alignment of the region 691 * @goal: preferred starting address of the region 692 * 693 * The goal is dropped if it can not be satisfied and the allocation will 694 * fall back to memory below @goal. 695 * 696 * Allocation may happen on any node in the system. 697 * 698 * The function panics if the request can not be satisfied. 699 */ 700void * __init __alloc_bootmem(unsigned long size, unsigned long align, 701 unsigned long goal) 702{ 703 unsigned long limit = 0; 704 705 return ___alloc_bootmem(size, align, goal, limit); 706} 707 708void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat, 709 unsigned long size, unsigned long align, 710 unsigned long goal, unsigned long limit) 711{ 712 void *ptr; 713 714 if (WARN_ON_ONCE(slab_is_available())) 715 return kzalloc(size, GFP_NOWAIT); 716again: 717 718 /* do not panic in alloc_bootmem_bdata() */ 719 if (limit && goal + size > limit) 720 limit = 0; 721 722 ptr = alloc_bootmem_bdata(pgdat->bdata, size, align, goal, limit); 723 if (ptr) 724 return ptr; 725 726 ptr = alloc_bootmem_core(size, align, goal, limit); 727 if (ptr) 728 return ptr; 729 730 if (goal) { 731 goal = 0; 732 goto again; 733 } 734 735 return NULL; 736} 737 738void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, 739 unsigned long align, unsigned long goal) 740{ 741 if (WARN_ON_ONCE(slab_is_available())) 742 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 743 744 return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0); 745} 746 747void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, 748 unsigned long align, unsigned long goal, 749 unsigned long limit) 750{ 751 void *ptr; 752 753 ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0); 754 if (ptr) 755 return ptr; 756 757 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size); 758 panic("Out of memory"); 759 return NULL; 760} 761 762/** 763 * __alloc_bootmem_node - allocate boot memory from a specific node 764 * @pgdat: node to allocate from 765 * @size: size of the request in bytes 766 * @align: alignment of the region 767 * @goal: preferred starting address of the region 768 * 769 * The goal is dropped if it can not be satisfied and the allocation will 770 * fall back to memory below @goal. 771 * 772 * Allocation may fall back to any node in the system if the specified node 773 * can not hold the requested memory. 774 * 775 * The function panics if the request can not be satisfied. 776 */ 777void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, 778 unsigned long align, unsigned long goal) 779{ 780 if (WARN_ON_ONCE(slab_is_available())) 781 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 782 783 return ___alloc_bootmem_node(pgdat, size, align, goal, 0); 784} 785 786void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size, 787 unsigned long align, unsigned long goal) 788{ 789#ifdef MAX_DMA32_PFN 790 unsigned long end_pfn; 791 792 if (WARN_ON_ONCE(slab_is_available())) 793 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 794 795 /* update goal according ...MAX_DMA32_PFN */ 796 end_pfn = pgdat_end_pfn(pgdat); 797 798 if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) && 799 (goal >> PAGE_SHIFT) < MAX_DMA32_PFN) { 800 void *ptr; 801 unsigned long new_goal; 802 803 new_goal = MAX_DMA32_PFN << PAGE_SHIFT; 804 ptr = alloc_bootmem_bdata(pgdat->bdata, size, align, 805 new_goal, 0); 806 if (ptr) 807 return ptr; 808 } 809#endif 810 811 return __alloc_bootmem_node(pgdat, size, align, goal); 812 813} 814 815#ifndef ARCH_LOW_ADDRESS_LIMIT 816#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL 817#endif 818 819/** 820 * __alloc_bootmem_low - allocate low boot memory 821 * @size: size of the request in bytes 822 * @align: alignment of the region 823 * @goal: preferred starting address of the region 824 * 825 * The goal is dropped if it can not be satisfied and the allocation will 826 * fall back to memory below @goal. 827 * 828 * Allocation may happen on any node in the system. 829 * 830 * The function panics if the request can not be satisfied. 831 */ 832void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, 833 unsigned long goal) 834{ 835 return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT); 836} 837 838void * __init __alloc_bootmem_low_nopanic(unsigned long size, 839 unsigned long align, 840 unsigned long goal) 841{ 842 return ___alloc_bootmem_nopanic(size, align, goal, 843 ARCH_LOW_ADDRESS_LIMIT); 844} 845 846/** 847 * __alloc_bootmem_low_node - allocate low boot memory from a specific node 848 * @pgdat: node to allocate from 849 * @size: size of the request in bytes 850 * @align: alignment of the region 851 * @goal: preferred starting address of the region 852 * 853 * The goal is dropped if it can not be satisfied and the allocation will 854 * fall back to memory below @goal. 855 * 856 * Allocation may fall back to any node in the system if the specified node 857 * can not hold the requested memory. 858 * 859 * The function panics if the request can not be satisfied. 860 */ 861void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, 862 unsigned long align, unsigned long goal) 863{ 864 if (WARN_ON_ONCE(slab_is_available())) 865 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 866 867 return ___alloc_bootmem_node(pgdat, size, align, 868 goal, ARCH_LOW_ADDRESS_LIMIT); 869} 870