1/* 2 * sparse memory mappings. 3 */ 4#include <linux/mm.h> 5#include <linux/slab.h> 6#include <linux/mmzone.h> 7#include <linux/bootmem.h> 8#include <linux/compiler.h> 9#include <linux/highmem.h> 10#include <linux/export.h> 11#include <linux/spinlock.h> 12#include <linux/vmalloc.h> 13 14#include "internal.h" 15#include <asm/dma.h> 16#include <asm/pgalloc.h> 17#include <asm/pgtable.h> 18 19/* 20 * Permanent SPARSEMEM data: 21 * 22 * 1) mem_section - memory sections, mem_map's for valid memory 23 */ 24#ifdef CONFIG_SPARSEMEM_EXTREME 25struct mem_section *mem_section[NR_SECTION_ROOTS] 26 ____cacheline_internodealigned_in_smp; 27#else 28struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 29 ____cacheline_internodealigned_in_smp; 30#endif 31EXPORT_SYMBOL(mem_section); 32 33#ifdef NODE_NOT_IN_PAGE_FLAGS 34/* 35 * If we did not store the node number in the page then we have to 36 * do a lookup in the section_to_node_table in order to find which 37 * node the page belongs to. 38 */ 39#if MAX_NUMNODES <= 256 40static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 41#else 42static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 43#endif 44 45int page_to_nid(const struct page *page) 46{ 47 return section_to_node_table[page_to_section(page)]; 48} 49EXPORT_SYMBOL(page_to_nid); 50 51static void set_section_nid(unsigned long section_nr, int nid) 52{ 53 section_to_node_table[section_nr] = nid; 54} 55#else /* !NODE_NOT_IN_PAGE_FLAGS */ 56static inline void set_section_nid(unsigned long section_nr, int nid) 57{ 58} 59#endif 60 61#ifdef CONFIG_SPARSEMEM_EXTREME 62static struct mem_section noinline __init_refok *sparse_index_alloc(int nid) 63{ 64 struct mem_section *section = NULL; 65 unsigned long array_size = SECTIONS_PER_ROOT * 66 sizeof(struct mem_section); 67 68 if (slab_is_available()) { 69 if (node_state(nid, N_HIGH_MEMORY)) 70 section = kzalloc_node(array_size, GFP_KERNEL, nid); 71 else 72 section = kzalloc(array_size, GFP_KERNEL); 73 } else { 74 section = memblock_virt_alloc_node(array_size, nid); 75 } 76 77 return section; 78} 79 80static int __meminit sparse_index_init(unsigned long section_nr, int nid) 81{ 82 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 83 struct mem_section *section; 84 85 if (mem_section[root]) 86 return -EEXIST; 87 88 section = sparse_index_alloc(nid); 89 if (!section) 90 return -ENOMEM; 91 92 mem_section[root] = section; 93 94 return 0; 95} 96#else /* !SPARSEMEM_EXTREME */ 97static inline int sparse_index_init(unsigned long section_nr, int nid) 98{ 99 return 0; 100} 101#endif 102 103/* 104 * Although written for the SPARSEMEM_EXTREME case, this happens 105 * to also work for the flat array case because 106 * NR_SECTION_ROOTS==NR_MEM_SECTIONS. 107 */ 108int __section_nr(struct mem_section* ms) 109{ 110 unsigned long root_nr; 111 struct mem_section* root; 112 113 for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { 114 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); 115 if (!root) 116 continue; 117 118 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) 119 break; 120 } 121 122 VM_BUG_ON(root_nr == NR_SECTION_ROOTS); 123 124 return (root_nr * SECTIONS_PER_ROOT) + (ms - root); 125} 126 127/* 128 * During early boot, before section_mem_map is used for an actual 129 * mem_map, we use section_mem_map to store the section's NUMA 130 * node. This keeps us from having to use another data structure. The 131 * node information is cleared just before we store the real mem_map. 132 */ 133static inline unsigned long sparse_encode_early_nid(int nid) 134{ 135 return (nid << SECTION_NID_SHIFT); 136} 137 138static inline int sparse_early_nid(struct mem_section *section) 139{ 140 return (section->section_mem_map >> SECTION_NID_SHIFT); 141} 142 143/* Validate the physical addressing limitations of the model */ 144void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 145 unsigned long *end_pfn) 146{ 147 unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 148 149 /* 150 * Sanity checks - do not allow an architecture to pass 151 * in larger pfns than the maximum scope of sparsemem: 152 */ 153 if (*start_pfn > max_sparsemem_pfn) { 154 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 155 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 156 *start_pfn, *end_pfn, max_sparsemem_pfn); 157 WARN_ON_ONCE(1); 158 *start_pfn = max_sparsemem_pfn; 159 *end_pfn = max_sparsemem_pfn; 160 } else if (*end_pfn > max_sparsemem_pfn) { 161 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 162 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 163 *start_pfn, *end_pfn, max_sparsemem_pfn); 164 WARN_ON_ONCE(1); 165 *end_pfn = max_sparsemem_pfn; 166 } 167} 168 169/* Record a memory area against a node. */ 170void __init memory_present(int nid, unsigned long start, unsigned long end) 171{ 172 unsigned long pfn; 173 174 start &= PAGE_SECTION_MASK; 175 mminit_validate_memmodel_limits(&start, &end); 176 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 177 unsigned long section = pfn_to_section_nr(pfn); 178 struct mem_section *ms; 179 180 sparse_index_init(section, nid); 181 set_section_nid(section, nid); 182 183 ms = __nr_to_section(section); 184 if (!ms->section_mem_map) 185 ms->section_mem_map = sparse_encode_early_nid(nid) | 186 SECTION_MARKED_PRESENT; 187 } 188} 189 190/* 191 * Only used by the i386 NUMA architecures, but relatively 192 * generic code. 193 */ 194unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, 195 unsigned long end_pfn) 196{ 197 unsigned long pfn; 198 unsigned long nr_pages = 0; 199 200 mminit_validate_memmodel_limits(&start_pfn, &end_pfn); 201 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 202 if (nid != early_pfn_to_nid(pfn)) 203 continue; 204 205 if (pfn_present(pfn)) 206 nr_pages += PAGES_PER_SECTION; 207 } 208 209 return nr_pages * sizeof(struct page); 210} 211 212/* 213 * Subtle, we encode the real pfn into the mem_map such that 214 * the identity pfn - section_mem_map will return the actual 215 * physical page frame number. 216 */ 217static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 218{ 219 return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 220} 221 222/* 223 * Decode mem_map from the coded memmap 224 */ 225struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 226{ 227 /* mask off the extra low bits of information */ 228 coded_mem_map &= SECTION_MAP_MASK; 229 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 230} 231 232static int __meminit sparse_init_one_section(struct mem_section *ms, 233 unsigned long pnum, struct page *mem_map, 234 unsigned long *pageblock_bitmap) 235{ 236 if (!present_section(ms)) 237 return -EINVAL; 238 239 ms->section_mem_map &= ~SECTION_MAP_MASK; 240 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | 241 SECTION_HAS_MEM_MAP; 242 ms->pageblock_flags = pageblock_bitmap; 243 244 return 1; 245} 246 247unsigned long usemap_size(void) 248{ 249 unsigned long size_bytes; 250 size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8; 251 size_bytes = roundup(size_bytes, sizeof(unsigned long)); 252 return size_bytes; 253} 254 255#ifdef CONFIG_MEMORY_HOTPLUG 256static unsigned long *__kmalloc_section_usemap(void) 257{ 258 return kmalloc(usemap_size(), GFP_KERNEL); 259} 260#endif /* CONFIG_MEMORY_HOTPLUG */ 261 262#ifdef CONFIG_MEMORY_HOTREMOVE 263static unsigned long * __init 264sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 265 unsigned long size) 266{ 267 unsigned long goal, limit; 268 unsigned long *p; 269 int nid; 270 /* 271 * A page may contain usemaps for other sections preventing the 272 * page being freed and making a section unremovable while 273 * other sections referencing the usemap remain active. Similarly, 274 * a pgdat can prevent a section being removed. If section A 275 * contains a pgdat and section B contains the usemap, both 276 * sections become inter-dependent. This allocates usemaps 277 * from the same section as the pgdat where possible to avoid 278 * this problem. 279 */ 280 goal = __pa(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT); 281 limit = goal + (1UL << PA_SECTION_SHIFT); 282 nid = early_pfn_to_nid(goal >> PAGE_SHIFT); 283again: 284 p = memblock_virt_alloc_try_nid_nopanic(size, 285 SMP_CACHE_BYTES, goal, limit, 286 nid); 287 if (!p && limit) { 288 limit = 0; 289 goto again; 290 } 291 return p; 292} 293 294static void __init check_usemap_section_nr(int nid, unsigned long *usemap) 295{ 296 unsigned long usemap_snr, pgdat_snr; 297 static unsigned long old_usemap_snr = NR_MEM_SECTIONS; 298 static unsigned long old_pgdat_snr = NR_MEM_SECTIONS; 299 struct pglist_data *pgdat = NODE_DATA(nid); 300 int usemap_nid; 301 302 usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT); 303 pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT); 304 if (usemap_snr == pgdat_snr) 305 return; 306 307 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr) 308 /* skip redundant message */ 309 return; 310 311 old_usemap_snr = usemap_snr; 312 old_pgdat_snr = pgdat_snr; 313 314 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr)); 315 if (usemap_nid != nid) { 316 printk(KERN_INFO 317 "node %d must be removed before remove section %ld\n", 318 nid, usemap_snr); 319 return; 320 } 321 /* 322 * There is a circular dependency. 323 * Some platforms allow un-removable section because they will just 324 * gather other removable sections for dynamic partitioning. 325 * Just notify un-removable section's number here. 326 */ 327 printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr, 328 pgdat_snr, nid); 329 printk(KERN_CONT 330 " have a circular dependency on usemap and pgdat allocations\n"); 331} 332#else 333static unsigned long * __init 334sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 335 unsigned long size) 336{ 337 return memblock_virt_alloc_node_nopanic(size, pgdat->node_id); 338} 339 340static void __init check_usemap_section_nr(int nid, unsigned long *usemap) 341{ 342} 343#endif /* CONFIG_MEMORY_HOTREMOVE */ 344 345static void __init sparse_early_usemaps_alloc_node(void *data, 346 unsigned long pnum_begin, 347 unsigned long pnum_end, 348 unsigned long usemap_count, int nodeid) 349{ 350 void *usemap; 351 unsigned long pnum; 352 unsigned long **usemap_map = (unsigned long **)data; 353 int size = usemap_size(); 354 355 usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid), 356 size * usemap_count); 357 if (!usemap) { 358 printk(KERN_WARNING "%s: allocation failed\n", __func__); 359 return; 360 } 361 362 for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 363 if (!present_section_nr(pnum)) 364 continue; 365 usemap_map[pnum] = usemap; 366 usemap += size; 367 check_usemap_section_nr(nodeid, usemap_map[pnum]); 368 } 369} 370 371#ifndef CONFIG_SPARSEMEM_VMEMMAP 372struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid) 373{ 374 struct page *map; 375 unsigned long size; 376 377 map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); 378 if (map) 379 return map; 380 381 size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); 382 map = memblock_virt_alloc_try_nid(size, 383 PAGE_SIZE, __pa(MAX_DMA_ADDRESS), 384 BOOTMEM_ALLOC_ACCESSIBLE, nid); 385 return map; 386} 387void __init sparse_mem_maps_populate_node(struct page **map_map, 388 unsigned long pnum_begin, 389 unsigned long pnum_end, 390 unsigned long map_count, int nodeid) 391{ 392 void *map; 393 unsigned long pnum; 394 unsigned long size = sizeof(struct page) * PAGES_PER_SECTION; 395 396 map = alloc_remap(nodeid, size * map_count); 397 if (map) { 398 for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 399 if (!present_section_nr(pnum)) 400 continue; 401 map_map[pnum] = map; 402 map += size; 403 } 404 return; 405 } 406 407 size = PAGE_ALIGN(size); 408 map = memblock_virt_alloc_try_nid(size * map_count, 409 PAGE_SIZE, __pa(MAX_DMA_ADDRESS), 410 BOOTMEM_ALLOC_ACCESSIBLE, nodeid); 411 if (map) { 412 for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 413 if (!present_section_nr(pnum)) 414 continue; 415 map_map[pnum] = map; 416 map += size; 417 } 418 return; 419 } 420 421 /* fallback */ 422 for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 423 struct mem_section *ms; 424 425 if (!present_section_nr(pnum)) 426 continue; 427 map_map[pnum] = sparse_mem_map_populate(pnum, nodeid); 428 if (map_map[pnum]) 429 continue; 430 ms = __nr_to_section(pnum); 431 printk(KERN_ERR "%s: sparsemem memory map backing failed " 432 "some memory will not be available.\n", __func__); 433 ms->section_mem_map = 0; 434 } 435} 436#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 437 438#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 439static void __init sparse_early_mem_maps_alloc_node(void *data, 440 unsigned long pnum_begin, 441 unsigned long pnum_end, 442 unsigned long map_count, int nodeid) 443{ 444 struct page **map_map = (struct page **)data; 445 sparse_mem_maps_populate_node(map_map, pnum_begin, pnum_end, 446 map_count, nodeid); 447} 448#else 449static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum) 450{ 451 struct page *map; 452 struct mem_section *ms = __nr_to_section(pnum); 453 int nid = sparse_early_nid(ms); 454 455 map = sparse_mem_map_populate(pnum, nid); 456 if (map) 457 return map; 458 459 printk(KERN_ERR "%s: sparsemem memory map backing failed " 460 "some memory will not be available.\n", __func__); 461 ms->section_mem_map = 0; 462 return NULL; 463} 464#endif 465 466void __weak __meminit vmemmap_populate_print_last(void) 467{ 468} 469 470/** 471 * alloc_usemap_and_memmap - memory alloction for pageblock flags and vmemmap 472 * @map: usemap_map for pageblock flags or mmap_map for vmemmap 473 */ 474static void __init alloc_usemap_and_memmap(void (*alloc_func) 475 (void *, unsigned long, unsigned long, 476 unsigned long, int), void *data) 477{ 478 unsigned long pnum; 479 unsigned long map_count; 480 int nodeid_begin = 0; 481 unsigned long pnum_begin = 0; 482 483 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 484 struct mem_section *ms; 485 486 if (!present_section_nr(pnum)) 487 continue; 488 ms = __nr_to_section(pnum); 489 nodeid_begin = sparse_early_nid(ms); 490 pnum_begin = pnum; 491 break; 492 } 493 map_count = 1; 494 for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) { 495 struct mem_section *ms; 496 int nodeid; 497 498 if (!present_section_nr(pnum)) 499 continue; 500 ms = __nr_to_section(pnum); 501 nodeid = sparse_early_nid(ms); 502 if (nodeid == nodeid_begin) { 503 map_count++; 504 continue; 505 } 506 /* ok, we need to take cake of from pnum_begin to pnum - 1*/ 507 alloc_func(data, pnum_begin, pnum, 508 map_count, nodeid_begin); 509 /* new start, update count etc*/ 510 nodeid_begin = nodeid; 511 pnum_begin = pnum; 512 map_count = 1; 513 } 514 /* ok, last chunk */ 515 alloc_func(data, pnum_begin, NR_MEM_SECTIONS, 516 map_count, nodeid_begin); 517} 518 519/* 520 * Allocate the accumulated non-linear sections, allocate a mem_map 521 * for each and record the physical to section mapping. 522 */ 523void __init sparse_init(void) 524{ 525 unsigned long pnum; 526 struct page *map; 527 unsigned long *usemap; 528 unsigned long **usemap_map; 529 int size; 530#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 531 int size2; 532 struct page **map_map; 533#endif 534 535 /* see include/linux/mmzone.h 'struct mem_section' definition */ 536 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section))); 537 538 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ 539 set_pageblock_order(); 540 541 /* 542 * map is using big page (aka 2M in x86 64 bit) 543 * usemap is less one page (aka 24 bytes) 544 * so alloc 2M (with 2M align) and 24 bytes in turn will 545 * make next 2M slip to one more 2M later. 546 * then in big system, the memory will have a lot of holes... 547 * here try to allocate 2M pages continuously. 548 * 549 * powerpc need to call sparse_init_one_section right after each 550 * sparse_early_mem_map_alloc, so allocate usemap_map at first. 551 */ 552 size = sizeof(unsigned long *) * NR_MEM_SECTIONS; 553 usemap_map = memblock_virt_alloc(size, 0); 554 if (!usemap_map) 555 panic("can not allocate usemap_map\n"); 556 alloc_usemap_and_memmap(sparse_early_usemaps_alloc_node, 557 (void *)usemap_map); 558 559#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 560 size2 = sizeof(struct page *) * NR_MEM_SECTIONS; 561 map_map = memblock_virt_alloc(size2, 0); 562 if (!map_map) 563 panic("can not allocate map_map\n"); 564 alloc_usemap_and_memmap(sparse_early_mem_maps_alloc_node, 565 (void *)map_map); 566#endif 567 568 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 569 if (!present_section_nr(pnum)) 570 continue; 571 572 usemap = usemap_map[pnum]; 573 if (!usemap) 574 continue; 575 576#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 577 map = map_map[pnum]; 578#else 579 map = sparse_early_mem_map_alloc(pnum); 580#endif 581 if (!map) 582 continue; 583 584 sparse_init_one_section(__nr_to_section(pnum), pnum, map, 585 usemap); 586 } 587 588 vmemmap_populate_print_last(); 589 590#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 591 memblock_free_early(__pa(map_map), size2); 592#endif 593 memblock_free_early(__pa(usemap_map), size); 594} 595 596#ifdef CONFIG_MEMORY_HOTPLUG 597#ifdef CONFIG_SPARSEMEM_VMEMMAP 598static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid) 599{ 600 /* This will make the necessary allocations eventually. */ 601 return sparse_mem_map_populate(pnum, nid); 602} 603static void __kfree_section_memmap(struct page *memmap) 604{ 605 unsigned long start = (unsigned long)memmap; 606 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); 607 608 vmemmap_free(start, end); 609} 610#ifdef CONFIG_MEMORY_HOTREMOVE 611static void free_map_bootmem(struct page *memmap) 612{ 613 unsigned long start = (unsigned long)memmap; 614 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); 615 616 vmemmap_free(start, end); 617} 618#endif /* CONFIG_MEMORY_HOTREMOVE */ 619#else 620static struct page *__kmalloc_section_memmap(void) 621{ 622 struct page *page, *ret; 623 unsigned long memmap_size = sizeof(struct page) * PAGES_PER_SECTION; 624 625 page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); 626 if (page) 627 goto got_map_page; 628 629 ret = vmalloc(memmap_size); 630 if (ret) 631 goto got_map_ptr; 632 633 return NULL; 634got_map_page: 635 ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); 636got_map_ptr: 637 638 return ret; 639} 640 641static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid) 642{ 643 return __kmalloc_section_memmap(); 644} 645 646static void __kfree_section_memmap(struct page *memmap) 647{ 648 if (is_vmalloc_addr(memmap)) 649 vfree(memmap); 650 else 651 free_pages((unsigned long)memmap, 652 get_order(sizeof(struct page) * PAGES_PER_SECTION)); 653} 654 655#ifdef CONFIG_MEMORY_HOTREMOVE 656static void free_map_bootmem(struct page *memmap) 657{ 658 unsigned long maps_section_nr, removing_section_nr, i; 659 unsigned long magic, nr_pages; 660 struct page *page = virt_to_page(memmap); 661 662 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 663 >> PAGE_SHIFT; 664 665 for (i = 0; i < nr_pages; i++, page++) { 666 magic = (unsigned long) page->lru.next; 667 668 BUG_ON(magic == NODE_INFO); 669 670 maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 671 removing_section_nr = page->private; 672 673 /* 674 * When this function is called, the removing section is 675 * logical offlined state. This means all pages are isolated 676 * from page allocator. If removing section's memmap is placed 677 * on the same section, it must not be freed. 678 * If it is freed, page allocator may allocate it which will 679 * be removed physically soon. 680 */ 681 if (maps_section_nr != removing_section_nr) 682 put_page_bootmem(page); 683 } 684} 685#endif /* CONFIG_MEMORY_HOTREMOVE */ 686#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 687 688/* 689 * returns the number of sections whose mem_maps were properly 690 * set. If this is <=0, then that means that the passed-in 691 * map was not consumed and must be freed. 692 */ 693int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn) 694{ 695 unsigned long section_nr = pfn_to_section_nr(start_pfn); 696 struct pglist_data *pgdat = zone->zone_pgdat; 697 struct mem_section *ms; 698 struct page *memmap; 699 unsigned long *usemap; 700 unsigned long flags; 701 int ret; 702 703 /* 704 * no locking for this, because it does its own 705 * plus, it does a kmalloc 706 */ 707 ret = sparse_index_init(section_nr, pgdat->node_id); 708 if (ret < 0 && ret != -EEXIST) 709 return ret; 710 memmap = kmalloc_section_memmap(section_nr, pgdat->node_id); 711 if (!memmap) 712 return -ENOMEM; 713 usemap = __kmalloc_section_usemap(); 714 if (!usemap) { 715 __kfree_section_memmap(memmap); 716 return -ENOMEM; 717 } 718 719 pgdat_resize_lock(pgdat, &flags); 720 721 ms = __pfn_to_section(start_pfn); 722 if (ms->section_mem_map & SECTION_MARKED_PRESENT) { 723 ret = -EEXIST; 724 goto out; 725 } 726 727 memset(memmap, 0, sizeof(struct page) * PAGES_PER_SECTION); 728 729 ms->section_mem_map |= SECTION_MARKED_PRESENT; 730 731 ret = sparse_init_one_section(ms, section_nr, memmap, usemap); 732 733out: 734 pgdat_resize_unlock(pgdat, &flags); 735 if (ret <= 0) { 736 kfree(usemap); 737 __kfree_section_memmap(memmap); 738 } 739 return ret; 740} 741 742#ifdef CONFIG_MEMORY_HOTREMOVE 743#ifdef CONFIG_MEMORY_FAILURE 744static void clear_hwpoisoned_pages(struct page *memmap, int nr_pages) 745{ 746 int i; 747 748 if (!memmap) 749 return; 750 751 for (i = 0; i < PAGES_PER_SECTION; i++) { 752 if (PageHWPoison(&memmap[i])) { 753 atomic_long_sub(1, &num_poisoned_pages); 754 ClearPageHWPoison(&memmap[i]); 755 } 756 } 757} 758#else 759static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages) 760{ 761} 762#endif 763 764static void free_section_usemap(struct page *memmap, unsigned long *usemap) 765{ 766 struct page *usemap_page; 767 768 if (!usemap) 769 return; 770 771 usemap_page = virt_to_page(usemap); 772 /* 773 * Check to see if allocation came from hot-plug-add 774 */ 775 if (PageSlab(usemap_page) || PageCompound(usemap_page)) { 776 kfree(usemap); 777 if (memmap) 778 __kfree_section_memmap(memmap); 779 return; 780 } 781 782 /* 783 * The usemap came from bootmem. This is packed with other usemaps 784 * on the section which has pgdat at boot time. Just keep it as is now. 785 */ 786 787 if (memmap) 788 free_map_bootmem(memmap); 789} 790 791void sparse_remove_one_section(struct zone *zone, struct mem_section *ms) 792{ 793 struct page *memmap = NULL; 794 unsigned long *usemap = NULL, flags; 795 struct pglist_data *pgdat = zone->zone_pgdat; 796 797 pgdat_resize_lock(pgdat, &flags); 798 if (ms->section_mem_map) { 799 usemap = ms->pageblock_flags; 800 memmap = sparse_decode_mem_map(ms->section_mem_map, 801 __section_nr(ms)); 802 ms->section_mem_map = 0; 803 ms->pageblock_flags = NULL; 804 } 805 pgdat_resize_unlock(pgdat, &flags); 806 807 clear_hwpoisoned_pages(memmap, PAGES_PER_SECTION); 808 free_section_usemap(memmap, usemap); 809} 810#endif /* CONFIG_MEMORY_HOTREMOVE */ 811#endif /* CONFIG_MEMORY_HOTPLUG */ 812