1/* 2 * linux/mm/memory_hotplug.c 3 * 4 * Copyright (C) 5 */ 6 7#include <linux/stddef.h> 8#include <linux/mm.h> 9#include <linux/swap.h> 10#include <linux/interrupt.h> 11#include <linux/pagemap.h> 12#include <linux/compiler.h> 13#include <linux/export.h> 14#include <linux/pagevec.h> 15#include <linux/writeback.h> 16#include <linux/slab.h> 17#include <linux/sysctl.h> 18#include <linux/cpu.h> 19#include <linux/memory.h> 20#include <linux/memory_hotplug.h> 21#include <linux/highmem.h> 22#include <linux/vmalloc.h> 23#include <linux/ioport.h> 24#include <linux/delay.h> 25#include <linux/migrate.h> 26#include <linux/page-isolation.h> 27#include <linux/pfn.h> 28#include <linux/suspend.h> 29#include <linux/mm_inline.h> 30#include <linux/firmware-map.h> 31#include <linux/stop_machine.h> 32#include <linux/hugetlb.h> 33#include <linux/memblock.h> 34#include <linux/bootmem.h> 35 36#include <asm/tlbflush.h> 37 38#include "internal.h" 39 40/* 41 * online_page_callback contains pointer to current page onlining function. 42 * Initially it is generic_online_page(). If it is required it could be 43 * changed by calling set_online_page_callback() for callback registration 44 * and restore_online_page_callback() for generic callback restore. 45 */ 46 47static void generic_online_page(struct page *page); 48 49static online_page_callback_t online_page_callback = generic_online_page; 50static DEFINE_MUTEX(online_page_callback_lock); 51 52/* The same as the cpu_hotplug lock, but for memory hotplug. */ 53static struct { 54 struct task_struct *active_writer; 55 struct mutex lock; /* Synchronizes accesses to refcount, */ 56 /* 57 * Also blocks the new readers during 58 * an ongoing mem hotplug operation. 59 */ 60 int refcount; 61 62#ifdef CONFIG_DEBUG_LOCK_ALLOC 63 struct lockdep_map dep_map; 64#endif 65} mem_hotplug = { 66 .active_writer = NULL, 67 .lock = __MUTEX_INITIALIZER(mem_hotplug.lock), 68 .refcount = 0, 69#ifdef CONFIG_DEBUG_LOCK_ALLOC 70 .dep_map = {.name = "mem_hotplug.lock" }, 71#endif 72}; 73 74/* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */ 75#define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map) 76#define memhp_lock_acquire() lock_map_acquire(&mem_hotplug.dep_map) 77#define memhp_lock_release() lock_map_release(&mem_hotplug.dep_map) 78 79void get_online_mems(void) 80{ 81 might_sleep(); 82 if (mem_hotplug.active_writer == current) 83 return; 84 memhp_lock_acquire_read(); 85 mutex_lock(&mem_hotplug.lock); 86 mem_hotplug.refcount++; 87 mutex_unlock(&mem_hotplug.lock); 88 89} 90 91void put_online_mems(void) 92{ 93 if (mem_hotplug.active_writer == current) 94 return; 95 mutex_lock(&mem_hotplug.lock); 96 97 if (WARN_ON(!mem_hotplug.refcount)) 98 mem_hotplug.refcount++; /* try to fix things up */ 99 100 if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer)) 101 wake_up_process(mem_hotplug.active_writer); 102 mutex_unlock(&mem_hotplug.lock); 103 memhp_lock_release(); 104 105} 106 107void mem_hotplug_begin(void) 108{ 109 mem_hotplug.active_writer = current; 110 111 memhp_lock_acquire(); 112 for (;;) { 113 mutex_lock(&mem_hotplug.lock); 114 if (likely(!mem_hotplug.refcount)) 115 break; 116 __set_current_state(TASK_UNINTERRUPTIBLE); 117 mutex_unlock(&mem_hotplug.lock); 118 schedule(); 119 } 120} 121 122void mem_hotplug_done(void) 123{ 124 mem_hotplug.active_writer = NULL; 125 mutex_unlock(&mem_hotplug.lock); 126 memhp_lock_release(); 127} 128 129/* add this memory to iomem resource */ 130static struct resource *register_memory_resource(u64 start, u64 size) 131{ 132 struct resource *res; 133 res = kzalloc(sizeof(struct resource), GFP_KERNEL); 134 BUG_ON(!res); 135 136 res->name = "System RAM"; 137 res->start = start; 138 res->end = start + size - 1; 139 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; 140 if (request_resource(&iomem_resource, res) < 0) { 141 pr_debug("System RAM resource %pR cannot be added\n", res); 142 kfree(res); 143 res = NULL; 144 } 145 return res; 146} 147 148static void release_memory_resource(struct resource *res) 149{ 150 if (!res) 151 return; 152 release_resource(res); 153 kfree(res); 154 return; 155} 156 157#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE 158void get_page_bootmem(unsigned long info, struct page *page, 159 unsigned long type) 160{ 161 page->lru.next = (struct list_head *) type; 162 SetPagePrivate(page); 163 set_page_private(page, info); 164 atomic_inc(&page->_count); 165} 166 167void put_page_bootmem(struct page *page) 168{ 169 unsigned long type; 170 171 type = (unsigned long) page->lru.next; 172 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || 173 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); 174 175 if (atomic_dec_return(&page->_count) == 1) { 176 ClearPagePrivate(page); 177 set_page_private(page, 0); 178 INIT_LIST_HEAD(&page->lru); 179 free_reserved_page(page); 180 } 181} 182 183#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE 184#ifndef CONFIG_SPARSEMEM_VMEMMAP 185static void register_page_bootmem_info_section(unsigned long start_pfn) 186{ 187 unsigned long *usemap, mapsize, section_nr, i; 188 struct mem_section *ms; 189 struct page *page, *memmap; 190 191 section_nr = pfn_to_section_nr(start_pfn); 192 ms = __nr_to_section(section_nr); 193 194 /* Get section's memmap address */ 195 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 196 197 /* 198 * Get page for the memmap's phys address 199 * XXX: need more consideration for sparse_vmemmap... 200 */ 201 page = virt_to_page(memmap); 202 mapsize = sizeof(struct page) * PAGES_PER_SECTION; 203 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; 204 205 /* remember memmap's page */ 206 for (i = 0; i < mapsize; i++, page++) 207 get_page_bootmem(section_nr, page, SECTION_INFO); 208 209 usemap = __nr_to_section(section_nr)->pageblock_flags; 210 page = virt_to_page(usemap); 211 212 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 213 214 for (i = 0; i < mapsize; i++, page++) 215 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 216 217} 218#else /* CONFIG_SPARSEMEM_VMEMMAP */ 219static void register_page_bootmem_info_section(unsigned long start_pfn) 220{ 221 unsigned long *usemap, mapsize, section_nr, i; 222 struct mem_section *ms; 223 struct page *page, *memmap; 224 225 if (!pfn_valid(start_pfn)) 226 return; 227 228 section_nr = pfn_to_section_nr(start_pfn); 229 ms = __nr_to_section(section_nr); 230 231 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 232 233 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); 234 235 usemap = __nr_to_section(section_nr)->pageblock_flags; 236 page = virt_to_page(usemap); 237 238 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 239 240 for (i = 0; i < mapsize; i++, page++) 241 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 242} 243#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 244 245void register_page_bootmem_info_node(struct pglist_data *pgdat) 246{ 247 unsigned long i, pfn, end_pfn, nr_pages; 248 int node = pgdat->node_id; 249 struct page *page; 250 struct zone *zone; 251 252 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; 253 page = virt_to_page(pgdat); 254 255 for (i = 0; i < nr_pages; i++, page++) 256 get_page_bootmem(node, page, NODE_INFO); 257 258 zone = &pgdat->node_zones[0]; 259 for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) { 260 if (zone_is_initialized(zone)) { 261 nr_pages = zone->wait_table_hash_nr_entries 262 * sizeof(wait_queue_head_t); 263 nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT; 264 page = virt_to_page(zone->wait_table); 265 266 for (i = 0; i < nr_pages; i++, page++) 267 get_page_bootmem(node, page, NODE_INFO); 268 } 269 } 270 271 pfn = pgdat->node_start_pfn; 272 end_pfn = pgdat_end_pfn(pgdat); 273 274 /* register section info */ 275 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 276 /* 277 * Some platforms can assign the same pfn to multiple nodes - on 278 * node0 as well as nodeN. To avoid registering a pfn against 279 * multiple nodes we check that this pfn does not already 280 * reside in some other nodes. 281 */ 282 if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node)) 283 register_page_bootmem_info_section(pfn); 284 } 285} 286#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ 287 288static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn, 289 unsigned long end_pfn) 290{ 291 unsigned long old_zone_end_pfn; 292 293 zone_span_writelock(zone); 294 295 old_zone_end_pfn = zone_end_pfn(zone); 296 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) 297 zone->zone_start_pfn = start_pfn; 298 299 zone->spanned_pages = max(old_zone_end_pfn, end_pfn) - 300 zone->zone_start_pfn; 301 302 zone_span_writeunlock(zone); 303} 304 305static void resize_zone(struct zone *zone, unsigned long start_pfn, 306 unsigned long end_pfn) 307{ 308 zone_span_writelock(zone); 309 310 if (end_pfn - start_pfn) { 311 zone->zone_start_pfn = start_pfn; 312 zone->spanned_pages = end_pfn - start_pfn; 313 } else { 314 /* 315 * make it consist as free_area_init_core(), 316 * if spanned_pages = 0, then keep start_pfn = 0 317 */ 318 zone->zone_start_pfn = 0; 319 zone->spanned_pages = 0; 320 } 321 322 zone_span_writeunlock(zone); 323} 324 325static void fix_zone_id(struct zone *zone, unsigned long start_pfn, 326 unsigned long end_pfn) 327{ 328 enum zone_type zid = zone_idx(zone); 329 int nid = zone->zone_pgdat->node_id; 330 unsigned long pfn; 331 332 for (pfn = start_pfn; pfn < end_pfn; pfn++) 333 set_page_links(pfn_to_page(pfn), zid, nid, pfn); 334} 335 336/* Can fail with -ENOMEM from allocating a wait table with vmalloc() or 337 * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */ 338static int __ref ensure_zone_is_initialized(struct zone *zone, 339 unsigned long start_pfn, unsigned long num_pages) 340{ 341 if (!zone_is_initialized(zone)) 342 return init_currently_empty_zone(zone, start_pfn, num_pages, 343 MEMMAP_HOTPLUG); 344 return 0; 345} 346 347static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2, 348 unsigned long start_pfn, unsigned long end_pfn) 349{ 350 int ret; 351 unsigned long flags; 352 unsigned long z1_start_pfn; 353 354 ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn); 355 if (ret) 356 return ret; 357 358 pgdat_resize_lock(z1->zone_pgdat, &flags); 359 360 /* can't move pfns which are higher than @z2 */ 361 if (end_pfn > zone_end_pfn(z2)) 362 goto out_fail; 363 /* the move out part must be at the left most of @z2 */ 364 if (start_pfn > z2->zone_start_pfn) 365 goto out_fail; 366 /* must included/overlap */ 367 if (end_pfn <= z2->zone_start_pfn) 368 goto out_fail; 369 370 /* use start_pfn for z1's start_pfn if z1 is empty */ 371 if (!zone_is_empty(z1)) 372 z1_start_pfn = z1->zone_start_pfn; 373 else 374 z1_start_pfn = start_pfn; 375 376 resize_zone(z1, z1_start_pfn, end_pfn); 377 resize_zone(z2, end_pfn, zone_end_pfn(z2)); 378 379 pgdat_resize_unlock(z1->zone_pgdat, &flags); 380 381 fix_zone_id(z1, start_pfn, end_pfn); 382 383 return 0; 384out_fail: 385 pgdat_resize_unlock(z1->zone_pgdat, &flags); 386 return -1; 387} 388 389static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2, 390 unsigned long start_pfn, unsigned long end_pfn) 391{ 392 int ret; 393 unsigned long flags; 394 unsigned long z2_end_pfn; 395 396 ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn); 397 if (ret) 398 return ret; 399 400 pgdat_resize_lock(z1->zone_pgdat, &flags); 401 402 /* can't move pfns which are lower than @z1 */ 403 if (z1->zone_start_pfn > start_pfn) 404 goto out_fail; 405 /* the move out part mast at the right most of @z1 */ 406 if (zone_end_pfn(z1) > end_pfn) 407 goto out_fail; 408 /* must included/overlap */ 409 if (start_pfn >= zone_end_pfn(z1)) 410 goto out_fail; 411 412 /* use end_pfn for z2's end_pfn if z2 is empty */ 413 if (!zone_is_empty(z2)) 414 z2_end_pfn = zone_end_pfn(z2); 415 else 416 z2_end_pfn = end_pfn; 417 418 resize_zone(z1, z1->zone_start_pfn, start_pfn); 419 resize_zone(z2, start_pfn, z2_end_pfn); 420 421 pgdat_resize_unlock(z1->zone_pgdat, &flags); 422 423 fix_zone_id(z2, start_pfn, end_pfn); 424 425 return 0; 426out_fail: 427 pgdat_resize_unlock(z1->zone_pgdat, &flags); 428 return -1; 429} 430 431static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, 432 unsigned long end_pfn) 433{ 434 unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat); 435 436 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) 437 pgdat->node_start_pfn = start_pfn; 438 439 pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) - 440 pgdat->node_start_pfn; 441} 442 443static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn) 444{ 445 struct pglist_data *pgdat = zone->zone_pgdat; 446 int nr_pages = PAGES_PER_SECTION; 447 int nid = pgdat->node_id; 448 int zone_type; 449 unsigned long flags; 450 int ret; 451 452 zone_type = zone - pgdat->node_zones; 453 ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages); 454 if (ret) 455 return ret; 456 457 pgdat_resize_lock(zone->zone_pgdat, &flags); 458 grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages); 459 grow_pgdat_span(zone->zone_pgdat, phys_start_pfn, 460 phys_start_pfn + nr_pages); 461 pgdat_resize_unlock(zone->zone_pgdat, &flags); 462 memmap_init_zone(nr_pages, nid, zone_type, 463 phys_start_pfn, MEMMAP_HOTPLUG); 464 return 0; 465} 466 467static int __meminit __add_section(int nid, struct zone *zone, 468 unsigned long phys_start_pfn) 469{ 470 int ret; 471 472 if (pfn_valid(phys_start_pfn)) 473 return -EEXIST; 474 475 ret = sparse_add_one_section(zone, phys_start_pfn); 476 477 if (ret < 0) 478 return ret; 479 480 ret = __add_zone(zone, phys_start_pfn); 481 482 if (ret < 0) 483 return ret; 484 485 return register_new_memory(nid, __pfn_to_section(phys_start_pfn)); 486} 487 488/* 489 * Reasonably generic function for adding memory. It is 490 * expected that archs that support memory hotplug will 491 * call this function after deciding the zone to which to 492 * add the new pages. 493 */ 494int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn, 495 unsigned long nr_pages) 496{ 497 unsigned long i; 498 int err = 0; 499 int start_sec, end_sec; 500 /* during initialize mem_map, align hot-added range to section */ 501 start_sec = pfn_to_section_nr(phys_start_pfn); 502 end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1); 503 504 for (i = start_sec; i <= end_sec; i++) { 505 err = __add_section(nid, zone, section_nr_to_pfn(i)); 506 507 /* 508 * EEXIST is finally dealt with by ioresource collision 509 * check. see add_memory() => register_memory_resource() 510 * Warning will be printed if there is collision. 511 */ 512 if (err && (err != -EEXIST)) 513 break; 514 err = 0; 515 } 516 517 return err; 518} 519EXPORT_SYMBOL_GPL(__add_pages); 520 521#ifdef CONFIG_MEMORY_HOTREMOVE 522/* find the smallest valid pfn in the range [start_pfn, end_pfn) */ 523static int find_smallest_section_pfn(int nid, struct zone *zone, 524 unsigned long start_pfn, 525 unsigned long end_pfn) 526{ 527 struct mem_section *ms; 528 529 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) { 530 ms = __pfn_to_section(start_pfn); 531 532 if (unlikely(!valid_section(ms))) 533 continue; 534 535 if (unlikely(pfn_to_nid(start_pfn) != nid)) 536 continue; 537 538 if (zone && zone != page_zone(pfn_to_page(start_pfn))) 539 continue; 540 541 return start_pfn; 542 } 543 544 return 0; 545} 546 547/* find the biggest valid pfn in the range [start_pfn, end_pfn). */ 548static int find_biggest_section_pfn(int nid, struct zone *zone, 549 unsigned long start_pfn, 550 unsigned long end_pfn) 551{ 552 struct mem_section *ms; 553 unsigned long pfn; 554 555 /* pfn is the end pfn of a memory section. */ 556 pfn = end_pfn - 1; 557 for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) { 558 ms = __pfn_to_section(pfn); 559 560 if (unlikely(!valid_section(ms))) 561 continue; 562 563 if (unlikely(pfn_to_nid(pfn) != nid)) 564 continue; 565 566 if (zone && zone != page_zone(pfn_to_page(pfn))) 567 continue; 568 569 return pfn; 570 } 571 572 return 0; 573} 574 575static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, 576 unsigned long end_pfn) 577{ 578 unsigned long zone_start_pfn = zone->zone_start_pfn; 579 unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */ 580 unsigned long zone_end_pfn = z; 581 unsigned long pfn; 582 struct mem_section *ms; 583 int nid = zone_to_nid(zone); 584 585 zone_span_writelock(zone); 586 if (zone_start_pfn == start_pfn) { 587 /* 588 * If the section is smallest section in the zone, it need 589 * shrink zone->zone_start_pfn and zone->zone_spanned_pages. 590 * In this case, we find second smallest valid mem_section 591 * for shrinking zone. 592 */ 593 pfn = find_smallest_section_pfn(nid, zone, end_pfn, 594 zone_end_pfn); 595 if (pfn) { 596 zone->zone_start_pfn = pfn; 597 zone->spanned_pages = zone_end_pfn - pfn; 598 } 599 } else if (zone_end_pfn == end_pfn) { 600 /* 601 * If the section is biggest section in the zone, it need 602 * shrink zone->spanned_pages. 603 * In this case, we find second biggest valid mem_section for 604 * shrinking zone. 605 */ 606 pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn, 607 start_pfn); 608 if (pfn) 609 zone->spanned_pages = pfn - zone_start_pfn + 1; 610 } 611 612 /* 613 * The section is not biggest or smallest mem_section in the zone, it 614 * only creates a hole in the zone. So in this case, we need not 615 * change the zone. But perhaps, the zone has only hole data. Thus 616 * it check the zone has only hole or not. 617 */ 618 pfn = zone_start_pfn; 619 for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) { 620 ms = __pfn_to_section(pfn); 621 622 if (unlikely(!valid_section(ms))) 623 continue; 624 625 if (page_zone(pfn_to_page(pfn)) != zone) 626 continue; 627 628 /* If the section is current section, it continues the loop */ 629 if (start_pfn == pfn) 630 continue; 631 632 /* If we find valid section, we have nothing to do */ 633 zone_span_writeunlock(zone); 634 return; 635 } 636 637 /* The zone has no valid section */ 638 zone->zone_start_pfn = 0; 639 zone->spanned_pages = 0; 640 zone_span_writeunlock(zone); 641} 642 643static void shrink_pgdat_span(struct pglist_data *pgdat, 644 unsigned long start_pfn, unsigned long end_pfn) 645{ 646 unsigned long pgdat_start_pfn = pgdat->node_start_pfn; 647 unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */ 648 unsigned long pgdat_end_pfn = p; 649 unsigned long pfn; 650 struct mem_section *ms; 651 int nid = pgdat->node_id; 652 653 if (pgdat_start_pfn == start_pfn) { 654 /* 655 * If the section is smallest section in the pgdat, it need 656 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages. 657 * In this case, we find second smallest valid mem_section 658 * for shrinking zone. 659 */ 660 pfn = find_smallest_section_pfn(nid, NULL, end_pfn, 661 pgdat_end_pfn); 662 if (pfn) { 663 pgdat->node_start_pfn = pfn; 664 pgdat->node_spanned_pages = pgdat_end_pfn - pfn; 665 } 666 } else if (pgdat_end_pfn == end_pfn) { 667 /* 668 * If the section is biggest section in the pgdat, it need 669 * shrink pgdat->node_spanned_pages. 670 * In this case, we find second biggest valid mem_section for 671 * shrinking zone. 672 */ 673 pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn, 674 start_pfn); 675 if (pfn) 676 pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1; 677 } 678 679 /* 680 * If the section is not biggest or smallest mem_section in the pgdat, 681 * it only creates a hole in the pgdat. So in this case, we need not 682 * change the pgdat. 683 * But perhaps, the pgdat has only hole data. Thus it check the pgdat 684 * has only hole or not. 685 */ 686 pfn = pgdat_start_pfn; 687 for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) { 688 ms = __pfn_to_section(pfn); 689 690 if (unlikely(!valid_section(ms))) 691 continue; 692 693 if (pfn_to_nid(pfn) != nid) 694 continue; 695 696 /* If the section is current section, it continues the loop */ 697 if (start_pfn == pfn) 698 continue; 699 700 /* If we find valid section, we have nothing to do */ 701 return; 702 } 703 704 /* The pgdat has no valid section */ 705 pgdat->node_start_pfn = 0; 706 pgdat->node_spanned_pages = 0; 707} 708 709static void __remove_zone(struct zone *zone, unsigned long start_pfn) 710{ 711 struct pglist_data *pgdat = zone->zone_pgdat; 712 int nr_pages = PAGES_PER_SECTION; 713 int zone_type; 714 unsigned long flags; 715 716 zone_type = zone - pgdat->node_zones; 717 718 pgdat_resize_lock(zone->zone_pgdat, &flags); 719 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); 720 shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages); 721 pgdat_resize_unlock(zone->zone_pgdat, &flags); 722} 723 724static int __remove_section(struct zone *zone, struct mem_section *ms) 725{ 726 unsigned long start_pfn; 727 int scn_nr; 728 int ret = -EINVAL; 729 730 if (!valid_section(ms)) 731 return ret; 732 733 ret = unregister_memory_section(ms); 734 if (ret) 735 return ret; 736 737 scn_nr = __section_nr(ms); 738 start_pfn = section_nr_to_pfn(scn_nr); 739 __remove_zone(zone, start_pfn); 740 741 sparse_remove_one_section(zone, ms); 742 return 0; 743} 744 745/** 746 * __remove_pages() - remove sections of pages from a zone 747 * @zone: zone from which pages need to be removed 748 * @phys_start_pfn: starting pageframe (must be aligned to start of a section) 749 * @nr_pages: number of pages to remove (must be multiple of section size) 750 * 751 * Generic helper function to remove section mappings and sysfs entries 752 * for the section of the memory we are removing. Caller needs to make 753 * sure that pages are marked reserved and zones are adjust properly by 754 * calling offline_pages(). 755 */ 756int __remove_pages(struct zone *zone, unsigned long phys_start_pfn, 757 unsigned long nr_pages) 758{ 759 unsigned long i; 760 int sections_to_remove; 761 resource_size_t start, size; 762 int ret = 0; 763 764 /* 765 * We can only remove entire sections 766 */ 767 BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK); 768 BUG_ON(nr_pages % PAGES_PER_SECTION); 769 770 start = phys_start_pfn << PAGE_SHIFT; 771 size = nr_pages * PAGE_SIZE; 772 ret = release_mem_region_adjustable(&iomem_resource, start, size); 773 if (ret) { 774 resource_size_t endres = start + size - 1; 775 776 pr_warn("Unable to release resource <%pa-%pa> (%d)\n", 777 &start, &endres, ret); 778 } 779 780 sections_to_remove = nr_pages / PAGES_PER_SECTION; 781 for (i = 0; i < sections_to_remove; i++) { 782 unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION; 783 ret = __remove_section(zone, __pfn_to_section(pfn)); 784 if (ret) 785 break; 786 } 787 return ret; 788} 789EXPORT_SYMBOL_GPL(__remove_pages); 790#endif /* CONFIG_MEMORY_HOTREMOVE */ 791 792int set_online_page_callback(online_page_callback_t callback) 793{ 794 int rc = -EINVAL; 795 796 get_online_mems(); 797 mutex_lock(&online_page_callback_lock); 798 799 if (online_page_callback == generic_online_page) { 800 online_page_callback = callback; 801 rc = 0; 802 } 803 804 mutex_unlock(&online_page_callback_lock); 805 put_online_mems(); 806 807 return rc; 808} 809EXPORT_SYMBOL_GPL(set_online_page_callback); 810 811int restore_online_page_callback(online_page_callback_t callback) 812{ 813 int rc = -EINVAL; 814 815 get_online_mems(); 816 mutex_lock(&online_page_callback_lock); 817 818 if (online_page_callback == callback) { 819 online_page_callback = generic_online_page; 820 rc = 0; 821 } 822 823 mutex_unlock(&online_page_callback_lock); 824 put_online_mems(); 825 826 return rc; 827} 828EXPORT_SYMBOL_GPL(restore_online_page_callback); 829 830void __online_page_set_limits(struct page *page) 831{ 832} 833EXPORT_SYMBOL_GPL(__online_page_set_limits); 834 835void __online_page_increment_counters(struct page *page) 836{ 837 adjust_managed_page_count(page, 1); 838} 839EXPORT_SYMBOL_GPL(__online_page_increment_counters); 840 841void __online_page_free(struct page *page) 842{ 843 __free_reserved_page(page); 844} 845EXPORT_SYMBOL_GPL(__online_page_free); 846 847static void generic_online_page(struct page *page) 848{ 849 __online_page_set_limits(page); 850 __online_page_increment_counters(page); 851 __online_page_free(page); 852} 853 854static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, 855 void *arg) 856{ 857 unsigned long i; 858 unsigned long onlined_pages = *(unsigned long *)arg; 859 struct page *page; 860 if (PageReserved(pfn_to_page(start_pfn))) 861 for (i = 0; i < nr_pages; i++) { 862 page = pfn_to_page(start_pfn + i); 863 (*online_page_callback)(page); 864 onlined_pages++; 865 } 866 *(unsigned long *)arg = onlined_pages; 867 return 0; 868} 869 870#ifdef CONFIG_MOVABLE_NODE 871/* 872 * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have 873 * normal memory. 874 */ 875static bool can_online_high_movable(struct zone *zone) 876{ 877 return true; 878} 879#else /* CONFIG_MOVABLE_NODE */ 880/* ensure every online node has NORMAL memory */ 881static bool can_online_high_movable(struct zone *zone) 882{ 883 return node_state(zone_to_nid(zone), N_NORMAL_MEMORY); 884} 885#endif /* CONFIG_MOVABLE_NODE */ 886 887/* check which state of node_states will be changed when online memory */ 888static void node_states_check_changes_online(unsigned long nr_pages, 889 struct zone *zone, struct memory_notify *arg) 890{ 891 int nid = zone_to_nid(zone); 892 enum zone_type zone_last = ZONE_NORMAL; 893 894 /* 895 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] 896 * contains nodes which have zones of 0...ZONE_NORMAL, 897 * set zone_last to ZONE_NORMAL. 898 * 899 * If we don't have HIGHMEM nor movable node, 900 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of 901 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. 902 */ 903 if (N_MEMORY == N_NORMAL_MEMORY) 904 zone_last = ZONE_MOVABLE; 905 906 /* 907 * if the memory to be online is in a zone of 0...zone_last, and 908 * the zones of 0...zone_last don't have memory before online, we will 909 * need to set the node to node_states[N_NORMAL_MEMORY] after 910 * the memory is online. 911 */ 912 if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY)) 913 arg->status_change_nid_normal = nid; 914 else 915 arg->status_change_nid_normal = -1; 916 917#ifdef CONFIG_HIGHMEM 918 /* 919 * If we have movable node, node_states[N_HIGH_MEMORY] 920 * contains nodes which have zones of 0...ZONE_HIGHMEM, 921 * set zone_last to ZONE_HIGHMEM. 922 * 923 * If we don't have movable node, node_states[N_NORMAL_MEMORY] 924 * contains nodes which have zones of 0...ZONE_MOVABLE, 925 * set zone_last to ZONE_MOVABLE. 926 */ 927 zone_last = ZONE_HIGHMEM; 928 if (N_MEMORY == N_HIGH_MEMORY) 929 zone_last = ZONE_MOVABLE; 930 931 if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY)) 932 arg->status_change_nid_high = nid; 933 else 934 arg->status_change_nid_high = -1; 935#else 936 arg->status_change_nid_high = arg->status_change_nid_normal; 937#endif 938 939 /* 940 * if the node don't have memory befor online, we will need to 941 * set the node to node_states[N_MEMORY] after the memory 942 * is online. 943 */ 944 if (!node_state(nid, N_MEMORY)) 945 arg->status_change_nid = nid; 946 else 947 arg->status_change_nid = -1; 948} 949 950static void node_states_set_node(int node, struct memory_notify *arg) 951{ 952 if (arg->status_change_nid_normal >= 0) 953 node_set_state(node, N_NORMAL_MEMORY); 954 955 if (arg->status_change_nid_high >= 0) 956 node_set_state(node, N_HIGH_MEMORY); 957 958 node_set_state(node, N_MEMORY); 959} 960 961 962/* Must be protected by mem_hotplug_begin() */ 963int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type) 964{ 965 unsigned long flags; 966 unsigned long onlined_pages = 0; 967 struct zone *zone; 968 int need_zonelists_rebuild = 0; 969 int nid; 970 int ret; 971 struct memory_notify arg; 972 973 /* 974 * This doesn't need a lock to do pfn_to_page(). 975 * The section can't be removed here because of the 976 * memory_block->state_mutex. 977 */ 978 zone = page_zone(pfn_to_page(pfn)); 979 980 if ((zone_idx(zone) > ZONE_NORMAL || 981 online_type == MMOP_ONLINE_MOVABLE) && 982 !can_online_high_movable(zone)) 983 return -EINVAL; 984 985 if (online_type == MMOP_ONLINE_KERNEL && 986 zone_idx(zone) == ZONE_MOVABLE) { 987 if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) 988 return -EINVAL; 989 } 990 if (online_type == MMOP_ONLINE_MOVABLE && 991 zone_idx(zone) == ZONE_MOVABLE - 1) { 992 if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) 993 return -EINVAL; 994 } 995 996 /* Previous code may changed the zone of the pfn range */ 997 zone = page_zone(pfn_to_page(pfn)); 998 999 arg.start_pfn = pfn; 1000 arg.nr_pages = nr_pages; 1001 node_states_check_changes_online(nr_pages, zone, &arg); 1002 1003 nid = pfn_to_nid(pfn); 1004 1005 ret = memory_notify(MEM_GOING_ONLINE, &arg); 1006 ret = notifier_to_errno(ret); 1007 if (ret) { 1008 memory_notify(MEM_CANCEL_ONLINE, &arg); 1009 return ret; 1010 } 1011 /* 1012 * If this zone is not populated, then it is not in zonelist. 1013 * This means the page allocator ignores this zone. 1014 * So, zonelist must be updated after online. 1015 */ 1016 mutex_lock(&zonelists_mutex); 1017 if (!populated_zone(zone)) { 1018 need_zonelists_rebuild = 1; 1019 build_all_zonelists(NULL, zone); 1020 } 1021 1022 ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages, 1023 online_pages_range); 1024 if (ret) { 1025 if (need_zonelists_rebuild) 1026 zone_pcp_reset(zone); 1027 mutex_unlock(&zonelists_mutex); 1028 printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n", 1029 (unsigned long long) pfn << PAGE_SHIFT, 1030 (((unsigned long long) pfn + nr_pages) 1031 << PAGE_SHIFT) - 1); 1032 memory_notify(MEM_CANCEL_ONLINE, &arg); 1033 return ret; 1034 } 1035 1036 zone->present_pages += onlined_pages; 1037 1038 pgdat_resize_lock(zone->zone_pgdat, &flags); 1039 zone->zone_pgdat->node_present_pages += onlined_pages; 1040 pgdat_resize_unlock(zone->zone_pgdat, &flags); 1041 1042 if (onlined_pages) { 1043 node_states_set_node(zone_to_nid(zone), &arg); 1044 if (need_zonelists_rebuild) 1045 build_all_zonelists(NULL, NULL); 1046 else 1047 zone_pcp_update(zone); 1048 } 1049 1050 mutex_unlock(&zonelists_mutex); 1051 1052 init_per_zone_wmark_min(); 1053 1054 if (onlined_pages) 1055 kswapd_run(zone_to_nid(zone)); 1056 1057 vm_total_pages = nr_free_pagecache_pages(); 1058 1059 writeback_set_ratelimit(); 1060 1061 if (onlined_pages) 1062 memory_notify(MEM_ONLINE, &arg); 1063 return 0; 1064} 1065#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ 1066 1067static void reset_node_present_pages(pg_data_t *pgdat) 1068{ 1069 struct zone *z; 1070 1071 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 1072 z->present_pages = 0; 1073 1074 pgdat->node_present_pages = 0; 1075} 1076 1077/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 1078static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start) 1079{ 1080 struct pglist_data *pgdat; 1081 unsigned long zones_size[MAX_NR_ZONES] = {0}; 1082 unsigned long zholes_size[MAX_NR_ZONES] = {0}; 1083 unsigned long start_pfn = PFN_DOWN(start); 1084 1085 pgdat = NODE_DATA(nid); 1086 if (!pgdat) { 1087 pgdat = arch_alloc_nodedata(nid); 1088 if (!pgdat) 1089 return NULL; 1090 1091 arch_refresh_nodedata(nid, pgdat); 1092 } else { 1093 /* Reset the nr_zones and classzone_idx to 0 before reuse */ 1094 pgdat->nr_zones = 0; 1095 pgdat->classzone_idx = 0; 1096 } 1097 1098 /* we can use NODE_DATA(nid) from here */ 1099 1100 /* init node's zones as empty zones, we don't have any present pages.*/ 1101 free_area_init_node(nid, zones_size, start_pfn, zholes_size); 1102 1103 /* 1104 * The node we allocated has no zone fallback lists. For avoiding 1105 * to access not-initialized zonelist, build here. 1106 */ 1107 mutex_lock(&zonelists_mutex); 1108 build_all_zonelists(pgdat, NULL); 1109 mutex_unlock(&zonelists_mutex); 1110 1111 /* 1112 * zone->managed_pages is set to an approximate value in 1113 * free_area_init_core(), which will cause 1114 * /sys/device/system/node/nodeX/meminfo has wrong data. 1115 * So reset it to 0 before any memory is onlined. 1116 */ 1117 reset_node_managed_pages(pgdat); 1118 1119 /* 1120 * When memory is hot-added, all the memory is in offline state. So 1121 * clear all zones' present_pages because they will be updated in 1122 * online_pages() and offline_pages(). 1123 */ 1124 reset_node_present_pages(pgdat); 1125 1126 return pgdat; 1127} 1128 1129static void rollback_node_hotadd(int nid, pg_data_t *pgdat) 1130{ 1131 arch_refresh_nodedata(nid, NULL); 1132 arch_free_nodedata(pgdat); 1133 return; 1134} 1135 1136 1137/** 1138 * try_online_node - online a node if offlined 1139 * 1140 * called by cpu_up() to online a node without onlined memory. 1141 */ 1142int try_online_node(int nid) 1143{ 1144 pg_data_t *pgdat; 1145 int ret; 1146 1147 if (node_online(nid)) 1148 return 0; 1149 1150 mem_hotplug_begin(); 1151 pgdat = hotadd_new_pgdat(nid, 0); 1152 if (!pgdat) { 1153 pr_err("Cannot online node %d due to NULL pgdat\n", nid); 1154 ret = -ENOMEM; 1155 goto out; 1156 } 1157 node_set_online(nid); 1158 ret = register_one_node(nid); 1159 BUG_ON(ret); 1160 1161 if (pgdat->node_zonelists->_zonerefs->zone == NULL) { 1162 mutex_lock(&zonelists_mutex); 1163 build_all_zonelists(NULL, NULL); 1164 mutex_unlock(&zonelists_mutex); 1165 } 1166 1167out: 1168 mem_hotplug_done(); 1169 return ret; 1170} 1171 1172static int check_hotplug_memory_range(u64 start, u64 size) 1173{ 1174 u64 start_pfn = PFN_DOWN(start); 1175 u64 nr_pages = size >> PAGE_SHIFT; 1176 1177 /* Memory range must be aligned with section */ 1178 if ((start_pfn & ~PAGE_SECTION_MASK) || 1179 (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) { 1180 pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n", 1181 (unsigned long long)start, 1182 (unsigned long long)size); 1183 return -EINVAL; 1184 } 1185 1186 return 0; 1187} 1188 1189/* 1190 * If movable zone has already been setup, newly added memory should be check. 1191 * If its address is higher than movable zone, it should be added as movable. 1192 * Without this check, movable zone may overlap with other zone. 1193 */ 1194static int should_add_memory_movable(int nid, u64 start, u64 size) 1195{ 1196 unsigned long start_pfn = start >> PAGE_SHIFT; 1197 pg_data_t *pgdat = NODE_DATA(nid); 1198 struct zone *movable_zone = pgdat->node_zones + ZONE_MOVABLE; 1199 1200 if (zone_is_empty(movable_zone)) 1201 return 0; 1202 1203 if (movable_zone->zone_start_pfn <= start_pfn) 1204 return 1; 1205 1206 return 0; 1207} 1208 1209int zone_for_memory(int nid, u64 start, u64 size, int zone_default) 1210{ 1211 if (should_add_memory_movable(nid, start, size)) 1212 return ZONE_MOVABLE; 1213 1214 return zone_default; 1215} 1216 1217/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 1218int __ref add_memory(int nid, u64 start, u64 size) 1219{ 1220 pg_data_t *pgdat = NULL; 1221 bool new_pgdat; 1222 bool new_node; 1223 struct resource *res; 1224 int ret; 1225 1226 ret = check_hotplug_memory_range(start, size); 1227 if (ret) 1228 return ret; 1229 1230 res = register_memory_resource(start, size); 1231 ret = -EEXIST; 1232 if (!res) 1233 return ret; 1234 1235 { /* Stupid hack to suppress address-never-null warning */ 1236 void *p = NODE_DATA(nid); 1237 new_pgdat = !p; 1238 } 1239 1240 mem_hotplug_begin(); 1241 1242 new_node = !node_online(nid); 1243 if (new_node) { 1244 pgdat = hotadd_new_pgdat(nid, start); 1245 ret = -ENOMEM; 1246 if (!pgdat) 1247 goto error; 1248 } 1249 1250 /* call arch's memory hotadd */ 1251 ret = arch_add_memory(nid, start, size); 1252 1253 if (ret < 0) 1254 goto error; 1255 1256 /* we online node here. we can't roll back from here. */ 1257 node_set_online(nid); 1258 1259 if (new_node) { 1260 ret = register_one_node(nid); 1261 /* 1262 * If sysfs file of new node can't create, cpu on the node 1263 * can't be hot-added. There is no rollback way now. 1264 * So, check by BUG_ON() to catch it reluctantly.. 1265 */ 1266 BUG_ON(ret); 1267 } 1268 1269 /* create new memmap entry */ 1270 firmware_map_add_hotplug(start, start + size, "System RAM"); 1271 1272 goto out; 1273 1274error: 1275 /* rollback pgdat allocation and others */ 1276 if (new_pgdat) 1277 rollback_node_hotadd(nid, pgdat); 1278 release_memory_resource(res); 1279 1280out: 1281 mem_hotplug_done(); 1282 return ret; 1283} 1284EXPORT_SYMBOL_GPL(add_memory); 1285 1286#ifdef CONFIG_MEMORY_HOTREMOVE 1287/* 1288 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy 1289 * set and the size of the free page is given by page_order(). Using this, 1290 * the function determines if the pageblock contains only free pages. 1291 * Due to buddy contraints, a free page at least the size of a pageblock will 1292 * be located at the start of the pageblock 1293 */ 1294static inline int pageblock_free(struct page *page) 1295{ 1296 return PageBuddy(page) && page_order(page) >= pageblock_order; 1297} 1298 1299/* Return the start of the next active pageblock after a given page */ 1300static struct page *next_active_pageblock(struct page *page) 1301{ 1302 /* Ensure the starting page is pageblock-aligned */ 1303 BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1)); 1304 1305 /* If the entire pageblock is free, move to the end of free page */ 1306 if (pageblock_free(page)) { 1307 int order; 1308 /* be careful. we don't have locks, page_order can be changed.*/ 1309 order = page_order(page); 1310 if ((order < MAX_ORDER) && (order >= pageblock_order)) 1311 return page + (1 << order); 1312 } 1313 1314 return page + pageblock_nr_pages; 1315} 1316 1317/* Checks if this range of memory is likely to be hot-removable. */ 1318int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) 1319{ 1320 struct page *page = pfn_to_page(start_pfn); 1321 struct page *end_page = page + nr_pages; 1322 1323 /* Check the starting page of each pageblock within the range */ 1324 for (; page < end_page; page = next_active_pageblock(page)) { 1325 if (!is_pageblock_removable_nolock(page)) 1326 return 0; 1327 cond_resched(); 1328 } 1329 1330 /* All pageblocks in the memory block are likely to be hot-removable */ 1331 return 1; 1332} 1333 1334/* 1335 * Confirm all pages in a range [start, end) is belongs to the same zone. 1336 */ 1337int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) 1338{ 1339 unsigned long pfn; 1340 struct zone *zone = NULL; 1341 struct page *page; 1342 int i; 1343 for (pfn = start_pfn; 1344 pfn < end_pfn; 1345 pfn += MAX_ORDER_NR_PAGES) { 1346 i = 0; 1347 /* This is just a CONFIG_HOLES_IN_ZONE check.*/ 1348 while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i)) 1349 i++; 1350 if (i == MAX_ORDER_NR_PAGES) 1351 continue; 1352 page = pfn_to_page(pfn + i); 1353 if (zone && page_zone(page) != zone) 1354 return 0; 1355 zone = page_zone(page); 1356 } 1357 return 1; 1358} 1359 1360/* 1361 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages 1362 * and hugepages). We scan pfn because it's much easier than scanning over 1363 * linked list. This function returns the pfn of the first found movable 1364 * page if it's found, otherwise 0. 1365 */ 1366static unsigned long scan_movable_pages(unsigned long start, unsigned long end) 1367{ 1368 unsigned long pfn; 1369 struct page *page; 1370 for (pfn = start; pfn < end; pfn++) { 1371 if (pfn_valid(pfn)) { 1372 page = pfn_to_page(pfn); 1373 if (PageLRU(page)) 1374 return pfn; 1375 if (PageHuge(page)) { 1376 if (page_huge_active(page)) 1377 return pfn; 1378 else 1379 pfn = round_up(pfn + 1, 1380 1 << compound_order(page)) - 1; 1381 } 1382 } 1383 } 1384 return 0; 1385} 1386 1387#define NR_OFFLINE_AT_ONCE_PAGES (256) 1388static int 1389do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) 1390{ 1391 unsigned long pfn; 1392 struct page *page; 1393 int move_pages = NR_OFFLINE_AT_ONCE_PAGES; 1394 int not_managed = 0; 1395 int ret = 0; 1396 LIST_HEAD(source); 1397 1398 for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) { 1399 if (!pfn_valid(pfn)) 1400 continue; 1401 page = pfn_to_page(pfn); 1402 1403 if (PageHuge(page)) { 1404 struct page *head = compound_head(page); 1405 pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1; 1406 if (compound_order(head) > PFN_SECTION_SHIFT) { 1407 ret = -EBUSY; 1408 break; 1409 } 1410 if (isolate_huge_page(page, &source)) 1411 move_pages -= 1 << compound_order(head); 1412 continue; 1413 } 1414 1415 if (!get_page_unless_zero(page)) 1416 continue; 1417 /* 1418 * We can skip free pages. And we can only deal with pages on 1419 * LRU. 1420 */ 1421 ret = isolate_lru_page(page); 1422 if (!ret) { /* Success */ 1423 put_page(page); 1424 list_add_tail(&page->lru, &source); 1425 move_pages--; 1426 inc_zone_page_state(page, NR_ISOLATED_ANON + 1427 page_is_file_cache(page)); 1428 1429 } else { 1430#ifdef CONFIG_DEBUG_VM 1431 printk(KERN_ALERT "removing pfn %lx from LRU failed\n", 1432 pfn); 1433 dump_page(page, "failed to remove from LRU"); 1434#endif 1435 put_page(page); 1436 /* Because we don't have big zone->lock. we should 1437 check this again here. */ 1438 if (page_count(page)) { 1439 not_managed++; 1440 ret = -EBUSY; 1441 break; 1442 } 1443 } 1444 } 1445 if (!list_empty(&source)) { 1446 if (not_managed) { 1447 putback_movable_pages(&source); 1448 goto out; 1449 } 1450 1451 /* 1452 * alloc_migrate_target should be improooooved!! 1453 * migrate_pages returns # of failed pages. 1454 */ 1455 ret = migrate_pages(&source, alloc_migrate_target, NULL, 0, 1456 MIGRATE_SYNC, MR_MEMORY_HOTPLUG); 1457 if (ret) 1458 putback_movable_pages(&source); 1459 } 1460out: 1461 return ret; 1462} 1463 1464/* 1465 * remove from free_area[] and mark all as Reserved. 1466 */ 1467static int 1468offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages, 1469 void *data) 1470{ 1471 __offline_isolated_pages(start, start + nr_pages); 1472 return 0; 1473} 1474 1475static void 1476offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 1477{ 1478 walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL, 1479 offline_isolated_pages_cb); 1480} 1481 1482/* 1483 * Check all pages in range, recoreded as memory resource, are isolated. 1484 */ 1485static int 1486check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages, 1487 void *data) 1488{ 1489 int ret; 1490 long offlined = *(long *)data; 1491 ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true); 1492 offlined = nr_pages; 1493 if (!ret) 1494 *(long *)data += offlined; 1495 return ret; 1496} 1497 1498static long 1499check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) 1500{ 1501 long offlined = 0; 1502 int ret; 1503 1504 ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined, 1505 check_pages_isolated_cb); 1506 if (ret < 0) 1507 offlined = (long)ret; 1508 return offlined; 1509} 1510 1511#ifdef CONFIG_MOVABLE_NODE 1512/* 1513 * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have 1514 * normal memory. 1515 */ 1516static bool can_offline_normal(struct zone *zone, unsigned long nr_pages) 1517{ 1518 return true; 1519} 1520#else /* CONFIG_MOVABLE_NODE */ 1521/* ensure the node has NORMAL memory if it is still online */ 1522static bool can_offline_normal(struct zone *zone, unsigned long nr_pages) 1523{ 1524 struct pglist_data *pgdat = zone->zone_pgdat; 1525 unsigned long present_pages = 0; 1526 enum zone_type zt; 1527 1528 for (zt = 0; zt <= ZONE_NORMAL; zt++) 1529 present_pages += pgdat->node_zones[zt].present_pages; 1530 1531 if (present_pages > nr_pages) 1532 return true; 1533 1534 present_pages = 0; 1535 for (; zt <= ZONE_MOVABLE; zt++) 1536 present_pages += pgdat->node_zones[zt].present_pages; 1537 1538 /* 1539 * we can't offline the last normal memory until all 1540 * higher memory is offlined. 1541 */ 1542 return present_pages == 0; 1543} 1544#endif /* CONFIG_MOVABLE_NODE */ 1545 1546static int __init cmdline_parse_movable_node(char *p) 1547{ 1548#ifdef CONFIG_MOVABLE_NODE 1549 /* 1550 * Memory used by the kernel cannot be hot-removed because Linux 1551 * cannot migrate the kernel pages. When memory hotplug is 1552 * enabled, we should prevent memblock from allocating memory 1553 * for the kernel. 1554 * 1555 * ACPI SRAT records all hotpluggable memory ranges. But before 1556 * SRAT is parsed, we don't know about it. 1557 * 1558 * The kernel image is loaded into memory at very early time. We 1559 * cannot prevent this anyway. So on NUMA system, we set any 1560 * node the kernel resides in as un-hotpluggable. 1561 * 1562 * Since on modern servers, one node could have double-digit 1563 * gigabytes memory, we can assume the memory around the kernel 1564 * image is also un-hotpluggable. So before SRAT is parsed, just 1565 * allocate memory near the kernel image to try the best to keep 1566 * the kernel away from hotpluggable memory. 1567 */ 1568 memblock_set_bottom_up(true); 1569 movable_node_enabled = true; 1570#else 1571 pr_warn("movable_node option not supported\n"); 1572#endif 1573 return 0; 1574} 1575early_param("movable_node", cmdline_parse_movable_node); 1576 1577/* check which state of node_states will be changed when offline memory */ 1578static void node_states_check_changes_offline(unsigned long nr_pages, 1579 struct zone *zone, struct memory_notify *arg) 1580{ 1581 struct pglist_data *pgdat = zone->zone_pgdat; 1582 unsigned long present_pages = 0; 1583 enum zone_type zt, zone_last = ZONE_NORMAL; 1584 1585 /* 1586 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] 1587 * contains nodes which have zones of 0...ZONE_NORMAL, 1588 * set zone_last to ZONE_NORMAL. 1589 * 1590 * If we don't have HIGHMEM nor movable node, 1591 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of 1592 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. 1593 */ 1594 if (N_MEMORY == N_NORMAL_MEMORY) 1595 zone_last = ZONE_MOVABLE; 1596 1597 /* 1598 * check whether node_states[N_NORMAL_MEMORY] will be changed. 1599 * If the memory to be offline is in a zone of 0...zone_last, 1600 * and it is the last present memory, 0...zone_last will 1601 * become empty after offline , thus we can determind we will 1602 * need to clear the node from node_states[N_NORMAL_MEMORY]. 1603 */ 1604 for (zt = 0; zt <= zone_last; zt++) 1605 present_pages += pgdat->node_zones[zt].present_pages; 1606 if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) 1607 arg->status_change_nid_normal = zone_to_nid(zone); 1608 else 1609 arg->status_change_nid_normal = -1; 1610 1611#ifdef CONFIG_HIGHMEM 1612 /* 1613 * If we have movable node, node_states[N_HIGH_MEMORY] 1614 * contains nodes which have zones of 0...ZONE_HIGHMEM, 1615 * set zone_last to ZONE_HIGHMEM. 1616 * 1617 * If we don't have movable node, node_states[N_NORMAL_MEMORY] 1618 * contains nodes which have zones of 0...ZONE_MOVABLE, 1619 * set zone_last to ZONE_MOVABLE. 1620 */ 1621 zone_last = ZONE_HIGHMEM; 1622 if (N_MEMORY == N_HIGH_MEMORY) 1623 zone_last = ZONE_MOVABLE; 1624 1625 for (; zt <= zone_last; zt++) 1626 present_pages += pgdat->node_zones[zt].present_pages; 1627 if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) 1628 arg->status_change_nid_high = zone_to_nid(zone); 1629 else 1630 arg->status_change_nid_high = -1; 1631#else 1632 arg->status_change_nid_high = arg->status_change_nid_normal; 1633#endif 1634 1635 /* 1636 * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE 1637 */ 1638 zone_last = ZONE_MOVABLE; 1639 1640 /* 1641 * check whether node_states[N_HIGH_MEMORY] will be changed 1642 * If we try to offline the last present @nr_pages from the node, 1643 * we can determind we will need to clear the node from 1644 * node_states[N_HIGH_MEMORY]. 1645 */ 1646 for (; zt <= zone_last; zt++) 1647 present_pages += pgdat->node_zones[zt].present_pages; 1648 if (nr_pages >= present_pages) 1649 arg->status_change_nid = zone_to_nid(zone); 1650 else 1651 arg->status_change_nid = -1; 1652} 1653 1654static void node_states_clear_node(int node, struct memory_notify *arg) 1655{ 1656 if (arg->status_change_nid_normal >= 0) 1657 node_clear_state(node, N_NORMAL_MEMORY); 1658 1659 if ((N_MEMORY != N_NORMAL_MEMORY) && 1660 (arg->status_change_nid_high >= 0)) 1661 node_clear_state(node, N_HIGH_MEMORY); 1662 1663 if ((N_MEMORY != N_HIGH_MEMORY) && 1664 (arg->status_change_nid >= 0)) 1665 node_clear_state(node, N_MEMORY); 1666} 1667 1668static int __ref __offline_pages(unsigned long start_pfn, 1669 unsigned long end_pfn, unsigned long timeout) 1670{ 1671 unsigned long pfn, nr_pages, expire; 1672 long offlined_pages; 1673 int ret, drain, retry_max, node; 1674 unsigned long flags; 1675 struct zone *zone; 1676 struct memory_notify arg; 1677 1678 /* at least, alignment against pageblock is necessary */ 1679 if (!IS_ALIGNED(start_pfn, pageblock_nr_pages)) 1680 return -EINVAL; 1681 if (!IS_ALIGNED(end_pfn, pageblock_nr_pages)) 1682 return -EINVAL; 1683 /* This makes hotplug much easier...and readable. 1684 we assume this for now. .*/ 1685 if (!test_pages_in_a_zone(start_pfn, end_pfn)) 1686 return -EINVAL; 1687 1688 zone = page_zone(pfn_to_page(start_pfn)); 1689 node = zone_to_nid(zone); 1690 nr_pages = end_pfn - start_pfn; 1691 1692 if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages)) 1693 return -EINVAL; 1694 1695 /* set above range as isolated */ 1696 ret = start_isolate_page_range(start_pfn, end_pfn, 1697 MIGRATE_MOVABLE, true); 1698 if (ret) 1699 return ret; 1700 1701 arg.start_pfn = start_pfn; 1702 arg.nr_pages = nr_pages; 1703 node_states_check_changes_offline(nr_pages, zone, &arg); 1704 1705 ret = memory_notify(MEM_GOING_OFFLINE, &arg); 1706 ret = notifier_to_errno(ret); 1707 if (ret) 1708 goto failed_removal; 1709 1710 pfn = start_pfn; 1711 expire = jiffies + timeout; 1712 drain = 0; 1713 retry_max = 5; 1714repeat: 1715 /* start memory hot removal */ 1716 ret = -EAGAIN; 1717 if (time_after(jiffies, expire)) 1718 goto failed_removal; 1719 ret = -EINTR; 1720 if (signal_pending(current)) 1721 goto failed_removal; 1722 ret = 0; 1723 if (drain) { 1724 lru_add_drain_all(); 1725 cond_resched(); 1726 drain_all_pages(zone); 1727 } 1728 1729 pfn = scan_movable_pages(start_pfn, end_pfn); 1730 if (pfn) { /* We have movable pages */ 1731 ret = do_migrate_range(pfn, end_pfn); 1732 if (!ret) { 1733 drain = 1; 1734 goto repeat; 1735 } else { 1736 if (ret < 0) 1737 if (--retry_max == 0) 1738 goto failed_removal; 1739 yield(); 1740 drain = 1; 1741 goto repeat; 1742 } 1743 } 1744 /* drain all zone's lru pagevec, this is asynchronous... */ 1745 lru_add_drain_all(); 1746 yield(); 1747 /* drain pcp pages, this is synchronous. */ 1748 drain_all_pages(zone); 1749 /* 1750 * dissolve free hugepages in the memory block before doing offlining 1751 * actually in order to make hugetlbfs's object counting consistent. 1752 */ 1753 dissolve_free_huge_pages(start_pfn, end_pfn); 1754 /* check again */ 1755 offlined_pages = check_pages_isolated(start_pfn, end_pfn); 1756 if (offlined_pages < 0) { 1757 ret = -EBUSY; 1758 goto failed_removal; 1759 } 1760 printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages); 1761 /* Ok, all of our target is isolated. 1762 We cannot do rollback at this point. */ 1763 offline_isolated_pages(start_pfn, end_pfn); 1764 /* reset pagetype flags and makes migrate type to be MOVABLE */ 1765 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1766 /* removal success */ 1767 adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages); 1768 zone->present_pages -= offlined_pages; 1769 1770 pgdat_resize_lock(zone->zone_pgdat, &flags); 1771 zone->zone_pgdat->node_present_pages -= offlined_pages; 1772 pgdat_resize_unlock(zone->zone_pgdat, &flags); 1773 1774 init_per_zone_wmark_min(); 1775 1776 if (!populated_zone(zone)) { 1777 zone_pcp_reset(zone); 1778 mutex_lock(&zonelists_mutex); 1779 build_all_zonelists(NULL, NULL); 1780 mutex_unlock(&zonelists_mutex); 1781 } else 1782 zone_pcp_update(zone); 1783 1784 node_states_clear_node(node, &arg); 1785 if (arg.status_change_nid >= 0) 1786 kswapd_stop(node); 1787 1788 vm_total_pages = nr_free_pagecache_pages(); 1789 writeback_set_ratelimit(); 1790 1791 memory_notify(MEM_OFFLINE, &arg); 1792 return 0; 1793 1794failed_removal: 1795 printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n", 1796 (unsigned long long) start_pfn << PAGE_SHIFT, 1797 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1); 1798 memory_notify(MEM_CANCEL_OFFLINE, &arg); 1799 /* pushback to free area */ 1800 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1801 return ret; 1802} 1803 1804/* Must be protected by mem_hotplug_begin() */ 1805int offline_pages(unsigned long start_pfn, unsigned long nr_pages) 1806{ 1807 return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ); 1808} 1809#endif /* CONFIG_MEMORY_HOTREMOVE */ 1810 1811/** 1812 * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn) 1813 * @start_pfn: start pfn of the memory range 1814 * @end_pfn: end pfn of the memory range 1815 * @arg: argument passed to func 1816 * @func: callback for each memory section walked 1817 * 1818 * This function walks through all present mem sections in range 1819 * [start_pfn, end_pfn) and call func on each mem section. 1820 * 1821 * Returns the return value of func. 1822 */ 1823int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn, 1824 void *arg, int (*func)(struct memory_block *, void *)) 1825{ 1826 struct memory_block *mem = NULL; 1827 struct mem_section *section; 1828 unsigned long pfn, section_nr; 1829 int ret; 1830 1831 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1832 section_nr = pfn_to_section_nr(pfn); 1833 if (!present_section_nr(section_nr)) 1834 continue; 1835 1836 section = __nr_to_section(section_nr); 1837 /* same memblock? */ 1838 if (mem) 1839 if ((section_nr >= mem->start_section_nr) && 1840 (section_nr <= mem->end_section_nr)) 1841 continue; 1842 1843 mem = find_memory_block_hinted(section, mem); 1844 if (!mem) 1845 continue; 1846 1847 ret = func(mem, arg); 1848 if (ret) { 1849 kobject_put(&mem->dev.kobj); 1850 return ret; 1851 } 1852 } 1853 1854 if (mem) 1855 kobject_put(&mem->dev.kobj); 1856 1857 return 0; 1858} 1859 1860#ifdef CONFIG_MEMORY_HOTREMOVE 1861static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) 1862{ 1863 int ret = !is_memblock_offlined(mem); 1864 1865 if (unlikely(ret)) { 1866 phys_addr_t beginpa, endpa; 1867 1868 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); 1869 endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1; 1870 pr_warn("removing memory fails, because memory " 1871 "[%pa-%pa] is onlined\n", 1872 &beginpa, &endpa); 1873 } 1874 1875 return ret; 1876} 1877 1878static int check_cpu_on_node(pg_data_t *pgdat) 1879{ 1880 int cpu; 1881 1882 for_each_present_cpu(cpu) { 1883 if (cpu_to_node(cpu) == pgdat->node_id) 1884 /* 1885 * the cpu on this node isn't removed, and we can't 1886 * offline this node. 1887 */ 1888 return -EBUSY; 1889 } 1890 1891 return 0; 1892} 1893 1894static void unmap_cpu_on_node(pg_data_t *pgdat) 1895{ 1896#ifdef CONFIG_ACPI_NUMA 1897 int cpu; 1898 1899 for_each_possible_cpu(cpu) 1900 if (cpu_to_node(cpu) == pgdat->node_id) 1901 numa_clear_node(cpu); 1902#endif 1903} 1904 1905static int check_and_unmap_cpu_on_node(pg_data_t *pgdat) 1906{ 1907 int ret; 1908 1909 ret = check_cpu_on_node(pgdat); 1910 if (ret) 1911 return ret; 1912 1913 /* 1914 * the node will be offlined when we come here, so we can clear 1915 * the cpu_to_node() now. 1916 */ 1917 1918 unmap_cpu_on_node(pgdat); 1919 return 0; 1920} 1921 1922/** 1923 * try_offline_node 1924 * 1925 * Offline a node if all memory sections and cpus of the node are removed. 1926 * 1927 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1928 * and online/offline operations before this call. 1929 */ 1930void try_offline_node(int nid) 1931{ 1932 pg_data_t *pgdat = NODE_DATA(nid); 1933 unsigned long start_pfn = pgdat->node_start_pfn; 1934 unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages; 1935 unsigned long pfn; 1936 int i; 1937 1938 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1939 unsigned long section_nr = pfn_to_section_nr(pfn); 1940 1941 if (!present_section_nr(section_nr)) 1942 continue; 1943 1944 if (pfn_to_nid(pfn) != nid) 1945 continue; 1946 1947 /* 1948 * some memory sections of this node are not removed, and we 1949 * can't offline node now. 1950 */ 1951 return; 1952 } 1953 1954 if (check_and_unmap_cpu_on_node(pgdat)) 1955 return; 1956 1957 /* 1958 * all memory/cpu of this node are removed, we can offline this 1959 * node now. 1960 */ 1961 node_set_offline(nid); 1962 unregister_one_node(nid); 1963 1964 /* free waittable in each zone */ 1965 for (i = 0; i < MAX_NR_ZONES; i++) { 1966 struct zone *zone = pgdat->node_zones + i; 1967 1968 /* 1969 * wait_table may be allocated from boot memory, 1970 * here only free if it's allocated by vmalloc. 1971 */ 1972 if (is_vmalloc_addr(zone->wait_table)) { 1973 vfree(zone->wait_table); 1974 zone->wait_table = NULL; 1975 } 1976 } 1977} 1978EXPORT_SYMBOL(try_offline_node); 1979 1980/** 1981 * remove_memory 1982 * 1983 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1984 * and online/offline operations before this call, as required by 1985 * try_offline_node(). 1986 */ 1987void __ref remove_memory(int nid, u64 start, u64 size) 1988{ 1989 int ret; 1990 1991 BUG_ON(check_hotplug_memory_range(start, size)); 1992 1993 mem_hotplug_begin(); 1994 1995 /* 1996 * All memory blocks must be offlined before removing memory. Check 1997 * whether all memory blocks in question are offline and trigger a BUG() 1998 * if this is not the case. 1999 */ 2000 ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL, 2001 check_memblock_offlined_cb); 2002 if (ret) 2003 BUG(); 2004 2005 /* remove memmap entry */ 2006 firmware_map_remove(start, start + size, "System RAM"); 2007 2008 arch_remove_memory(start, size); 2009 2010 try_offline_node(nid); 2011 2012 mem_hotplug_done(); 2013} 2014EXPORT_SYMBOL_GPL(remove_memory); 2015#endif /* CONFIG_MEMORY_HOTREMOVE */ 2016