250 	struct zone *zone;  in register_page_bootmem_info_node()  local
258 	zone = &pgdat->node_zones[0];  in register_page_bootmem_info_node()
259 	for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) {  in register_page_bootmem_info_node()
260 		if (zone_is_initialized(zone)) {  in register_page_bootmem_info_node()
261 			nr_pages = zone->wait_table_hash_nr_entries  in register_page_bootmem_info_node()
264 			page = virt_to_page(zone->wait_table);  in register_page_bootmem_info_node()
288 static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn,  in grow_zone_span()  argument
293 	zone_span_writelock(zone);  in grow_zone_span()
295 	old_zone_end_pfn = zone_end_pfn(zone);  in grow_zone_span()
296 	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)  in grow_zone_span()
297 		zone->zone_start_pfn = start_pfn;  in grow_zone_span()
299 	zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -  in grow_zone_span()
300 				zone->zone_start_pfn;  in grow_zone_span()
302 	zone_span_writeunlock(zone);  in grow_zone_span()
305 static void resize_zone(struct zone *zone, unsigned long start_pfn,  in resize_zone()  argument
308 	zone_span_writelock(zone);  in resize_zone()
311 		zone->zone_start_pfn = start_pfn;  in resize_zone()
312 		zone->spanned_pages = end_pfn - start_pfn;  in resize_zone()
318 		zone->zone_start_pfn = 0;  in resize_zone()
319 		zone->spanned_pages = 0;  in resize_zone()
322 	zone_span_writeunlock(zone);  in resize_zone()
325 static void fix_zone_id(struct zone *zone, unsigned long start_pfn,  in fix_zone_id()  argument
328 	enum zone_type zid = zone_idx(zone);  in fix_zone_id()
329 	int nid = zone->zone_pgdat->node_id;  in fix_zone_id()
338 static int __ref ensure_zone_is_initialized(struct zone *zone,  in ensure_zone_is_initialized()  argument
341 	if (!zone_is_initialized(zone))  in ensure_zone_is_initialized()
342 		return init_currently_empty_zone(zone, start_pfn, num_pages);  in ensure_zone_is_initialized()
347 static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,  in move_pfn_range_left()
389 static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,  in move_pfn_range_right()
443 static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)  in __add_zone()  argument
445 	struct pglist_data *pgdat = zone->zone_pgdat;  in __add_zone()
452 	zone_type = zone - pgdat->node_zones;  in __add_zone()
453 	ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages);  in __add_zone()
457 	pgdat_resize_lock(zone->zone_pgdat, &flags);  in __add_zone()
458 	grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);  in __add_zone()
459 	grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,  in __add_zone()
461 	pgdat_resize_unlock(zone->zone_pgdat, &flags);  in __add_zone()
475 static int __meminit __add_section(int nid, struct zone *zone,  in __add_section()  argument
483 	ret = sparse_add_one_section(zone, phys_start_pfn);  in __add_section()
488 	ret = __add_zone(zone, phys_start_pfn);  in __add_section()
502 int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,  in __add_pages()  argument
513 		err = __add_section(nid, zone, section_nr_to_pfn(i));  in __add_pages()
532 static int find_smallest_section_pfn(int nid, struct zone *zone,  in find_smallest_section_pfn()  argument
547 		if (zone && zone != page_zone(pfn_to_page(start_pfn)))  in find_smallest_section_pfn()
557 static int find_biggest_section_pfn(int nid, struct zone *zone,  in find_biggest_section_pfn()  argument
575 		if (zone && zone != page_zone(pfn_to_page(pfn)))  in find_biggest_section_pfn()
584 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,  in shrink_zone_span()  argument
587 	unsigned long zone_start_pfn = zone->zone_start_pfn;  in shrink_zone_span()
588 	unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */  in shrink_zone_span()
592 	int nid = zone_to_nid(zone);  in shrink_zone_span()
594 	zone_span_writelock(zone);  in shrink_zone_span()
602 		pfn = find_smallest_section_pfn(nid, zone, end_pfn,  in shrink_zone_span()
605 			zone->zone_start_pfn = pfn;  in shrink_zone_span()
606 			zone->spanned_pages = zone_end_pfn - pfn;  in shrink_zone_span()
615 		pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,  in shrink_zone_span()
618 			zone->spanned_pages = pfn - zone_start_pfn + 1;  in shrink_zone_span()
634 		if (page_zone(pfn_to_page(pfn)) != zone)  in shrink_zone_span()
642 		zone_span_writeunlock(zone);  in shrink_zone_span()
647 	zone->zone_start_pfn = 0;  in shrink_zone_span()
648 	zone->spanned_pages = 0;  in shrink_zone_span()
649 	zone_span_writeunlock(zone);  in shrink_zone_span()
718 static void __remove_zone(struct zone *zone, unsigned long start_pfn)  in __remove_zone()  argument
720 	struct pglist_data *pgdat = zone->zone_pgdat;  in __remove_zone()
725 	zone_type = zone - pgdat->node_zones;  in __remove_zone()
727 	pgdat_resize_lock(zone->zone_pgdat, &flags);  in __remove_zone()
728 	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);  in __remove_zone()
730 	pgdat_resize_unlock(zone->zone_pgdat, &flags);  in __remove_zone()
733 static int __remove_section(struct zone *zone, struct mem_section *ms)  in __remove_section()  argument
748 	__remove_zone(zone, start_pfn);  in __remove_section()
750 	sparse_remove_one_section(zone, ms);  in __remove_section()
765 int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,  in __remove_pages()  argument
783 	if (!is_dev_zone(zone))  in __remove_pages()
795 		ret = __remove_section(zone, __pfn_to_section(pfn));  in __remove_pages()
887 static bool can_online_high_movable(struct zone *zone)  in can_online_high_movable()  argument
893 static bool can_online_high_movable(struct zone *zone)  in can_online_high_movable()  argument
895 	return node_state(zone_to_nid(zone), N_NORMAL_MEMORY);  in can_online_high_movable()
901 	struct zone *zone, struct memory_notify *arg)  in node_states_check_changes_online()  argument
903 	int nid = zone_to_nid(zone);  in node_states_check_changes_online()
924 	if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY))  in node_states_check_changes_online()
943 	if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY))  in node_states_check_changes_online()
979 	struct zone *zone;  in online_pages()  local
990 	zone = page_zone(pfn_to_page(pfn));  in online_pages()
992 	if ((zone_idx(zone) > ZONE_NORMAL ||  in online_pages()
994 	    !can_online_high_movable(zone))  in online_pages()
998 	    zone_idx(zone) == ZONE_MOVABLE) {  in online_pages()
999 		if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages))  in online_pages()
1003 	    zone_idx(zone) == ZONE_MOVABLE - 1) {  in online_pages()
1004 		if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages))  in online_pages()
1009 	zone = page_zone(pfn_to_page(pfn));  in online_pages()
1013 	node_states_check_changes_online(nr_pages, zone, &arg);  in online_pages()
1029 	if (!populated_zone(zone)) {  in online_pages()
1031 		build_all_zonelists(NULL, zone);  in online_pages()
1038 			zone_pcp_reset(zone);  in online_pages()
1048 	zone->present_pages += onlined_pages;  in online_pages()
1050 	pgdat_resize_lock(zone->zone_pgdat, &flags);  in online_pages()
1051 	zone->zone_pgdat->node_present_pages += onlined_pages;  in online_pages()
1052 	pgdat_resize_unlock(zone->zone_pgdat, &flags);  in online_pages()
1055 		node_states_set_node(zone_to_nid(zone), &arg);  in online_pages()
1059 			zone_pcp_update(zone);  in online_pages()
1067 		kswapd_run(zone_to_nid(zone));  in online_pages()
1081 	struct zone *z;  in reset_node_present_pages()
1173 	if (pgdat->node_zonelists->_zonerefs->zone == NULL) {  in try_online_node()
1210 	struct zone *movable_zone = pgdat->node_zones + ZONE_MOVABLE;  in should_add_memory_movable()
1379 	struct zone *zone = NULL;  in test_pages_in_a_zone()  local
1398 			if (zone && page_zone(page) != zone)  in test_pages_in_a_zone()
1400 			zone = page_zone(page);  in test_pages_in_a_zone()
1562 static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)  in can_offline_normal()  argument
1568 static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)  in can_offline_normal()  argument
1570 	struct pglist_data *pgdat = zone->zone_pgdat;  in can_offline_normal()
1625 		struct zone *zone, struct memory_notify *arg)  in node_states_check_changes_offline()  argument
1627 	struct pglist_data *pgdat = zone->zone_pgdat;  in node_states_check_changes_offline()
1652 	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)  in node_states_check_changes_offline()
1653 		arg->status_change_nid_normal = zone_to_nid(zone);  in node_states_check_changes_offline()
1673 	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)  in node_states_check_changes_offline()
1674 		arg->status_change_nid_high = zone_to_nid(zone);  in node_states_check_changes_offline()
1695 		arg->status_change_nid = zone_to_nid(zone);  in node_states_check_changes_offline()
1721 	struct zone *zone;  in __offline_pages()  local
1734 	zone = page_zone(pfn_to_page(start_pfn));  in __offline_pages()
1735 	node = zone_to_nid(zone);  in __offline_pages()
1738 	if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages))  in __offline_pages()
1749 	node_states_check_changes_offline(nr_pages, zone, &arg);  in __offline_pages()
1772 		drain_all_pages(zone);  in __offline_pages()
1794 	drain_all_pages(zone);  in __offline_pages()
1814 	zone->present_pages -= offlined_pages;  in __offline_pages()
1816 	pgdat_resize_lock(zone->zone_pgdat, &flags);  in __offline_pages()
1817 	zone->zone_pgdat->node_present_pages -= offlined_pages;  in __offline_pages()
1818 	pgdat_resize_unlock(zone->zone_pgdat, &flags);  in __offline_pages()
1822 	if (!populated_zone(zone)) {  in __offline_pages()
1823 		zone_pcp_reset(zone);  in __offline_pages()
1828 		zone_pcp_update(zone);  in __offline_pages()
2012 		struct zone *zone = pgdat->node_zones + i;  in try_offline_node()  local
2018 		if (is_vmalloc_addr(zone->wait_table)) {  in try_offline_node()
2019 			vfree(zone->wait_table);  in try_offline_node()
2020 			zone->wait_table = NULL;  in try_offline_node()