299 	struct mem_zone_bm_rtree *zone;  member
357 static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,  in add_rtree_block()  argument
364 	block_nr = zone->blocks;  in add_rtree_block()
374 	for (i = zone->levels; i < levels_needed; i++) {  in add_rtree_block()
376 					&zone->nodes);  in add_rtree_block()
380 		node->data[0] = (unsigned long)zone->rtree;  in add_rtree_block()
381 		zone->rtree = node;  in add_rtree_block()
382 		zone->levels += 1;  in add_rtree_block()
386 	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);  in add_rtree_block()
391 	node = zone->rtree;  in add_rtree_block()
392 	dst = &zone->rtree;  in add_rtree_block()
393 	block_nr = zone->blocks;  in add_rtree_block()
394 	for (i = zone->levels; i > 0; i--) {  in add_rtree_block()
399 						&zone->nodes);  in add_rtree_block()
411 	zone->blocks += 1;  in add_rtree_block()
417 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
432 	struct mem_zone_bm_rtree *zone;  in create_zone_bm_rtree()  local
437 	zone  = chain_alloc(ca, sizeof(struct mem_zone_bm_rtree));  in create_zone_bm_rtree()
438 	if (!zone)  in create_zone_bm_rtree()
441 	INIT_LIST_HEAD(&zone->nodes);  in create_zone_bm_rtree()
442 	INIT_LIST_HEAD(&zone->leaves);  in create_zone_bm_rtree()
443 	zone->start_pfn = start;  in create_zone_bm_rtree()
444 	zone->end_pfn = end;  in create_zone_bm_rtree()
448 		if (add_rtree_block(zone, gfp_mask, safe_needed, ca)) {  in create_zone_bm_rtree()
449 			free_zone_bm_rtree(zone, PG_UNSAFE_CLEAR);  in create_zone_bm_rtree()
454 	return zone;  in create_zone_bm_rtree()
464 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,  in free_zone_bm_rtree()  argument
469 	list_for_each_entry(node, &zone->nodes, list)  in free_zone_bm_rtree()
472 	list_for_each_entry(node, &zone->leaves, list)  in free_zone_bm_rtree()
478 	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,  in memory_bm_position_reset()
480 	bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in memory_bm_position_reset()
516 	struct zone *zone;  in create_mem_extents()  local
520 	for_each_populated_zone(zone) {  in create_mem_extents()
524 		zone_start = zone->zone_start_pfn;  in create_mem_extents()
525 		zone_end = zone_end_pfn(zone);  in create_mem_extents()
586 		struct mem_zone_bm_rtree *zone;  in memory_bm_create()  local
588 		zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,  in memory_bm_create()
590 		if (!zone) {  in memory_bm_create()
594 		list_add_tail(&zone->list, &bm->zones);  in memory_bm_create()
614 	struct mem_zone_bm_rtree *zone;  in memory_bm_free()  local
616 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_free()
617 		free_zone_bm_rtree(zone, clear_nosave_free);  in memory_bm_free()
637 	struct mem_zone_bm_rtree *curr, *zone;  in memory_bm_find_bit()  local
641 	zone = bm->cur.zone;  in memory_bm_find_bit()
643 	if (pfn >= zone->start_pfn && pfn < zone->end_pfn)  in memory_bm_find_bit()
646 	zone = NULL;  in memory_bm_find_bit()
651 			zone = curr;  in memory_bm_find_bit()
656 	if (!zone)  in memory_bm_find_bit()
666 	if (((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)  in memory_bm_find_bit()
669 	node      = zone->rtree;  in memory_bm_find_bit()
670 	block_nr  = (pfn - zone->start_pfn) >> BM_BLOCK_SHIFT;  in memory_bm_find_bit()
672 	for (i = zone->levels; i > 0; i--) {  in memory_bm_find_bit()
683 	bm->cur.zone = zone;  in memory_bm_find_bit()
685 	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;  in memory_bm_find_bit()
689 	*bit_nr = (pfn - zone->start_pfn) & BM_BLOCK_MASK;  in memory_bm_find_bit()
770 	if (&bm->cur.node->list != &bm->cur.zone->leaves) {  in rtree_next_node()
778 	bm->cur.zone = list_entry(bm->cur.zone->list.next,  in rtree_next_node()
780 	if (&bm->cur.zone->list != &bm->zones) {  in rtree_next_node()
781 		bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in rtree_next_node()
808 		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;  in memory_bm_next_pfn()
813 			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;  in memory_bm_next_pfn()
1041 unsigned int snapshot_additional_pages(struct zone *zone)  in snapshot_additional_pages()  argument
1045 	rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);  in snapshot_additional_pages()
1064 	struct zone *zone;  in count_free_highmem_pages()  local
1067 	for_each_populated_zone(zone)  in count_free_highmem_pages()
1068 		if (is_highmem(zone))  in count_free_highmem_pages()
1069 			cnt += zone_page_state(zone, NR_FREE_PAGES);  in count_free_highmem_pages()
1081 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)  in saveable_highmem_page()  argument
1089 	if (page_zone(page) != zone)  in saveable_highmem_page()
1111 	struct zone *zone;  in count_highmem_pages()  local
1114 	for_each_populated_zone(zone) {  in count_highmem_pages()
1117 		if (!is_highmem(zone))  in count_highmem_pages()
1120 		mark_free_pages(zone);  in count_highmem_pages()
1121 		max_zone_pfn = zone_end_pfn(zone);  in count_highmem_pages()
1122 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_highmem_pages()
1123 			if (saveable_highmem_page(zone, pfn))  in count_highmem_pages()
1129 static inline void *saveable_highmem_page(struct zone *z, unsigned long p)  in saveable_highmem_page()
1143 static struct page *saveable_page(struct zone *zone, unsigned long pfn)  in saveable_page()  argument
1151 	if (page_zone(page) != zone)  in saveable_page()
1176 	struct zone *zone;  in count_data_pages()  local
1180 	for_each_populated_zone(zone) {  in count_data_pages()
1181 		if (is_highmem(zone))  in count_data_pages()
1184 		mark_free_pages(zone);  in count_data_pages()
1185 		max_zone_pfn = zone_end_pfn(zone);  in count_data_pages()
1186 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_data_pages()
1187 			if (saveable_page(zone, pfn))  in count_data_pages()
1225 page_is_saveable(struct zone *zone, unsigned long pfn)  in page_is_saveable()  argument
1227 	return is_highmem(zone) ?  in page_is_saveable()
1228 		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);  in page_is_saveable()
1259 #define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)  argument
1271 	struct zone *zone;  in copy_data_pages()  local
1274 	for_each_populated_zone(zone) {  in copy_data_pages()
1277 		mark_free_pages(zone);  in copy_data_pages()
1278 		max_zone_pfn = zone_end_pfn(zone);  in copy_data_pages()
1279 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in copy_data_pages()
1280 			if (page_is_saveable(zone, pfn))  in copy_data_pages()
1561 	struct zone *zone;  in hibernate_preallocate_memory()  local
1593 	for_each_populated_zone(zone) {  in hibernate_preallocate_memory()
1594 		size += snapshot_additional_pages(zone);  in hibernate_preallocate_memory()
1595 		if (is_highmem(zone))  in hibernate_preallocate_memory()
1596 			highmem += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1598 			count += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1738 	struct zone *zone;  in enough_free_mem()  local
1741 	for_each_populated_zone(zone)  in enough_free_mem()
1742 		if (!is_highmem(zone))  in enough_free_mem()
1743 			free += zone_page_state(zone, NR_FREE_PAGES);  in enough_free_mem()
2010 	struct zone *zone;  in mark_unsafe_pages()  local
2014 	for_each_populated_zone(zone) {  in mark_unsafe_pages()
2015 		max_zone_pfn = zone_end_pfn(zone);  in mark_unsafe_pages()
2016 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in mark_unsafe_pages()