103 				unsigned long end_pfn, struct zone *zone)  in pageblock_pfn_to_page()  argument
116 	if (page_zone(start_page) != zone)  in pageblock_pfn_to_page()
138 void defer_compaction(struct zone *zone, int order)  in defer_compaction()  argument
140 	zone->compact_considered = 0;  in defer_compaction()
141 	zone->compact_defer_shift++;  in defer_compaction()
143 	if (order < zone->compact_order_failed)  in defer_compaction()
144 		zone->compact_order_failed = order;  in defer_compaction()
146 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)  in defer_compaction()
147 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;  in defer_compaction()
149 	trace_mm_compaction_defer_compaction(zone, order);  in defer_compaction()
153 bool compaction_deferred(struct zone *zone, int order)  in compaction_deferred()  argument
155 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;  in compaction_deferred()
157 	if (order < zone->compact_order_failed)  in compaction_deferred()
161 	if (++zone->compact_considered > defer_limit)  in compaction_deferred()
162 		zone->compact_considered = defer_limit;  in compaction_deferred()
164 	if (zone->compact_considered >= defer_limit)  in compaction_deferred()
167 	trace_mm_compaction_deferred(zone, order);  in compaction_deferred()
177 void compaction_defer_reset(struct zone *zone, int order,  in compaction_defer_reset()  argument
181 		zone->compact_considered = 0;  in compaction_defer_reset()
182 		zone->compact_defer_shift = 0;  in compaction_defer_reset()
184 	if (order >= zone->compact_order_failed)  in compaction_defer_reset()
185 		zone->compact_order_failed = order + 1;  in compaction_defer_reset()
187 	trace_mm_compaction_defer_reset(zone, order);  in compaction_defer_reset()
191 bool compaction_restarting(struct zone *zone, int order)  in compaction_restarting()  argument
193 	if (order < zone->compact_order_failed)  in compaction_restarting()
196 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&  in compaction_restarting()
197 		zone->compact_considered >= 1UL << zone->compact_defer_shift;  in compaction_restarting()
215 static void __reset_isolation_suitable(struct zone *zone)  in __reset_isolation_suitable()  argument
217 	unsigned long start_pfn = zone->zone_start_pfn;  in __reset_isolation_suitable()
218 	unsigned long end_pfn = zone_end_pfn(zone);  in __reset_isolation_suitable()
221 	zone->compact_cached_migrate_pfn[0] = start_pfn;  in __reset_isolation_suitable()
222 	zone->compact_cached_migrate_pfn[1] = start_pfn;  in __reset_isolation_suitable()
223 	zone->compact_cached_free_pfn = end_pfn;  in __reset_isolation_suitable()
224 	zone->compact_blockskip_flush = false;  in __reset_isolation_suitable()
236 		if (zone != page_zone(page))  in __reset_isolation_suitable()
248 		struct zone *zone = &pgdat->node_zones[zoneid];  in reset_isolation_suitable()  local
249 		if (!populated_zone(zone))  in reset_isolation_suitable()
253 		if (zone->compact_blockskip_flush)  in reset_isolation_suitable()
254 			__reset_isolation_suitable(zone);  in reset_isolation_suitable()
266 	struct zone *zone = cc->zone;  in update_pageblock_skip()  local
284 		if (pfn > zone->compact_cached_migrate_pfn[0])  in update_pageblock_skip()
285 			zone->compact_cached_migrate_pfn[0] = pfn;  in update_pageblock_skip()
287 		    pfn > zone->compact_cached_migrate_pfn[1])  in update_pageblock_skip()
288 			zone->compact_cached_migrate_pfn[1] = pfn;  in update_pageblock_skip()
290 		if (pfn < zone->compact_cached_free_pfn)  in update_pageblock_skip()
291 			zone->compact_cached_free_pfn = pfn;  in update_pageblock_skip()
424 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
453 			locked = compact_trylock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
508 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
560 		if (!pageblock_pfn_to_page(pfn, block_end_pfn, cc->zone))  in isolate_freepages_range()
595 static void acct_isolated(struct zone *zone, struct compact_control *cc)  in acct_isolated()  argument
606 	mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]);  in acct_isolated()
607 	mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]);  in acct_isolated()
611 static bool too_many_isolated(struct zone *zone)  in too_many_isolated()  argument
615 	inactive = zone_page_state(zone, NR_INACTIVE_FILE) +  in too_many_isolated()
616 					zone_page_state(zone, NR_INACTIVE_ANON);  in too_many_isolated()
617 	active = zone_page_state(zone, NR_ACTIVE_FILE) +  in too_many_isolated()
618 					zone_page_state(zone, NR_ACTIVE_ANON);  in too_many_isolated()
619 	isolated = zone_page_state(zone, NR_ISOLATED_FILE) +  in too_many_isolated()
620 					zone_page_state(zone, NR_ISOLATED_ANON);  in too_many_isolated()
647 	struct zone *zone = cc->zone;  in isolate_migratepages_block()  local
661 	while (unlikely(too_many_isolated(zone))) {  in isolate_migratepages_block()
683 		    && compact_unlock_should_abort(&zone->lru_lock, flags,  in isolate_migratepages_block()
761 			locked = compact_trylock_irqsave(&zone->lru_lock,  in isolate_migratepages_block()
775 		lruvec = mem_cgroup_page_lruvec(page, zone);  in isolate_migratepages_block()
806 		spin_unlock_irqrestore(&zone->lru_lock, flags);  in isolate_migratepages_block()
850 		if (!pageblock_pfn_to_page(pfn, block_end_pfn, cc->zone))  in isolate_migratepages_range()
862 	acct_isolated(cc->zone, cc);  in isolate_migratepages_range()
898 	struct zone *zone = cc->zone;  in isolate_freepages()  local
920 						zone_end_pfn(zone));  in isolate_freepages()
944 									zone);  in isolate_freepages()
1053 static isolate_migrate_t isolate_migratepages(struct zone *zone,  in isolate_migratepages()  argument
1087 		page = pageblock_pfn_to_page(low_pfn, end_pfn, zone);  in isolate_migratepages()
1109 			acct_isolated(zone, cc);  in isolate_migratepages()
1121 	acct_isolated(zone, cc);  in isolate_migratepages()
1132 static int __compact_finished(struct zone *zone, struct compact_control *cc,  in __compact_finished()  argument
1144 		zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;  in __compact_finished()
1145 		zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;  in __compact_finished()
1146 		zone->compact_cached_free_pfn = zone_end_pfn(zone);  in __compact_finished()
1155 			zone->compact_blockskip_flush = true;  in __compact_finished()
1168 	watermark = low_wmark_pages(zone);  in __compact_finished()
1170 	if (!zone_watermark_ok(zone, cc->order, watermark, cc->classzone_idx,  in __compact_finished()
1176 		struct free_area *area = &zone->free_area[order];  in __compact_finished()
1201 static int compact_finished(struct zone *zone, struct compact_control *cc,  in compact_finished()  argument
1206 	ret = __compact_finished(zone, cc, migratetype);  in compact_finished()
1207 	trace_mm_compaction_finished(zone, cc->order, ret);  in compact_finished()
1221 static unsigned long __compaction_suitable(struct zone *zone, int order,  in __compaction_suitable()  argument
1234 	watermark = low_wmark_pages(zone);  in __compaction_suitable()
1239 	if (zone_watermark_ok(zone, order, watermark, classzone_idx,  in __compaction_suitable()
1249 	if (!zone_watermark_ok(zone, 0, watermark, classzone_idx, alloc_flags))  in __compaction_suitable()
1263 	fragindex = fragmentation_index(zone, order);  in __compaction_suitable()
1270 unsigned long compaction_suitable(struct zone *zone, int order,  in compaction_suitable()  argument
1275 	ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx);  in compaction_suitable()
1276 	trace_mm_compaction_suitable(zone, order, ret);  in compaction_suitable()
1283 static int compact_zone(struct zone *zone, struct compact_control *cc)  in compact_zone()  argument
1286 	unsigned long start_pfn = zone->zone_start_pfn;  in compact_zone()
1287 	unsigned long end_pfn = zone_end_pfn(zone);  in compact_zone()
1292 	ret = compaction_suitable(zone, cc->order, cc->alloc_flags,  in compact_zone()
1309 	if (compaction_restarting(zone, cc->order) && !current_is_kswapd())  in compact_zone()
1310 		__reset_isolation_suitable(zone);  in compact_zone()
1317 	cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync];  in compact_zone()
1318 	cc->free_pfn = zone->compact_cached_free_pfn;  in compact_zone()
1321 		zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
1325 		zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
1326 		zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
1334 	while ((ret = compact_finished(zone, cc, migratetype)) ==  in compact_zone()
1339 		switch (isolate_migratepages(zone, cc)) {  in compact_zone()
1403 				drain_local_pages(zone);  in compact_zone()
1428 		if (free_pfn > zone->compact_cached_free_pfn)  in compact_zone()
1429 			zone->compact_cached_free_pfn = free_pfn;  in compact_zone()
1438 static unsigned long compact_zone_order(struct zone *zone, int order,  in compact_zone_order()  argument
1448 		.zone = zone,  in compact_zone_order()
1456 	ret = compact_zone(zone, &cc);  in compact_zone_order()
1486 	struct zone *zone;  in try_to_compact_pages()  local
1499 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,  in try_to_compact_pages()
1504 		if (compaction_deferred(zone, order))  in try_to_compact_pages()
1507 		status = compact_zone_order(zone, order, gfp_mask, mode,  in try_to_compact_pages()
1518 		if (zone_watermark_ok(zone, order, low_wmark_pages(zone),  in try_to_compact_pages()
1526 			compaction_defer_reset(zone, order, false);  in try_to_compact_pages()
1547 			defer_compaction(zone, order);  in try_to_compact_pages()
1587 	struct zone *zone;  in __compact_pgdat()  local
1591 		zone = &pgdat->node_zones[zoneid];  in __compact_pgdat()
1592 		if (!populated_zone(zone))  in __compact_pgdat()
1597 		cc->zone = zone;  in __compact_pgdat()
1607 			__reset_isolation_suitable(zone);  in __compact_pgdat()
1609 		if (cc->order == -1 || !compaction_deferred(zone, cc->order))  in __compact_pgdat()
1610 			compact_zone(zone, cc);  in __compact_pgdat()
1613 			if (zone_watermark_ok(zone, cc->order,  in __compact_pgdat()
1614 						low_wmark_pages(zone), 0, 0))  in __compact_pgdat()
1615 				compaction_defer_reset(zone, cc->order, false);  in __compact_pgdat()