92 				unsigned long end_pfn, struct zone *zone)  in pageblock_pfn_to_page()  argument
105 	if (page_zone(start_page) != zone)  in pageblock_pfn_to_page()
127 void defer_compaction(struct zone *zone, int order)  in defer_compaction()  argument
129 	zone->compact_considered = 0;  in defer_compaction()
130 	zone->compact_defer_shift++;  in defer_compaction()
132 	if (order < zone->compact_order_failed)  in defer_compaction()
133 		zone->compact_order_failed = order;  in defer_compaction()
135 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)  in defer_compaction()
136 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;  in defer_compaction()
138 	trace_mm_compaction_defer_compaction(zone, order);  in defer_compaction()
142 bool compaction_deferred(struct zone *zone, int order)  in compaction_deferred()  argument
144 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;  in compaction_deferred()
146 	if (order < zone->compact_order_failed)  in compaction_deferred()
150 	if (++zone->compact_considered > defer_limit)  in compaction_deferred()
151 		zone->compact_considered = defer_limit;  in compaction_deferred()
153 	if (zone->compact_considered >= defer_limit)  in compaction_deferred()
156 	trace_mm_compaction_deferred(zone, order);  in compaction_deferred()
166 void compaction_defer_reset(struct zone *zone, int order,  in compaction_defer_reset()  argument
170 		zone->compact_considered = 0;  in compaction_defer_reset()
171 		zone->compact_defer_shift = 0;  in compaction_defer_reset()
173 	if (order >= zone->compact_order_failed)  in compaction_defer_reset()
174 		zone->compact_order_failed = order + 1;  in compaction_defer_reset()
176 	trace_mm_compaction_defer_reset(zone, order);  in compaction_defer_reset()
180 bool compaction_restarting(struct zone *zone, int order)  in compaction_restarting()  argument
182 	if (order < zone->compact_order_failed)  in compaction_restarting()
185 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&  in compaction_restarting()
186 		zone->compact_considered >= 1UL << zone->compact_defer_shift;  in compaction_restarting()
199 static void reset_cached_positions(struct zone *zone)  in reset_cached_positions()  argument
201 	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;  in reset_cached_positions()
202 	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;  in reset_cached_positions()
203 	zone->compact_cached_free_pfn = zone_end_pfn(zone);  in reset_cached_positions()
211 static void __reset_isolation_suitable(struct zone *zone)  in __reset_isolation_suitable()  argument
213 	unsigned long start_pfn = zone->zone_start_pfn;  in __reset_isolation_suitable()
214 	unsigned long end_pfn = zone_end_pfn(zone);  in __reset_isolation_suitable()
217 	zone->compact_blockskip_flush = false;  in __reset_isolation_suitable()
229 		if (zone != page_zone(page))  in __reset_isolation_suitable()
235 	reset_cached_positions(zone);  in __reset_isolation_suitable()
243 		struct zone *zone = &pgdat->node_zones[zoneid];  in reset_isolation_suitable()  local
244 		if (!populated_zone(zone))  in reset_isolation_suitable()
248 		if (zone->compact_blockskip_flush)  in reset_isolation_suitable()
249 			__reset_isolation_suitable(zone);  in reset_isolation_suitable()
261 	struct zone *zone = cc->zone;  in update_pageblock_skip()  local
279 		if (pfn > zone->compact_cached_migrate_pfn[0])  in update_pageblock_skip()
280 			zone->compact_cached_migrate_pfn[0] = pfn;  in update_pageblock_skip()
282 		    pfn > zone->compact_cached_migrate_pfn[1])  in update_pageblock_skip()
283 			zone->compact_cached_migrate_pfn[1] = pfn;  in update_pageblock_skip()
285 		if (pfn < zone->compact_cached_free_pfn)  in update_pageblock_skip()
286 			zone->compact_cached_free_pfn = pfn;  in update_pageblock_skip()
419 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
466 			locked = compact_trylock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
528 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
580 		if (!pageblock_pfn_to_page(pfn, block_end_pfn, cc->zone))  in isolate_freepages_range()
615 static void acct_isolated(struct zone *zone, struct compact_control *cc)  in acct_isolated()  argument
626 	mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]);  in acct_isolated()
627 	mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]);  in acct_isolated()
631 static bool too_many_isolated(struct zone *zone)  in too_many_isolated()  argument
635 	inactive = zone_page_state(zone, NR_INACTIVE_FILE) +  in too_many_isolated()
636 					zone_page_state(zone, NR_INACTIVE_ANON);  in too_many_isolated()
637 	active = zone_page_state(zone, NR_ACTIVE_FILE) +  in too_many_isolated()
638 					zone_page_state(zone, NR_ACTIVE_ANON);  in too_many_isolated()
639 	isolated = zone_page_state(zone, NR_ISOLATED_FILE) +  in too_many_isolated()
640 					zone_page_state(zone, NR_ISOLATED_ANON);  in too_many_isolated()
667 	struct zone *zone = cc->zone;  in isolate_migratepages_block()  local
681 	while (unlikely(too_many_isolated(zone))) {  in isolate_migratepages_block()
705 		    && compact_unlock_should_abort(&zone->lru_lock, flags,  in isolate_migratepages_block()
782 			locked = compact_trylock_irqsave(&zone->lru_lock,  in isolate_migratepages_block()
802 		lruvec = mem_cgroup_page_lruvec(page, zone);  in isolate_migratepages_block()
833 		spin_unlock_irqrestore(&zone->lru_lock, flags);  in isolate_migratepages_block()
877 		if (!pageblock_pfn_to_page(pfn, block_end_pfn, cc->zone))  in isolate_migratepages_range()
889 	acct_isolated(cc->zone, cc);  in isolate_migratepages_range()
935 	struct zone *zone = cc->zone;  in isolate_freepages()  local
957 						zone_end_pfn(zone));  in isolate_freepages()
980 									zone);  in isolate_freepages()
1095 static isolate_migrate_t isolate_migratepages(struct zone *zone,  in isolate_migratepages()  argument
1130 		page = pageblock_pfn_to_page(low_pfn, end_pfn, zone);  in isolate_migratepages()
1153 			acct_isolated(zone, cc);  in isolate_migratepages()
1174 	acct_isolated(zone, cc);  in isolate_migratepages()
1190 static int __compact_finished(struct zone *zone, struct compact_control *cc,  in __compact_finished()  argument
1202 		reset_cached_positions(zone);  in __compact_finished()
1211 			zone->compact_blockskip_flush = true;  in __compact_finished()
1220 	watermark = low_wmark_pages(zone);  in __compact_finished()
1222 	if (!zone_watermark_ok(zone, cc->order, watermark, cc->classzone_idx,  in __compact_finished()
1228 		struct free_area *area = &zone->free_area[order];  in __compact_finished()
1253 static int compact_finished(struct zone *zone, struct compact_control *cc,  in compact_finished()  argument
1258 	ret = __compact_finished(zone, cc, migratetype);  in compact_finished()
1259 	trace_mm_compaction_finished(zone, cc->order, ret);  in compact_finished()
1273 static unsigned long __compaction_suitable(struct zone *zone, int order,  in __compaction_suitable()  argument
1282 	watermark = low_wmark_pages(zone);  in __compaction_suitable()
1287 	if (zone_watermark_ok(zone, order, watermark, classzone_idx,  in __compaction_suitable()
1297 	if (!zone_watermark_ok(zone, 0, watermark, classzone_idx, alloc_flags))  in __compaction_suitable()
1311 	fragindex = fragmentation_index(zone, order);  in __compaction_suitable()
1318 unsigned long compaction_suitable(struct zone *zone, int order,  in compaction_suitable()  argument
1323 	ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx);  in compaction_suitable()
1324 	trace_mm_compaction_suitable(zone, order, ret);  in compaction_suitable()
1331 static int compact_zone(struct zone *zone, struct compact_control *cc)  in compact_zone()  argument
1334 	unsigned long start_pfn = zone->zone_start_pfn;  in compact_zone()
1335 	unsigned long end_pfn = zone_end_pfn(zone);  in compact_zone()
1339 	ret = compaction_suitable(zone, cc->order, cc->alloc_flags,  in compact_zone()
1356 	if (compaction_restarting(zone, cc->order) && !current_is_kswapd())  in compact_zone()
1357 		__reset_isolation_suitable(zone);  in compact_zone()
1364 	cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync];  in compact_zone()
1365 	cc->free_pfn = zone->compact_cached_free_pfn;  in compact_zone()
1368 		zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
1372 		zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
1373 		zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
1382 	while ((ret = compact_finished(zone, cc, migratetype)) ==  in compact_zone()
1386 		switch (isolate_migratepages(zone, cc)) {  in compact_zone()
1440 				drain_local_pages(zone);  in compact_zone()
1465 		if (free_pfn > zone->compact_cached_free_pfn)  in compact_zone()
1466 			zone->compact_cached_free_pfn = free_pfn;  in compact_zone()
1478 static unsigned long compact_zone_order(struct zone *zone, int order,  in compact_zone_order()  argument
1488 		.zone = zone,  in compact_zone_order()
1496 	ret = compact_zone(zone, &cc);  in compact_zone_order()
1526 	struct zone *zone;  in try_to_compact_pages()  local
1539 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,  in try_to_compact_pages()
1544 		if (compaction_deferred(zone, order))  in try_to_compact_pages()
1547 		status = compact_zone_order(zone, order, gfp_mask, mode,  in try_to_compact_pages()
1558 		if (zone_watermark_ok(zone, order, low_wmark_pages(zone),  in try_to_compact_pages()
1566 			compaction_defer_reset(zone, order, false);  in try_to_compact_pages()
1587 			defer_compaction(zone, order);  in try_to_compact_pages()
1627 	struct zone *zone;  in __compact_pgdat()  local
1631 		zone = &pgdat->node_zones[zoneid];  in __compact_pgdat()
1632 		if (!populated_zone(zone))  in __compact_pgdat()
1637 		cc->zone = zone;  in __compact_pgdat()
1647 			__reset_isolation_suitable(zone);  in __compact_pgdat()
1650 				!compaction_deferred(zone, cc->order))  in __compact_pgdat()
1651 			compact_zone(zone, cc);  in __compact_pgdat()
1654 			if (zone_watermark_ok(zone, cc->order,  in __compact_pgdat()
1655 						low_wmark_pages(zone), 0, 0))  in __compact_pgdat()
1656 				compaction_defer_reset(zone, cc->order, false);  in __compact_pgdat()