201 static inline bool isolation_suitable(struct compact_control *cc,  in isolation_suitable()  argument
204 	if (cc->ignore_skip_hint)  in isolation_suitable()
262 static void update_pageblock_skip(struct compact_control *cc,  in update_pageblock_skip()  argument
266 	struct zone *zone = cc->zone;  in update_pageblock_skip()
269 	if (cc->ignore_skip_hint)  in update_pageblock_skip()
286 		if (cc->mode != MIGRATE_ASYNC &&  in update_pageblock_skip()
295 static inline bool isolation_suitable(struct compact_control *cc,  in isolation_suitable()  argument
301 static void update_pageblock_skip(struct compact_control *cc,  in update_pageblock_skip()  argument
317 						struct compact_control *cc)  in compact_trylock_irqsave()  argument
319 	if (cc->mode == MIGRATE_ASYNC) {  in compact_trylock_irqsave()
321 			cc->contended = COMPACT_CONTENDED_LOCK;  in compact_trylock_irqsave()
347 		unsigned long flags, bool *locked, struct compact_control *cc)  in compact_unlock_should_abort()  argument
355 		cc->contended = COMPACT_CONTENDED_SCHED;  in compact_unlock_should_abort()
360 		if (cc->mode == MIGRATE_ASYNC) {  in compact_unlock_should_abort()
361 			cc->contended = COMPACT_CONTENDED_SCHED;  in compact_unlock_should_abort()
379 static inline bool compact_should_abort(struct compact_control *cc)  in compact_should_abort()  argument
383 		if (cc->mode == MIGRATE_ASYNC) {  in compact_should_abort()
384 			cc->contended = COMPACT_CONTENDED_SCHED;  in compact_should_abort()
399 static unsigned long isolate_freepages_block(struct compact_control *cc,  in isolate_freepages_block()  argument
424 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
425 								&locked, cc))  in isolate_freepages_block()
453 			locked = compact_trylock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
454 								&flags, cc);  in isolate_freepages_block()
473 			cc->nr_freepages += isolated;  in isolate_freepages_block()
475 				cc->nr_migratepages <= cc->nr_freepages) {  in isolate_freepages_block()
508 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
512 		update_pageblock_skip(cc, valid_page, total_isolated, false);  in isolate_freepages_block()
534 isolate_freepages_range(struct compact_control *cc,  in isolate_freepages_range()  argument
560 		if (!pageblock_pfn_to_page(pfn, block_end_pfn, cc->zone))  in isolate_freepages_range()
563 		isolated = isolate_freepages_block(cc, &isolate_start_pfn,  in isolate_freepages_range()
595 static void acct_isolated(struct zone *zone, struct compact_control *cc)  in acct_isolated()  argument
600 	if (list_empty(&cc->migratepages))  in acct_isolated()
603 	list_for_each_entry(page, &cc->migratepages, lru)  in acct_isolated()
644 isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,  in isolate_migratepages_block()  argument
647 	struct zone *zone = cc->zone;  in isolate_migratepages_block()
649 	struct list_head *migratelist = &cc->migratepages;  in isolate_migratepages_block()
663 		if (cc->mode == MIGRATE_ASYNC)  in isolate_migratepages_block()
672 	if (compact_should_abort(cc))  in isolate_migratepages_block()
684 								&locked, cc))  in isolate_migratepages_block()
762 								&flags, cc);  in isolate_migratepages_block()
788 		cc->nr_migratepages++;  in isolate_migratepages_block()
792 		if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) {  in isolate_migratepages_block()
813 		update_pageblock_skip(cc, valid_page, nr_isolated, true);  in isolate_migratepages_block()
836 isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,  in isolate_migratepages_range()  argument
850 		if (!pageblock_pfn_to_page(pfn, block_end_pfn, cc->zone))  in isolate_migratepages_range()
853 		pfn = isolate_migratepages_block(cc, pfn, block_end_pfn,  in isolate_migratepages_range()
859 		if (cc->nr_migratepages == COMPACT_CLUSTER_MAX)  in isolate_migratepages_range()
862 	acct_isolated(cc->zone, cc);  in isolate_migratepages_range()
896 static void isolate_freepages(struct compact_control *cc)  in isolate_freepages()  argument
898 	struct zone *zone = cc->zone;  in isolate_freepages()
904 	struct list_head *freelist = &cc->freepages;  in isolate_freepages()
917 	isolate_start_pfn = cc->free_pfn;  in isolate_freepages()
918 	block_start_pfn = cc->free_pfn & ~(pageblock_nr_pages-1);  in isolate_freepages()
921 	low_pfn = ALIGN(cc->migrate_pfn + 1, pageblock_nr_pages);  in isolate_freepages()
929 			cc->nr_migratepages > cc->nr_freepages;  in isolate_freepages()
940 						&& compact_should_abort(cc))  in isolate_freepages()
953 		if (!isolation_suitable(cc, page))  in isolate_freepages()
957 		isolate_freepages_block(cc, &isolate_start_pfn,  in isolate_freepages()
969 		cc->free_pfn = (isolate_start_pfn < block_end_pfn) ?  in isolate_freepages()
977 		if (cc->contended)  in isolate_freepages()
989 		cc->free_pfn = cc->migrate_pfn;  in isolate_freepages()
1000 	struct compact_control *cc = (struct compact_control *)data;  in compaction_alloc()  local
1007 	if (list_empty(&cc->freepages)) {  in compaction_alloc()
1008 		if (!cc->contended)  in compaction_alloc()
1009 			isolate_freepages(cc);  in compaction_alloc()
1011 		if (list_empty(&cc->freepages))  in compaction_alloc()
1015 	freepage = list_entry(cc->freepages.next, struct page, lru);  in compaction_alloc()
1017 	cc->nr_freepages--;  in compaction_alloc()
1029 	struct compact_control *cc = (struct compact_control *)data;  in compaction_free()  local
1031 	list_add(&page->lru, &cc->freepages);  in compaction_free()
1032 	cc->nr_freepages++;  in compaction_free()
1054 					struct compact_control *cc)  in isolate_migratepages()  argument
1060 		(cc->mode == MIGRATE_ASYNC ? ISOLATE_ASYNC_MIGRATE : 0);  in isolate_migratepages()
1066 	low_pfn = cc->migrate_pfn;  in isolate_migratepages()
1075 	for (; end_pfn <= cc->free_pfn;  in isolate_migratepages()
1084 						&& compact_should_abort(cc))  in isolate_migratepages()
1092 		if (!isolation_suitable(cc, page))  in isolate_migratepages()
1100 		if (cc->mode == MIGRATE_ASYNC &&  in isolate_migratepages()
1105 		low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn,  in isolate_migratepages()
1108 		if (!low_pfn || cc->contended) {  in isolate_migratepages()
1109 			acct_isolated(zone, cc);  in isolate_migratepages()
1121 	acct_isolated(zone, cc);  in isolate_migratepages()
1127 	cc->migrate_pfn = (end_pfn <= cc->free_pfn) ? low_pfn : cc->free_pfn;  in isolate_migratepages()
1129 	return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;  in isolate_migratepages()
1132 static int __compact_finished(struct zone *zone, struct compact_control *cc,  in __compact_finished()  argument
1138 	if (cc->contended || fatal_signal_pending(current))  in __compact_finished()
1142 	if (cc->free_pfn <= cc->migrate_pfn) {  in __compact_finished()
1164 	if (cc->order == -1)  in __compact_finished()
1170 	if (!zone_watermark_ok(zone, cc->order, watermark, cc->classzone_idx,  in __compact_finished()
1171 							cc->alloc_flags))  in __compact_finished()
1175 	for (order = cc->order; order < MAX_ORDER; order++) {  in __compact_finished()
1201 static int compact_finished(struct zone *zone, struct compact_control *cc,  in compact_finished()  argument
1206 	ret = __compact_finished(zone, cc, migratetype);  in compact_finished()
1207 	trace_mm_compaction_finished(zone, cc->order, ret);  in compact_finished()
1283 static int compact_zone(struct zone *zone, struct compact_control *cc)  in compact_zone()  argument
1288 	const int migratetype = gfpflags_to_migratetype(cc->gfp_mask);  in compact_zone()
1289 	const bool sync = cc->mode != MIGRATE_ASYNC;  in compact_zone()
1292 	ret = compaction_suitable(zone, cc->order, cc->alloc_flags,  in compact_zone()
1293 							cc->classzone_idx);  in compact_zone()
1309 	if (compaction_restarting(zone, cc->order) && !current_is_kswapd())  in compact_zone()
1317 	cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync];  in compact_zone()
1318 	cc->free_pfn = zone->compact_cached_free_pfn;  in compact_zone()
1319 	if (cc->free_pfn < start_pfn || cc->free_pfn > end_pfn) {  in compact_zone()
1320 		cc->free_pfn = end_pfn & ~(pageblock_nr_pages-1);  in compact_zone()
1321 		zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
1323 	if (cc->migrate_pfn < start_pfn || cc->migrate_pfn > end_pfn) {  in compact_zone()
1324 		cc->migrate_pfn = start_pfn;  in compact_zone()
1325 		zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
1326 		zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
1329 	trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,  in compact_zone()
1330 				cc->free_pfn, end_pfn, sync);  in compact_zone()
1334 	while ((ret = compact_finished(zone, cc, migratetype)) ==  in compact_zone()
1337 		unsigned long isolate_start_pfn = cc->migrate_pfn;  in compact_zone()
1339 		switch (isolate_migratepages(zone, cc)) {  in compact_zone()
1342 			putback_movable_pages(&cc->migratepages);  in compact_zone()
1343 			cc->nr_migratepages = 0;  in compact_zone()
1356 		err = migrate_pages(&cc->migratepages, compaction_alloc,  in compact_zone()
1357 				compaction_free, (unsigned long)cc, cc->mode,  in compact_zone()
1360 		trace_mm_compaction_migratepages(cc->nr_migratepages, err,  in compact_zone()
1361 							&cc->migratepages);  in compact_zone()
1364 		cc->nr_migratepages = 0;  in compact_zone()
1366 			putback_movable_pages(&cc->migratepages);  in compact_zone()
1371 			if (err == -ENOMEM && cc->free_pfn > cc->migrate_pfn) {  in compact_zone()
1395 		if (cc->order > 0 && last_migrated_pfn) {  in compact_zone()
1398 				cc->migrate_pfn & ~((1UL << cc->order) - 1);  in compact_zone()
1417 	if (cc->nr_freepages > 0) {  in compact_zone()
1418 		unsigned long free_pfn = release_freepages(&cc->freepages);  in compact_zone()
1420 		cc->nr_freepages = 0;  in compact_zone()
1432 	trace_mm_compaction_end(start_pfn, cc->migrate_pfn,  in compact_zone()
1433 				cc->free_pfn, end_pfn, sync, ret);  in compact_zone()
1443 	struct compact_control cc = {  in compact_zone_order()  local
1453 	INIT_LIST_HEAD(&cc.freepages);  in compact_zone_order()
1454 	INIT_LIST_HEAD(&cc.migratepages);  in compact_zone_order()
1456 	ret = compact_zone(zone, &cc);  in compact_zone_order()
1458 	VM_BUG_ON(!list_empty(&cc.freepages));  in compact_zone_order()
1459 	VM_BUG_ON(!list_empty(&cc.migratepages));  in compact_zone_order()
1461 	*contended = cc.contended;  in compact_zone_order()
1584 static void __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc)  in __compact_pgdat()  argument
1595 		cc->nr_freepages = 0;  in __compact_pgdat()
1596 		cc->nr_migratepages = 0;  in __compact_pgdat()
1597 		cc->zone = zone;  in __compact_pgdat()
1598 		INIT_LIST_HEAD(&cc->freepages);  in __compact_pgdat()
1599 		INIT_LIST_HEAD(&cc->migratepages);  in __compact_pgdat()
1606 		if (cc->order == -1)  in __compact_pgdat()
1609 		if (cc->order == -1 || !compaction_deferred(zone, cc->order))  in __compact_pgdat()
1610 			compact_zone(zone, cc);  in __compact_pgdat()
1612 		if (cc->order > 0) {  in __compact_pgdat()
1613 			if (zone_watermark_ok(zone, cc->order,  in __compact_pgdat()
1615 				compaction_defer_reset(zone, cc->order, false);  in __compact_pgdat()
1618 		VM_BUG_ON(!list_empty(&cc->freepages));  in __compact_pgdat()
1619 		VM_BUG_ON(!list_empty(&cc->migratepages));  in __compact_pgdat()
1625 	struct compact_control cc = {  in compact_pgdat()  local
1633 	__compact_pgdat(pgdat, &cc);  in compact_pgdat()
1638 	struct compact_control cc = {  in compact_node()  local
1644 	__compact_pgdat(NODE_DATA(nid), &cc);  in compact_node()