19 static void list_lru_register(struct list_lru *lru)  in list_lru_register()  argument
22 	list_add(&lru->list, &list_lrus);  in list_lru_register()
26 static void list_lru_unregister(struct list_lru *lru)  in list_lru_unregister()  argument
29 	list_del(&lru->list);  in list_lru_unregister()
33 static void list_lru_register(struct list_lru *lru)  in list_lru_register()  argument
37 static void list_lru_unregister(struct list_lru *lru)  in list_lru_unregister()  argument
43 static inline bool list_lru_memcg_aware(struct list_lru *lru)  in list_lru_memcg_aware()  argument
45 	return !!lru->node[0].memcg_lrus;  in list_lru_memcg_aware()
57 		return nlru->memcg_lrus->lru[idx];  in list_lru_from_memcg_idx()
59 	return &nlru->lru;  in list_lru_from_memcg_idx()
68 		return &nlru->lru;  in list_lru_from_kmem()
72 		return &nlru->lru;  in list_lru_from_kmem()
77 static inline bool list_lru_memcg_aware(struct list_lru *lru)  in list_lru_memcg_aware()  argument
85 	return &nlru->lru;  in list_lru_from_memcg_idx()
91 	return &nlru->lru;  in list_lru_from_kmem()
95 bool list_lru_add(struct list_lru *lru, struct list_head *item)  in list_lru_add()  argument
98 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_add()
114 bool list_lru_del(struct list_lru *lru, struct list_head *item)  in list_lru_del()  argument
117 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_del()
148 static unsigned long __list_lru_count_one(struct list_lru *lru,  in __list_lru_count_one()  argument
151 	struct list_lru_node *nlru = &lru->node[nid];  in __list_lru_count_one()
163 unsigned long list_lru_count_one(struct list_lru *lru,  in list_lru_count_one()  argument
166 	return __list_lru_count_one(lru, nid, memcg_cache_id(memcg));  in list_lru_count_one()
170 unsigned long list_lru_count_node(struct list_lru *lru, int nid)  in list_lru_count_node()  argument
175 	count += __list_lru_count_one(lru, nid, -1);  in list_lru_count_node()
176 	if (list_lru_memcg_aware(lru)) {  in list_lru_count_node()
178 			count += __list_lru_count_one(lru, nid, memcg_idx);  in list_lru_count_node()
185 __list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,  in __list_lru_walk_one()  argument
190 	struct list_lru_node *nlru = &lru->node[nid];  in __list_lru_walk_one()
245 list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,  in list_lru_walk_one()  argument
249 	return __list_lru_walk_one(lru, nid, memcg_cache_id(memcg),  in list_lru_walk_one()
254 unsigned long list_lru_walk_node(struct list_lru *lru, int nid,  in list_lru_walk_node()  argument
261 	isolated += __list_lru_walk_one(lru, nid, -1, isolate, cb_arg,  in list_lru_walk_node()
263 	if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) {  in list_lru_walk_node()
265 			isolated += __list_lru_walk_one(lru, nid, memcg_idx,  in list_lru_walk_node()
288 		kfree(memcg_lrus->lru[i]);  in __memcg_destroy_list_lru_node()
304 		memcg_lrus->lru[i] = l;  in __memcg_init_list_lru_node()
376 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)  in memcg_init_list_lru()  argument
382 			lru->node[i].memcg_lrus = NULL;  in memcg_init_list_lru()
383 		else if (memcg_init_list_lru_node(&lru->node[i]))  in memcg_init_list_lru()
389 		memcg_destroy_list_lru_node(&lru->node[i]);  in memcg_init_list_lru()
393 static void memcg_destroy_list_lru(struct list_lru *lru)  in memcg_destroy_list_lru()  argument
397 	if (!list_lru_memcg_aware(lru))  in memcg_destroy_list_lru()
401 		memcg_destroy_list_lru_node(&lru->node[i]);  in memcg_destroy_list_lru()
404 static int memcg_update_list_lru(struct list_lru *lru,  in memcg_update_list_lru()  argument
409 	if (!list_lru_memcg_aware(lru))  in memcg_update_list_lru()
413 		if (memcg_update_list_lru_node(&lru->node[i],  in memcg_update_list_lru()
420 		memcg_cancel_update_list_lru_node(&lru->node[i],  in memcg_update_list_lru()
425 static void memcg_cancel_update_list_lru(struct list_lru *lru,  in memcg_cancel_update_list_lru()  argument
430 	if (!list_lru_memcg_aware(lru))  in memcg_cancel_update_list_lru()
434 		memcg_cancel_update_list_lru_node(&lru->node[i],  in memcg_cancel_update_list_lru()
441 	struct list_lru *lru;  in memcg_update_all_list_lrus()  local
445 	list_for_each_entry(lru, &list_lrus, list) {  in memcg_update_all_list_lrus()
446 		ret = memcg_update_list_lru(lru, old_size, new_size);  in memcg_update_all_list_lrus()
454 	list_for_each_entry_continue_reverse(lru, &list_lrus, list)  in memcg_update_all_list_lrus()
455 		memcg_cancel_update_list_lru(lru, old_size, new_size);  in memcg_update_all_list_lrus()
480 static void memcg_drain_list_lru(struct list_lru *lru,  in memcg_drain_list_lru()  argument
485 	if (!list_lru_memcg_aware(lru))  in memcg_drain_list_lru()
489 		memcg_drain_list_lru_node(&lru->node[i], src_idx, dst_idx);  in memcg_drain_list_lru()
494 	struct list_lru *lru;  in memcg_drain_all_list_lrus()  local
497 	list_for_each_entry(lru, &list_lrus, list)  in memcg_drain_all_list_lrus()
498 		memcg_drain_list_lru(lru, src_idx, dst_idx);  in memcg_drain_all_list_lrus()
502 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)  in memcg_init_list_lru()  argument
507 static void memcg_destroy_list_lru(struct list_lru *lru)  in memcg_destroy_list_lru()  argument
512 int __list_lru_init(struct list_lru *lru, bool memcg_aware,  in __list_lru_init()  argument
516 	size_t size = sizeof(*lru->node) * nr_node_ids;  in __list_lru_init()
521 	lru->node = kzalloc(size, GFP_KERNEL);  in __list_lru_init()
522 	if (!lru->node)  in __list_lru_init()
526 		spin_lock_init(&lru->node[i].lock);  in __list_lru_init()
528 			lockdep_set_class(&lru->node[i].lock, key);  in __list_lru_init()
529 		init_one_lru(&lru->node[i].lru);  in __list_lru_init()
532 	err = memcg_init_list_lru(lru, memcg_aware);  in __list_lru_init()
534 		kfree(lru->node);  in __list_lru_init()
538 	list_lru_register(lru);  in __list_lru_init()
545 void list_lru_destroy(struct list_lru *lru)  in list_lru_destroy()  argument
548 	if (!lru->node)  in list_lru_destroy()
553 	list_lru_unregister(lru);  in list_lru_destroy()
555 	memcg_destroy_list_lru(lru);  in list_lru_destroy()
556 	kfree(lru->node);  in list_lru_destroy()
557 	lru->node = NULL;  in list_lru_destroy()