19 static void list_lru_register(struct list_lru *lru)  in list_lru_register()  argument
22 	list_add(&lru->list, &list_lrus);  in list_lru_register()
26 static void list_lru_unregister(struct list_lru *lru)  in list_lru_unregister()  argument
29 	list_del(&lru->list);  in list_lru_unregister()
33 static void list_lru_register(struct list_lru *lru)  in list_lru_register()  argument
37 static void list_lru_unregister(struct list_lru *lru)  in list_lru_unregister()  argument
43 static inline bool list_lru_memcg_aware(struct list_lru *lru)  in list_lru_memcg_aware()  argument
49 	return !!lru->node[0].memcg_lrus;  in list_lru_memcg_aware()
61 		return nlru->memcg_lrus->lru[idx];  in list_lru_from_memcg_idx()
63 	return &nlru->lru;  in list_lru_from_memcg_idx()
82 		return &nlru->lru;  in list_lru_from_kmem()
86 		return &nlru->lru;  in list_lru_from_kmem()
91 static inline bool list_lru_memcg_aware(struct list_lru *lru)  in list_lru_memcg_aware()  argument
99 	return &nlru->lru;  in list_lru_from_memcg_idx()
105 	return &nlru->lru;  in list_lru_from_kmem()
109 bool list_lru_add(struct list_lru *lru, struct list_head *item)  in list_lru_add()  argument
112 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_add()
128 bool list_lru_del(struct list_lru *lru, struct list_head *item)  in list_lru_del()  argument
131 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_del()
162 static unsigned long __list_lru_count_one(struct list_lru *lru,  in __list_lru_count_one()  argument
165 	struct list_lru_node *nlru = &lru->node[nid];  in __list_lru_count_one()
177 unsigned long list_lru_count_one(struct list_lru *lru,  in list_lru_count_one()  argument
180 	return __list_lru_count_one(lru, nid, memcg_cache_id(memcg));  in list_lru_count_one()
184 unsigned long list_lru_count_node(struct list_lru *lru, int nid)  in list_lru_count_node()  argument
189 	count += __list_lru_count_one(lru, nid, -1);  in list_lru_count_node()
190 	if (list_lru_memcg_aware(lru)) {  in list_lru_count_node()
192 			count += __list_lru_count_one(lru, nid, memcg_idx);  in list_lru_count_node()
199 __list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,  in __list_lru_walk_one()  argument
204 	struct list_lru_node *nlru = &lru->node[nid];  in __list_lru_walk_one()
259 list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,  in list_lru_walk_one()  argument
263 	return __list_lru_walk_one(lru, nid, memcg_cache_id(memcg),  in list_lru_walk_one()
268 unsigned long list_lru_walk_node(struct list_lru *lru, int nid,  in list_lru_walk_node()  argument
275 	isolated += __list_lru_walk_one(lru, nid, -1, isolate, cb_arg,  in list_lru_walk_node()
277 	if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) {  in list_lru_walk_node()
279 			isolated += __list_lru_walk_one(lru, nid, memcg_idx,  in list_lru_walk_node()
302 		kfree(memcg_lrus->lru[i]);  in __memcg_destroy_list_lru_node()
318 		memcg_lrus->lru[i] = l;  in __memcg_init_list_lru_node()
390 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)  in memcg_init_list_lru()  argument
398 		if (memcg_init_list_lru_node(&lru->node[i]))  in memcg_init_list_lru()
404 		if (!lru->node[i].memcg_lrus)  in memcg_init_list_lru()
406 		memcg_destroy_list_lru_node(&lru->node[i]);  in memcg_init_list_lru()
411 static void memcg_destroy_list_lru(struct list_lru *lru)  in memcg_destroy_list_lru()  argument
415 	if (!list_lru_memcg_aware(lru))  in memcg_destroy_list_lru()
419 		memcg_destroy_list_lru_node(&lru->node[i]);  in memcg_destroy_list_lru()
422 static int memcg_update_list_lru(struct list_lru *lru,  in memcg_update_list_lru()  argument
427 	if (!list_lru_memcg_aware(lru))  in memcg_update_list_lru()
431 		if (memcg_update_list_lru_node(&lru->node[i],  in memcg_update_list_lru()
438 		if (!lru->node[i].memcg_lrus)  in memcg_update_list_lru()
441 		memcg_cancel_update_list_lru_node(&lru->node[i],  in memcg_update_list_lru()
447 static void memcg_cancel_update_list_lru(struct list_lru *lru,  in memcg_cancel_update_list_lru()  argument
452 	if (!list_lru_memcg_aware(lru))  in memcg_cancel_update_list_lru()
456 		memcg_cancel_update_list_lru_node(&lru->node[i],  in memcg_cancel_update_list_lru()
463 	struct list_lru *lru;  in memcg_update_all_list_lrus()  local
467 	list_for_each_entry(lru, &list_lrus, list) {  in memcg_update_all_list_lrus()
468 		ret = memcg_update_list_lru(lru, old_size, new_size);  in memcg_update_all_list_lrus()
476 	list_for_each_entry_continue_reverse(lru, &list_lrus, list)  in memcg_update_all_list_lrus()
477 		memcg_cancel_update_list_lru(lru, old_size, new_size);  in memcg_update_all_list_lrus()
502 static void memcg_drain_list_lru(struct list_lru *lru,  in memcg_drain_list_lru()  argument
507 	if (!list_lru_memcg_aware(lru))  in memcg_drain_list_lru()
511 		memcg_drain_list_lru_node(&lru->node[i], src_idx, dst_idx);  in memcg_drain_list_lru()
516 	struct list_lru *lru;  in memcg_drain_all_list_lrus()  local
519 	list_for_each_entry(lru, &list_lrus, list)  in memcg_drain_all_list_lrus()
520 		memcg_drain_list_lru(lru, src_idx, dst_idx);  in memcg_drain_all_list_lrus()
524 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)  in memcg_init_list_lru()  argument
529 static void memcg_destroy_list_lru(struct list_lru *lru)  in memcg_destroy_list_lru()  argument
534 int __list_lru_init(struct list_lru *lru, bool memcg_aware,  in __list_lru_init()  argument
538 	size_t size = sizeof(*lru->node) * nr_node_ids;  in __list_lru_init()
543 	lru->node = kzalloc(size, GFP_KERNEL);  in __list_lru_init()
544 	if (!lru->node)  in __list_lru_init()
548 		spin_lock_init(&lru->node[i].lock);  in __list_lru_init()
550 			lockdep_set_class(&lru->node[i].lock, key);  in __list_lru_init()
551 		init_one_lru(&lru->node[i].lru);  in __list_lru_init()
554 	err = memcg_init_list_lru(lru, memcg_aware);  in __list_lru_init()
556 		kfree(lru->node);  in __list_lru_init()
560 	list_lru_register(lru);  in __list_lru_init()
567 void list_lru_destroy(struct list_lru *lru)  in list_lru_destroy()  argument
570 	if (!lru->node)  in list_lru_destroy()
575 	list_lru_unregister(lru);  in list_lru_destroy()
577 	memcg_destroy_list_lru(lru);  in list_lru_destroy()
578 	kfree(lru->node);  in list_lru_destroy()
579 	lru->node = NULL;  in list_lru_destroy()