53 list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)  in list_lru_from_memcg_idx()  argument
59 	lockdep_assert_held(&nlru->lock);  in list_lru_from_memcg_idx()
60 	if (nlru->memcg_lrus && idx >= 0)  in list_lru_from_memcg_idx()
61 		return nlru->memcg_lrus->lru[idx];  in list_lru_from_memcg_idx()
63 	return &nlru->lru;  in list_lru_from_memcg_idx()
77 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)  in list_lru_from_kmem()  argument
81 	if (!nlru->memcg_lrus)  in list_lru_from_kmem()
82 		return &nlru->lru;  in list_lru_from_kmem()
86 		return &nlru->lru;  in list_lru_from_kmem()
88 	return list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));  in list_lru_from_kmem()
97 list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)  in list_lru_from_memcg_idx()  argument
99 	return &nlru->lru;  in list_lru_from_memcg_idx()
103 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)  in list_lru_from_kmem()  argument
105 	return &nlru->lru;  in list_lru_from_kmem()
112 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_add()  local
115 	spin_lock(&nlru->lock);  in list_lru_add()
117 		l = list_lru_from_kmem(nlru, item);  in list_lru_add()
120 		spin_unlock(&nlru->lock);  in list_lru_add()
123 	spin_unlock(&nlru->lock);  in list_lru_add()
131 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_del()  local
134 	spin_lock(&nlru->lock);  in list_lru_del()
136 		l = list_lru_from_kmem(nlru, item);  in list_lru_del()
139 		spin_unlock(&nlru->lock);  in list_lru_del()
142 	spin_unlock(&nlru->lock);  in list_lru_del()
165 	struct list_lru_node *nlru = &lru->node[nid];  in __list_lru_count_one()  local
169 	spin_lock(&nlru->lock);  in __list_lru_count_one()
170 	l = list_lru_from_memcg_idx(nlru, memcg_idx);  in __list_lru_count_one()
172 	spin_unlock(&nlru->lock);  in __list_lru_count_one()
204 	struct list_lru_node *nlru = &lru->node[nid];  in __list_lru_walk_one()  local
209 	spin_lock(&nlru->lock);  in __list_lru_walk_one()
210 	l = list_lru_from_memcg_idx(nlru, memcg_idx);  in __list_lru_walk_one()
223 		ret = isolate(item, l, &nlru->lock, cb_arg);  in __list_lru_walk_one()
226 			assert_spin_locked(&nlru->lock);  in __list_lru_walk_one()
247 			assert_spin_locked(&nlru->lock);  in __list_lru_walk_one()
254 	spin_unlock(&nlru->lock);  in __list_lru_walk_one()
326 static int memcg_init_list_lru_node(struct list_lru_node *nlru)  in memcg_init_list_lru_node()  argument
330 	nlru->memcg_lrus = kmalloc(size * sizeof(void *), GFP_KERNEL);  in memcg_init_list_lru_node()
331 	if (!nlru->memcg_lrus)  in memcg_init_list_lru_node()
334 	if (__memcg_init_list_lru_node(nlru->memcg_lrus, 0, size)) {  in memcg_init_list_lru_node()
335 		kfree(nlru->memcg_lrus);  in memcg_init_list_lru_node()
342 static void memcg_destroy_list_lru_node(struct list_lru_node *nlru)  in memcg_destroy_list_lru_node()  argument
344 	__memcg_destroy_list_lru_node(nlru->memcg_lrus, 0, memcg_nr_cache_ids);  in memcg_destroy_list_lru_node()
345 	kfree(nlru->memcg_lrus);  in memcg_destroy_list_lru_node()
348 static int memcg_update_list_lru_node(struct list_lru_node *nlru,  in memcg_update_list_lru_node()  argument
355 	old = nlru->memcg_lrus;  in memcg_update_list_lru_node()
374 	spin_lock_irq(&nlru->lock);  in memcg_update_list_lru_node()
375 	nlru->memcg_lrus = new;  in memcg_update_list_lru_node()
376 	spin_unlock_irq(&nlru->lock);  in memcg_update_list_lru_node()
382 static void memcg_cancel_update_list_lru_node(struct list_lru_node *nlru,  in memcg_cancel_update_list_lru_node()  argument
387 	__memcg_destroy_list_lru_node(nlru->memcg_lrus, old_size, new_size);  in memcg_cancel_update_list_lru_node()
481 static void memcg_drain_list_lru_node(struct list_lru_node *nlru,  in memcg_drain_list_lru_node()  argument
490 	spin_lock_irq(&nlru->lock);  in memcg_drain_list_lru_node()
492 	src = list_lru_from_memcg_idx(nlru, src_idx);  in memcg_drain_list_lru_node()
493 	dst = list_lru_from_memcg_idx(nlru, dst_idx);  in memcg_drain_list_lru_node()
499 	spin_unlock_irq(&nlru->lock);  in memcg_drain_list_lru_node()