80 static struct mem_cgroup *root_mem_cgroup __read_mostly;
152 	struct mem_cgroup *memcg;
166 	int (*register_event)(struct mem_cgroup *memcg,
173 	void (*unregister_event)(struct mem_cgroup *memcg,
185 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
186 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
200 	struct mem_cgroup *from;
201 	struct mem_cgroup *to;
250 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)  in memcg_to_vmpressure()
259 	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;  in vmpressure_to_css()
262 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)  in mem_cgroup_is_root()
273 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)  in mem_cgroup_id()
284 static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)  in mem_cgroup_from_id()
298 		struct mem_cgroup *memcg;  in sock_update_memcg()
332 		struct mem_cgroup *memcg;  in sock_release_memcg()
339 struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)  in tcp_proto_cgroup()
405 mem_cgroup_zone_zoneinfo(struct mem_cgroup *memcg, struct zone *zone)  in mem_cgroup_zone_zoneinfo()
433 	struct mem_cgroup *memcg;  in mem_cgroup_css_from_page()
437 	memcg = page->mem_cgroup;  in mem_cgroup_css_from_page()
461 	struct mem_cgroup *memcg;  in page_cgroup_ino()
465 	memcg = READ_ONCE(page->mem_cgroup);  in page_cgroup_ino()
475 mem_cgroup_page_zoneinfo(struct mem_cgroup *memcg, struct page *page)  in mem_cgroup_page_zoneinfo()
549 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)  in soft_limit_excess()
561 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)  in mem_cgroup_update_tree()
596 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)  in mem_cgroup_remove_from_trees()
670 mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx)  in mem_cgroup_read_stat()
687 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,  in mem_cgroup_read_events()
698 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,  in mem_cgroup_charge_statistics()
728 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,  in mem_cgroup_node_nr_lru_pages()
751 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,  in mem_cgroup_nr_lru_pages()
762 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,  in mem_cgroup_event_ratelimit()
794 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)  in memcg_check_events()
818 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)  in mem_cgroup_from_task()
832 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)  in get_mem_cgroup_from_mm()
834 	struct mem_cgroup *memcg = NULL;  in get_mem_cgroup_from_mm()
872 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,  in mem_cgroup_iter()
873 				   struct mem_cgroup *prev,  in mem_cgroup_iter()
878 	struct mem_cgroup *memcg = NULL;  in mem_cgroup_iter()
879 	struct mem_cgroup *pos = NULL;  in mem_cgroup_iter()
996 void mem_cgroup_iter_break(struct mem_cgroup *root,  in mem_cgroup_iter_break()
997 			   struct mem_cgroup *prev)  in mem_cgroup_iter_break()
1005 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)  in invalidate_reclaim_iterators()
1007 	struct mem_cgroup *memcg = dead_memcg;  in invalidate_reclaim_iterators()
1052 				      struct mem_cgroup *memcg)  in mem_cgroup_zone_lruvec()
1087 	struct mem_cgroup *memcg;  in mem_cgroup_page_lruvec()
1095 	memcg = page->mem_cgroup;  in mem_cgroup_page_lruvec()
1140 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)  in task_in_mem_cgroup()
1142 	struct mem_cgroup *task_memcg;  in task_in_mem_cgroup()
1167 	container_of(counter, struct mem_cgroup, member)
1176 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)  in mem_cgroup_margin()
1204 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)  in mem_cgroup_under_move()
1206 	struct mem_cgroup *from;  in mem_cgroup_under_move()
1207 	struct mem_cgroup *to;  in mem_cgroup_under_move()
1226 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)  in mem_cgroup_wait_acct_move()
1251 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)  in mem_cgroup_print_oom_info()
1255 	struct mem_cgroup *iter;  in mem_cgroup_print_oom_info()
1309 static int mem_cgroup_count_children(struct mem_cgroup *memcg)  in mem_cgroup_count_children()
1312 	struct mem_cgroup *iter;  in mem_cgroup_count_children()
1322 static unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)  in mem_cgroup_get_limit()
1336 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,  in mem_cgroup_out_of_memory()
1345 	struct mem_cgroup *iter;  in mem_cgroup_out_of_memory()
1429 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,  in test_mem_cgroup_node_reclaimable()
1448 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)  in mem_cgroup_may_update_nodemask()
1485 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)  in mem_cgroup_select_victim_node()
1508 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)  in mem_cgroup_select_victim_node()
1514 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,  in mem_cgroup_soft_reclaim()
1519 	struct mem_cgroup *victim = NULL;  in mem_cgroup_soft_reclaim()
1577 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)  in mem_cgroup_oom_trylock()
1579 	struct mem_cgroup *iter, *failed = NULL;  in mem_cgroup_oom_trylock()
1616 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)  in mem_cgroup_oom_unlock()
1618 	struct mem_cgroup *iter;  in mem_cgroup_oom_unlock()
1627 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)  in mem_cgroup_mark_under_oom()
1629 	struct mem_cgroup *iter;  in mem_cgroup_mark_under_oom()
1637 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)  in mem_cgroup_unmark_under_oom()
1639 	struct mem_cgroup *iter;  in mem_cgroup_unmark_under_oom()
1655 	struct mem_cgroup *memcg;
1662 	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;  in memcg_oom_wake_function()
1663 	struct mem_cgroup *oom_wait_memcg;  in memcg_oom_wake_function()
1675 static void memcg_oom_recover(struct mem_cgroup *memcg)  in memcg_oom_recover()
1689 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)  in mem_cgroup_oom()
1732 	struct mem_cgroup *memcg = current->memcg_in_oom;  in mem_cgroup_oom_synchronize()
1796 struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page)  in mem_cgroup_begin_page_stat()
1798 	struct mem_cgroup *memcg;  in mem_cgroup_begin_page_stat()
1818 	memcg = page->mem_cgroup;  in mem_cgroup_begin_page_stat()
1826 	if (memcg != page->mem_cgroup) {  in mem_cgroup_begin_page_stat()
1847 void mem_cgroup_end_page_stat(struct mem_cgroup *memcg)  in mem_cgroup_end_page_stat()
1868 	struct mem_cgroup *cached; /* this never be root cgroup */
1888 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)  in consume_stock()
1910 	struct mem_cgroup *old = stock->cached;  in drain_stock()
1937 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)  in refill_stock()
1953 static void drain_all_stock(struct mem_cgroup *root_memcg)  in drain_all_stock()
1965 		struct mem_cgroup *memcg;  in drain_all_stock()
2009 	struct mem_cgroup *memcg, *pos;  in mem_cgroup_handle_over_high()
2027 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,  in try_charge()
2032 	struct mem_cgroup *mem_over_limit;  in try_charge()
2166 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)  in cancel_charge()
2209 static void commit_charge(struct page *page, struct mem_cgroup *memcg,  in commit_charge()
2214 	VM_BUG_ON_PAGE(page->mem_cgroup, page);  in commit_charge()
2237 	page->mem_cgroup = memcg;  in commit_charge()
2290 	struct mem_cgroup *memcg;
2299 	struct mem_cgroup *memcg = cw->memcg;  in memcg_kmem_cache_create_func()
2311 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,  in __memcg_schedule_kmem_cache_create()
2329 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,  in memcg_schedule_kmem_cache_create()
2363 	struct mem_cgroup *memcg;  in __memcg_kmem_get_cache()
2406 			      struct mem_cgroup *memcg)  in __memcg_kmem_charge_memcg()
2424 	page->mem_cgroup = memcg;  in __memcg_kmem_charge_memcg()
2431 	struct mem_cgroup *memcg;  in __memcg_kmem_charge()
2442 	struct mem_cgroup *memcg = page->mem_cgroup;  in __memcg_kmem_uncharge()
2455 	page->mem_cgroup = NULL;  in __memcg_kmem_uncharge()
2476 		head[i].mem_cgroup = head->mem_cgroup;  in mem_cgroup_split_huge_fixup()
2478 	__this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE],  in mem_cgroup_split_huge_fixup()
2484 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,  in mem_cgroup_swap_statistics()
2506 				struct mem_cgroup *from, struct mem_cgroup *to)  in mem_cgroup_move_swap_account()
2522 				struct mem_cgroup *from, struct mem_cgroup *to)  in mem_cgroup_move_swap_account()
2530 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,  in mem_cgroup_resize_limit()
2585 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,  in mem_cgroup_resize_memsw_limit()
2716 static inline bool memcg_has_children(struct mem_cgroup *memcg)  in memcg_has_children()
2740 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)  in mem_cgroup_force_empty()
2770 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));  in mem_cgroup_force_empty_write()
2787 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_hierarchy_write()
2788 	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);  in mem_cgroup_hierarchy_write()
2818 static unsigned long tree_stat(struct mem_cgroup *memcg,  in tree_stat()
2821 	struct mem_cgroup *iter;  in tree_stat()
2830 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)  in mem_cgroup_usage()
2859 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_read_u64()
2897 static int memcg_activate_kmem(struct mem_cgroup *memcg,  in memcg_activate_kmem()
2954 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,  in memcg_update_kmem_limit()
2968 static int memcg_propagate_kmem(struct mem_cgroup *memcg)  in memcg_propagate_kmem()
2971 	struct mem_cgroup *parent = parent_mem_cgroup(memcg);  in memcg_propagate_kmem()
2987 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,  in memcg_update_kmem_limit()
3001 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));  in mem_cgroup_write()
3039 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));  in mem_cgroup_reset()
3080 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_move_charge_write()
3119 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));  in memcg_numa_stat_show()
3133 		struct mem_cgroup *iter;  in memcg_numa_stat_show()
3155 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));  in memcg_stat_show()
3157 	struct mem_cgroup *mi;  in memcg_stat_show()
3251 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_swappiness_read()
3259 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_swappiness_write()
3272 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)  in __mem_cgroup_threshold()
3323 static void mem_cgroup_threshold(struct mem_cgroup *memcg)  in mem_cgroup_threshold()
3348 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)  in mem_cgroup_oom_notify_cb()
3361 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)  in mem_cgroup_oom_notify()
3363 	struct mem_cgroup *iter;  in mem_cgroup_oom_notify()
3369 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,  in __mem_cgroup_usage_register_event()
3451 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,  in mem_cgroup_usage_register_event()
3457 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,  in memsw_cgroup_usage_register_event()
3463 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,  in __mem_cgroup_usage_unregister_event()
3542 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,  in mem_cgroup_usage_unregister_event()
3548 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,  in memsw_cgroup_usage_unregister_event()
3554 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,  in mem_cgroup_oom_register_event()
3576 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,  in mem_cgroup_oom_unregister_event()
3595 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));  in mem_cgroup_oom_control_read()
3605 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_oom_control_write()
3619 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)  in memcg_init_kmem()
3630 static void memcg_deactivate_kmem(struct mem_cgroup *memcg)  in memcg_deactivate_kmem()
3633 	struct mem_cgroup *parent, *child;  in memcg_deactivate_kmem()
3679 static void memcg_destroy_kmem(struct mem_cgroup *memcg)  in memcg_destroy_kmem()
3689 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)  in memcg_init_kmem()
3694 static void memcg_deactivate_kmem(struct mem_cgroup *memcg)  in memcg_deactivate_kmem()
3698 static void memcg_destroy_kmem(struct mem_cgroup *memcg)  in memcg_destroy_kmem()
3705 struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)  in mem_cgroup_cgwb_list()
3710 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)  in memcg_wb_domain_init()
3715 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)  in memcg_wb_domain_exit()
3720 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)  in memcg_wb_domain_size_changed()
3727 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);  in mem_cgroup_wb_domain()
3757 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);  in mem_cgroup_wb_stats()
3758 	struct mem_cgroup *parent;  in mem_cgroup_wb_stats()
3779 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)  in memcg_wb_domain_init()
3784 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)  in memcg_wb_domain_exit()
3788 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)  in memcg_wb_domain_size_changed()
3816 	struct mem_cgroup *memcg = event->memcg;  in memcg_event_remove()
3840 	struct mem_cgroup *memcg = event->memcg;  in memcg_event_wake()
3890 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in memcg_write_event_control()
4127 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)  in alloc_mem_cgroup_per_zone_info()
4157 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)  in free_mem_cgroup_per_zone_info()
4162 static struct mem_cgroup *mem_cgroup_alloc(void)  in mem_cgroup_alloc()
4164 	struct mem_cgroup *memcg;  in mem_cgroup_alloc()
4167 	size = sizeof(struct mem_cgroup);  in mem_cgroup_alloc()
4201 static void __mem_cgroup_free(struct mem_cgroup *memcg)  in __mem_cgroup_free()
4218 struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)  in parent_mem_cgroup()
4229 	struct mem_cgroup *memcg;  in mem_cgroup_css_alloc()
4276 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_css_online()
4277 	struct mem_cgroup *parent = mem_cgroup_from_css(css->parent);  in mem_cgroup_css_online()
4335 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_css_offline()
4359 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_css_released()
4366 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_css_free()
4387 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in mem_cgroup_css_reset()
4548 				   struct mem_cgroup *from,  in mem_cgroup_move_account()
4549 				   struct mem_cgroup *to)  in mem_cgroup_move_account()
4575 	if (page->mem_cgroup != from)  in mem_cgroup_move_account()
4619 	page->mem_cgroup = to;  in mem_cgroup_move_account()
4658 		if (page->mem_cgroup == mc.from) {  in get_mctgt_type()
4692 	if (page->mem_cgroup == mc.from) {  in get_mctgt_type_thp()
4766 	struct mem_cgroup *from = mc.from;  in __mem_cgroup_clear_mc()
4767 	struct mem_cgroup *to = mc.to;  in __mem_cgroup_clear_mc()
4827 	struct mem_cgroup *memcg;  in mem_cgroup_can_attach()
4828 	struct mem_cgroup *from;  in mem_cgroup_can_attach()
5077 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);  in memory_current_read()
5084 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));  in memory_low_show()
5098 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));  in memory_low_write()
5114 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));  in memory_high_show()
5128 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));  in memory_high_write()
5151 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));  in memory_max_show()
5165 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));  in memory_max_write()
5213 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));  in memory_events_show()
5250 		.file_offset = offsetof(struct mem_cgroup, events_file),
5280 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)  in mem_cgroup_low()
5327 			  gfp_t gfp_mask, struct mem_cgroup **memcgp)  in mem_cgroup_try_charge()
5329 	struct mem_cgroup *memcg = NULL;  in mem_cgroup_try_charge()
5345 		if (page->mem_cgroup)  in mem_cgroup_try_charge()
5392 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,  in mem_cgroup_commit_charge()
5440 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg)  in mem_cgroup_cancel_charge()
5462 static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,  in uncharge_batch()
5491 	struct mem_cgroup *memcg = NULL;  in uncharge_list()
5509 		if (!page->mem_cgroup)  in uncharge_list()
5518 		if (memcg != page->mem_cgroup) {  in uncharge_list()
5524 			memcg = page->mem_cgroup;  in uncharge_list()
5538 		page->mem_cgroup = NULL;  in uncharge_list()
5561 	if (!page->mem_cgroup)  in mem_cgroup_uncharge()
5596 	struct mem_cgroup *memcg;  in mem_cgroup_replace_page()
5609 	if (newpage->mem_cgroup)  in mem_cgroup_replace_page()
5613 	memcg = oldpage->mem_cgroup;  in mem_cgroup_replace_page()
5618 	oldpage->mem_cgroup = NULL;  in mem_cgroup_replace_page()
5673 	struct mem_cgroup *memcg;  in mem_cgroup_swapout()
5682 	memcg = page->mem_cgroup;  in mem_cgroup_swapout()
5692 	page->mem_cgroup = NULL;  in mem_cgroup_swapout()
5716 	struct mem_cgroup *memcg;  in mem_cgroup_uncharge_swap()