1/* memcontrol.h - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published by 11 * the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 */ 19 20#ifndef _LINUX_MEMCONTROL_H 21#define _LINUX_MEMCONTROL_H 22#include <linux/cgroup.h> 23#include <linux/vm_event_item.h> 24#include <linux/hardirq.h> 25#include <linux/jump_label.h> 26#include <linux/page_counter.h> 27#include <linux/vmpressure.h> 28#include <linux/eventfd.h> 29#include <linux/mmzone.h> 30#include <linux/writeback.h> 31 32struct mem_cgroup; 33struct page; 34struct mm_struct; 35struct kmem_cache; 36 37/* 38 * The corresponding mem_cgroup_stat_names is defined in mm/memcontrol.c, 39 * These two lists should keep in accord with each other. 40 */ 41enum mem_cgroup_stat_index { 42 /* 43 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. 44 */ 45 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ 46 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ 47 MEM_CGROUP_STAT_RSS_HUGE, /* # of pages charged as anon huge */ 48 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ 49 MEM_CGROUP_STAT_DIRTY, /* # of dirty pages in page cache */ 50 MEM_CGROUP_STAT_WRITEBACK, /* # of pages under writeback */ 51 MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */ 52 MEM_CGROUP_STAT_NSTATS, 53}; 54 55struct mem_cgroup_reclaim_cookie { 56 struct zone *zone; 57 int priority; 58 unsigned int generation; 59}; 60 61enum mem_cgroup_events_index { 62 MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */ 63 MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */ 64 MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */ 65 MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */ 66 MEM_CGROUP_EVENTS_NSTATS, 67 /* default hierarchy events */ 68 MEMCG_LOW = MEM_CGROUP_EVENTS_NSTATS, 69 MEMCG_HIGH, 70 MEMCG_MAX, 71 MEMCG_OOM, 72 MEMCG_NR_EVENTS, 73}; 74 75/* 76 * Per memcg event counter is incremented at every pagein/pageout. With THP, 77 * it will be incremated by the number of pages. This counter is used for 78 * for trigger some periodic events. This is straightforward and better 79 * than using jiffies etc. to handle periodic memcg event. 80 */ 81enum mem_cgroup_events_target { 82 MEM_CGROUP_TARGET_THRESH, 83 MEM_CGROUP_TARGET_SOFTLIMIT, 84 MEM_CGROUP_TARGET_NUMAINFO, 85 MEM_CGROUP_NTARGETS, 86}; 87 88/* 89 * Bits in struct cg_proto.flags 90 */ 91enum cg_proto_flags { 92 /* Currently active and new sockets should be assigned to cgroups */ 93 MEMCG_SOCK_ACTIVE, 94 /* It was ever activated; we must disarm static keys on destruction */ 95 MEMCG_SOCK_ACTIVATED, 96}; 97 98struct cg_proto { 99 struct page_counter memory_allocated; /* Current allocated memory. */ 100 struct percpu_counter sockets_allocated; /* Current number of sockets. */ 101 int memory_pressure; 102 long sysctl_mem[3]; 103 unsigned long flags; 104 /* 105 * memcg field is used to find which memcg we belong directly 106 * Each memcg struct can hold more than one cg_proto, so container_of 107 * won't really cut. 108 * 109 * The elegant solution would be having an inverse function to 110 * proto_cgroup in struct proto, but that means polluting the structure 111 * for everybody, instead of just for memcg users. 112 */ 113 struct mem_cgroup *memcg; 114}; 115 116#ifdef CONFIG_MEMCG 117struct mem_cgroup_stat_cpu { 118 long count[MEM_CGROUP_STAT_NSTATS]; 119 unsigned long events[MEMCG_NR_EVENTS]; 120 unsigned long nr_page_events; 121 unsigned long targets[MEM_CGROUP_NTARGETS]; 122}; 123 124struct mem_cgroup_reclaim_iter { 125 struct mem_cgroup *position; 126 /* scan generation, increased every round-trip */ 127 unsigned int generation; 128}; 129 130/* 131 * per-zone information in memory controller. 132 */ 133struct mem_cgroup_per_zone { 134 struct lruvec lruvec; 135 unsigned long lru_size[NR_LRU_LISTS]; 136 137 struct mem_cgroup_reclaim_iter iter[DEF_PRIORITY + 1]; 138 139 struct rb_node tree_node; /* RB tree node */ 140 unsigned long usage_in_excess;/* Set to the value by which */ 141 /* the soft limit is exceeded*/ 142 bool on_tree; 143 struct mem_cgroup *memcg; /* Back pointer, we cannot */ 144 /* use container_of */ 145}; 146 147struct mem_cgroup_per_node { 148 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; 149}; 150 151struct mem_cgroup_threshold { 152 struct eventfd_ctx *eventfd; 153 unsigned long threshold; 154}; 155 156/* For threshold */ 157struct mem_cgroup_threshold_ary { 158 /* An array index points to threshold just below or equal to usage. */ 159 int current_threshold; 160 /* Size of entries[] */ 161 unsigned int size; 162 /* Array of thresholds */ 163 struct mem_cgroup_threshold entries[0]; 164}; 165 166struct mem_cgroup_thresholds { 167 /* Primary thresholds array */ 168 struct mem_cgroup_threshold_ary *primary; 169 /* 170 * Spare threshold array. 171 * This is needed to make mem_cgroup_unregister_event() "never fail". 172 * It must be able to store at least primary->size - 1 entries. 173 */ 174 struct mem_cgroup_threshold_ary *spare; 175}; 176 177/* 178 * The memory controller data structure. The memory controller controls both 179 * page cache and RSS per cgroup. We would eventually like to provide 180 * statistics based on the statistics developed by Rik Van Riel for clock-pro, 181 * to help the administrator determine what knobs to tune. 182 */ 183struct mem_cgroup { 184 struct cgroup_subsys_state css; 185 186 /* Accounted resources */ 187 struct page_counter memory; 188 struct page_counter memsw; 189 struct page_counter kmem; 190 191 /* Normal memory consumption range */ 192 unsigned long low; 193 unsigned long high; 194 195 unsigned long soft_limit; 196 197 /* vmpressure notifications */ 198 struct vmpressure vmpressure; 199 200 /* css_online() has been completed */ 201 int initialized; 202 203 /* 204 * Should the accounting and control be hierarchical, per subtree? 205 */ 206 bool use_hierarchy; 207 208 /* protected by memcg_oom_lock */ 209 bool oom_lock; 210 int under_oom; 211 212 int swappiness; 213 /* OOM-Killer disable */ 214 int oom_kill_disable; 215 216 /* handle for "memory.events" */ 217 struct cgroup_file events_file; 218 219 /* protect arrays of thresholds */ 220 struct mutex thresholds_lock; 221 222 /* thresholds for memory usage. RCU-protected */ 223 struct mem_cgroup_thresholds thresholds; 224 225 /* thresholds for mem+swap usage. RCU-protected */ 226 struct mem_cgroup_thresholds memsw_thresholds; 227 228 /* For oom notifier event fd */ 229 struct list_head oom_notify; 230 231 /* 232 * Should we move charges of a task when a task is moved into this 233 * mem_cgroup ? And what type of charges should we move ? 234 */ 235 unsigned long move_charge_at_immigrate; 236 /* 237 * set > 0 if pages under this cgroup are moving to other cgroup. 238 */ 239 atomic_t moving_account; 240 /* taken only while moving_account > 0 */ 241 spinlock_t move_lock; 242 struct task_struct *move_lock_task; 243 unsigned long move_lock_flags; 244 /* 245 * percpu counter. 246 */ 247 struct mem_cgroup_stat_cpu __percpu *stat; 248 249#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET) 250 struct cg_proto tcp_mem; 251#endif 252#if defined(CONFIG_MEMCG_KMEM) 253 /* Index in the kmem_cache->memcg_params.memcg_caches array */ 254 int kmemcg_id; 255 bool kmem_acct_activated; 256 bool kmem_acct_active; 257#endif 258 259 int last_scanned_node; 260#if MAX_NUMNODES > 1 261 nodemask_t scan_nodes; 262 atomic_t numainfo_events; 263 atomic_t numainfo_updating; 264#endif 265 266#ifdef CONFIG_CGROUP_WRITEBACK 267 struct list_head cgwb_list; 268 struct wb_domain cgwb_domain; 269#endif 270 271 /* List of events which userspace want to receive */ 272 struct list_head event_list; 273 spinlock_t event_list_lock; 274 275 struct mem_cgroup_per_node *nodeinfo[0]; 276 /* WARNING: nodeinfo must be the last member here */ 277}; 278extern struct cgroup_subsys_state *mem_cgroup_root_css; 279 280/** 281 * mem_cgroup_events - count memory events against a cgroup 282 * @memcg: the memory cgroup 283 * @idx: the event index 284 * @nr: the number of events to account for 285 */ 286static inline void mem_cgroup_events(struct mem_cgroup *memcg, 287 enum mem_cgroup_events_index idx, 288 unsigned int nr) 289{ 290 this_cpu_add(memcg->stat->events[idx], nr); 291 cgroup_file_notify(&memcg->events_file); 292} 293 294bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg); 295 296int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, 297 gfp_t gfp_mask, struct mem_cgroup **memcgp); 298void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, 299 bool lrucare); 300void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg); 301void mem_cgroup_uncharge(struct page *page); 302void mem_cgroup_uncharge_list(struct list_head *page_list); 303 304void mem_cgroup_replace_page(struct page *oldpage, struct page *newpage); 305 306struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *); 307struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *); 308 309bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg); 310struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p); 311struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg); 312 313static inline 314struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ 315 return css ? container_of(css, struct mem_cgroup, css) : NULL; 316} 317 318struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *, 319 struct mem_cgroup *, 320 struct mem_cgroup_reclaim_cookie *); 321void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); 322 323static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, 324 struct mem_cgroup *root) 325{ 326 if (root == memcg) 327 return true; 328 if (!root->use_hierarchy) 329 return false; 330 return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup); 331} 332 333static inline bool mm_match_cgroup(struct mm_struct *mm, 334 struct mem_cgroup *memcg) 335{ 336 struct mem_cgroup *task_memcg; 337 bool match = false; 338 339 rcu_read_lock(); 340 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 341 if (task_memcg) 342 match = mem_cgroup_is_descendant(task_memcg, memcg); 343 rcu_read_unlock(); 344 return match; 345} 346 347struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page); 348ino_t page_cgroup_ino(struct page *page); 349 350static inline bool mem_cgroup_disabled(void) 351{ 352 return !cgroup_subsys_enabled(memory_cgrp_subsys); 353} 354 355/* 356 * For memory reclaim. 357 */ 358int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); 359 360void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 361 int nr_pages); 362 363static inline bool mem_cgroup_lruvec_online(struct lruvec *lruvec) 364{ 365 struct mem_cgroup_per_zone *mz; 366 struct mem_cgroup *memcg; 367 368 if (mem_cgroup_disabled()) 369 return true; 370 371 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec); 372 memcg = mz->memcg; 373 374 return !!(memcg->css.flags & CSS_ONLINE); 375} 376 377static inline 378unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) 379{ 380 struct mem_cgroup_per_zone *mz; 381 382 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec); 383 return mz->lru_size[lru]; 384} 385 386static inline bool mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec) 387{ 388 unsigned long inactive_ratio; 389 unsigned long inactive; 390 unsigned long active; 391 unsigned long gb; 392 393 inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON); 394 active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON); 395 396 gb = (inactive + active) >> (30 - PAGE_SHIFT); 397 if (gb) 398 inactive_ratio = int_sqrt(10 * gb); 399 else 400 inactive_ratio = 1; 401 402 return inactive * inactive_ratio < active; 403} 404 405void mem_cgroup_handle_over_high(void); 406 407void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, 408 struct task_struct *p); 409 410static inline void mem_cgroup_oom_enable(void) 411{ 412 WARN_ON(current->memcg_may_oom); 413 current->memcg_may_oom = 1; 414} 415 416static inline void mem_cgroup_oom_disable(void) 417{ 418 WARN_ON(!current->memcg_may_oom); 419 current->memcg_may_oom = 0; 420} 421 422static inline bool task_in_memcg_oom(struct task_struct *p) 423{ 424 return p->memcg_in_oom; 425} 426 427bool mem_cgroup_oom_synchronize(bool wait); 428 429#ifdef CONFIG_MEMCG_SWAP 430extern int do_swap_account; 431#endif 432 433struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page); 434void mem_cgroup_end_page_stat(struct mem_cgroup *memcg); 435 436/** 437 * mem_cgroup_update_page_stat - update page state statistics 438 * @memcg: memcg to account against 439 * @idx: page state item to account 440 * @val: number of pages (positive or negative) 441 * 442 * See mem_cgroup_begin_page_stat() for locking requirements. 443 */ 444static inline void mem_cgroup_update_page_stat(struct mem_cgroup *memcg, 445 enum mem_cgroup_stat_index idx, int val) 446{ 447 VM_BUG_ON(!rcu_read_lock_held()); 448 449 if (memcg) 450 this_cpu_add(memcg->stat->count[idx], val); 451} 452 453static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg, 454 enum mem_cgroup_stat_index idx) 455{ 456 mem_cgroup_update_page_stat(memcg, idx, 1); 457} 458 459static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg, 460 enum mem_cgroup_stat_index idx) 461{ 462 mem_cgroup_update_page_stat(memcg, idx, -1); 463} 464 465unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, 466 gfp_t gfp_mask, 467 unsigned long *total_scanned); 468 469static inline void mem_cgroup_count_vm_event(struct mm_struct *mm, 470 enum vm_event_item idx) 471{ 472 struct mem_cgroup *memcg; 473 474 if (mem_cgroup_disabled()) 475 return; 476 477 rcu_read_lock(); 478 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 479 if (unlikely(!memcg)) 480 goto out; 481 482 switch (idx) { 483 case PGFAULT: 484 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]); 485 break; 486 case PGMAJFAULT: 487 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]); 488 break; 489 default: 490 BUG(); 491 } 492out: 493 rcu_read_unlock(); 494} 495#ifdef CONFIG_TRANSPARENT_HUGEPAGE 496void mem_cgroup_split_huge_fixup(struct page *head); 497#endif 498 499#else /* CONFIG_MEMCG */ 500struct mem_cgroup; 501 502static inline void mem_cgroup_events(struct mem_cgroup *memcg, 503 enum mem_cgroup_events_index idx, 504 unsigned int nr) 505{ 506} 507 508static inline bool mem_cgroup_low(struct mem_cgroup *root, 509 struct mem_cgroup *memcg) 510{ 511 return false; 512} 513 514static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, 515 gfp_t gfp_mask, 516 struct mem_cgroup **memcgp) 517{ 518 *memcgp = NULL; 519 return 0; 520} 521 522static inline void mem_cgroup_commit_charge(struct page *page, 523 struct mem_cgroup *memcg, 524 bool lrucare) 525{ 526} 527 528static inline void mem_cgroup_cancel_charge(struct page *page, 529 struct mem_cgroup *memcg) 530{ 531} 532 533static inline void mem_cgroup_uncharge(struct page *page) 534{ 535} 536 537static inline void mem_cgroup_uncharge_list(struct list_head *page_list) 538{ 539} 540 541static inline void mem_cgroup_replace_page(struct page *old, struct page *new) 542{ 543} 544 545static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone, 546 struct mem_cgroup *memcg) 547{ 548 return &zone->lruvec; 549} 550 551static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page, 552 struct zone *zone) 553{ 554 return &zone->lruvec; 555} 556 557static inline bool mm_match_cgroup(struct mm_struct *mm, 558 struct mem_cgroup *memcg) 559{ 560 return true; 561} 562 563static inline bool task_in_mem_cgroup(struct task_struct *task, 564 const struct mem_cgroup *memcg) 565{ 566 return true; 567} 568 569static inline struct mem_cgroup * 570mem_cgroup_iter(struct mem_cgroup *root, 571 struct mem_cgroup *prev, 572 struct mem_cgroup_reclaim_cookie *reclaim) 573{ 574 return NULL; 575} 576 577static inline void mem_cgroup_iter_break(struct mem_cgroup *root, 578 struct mem_cgroup *prev) 579{ 580} 581 582static inline bool mem_cgroup_disabled(void) 583{ 584 return true; 585} 586 587static inline bool 588mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec) 589{ 590 return true; 591} 592 593static inline bool mem_cgroup_lruvec_online(struct lruvec *lruvec) 594{ 595 return true; 596} 597 598static inline unsigned long 599mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) 600{ 601 return 0; 602} 603 604static inline void 605mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 606 int increment) 607{ 608} 609 610static inline void 611mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 612{ 613} 614 615static inline struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page) 616{ 617 return NULL; 618} 619 620static inline void mem_cgroup_end_page_stat(struct mem_cgroup *memcg) 621{ 622} 623 624static inline void mem_cgroup_handle_over_high(void) 625{ 626} 627 628static inline void mem_cgroup_oom_enable(void) 629{ 630} 631 632static inline void mem_cgroup_oom_disable(void) 633{ 634} 635 636static inline bool task_in_memcg_oom(struct task_struct *p) 637{ 638 return false; 639} 640 641static inline bool mem_cgroup_oom_synchronize(bool wait) 642{ 643 return false; 644} 645 646static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg, 647 enum mem_cgroup_stat_index idx) 648{ 649} 650 651static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg, 652 enum mem_cgroup_stat_index idx) 653{ 654} 655 656static inline 657unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, 658 gfp_t gfp_mask, 659 unsigned long *total_scanned) 660{ 661 return 0; 662} 663 664static inline void mem_cgroup_split_huge_fixup(struct page *head) 665{ 666} 667 668static inline 669void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx) 670{ 671} 672#endif /* CONFIG_MEMCG */ 673 674enum { 675 UNDER_LIMIT, 676 SOFT_LIMIT, 677 OVER_LIMIT, 678}; 679 680#ifdef CONFIG_CGROUP_WRITEBACK 681 682struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg); 683struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb); 684void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 685 unsigned long *pheadroom, unsigned long *pdirty, 686 unsigned long *pwriteback); 687 688#else /* CONFIG_CGROUP_WRITEBACK */ 689 690static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 691{ 692 return NULL; 693} 694 695static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb, 696 unsigned long *pfilepages, 697 unsigned long *pheadroom, 698 unsigned long *pdirty, 699 unsigned long *pwriteback) 700{ 701} 702 703#endif /* CONFIG_CGROUP_WRITEBACK */ 704 705struct sock; 706#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM) 707void sock_update_memcg(struct sock *sk); 708void sock_release_memcg(struct sock *sk); 709#else 710static inline void sock_update_memcg(struct sock *sk) 711{ 712} 713static inline void sock_release_memcg(struct sock *sk) 714{ 715} 716#endif /* CONFIG_INET && CONFIG_MEMCG_KMEM */ 717 718#ifdef CONFIG_MEMCG_KMEM 719extern struct static_key memcg_kmem_enabled_key; 720 721extern int memcg_nr_cache_ids; 722void memcg_get_cache_ids(void); 723void memcg_put_cache_ids(void); 724 725/* 726 * Helper macro to loop through all memcg-specific caches. Callers must still 727 * check if the cache is valid (it is either valid or NULL). 728 * the slab_mutex must be held when looping through those caches 729 */ 730#define for_each_memcg_cache_index(_idx) \ 731 for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++) 732 733static inline bool memcg_kmem_enabled(void) 734{ 735 return static_key_false(&memcg_kmem_enabled_key); 736} 737 738static inline bool memcg_kmem_is_active(struct mem_cgroup *memcg) 739{ 740 return memcg->kmem_acct_active; 741} 742 743/* 744 * In general, we'll do everything in our power to not incur in any overhead 745 * for non-memcg users for the kmem functions. Not even a function call, if we 746 * can avoid it. 747 * 748 * Therefore, we'll inline all those functions so that in the best case, we'll 749 * see that kmemcg is off for everybody and proceed quickly. If it is on, 750 * we'll still do most of the flag checking inline. We check a lot of 751 * conditions, but because they are pretty simple, they are expected to be 752 * fast. 753 */ 754int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, 755 struct mem_cgroup *memcg); 756int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order); 757void __memcg_kmem_uncharge(struct page *page, int order); 758 759/* 760 * helper for acessing a memcg's index. It will be used as an index in the 761 * child cache array in kmem_cache, and also to derive its name. This function 762 * will return -1 when this is not a kmem-limited memcg. 763 */ 764static inline int memcg_cache_id(struct mem_cgroup *memcg) 765{ 766 return memcg ? memcg->kmemcg_id : -1; 767} 768 769struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep); 770void __memcg_kmem_put_cache(struct kmem_cache *cachep); 771 772static inline bool __memcg_kmem_bypass(gfp_t gfp) 773{ 774 if (!memcg_kmem_enabled()) 775 return true; 776 if (gfp & __GFP_NOACCOUNT) 777 return true; 778 if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD)) 779 return true; 780 return false; 781} 782 783/** 784 * memcg_kmem_charge: charge a kmem page 785 * @page: page to charge 786 * @gfp: reclaim mode 787 * @order: allocation order 788 * 789 * Returns 0 on success, an error code on failure. 790 */ 791static __always_inline int memcg_kmem_charge(struct page *page, 792 gfp_t gfp, int order) 793{ 794 if (__memcg_kmem_bypass(gfp)) 795 return 0; 796 return __memcg_kmem_charge(page, gfp, order); 797} 798 799/** 800 * memcg_kmem_uncharge: uncharge a kmem page 801 * @page: page to uncharge 802 * @order: allocation order 803 */ 804static __always_inline void memcg_kmem_uncharge(struct page *page, int order) 805{ 806 if (memcg_kmem_enabled()) 807 __memcg_kmem_uncharge(page, order); 808} 809 810/** 811 * memcg_kmem_get_cache: selects the correct per-memcg cache for allocation 812 * @cachep: the original global kmem cache 813 * @gfp: allocation flags. 814 * 815 * All memory allocated from a per-memcg cache is charged to the owner memcg. 816 */ 817static __always_inline struct kmem_cache * 818memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) 819{ 820 if (__memcg_kmem_bypass(gfp)) 821 return cachep; 822 return __memcg_kmem_get_cache(cachep); 823} 824 825static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep) 826{ 827 if (memcg_kmem_enabled()) 828 __memcg_kmem_put_cache(cachep); 829} 830#else 831#define for_each_memcg_cache_index(_idx) \ 832 for (; NULL; ) 833 834static inline bool memcg_kmem_enabled(void) 835{ 836 return false; 837} 838 839static inline bool memcg_kmem_is_active(struct mem_cgroup *memcg) 840{ 841 return false; 842} 843 844static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order) 845{ 846 return 0; 847} 848 849static inline void memcg_kmem_uncharge(struct page *page, int order) 850{ 851} 852 853static inline int memcg_cache_id(struct mem_cgroup *memcg) 854{ 855 return -1; 856} 857 858static inline void memcg_get_cache_ids(void) 859{ 860} 861 862static inline void memcg_put_cache_ids(void) 863{ 864} 865 866static inline struct kmem_cache * 867memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) 868{ 869 return cachep; 870} 871 872static inline void memcg_kmem_put_cache(struct kmem_cache *cachep) 873{ 874} 875#endif /* CONFIG_MEMCG_KMEM */ 876#endif /* _LINUX_MEMCONTROL_H */ 877