root/include/linux/page_ref.h

INCLUDED FROM

DEFINITIONS

1. __page_ref_set
2. __page_ref_mod
3. __page_ref_mod_and_test
4. __page_ref_mod_and_return
5. __page_ref_mod_unless
6. __page_ref_freeze
7. __page_ref_unfreeze
8. page_ref_count
9. page_count
10. set_page_count
11. init_page_count
12. page_ref_add
13. page_ref_sub
14. page_ref_inc
15. page_ref_dec
16. page_ref_sub_and_test
17. page_ref_inc_return
18. page_ref_dec_and_test
19. page_ref_dec_return
20. page_ref_add_unless
21. page_ref_freeze
22. page_ref_unfreeze

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_PAGE_REF_H
   3 #define _LINUX_PAGE_REF_H
   4 
   5 #include <linux/atomic.h>
   6 #include <linux/mm_types.h>
   7 #include <linux/page-flags.h>
   8 #include <linux/tracepoint-defs.h>
   9 
  10 extern struct tracepoint __tracepoint_page_ref_set;
  11 extern struct tracepoint __tracepoint_page_ref_mod;
  12 extern struct tracepoint __tracepoint_page_ref_mod_and_test;
  13 extern struct tracepoint __tracepoint_page_ref_mod_and_return;
  14 extern struct tracepoint __tracepoint_page_ref_mod_unless;
  15 extern struct tracepoint __tracepoint_page_ref_freeze;
  16 extern struct tracepoint __tracepoint_page_ref_unfreeze;
  17 
  18 #ifdef CONFIG_DEBUG_PAGE_REF
  19 
  20 /*
  21  * Ideally we would want to use the trace_<tracepoint>_enabled() helper
  22  * functions. But due to include header file issues, that is not
  23  * feasible. Instead we have to open code the static key functions.
  24  *
  25  * See trace_##name##_enabled(void) in include/linux/tracepoint.h
  26  */
  27 #define page_ref_tracepoint_active(t) static_key_false(&(t).key)
  28 
  29 extern void __page_ref_set(struct page *page, int v);
  30 extern void __page_ref_mod(struct page *page, int v);
  31 extern void __page_ref_mod_and_test(struct page *page, int v, int ret);
  32 extern void __page_ref_mod_and_return(struct page *page, int v, int ret);
  33 extern void __page_ref_mod_unless(struct page *page, int v, int u);
  34 extern void __page_ref_freeze(struct page *page, int v, int ret);
  35 extern void __page_ref_unfreeze(struct page *page, int v);
  36 
  37 #else
  38 
  39 #define page_ref_tracepoint_active(t) false
  40 
  41 static inline void __page_ref_set(struct page *page, int v)
  42 {
  43 }
  44 static inline void __page_ref_mod(struct page *page, int v)
  45 {
  46 }
  47 static inline void __page_ref_mod_and_test(struct page *page, int v, int ret)
  48 {
  49 }
  50 static inline void __page_ref_mod_and_return(struct page *page, int v, int ret)
  51 {
  52 }
  53 static inline void __page_ref_mod_unless(struct page *page, int v, int u)
  54 {
  55 }
  56 static inline void __page_ref_freeze(struct page *page, int v, int ret)
  57 {
  58 }
  59 static inline void __page_ref_unfreeze(struct page *page, int v)
  60 {
  61 }
  62 
  63 #endif
  64 
  65 static inline int page_ref_count(struct page *page)
  66 {
  67         return atomic_read(&page->_refcount);
  68 }
  69 
  70 static inline int page_count(struct page *page)
  71 {
  72         return atomic_read(&compound_head(page)->_refcount);
  73 }
  74 
  75 static inline void set_page_count(struct page *page, int v)
  76 {
  77         atomic_set(&page->_refcount, v);
  78         if (page_ref_tracepoint_active(__tracepoint_page_ref_set))
  79                 __page_ref_set(page, v);
  80 }
  81 
  82 /*
  83  * Setup the page count before being freed into the page allocator for
  84  * the first time (boot or memory hotplug)
  85  */
  86 static inline void init_page_count(struct page *page)
  87 {
  88         set_page_count(page, 1);
  89 }
  90 
  91 static inline void page_ref_add(struct page *page, int nr)
  92 {
  93         atomic_add(nr, &page->_refcount);
  94         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
  95                 __page_ref_mod(page, nr);
  96 }
  97 
  98 static inline void page_ref_sub(struct page *page, int nr)
  99 {
 100         atomic_sub(nr, &page->_refcount);
 101         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
 102                 __page_ref_mod(page, -nr);
 103 }
 104 
 105 static inline void page_ref_inc(struct page *page)
 106 {
 107         atomic_inc(&page->_refcount);
 108         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
 109                 __page_ref_mod(page, 1);
 110 }
 111 
 112 static inline void page_ref_dec(struct page *page)
 113 {
 114         atomic_dec(&page->_refcount);
 115         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
 116                 __page_ref_mod(page, -1);
 117 }
 118 
 119 static inline int page_ref_sub_and_test(struct page *page, int nr)
 120 {
 121         int ret = atomic_sub_and_test(nr, &page->_refcount);
 122 
 123         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test))
 124                 __page_ref_mod_and_test(page, -nr, ret);
 125         return ret;
 126 }
 127 
 128 static inline int page_ref_inc_return(struct page *page)
 129 {
 130         int ret = atomic_inc_return(&page->_refcount);
 131 
 132         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return))
 133                 __page_ref_mod_and_return(page, 1, ret);
 134         return ret;
 135 }
 136 
 137 static inline int page_ref_dec_and_test(struct page *page)
 138 {
 139         int ret = atomic_dec_and_test(&page->_refcount);
 140 
 141         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test))
 142                 __page_ref_mod_and_test(page, -1, ret);
 143         return ret;
 144 }
 145 
 146 static inline int page_ref_dec_return(struct page *page)
 147 {
 148         int ret = atomic_dec_return(&page->_refcount);
 149 
 150         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return))
 151                 __page_ref_mod_and_return(page, -1, ret);
 152         return ret;
 153 }
 154 
 155 static inline int page_ref_add_unless(struct page *page, int nr, int u)
 156 {
 157         int ret = atomic_add_unless(&page->_refcount, nr, u);
 158 
 159         if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_unless))
 160                 __page_ref_mod_unless(page, nr, ret);
 161         return ret;
 162 }
 163 
 164 static inline int page_ref_freeze(struct page *page, int count)
 165 {
 166         int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count);
 167 
 168         if (page_ref_tracepoint_active(__tracepoint_page_ref_freeze))
 169                 __page_ref_freeze(page, count, ret);
 170         return ret;
 171 }
 172 
 173 static inline void page_ref_unfreeze(struct page *page, int count)
 174 {
 175         VM_BUG_ON_PAGE(page_count(page) != 0, page);
 176         VM_BUG_ON(count == 0);
 177 
 178         atomic_set_release(&page->_refcount, count);
 179         if (page_ref_tracepoint_active(__tracepoint_page_ref_unfreeze))
 180                 __page_ref_unfreeze(page, count);
 181 }
 182 
 183 #endif