24 	struct percpu_counter *fbc = addr;  in percpu_counter_fixup_free()  local
28 		percpu_counter_destroy(fbc);  in percpu_counter_fixup_free()
29 		debug_object_free(fbc, &percpu_counter_debug_descr);  in percpu_counter_fixup_free()
41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
43 	debug_object_init(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
44 	debug_object_activate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
49 	debug_object_deactivate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
50 	debug_object_free(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)  in percpu_counter_set()  argument
65 	raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_set()
67 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_set()
70 	fbc->count = amount;  in percpu_counter_set()
71 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_set()
75 void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)  in __percpu_counter_add()  argument
80 	count = __this_cpu_read(*fbc->counters) + amount;  in __percpu_counter_add()
83 		raw_spin_lock_irqsave(&fbc->lock, flags);  in __percpu_counter_add()
84 		fbc->count += count;  in __percpu_counter_add()
85 		__this_cpu_sub(*fbc->counters, count - amount);  in __percpu_counter_add()
86 		raw_spin_unlock_irqrestore(&fbc->lock, flags);  in __percpu_counter_add()
88 		this_cpu_add(*fbc->counters, amount);  in __percpu_counter_add()
98 s64 __percpu_counter_sum(struct percpu_counter *fbc)  in __percpu_counter_sum()  argument
104 	raw_spin_lock_irqsave(&fbc->lock, flags);  in __percpu_counter_sum()
105 	ret = fbc->count;  in __percpu_counter_sum()
107 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in __percpu_counter_sum()
110 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in __percpu_counter_sum()
115 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,  in __percpu_counter_init()  argument
120 	raw_spin_lock_init(&fbc->lock);  in __percpu_counter_init()
121 	lockdep_set_class(&fbc->lock, key);  in __percpu_counter_init()
122 	fbc->count = amount;  in __percpu_counter_init()
123 	fbc->counters = alloc_percpu_gfp(s32, gfp);  in __percpu_counter_init()
124 	if (!fbc->counters)  in __percpu_counter_init()
127 	debug_percpu_counter_activate(fbc);  in __percpu_counter_init()
130 	INIT_LIST_HEAD(&fbc->list);  in __percpu_counter_init()
132 	list_add(&fbc->list, &percpu_counters);  in __percpu_counter_init()
139 void percpu_counter_destroy(struct percpu_counter *fbc)  in percpu_counter_destroy()  argument
143 	if (!fbc->counters)  in percpu_counter_destroy()
146 	debug_percpu_counter_deactivate(fbc);  in percpu_counter_destroy()
150 	list_del(&fbc->list);  in percpu_counter_destroy()
153 	free_percpu(fbc->counters);  in percpu_counter_destroy()
154 	fbc->counters = NULL;  in percpu_counter_destroy()
173 	struct percpu_counter *fbc;  in percpu_counter_hotcpu_callback()  local
181 	list_for_each_entry(fbc, &percpu_counters, list) {  in percpu_counter_hotcpu_callback()
185 		raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_hotcpu_callback()
186 		pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_hotcpu_callback()
187 		fbc->count += *pcount;  in percpu_counter_hotcpu_callback()
189 		raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_hotcpu_callback()
200 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)  in __percpu_counter_compare()  argument
204 	count = percpu_counter_read(fbc);  in __percpu_counter_compare()
213 	count = percpu_counter_sum(fbc);  in __percpu_counter_compare()