93 int fprop_local_init_single(struct fprop_local_single *pl)  in fprop_local_init_single()  argument
95 	pl->events = 0;  in fprop_local_init_single()
96 	pl->period = 0;  in fprop_local_init_single()
97 	raw_spin_lock_init(&pl->lock);  in fprop_local_init_single()
101 void fprop_local_destroy_single(struct fprop_local_single *pl)  in fprop_local_destroy_single()  argument
106 					struct fprop_local_single *pl)  in fprop_reflect_period_single()  argument
112 	if (pl->period == period)  in fprop_reflect_period_single()
114 	raw_spin_lock_irqsave(&pl->lock, flags);  in fprop_reflect_period_single()
116 	if (pl->period >= period) {  in fprop_reflect_period_single()
117 		raw_spin_unlock_irqrestore(&pl->lock, flags);  in fprop_reflect_period_single()
121 	if (period - pl->period < BITS_PER_LONG)  in fprop_reflect_period_single()
122 		pl->events >>= period - pl->period;  in fprop_reflect_period_single()
124 		pl->events = 0;  in fprop_reflect_period_single()
125 	pl->period = period;  in fprop_reflect_period_single()
126 	raw_spin_unlock_irqrestore(&pl->lock, flags);  in fprop_reflect_period_single()
130 void __fprop_inc_single(struct fprop_global *p, struct fprop_local_single *pl)  in __fprop_inc_single()  argument
132 	fprop_reflect_period_single(p, pl);  in __fprop_inc_single()
133 	pl->events++;  in __fprop_inc_single()
139 			   struct fprop_local_single *pl,  in fprop_fraction_single()  argument
147 		fprop_reflect_period_single(p, pl);  in fprop_fraction_single()
148 		num = pl->events;  in fprop_fraction_single()
171 int fprop_local_init_percpu(struct fprop_local_percpu *pl, gfp_t gfp)  in fprop_local_init_percpu()  argument
175 	err = percpu_counter_init(&pl->events, 0, gfp);  in fprop_local_init_percpu()
178 	pl->period = 0;  in fprop_local_init_percpu()
179 	raw_spin_lock_init(&pl->lock);  in fprop_local_init_percpu()
183 void fprop_local_destroy_percpu(struct fprop_local_percpu *pl)  in fprop_local_destroy_percpu()  argument
185 	percpu_counter_destroy(&pl->events);  in fprop_local_destroy_percpu()
189 					struct fprop_local_percpu *pl)  in fprop_reflect_period_percpu()  argument
195 	if (pl->period == period)  in fprop_reflect_period_percpu()
197 	raw_spin_lock_irqsave(&pl->lock, flags);  in fprop_reflect_period_percpu()
199 	if (pl->period >= period) {  in fprop_reflect_period_percpu()
200 		raw_spin_unlock_irqrestore(&pl->lock, flags);  in fprop_reflect_period_percpu()
204 	if (period - pl->period < BITS_PER_LONG) {  in fprop_reflect_period_percpu()
205 		s64 val = percpu_counter_read(&pl->events);  in fprop_reflect_period_percpu()
208 			val = percpu_counter_sum(&pl->events);  in fprop_reflect_period_percpu()
210 		__percpu_counter_add(&pl->events,  in fprop_reflect_period_percpu()
211 			-val + (val >> (period-pl->period)), PROP_BATCH);  in fprop_reflect_period_percpu()
213 		percpu_counter_set(&pl->events, 0);  in fprop_reflect_period_percpu()
214 	pl->period = period;  in fprop_reflect_period_percpu()
215 	raw_spin_unlock_irqrestore(&pl->lock, flags);  in fprop_reflect_period_percpu()
219 void __fprop_inc_percpu(struct fprop_global *p, struct fprop_local_percpu *pl)  in __fprop_inc_percpu()  argument
221 	fprop_reflect_period_percpu(p, pl);  in __fprop_inc_percpu()
222 	__percpu_counter_add(&pl->events, 1, PROP_BATCH);  in __fprop_inc_percpu()
227 			   struct fprop_local_percpu *pl,  in fprop_fraction_percpu()  argument
235 		fprop_reflect_period_percpu(p, pl);  in fprop_fraction_percpu()
236 		num = percpu_counter_read_positive(&pl->events);  in fprop_fraction_percpu()
259 			    struct fprop_local_percpu *pl, int max_frac)  in __fprop_inc_percpu_max()  argument
264 		fprop_fraction_percpu(p, pl, &numerator, &denominator);  in __fprop_inc_percpu_max()
269 		fprop_reflect_period_percpu(p, pl);  in __fprop_inc_percpu_max()
270 	__percpu_counter_add(&pl->events, 1, PROP_BATCH);  in __fprop_inc_percpu_max()