60 static inline void steal_tags(struct percpu_ida *pool,  in steal_tags()  argument
63 	unsigned cpus_have_tags, cpu = pool->cpu_last_stolen;  in steal_tags()
66 	for (cpus_have_tags = cpumask_weight(&pool->cpus_have_tags);  in steal_tags()
68 		cpu = cpumask_next(cpu, &pool->cpus_have_tags);  in steal_tags()
71 			cpu = cpumask_first(&pool->cpus_have_tags);  in steal_tags()
76 		pool->cpu_last_stolen = cpu;  in steal_tags()
77 		remote = per_cpu_ptr(pool->tag_cpu, cpu);  in steal_tags()
79 		cpumask_clear_cpu(cpu, &pool->cpus_have_tags);  in steal_tags()
106 static inline void alloc_global_tags(struct percpu_ida *pool,  in alloc_global_tags()  argument
110 		  pool->freelist, &pool->nr_free,  in alloc_global_tags()
111 		  min(pool->nr_free, pool->percpu_batch_size));  in alloc_global_tags()
144 int percpu_ida_alloc(struct percpu_ida *pool, int state)  in percpu_ida_alloc()  argument
152 	tags = this_cpu_ptr(pool->tag_cpu);  in percpu_ida_alloc()
162 		spin_lock(&pool->lock);  in percpu_ida_alloc()
172 			prepare_to_wait(&pool->wait, &wait, state);  in percpu_ida_alloc()
175 			alloc_global_tags(pool, tags);  in percpu_ida_alloc()
177 			steal_tags(pool, tags);  in percpu_ida_alloc()
183 						&pool->cpus_have_tags);  in percpu_ida_alloc()
186 		spin_unlock(&pool->lock);  in percpu_ida_alloc()
200 		tags = this_cpu_ptr(pool->tag_cpu);  in percpu_ida_alloc()
203 		finish_wait(&pool->wait, &wait);  in percpu_ida_alloc()
216 void percpu_ida_free(struct percpu_ida *pool, unsigned tag)  in percpu_ida_free()  argument
222 	BUG_ON(tag >= pool->nr_tags);  in percpu_ida_free()
225 	tags = this_cpu_ptr(pool->tag_cpu);  in percpu_ida_free()
235 				&pool->cpus_have_tags);  in percpu_ida_free()
236 		wake_up(&pool->wait);  in percpu_ida_free()
239 	if (nr_free == pool->percpu_max_size) {  in percpu_ida_free()
240 		spin_lock(&pool->lock);  in percpu_ida_free()
246 		if (tags->nr_free == pool->percpu_max_size) {  in percpu_ida_free()
247 			move_tags(pool->freelist, &pool->nr_free,  in percpu_ida_free()
249 				  pool->percpu_batch_size);  in percpu_ida_free()
251 			wake_up(&pool->wait);  in percpu_ida_free()
253 		spin_unlock(&pool->lock);  in percpu_ida_free()
266 void percpu_ida_destroy(struct percpu_ida *pool)  in percpu_ida_destroy()  argument
268 	free_percpu(pool->tag_cpu);  in percpu_ida_destroy()
269 	free_pages((unsigned long) pool->freelist,  in percpu_ida_destroy()
270 		   get_order(pool->nr_tags * sizeof(unsigned)));  in percpu_ida_destroy()
286 int __percpu_ida_init(struct percpu_ida *pool, unsigned long nr_tags,  in __percpu_ida_init()  argument
291 	memset(pool, 0, sizeof(*pool));  in __percpu_ida_init()
293 	init_waitqueue_head(&pool->wait);  in __percpu_ida_init()
294 	spin_lock_init(&pool->lock);  in __percpu_ida_init()
295 	pool->nr_tags = nr_tags;  in __percpu_ida_init()
296 	pool->percpu_max_size = max_size;  in __percpu_ida_init()
297 	pool->percpu_batch_size = batch_size;  in __percpu_ida_init()
306 	pool->freelist = (void *) __get_free_pages(GFP_KERNEL, order);  in __percpu_ida_init()
307 	if (!pool->freelist)  in __percpu_ida_init()
311 		pool->freelist[i] = i;  in __percpu_ida_init()
313 	pool->nr_free = nr_tags;  in __percpu_ida_init()
315 	pool->tag_cpu = __alloc_percpu(sizeof(struct percpu_ida_cpu) +  in __percpu_ida_init()
316 				       pool->percpu_max_size * sizeof(unsigned),  in __percpu_ida_init()
318 	if (!pool->tag_cpu)  in __percpu_ida_init()
322 		spin_lock_init(&per_cpu_ptr(pool->tag_cpu, cpu)->lock);  in __percpu_ida_init()
326 	percpu_ida_destroy(pool);  in __percpu_ida_init()
341 int percpu_ida_for_each_free(struct percpu_ida *pool, percpu_ida_cb fn,  in percpu_ida_for_each_free()  argument
350 		remote = per_cpu_ptr(pool->tag_cpu, cpu);  in percpu_ida_for_each_free()
362 	spin_lock(&pool->lock);  in percpu_ida_for_each_free()
363 	for (i = 0; i < pool->nr_free; i++) {  in percpu_ida_for_each_free()
364 		err = fn(pool->freelist[i], data);  in percpu_ida_for_each_free()
368 	spin_unlock(&pool->lock);  in percpu_ida_for_each_free()
382 unsigned percpu_ida_free_tags(struct percpu_ida *pool, int cpu)  in percpu_ida_free_tags()  argument
386 		return pool->nr_free;  in percpu_ida_free_tags()
387 	remote = per_cpu_ptr(pool->tag_cpu, cpu);  in percpu_ida_free_tags()