24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
27 	const int nr = pool->curr_nr;  in poison_error()
33 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
60 	if (pool->free == mempool_free_slab || pool->free == mempool_kfree)  in check_element()
61 		__check_element(pool, element, ksize(element));  in check_element()
64 	if (pool->free == mempool_free_pages) {  in check_element()
65 		int order = (int)(long)pool->pool_data;  in check_element()
68 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));  in check_element()
81 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
84 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in poison_element()
88 	if (pool->alloc == mempool_alloc_pages) {  in poison_element()
89 		int order = (int)(long)pool->pool_data;  in poison_element()
97 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
100 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
105 static void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
107 	if (pool->alloc == mempool_alloc_slab)  in kasan_poison_element()
108 		kasan_slab_free(pool->pool_data, element);  in kasan_poison_element()
109 	if (pool->alloc == mempool_kmalloc)  in kasan_poison_element()
111 	if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
112 		kasan_free_pages(element, (unsigned long)pool->pool_data);  in kasan_poison_element()
115 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
117 	if (pool->alloc == mempool_alloc_slab)  in kasan_unpoison_element()
118 		kasan_slab_alloc(pool->pool_data, element);  in kasan_unpoison_element()
119 	if (pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
120 		kasan_krealloc(element, (size_t)pool->pool_data);  in kasan_unpoison_element()
121 	if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
122 		kasan_alloc_pages(element, (unsigned long)pool->pool_data);  in kasan_unpoison_element()
125 static void add_element(mempool_t *pool, void *element)  in add_element()  argument
127 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
128 	poison_element(pool, element);  in add_element()
129 	kasan_poison_element(pool, element);  in add_element()
130 	pool->elements[pool->curr_nr++] = element;  in add_element()
133 static void *remove_element(mempool_t *pool)  in remove_element()  argument
135 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
137 	BUG_ON(pool->curr_nr < 0);  in remove_element()
138 	check_element(pool, element);  in remove_element()
139 	kasan_unpoison_element(pool, element);  in remove_element()
151 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
153 	while (pool->curr_nr) {  in mempool_destroy()
154 		void *element = remove_element(pool);  in mempool_destroy()
155 		pool->free(element, pool->pool_data);  in mempool_destroy()
157 	kfree(pool->elements);  in mempool_destroy()
158 	kfree(pool);  in mempool_destroy()
188 	mempool_t *pool;  in mempool_create_node()  local
189 	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);  in mempool_create_node()
190 	if (!pool)  in mempool_create_node()
192 	pool->elements = kmalloc_node(min_nr * sizeof(void *),  in mempool_create_node()
194 	if (!pool->elements) {  in mempool_create_node()
195 		kfree(pool);  in mempool_create_node()
198 	spin_lock_init(&pool->lock);  in mempool_create_node()
199 	pool->min_nr = min_nr;  in mempool_create_node()
200 	pool->pool_data = pool_data;  in mempool_create_node()
201 	init_waitqueue_head(&pool->wait);  in mempool_create_node()
202 	pool->alloc = alloc_fn;  in mempool_create_node()
203 	pool->free = free_fn;  in mempool_create_node()
208 	while (pool->curr_nr < pool->min_nr) {  in mempool_create_node()
211 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_create_node()
213 			mempool_destroy(pool);  in mempool_create_node()
216 		add_element(pool, element);  in mempool_create_node()
218 	return pool;  in mempool_create_node()
238 int mempool_resize(mempool_t *pool, int new_min_nr)  in mempool_resize()  argument
247 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
248 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
249 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
250 			element = remove_element(pool);  in mempool_resize()
251 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
252 			pool->free(element, pool->pool_data);  in mempool_resize()
253 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
255 		pool->min_nr = new_min_nr;  in mempool_resize()
258 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
266 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
267 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
269 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
273 	memcpy(new_elements, pool->elements,  in mempool_resize()
274 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
275 	kfree(pool->elements);  in mempool_resize()
276 	pool->elements = new_elements;  in mempool_resize()
277 	pool->min_nr = new_min_nr;  in mempool_resize()
279 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
280 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
281 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
284 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
285 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
286 			add_element(pool, element);  in mempool_resize()
288 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
289 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
294 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
312 void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc()  argument
330 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
334 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc()
335 	if (likely(pool->curr_nr)) {  in mempool_alloc()
336 		element = remove_element(pool);  in mempool_alloc()
337 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
353 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
360 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
366 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc()
368 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
376 	finish_wait(&pool->wait, &wait);  in mempool_alloc()
389 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
429 	if (unlikely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
430 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
431 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
432 			add_element(pool, element);  in mempool_free()
433 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
434 			wake_up(&pool->wait);  in mempool_free()
437 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
439 	pool->free(element, pool->pool_data);  in mempool_free()