24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
27 	const int nr = pool->curr_nr;  in poison_error()
33 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
60 	if (pool->free == mempool_free_slab || pool->free == mempool_kfree)  in check_element()
61 		__check_element(pool, element, ksize(element));  in check_element()
64 	if (pool->free == mempool_free_pages) {  in check_element()
65 		int order = (int)(long)pool->pool_data;  in check_element()
68 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));  in check_element()
81 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
84 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in poison_element()
88 	if (pool->alloc == mempool_alloc_pages) {  in poison_element()
89 		int order = (int)(long)pool->pool_data;  in poison_element()
97 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
100 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
105 static void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
107 	if (pool->alloc == mempool_alloc_slab)  in kasan_poison_element()
108 		kasan_slab_free(pool->pool_data, element);  in kasan_poison_element()
109 	if (pool->alloc == mempool_kmalloc)  in kasan_poison_element()
111 	if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
112 		kasan_free_pages(element, (unsigned long)pool->pool_data);  in kasan_poison_element()
115 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
117 	if (pool->alloc == mempool_alloc_slab)  in kasan_unpoison_element()
118 		kasan_slab_alloc(pool->pool_data, element);  in kasan_unpoison_element()
119 	if (pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
120 		kasan_krealloc(element, (size_t)pool->pool_data);  in kasan_unpoison_element()
121 	if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
122 		kasan_alloc_pages(element, (unsigned long)pool->pool_data);  in kasan_unpoison_element()
125 static void add_element(mempool_t *pool, void *element)  in add_element()  argument
127 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
128 	poison_element(pool, element);  in add_element()
129 	kasan_poison_element(pool, element);  in add_element()
130 	pool->elements[pool->curr_nr++] = element;  in add_element()
133 static void *remove_element(mempool_t *pool)  in remove_element()  argument
135 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
137 	BUG_ON(pool->curr_nr < 0);  in remove_element()
138 	check_element(pool, element);  in remove_element()
139 	kasan_unpoison_element(pool, element);  in remove_element()
151 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
153 	if (unlikely(!pool))  in mempool_destroy()
156 	while (pool->curr_nr) {  in mempool_destroy()
157 		void *element = remove_element(pool);  in mempool_destroy()
158 		pool->free(element, pool->pool_data);  in mempool_destroy()
160 	kfree(pool->elements);  in mempool_destroy()
161 	kfree(pool);  in mempool_destroy()
191 	mempool_t *pool;  in mempool_create_node()  local
192 	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);  in mempool_create_node()
193 	if (!pool)  in mempool_create_node()
195 	pool->elements = kmalloc_node(min_nr * sizeof(void *),  in mempool_create_node()
197 	if (!pool->elements) {  in mempool_create_node()
198 		kfree(pool);  in mempool_create_node()
201 	spin_lock_init(&pool->lock);  in mempool_create_node()
202 	pool->min_nr = min_nr;  in mempool_create_node()
203 	pool->pool_data = pool_data;  in mempool_create_node()
204 	init_waitqueue_head(&pool->wait);  in mempool_create_node()
205 	pool->alloc = alloc_fn;  in mempool_create_node()
206 	pool->free = free_fn;  in mempool_create_node()
211 	while (pool->curr_nr < pool->min_nr) {  in mempool_create_node()
214 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_create_node()
216 			mempool_destroy(pool);  in mempool_create_node()
219 		add_element(pool, element);  in mempool_create_node()
221 	return pool;  in mempool_create_node()
241 int mempool_resize(mempool_t *pool, int new_min_nr)  in mempool_resize()  argument
250 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
251 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
252 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
253 			element = remove_element(pool);  in mempool_resize()
254 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
255 			pool->free(element, pool->pool_data);  in mempool_resize()
256 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
258 		pool->min_nr = new_min_nr;  in mempool_resize()
261 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
269 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
270 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
272 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
276 	memcpy(new_elements, pool->elements,  in mempool_resize()
277 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
278 	kfree(pool->elements);  in mempool_resize()
279 	pool->elements = new_elements;  in mempool_resize()
280 	pool->min_nr = new_min_nr;  in mempool_resize()
282 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
283 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
284 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
287 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
288 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
289 			add_element(pool, element);  in mempool_resize()
291 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
292 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
297 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
315 void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc()  argument
333 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
337 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc()
338 	if (likely(pool->curr_nr)) {  in mempool_alloc()
339 		element = remove_element(pool);  in mempool_alloc()
340 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
356 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
363 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
369 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc()
371 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
379 	finish_wait(&pool->wait, &wait);  in mempool_alloc()
392 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
432 	if (unlikely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
433 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
434 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
435 			add_element(pool, element);  in mempool_free()
436 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
437 			wake_up(&pool->wait);  in mempool_free()
440 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
442 	pool->free(element, pool->pool_data);  in mempool_free()