95 	void                     (*flush_function)(struct ib_fmr_pool *pool,
114 static inline struct ib_pool_fmr *ib_fmr_cache_lookup(struct ib_fmr_pool *pool,  in ib_fmr_cache_lookup()  argument
122 	if (!pool->cache_bucket)  in ib_fmr_cache_lookup()
125 	bucket = pool->cache_bucket + ib_fmr_hash(*page_list);  in ib_fmr_cache_lookup()
137 static void ib_fmr_batch_release(struct ib_fmr_pool *pool)  in ib_fmr_batch_release()  argument
144 	spin_lock_irq(&pool->pool_lock);  in ib_fmr_batch_release()
146 	list_for_each_entry(fmr, &pool->dirty_list, list) {  in ib_fmr_batch_release()
159 	list_splice_init(&pool->dirty_list, &unmap_list);  in ib_fmr_batch_release()
160 	pool->dirty_len = 0;  in ib_fmr_batch_release()
162 	spin_unlock_irq(&pool->pool_lock);  in ib_fmr_batch_release()
172 	spin_lock_irq(&pool->pool_lock);  in ib_fmr_batch_release()
173 	list_splice(&unmap_list, &pool->free_list);  in ib_fmr_batch_release()
174 	spin_unlock_irq(&pool->pool_lock);  in ib_fmr_batch_release()
179 	struct ib_fmr_pool *pool = pool_ptr;  in ib_fmr_cleanup_thread()  local
182 		if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0) {  in ib_fmr_cleanup_thread()
183 			ib_fmr_batch_release(pool);  in ib_fmr_cleanup_thread()
185 			atomic_inc(&pool->flush_ser);  in ib_fmr_cleanup_thread()
186 			wake_up_interruptible(&pool->force_wait);  in ib_fmr_cleanup_thread()
188 			if (pool->flush_function)  in ib_fmr_cleanup_thread()
189 				pool->flush_function(pool, pool->flush_arg);  in ib_fmr_cleanup_thread()
193 		if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) >= 0 &&  in ib_fmr_cleanup_thread()
214 	struct ib_fmr_pool *pool;  in ib_create_fmr_pool()  local
251 	pool = kmalloc(sizeof *pool, GFP_KERNEL);  in ib_create_fmr_pool()
252 	if (!pool) {  in ib_create_fmr_pool()
257 	pool->cache_bucket   = NULL;  in ib_create_fmr_pool()
259 	pool->flush_function = params->flush_function;  in ib_create_fmr_pool()
260 	pool->flush_arg      = params->flush_arg;  in ib_create_fmr_pool()
262 	INIT_LIST_HEAD(&pool->free_list);  in ib_create_fmr_pool()
263 	INIT_LIST_HEAD(&pool->dirty_list);  in ib_create_fmr_pool()
266 		pool->cache_bucket =  in ib_create_fmr_pool()
267 			kmalloc(IB_FMR_HASH_SIZE * sizeof *pool->cache_bucket,  in ib_create_fmr_pool()
269 		if (!pool->cache_bucket) {  in ib_create_fmr_pool()
276 			INIT_HLIST_HEAD(pool->cache_bucket + i);  in ib_create_fmr_pool()
279 	pool->pool_size       = 0;  in ib_create_fmr_pool()
280 	pool->max_pages       = params->max_pages_per_fmr;  in ib_create_fmr_pool()
281 	pool->max_remaps      = max_remaps;  in ib_create_fmr_pool()
282 	pool->dirty_watermark = params->dirty_watermark;  in ib_create_fmr_pool()
283 	pool->dirty_len       = 0;  in ib_create_fmr_pool()
284 	spin_lock_init(&pool->pool_lock);  in ib_create_fmr_pool()
285 	atomic_set(&pool->req_ser,   0);  in ib_create_fmr_pool()
286 	atomic_set(&pool->flush_ser, 0);  in ib_create_fmr_pool()
287 	init_waitqueue_head(&pool->force_wait);  in ib_create_fmr_pool()
289 	pool->thread = kthread_run(ib_fmr_cleanup_thread,  in ib_create_fmr_pool()
290 				   pool,  in ib_create_fmr_pool()
293 	if (IS_ERR(pool->thread)) {  in ib_create_fmr_pool()
295 		ret = PTR_ERR(pool->thread);  in ib_create_fmr_pool()
303 			.max_maps   = pool->max_remaps,  in ib_create_fmr_pool()
308 		if (pool->cache_bucket)  in ib_create_fmr_pool()
319 			fmr->pool             = pool;  in ib_create_fmr_pool()
332 			list_add_tail(&fmr->list, &pool->free_list);  in ib_create_fmr_pool()
333 			++pool->pool_size;  in ib_create_fmr_pool()
337 	return pool;  in ib_create_fmr_pool()
340 	kfree(pool->cache_bucket);  in ib_create_fmr_pool()
341 	kfree(pool);  in ib_create_fmr_pool()
346 	ib_destroy_fmr_pool(pool);  in ib_create_fmr_pool()
358 void ib_destroy_fmr_pool(struct ib_fmr_pool *pool)  in ib_destroy_fmr_pool()  argument
365 	kthread_stop(pool->thread);  in ib_destroy_fmr_pool()
366 	ib_fmr_batch_release(pool);  in ib_destroy_fmr_pool()
369 	list_for_each_entry_safe(fmr, tmp, &pool->free_list, list) {  in ib_destroy_fmr_pool()
381 	if (i < pool->pool_size)  in ib_destroy_fmr_pool()
383 		       pool->pool_size - i);  in ib_destroy_fmr_pool()
385 	kfree(pool->cache_bucket);  in ib_destroy_fmr_pool()
386 	kfree(pool);  in ib_destroy_fmr_pool()
396 int ib_flush_fmr_pool(struct ib_fmr_pool *pool)  in ib_flush_fmr_pool()  argument
407 	spin_lock_irq(&pool->pool_lock);  in ib_flush_fmr_pool()
408 	list_for_each_entry_safe(fmr, next, &pool->free_list, list) {  in ib_flush_fmr_pool()
410 			list_move(&fmr->list, &pool->dirty_list);  in ib_flush_fmr_pool()
412 	spin_unlock_irq(&pool->pool_lock);  in ib_flush_fmr_pool()
414 	serial = atomic_inc_return(&pool->req_ser);  in ib_flush_fmr_pool()
415 	wake_up_process(pool->thread);  in ib_flush_fmr_pool()
417 	if (wait_event_interruptible(pool->force_wait,  in ib_flush_fmr_pool()
418 				     atomic_read(&pool->flush_ser) - serial >= 0))  in ib_flush_fmr_pool()
439 	struct ib_fmr_pool *pool = pool_handle;  in ib_fmr_pool_map_phys()  local
444 	if (list_len < 1 || list_len > pool->max_pages)  in ib_fmr_pool_map_phys()
447 	spin_lock_irqsave(&pool->pool_lock, flags);  in ib_fmr_pool_map_phys()
448 	fmr = ib_fmr_cache_lookup(pool,  in ib_fmr_pool_map_phys()
459 		spin_unlock_irqrestore(&pool->pool_lock, flags);  in ib_fmr_pool_map_phys()
464 	if (list_empty(&pool->free_list)) {  in ib_fmr_pool_map_phys()
465 		spin_unlock_irqrestore(&pool->pool_lock, flags);  in ib_fmr_pool_map_phys()
469 	fmr = list_entry(pool->free_list.next, struct ib_pool_fmr, list);  in ib_fmr_pool_map_phys()
472 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in ib_fmr_pool_map_phys()
478 		spin_lock_irqsave(&pool->pool_lock, flags);  in ib_fmr_pool_map_phys()
479 		list_add(&fmr->list, &pool->free_list);  in ib_fmr_pool_map_phys()
480 		spin_unlock_irqrestore(&pool->pool_lock, flags);  in ib_fmr_pool_map_phys()
490 	if (pool->cache_bucket) {  in ib_fmr_pool_map_phys()
495 		spin_lock_irqsave(&pool->pool_lock, flags);  in ib_fmr_pool_map_phys()
497 			       pool->cache_bucket + ib_fmr_hash(fmr->page_list[0]));  in ib_fmr_pool_map_phys()
498 		spin_unlock_irqrestore(&pool->pool_lock, flags);  in ib_fmr_pool_map_phys()
514 	struct ib_fmr_pool *pool;  in ib_fmr_pool_unmap()  local
517 	pool = fmr->pool;  in ib_fmr_pool_unmap()
519 	spin_lock_irqsave(&pool->pool_lock, flags);  in ib_fmr_pool_unmap()
523 		if (fmr->remap_count < pool->max_remaps) {  in ib_fmr_pool_unmap()
524 			list_add_tail(&fmr->list, &pool->free_list);  in ib_fmr_pool_unmap()
526 			list_add_tail(&fmr->list, &pool->dirty_list);  in ib_fmr_pool_unmap()
527 			if (++pool->dirty_len >= pool->dirty_watermark) {  in ib_fmr_pool_unmap()
528 				atomic_inc(&pool->req_ser);  in ib_fmr_pool_unmap()
529 				wake_up_process(pool->thread);  in ib_fmr_pool_unmap()
540 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in ib_fmr_pool_unmap()