49 	struct rds_ib_mr_pool	*pool;  member
106 static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool, int free_all, struct rds_ib_mr **);
241 	struct rds_ib_mr_pool *pool;  in rds_ib_create_mr_pool()  local
243 	pool = kzalloc(sizeof(*pool), GFP_KERNEL);  in rds_ib_create_mr_pool()
244 	if (!pool)  in rds_ib_create_mr_pool()
247 	pool->pool_type = pool_type;  in rds_ib_create_mr_pool()
248 	init_llist_head(&pool->free_list);  in rds_ib_create_mr_pool()
249 	init_llist_head(&pool->drop_list);  in rds_ib_create_mr_pool()
250 	init_llist_head(&pool->clean_list);  in rds_ib_create_mr_pool()
251 	mutex_init(&pool->flush_lock);  in rds_ib_create_mr_pool()
252 	init_waitqueue_head(&pool->flush_wait);  in rds_ib_create_mr_pool()
253 	INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker);  in rds_ib_create_mr_pool()
257 		pool->fmr_attr.max_pages = RDS_FMR_1M_MSG_SIZE + 1;  in rds_ib_create_mr_pool()
258 		pool->max_items = RDS_FMR_1M_POOL_SIZE;  in rds_ib_create_mr_pool()
261 		pool->fmr_attr.max_pages = RDS_FMR_8K_MSG_SIZE + 1;  in rds_ib_create_mr_pool()
262 		pool->max_items = RDS_FMR_8K_POOL_SIZE;  in rds_ib_create_mr_pool()
265 	pool->max_free_pinned = pool->max_items * pool->fmr_attr.max_pages / 4;  in rds_ib_create_mr_pool()
266 	pool->fmr_attr.max_maps = rds_ibdev->fmr_max_remaps;  in rds_ib_create_mr_pool()
267 	pool->fmr_attr.page_shift = PAGE_SHIFT;  in rds_ib_create_mr_pool()
268 	pool->max_items_soft = rds_ibdev->max_fmrs * 3 / 4;  in rds_ib_create_mr_pool()
270 	return pool;  in rds_ib_create_mr_pool()
281 void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *pool)  in rds_ib_destroy_mr_pool()  argument
283 	cancel_delayed_work_sync(&pool->flush_worker);  in rds_ib_destroy_mr_pool()
284 	rds_ib_flush_mr_pool(pool, 1, NULL);  in rds_ib_destroy_mr_pool()
285 	WARN_ON(atomic_read(&pool->item_count));  in rds_ib_destroy_mr_pool()
286 	WARN_ON(atomic_read(&pool->free_pinned));  in rds_ib_destroy_mr_pool()
287 	kfree(pool);  in rds_ib_destroy_mr_pool()
290 static inline struct rds_ib_mr *rds_ib_reuse_fmr(struct rds_ib_mr_pool *pool)  in rds_ib_reuse_fmr()  argument
299 	ret = llist_del_first(&pool->clean_list);  in rds_ib_reuse_fmr()
323 	struct rds_ib_mr_pool *pool;  in rds_ib_alloc_fmr()  local
328 		pool = rds_ibdev->mr_8k_pool;  in rds_ib_alloc_fmr()
330 		pool = rds_ibdev->mr_1m_pool;  in rds_ib_alloc_fmr()
332 	if (atomic_read(&pool->dirty_count) >= pool->max_items / 10)  in rds_ib_alloc_fmr()
333 		queue_delayed_work(rds_ib_fmr_wq, &pool->flush_worker, 10);  in rds_ib_alloc_fmr()
336 	if (atomic_read(&pool->dirty_count) >=  pool->max_items * 9 / 10) {  in rds_ib_alloc_fmr()
337 		if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_alloc_fmr()
338 			pool = rds_ibdev->mr_1m_pool;  in rds_ib_alloc_fmr()
340 			pool = rds_ibdev->mr_8k_pool;  in rds_ib_alloc_fmr()
344 		ibmr = rds_ib_reuse_fmr(pool);  in rds_ib_alloc_fmr()
357 		if (atomic_inc_return(&pool->item_count) <= pool->max_items)  in rds_ib_alloc_fmr()
360 		atomic_dec(&pool->item_count);  in rds_ib_alloc_fmr()
363 			if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_alloc_fmr()
371 		if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_alloc_fmr()
375 		rds_ib_flush_mr_pool(pool, 0, &ibmr);  in rds_ib_alloc_fmr()
391 			&pool->fmr_attr);  in rds_ib_alloc_fmr()
399 	ibmr->pool = pool;  in rds_ib_alloc_fmr()
400 	if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_alloc_fmr()
413 	atomic_dec(&pool->item_count);  in rds_ib_alloc_fmr()
460 	if (page_cnt > ibmr->pool->fmr_attr.max_pages)  in rds_ib_map_fmr()
492 	if (ibmr->pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_map_fmr()
558 		struct rds_ib_mr_pool *pool = ibmr->pool;  in rds_ib_teardown_mr()  local
560 		atomic_sub(pinned, &pool->free_pinned);  in rds_ib_teardown_mr()
564 static inline unsigned int rds_ib_flush_goal(struct rds_ib_mr_pool *pool, int free_all)  in rds_ib_flush_goal()  argument
568 	item_count = atomic_read(&pool->item_count);  in rds_ib_flush_goal()
602 static void list_to_llist_nodes(struct rds_ib_mr_pool *pool,  in list_to_llist_nodes()  argument
626 static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,  in rds_ib_flush_mr_pool()  argument
638 	if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_flush_mr_pool()
645 		while (!mutex_trylock(&pool->flush_lock)) {  in rds_ib_flush_mr_pool()
646 			ibmr = rds_ib_reuse_fmr(pool);  in rds_ib_flush_mr_pool()
649 				finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
653 			prepare_to_wait(&pool->flush_wait, &wait,  in rds_ib_flush_mr_pool()
655 			if (llist_empty(&pool->clean_list))  in rds_ib_flush_mr_pool()
658 			ibmr = rds_ib_reuse_fmr(pool);  in rds_ib_flush_mr_pool()
661 				finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
665 		finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
667 		mutex_lock(&pool->flush_lock);  in rds_ib_flush_mr_pool()
670 		ibmr = rds_ib_reuse_fmr(pool);  in rds_ib_flush_mr_pool()
680 	dirty_to_clean = llist_append_to_list(&pool->drop_list, &unmap_list);  in rds_ib_flush_mr_pool()
681 	dirty_to_clean += llist_append_to_list(&pool->free_list, &unmap_list);  in rds_ib_flush_mr_pool()
683 		llist_append_to_list(&pool->clean_list, &unmap_list);  in rds_ib_flush_mr_pool()
685 	free_goal = rds_ib_flush_goal(pool, free_all);  in rds_ib_flush_mr_pool()
703 		    ibmr->remap_count >= pool->fmr_attr.max_maps) {  in rds_ib_flush_mr_pool()
704 			if (ibmr->pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_flush_mr_pool()
727 		list_to_llist_nodes(pool, &unmap_list, &clean_nodes, &clean_tail);  in rds_ib_flush_mr_pool()
733 			llist_add_batch(clean_nodes->next, clean_tail, &pool->clean_list);  in rds_ib_flush_mr_pool()
737 	atomic_sub(unpinned, &pool->free_pinned);  in rds_ib_flush_mr_pool()
738 	atomic_sub(dirty_to_clean, &pool->dirty_count);  in rds_ib_flush_mr_pool()
739 	atomic_sub(nfreed, &pool->item_count);  in rds_ib_flush_mr_pool()
742 	mutex_unlock(&pool->flush_lock);  in rds_ib_flush_mr_pool()
743 	if (waitqueue_active(&pool->flush_wait))  in rds_ib_flush_mr_pool()
744 		wake_up(&pool->flush_wait);  in rds_ib_flush_mr_pool()
751 	struct rds_ib_mr_pool *pool = container_of(work, struct rds_ib_mr_pool, flush_worker.work);  in rds_ib_mr_pool_flush_worker()  local
753 	rds_ib_flush_mr_pool(pool, 0, NULL);  in rds_ib_mr_pool_flush_worker()
759 	struct rds_ib_mr_pool *pool = ibmr->pool;  in rds_ib_free_mr()  local
765 	if (ibmr->remap_count >= pool->fmr_attr.max_maps)  in rds_ib_free_mr()
766 		llist_add(&ibmr->llnode, &pool->drop_list);  in rds_ib_free_mr()
768 		llist_add(&ibmr->llnode, &pool->free_list);  in rds_ib_free_mr()
770 	atomic_add(ibmr->sg_len, &pool->free_pinned);  in rds_ib_free_mr()
771 	atomic_inc(&pool->dirty_count);  in rds_ib_free_mr()
774 	if (atomic_read(&pool->free_pinned) >= pool->max_free_pinned ||  in rds_ib_free_mr()
775 	    atomic_read(&pool->dirty_count) >= pool->max_items / 5)  in rds_ib_free_mr()
776 		queue_delayed_work(rds_ib_fmr_wq, &pool->flush_worker, 10);  in rds_ib_free_mr()
780 			rds_ib_flush_mr_pool(pool, 0, NULL);  in rds_ib_free_mr()
786 					   &pool->flush_worker, 10);  in rds_ib_free_mr()