79 static int rds_iw_init_reg(struct rds_iw_mr_pool *pool, struct rds_iw_mr *ibmr);
81 			  struct rds_iw_mr *ibmr,
83 static void rds_iw_free_fastreg(struct rds_iw_mr_pool *pool, struct rds_iw_mr *ibmr);
88 static void rds_iw_destroy_fastreg(struct rds_iw_mr_pool *pool, struct rds_iw_mr *ibmr);
366 	struct rds_iw_mr *ibmr = NULL;  local
371 		ibmr = list_entry(pool->clean_list.next, struct rds_iw_mr, mapping.m_list);
372 		list_del_init(&ibmr->mapping.m_list);
376 	return ibmr;
382 	struct rds_iw_mr *ibmr = NULL;  local
386 		ibmr = rds_iw_reuse_fmr(pool);
387 		if (ibmr)
388 			return ibmr;
414 	ibmr = kzalloc(sizeof(*ibmr), GFP_KERNEL);
415 	if (!ibmr) {
420 	spin_lock_init(&ibmr->mapping.m_lock);
421 	INIT_LIST_HEAD(&ibmr->mapping.m_list);
422 	ibmr->mapping.m_mr = ibmr;
424 	err = rds_iw_init_reg(pool, ibmr);
429 	return ibmr;
432 	if (ibmr) {
433 		rds_iw_destroy_fastreg(pool, ibmr);
434 		kfree(ibmr);
442 	struct rds_iw_mr *ibmr = trans_private;  local
443 	struct rds_iw_device *rds_iwdev = ibmr->device;
447 		ib_dma_sync_sg_for_cpu(rds_iwdev->dev, ibmr->mapping.m_sg.list,
448 			ibmr->mapping.m_sg.dma_len, DMA_BIDIRECTIONAL);
451 		ib_dma_sync_sg_for_device(rds_iwdev->dev, ibmr->mapping.m_sg.list,
452 			ibmr->mapping.m_sg.dma_len, DMA_BIDIRECTIONAL);
465 	struct rds_iw_mr *ibmr, *next;  local
500 	list_for_each_entry_safe(ibmr, next, &kill_list, mapping.m_list) {
502 		list_del(&ibmr->mapping.m_list);
503 		rds_iw_destroy_fastreg(pool, ibmr);
504 		kfree(ibmr);
532 	struct rds_iw_mr *ibmr = trans_private;  local
533 	struct rds_iw_mr_pool *pool = ibmr->device->mr_pool;
535 	rdsdebug("RDS/IW: free_mr nents %u\n", ibmr->mapping.m_sg.len);
540 	rds_iw_free_fastreg(pool, ibmr);
574 	struct rds_iw_mr *ibmr = NULL;  local
597 	ibmr = rds_iw_alloc_mr(rds_iwdev);
598 	if (IS_ERR(ibmr))
599 		return ibmr;
601 	ibmr->cm_id = cm_id;
602 	ibmr->device = rds_iwdev;
604 	ret = rds_iw_map_reg(rds_iwdev->mr_pool, ibmr, sg, nents);
606 		*key_ret = ibmr->mr->rkey;
612 		if (ibmr)
613 			rds_iw_free_mr(ibmr, 0);
614 		ibmr = ERR_PTR(ret);
616 	return ibmr;
642 			   struct rds_iw_mr *ibmr)  argument
657 	ibmr->mr = mr;
663 	struct rds_iw_mr *ibmr = mapping->m_mr;  local
669 	n = ib_map_mr_sg_zbva(ibmr->mr, m_sg->list, m_sg->len, PAGE_SIZE);
677 	reg_wr.mr = ibmr->mr;
689 	ib_update_fast_reg_key(ibmr->mr, ibmr->remap_count++);
690 	mapping->m_rkey = ibmr->mr->rkey;
693 	ret = ib_post_send(ibmr->cm_id->qp, &reg_wr.wr, &failed_wr);
701 static int rds_iw_rdma_fastreg_inv(struct rds_iw_mr *ibmr)  argument
706 	if (!ibmr->cm_id->qp || !ibmr->mr)
712 	s_wr.ex.invalidate_rkey = ibmr->mr->rkey;
716 	ret = ib_post_send(ibmr->cm_id->qp, &s_wr, &failed_wr);
727 			  struct rds_iw_mr *ibmr,  argument
732 	struct rds_iw_mapping *mapping = &ibmr->mapping;
765 		struct rds_iw_mr *ibmr)  argument
770 	if (!ibmr->mapping.m_sg.dma_len)
773 	ret = rds_iw_rdma_fastreg_inv(ibmr);
780 	list_add_tail(&ibmr->mapping.m_list, &pool->dirty_list);
781 	atomic_add(ibmr->mapping.m_sg.len, &pool->free_pinned);
833 		struct rds_iw_mr *ibmr)  argument
835 	if (ibmr->mr)
836 		ib_dereg_mr(ibmr->mr);