418 	struct rpcrdma_ia *ia = &xprt->rx_ia;  in rpcrdma_conn_upcall()  local
423 	struct ib_qp_attr *attr = &ia->ri_qp_attr;  in rpcrdma_conn_upcall()
424 	struct ib_qp_init_attr *iattr = &ia->ri_qp_init_attr;  in rpcrdma_conn_upcall()
430 		ia->ri_async_rc = 0;  in rpcrdma_conn_upcall()
431 		complete(&ia->ri_done);  in rpcrdma_conn_upcall()
434 		ia->ri_async_rc = -EHOSTUNREACH;  in rpcrdma_conn_upcall()
437 		complete(&ia->ri_done);  in rpcrdma_conn_upcall()
440 		ia->ri_async_rc = -ENETUNREACH;  in rpcrdma_conn_upcall()
443 		complete(&ia->ri_done);  in rpcrdma_conn_upcall()
447 		ib_query_qp(ia->ri_id->qp, attr,  in rpcrdma_conn_upcall()
490 			ia->ri_id->device->name,  in rpcrdma_conn_upcall()
491 			ia->ri_ops->ro_displayname,  in rpcrdma_conn_upcall()
505 			struct rpcrdma_ia *ia, struct sockaddr *addr)  in rpcrdma_create_id()  argument
510 	init_completion(&ia->ri_done);  in rpcrdma_create_id()
520 	ia->ri_async_rc = -ETIMEDOUT;  in rpcrdma_create_id()
527 	wait_for_completion_interruptible_timeout(&ia->ri_done,  in rpcrdma_create_id()
529 	rc = ia->ri_async_rc;  in rpcrdma_create_id()
533 	ia->ri_async_rc = -ETIMEDOUT;  in rpcrdma_create_id()
540 	wait_for_completion_interruptible_timeout(&ia->ri_done,  in rpcrdma_create_id()
542 	rc = ia->ri_async_rc;  in rpcrdma_create_id()
583 	struct rpcrdma_ia *ia = &xprt->rx_ia;  in rpcrdma_ia_open()  local
584 	struct ib_device_attr *devattr = &ia->ri_devattr;  in rpcrdma_ia_open()
586 	ia->ri_id = rpcrdma_create_id(xprt, ia, addr);  in rpcrdma_ia_open()
587 	if (IS_ERR(ia->ri_id)) {  in rpcrdma_ia_open()
588 		rc = PTR_ERR(ia->ri_id);  in rpcrdma_ia_open()
592 	ia->ri_pd = ib_alloc_pd(ia->ri_id->device);  in rpcrdma_ia_open()
593 	if (IS_ERR(ia->ri_pd)) {  in rpcrdma_ia_open()
594 		rc = PTR_ERR(ia->ri_pd);  in rpcrdma_ia_open()
600 	rc = ib_query_device(ia->ri_id->device, devattr);  in rpcrdma_ia_open()
608 		ia->ri_have_dma_lkey = 1;  in rpcrdma_ia_open()
609 		ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey;  in rpcrdma_ia_open()
624 		if (!ia->ri_id->device->alloc_fmr) {  in rpcrdma_ia_open()
641 		ia->ri_ops = &rpcrdma_frwr_memreg_ops;  in rpcrdma_ia_open()
644 		ia->ri_ops = &rpcrdma_physical_memreg_ops;  in rpcrdma_ia_open()
650 		ia->ri_ops = &rpcrdma_fmr_memreg_ops;  in rpcrdma_ia_open()
651 		if (ia->ri_have_dma_lkey)  in rpcrdma_ia_open()
655 		ia->ri_bind_mem = ib_get_dma_mr(ia->ri_pd, mem_priv);  in rpcrdma_ia_open()
656 		if (IS_ERR(ia->ri_bind_mem)) {  in rpcrdma_ia_open()
659 				__func__, PTR_ERR(ia->ri_bind_mem));  in rpcrdma_ia_open()
671 		__func__, ia->ri_ops->ro_displayname);  in rpcrdma_ia_open()
674 	ia->ri_memreg_strategy = memreg;  in rpcrdma_ia_open()
676 	rwlock_init(&ia->ri_qplock);  in rpcrdma_ia_open()
680 	ib_dealloc_pd(ia->ri_pd);  in rpcrdma_ia_open()
681 	ia->ri_pd = NULL;  in rpcrdma_ia_open()
683 	rdma_destroy_id(ia->ri_id);  in rpcrdma_ia_open()
684 	ia->ri_id = NULL;  in rpcrdma_ia_open()
695 rpcrdma_ia_close(struct rpcrdma_ia *ia)  in rpcrdma_ia_close()  argument
700 	if (ia->ri_bind_mem != NULL) {  in rpcrdma_ia_close()
701 		rc = ib_dereg_mr(ia->ri_bind_mem);  in rpcrdma_ia_close()
705 	if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) {  in rpcrdma_ia_close()
706 		if (ia->ri_id->qp)  in rpcrdma_ia_close()
707 			rdma_destroy_qp(ia->ri_id);  in rpcrdma_ia_close()
708 		rdma_destroy_id(ia->ri_id);  in rpcrdma_ia_close()
709 		ia->ri_id = NULL;  in rpcrdma_ia_close()
711 	if (ia->ri_pd != NULL && !IS_ERR(ia->ri_pd)) {  in rpcrdma_ia_close()
712 		rc = ib_dealloc_pd(ia->ri_pd);  in rpcrdma_ia_close()
722 rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,  in rpcrdma_ep_create()  argument
725 	struct ib_device_attr *devattr = &ia->ri_devattr;  in rpcrdma_ep_create()
737 	rc = ia->ri_ops->ro_open(ia, ep, cdata);  in rpcrdma_ep_create()
749 		ep->rep_padbuf = rpcrdma_alloc_regbuf(ia, cdata->padding,  in rpcrdma_ep_create()
774 	sendcq = ib_create_cq(ia->ri_id->device, rpcrdma_sendcq_upcall,  in rpcrdma_ep_create()
791 	recvcq = ib_create_cq(ia->ri_id->device, rpcrdma_recvcq_upcall,  in rpcrdma_ep_create()
838 	rpcrdma_free_regbuf(ia, ep->rep_padbuf);  in rpcrdma_ep_create()
850 rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)  in rpcrdma_ep_destroy()  argument
859 	if (ia->ri_id->qp) {  in rpcrdma_ep_destroy()
860 		rpcrdma_ep_disconnect(ep, ia);  in rpcrdma_ep_destroy()
861 		rdma_destroy_qp(ia->ri_id);  in rpcrdma_ep_destroy()
862 		ia->ri_id->qp = NULL;  in rpcrdma_ep_destroy()
865 	rpcrdma_free_regbuf(ia, ep->rep_padbuf);  in rpcrdma_ep_destroy()
884 rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)  in rpcrdma_ep_connect()  argument
895 		rpcrdma_ep_disconnect(ep, ia);  in rpcrdma_ep_connect()
898 		xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);  in rpcrdma_ep_connect()
899 		ia->ri_ops->ro_reset(xprt);  in rpcrdma_ep_connect()
901 		id = rpcrdma_create_id(xprt, ia,  in rpcrdma_ep_connect()
914 		if (ia->ri_id->device != id->device) {  in rpcrdma_ep_connect()
922 		rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);  in rpcrdma_ep_connect()
931 		write_lock(&ia->ri_qplock);  in rpcrdma_ep_connect()
932 		old = ia->ri_id;  in rpcrdma_ep_connect()
933 		ia->ri_id = id;  in rpcrdma_ep_connect()
934 		write_unlock(&ia->ri_qplock);  in rpcrdma_ep_connect()
940 		rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);  in rpcrdma_ep_connect()
951 	rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);  in rpcrdma_ep_connect()
1005 rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)  in rpcrdma_ep_disconnect()  argument
1010 	rc = rdma_disconnect(ia->ri_id);  in rpcrdma_ep_disconnect()
1040 	struct rpcrdma_ia *ia = &r_xprt->rx_ia;  in rpcrdma_create_rep()  local
1049 	rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize,  in rpcrdma_create_rep()
1069 	struct rpcrdma_ia *ia = &r_xprt->rx_ia;  in rpcrdma_buffer_create()  local
1101 	rc = ia->ri_ops->ro_init(r_xprt);  in rpcrdma_buffer_create()
1135 rpcrdma_destroy_rep(struct rpcrdma_ia *ia, struct rpcrdma_rep *rep)  in rpcrdma_destroy_rep()  argument
1140 	rpcrdma_free_regbuf(ia, rep->rr_rdmabuf);  in rpcrdma_destroy_rep()
1145 rpcrdma_destroy_req(struct rpcrdma_ia *ia, struct rpcrdma_req *req)  in rpcrdma_destroy_req()  argument
1150 	rpcrdma_free_regbuf(ia, req->rl_sendbuf);  in rpcrdma_destroy_req()
1151 	rpcrdma_free_regbuf(ia, req->rl_rdmabuf);  in rpcrdma_destroy_req()
1158 	struct rpcrdma_ia *ia = rdmab_to_ia(buf);  in rpcrdma_buffer_destroy()  local
1170 			rpcrdma_destroy_rep(ia, buf->rb_recv_bufs[i]);  in rpcrdma_buffer_destroy()
1172 			rpcrdma_destroy_req(ia, buf->rb_send_bufs[i]);  in rpcrdma_buffer_destroy()
1175 	ia->ri_ops->ro_destroy(buf);  in rpcrdma_buffer_destroy()
1223 rpcrdma_retry_local_inv(struct rpcrdma_mw *r, struct rpcrdma_ia *ia)  in rpcrdma_retry_local_inv()  argument
1226 				container_of(ia, struct rpcrdma_xprt, rx_ia);  in rpcrdma_retry_local_inv()
1244 	read_lock(&ia->ri_qplock);  in rpcrdma_retry_local_inv()
1245 	rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr);  in rpcrdma_retry_local_inv()
1246 	read_unlock(&ia->ri_qplock);  in rpcrdma_retry_local_inv()
1259 	struct rpcrdma_ia *ia = rdmab_to_ia(buf);  in rpcrdma_retry_flushed_linv()  local
1266 		rpcrdma_retry_local_inv(r, ia);  in rpcrdma_retry_flushed_linv()
1335 	struct rpcrdma_ia *ia = rdmab_to_ia(buffers);  in rpcrdma_buffer_get()  local
1360 	switch (ia->ri_memreg_strategy) {  in rpcrdma_buffer_get()
1384 	struct rpcrdma_ia *ia = rdmab_to_ia(buffers);  in rpcrdma_buffer_put()  local
1389 	switch (ia->ri_memreg_strategy) {  in rpcrdma_buffer_put()
1448 rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len,  in rpcrdma_register_internal()  argument
1458 	iov->addr = ib_dma_map_single(ia->ri_id->device,  in rpcrdma_register_internal()
1460 	if (ib_dma_mapping_error(ia->ri_id->device, iov->addr))  in rpcrdma_register_internal()
1465 	if (ia->ri_have_dma_lkey) {  in rpcrdma_register_internal()
1467 		iov->lkey = ia->ri_dma_lkey;  in rpcrdma_register_internal()
1469 	} else if (ia->ri_bind_mem != NULL) {  in rpcrdma_register_internal()
1471 		iov->lkey = ia->ri_bind_mem->lkey;  in rpcrdma_register_internal()
1477 	mr = ib_reg_phys_mr(ia->ri_pd, &ipb, 1,  in rpcrdma_register_internal()
1499 rpcrdma_deregister_internal(struct rpcrdma_ia *ia,  in rpcrdma_deregister_internal()  argument
1504 	ib_dma_unmap_single(ia->ri_id->device,  in rpcrdma_deregister_internal()
1532 rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)  in rpcrdma_alloc_regbuf()  argument
1544 	rc = rpcrdma_register_internal(ia, rb->rg_base, size,  in rpcrdma_alloc_regbuf()
1563 rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)  in rpcrdma_free_regbuf()  argument
1566 		rpcrdma_deregister_internal(ia, rb->rg_mr, &rb->rg_iov);  in rpcrdma_free_regbuf()
1577 rpcrdma_ep_post(struct rpcrdma_ia *ia,  in rpcrdma_ep_post()  argument
1586 		rc = rpcrdma_ep_post_recv(ia, ep, rep);  in rpcrdma_ep_post()
1598 		ib_dma_sync_single_for_device(ia->ri_id->device,  in rpcrdma_ep_post()
1601 	ib_dma_sync_single_for_device(ia->ri_id->device,  in rpcrdma_ep_post()
1604 	ib_dma_sync_single_for_device(ia->ri_id->device,  in rpcrdma_ep_post()
1615 	rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail);  in rpcrdma_ep_post()
1627 rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,  in rpcrdma_ep_post_recv()  argument
1639 	ib_dma_sync_single_for_cpu(ia->ri_id->device,  in rpcrdma_ep_post_recv()
1644 	rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);  in rpcrdma_ep_post_recv()