279 	struct rpcrdma_ia *ia = &xprt->rx_ia;  in rpcrdma_conn_upcall()  local
284 	struct ib_qp_attr *attr = &ia->ri_qp_attr;  in rpcrdma_conn_upcall()
285 	struct ib_qp_init_attr *iattr = &ia->ri_qp_init_attr;  in rpcrdma_conn_upcall()
291 		ia->ri_async_rc = 0;  in rpcrdma_conn_upcall()
292 		complete(&ia->ri_done);  in rpcrdma_conn_upcall()
295 		ia->ri_async_rc = -EHOSTUNREACH;  in rpcrdma_conn_upcall()
298 		complete(&ia->ri_done);  in rpcrdma_conn_upcall()
301 		ia->ri_async_rc = -ENETUNREACH;  in rpcrdma_conn_upcall()
304 		complete(&ia->ri_done);  in rpcrdma_conn_upcall()
308 		ib_query_qp(ia->ri_id->qp, attr,  in rpcrdma_conn_upcall()
351 			ia->ri_device->name,  in rpcrdma_conn_upcall()
352 			ia->ri_ops->ro_displayname,  in rpcrdma_conn_upcall()
374 			struct rpcrdma_ia *ia, struct sockaddr *addr)  in rpcrdma_create_id()  argument
379 	init_completion(&ia->ri_done);  in rpcrdma_create_id()
390 	ia->ri_async_rc = -ETIMEDOUT;  in rpcrdma_create_id()
397 	wait_for_completion_interruptible_timeout(&ia->ri_done,  in rpcrdma_create_id()
405 	if (!ia->ri_async_rc && !try_module_get(id->device->owner)) {  in rpcrdma_create_id()
408 		ia->ri_async_rc = -ENODEV;  in rpcrdma_create_id()
410 	rc = ia->ri_async_rc;  in rpcrdma_create_id()
414 	ia->ri_async_rc = -ETIMEDOUT;  in rpcrdma_create_id()
421 	wait_for_completion_interruptible_timeout(&ia->ri_done,  in rpcrdma_create_id()
423 	rc = ia->ri_async_rc;  in rpcrdma_create_id()
464 	struct rpcrdma_ia *ia = &xprt->rx_ia;  in rpcrdma_ia_open()  local
465 	struct ib_device_attr *devattr = &ia->ri_devattr;  in rpcrdma_ia_open()
468 	ia->ri_dma_mr = NULL;  in rpcrdma_ia_open()
470 	ia->ri_id = rpcrdma_create_id(xprt, ia, addr);  in rpcrdma_ia_open()
471 	if (IS_ERR(ia->ri_id)) {  in rpcrdma_ia_open()
472 		rc = PTR_ERR(ia->ri_id);  in rpcrdma_ia_open()
475 	ia->ri_device = ia->ri_id->device;  in rpcrdma_ia_open()
477 	ia->ri_pd = ib_alloc_pd(ia->ri_device);  in rpcrdma_ia_open()
478 	if (IS_ERR(ia->ri_pd)) {  in rpcrdma_ia_open()
479 		rc = PTR_ERR(ia->ri_pd);  in rpcrdma_ia_open()
485 	rc = ib_query_device(ia->ri_device, devattr);  in rpcrdma_ia_open()
501 		if (!ia->ri_device->alloc_fmr) {  in rpcrdma_ia_open()
511 		ia->ri_ops = &rpcrdma_frwr_memreg_ops;  in rpcrdma_ia_open()
514 		ia->ri_ops = &rpcrdma_physical_memreg_ops;  in rpcrdma_ia_open()
517 		ia->ri_ops = &rpcrdma_fmr_memreg_ops;  in rpcrdma_ia_open()
526 		__func__, ia->ri_ops->ro_displayname);  in rpcrdma_ia_open()
528 	rwlock_init(&ia->ri_qplock);  in rpcrdma_ia_open()
532 	ib_dealloc_pd(ia->ri_pd);  in rpcrdma_ia_open()
533 	ia->ri_pd = NULL;  in rpcrdma_ia_open()
535 	rpcrdma_destroy_id(ia->ri_id);  in rpcrdma_ia_open()
536 	ia->ri_id = NULL;  in rpcrdma_ia_open()
547 rpcrdma_ia_close(struct rpcrdma_ia *ia)  in rpcrdma_ia_close()  argument
550 	if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) {  in rpcrdma_ia_close()
551 		if (ia->ri_id->qp)  in rpcrdma_ia_close()
552 			rdma_destroy_qp(ia->ri_id);  in rpcrdma_ia_close()
553 		rpcrdma_destroy_id(ia->ri_id);  in rpcrdma_ia_close()
554 		ia->ri_id = NULL;  in rpcrdma_ia_close()
558 	if (ia->ri_pd && !IS_ERR(ia->ri_pd))  in rpcrdma_ia_close()
559 		ib_dealloc_pd(ia->ri_pd);  in rpcrdma_ia_close()
566 rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,  in rpcrdma_ep_create()  argument
569 	struct ib_device_attr *devattr = &ia->ri_devattr;  in rpcrdma_ep_create()
597 	rc = ia->ri_ops->ro_open(ia, ep, cdata);  in rpcrdma_ep_create()
628 	sendcq = ib_create_cq(ia->ri_device, rpcrdma_sendcq_upcall,  in rpcrdma_ep_create()
645 	recvcq = ib_create_cq(ia->ri_device, rpcrdma_recvcq_upcall,  in rpcrdma_ep_create()
691 	if (ia->ri_dma_mr)  in rpcrdma_ep_create()
692 		ib_dereg_mr(ia->ri_dma_mr);  in rpcrdma_ep_create()
704 rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)  in rpcrdma_ep_destroy()  argument
713 	if (ia->ri_id->qp)  in rpcrdma_ep_destroy()
714 		rpcrdma_ep_disconnect(ep, ia);  in rpcrdma_ep_destroy()
719 	if (ia->ri_id->qp) {  in rpcrdma_ep_destroy()
720 		rdma_destroy_qp(ia->ri_id);  in rpcrdma_ep_destroy()
721 		ia->ri_id->qp = NULL;  in rpcrdma_ep_destroy()
734 	if (ia->ri_dma_mr) {  in rpcrdma_ep_destroy()
735 		rc = ib_dereg_mr(ia->ri_dma_mr);  in rpcrdma_ep_destroy()
745 rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)  in rpcrdma_ep_connect()  argument
756 		rpcrdma_ep_disconnect(ep, ia);  in rpcrdma_ep_connect()
759 		xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);  in rpcrdma_ep_connect()
760 		id = rpcrdma_create_id(xprt, ia,  in rpcrdma_ep_connect()
773 		if (ia->ri_device != id->device) {  in rpcrdma_ep_connect()
781 		rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);  in rpcrdma_ep_connect()
790 		write_lock(&ia->ri_qplock);  in rpcrdma_ep_connect()
791 		old = ia->ri_id;  in rpcrdma_ep_connect()
792 		ia->ri_id = id;  in rpcrdma_ep_connect()
793 		write_unlock(&ia->ri_qplock);  in rpcrdma_ep_connect()
799 		rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);  in rpcrdma_ep_connect()
810 	rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);  in rpcrdma_ep_connect()
850 		r_xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);  in rpcrdma_ep_connect()
878 rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)  in rpcrdma_ep_disconnect()  argument
883 	rc = rdma_disconnect(ia->ri_id);  in rpcrdma_ep_disconnect()
918 	struct rpcrdma_ia *ia = &r_xprt->rx_ia;  in rpcrdma_create_rep()  local
927 	rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize,  in rpcrdma_create_rep()
934 	rep->rr_device = ia->ri_device;  in rpcrdma_create_rep()
949 	struct rpcrdma_ia *ia = &r_xprt->rx_ia;  in rpcrdma_buffer_create()  local
956 	rc = ia->ri_ops->ro_init(r_xprt);  in rpcrdma_buffer_create()
1020 rpcrdma_destroy_rep(struct rpcrdma_ia *ia, struct rpcrdma_rep *rep)  in rpcrdma_destroy_rep()  argument
1022 	rpcrdma_free_regbuf(ia, rep->rr_rdmabuf);  in rpcrdma_destroy_rep()
1027 rpcrdma_destroy_req(struct rpcrdma_ia *ia, struct rpcrdma_req *req)  in rpcrdma_destroy_req()  argument
1029 	rpcrdma_free_regbuf(ia, req->rl_sendbuf);  in rpcrdma_destroy_req()
1030 	rpcrdma_free_regbuf(ia, req->rl_rdmabuf);  in rpcrdma_destroy_req()
1037 	struct rpcrdma_ia *ia = rdmab_to_ia(buf);  in rpcrdma_buffer_destroy()  local
1043 		rpcrdma_destroy_rep(ia, rep);  in rpcrdma_buffer_destroy()
1055 		rpcrdma_destroy_req(ia, req);  in rpcrdma_buffer_destroy()
1060 	ia->ri_ops->ro_destroy(buf);  in rpcrdma_buffer_destroy()
1200 rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)  in rpcrdma_alloc_regbuf()  argument
1210 	iov->addr = ib_dma_map_single(ia->ri_device,  in rpcrdma_alloc_regbuf()
1213 	if (ib_dma_mapping_error(ia->ri_device, iov->addr))  in rpcrdma_alloc_regbuf()
1217 	iov->lkey = ia->ri_pd->local_dma_lkey;  in rpcrdma_alloc_regbuf()
1234 rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)  in rpcrdma_free_regbuf()  argument
1242 	ib_dma_unmap_single(ia->ri_device,  in rpcrdma_free_regbuf()
1253 rpcrdma_ep_post(struct rpcrdma_ia *ia,  in rpcrdma_ep_post()  argument
1257 	struct ib_device *device = ia->ri_device;  in rpcrdma_ep_post()
1264 		rc = rpcrdma_ep_post_recv(ia, ep, rep);  in rpcrdma_ep_post()
1289 	rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail);  in rpcrdma_ep_post()
1301 rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,  in rpcrdma_ep_post_recv()  argument
1313 	ib_dma_sync_single_for_cpu(ia->ri_device,  in rpcrdma_ep_post_recv()
1318 	rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);  in rpcrdma_ep_post_recv()
1337 	struct rpcrdma_ia *ia = &r_xprt->rx_ia;  in rpcrdma_ep_post_extra_recv()  local
1350 		rc = rpcrdma_ep_post_recv(ia, ep, rep);  in rpcrdma_ep_post_extra_recv()