71 int cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,  in cxio_hal_cq_op()  argument
82 	ret = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_OP, &setup);  in cxio_hal_cq_op()
114 				       rdev_p->dev_name);  in cxio_hal_cq_op()
125 static int cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)  in cxio_hal_clear_cq_ctx()  argument
134 	return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));  in cxio_hal_clear_cq_ctx()
137 static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)  in cxio_hal_clear_qp_ctx()  argument
155 	return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);  in cxio_hal_clear_qp_ctx()
158 int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel)  in cxio_create_cq()  argument
164 	cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);  in cxio_create_cq()
172 	cq->queue = dma_alloc_coherent(&(rdev_p->rnic_info.pdev->dev), size,  in cxio_create_cq()
185 	if (rdev_p->t3cdev_p->type != T3A)  in cxio_create_cq()
189 	return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));  in cxio_create_cq()
193 int cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)  in cxio_resize_cq()  argument
202 	return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));  in cxio_resize_cq()
206 static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)  in get_qpid()  argument
220 		qpid = cxio_hal_get_qpid(rdev_p->rscp);  in get_qpid()
223 		for (i = qpid+1; i & rdev_p->qpmask; i++) {  in get_qpid()
237 static void put_qpid(struct cxio_rdev *rdev_p, u32 qpid,  in put_qpid()  argument
252 void cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)  in cxio_release_ucontext()  argument
261 		if (!(entry->qpid & rdev_p->qpmask))  in cxio_release_ucontext()
262 			cxio_hal_put_qpid(rdev_p->rscp, entry->qpid);  in cxio_release_ucontext()
268 void cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)  in cxio_init_ucontext()  argument
274 int cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,  in cxio_create_qp()  argument
280 	wq->qpid = get_qpid(rdev_p, uctx);  in cxio_create_qp()
288 	wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);  in cxio_create_qp()
296 	wq->queue = dma_alloc_coherent(&(rdev_p->rnic_info.pdev->dev),  in cxio_create_qp()
304 	wq->doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr;  in cxio_create_qp()
306 		wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +  in cxio_create_qp()
307 					(wq->qpid << rdev_p->qpshift);  in cxio_create_qp()
308 	wq->rdev = rdev_p;  in cxio_create_qp()
315 	cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize);  in cxio_create_qp()
319 	put_qpid(rdev_p, wq->qpid, uctx);  in cxio_create_qp()
323 int cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)  in cxio_destroy_cq()  argument
326 	err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);  in cxio_destroy_cq()
328 	dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),  in cxio_destroy_cq()
332 	cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);  in cxio_destroy_cq()
336 int cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,  in cxio_destroy_qp()  argument
339 	dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),  in cxio_destroy_qp()
344 	cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2));  in cxio_destroy_qp()
346 	put_qpid(rdev_p, wq->qpid, uctx);  in cxio_destroy_qp()
500 static int cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)  in cxio_hal_init_ctrl_cq()  argument
511 	return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));  in cxio_hal_init_ctrl_cq()
514 static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)  in cxio_hal_init_ctrl_qp()  argument
527 	err = cxio_hal_init_ctrl_cq(rdev_p);  in cxio_hal_init_ctrl_qp()
532 	rdev_p->ctrl_qp.workq = dma_alloc_coherent(  in cxio_hal_init_ctrl_qp()
533 					&(rdev_p->rnic_info.pdev->dev),  in cxio_hal_init_ctrl_qp()
536 					&(rdev_p->ctrl_qp.dma_addr),  in cxio_hal_init_ctrl_qp()
538 	if (!rdev_p->ctrl_qp.workq) {  in cxio_hal_init_ctrl_qp()
543 	dma_unmap_addr_set(&rdev_p->ctrl_qp, mapping,  in cxio_hal_init_ctrl_qp()
544 			   rdev_p->ctrl_qp.dma_addr);  in cxio_hal_init_ctrl_qp()
545 	rdev_p->ctrl_qp.doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr;  in cxio_hal_init_ctrl_qp()
546 	memset(rdev_p->ctrl_qp.workq, 0,  in cxio_hal_init_ctrl_qp()
549 	mutex_init(&rdev_p->ctrl_qp.lock);  in cxio_hal_init_ctrl_qp()
550 	init_waitqueue_head(&rdev_p->ctrl_qp.waitq);  in cxio_hal_init_ctrl_qp()
553 	base_addr = rdev_p->ctrl_qp.dma_addr;  in cxio_hal_init_ctrl_qp()
575 	     (unsigned long long) rdev_p->ctrl_qp.dma_addr,  in cxio_hal_init_ctrl_qp()
576 	     rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);  in cxio_hal_init_ctrl_qp()
578 	return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);  in cxio_hal_init_ctrl_qp()
584 static int cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)  in cxio_hal_destroy_ctrl_qp()  argument
586 	dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),  in cxio_hal_destroy_ctrl_qp()
588 			  * sizeof(union t3_wr), rdev_p->ctrl_qp.workq,  in cxio_hal_destroy_ctrl_qp()
589 			  dma_unmap_addr(&rdev_p->ctrl_qp, mapping));  in cxio_hal_destroy_ctrl_qp()
590 	return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);  in cxio_hal_destroy_ctrl_qp()
597 static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,  in cxio_hal_ctrl_qp_write_mem()  argument
609 	     __func__, rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len,  in cxio_hal_ctrl_qp_write_mem()
613 		if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr,  in cxio_hal_ctrl_qp_write_mem()
617 			     rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i);  in cxio_hal_ctrl_qp_write_mem()
618 			if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,  in cxio_hal_ctrl_qp_write_mem()
619 					     !Q_FULL(rdev_p->ctrl_qp.rptr,  in cxio_hal_ctrl_qp_write_mem()
620 						     rdev_p->ctrl_qp.wptr,  in cxio_hal_ctrl_qp_write_mem()
629 		wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %  in cxio_hal_ctrl_qp_write_mem()
671 		wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %  in cxio_hal_ctrl_qp_write_mem()
675 		((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr;  in cxio_hal_ctrl_qp_write_mem()
682 			       Q_GENBIT(rdev_p->ctrl_qp.wptr,  in cxio_hal_ctrl_qp_write_mem()
686 			ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);  in cxio_hal_ctrl_qp_write_mem()
688 		rdev_p->ctrl_qp.wptr++;  in cxio_hal_ctrl_qp_write_mem()
697 static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,  in __cxio_tpt_op()  argument
708 	if (cxio_fatal_error(rdev_p))  in __cxio_tpt_op()
715 		stag_idx = cxio_hal_get_stag(rdev_p->rscp);  in __cxio_tpt_op()
723 	mutex_lock(&rdev_p->ctrl_qp.lock);  in __cxio_tpt_op()
738 		tpt.rsvd_pbl_addr = cpu_to_be32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, pbl_addr)>>3));  in __cxio_tpt_op()
745 	err = cxio_hal_ctrl_qp_write_mem(rdev_p,  in __cxio_tpt_op()
747 				       (rdev_p->rnic_info.tpt_base >> 5),  in __cxio_tpt_op()
752 		cxio_hal_put_stag(rdev_p->rscp, stag_idx);  in __cxio_tpt_op()
754 	wptr = rdev_p->ctrl_qp.wptr;  in __cxio_tpt_op()
755 	mutex_unlock(&rdev_p->ctrl_qp.lock);  in __cxio_tpt_op()
757 		if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,  in __cxio_tpt_op()
758 					     SEQ32_GE(rdev_p->ctrl_qp.rptr,  in __cxio_tpt_op()
764 int cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl,  in cxio_write_pbl()  argument
771 	     __func__, pbl_addr, rdev_p->rnic_info.pbl_base,  in cxio_write_pbl()
774 	mutex_lock(&rdev_p->ctrl_qp.lock);  in cxio_write_pbl()
775 	err = cxio_hal_ctrl_qp_write_mem(rdev_p, pbl_addr >> 5, pbl_size << 3,  in cxio_write_pbl()
777 	wptr = rdev_p->ctrl_qp.wptr;  in cxio_write_pbl()
778 	mutex_unlock(&rdev_p->ctrl_qp.lock);  in cxio_write_pbl()
782 	if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,  in cxio_write_pbl()
783 				     SEQ32_GE(rdev_p->ctrl_qp.rptr,  in cxio_write_pbl()
790 int cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,  in cxio_register_phys_mem()  argument
795 	return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,  in cxio_register_phys_mem()
799 int cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,  in cxio_reregister_phys_mem()  argument
803 	return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,  in cxio_reregister_phys_mem()
807 int cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,  in cxio_dereg_mem()  argument
810 	return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,  in cxio_dereg_mem()
814 int cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)  in cxio_allocate_window()  argument
817 	return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,  in cxio_allocate_window()
821 int cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)  in cxio_deallocate_window()  argument
823 	return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,  in cxio_deallocate_window()
827 int cxio_allocate_stag(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr)  in cxio_allocate_stag()  argument
830 	return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_NON_SHARED_MR,  in cxio_allocate_stag()
834 int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)  in cxio_rdma_init()  argument
840 	PDBG("%s rdev_p %p\n", __func__, rdev_p);  in cxio_rdma_init()
850 	wqe->rq_addr = cpu_to_be32(attr->rq_addr - rdev_p->rnic_info.rqt_base);  in cxio_rdma_init()
865 	return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);  in cxio_rdma_init()
881 	struct cxio_rdev *rdev_p = NULL;  in cxio_hal_ev_handler()  local
895 	rdev_p = (struct cxio_rdev *)t3cdev_p->ulp;  in cxio_hal_ev_handler()
896 	if (!rdev_p) {  in cxio_hal_ev_handler()
902 		rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;  in cxio_hal_ev_handler()
903 		wake_up_interruptible(&rdev_p->ctrl_qp.waitq);  in cxio_hal_ev_handler()
908 		(*cxio_ev_cb) (rdev_p, skb);  in cxio_hal_ev_handler()
916 int cxio_rdev_open(struct cxio_rdev *rdev_p)  in cxio_rdev_open()  argument
920 	if (strlen(rdev_p->dev_name)) {  in cxio_rdev_open()
921 		if (cxio_hal_find_rdev_by_name(rdev_p->dev_name)) {  in cxio_rdev_open()
924 		netdev_p = dev_get_by_name(&init_net, rdev_p->dev_name);  in cxio_rdev_open()
929 	} else if (rdev_p->t3cdev_p) {  in cxio_rdev_open()
930 		if (cxio_hal_find_rdev_by_t3cdev(rdev_p->t3cdev_p)) {  in cxio_rdev_open()
933 		netdev_p = rdev_p->t3cdev_p->lldev;  in cxio_rdev_open()
934 		strncpy(rdev_p->dev_name, rdev_p->t3cdev_p->name,  in cxio_rdev_open()
941 	list_add_tail(&rdev_p->entry, &rdev_list);  in cxio_rdev_open()
943 	PDBG("%s opening rnic dev %s\n", __func__, rdev_p->dev_name);  in cxio_rdev_open()
944 	memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));  in cxio_rdev_open()
945 	if (!rdev_p->t3cdev_p)  in cxio_rdev_open()
946 		rdev_p->t3cdev_p = dev2t3cdev(netdev_p);  in cxio_rdev_open()
947 	rdev_p->t3cdev_p->ulp = (void *) rdev_p;  in cxio_rdev_open()
949 	err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_EMBEDDED_INFO,  in cxio_rdev_open()
950 					 &(rdev_p->fw_info));  in cxio_rdev_open()
953 		     __func__, rdev_p->t3cdev_p, err);  in cxio_rdev_open()
956 	if (G_FW_VERSION_MAJOR(rdev_p->fw_info.fw_vers) != CXIO_FW_MAJ) {  in cxio_rdev_open()
960 		       G_FW_VERSION_MAJOR(rdev_p->fw_info.fw_vers));  in cxio_rdev_open()
965 	err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_GET_PARAMS,  in cxio_rdev_open()
966 					 &(rdev_p->rnic_info));  in cxio_rdev_open()
969 		     __func__, rdev_p->t3cdev_p, err);  in cxio_rdev_open()
972 	err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_PORTS,  in cxio_rdev_open()
973 				    &(rdev_p->port_info));  in cxio_rdev_open()
976 		     __func__, rdev_p->t3cdev_p, err);  in cxio_rdev_open()
984 	cxio_init_ucontext(rdev_p, &rdev_p->uctx);  in cxio_rdev_open()
985 	rdev_p->qpshift = PAGE_SHIFT -  in cxio_rdev_open()
987 			            ilog2(rdev_p->rnic_info.udbell_len >>  in cxio_rdev_open()
989 	rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;  in cxio_rdev_open()
990 	rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;  in cxio_rdev_open()
993 	     __func__, rdev_p->dev_name, rdev_p->rnic_info.tpt_base,  in cxio_rdev_open()
994 	     rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p),  in cxio_rdev_open()
995 	     rdev_p->rnic_info.pbl_base,  in cxio_rdev_open()
996 	     rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,  in cxio_rdev_open()
997 	     rdev_p->rnic_info.rqt_top);  in cxio_rdev_open()
1000 	     rdev_p->rnic_info.udbell_len,  in cxio_rdev_open()
1001 	     rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,  in cxio_rdev_open()
1002 	     rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);  in cxio_rdev_open()
1004 	err = cxio_hal_init_ctrl_qp(rdev_p);  in cxio_rdev_open()
1010 	err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,  in cxio_rdev_open()
1018 	err = cxio_hal_pblpool_create(rdev_p);  in cxio_rdev_open()
1024 	err = cxio_hal_rqtpool_create(rdev_p);  in cxio_rdev_open()
1032 	cxio_hal_pblpool_destroy(rdev_p);  in cxio_rdev_open()
1034 	cxio_hal_destroy_resource(rdev_p->rscp);  in cxio_rdev_open()
1036 	cxio_hal_destroy_ctrl_qp(rdev_p);  in cxio_rdev_open()
1038 	rdev_p->t3cdev_p->ulp = NULL;  in cxio_rdev_open()
1039 	list_del(&rdev_p->entry);  in cxio_rdev_open()
1043 void cxio_rdev_close(struct cxio_rdev *rdev_p)  in cxio_rdev_close()  argument
1045 	if (rdev_p) {  in cxio_rdev_close()
1046 		cxio_hal_pblpool_destroy(rdev_p);  in cxio_rdev_close()
1047 		cxio_hal_rqtpool_destroy(rdev_p);  in cxio_rdev_close()
1048 		list_del(&rdev_p->entry);  in cxio_rdev_close()
1049 		cxio_hal_destroy_ctrl_qp(rdev_p);  in cxio_rdev_close()
1050 		cxio_hal_destroy_resource(rdev_p->rscp);  in cxio_rdev_close()
1051 		rdev_p->t3cdev_p->ulp = NULL;  in cxio_rdev_close()