94 static void dealloc_oc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)  in dealloc_oc_sq()  argument
96 	c4iw_ocqp_pool_free(rdev, sq->dma_addr, sq->memsize);  in dealloc_oc_sq()
99 static void dealloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)  in dealloc_host_sq()  argument
101 	dma_free_coherent(&(rdev->lldi.pdev->dev), sq->memsize, sq->queue,  in dealloc_host_sq()
102 			  pci_unmap_addr(sq, mapping));  in dealloc_host_sq()
105 static void dealloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)  in dealloc_sq()  argument
107 	if (t4_sq_onchip(sq))  in dealloc_sq()
108 		dealloc_oc_sq(rdev, sq);  in dealloc_sq()
110 		dealloc_host_sq(rdev, sq);  in dealloc_sq()
113 static int alloc_oc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)  in alloc_oc_sq()  argument
117 	sq->dma_addr = c4iw_ocqp_pool_alloc(rdev, sq->memsize);  in alloc_oc_sq()
118 	if (!sq->dma_addr)  in alloc_oc_sq()
120 	sq->phys_addr = rdev->oc_mw_pa + sq->dma_addr -  in alloc_oc_sq()
122 	sq->queue = (__force union t4_wr *)(rdev->oc_mw_kva + sq->dma_addr -  in alloc_oc_sq()
124 	sq->flags |= T4_SQ_ONCHIP;  in alloc_oc_sq()
128 static int alloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)  in alloc_host_sq()  argument
130 	sq->queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev), sq->memsize,  in alloc_host_sq()
131 				       &(sq->dma_addr), GFP_KERNEL);  in alloc_host_sq()
132 	if (!sq->queue)  in alloc_host_sq()
134 	sq->phys_addr = virt_to_phys(sq->queue);  in alloc_host_sq()
135 	pci_unmap_addr_set(sq, mapping, sq->dma_addr);  in alloc_host_sq()
139 static int alloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq, int user)  in alloc_sq()  argument
143 		ret = alloc_oc_sq(rdev, sq);  in alloc_sq()
145 		ret = alloc_host_sq(rdev, sq);  in alloc_sq()
159 	dealloc_sq(rdev, &wq->sq);  in destroy_qp()
162 	kfree(wq->sq.sw_sq);  in destroy_qp()
164 	c4iw_put_qpid(rdev, wq->sq.qid, uctx);  in destroy_qp()
181 	wq->sq.qid = c4iw_get_qpid(rdev, uctx);  in create_qp()
182 	if (!wq->sq.qid)  in create_qp()
192 		wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq,  in create_qp()
194 		if (!wq->sq.sw_sq) {  in create_qp()
217 	ret = alloc_sq(rdev, &wq->sq, user);  in create_qp()
220 	memset(wq->sq.queue, 0, wq->sq.memsize);  in create_qp()
221 	dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr);  in create_qp()
231 		__func__, wq->sq.queue,  in create_qp()
232 		(unsigned long long)virt_to_phys(wq->sq.queue),  in create_qp()
243 		off = (wq->sq.qid << rdev->qpshift) & PAGE_MASK;  in create_qp()
245 			wq->sq.udb = (u64 __iomem *)(rdev->bar2_pa + off);  in create_qp()
247 			off += 128 * (wq->sq.qid & rdev->qpmask) + 8;  in create_qp()
248 			wq->sq.udb = (u64 __iomem *)(rdev->bar2_kva + off);  in create_qp()
286 	eqsize = wq->sq.size * T4_SQ_NUM_SLOTS +  in create_qp()
293 		(t4_sq_onchip(&wq->sq) ? FW_RI_RES_WR_ONCHIP_F : 0) |  in create_qp()
303 	res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid);  in create_qp()
304 	res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr);  in create_qp()
335 	ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, __func__);  in create_qp()
340 	     __func__, wq->sq.qid, wq->rq.qid, wq->db,  in create_qp()
341 	     (__force unsigned long) wq->sq.udb,  in create_qp()
350 	dealloc_sq(rdev, &wq->sq);  in create_qp()
356 	kfree(wq->sq.sw_sq);  in create_qp()
360 	c4iw_put_qpid(rdev, wq->sq.qid, uctx);  in create_qp()
364 static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp,  in build_immd()  argument
380 			if (dstp == (u8 *)&sq->queue[sq->size])  in build_immd()
381 				dstp = (u8 *)sq->queue;  in build_immd()
382 			if (rem <= (u8 *)&sq->queue[sq->size] - dstp)  in build_immd()
385 				len = (u8 *)&sq->queue[sq->size] - dstp;  in build_immd()
434 static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,  in build_rdma_send()  argument
472 			ret = build_immd(sq, wqe->send.u.immd_src, wr,  in build_rdma_send()
479 			ret = build_isgl((__be64 *)sq->queue,  in build_rdma_send()
480 					 (__be64 *)&sq->queue[sq->size],  in build_rdma_send()
501 static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,  in build_rdma_write()  argument
515 			ret = build_immd(sq, wqe->write.u.immd_src, wr,  in build_rdma_write()
522 			ret = build_isgl((__be64 *)sq->queue,  in build_rdma_write()
523 					 (__be64 *)&sq->queue[sq->size],  in build_rdma_write()
588 static int build_fastreg(struct t4_sq *sq, union t4_wr *wqe,  in build_fastreg()  argument
644 			if (++p == (__be64 *)&sq->queue[sq->size])  in build_fastreg()
645 				p = (__be64 *)sq->queue;  in build_fastreg()
651 			if (++p == (__be64 *)&sq->queue[sq->size])  in build_fastreg()
652 				p = (__be64 *)sq->queue;  in build_fastreg()
699 		qhp->wq.sq.wq_pidx_inc += inc;  in ring_kernel_sq_db()
755 		wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue +  in c4iw_post_send()
756 		      qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE);  in c4iw_post_send()
763 		swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx];  in c4iw_post_send()
774 			err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16);  in c4iw_post_send()
779 			err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16);  in c4iw_post_send()
793 			if (!qhp->wq.sq.oldest_read)  in c4iw_post_send()
794 				qhp->wq.sq.oldest_read = swsqe;  in c4iw_post_send()
799 			err = build_fastreg(&qhp->wq.sq, wqe, wr, &len16,  in c4iw_post_send()
820 		swsqe->idx = qhp->wq.sq.pidx;  in c4iw_post_send()
832 		init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16);  in c4iw_post_send()
835 		     __func__, (unsigned long long)wr->wr_id, qhp->wq.sq.pidx,  in c4iw_post_send()
1075 	PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid,  in post_terminate()
1196 	PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid,  in rdma_fini()
1220 			     qhp->wq.sq.qid, __func__);  in rdma_fini()
1258 	     qhp->wq.sq.qid, qhp->ep->hwtid, qhp->ep->ird, qhp->ep->ord);  in rdma_init()
1304 	wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid);  in rdma_init()
1305 	wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid);  in rdma_init()
1324 				  qhp->ep->hwtid, qhp->wq.sq.qid, __func__);  in rdma_init()
1348 	     qhp, qhp->wq.sq.qid, qhp->wq.rq.qid, qhp->ep, qhp->attr.state,  in c4iw_modify_qp()
1531 	     qhp->wq.sq.qid);  in c4iw_modify_qp()
1588 	remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);  in c4iw_destroy_qp()
1603 	PDBG("%s ib_qp %p qpid 0x%0x\n", __func__, ib_qp, qhp->wq.sq.qid);  in c4iw_destroy_qp()
1654 	qhp->wq.sq.size = sqsize;  in c4iw_create_qp()
1655 	qhp->wq.sq.memsize =  in c4iw_create_qp()
1657 		sizeof(*qhp->wq.sq.queue) + 16 * sizeof(__be64);  in c4iw_create_qp()
1658 	qhp->wq.sq.flush_cidx = -1;  in c4iw_create_qp()
1665 		qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE);  in c4iw_create_qp()
1700 	ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid);  in c4iw_create_qp()
1725 		if (t4_sq_onchip(&qhp->wq.sq)) {  in c4iw_create_qp()
1735 		uresp.sqid = qhp->wq.sq.qid;  in c4iw_create_qp()
1736 		uresp.sq_size = qhp->wq.sq.size;  in c4iw_create_qp()
1737 		uresp.sq_memsize = qhp->wq.sq.memsize;  in c4iw_create_qp()
1761 		mm1->addr = qhp->wq.sq.phys_addr;  in c4iw_create_qp()
1762 		mm1->len = PAGE_ALIGN(qhp->wq.sq.memsize);  in c4iw_create_qp()
1769 		mm3->addr = (__force unsigned long)qhp->wq.sq.udb;  in c4iw_create_qp()
1784 	qhp->ibqp.qp_num = qhp->wq.sq.qid;  in c4iw_create_qp()
1789 	     qhp->wq.sq.qid, qhp->wq.sq.size, qhp->wq.sq.memsize,  in c4iw_create_qp()
1804 	remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);  in c4iw_create_qp()