218 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_bind_mw()  local
225 	if (nesqp->ibqp_state > IB_QPS_RTS)  in nes_bind_mw()
228 	spin_lock_irqsave(&nesqp->lock, flags);  in nes_bind_mw()
230 	head = nesqp->hwqp.sq_head;  in nes_bind_mw()
231 	qsize = nesqp->hwqp.sq_tail;  in nes_bind_mw()
234 	if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) {  in nes_bind_mw()
235 		spin_unlock_irqrestore(&nesqp->lock, flags);  in nes_bind_mw()
239 	wqe = &nesqp->hwqp.sq_vbase[head];  in nes_bind_mw()
241 	nes_fill_init_qp_wqe(wqe, nesqp, head);  in nes_bind_mw()
270 	nesqp->hwqp.sq_head = head;  in nes_bind_mw()
274 			(1 << 24) | 0x00800000 | nesqp->hwqp.qp_id);  in nes_bind_mw()
276 	spin_unlock_irqrestore(&nesqp->lock, flags);  in nes_bind_mw()
720 	struct nes_qp *nesqp;  in nes_mmap()  local
733 		nesqp = nes_ucontext->mmap_nesqp[index];  in nes_mmap()
734 		if (nesqp == NULL) {  in nes_mmap()
739 				virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT,  in nes_mmap()
745 		vma->vm_private_data = nesqp;  in nes_mmap()
907 static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,  in nes_setup_virt_qp()  argument
931 …nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1])…  in nes_setup_virt_qp()
938 	nesqp->hwqp.sq_vbase = kmap(nespbl->page);  in nes_setup_virt_qp()
939 	nesqp->page = nespbl->page;  in nes_setup_virt_qp()
940 	if (!nesqp->hwqp.sq_vbase) {  in nes_setup_virt_qp()
949 …nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1])…  in nes_setup_virt_qp()
954 		  nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase,  in nes_setup_virt_qp()
955 		  nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase);  in nes_setup_virt_qp()
961 		kunmap(nesqp->page);  in nes_setup_virt_qp()
968 	nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase);  in nes_setup_virt_qp()
969 	pblbuffer = nesqp->pbl_vbase;  in nes_setup_virt_qp()
970 	if (!nesqp->pbl_vbase) {  in nes_setup_virt_qp()
978 		kunmap(nesqp->page);  in nes_setup_virt_qp()
981 	memset(nesqp->pbl_vbase, 0, 256);  in nes_setup_virt_qp()
996 	nesqp->qp_mem_size =  in nes_setup_virt_qp()
999 	nesqp->qp_mem_size += PAGE_SIZE - 1;  in nes_setup_virt_qp()
1000 	nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);  in nes_setup_virt_qp()
1002 	mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,  in nes_setup_virt_qp()
1003 			&nesqp->hwqp.q2_pbase);  in nes_setup_virt_qp()
1006 		pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase);  in nes_setup_virt_qp()
1007 		nesqp->pbl_vbase = NULL;  in nes_setup_virt_qp()
1011 		kunmap(nesqp->page);  in nes_setup_virt_qp()
1014 	nesqp->sq_kmapped = 1;  in nes_setup_virt_qp()
1015 	nesqp->hwqp.q2_vbase = mem;  in nes_setup_virt_qp()
1017 	memset(nesqp->hwqp.q2_vbase, 0, 256);  in nes_setup_virt_qp()
1018 	nesqp->nesqp_context = mem;  in nes_setup_virt_qp()
1019 	memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));  in nes_setup_virt_qp()
1020 	nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;  in nes_setup_virt_qp()
1029 static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic,  in nes_setup_mmap_qp()  argument
1035 	nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) +  in nes_setup_mmap_qp()
1040 	nesqp->qp_mem_size += PAGE_SIZE - 1;  in nes_setup_mmap_qp()
1041 	nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);  in nes_setup_mmap_qp()
1043 	mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,  in nes_setup_mmap_qp()
1044 			&nesqp->hwqp.sq_pbase);  in nes_setup_mmap_qp()
1049 			mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size);  in nes_setup_mmap_qp()
1051 	memset(mem, 0, nesqp->qp_mem_size);  in nes_setup_mmap_qp()
1053 	nesqp->hwqp.sq_vbase = mem;  in nes_setup_mmap_qp()
1056 	nesqp->hwqp.rq_vbase = mem;  in nes_setup_mmap_qp()
1057 	nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase +  in nes_setup_mmap_qp()
1061 	nesqp->hwqp.q2_vbase = mem;  in nes_setup_mmap_qp()
1062 	nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase +  in nes_setup_mmap_qp()
1065 	memset(nesqp->hwqp.q2_vbase, 0, 256);  in nes_setup_mmap_qp()
1067 	nesqp->nesqp_context = mem;  in nes_setup_mmap_qp()
1068 	nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;  in nes_setup_mmap_qp()
1069 	memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));  in nes_setup_mmap_qp()
1078 		struct nes_qp *nesqp, int virt_wqs)  in nes_free_qp_mem()  argument
1083 		pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size,  in nes_free_qp_mem()
1084 				nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase);  in nes_free_qp_mem()
1089 …pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase…  in nes_free_qp_mem()
1090 		pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase );  in nes_free_qp_mem()
1091 		nesqp->pbl_vbase = NULL;  in nes_free_qp_mem()
1092 		if (nesqp->sq_kmapped) {  in nes_free_qp_mem()
1093 			nesqp->sq_kmapped = 0;  in nes_free_qp_mem()
1094 			kunmap(nesqp->page);  in nes_free_qp_mem()
1112 	struct nes_qp *nesqp;  in nes_create_qp()  local
1169 			mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL);  in nes_create_qp()
1179 			nesqp = (struct nes_qp *)(unsigned long)u64nesqp;  in nes_create_qp()
1182 			nesqp->allocated_buffer = mem;  in nes_create_qp()
1187 					kfree(nesqp->allocated_buffer);  in nes_create_qp()
1195 					nesqp->nesuqp_addr = req.user_qp_buffer;  in nes_create_qp()
1197 					nesqp->user_mode = 1;  in nes_create_qp()
1214 							kfree(nesqp->allocated_buffer);  in nes_create_qp()
1220 					nesqp->mmap_sq_db_index =  in nes_create_qp()
1225 					if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) {  in nes_create_qp()
1234 						kfree(nesqp->allocated_buffer);  in nes_create_qp()
1237 					set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);  in nes_create_qp()
1238 					nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp;  in nes_create_qp()
1239 					nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1;  in nes_create_qp()
1242 					kfree(nesqp->allocated_buffer);  in nes_create_qp()
1246 			err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) :  in nes_create_qp()
1247 					nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size);  in nes_create_qp()
1252 				kfree(nesqp->allocated_buffer);  in nes_create_qp()
1256 			nesqp->hwqp.sq_size = sq_size;  in nes_create_qp()
1257 			nesqp->hwqp.sq_encoded_size = sq_encoded_size;  in nes_create_qp()
1258 			nesqp->hwqp.sq_head = 1;  in nes_create_qp()
1259 			nesqp->hwqp.rq_size = rq_size;  in nes_create_qp()
1260 			nesqp->hwqp.rq_encoded_size = rq_encoded_size;  in nes_create_qp()
1264 			nesqp->hwqp.qp_id = qp_num;  in nes_create_qp()
1265 			nesqp->ibqp.qp_num = nesqp->hwqp.qp_id;  in nes_create_qp()
1266 			nesqp->nespd = nespd;  in nes_create_qp()
1269 			nesqp->nesscq = nescq;  in nes_create_qp()
1271 			nesqp->nesrcq = nescq;  in nes_create_qp()
1273 			nesqp->nesqp_context->misc |= cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) <<  in nes_create_qp()
1275 			nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.rq_encoded_size <<  in nes_create_qp()
1277 			nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size <<  in nes_create_qp()
1280 				nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN);  in nes_create_qp()
1281 				nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN);  in nes_create_qp()
1283 			nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number +  in nes_create_qp()
1284 					((u32)nesqp->nesrcq->hw_cq.cq_number << 16));  in nes_create_qp()
1285 			u64temp = (u64)nesqp->hwqp.sq_pbase;  in nes_create_qp()
1286 			nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1287 			nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1291 				u64temp = (u64)nesqp->hwqp.sq_pbase;  in nes_create_qp()
1292 				nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1293 				nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1294 				u64temp = (u64)nesqp->hwqp.rq_pbase;  in nes_create_qp()
1295 				nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1296 				nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1298 				u64temp = (u64)nesqp->pbl_pbase;  in nes_create_qp()
1299 				nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1300 				nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1307 			nesqp->nesqp_context->misc2 |= cpu_to_le32(  in nes_create_qp()
1317 			nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32((u32)nesqp->nespd->pd_id << 16);  in nes_create_qp()
1318 			u64temp = (u64)nesqp->hwqp.q2_pbase;  in nes_create_qp()
1319 			nesqp->nesqp_context->q2_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1320 			nesqp->nesqp_context->q2_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1321 			nesqp->nesqp_context->aeq_token_low =  cpu_to_le32((u32)((unsigned long)(nesqp)));  in nes_create_qp()
1322 …	nesqp->nesqp_context->aeq_token_high =  cpu_to_le32((u32)(upper_32_bits((unsigned long)(nesqp))));  in nes_create_qp()
1323 			nesqp->nesqp_context->ird_ord_sizes = cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM |  in nes_create_qp()
1329 				nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC);  in nes_create_qp()
1338 				nes_free_qp_mem(nesdev, nesqp,virt_wqs);  in nes_create_qp()
1339 				kfree(nesqp->allocated_buffer);  in nes_create_qp()
1355 			set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);  in nes_create_qp()
1357 			u64temp = (u64)nesqp->nesqp_context_pbase;  in nes_create_qp()
1365 					nesqp->hwqp.qp_id);  in nes_create_qp()
1371 					nesqp->hwqp.qp_id, ret, nesdev->cqp.sq_head, nesdev->cqp.sq_tail,  in nes_create_qp()
1376 				nes_free_qp_mem(nesdev, nesqp,virt_wqs);  in nes_create_qp()
1377 				kfree(nesqp->allocated_buffer);  in nes_create_qp()
1388 				uresp.mmap_sq_db_index = nesqp->mmap_sq_db_index;  in nes_create_qp()
1392 				uresp.qp_id = nesqp->hwqp.qp_id;  in nes_create_qp()
1396 					nes_free_qp_mem(nesdev, nesqp,virt_wqs);  in nes_create_qp()
1397 					kfree(nesqp->allocated_buffer);  in nes_create_qp()
1403 					nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp));  in nes_create_qp()
1404 			spin_lock_init(&nesqp->lock);  in nes_create_qp()
1405 			nes_add_ref(&nesqp->ibqp);  in nes_create_qp()
1412 	nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR);  in nes_create_qp()
1413 	init_timer(&nesqp->terminate_timer);  in nes_create_qp()
1414 	nesqp->terminate_timer.function = nes_terminate_timeout;  in nes_create_qp()
1415 	nesqp->terminate_timer.data = (unsigned long)nesqp;  in nes_create_qp()
1418 	nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp;  in nes_create_qp()
1422 	return &nesqp->ibqp;  in nes_create_qp()
1428 static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq)  in nes_clean_cq()  argument
1445 		if (u64temp == (u64)(unsigned long)nesqp) {  in nes_clean_cq()
1464 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_destroy_qp()  local
1472 	nesqp->destroyed = 1;  in nes_destroy_qp()
1475 	if (nesqp->ibqp_state >= IB_QPS_INIT && nesqp->ibqp_state <= IB_QPS_RTS) {  in nes_destroy_qp()
1478 		nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL);  in nes_destroy_qp()
1481 	if (((nesqp->ibqp_state == IB_QPS_INIT) ||  in nes_destroy_qp()
1482 			(nesqp->ibqp_state == IB_QPS_RTR)) && (nesqp->cm_id)) {  in nes_destroy_qp()
1483 		cm_id = nesqp->cm_id;  in nes_destroy_qp()
1493 				nesqp->hwqp.qp_id, cm_id, atomic_read(&nesqp->refcount));  in nes_destroy_qp()
1501 	if (nesqp->user_mode) {  in nes_destroy_qp()
1504 			clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);  in nes_destroy_qp()
1505 			nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL;  in nes_destroy_qp()
1506 			if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) {  in nes_destroy_qp()
1507 				nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index;  in nes_destroy_qp()
1510 		if (nesqp->pbl_pbase && nesqp->sq_kmapped) {  in nes_destroy_qp()
1511 			nesqp->sq_kmapped = 0;  in nes_destroy_qp()
1512 			kunmap(nesqp->page);  in nes_destroy_qp()
1516 		if (nesqp->nesscq)  in nes_destroy_qp()
1517 			nes_clean_cq(nesqp, nesqp->nesscq);  in nes_destroy_qp()
1519 		if ((nesqp->nesrcq) && (nesqp->nesrcq != nesqp->nesscq))  in nes_destroy_qp()
1520 			nes_clean_cq(nesqp, nesqp->nesrcq);  in nes_destroy_qp()
1522 	nes_rem_ref(&nesqp->ibqp);  in nes_destroy_qp()
2822 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_query_qp()  local
2827 	attr->cap.max_send_wr = nesqp->hwqp.sq_size;  in nes_query_qp()
2828 	attr->cap.max_recv_wr = nesqp->hwqp.rq_size;  in nes_query_qp()
2835 	init_attr->event_handler = nesqp->ibqp.event_handler;  in nes_query_qp()
2836 	init_attr->qp_context = nesqp->ibqp.qp_context;  in nes_query_qp()
2837 	init_attr->send_cq = nesqp->ibqp.send_cq;  in nes_query_qp()
2838 	init_attr->recv_cq = nesqp->ibqp.recv_cq;  in nes_query_qp()
2839 	init_attr->srq = nesqp->ibqp.srq;  in nes_query_qp()
2849 int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp,  in nes_hw_modify_qp()  argument
2860 			nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));  in nes_hw_modify_qp()
2879 	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);  in nes_hw_modify_qp()
2880 …set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_…  in nes_hw_modify_qp()
2900 				nesqp->hwqp.qp_id, ret, cqp_request->major_code, cqp_request->minor_code);  in nes_hw_modify_qp()
2905 					nesqp->hwqp.qp_id, cqp_request->major_code,  in nes_hw_modify_qp()
2929 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_modify_qp()  local
2944 			nesqp->hwqp.qp_id, attr->qp_state, nesqp->ibqp_state,  in nes_modify_qp()
2945 			nesqp->iwarp_state, atomic_read(&nesqp->refcount));  in nes_modify_qp()
2947 	spin_lock_irqsave(&nesqp->lock, qplockflags);  in nes_modify_qp()
2951 			nesqp->hwqp.qp_id, nesqp->hw_iwarp_state,  in nes_modify_qp()
2952 			nesqp->hw_tcp_state, attr->qp_access_flags, attr_mask);  in nes_modify_qp()
2958 						nesqp->hwqp.qp_id);  in nes_modify_qp()
2959 				if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_IDLE) {  in nes_modify_qp()
2960 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
2968 						nesqp->hwqp.qp_id);  in nes_modify_qp()
2969 				if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_IDLE) {  in nes_modify_qp()
2970 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
2978 						nesqp->hwqp.qp_id);  in nes_modify_qp()
2979 				if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_RTS) {  in nes_modify_qp()
2980 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
2983 				if (nesqp->cm_id == NULL) {  in nes_modify_qp()
2985 							nesqp->hwqp.qp_id );  in nes_modify_qp()
2986 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
2990 				if (nesqp->iwarp_state != NES_CQP_QP_IWARP_STATE_RTS)  in nes_modify_qp()
2994 				nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_ESTABLISHED;  in nes_modify_qp()
2995 				nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_RTS;  in nes_modify_qp()
2996 				nesqp->hte_added = 1;  in nes_modify_qp()
3001 						nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, nesqp->hwqp.sq_tail);  in nes_modify_qp()
3002 				if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_CLOSING) {  in nes_modify_qp()
3003 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3006 					if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_CLOSING) {  in nes_modify_qp()
3009 								nesqp->hwqp.qp_id);  in nes_modify_qp()
3010 						spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3013 					if (nesqp->hw_iwarp_state != NES_AEQE_IWARP_STATE_RTS) {  in nes_modify_qp()
3016 								nesqp->hwqp.qp_id);  in nes_modify_qp()
3019 					switch (nesqp->hw_iwarp_state) {  in nes_modify_qp()
3031 							nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;  in nes_modify_qp()
3038 						nesqp->hwqp.qp_id);  in nes_modify_qp()
3039 				if (nesqp->iwarp_state>=(u32)NES_CQP_QP_IWARP_STATE_TERMINATE) {  in nes_modify_qp()
3040 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3045 				nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;  in nes_modify_qp()
3050 				if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_ERROR) {  in nes_modify_qp()
3051 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3055 						nesqp->hwqp.qp_id);  in nes_modify_qp()
3056 				if (nesqp->term_flags)  in nes_modify_qp()
3057 					del_timer(&nesqp->terminate_timer);  in nes_modify_qp()
3061 					if (nesqp->hte_added) {  in nes_modify_qp()
3064 						nesqp->hte_added = 0;  in nes_modify_qp()
3066 				if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) &&  in nes_modify_qp()
3068 						(nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) {  in nes_modify_qp()
3072 							nesqp->hwqp.qp_id, nesqp->hw_tcp_state);  in nes_modify_qp()
3076 				nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;  in nes_modify_qp()
3079 				spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3084 		nesqp->ibqp_state = attr->qp_state;  in nes_modify_qp()
3085 		nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK;  in nes_modify_qp()
3087 				nesqp->iwarp_state);  in nes_modify_qp()
3092 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN |  in nes_modify_qp()
3097 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN);  in nes_modify_qp()
3101 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN);  in nes_modify_qp()
3105 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN);  in nes_modify_qp()
3109 		if (nesqp->user_mode) {  in nes_modify_qp()
3110 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN |  in nes_modify_qp()
3116 	original_last_aeq = nesqp->last_aeq;  in nes_modify_qp()
3117 	spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3126 		ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 1);  in nes_modify_qp()
3130 					next_iwarp_state, nesqp->hwqp.qp_id);  in nes_modify_qp()
3134 	if ((issue_modify_qp) && (nesqp->ibqp_state > IB_QPS_RTS)) {  in nes_modify_qp()
3137 				nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3138 				original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3141 				if (nesqp->cm_id && nesqp->hw_tcp_state != 0) {  in nes_modify_qp()
3144 							nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3145 							original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3147 					spin_lock_irqsave(&nesqp->lock, qplockflags);  in nes_modify_qp()
3148 					nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;  in nes_modify_qp()
3149 					nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT;  in nes_modify_qp()
3150 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3151 					nes_cm_disconn(nesqp);  in nes_modify_qp()
3154 							nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));  in nes_modify_qp()
3157 				spin_lock_irqsave(&nesqp->lock, qplockflags);  in nes_modify_qp()
3158 				if (nesqp->cm_id) {  in nes_modify_qp()
3160 					if (atomic_inc_return(&nesqp->close_timer_started) == 1) {  in nes_modify_qp()
3161 						nesqp->cm_id->add_ref(nesqp->cm_id);  in nes_modify_qp()
3165 								nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3166 								original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3167 						schedule_nes_timer(nesqp->cm_node, (struct sk_buff *) nesqp, NES_TIMER_TYPE_CLOSE, 1, 0);  in nes_modify_qp()
3169 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3171 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3175 							nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3176 							original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3182 					nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3183 					original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3188 				nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3189 				original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3195 			nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));  in nes_modify_qp()
3266 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_post_send()  local
3269 	u32 qsize = nesqp->hwqp.sq_size;  in nes_post_send()
3275 	if (nesqp->ibqp_state > IB_QPS_RTS) {  in nes_post_send()
3280 	spin_lock_irqsave(&nesqp->lock, flags);  in nes_post_send()
3282 	head = nesqp->hwqp.sq_head;  in nes_post_send()
3286 		if (nesqp->term_flags) {  in nes_post_send()
3292 		if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) {  in nes_post_send()
3297 		wqe = &nesqp->hwqp.sq_vbase[head];  in nes_post_send()
3300 		nes_fill_init_qp_wqe(wqe, nesqp, head);  in nes_post_send()
3505 		if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all)  in nes_post_send()
3518 	nesqp->hwqp.sq_head = head;  in nes_post_send()
3524 				(counter << 24) | 0x00800000 | nesqp->hwqp.qp_id);  in nes_post_send()
3527 	spin_unlock_irqrestore(&nesqp->lock, flags);  in nes_post_send()
3546 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_post_recv()  local
3550 	u32 qsize = nesqp->hwqp.rq_size;  in nes_post_recv()
3556 	if (nesqp->ibqp_state > IB_QPS_RTS) {  in nes_post_recv()
3561 	spin_lock_irqsave(&nesqp->lock, flags);  in nes_post_recv()
3563 	head = nesqp->hwqp.rq_head;  in nes_post_recv()
3567 		if (nesqp->term_flags) {  in nes_post_recv()
3577 		if (((head + (2 * qsize) - nesqp->hwqp.rq_tail) % qsize) == (qsize - 1)) {  in nes_post_recv()
3583 		wqe = &nesqp->hwqp.rq_vbase[head];  in nes_post_recv()
3587 		nes_fill_init_qp_wqe(wqe, nesqp, head);  in nes_post_recv()
3612 	nesqp->hwqp.rq_head = head;  in nes_post_recv()
3617 		nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter<<24) | nesqp->hwqp.qp_id);  in nes_post_recv()
3620 	spin_unlock_irqrestore(&nesqp->lock, flags);  in nes_post_recv()
3640 	struct nes_qp *nesqp;  in nes_poll_cq()  local
3678 			nesqp = (struct nes_qp *)(unsigned long)u64temp;  in nes_poll_cq()
3695 			entry->qp = &nesqp->ibqp;  in nes_poll_cq()
3696 			entry->src_qp = nesqp->hwqp.qp_id;  in nes_poll_cq()
3699 				if (nesqp->skip_lsmm) {  in nes_poll_cq()
3700 					nesqp->skip_lsmm = 0;  in nes_poll_cq()
3701 					nesqp->hwqp.sq_tail++;  in nes_poll_cq()
3705 				wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3707 						((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3709 				entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3712 				switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3721 						entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3739 				nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1);  in nes_poll_cq()
3740 				if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)) {  in nes_poll_cq()
3742 					wq_tail = nesqp->hwqp.sq_tail;  in nes_poll_cq()
3747 …wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_…  in nes_poll_cq()
3748 …((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_ID…  in nes_poll_cq()
3751 				nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1);  in nes_poll_cq()
3752 				if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)) {  in nes_poll_cq()
3754 					wq_tail = nesqp->hwqp.rq_tail;  in nes_poll_cq()